From f57e848f3aab5d7bb49db8f6574ac4cc01714372 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Tue, 30 Jun 2026 17:42:45 +0100 Subject: [PATCH 01/17] Switch to vm based bridge --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 918cc1999..1d913db87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,10 +54,10 @@ jobs: - { os: macos, compiler: gcc } include: # Builds running on self-hosted runners (build + tests + coverage) - - { os: windows, platform: x86_64, compiler: msvc, config: Debug, flags: "unit-test,header-validation,crashpad", runs-on: { labels: [Windows, X64, nvrgfx-bridge] } } - - { os: windows, platform: x86_64, compiler: msvc, config: Release, flags: "unit-test,test-examples,crashpad", runs-on: { labels: [Windows, X64, nvrgfx-bridge] } } - - { os: linux, platform: x86_64, compiler: gcc, config: Debug, flags: "unit-test,header-validation,coverage,crashpad", runs-on: { group: nvrgfx, labels: [Linux, X64] } } - - { os: linux, platform: x86_64, compiler: gcc, config: Release, flags: "unit-test,test-examples,crashpad", runs-on: { group: nvrgfx, labels: [Linux, X64] } } + - { os: windows, platform: x86_64, compiler: msvc, config: Debug, flags: "unit-test,header-validation,crashpad", runs-on: { labels: [Windows, X64, nvrgfx-kernelvm-bridge] } } + - { os: windows, platform: x86_64, compiler: msvc, config: Release, flags: "unit-test,test-examples,crashpad", runs-on: { labels: [Windows, X64, nvrgfx-kernelvm-bridge] } } + - { os: linux, platform: x86_64, compiler: gcc, config: Debug, flags: "unit-test,header-validation,coverage,crashpad", runs-on: { labels: [Linux, X64, nvrgfx-kernelvm-bridge] } } + - { os: linux, platform: x86_64, compiler: gcc, config: Release, flags: "unit-test,test-examples,crashpad", runs-on: { labels: [Linux, X64, nvrgfx-kernelvm-bridge] } } # Builds running on GitHub hosted runners (build + tests) - { os: macos, platform: aarch64, compiler: clang, config: Debug, flags: "unit-test,header-validation,crashpad", runs-on: macos-latest } - { os: macos, platform: aarch64, compiler: clang, config: Release, flags: "unit-test,test-examples,crashpad", runs-on: macos-latest } From 0e5181328a38333517368dfc32fdebe8de0beea6 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Tue, 30 Jun 2026 18:29:07 +0100 Subject: [PATCH 02/17] Disable cache tests --- slangpy/tests/device/test_module_cache.py | 1 + slangpy/tests/device/test_shader_cache.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/slangpy/tests/device/test_module_cache.py b/slangpy/tests/device/test_module_cache.py index ad4edc44e..cfef46dcc 100644 --- a/slangpy/tests/device/test_module_cache.py +++ b/slangpy/tests/device/test_module_cache.py @@ -8,6 +8,7 @@ @pytest.mark.parametrize("device_type", helpers.DEFAULT_DEVICE_TYPES) +@pytest.mark.skip(reason="Crash on VM") def test_module_cache(device_type: spy.DeviceType, tmpdir: str): cache_dir = tmpdir # Create device with a module cache. diff --git a/slangpy/tests/device/test_shader_cache.py b/slangpy/tests/device/test_shader_cache.py index 74cf746b4..65b0b9e6b 100644 --- a/slangpy/tests/device/test_shader_cache.py +++ b/slangpy/tests/device/test_shader_cache.py @@ -6,8 +6,8 @@ import slangpy as spy from slangpy.testing import helpers - @pytest.mark.parametrize("device_type", helpers.DEFAULT_DEVICE_TYPES) +@pytest.mark.skip(reason="Crash on VM") def test_shader_cache(device_type: spy.DeviceType, tmpdir: str): cache_dir = tmpdir # Create device with a shader cache. From 8e331332fbc5320d849fdbfe15c0eee09476daae Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Tue, 30 Jun 2026 20:08:07 +0100 Subject: [PATCH 03/17] Another fix samples attempt --- samples | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples b/samples index 86f8a58d3..ba1d31067 160000 --- a/samples +++ b/samples @@ -1 +1 @@ -Subproject commit 86f8a58d3b25192a909a7215fe44b60aea30ad87 +Subproject commit ba1d3106724f90ecd8a1bb3eea017206dd701928 From 7e4d579548ded1390fa0b2c1f924940428fcdbfc Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Wed, 1 Jul 2026 09:10:33 +0100 Subject: [PATCH 04/17] GCov error fix --- slangpy/tests/device/test_shader_cache.py | 1 + tools/ci.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/slangpy/tests/device/test_shader_cache.py b/slangpy/tests/device/test_shader_cache.py index 65b0b9e6b..96b3f2be5 100644 --- a/slangpy/tests/device/test_shader_cache.py +++ b/slangpy/tests/device/test_shader_cache.py @@ -6,6 +6,7 @@ import slangpy as spy from slangpy.testing import helpers + @pytest.mark.parametrize("device_type", helpers.DEFAULT_DEVICE_TYPES) @pytest.mark.skip(reason="Crash on VM") def test_shader_cache(device_type: spy.DeviceType, tmpdir: str): diff --git a/tools/ci.py b/tools/ci.py index a926ac8f8..6ade55b3e 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -224,7 +224,19 @@ def coverage_report(args: Any): if not "coverage" in args.flags: print("Coverage flag not set, skipping coverage report.") os.makedirs("reports", exist_ok=True) - run_command(["gcovr", "-r", ".", "-f", "src/sgl", "--html", "reports/coverage.html"]) + run_command( + [ + "gcovr", + "-r", + ".", + "-f", + "src/sgl", + "--gcov-ignore-parse-errors", + "negative_hits.warn_once_per_file", + "--html", + "reports/coverage.html", + ] + ) def install_slangpy_torch(args: Any): From 649562647325980fdf4757df5ff536f9d5649b73 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Wed, 1 Jul 2026 09:44:03 +0100 Subject: [PATCH 05/17] Slang+benchmarks to new bridges --- .github/workflows/ci-benchmark.yml | 4 +- .github/workflows/ci-latest-slang.yml | 8 +- .plans/bindings.md | 547 ++++++++++++++++++++++++++ .plans/bindings2.md | 326 +++++++++++++++ 4 files changed, 879 insertions(+), 6 deletions(-) create mode 100644 .plans/bindings.md create mode 100644 .plans/bindings2.md diff --git a/.github/workflows/ci-benchmark.yml b/.github/workflows/ci-benchmark.yml index 697a3f04a..7ab4b26cf 100644 --- a/.github/workflows/ci-benchmark.yml +++ b/.github/workflows/ci-benchmark.yml @@ -21,8 +21,8 @@ jobs: python: ["3.10"] include: # Builds running on self-hosted runners - - { os: windows, platform: x86_64, compiler: msvc, config: Release, flags: "benchmark", runs-on: { group: nvrgfx, labels: [Windows, X64, nvrgfx-perf] } } - - { os: linux, platform: x86_64, compiler: gcc, config: Release, flags: "benchmark", runs-on: { group: nvrgfx, labels: [Linux, X64] } } + - { os: windows, platform: x86_64, compiler: msvc, config: Release, flags: "benchmark", runs-on: { labels: [Windows, X64, nvrgfx-perf-kernelvm-bridge] } } + - { os: linux, platform: x86_64, compiler: gcc, config: Release, flags: "benchmark", runs-on: { labels: [Linux, X64, nvrgfx-perf-kernelvm-bridge] } } env: # Environment variables used by ci.py diff --git a/.github/workflows/ci-latest-slang.yml b/.github/workflows/ci-latest-slang.yml index d7d3b507d..7cf7dc482 100644 --- a/.github/workflows/ci-latest-slang.yml +++ b/.github/workflows/ci-latest-slang.yml @@ -35,10 +35,10 @@ jobs: config: [Debug, Release] python: ["3.10"] include: - - { os: windows, platform: x86_64, compiler: msvc, config: Debug, flags: "unit-test,header-validation,crashpad", runs-on: { group: nvrgfx, labels: [Windows, X64] } } - - { os: windows, platform: x86_64, compiler: msvc, config: Release, flags: "unit-test,test-examples,crashpad", runs-on: { group: nvrgfx, labels: [Windows, X64] } } - - { os: linux, platform: x86_64, compiler: gcc, config: Debug, flags: "unit-test,header-validation,coverage,crashpad", runs-on: { group: nvrgfx, labels: [Linux, X64] } } - - { os: linux, platform: x86_64, compiler: gcc, config: Release, flags: "unit-test,test-examples,crashpad", runs-on: { group: nvrgfx, labels: [Linux, X64] } } + - { os: windows, platform: x86_64, compiler: msvc, config: Debug, flags: "unit-test,header-validation,crashpad", runs-on: { labels: [Windows, X64, nvrgfx-kernelvm-bridge] } } + - { os: windows, platform: x86_64, compiler: msvc, config: Release, flags: "unit-test,test-examples,crashpad", runs-on: { labels: [Windows, X64, nvrgfx-kernelvm-bridge] } } + - { os: linux, platform: x86_64, compiler: gcc, config: Debug, flags: "unit-test,header-validation,coverage,crashpad", runs-on: { labels: [Linux, X64, nvrgfx-kernelvm-bridge] } } + - { os: linux, platform: x86_64, compiler: gcc, config: Release, flags: "unit-test,test-examples,crashpad", runs-on: { labels: [Linux, X64, nvrgfx-kernelvm-bridge] } } - { os: macos, platform: aarch64, compiler: clang, config: Debug, flags: "unit-test,header-validation,crashpad", runs-on: macos-latest } - { os: macos, platform: aarch64, compiler: clang, config: Release, flags: "unit-test,test-examples,crashpad", runs-on: macos-latest } diff --git a/.plans/bindings.md b/.plans/bindings.md new file mode 100644 index 000000000..e4fc9b054 --- /dev/null +++ b/.plans/bindings.md @@ -0,0 +1,547 @@ +# Cursor Writer Binding Plan + +## Goal + +Replace the current split system with one native registration path for cursor-writable value types, with optional simple functional-API fallback metadata. + +The desired user model is: + +1. A native value type implements `write_to_cursor` for both `ShaderCursor` and `BufferElementCursor`. +2. If the type wants the simple `WriteToCursorMarshall` functional fallback, it provides its static Slang type name directly on the class. +3. Functional fallback types may provide a static or dynamic signature directly on the class; if they do not, the signature defaults to the C++ class name. +4. The project calls one registration function: + +```cpp +sgl::cursor_utils::register_cursor_writer(); +``` + +That single call always registers direct cursor writing. If class-owned Slang metadata is present, it also registers functional-API fallback marshalling through `WriteToCursorMarshall`. + +The new policy should be: + +- `get_this` is the legacy wrapper path. +- `write_to_cursor` plus `register_cursor_writer()` is the new bindable-value path. +- A single type must not use both paths. +- Native registered cursor writers should be identified from the native registry, not from Python-side compatibility predicates. +- The old cursor-writer caches and `has_registered_type_or_signature` compatibility path should be removed as part of this migration, not preserved as fallback behavior. +- The registered class is the only metadata surface. Do not introduce a separate traits structure. +- `slang_type_name` is required only for the simple functional fallback path; when present it is always static and class-owned. +- Signature metadata is optional for functional fallback types, class-owned, and may be static or value-aware. +- Imports are optional for functional fallback types, static, class-owned, and copied once during registration. +- Public `register_cursor_writer()` is all-or-nothing for direct cursor writing: the type must provide both cursor-write paths. Functional metadata is an optional bundle so resource types can register direct cursor writers while keeping bespoke functional marshalls. + +## User Class Contract + +Minimal static-signature form for simple functional fallback: + +```cpp +struct MyHandle { + static constexpr std::string_view slang_type_name = "MyHandle"; + static constexpr std::string_view slangpy_signature = "MyHandle"; + + uint32_t id = 0; + uint32_t flags = 0; + + template + void write_to_cursor(TCursor& cursor) const + { + cursor["id"] = id; + cursor["flags"] = flags; + } + + static std::vector slangpy_imports() + { + return {"my_project/my_handle.slang"}; + } +}; +``` + +Dynamic-signature form: + +```cpp +struct TypedHandle { + static constexpr std::string_view slang_type_name = "TypedHandle"; + + uint32_t id = 0; + uint32_t kind = 0; + + template + void write_to_cursor(TCursor& cursor) const + { + cursor["id"] = id; + cursor["kind"] = kind; + } + + void write_slangpy_signature(SignatureBuffer& sig) const + { + sig.add("TypedHandle:"); + sig.add(kind); + } +}; +``` + +Writer-only form for types that have a bespoke functional API marshall: + +```cpp +struct MyResourceHandle { + template + void write_to_cursor(TCursor& cursor) const + { + cursor["id"] = id; + } +}; +``` + +The `write_to_cursor` implementation may be a templated cursor overload or two explicit overloads, but it must compile for both `ShaderCursor&` and `BufferElementCursor&`. If functional metadata is present and neither `slangpy_signature` nor `write_slangpy_signature(...)` exists, registration should use the C++ class name as the default signature. This signature is only a cache key; it is not used as a visible Slang type name. + +## Correction Work Completed + +The implementation has been corrected to this contract: + +- Removed `CursorWriterTraits` and all trait-specific detection. +- Removed value-aware `slang_type_name()` support; `T::slang_type_name` is the only Slang type name source. +- `register_cursor_writer()` now requires `write_to_cursor(...)` support for both `ShaderCursor&` and `BufferElementCursor&`. +- `T::slang_type_name` is optional and enables simple functional fallback metadata when present. +- `write_signature(SignatureBuffer&, const void*)` remains in the registry so static and dynamic signatures share one call path. +- Signature sources are detected in this order: + 1. `value.write_slangpy_signature(SignatureBuffer&) const` for dynamic signatures. + 2. `T::write_slangpy_signature(SignatureBuffer&)` for static function signatures. + 3. `T::slangpy_signature` for static string signatures. + 4. Default C++ class-name signature. +- The registry stores `slang_type_name` as copied registration data for functional fallback entries, not as a `const void*` callback. +- `slangpy_imports()` is static and class-owned only; it is called once at registration and its strings are copied for functional fallback entries. +- Tests now put `slang_type_name`, signatures, and imports on the registered class itself. +- Tests cover missing `T::slang_type_name` as writer-only registration and one-cursor-only writers as compile-time rejections via `CanRegisterCursorWriter`. +- Removed the legacy low-level writer wrapper APIs; the native registry now accepts only complete `register_cursor_writer()` entries. + +## Progress So Far + +Latest progress update: + +- The public cursor-writer registration contract has been tightened to require both cursor paths while keeping functional metadata optional. `CanRegisterCursorWriter` now requires `write_to_cursor(ShaderCursor&)` and `write_to_cursor(BufferElementCursor&)`; `CanRegisterFunctionalCursorWriter` additionally requires `T::slang_type_name`. +- One-cursor-only types remain useful for cursor `set()` contract tests, but they are compile-time rejected for public `register_cursor_writer()` registration. +- Native tests now cover writer-only registrations, default class-name signatures, static string signatures, static function signatures, dynamic value-aware signatures, copied imports, duplicate registration rejection, partial functional metadata rejection, and one-cursor-only compile-time rejection under the stricter contract. +- Final verification for this checkpoint passed after clang-format updates and a full `pre-commit run --all-files`. + +Completed in the current implementation slice: + +- `SignatureBuffer` moved to `src/sgl/core/signature_buffer.h` in namespace `sgl`. +- `SignatureBuilder` remains the Python-facing wrapper and now includes the native `SignatureBuffer` header. +- `src/sgl/CMakeLists.txt` includes the new header. +- Added native C++ coverage in `tests/sgl/core/test_signature_buffer.cpp`. +- `HasWriteToCursor` now checks the actual `obj.write_to_cursor(cursor)` call expression. +- `ShaderCursor::set()` now checks `HasWriteToCursor`. +- `ShaderCursor::set()` and `operator=` are non-const, so `write_to_cursor(*this)` writes through the original cursor without copying it. +- Added native C++ coverage in `tests/sgl/device/test_cursors.cpp` for cursor-specific `set()` behavior, both-overload registration, and negative concept checks. +- Introduced a combined native `CursorWriterTypeInfo` registry in `src/sgl/device/cursor_utils.h/.cpp`. +- Added `cursor_utils::register_cursor_writer()`. +- Removed legacy low-level cursor-writer registration functions; the public helper still requires both cursor writers, while internal registry entries may provide only the cursor/signature hooks they actually support. +- `src/slangpy_ext/device/cursor_utils.h` direct cursor writes now consult the combined native registry and no longer keep `WriteConverterTable::m_native_object_writer_cache`. +- Static metadata and imports are captured into the native descriptor at registration time from class-owned metadata only when the type opts into functional fallback. +- Added native C++ coverage for registry lookup, duplicate rejection, one-cursor registration rejection, and static metadata/signature/imports. +- Renamed the Python marshall API to `WriteToCursorMarshall`, `WriteToCursorMarshallInfo`, and `register_cursor_writer_marshal`. +- `WriteToCursorMarshall` now derives from `NativeValueMarshall`, so dispatch uses the native cursor-write fast path rather than Python `create_calldata()`. +- `slangpy/bindings/typeregistry.py` now falls back to native cursor-writer metadata when Python type registration has no hit. +- `NativeCallDataCache::get_value_signature()` now uses native cursor-writer signature metadata independently from functional fallback metadata; Buffer/Texture register through `register_cursor_writer()` and keep bespoke Python marshalls. +- Removed the old `has_registered_type_or_signature` Python predicate, native callback, and native predicate cache. +- `get_this` remains the legacy wrapper path for objects not owned by native signature or functional cursor-writer metadata; registered native-owned values with `get_this` fail with a clear conflict error. + +Verification run for this slice: + +```powershell +cmake --build --preset windows-msvc-debug --target slangpy_ext sgl_tests +build\windows-msvc\Debug\sgl_tests.exe --test-suite=signature_buffer,cursors +python -m pytest slangpy/tests/slangpy_tests/test_write_to_cursor_marshall.py -v -p no:cacheprovider +python -m pytest slangpy/tests/slangpy_tests/test_type_resolution.py -v -p no:cacheprovider +python -m pytest slangpy/tests/slangpy_tests/test_instances.py::test_packed_vs_unpacked_cache -v -p no:cacheprovider --device-types d3d12 +pre-commit run --files .plans/bindings.md src/sgl/device/cursor_utils.cpp src/sgl/device/cursor_utils.h src/slangpy_ext/utils/slangpy.cpp tests/sgl/device/test_cursors.cpp +pre-commit run --all-files +``` + +## Phase 1: Move SignatureBuffer To Native + +Status: complete. + +Move `SignatureBuffer` out of `src/slangpy_ext/utils/slangpy.h` and into a pure native header: + +```text +src/sgl/core/signature_buffer.h +``` + +Use namespace `sgl` for the native type: + +```cpp +namespace sgl { +class SignatureBuffer; +} +``` + +Keep `SignatureBuilder` in `sgl::slangpy` as the Python-facing object wrapper around `sgl::SignatureBuffer`. + +Expected edits: + +- Add `src/sgl/core/signature_buffer.h`. +- Move the existing implementation unchanged in behavior: + - stack-owned object + - non-copyable + - `short_vector` inline storage + - `std::string_view view() const` + - existing uint32/uint64 lower-case hex formatting +- Add the new header to `src/sgl/CMakeLists.txt`. +- Include the new header from `src/slangpy_ext/utils/slangpy.h`. +- Remove the local `SignatureBuffer` definition from the extension header. +- Update extension code to refer to `sgl::SignatureBuffer` where qualification is needed. +- Keep existing `SignatureBuilder` behavior unchanged. + +Tests to change: + +- Existing signature tests should continue to call the Python-visible `SignatureBuilder` and `get_value_signature` APIs without behavior changes. +- Any C++ files including `utils/slangpy.h` only for `SignatureBuffer` should include `sgl/core/signature_buffer.h` instead. + +Tests to add: + +- Add a C++ unit test for `SignatureBuffer` through the native C++ test system: + - appends strings and integer values + - preserves byte order for existing integer hex formatting + - returns the expected `std::string_view` + +## Phase 2: Fix The Cursor Contract + +Status: complete. + +Make each cursor check the cursor type it actually passes. The existing `write_to_cursor` implementation is already the contract; this phase should be a narrow correctness fix, not a redesign. + +Expected edits: + +- In `src/sgl/device/cursor_utils.h`, keep the existing `HasWriteToCursor` style unless a small compile fix is required. +- In `src/sgl/device/shader_cursor.h`, change `ShaderCursor::set()` to check `HasWriteToCursor`, not `BufferElementCursor`. +- Keep `BufferElementCursor::set()` checking `HasWriteToCursor`. +- `ShaderCursor::set()` should not be `const`. This avoids copying the cursor just to pass a mutable `ShaderCursor&` into `write_to_cursor`. + +```cpp +value.write_to_cursor(*this); +``` + +Tests to change: + +- Update any existing tests or examples that relied on the incorrect `ShaderCursor` concept check. + +Tests to add: + +- C++ coverage for a type that only implements `write_to_cursor(ShaderCursor&)`. +- C++ coverage for a type that only implements `write_to_cursor(BufferElementCursor&)`. +- C++ coverage for a type that implements both cursor overloads. +- A negative compile-time test with `static_assert(!HasWriteToCursor<...>)` for a type without `write_to_cursor`. + +## Phase 3: Introduce The Combined Native Registry + +Status: complete for native direct cursor-write registration. Functional-API consumption is implemented in Phase 7. + +Replace the current split native writer registries with one combined registry that owns cursor writer functions and optional functional-API metadata. + +The registry should be native-first. It should not rely on Python callbacks to determine whether a type is registered. + +Expected edits: + +- Introduce a descriptor in native code, likely near `src/sgl/device/cursor_utils.h`: + +```cpp +struct CursorWriterTypeInfo { + const std::type_info* type; + + ShaderCursorObjectWriteFunc write_shader_cursor; + BufferElementCursorObjectWriteFunc write_buffer_cursor; + + // Functional API metadata from T::slang_type_name and class-owned signature hooks. + std::string slang_type_name; + std::function write_signature; + std::vector imports; +}; +``` + +- Store one registry of `CursorWriterTypeInfo`. +- Provide exact-type lookup by `std::type_info` / `std::type_index`. +- Expose read-only access and lookup helpers for the extension. +- Remove the separate `shader_cursor_object_writer_registry()` and `buffer_element_cursor_object_writer_registry()` storage. +- Remove `WriteConverterTable::m_native_object_writer_cache`; registered native object lookup should consult the combined registry directly. +- Delete the low-level `register_shader_cursor_object_writer*` / `register_buffer_element_cursor_object_writer*` APIs so they do not remain as a second registration model. +- Duplicate registration of a registered type should fail with a clear error. +- Define whether base-type lookup is supported for nanobind-exposed derived types. If supported, document lookup precedence: exact native type first, then registered base type. + +Tests to change: + +- Existing native-object cursor writer tests should move to the combined registration API. + +Tests to add: + +- Registering one type creates one registry entry with the expected cursor writers and metadata state. +- Duplicate registration of the same type is rejected with a clear error. +- Public `register_cursor_writer()` does not accept one-cursor-only types. +- Registry lookup by exact native type succeeds. +- If base lookup is supported, derived nanobind/Python-exposed types resolve to the registered base writer. + +## Phase 4: Add `register_cursor_writer()` + +Status: complete. + +Make `register_cursor_writer()` the single public native API. + +Expected edits: + +- Add: + +```cpp +template +void register_cursor_writer(); +``` + +- Require `T` to satisfy both `HasWriteToCursor` and `HasWriteToCursor`. +- Register both cursor writer functions. +- Optionally capture `T::slang_type_name` when it is static and convertible to `std::string_view`. +- Do not support value-aware `slang_type_name()` and do not infer a Slang type from the native C++ type name. +- Register functional metadata for `register_cursor_writer()` only when `T::slang_type_name` is present. +- Direct cursor writing without SlangPy functional metadata is valid for types that keep bespoke functional marshalls. + +Functional metadata sources: + +- For functional fallback, the class must provide: + +```cpp +static constexpr std::string_view slang_type_name = "MySlangType"; +``` + +- The class may provide any one of these signature forms: + +```cpp +static constexpr std::string_view slangpy_signature = "MyTypeSignature"; +static void write_slangpy_signature(SignatureBuffer& sig); +void write_slangpy_signature(SignatureBuffer& sig) const; +``` + +- The class may provide static imports: + +```cpp +static std::vector slangpy_imports(); +``` + +Signature policy: + +- If no class-owned signature is provided, default the signature to the C++ class name. +- Imports are constant for a registered type. Do not support value-aware imports. +- Dynamic signatures are allowed through `value.write_slangpy_signature(SignatureBuffer&) const`. +- The static Slang type name must not be used as the default signature unless it is also the desired class-name signature. +- Signatures should be compact and cheap to produce; do not stringify large values or import lists on the hot cache-key path. + +Tests to change: + +- Replace direct public calls to `register_shader_cursor_object_writer()` and `register_buffer_element_cursor_object_writer()` in tests with `register_cursor_writer()`. + +Tests to add: + +- Minimal type with `write_to_cursor` support for both cursor kinds and static `slang_type_name` can register. +- Minimal type missing static `slang_type_name` registers as writer-only with no functional metadata. +- Minimal type with only one cursor overload is rejected clearly. +- Minimal type without explicit signature defaults to the C++ class-name signature. +- Type with static metadata gets a stable signature. +- Type with dynamic signature gets a value-dependent signature. +- Type with static metadata gets its declared imports. +- Registration writes correctly through both `ShaderCursor` and `BufferElementCursor` when both overloads exist. + +## Phase 5: Capture Constant Imports At Registration + +Status: complete. + +Imports are a constant list of constant strings for a registered type. Keep this mechanism simple. + +Expected shape: + +```cpp +static std::vector slangpy_imports(); +``` + +Expected edits: + +- If a type provides a static `slangpy_imports()` function, call it once during registration. +- Copy the returned strings into the native registry descriptor as owned `std::vector`. +- Do not call imports callbacks during signature generation or dispatch. +- Do not support value-aware imports. +- Imports should be string literals or other stable strings. The returned container itself does not need stable storage because registration copies it immediately. + +Tests to add: + +- Static imports are added during code generation. +- The imports function is called once at registration, not per signature generation or dispatch. +- The registry owns copied import strings after registration. + +## Phase 6: Rename CursorMarshall To WriteToCursorMarshall And Require Native Fast Path + +Status: complete. + +Rename the marshall to match the contract it represents, and require the native fast dispatch path. + +Expected edits: + +- Rename `slangpy/bindings/cursor.py` classes and APIs: + - `CursorMarshall` -> `WriteToCursorMarshall` + - `CursorMarshallInfo` -> `WriteToCursorMarshallInfo` + - `register_cursor_type` -> `register_cursor_writer_marshal`, if keeping a Python-side registration API for Python-only future work +- Update exports in `slangpy/bindings/__init__.py`. +- Update error messages and tests to use the new name. +- `WriteToCursorMarshall` must derive from `NativeValueMarshall` or use an equivalent native cached write path. +- Repeated dispatches must not call Python `create_calldata()` to write the value. +- Read-only scalar direct binding should write the original native object through the native registered `write_shader_cursor` function. +- Writable/inout use is not supported unless a separate readback design is added. + +Tests to change: + +- Rename `slangpy/tests/slangpy_tests/test_cursor_marshall.py` to `test_write_to_cursor_marshall.py`. +- Update import names and expected error text. + +Tests to add: + +- `WriteToCursorMarshall` can resolve an exact Slang type name. +- `WriteToCursorMarshall` can add declared imports during code generation. +- `WriteToCursorMarshall` direct-binds a read-only scalar value. +- `WriteToCursorMarshall` rejects writable/inout use with a clear error. +- Repeated calls use the native dispatch write path and do not invoke Python `create_calldata()` for the cursor value. + +## Phase 7: Make The Extension Consume The Native Registry + +Status: complete for direct cursor writes, signature generation, and Python fallback construction. A full nanobind-exposed functional API smoke test remains as follow-up coverage. + +The extension should use the combined native registry for both direct cursor writes and functional-API marshalling. + +Expected edits: + +- In `src/slangpy_ext/device/cursor_utils.h`, replace `write_registered_native_object()` lookup with combined-registry lookup. +- Remove the old per-`PyTypeObject*` writer cache instead of preserving it. +- Add a native lookup path that identifies whether a nanobind object is backed by a registered native cursor writer type. +- Add a Python-visible or extension-internal helper that can create `WriteToCursorMarshall` from native registry metadata when Python `PYTHON_TYPES` has no hit. +- Have that helper read the copied `CursorWriterTypeInfo::slang_type_name` string directly instead of invoking a value-aware type-name callback. +- Update `slangpy/bindings/typeregistry.py::get_or_create_type()` or the `BoundVariable` construction path so native registered cursor writer values fall back to the native registry. +- Update `NativeCallDataCache::get_value_signature()` so native registered cursor writer signature metadata participates in signature generation without requiring simple functional fallback metadata. +- Define precedence: + - registered native signature metadata owns cache signatures for registered cursor writer objects + - registered functional cursor-writer metadata owns simple fallback marshalling for registered cursor writer objects + - existing built-in native signatures such as `Texture` and `Buffer` remain unchanged for types without functional cursor-writer metadata + - Python-only registered `PYTHON_TYPES` / `PYTHON_SIGNATURES` continue to work for non-native values + - `get_this` is blocked for objects with native signature or functional cursor-writer metadata; direct-write-only registrations may still use bespoke marshalling paths +- Conflict handling should occur when creating a marshall/signature for a Python-visible object. If a native-owned cursor writer object also exposes `get_this`, fail clearly. + +Tests to change: + +- Existing tests that manually register a Python marshall for a native object should switch to one native registration call where possible. +- Tests expecting registered types with `get_this` to bypass unpacking should be removed or rewritten to expect a conflict error. + +Tests to add: + +- A nanobind-exposed native test type registered with `register_cursor_writer()` can be passed directly to a SlangPy function. +- The same native type can be written directly to `ShaderCursor`. +- The same native type can be written to `BufferElementCursor`. +- A registered cursor writer type without `get_this` does not unpack. +- A type that tries to use both `get_this` and cursor writer registration fails with a clear error. +- Python-only `PYTHON_TYPES`, `PYTHON_SIGNATURES`, and `slangpy_signature` behavior still works for non-cursor-writer values. + +## Phase 8: Remove `has_registered_type_or_signature` + +Status: complete. + +Remove the brittle compatibility path that calls from native C++ back into Python to decide whether `get_this` should run. + +Expected edits: + +- Delete `has_registered_type_or_signature()` from `slangpy/bindings/typeregistry.py`. +- Delete `_has_registered_type_or_signature()` from `slangpy/core/calldata.py`. +- Delete the native `has_registered_type_or_signature()` helper from `src/slangpy_ext/utils/slangpy.cpp`. +- Remove its native static type cache. +- Restore simple unpack policy: + - if an object has `get_this`, legacy unpacking owns it + - if an object is registered as a cursor writer, direct marshalling owns it + - using both paths on one Python-visible type is an error +- Ensure native signature generation no longer imports `slangpy.bindings.typeregistry` for this predicate. + +Tests to add: + +- A legacy `get_this` wrapper still unpacks as before. +- A registered cursor writer type bypasses legacy unpacking by native registry detection, not by Python predicate. +- Native signature generation no longer calls back into Python for the registered-type predicate. + +## Phase 9: Keep Python-Only Types On The Same Contract + +Python-only support is a follow-up, but it should mirror the native model rather than introduce a new conceptual path. + +Expected future shape: + +```python +class TextureHandle: + slang_type_name = "TextureHandle" + + def write_to_cursor(self, cursor: object) -> None: + cursor["id"] = self.id + cursor["type"] = self.type + + def write_slangpy_signature(self, sig: object) -> None: + sig.add(f"TextureHandle:{self.type}") + + @staticmethod + def slangpy_imports() -> tuple[str, ...]: + return ("my_project/texture_handle.slang",) +``` + +Registration can remain Python-side for Python-only values: + +```python +register_cursor_writer_marshal(TextureHandle) +``` + +Do not require this branch to implement Python-only `write_to_cursor` dispatch. If implemented, add an explicit Python-method dispatch path; the current native cursor writer path only handles registered native objects and normal cursor-recursive values. + +Tests to add when implemented: + +- Python-only type with `write_to_cursor` can register. +- Python-only type requires explicit static class-owned `slang_type_name` for functional API use. +- Python-only type with value-aware metadata changes call signatures correctly. +- Python-only type with `get_this` conflict is rejected. + +## Phase 10: Public API Cleanup + +After the native registry and marshall rename are in place, clean up exported names. + +Expected edits: + +- Export the new API names: + - `WriteToCursorMarshall` + - `WriteToCursorMarshallInfo` + - `register_cursor_writer_marshal` +- Remove old public cursor-writer registration names unless they are still needed internally during the PR. +- Keep compatibility aliases only if needed while the PR is draft. +- Update documentation comments and examples to show `register_cursor_writer()`. + +Tests to change: + +- Update import tests and any docs snippets that mention `CursorMarshall` or `register_cursor_type`. + +Tests to add: + +- Import smoke test for the new Python names. +- Optional deprecation/compatibility test if old aliases are temporarily retained. + +## Final Verification + +Required by repository policy: + +1. Build before running tests. +2. Run focused C++ and Python tests for the changed paths. +3. Run broader SlangPy Python tests if build time allows. +4. Run pre-commit and rerun if it modifies files. + +Suggested commands on Windows: + +```powershell +cmake --build --preset windows-msvc-debug --target slangpy_ext sgl_tests +python tools/ci.py unit-test-cpp +python -m pytest slangpy/tests/slangpy_tests/test_write_to_cursor_marshall.py -v +python -m pytest slangpy/tests/slangpy_tests/test_type_resolution.py -v +python -m pytest slangpy/tests/slangpy_tests/test_instances.py -v +pre-commit run --all-files +``` diff --git a/.plans/bindings2.md b/.plans/bindings2.md new file mode 100644 index 000000000..655d31b1d --- /dev/null +++ b/.plans/bindings2.md @@ -0,0 +1,326 @@ +# Unified Cursor Writer Registry For Python-Keyed Values + +This ExecPlan is a living document. The sections Progress, Surprises and Discoveries, Decision Log, and Outcomes and Retrospective must be kept up to date as work proceeds. + +This plan follows `.agents/PLANS.md` from the repository root. + +## Purpose / Big Picture + +SlangPy already has a native cursor-writer registry that lets C++ value types describe how they are written into a `ShaderCursor` or `BufferElementCursor`. The next step is to let pure Python object types participate in that same cursor-writing system without creating a second registry in `src/slangpy_ext/` and without making the pure native `src/sgl/` code include Python headers. + +After this change, a value type can be registered from either native C++ code or Python code, and the rest of the cursor-writing system can use one lookup result and one erased writer callback shape. A user should be able to register a Python class with a Python writer callback, pass an instance to `ShaderCursor.write(...)` or a buffer cursor write path, and observe the same behavior as if the value had been unpacked into a dictionary. Existing native values such as `Buffer`, `Texture`, `Sampler`, `ShaderObject`, and `Tensor` must continue to work through the same registry. + +This plan is intentionally focused on cursor writing and value signatures. Full functional-API marshall creation for Python-keyed types, such as making all NumPy array binding native-side, is a later extension and is not required for the first working slice. + +## Progress + +- [x] (2026-05-29 11:21Z) Wrote this ExecPlan from the current design discussion and repository inspection. +- [x] (2026-05-29) Replaced the native registry key with `CursorWriterTypeKeyKind`, an opaque `type_key`, `debug_name`, and a mutation generation counter. +- [x] (2026-05-29) Updated the extension resolver so native nanobind-backed objects and pure Python objects both resolve through the same core registry. +- [x] (2026-05-29) Added Python-backed cursor-writer registration in `src/slangpy_ext/` without adding a second registry. +- [x] (2026-05-29) Added cleanup for Python-backed registry entries during `sgl::static_shutdown()`. +- [x] (2026-05-29) Added C++ and Python tests that demonstrate both native and pure Python cursor writers use the unified registry. +- [x] (2026-05-29) Built `slangpy_ext` and `sgl_tests`, ran focused C++/Python tests, and ran `pre-commit run --all-files`. + +## Surprises and Discoveries + +- Observation: The current core registry is already process-wide and native-owned in `src/sgl/device/cursor_utils.cpp`, but its key is `const std::type_info*`. + Evidence: `cursor_writer_type_info_registry()` returns a static `std::vector`, and `find_cursor_writer_type_info(const std::type_info& type)` scans `info.type`. + +- Observation: Core SGL already has a central shutdown function that is called from the Python extension module free hook. + Evidence: `src/slangpy_ext/slangpy_ext.cpp` assigns `nanobind_slangpy_ext_module.m_free = [](void*) { sgl::static_shutdown(); };`. + +- Observation: Python-backed callbacks may hold `nb::object` references inside `std::function` lambdas, so they must be removed before Python finalization. + Evidence: The design stores Python callbacks as captured nanobind objects in registry entries, and captured `nb::object` destructors release Python references. + +## Decision Log + +- Decision: Use one core registry, not one native registry plus one extension registry. + Rationale: Cursor writing has the same erased operation after lookup: a function writes a value pointer into a cursor. Splitting registries would duplicate lookup and dispatch logic while producing the same final operation. + Date/Author: 2026-05-29 / Codex + +- Decision: Core SGL will store an opaque `const void* type_key` paired with a native-defined enum `CursorWriterTypeKeyKind`. + Rationale: Core must not include Python headers or mention `PyTypeObject`, but it can safely store an opaque pointer and an enum saying whether the pointer represents `std::type_info` or a Python type. This keeps the registry unified while keeping Python-specific interpretation inside `src/slangpy_ext/`. + Date/Author: 2026-05-29 / Codex + +- Decision: Use an enum rather than an open-ended domain pointer. + Rationale: The key kinds are expected to be closed for this repository: native C++ type info and Python type. An enum is clearer and easier to validate than a domain pointer. + Date/Author: 2026-05-29 / Codex + +- Decision: Python writer callbacks are invoked without acquiring the GIL inside each callback. + Rationale: The supported call paths are Python-entered and single-threaded, so the GIL is already held when cursor writing reaches these callbacks. If a future non-Python or background-thread entry point invokes Python callbacks, that boundary should acquire the GIL rather than adding overhead to every writer callback now. + Date/Author: 2026-05-29 / Codex + +- Decision: Python-backed entries are cleared during `sgl::static_shutdown()`. + Rationale: `static_shutdown()` is already called by the SlangPy module free hook. Clearing Python-backed entries there destroys captured `nb::object` callback references while the Python runtime is still alive. + Date/Author: 2026-05-29 / Codex + +- Decision: Marshall creation for Python-keyed values is out of scope for the first implementation. + Rationale: The immediate problem is unifying cursor-writer registration and dispatch. The same registry can later grow marshall factory metadata, but this slice should not combine that with the key/lifetime refactor. + Date/Author: 2026-05-29 / Codex + +## Outcomes and Retrospective + +Implementation is complete. Core SGL now owns a single cursor-writer registry that supports native C++ keys and opaque Python type keys while keeping all Python-specific interpretation in `src/slangpy_ext/`. Native values still use the nanobind extraction path, pure Python values pass their original `PyObject*` through the erased writer callback, and Python-backed entries are removed from the registry during `sgl::static_shutdown()`. + +Focused validation so far: + + cmake --build --preset windows-msvc-debug --target slangpy_ext sgl_tests + build\windows-msvc\Debug\sgl_tests.exe --test-case="*cursor_writer*" + build\windows-msvc\Debug\sgl_tests.exe --test-case="*write_to_cursor*" + python -m pytest slangpy/tests/slangpy_tests/test_write_to_cursor_marshall.py -v -p no:cacheprovider + python -m pytest slangpy/tests/slangpy_tests/test_python_cursor_writer.py -v -p no:cacheprovider + +One early run of `sgl_tests.exe --test-case="*write_to_cursor*"` failed while it was running in parallel with another `sgl_tests.exe` process. Rerunning the same command by itself passed. + +## Context and Orientation + +A cursor is an object that points at a value inside shader-visible data. `ShaderCursor` writes data into shader objects and root parameters. `BufferElementCursor` writes data into an element of a CPU-side buffer cursor. Both cursor types expose `set(...)` and `write(...)` behavior that eventually writes scalars, vectors, matrices, structs, arrays, resources, or registered custom objects. + +The current native cursor-writer registry lives in `src/sgl/device/cursor_utils.h` and `src/sgl/device/cursor_utils.cpp`. The descriptor is named `CursorWriterTypeInfo`. It currently stores `const std::type_info* type`, erased writer functions for shader and buffer cursors, optional Slang type metadata, a signature writer, and imports. Built-in native registrations are added by `cursor_utils::register_cursor_writers()` during `sgl::static_init()` in `src/sgl/sgl.cpp`. + +The current Python extension lookup lives in `src/slangpy_ext/device/cursor_writer.h` and `src/slangpy_ext/device/cursor_writer.cpp`. It takes a `nanobind` Python object, asks nanobind whether the object is backed by one of the registered C++ types, and returns a `NativeCursorWriterValue` containing the registry entry and the extracted C++ pointer. The hot-path lookup caches by `PyTypeObject*`, but only for native nanobind-backed types. + +The current Python cursor write tables live mainly in `src/slangpy_ext/device/cursor_utils.h`. `WriteConverterTable::write_registered_native_object(...)` calls `sgl::slangpy::find_native_cursor_writer(...)`. The shader cursor binding in `src/slangpy_ext/device/shader_cursor.cpp` uses `WriteConverterTable` to implement `write_shader_cursor(...)`. + +`std::type_info` is the C++ runtime type descriptor used by `typeid(T)`. Nanobind uses it to test and extract C++ objects from Python wrappers. `PyTypeObject` is the CPython runtime type descriptor for Python classes, but this plan never exposes that type name to core SGL. Core stores Python type pointers only as opaque `const void*` values and lets the extension interpret them. + +`SignatureBuffer` is the native cache-signature builder in `src/sgl/core/signature_buffer.h`. A cursor-writer entry must be able to add a stable signature for a value so SlangPy can cache generated call data. Native values do this through C++ callbacks today; Python-backed entries may use a static signature string or call a Python signature callback. + +## Plan of Work + +Milestone 1 changes the core registry key without adding Python support. In `src/sgl/device/cursor_utils.h`, introduce: + + enum class CursorWriterTypeKeyKind { + native_type_info, + python_type, + }; + +Replace `CursorWriterTypeInfo::type` with: + + CursorWriterTypeKeyKind key_kind{CursorWriterTypeKeyKind::native_type_info}; + const void* type_key{nullptr}; + std::string debug_name; + +Keep the existing erased writer fields, `slang_type_name`, `write_signature`, and `imports`. Native registration through `register_cursor_writer()` should set `key_kind` to `native_type_info`, `type_key` to `&typeid(T)`, and `debug_name` to `detail::type_name()` or `typeid(T).name()`. + +In `src/sgl/device/cursor_utils.cpp`, update `register_cursor_writer_type(...)` to validate `type_key`, use `debug_name` in error messages, and reject duplicates by the pair `(key_kind, type_key)`. Keep `find_cursor_writer_type_info(const std::type_info& type)`, but make it search only entries whose `key_kind` is `native_type_info` and compare `*static_cast(info.type_key)` against `type`. Add: + + SGL_API void unregister_cursor_writer_types(CursorWriterTypeKeyKind key_kind); + SGL_API uint64_t cursor_writer_registry_generation(); + +`unregister_cursor_writer_types(...)` removes all entries of that key kind. `cursor_writer_registry_generation()` returns a monotonically increasing counter that changes whenever entries are added or removed. The generation lets extension caches drop stale pointers after vector reallocation or cleanup. + +Milestone 2 updates the existing extension resolver to understand the new native key representation. In `src/slangpy_ext/device/cursor_writer.cpp`, replace all `info.type` uses with native-only helper code: + + const std::type_info& native_type_info(const CursorWriterTypeInfo& info) + +This helper asserts that `info.key_kind == CursorWriterTypeKeyKind::native_type_info` and casts `info.type_key` back to `const std::type_info*`. `native_cursor_writer_pointer(...)` continues to take `const std::type_info&` because nanobind needs that exact type descriptor. + +The uncached native lookup should skip non-native entries while doing nanobind `nb_type_isinstance(...)` scans. The exact-type fast path should keep using `cursor_utils::find_cursor_writer_type_info(...)`. The cache should store the registry generation, not only registry size. On each lookup, if `cursor_utils::cursor_writer_registry_generation()` has changed, clear the cache before reading a cached pointer. + +Milestone 3 adds Python-keyed registration in the extension while still writing into the same core registry. In `src/slangpy_ext/device/cursor_writer.h`, add an extension-only registration API. The exact shape can be adjusted to match nanobind conventions, but it must accept a Python type and optional callbacks: + + void register_python_cursor_writer_type( + nb::type_object python_type, + nb::object write_shader_cursor, + nb::object write_buffer_cursor, + nb::object write_signature, + std::string slang_type_name, + std::vector imports + ); + +If a callback is `None`, leave the corresponding erased function empty. At least one cursor writer and one signature writer must exist, matching the core validation contract. A static signature string may be converted into a C++ lambda before registration so callers do not need to provide a dynamic Python signature callback. + +The Python registration function builds a `CursorWriterTypeInfo` with `key_kind = CursorWriterTypeKeyKind::python_type`, `type_key = python_type.ptr()`, and a readable `debug_name` such as `module.qualname`. The writer lambdas capture `nb::object` callbacks. They receive `const void* value`, interpret it as the original `PyObject*`, create a borrowed `nb::object`, and call the Python callback: + + callback(cursor, nb::borrow(reinterpret_cast(const_cast(value)))); + +Do not acquire the GIL inside these lambdas. Add a comment stating that Python cursor-writer callbacks are invoked only from Python-entered paths where the GIL is already held. + +Milestone 4 unifies lookup results for native and Python-keyed entries. Rename the extension result type from `NativeCursorWriterValue` to a neutral name such as `CursorWriterValue`, and rename `find_native_cursor_writer(...)` to `find_cursor_writer(...)`. If this churn is too broad, keep the old names as wrappers during the transition, but new code should use the neutral names. + +The unified resolver in `src/slangpy_ext/device/cursor_writer.cpp` should handle both key kinds. For native entries, it keeps the current nanobind behavior and returns the extracted C++ pointer. For Python entries, it should first compare exact Python type pointer: + + Py_TYPE(obj.ptr()) == reinterpret_cast(const_cast(info.type_key)) + +If exact comparison fails, it may use `PyObject_IsInstance(...)` or an explicit MRO scan so registrations on Python base classes also work. Return the original Python object pointer as `value` for Python entries. Cache both hits and misses by `PyTypeObject*` and registry generation. + +Update `src/slangpy_ext/device/cursor_utils.h` so `WriteConverterTable::write_registered_native_object(...)` becomes `write_registered_object(...)` or similarly neutral and calls the unified resolver. Its call site should not know whether the value pointer is a C++ object or a Python object; it only invokes the erased writer stored in the registry entry. + +Milestone 5 wires cleanup into shutdown. In `src/sgl/sgl.cpp`, call: + + cursor_utils::unregister_cursor_writer_types(CursorWriterTypeKeyKind::python_type); + +near the start of `sgl::static_shutdown()` after the reference-count early return and before waiting for tasks or tearing down other subsystems. This preserves process-lifetime native registrations while destroying Python-backed callbacks before interpreter finalization. If later testing shows `static_shutdown()` can run after Python finalization in some embedding scenario, add an explicit call from the existing `atexit` hook in `src/slangpy_ext/slangpy_ext.cpp` while the GIL is held, but keep the core cleanup API as the owner of registry removal. + +Milestone 6 exposes a Python-level registration function only if needed for tests and user-facing use. The existing `slangpy/bindings/cursor.py::register_cursor_writer_marshal(...)` currently registers functional fallback metadata in Python dictionaries; it does not register runtime cursor writer callbacks into the core registry. Add a separate public helper only if the implementation needs pure Python users to register cursor writers directly. A possible name is `register_cursor_writer_type(...)`, but avoid overloading it with the existing functional marshall helper unless both operations are intentionally combined. The helper should call the native extension function from Milestone 3. + +Milestone 7 adds tests and keeps existing behavior stable. Native C++ tests under `tests/sgl/device/test_cursors.cpp` should cover key-kind duplicate rejection, native lookup after the opaque key migration, and unregistering only Python-kind entries. Python tests under `slangpy/tests/slangpy_tests/` should register a small Python class with a cursor writer callback, write it into a struct-shaped cursor, and verify the resulting data matches the callback's fields. Existing tests for `WriteToCursorMarshall`, native resource cursor writing, tensors, and descriptor handles should still pass. + +## Concrete Steps + +Work from the repository root: + + cd C:\sw\slangpy + +Before editing, inspect the current relevant code: + + rg -n "CursorWriterTypeInfo|find_native_cursor_writer|register_cursor_writer_type|static_shutdown" src/sgl src/slangpy_ext slangpy tests + +Implement Milestone 1 in `src/sgl/device/cursor_utils.h` and `src/sgl/device/cursor_utils.cpp`. Keep the public API source-compatible where possible by preserving `find_cursor_writer_type_info(const std::type_info&)`. + +Implement Milestones 2 through 4 in `src/slangpy_ext/device/cursor_writer.h`, `src/slangpy_ext/device/cursor_writer.cpp`, `src/slangpy_ext/device/cursor_utils.h`, and any direct callers found by `rg "find_native_cursor_writer|NativeCursorWriterValue|get_native_cursor_writer_type_info"`. + +Implement Milestone 5 in `src/sgl/sgl.cpp`. If the extension needs a test-only cleanup function, put it in `src/slangpy_ext/device/cursor_writer.cpp` and expose it only as a private `_...` function through the nanobind module. + +Add or update tests. Prefer focused test names that state the behavior: + + register_cursor_writer_rejects_duplicate_key_kind_and_key + unregister_cursor_writer_types_removes_python_keyed_entries + test_python_cursor_writer_callback_writes_struct + +Build before running tests, as required by this repository: + + cmake --build --preset windows-msvc-debug --target slangpy_ext sgl_tests + +Run focused C++ tests. Adjust the exact doctest filters to match the final test names: + + build\windows-msvc\Debug\sgl_tests.exe --test-case="*cursor_writer*" + build\windows-msvc\Debug\sgl_tests.exe --test-case="*write_to_cursor*" + +Run focused Python tests: + + python -m pytest slangpy/tests/slangpy_tests/test_write_to_cursor_marshall.py -v -p no:cacheprovider + python -m pytest slangpy/tests/slangpy_tests/test_python_cursor_writer.py -v -p no:cacheprovider + +Run pre-commit after implementation: + + pre-commit run --all-files + +If pre-commit modifies files, inspect the diff, then rerun `pre-commit run --all-files` until it passes. + +## Validation and Acceptance + +The implementation is accepted when all of these behaviors are true. + +Existing native cursor writers still work. Passing a native `Buffer`, `Texture`, `Sampler`, `ShaderObject`, `AccelerationStructure`, or `Tensor` to the existing cursor-writing paths succeeds exactly as before. The focused C++ cursor tests and existing Python write-to-cursor marshall tests pass. + +A pure Python type can be registered without adding another registry. The registration stores a `CursorWriterTypeInfo` in the core `cursor_utils` registry with `key_kind == CursorWriterTypeKeyKind::python_type` and an opaque Python type pointer. The core registry does not include Python headers and does not mention `PyTypeObject`. + +The unified resolver returns the same result shape for native and Python values. For native values, `value` is the extracted C++ pointer. For Python values, `value` is the original Python object pointer. The writer call site invokes `info.write_shader_cursor(cursor, value)` or `info.write_buffer_cursor(cursor, value)` without branching on native versus Python. + +Python-backed callbacks are destroyed during shutdown. A test or debug assertion should show that calling `cursor_utils::unregister_cursor_writer_types(CursorWriterTypeKeyKind::python_type)` removes Python-keyed entries and advances the registry generation. Static shutdown calls this cleanup path. Native entries are not removed by this cleanup. + +The extension lookup cache cannot return stale pointers after registration or cleanup. Any lookup cache in `src/slangpy_ext/device/cursor_writer.cpp` compares a stored generation against `cursor_utils::cursor_writer_registry_generation()` and clears itself when the generation changes. + +Expected successful command summary: + + cmake --build --preset windows-msvc-debug --target slangpy_ext sgl_tests + # build completes with no errors + + build\windows-msvc\Debug\sgl_tests.exe --test-case="*cursor_writer*" + # all selected doctest cases pass + + python -m pytest slangpy/tests/slangpy_tests/test_python_cursor_writer.py -v -p no:cacheprovider + # the pure Python cursor writer test passes + + pre-commit run --all-files + # all hooks pass + +## Idempotence and Recovery + +The implementation should be safe to apply incrementally. The opaque key migration is mechanical and can be verified before adding Python entries. If a later milestone fails, keep Milestone 1 compiling and temporarily leave Python registration unexposed. + +`register_cursor_writer_type(...)` must continue to reject duplicate registrations for the same `(key_kind, type_key)` pair. Re-running `sgl::static_init()` without a matching process restart should not silently add duplicate native entries. This plan does not require changing existing native-entry lifetime, but it must not make duplicate behavior worse. + +`unregister_cursor_writer_types(CursorWriterTypeKeyKind::python_type)` must be safe to call more than once. Calling it when no Python entries exist should leave the registry unchanged except that it may advance the generation only if entries were actually removed. Prefer advancing the generation only on mutation so caches are not cleared unnecessarily. + +If a Python callback raises, preserve the existing cursor-write error wrapping style in `WriteConverterTable`: include the Python value type in the error message and keep the cursor path stack context. Do not swallow the exception. + +If a test creates Python cursor-writer registrations, make it unregister them in a `finally` block or through a test fixture so later tests do not inherit registrations. + +## Artifacts and Notes + +The central native type shape should end up close to this. The exact field order may change to match local style: + + enum class CursorWriterTypeKeyKind { + native_type_info, + python_type, + }; + + struct CursorWriterTypeInfo { + CursorWriterTypeKeyKind key_kind{CursorWriterTypeKeyKind::native_type_info}; + const void* type_key{nullptr}; + std::string debug_name; + + ShaderCursorObjectWriteFunc write_shader_cursor; + BufferElementCursorObjectWriteFunc write_buffer_cursor; + + std::string slang_type_name; + std::function write_signature; + std::vector imports; + }; + +Native registration should remain concise: + + template + void register_cursor_writer() + requires(CanRegisterCursorWriter) + { + CursorWriterTypeInfo info; + info.key_kind = CursorWriterTypeKeyKind::native_type_info; + info.type_key = &typeid(T); + info.debug_name = std::string(detail::type_name()); + ... + register_cursor_writer_type(std::move(info)); + } + +Python registration should be visibly the same registry operation: + + CursorWriterTypeInfo info; + info.key_kind = CursorWriterTypeKeyKind::python_type; + info.type_key = python_type.ptr(); + info.debug_name = python_debug_name(python_type); + info.write_shader_cursor = [callback = nb::object(write_shader_cursor)](ShaderCursor& cursor, const void* value) + { + callback(cursor, nb::borrow(reinterpret_cast(const_cast(value)))); + return true; + }; + cursor_utils::register_cursor_writer_type(std::move(info)); + +Do not place this Python lambda in core SGL. The lambda is constructed in `src/slangpy_ext/`, where `nanobind.h` and Python types are already available. Core SGL only stores and later destroys the erased `std::function`. + +## Interfaces and Dependencies + +The core native API in `src/sgl/device/cursor_utils.h` must provide: + + enum class CursorWriterTypeKeyKind; + struct CursorWriterTypeInfo; + void register_cursor_writer_type(CursorWriterTypeInfo info); + std::span cursor_writer_type_infos(); + const CursorWriterTypeInfo* find_cursor_writer_type_info(const std::type_info& type); + void unregister_cursor_writer_types(CursorWriterTypeKeyKind key_kind); + uint64_t cursor_writer_registry_generation(); + template void register_cursor_writer(); + +Core SGL must not include `nanobind.h`, `Python.h`, or mention `PyTypeObject`. + +The extension API in `src/slangpy_ext/device/cursor_writer.h` should provide neutral lookup names: + + struct CursorWriterValue { + const cursor_utils::CursorWriterTypeInfo* info; + const void* value; + }; + + std::optional find_cursor_writer(nb::handle obj); + nb::object get_cursor_writer_type_info(nb::handle obj); + void register_python_cursor_writer_type(...); + +If existing Python or C++ code still expects `_get_native_cursor_writer_type_info` or `find_native_cursor_writer`, keep compatibility wrappers temporarily, but route them through the new neutral functions. + +The Python-facing helper, if added, belongs near `slangpy/bindings/cursor.py` because that file already owns `WriteToCursorMarshall` registration concepts. Keep the naming clear so users can tell whether they are registering runtime cursor callbacks, functional marshall metadata, or both. + +## Revision Notes + +2026-05-29: Initial ExecPlan created. It captures the agreed design: one core registry, an enum key kind plus opaque key, Python callbacks stored as erased lambdas constructed in the extension, no GIL acquisition inside hot callbacks, and cleanup of Python-backed entries during `sgl::static_shutdown()`. From ca2b820b78fc2c52ada41be69787676b955683e4 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Wed, 1 Jul 2026 10:14:55 +0100 Subject: [PATCH 06/17] Try disabling linux package installs --- .github/actions/build-and-test-with-slang/action.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/build-and-test-with-slang/action.yml b/.github/actions/build-and-test-with-slang/action.yml index 86c255d52..c9c5b2c86 100644 --- a/.github/actions/build-and-test-with-slang/action.yml +++ b/.github/actions/build-and-test-with-slang/action.yml @@ -42,12 +42,12 @@ runs: git submodule foreach --recursive git clean -ffdx git submodule foreach --recursive git reset --hard - # Setup Linux. - - name: Setup Linux - if: startsWith(inputs.os, 'linux') - shell: bash - run: | - sudo apt update && sudo apt install -y libxinerama-dev libxcursor-dev xorg-dev libglu1-mesa-dev pkg-config +# # Setup Linux. +# - name: Setup Linux +# if: startsWith(inputs.os, 'linux') +# shell: bash +# run: | +# sudo apt update && sudo apt install -y libxinerama-dev libxcursor-dev xorg-dev libglu1-mesa-dev pkg-config # Setup Python. - name: Setup Python ${{ inputs.python }} From 94ac951a9fa3c9816cb875be9642749bb1931a2d Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Wed, 1 Jul 2026 11:44:03 +0100 Subject: [PATCH 07/17] Disable gpu clock locking for now on linux --- .github/workflows/ci-benchmark.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-benchmark.yml b/.github/workflows/ci-benchmark.yml index 7ab4b26cf..fa36a6c6b 100644 --- a/.github/workflows/ci-benchmark.yml +++ b/.github/workflows/ci-benchmark.yml @@ -44,12 +44,6 @@ jobs: git submodule foreach --recursive git clean -ffdx git submodule foreach --recursive git reset --hard - # Setup Linux. - - name: Setup Linux - if: startsWith(matrix.os, 'linux') && contains(matrix.runs-on, 'ubuntu-') - run: | - sudo apt update && sudo apt install -y libxinerama-dev libxcursor-dev xorg-dev libglu1-mesa-dev pkg-config - # Setup Python. - name: Setup Python ${{ matrix.python }} uses: actions/setup-python@v6 @@ -115,6 +109,10 @@ jobs: run: python tools/ci.py build # Benchmark (Python) - - name: Benchmark (Python) - if: contains(matrix.flags, 'benchmark') + - name: Benchmark (Python, Windows, GPU Clock Locked) + if: contains(matrix.flags, 'benchmark') && runner.os == 'Windows' run: python tools/ci.py benchmark-python --run-id "${{ github.run_id }}" --mongodb-connection-string "${{ secrets.BENCHMARK_MONGODB_CONNECTION_STRING }}" --mongodb-database-name "nvr-ci" --lock-gpu-clocks + + - name: Benchmark (Python, Linux, GPU Clock Unlocked) + if: contains(matrix.flags, 'benchmark') && runner.os == 'Linux' + run: python tools/ci.py benchmark-python --run-id "${{ github.run_id }}" --mongodb-connection-string "${{ secrets.BENCHMARK_MONGODB_CONNECTION_STRING }}" --mongodb-database-name "nvr-ci" From 12c0b9175cf8a61d0de6fbe3be4abb62b5b01203 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Wed, 1 Jul 2026 15:31:22 +0100 Subject: [PATCH 08/17] PR cleanup --- .../build-and-test-with-slang/action.yml | 7 - .plans/bindings.md | 547 ------------------ .plans/bindings2.md | 326 ----------- 3 files changed, 880 deletions(-) delete mode 100644 .plans/bindings.md delete mode 100644 .plans/bindings2.md diff --git a/.github/actions/build-and-test-with-slang/action.yml b/.github/actions/build-and-test-with-slang/action.yml index c9c5b2c86..bab81064c 100644 --- a/.github/actions/build-and-test-with-slang/action.yml +++ b/.github/actions/build-and-test-with-slang/action.yml @@ -42,13 +42,6 @@ runs: git submodule foreach --recursive git clean -ffdx git submodule foreach --recursive git reset --hard -# # Setup Linux. -# - name: Setup Linux -# if: startsWith(inputs.os, 'linux') -# shell: bash -# run: | -# sudo apt update && sudo apt install -y libxinerama-dev libxcursor-dev xorg-dev libglu1-mesa-dev pkg-config - # Setup Python. - name: Setup Python ${{ inputs.python }} uses: actions/setup-python@v6 diff --git a/.plans/bindings.md b/.plans/bindings.md deleted file mode 100644 index e4fc9b054..000000000 --- a/.plans/bindings.md +++ /dev/null @@ -1,547 +0,0 @@ -# Cursor Writer Binding Plan - -## Goal - -Replace the current split system with one native registration path for cursor-writable value types, with optional simple functional-API fallback metadata. - -The desired user model is: - -1. A native value type implements `write_to_cursor` for both `ShaderCursor` and `BufferElementCursor`. -2. If the type wants the simple `WriteToCursorMarshall` functional fallback, it provides its static Slang type name directly on the class. -3. Functional fallback types may provide a static or dynamic signature directly on the class; if they do not, the signature defaults to the C++ class name. -4. The project calls one registration function: - -```cpp -sgl::cursor_utils::register_cursor_writer(); -``` - -That single call always registers direct cursor writing. If class-owned Slang metadata is present, it also registers functional-API fallback marshalling through `WriteToCursorMarshall`. - -The new policy should be: - -- `get_this` is the legacy wrapper path. -- `write_to_cursor` plus `register_cursor_writer()` is the new bindable-value path. -- A single type must not use both paths. -- Native registered cursor writers should be identified from the native registry, not from Python-side compatibility predicates. -- The old cursor-writer caches and `has_registered_type_or_signature` compatibility path should be removed as part of this migration, not preserved as fallback behavior. -- The registered class is the only metadata surface. Do not introduce a separate traits structure. -- `slang_type_name` is required only for the simple functional fallback path; when present it is always static and class-owned. -- Signature metadata is optional for functional fallback types, class-owned, and may be static or value-aware. -- Imports are optional for functional fallback types, static, class-owned, and copied once during registration. -- Public `register_cursor_writer()` is all-or-nothing for direct cursor writing: the type must provide both cursor-write paths. Functional metadata is an optional bundle so resource types can register direct cursor writers while keeping bespoke functional marshalls. - -## User Class Contract - -Minimal static-signature form for simple functional fallback: - -```cpp -struct MyHandle { - static constexpr std::string_view slang_type_name = "MyHandle"; - static constexpr std::string_view slangpy_signature = "MyHandle"; - - uint32_t id = 0; - uint32_t flags = 0; - - template - void write_to_cursor(TCursor& cursor) const - { - cursor["id"] = id; - cursor["flags"] = flags; - } - - static std::vector slangpy_imports() - { - return {"my_project/my_handle.slang"}; - } -}; -``` - -Dynamic-signature form: - -```cpp -struct TypedHandle { - static constexpr std::string_view slang_type_name = "TypedHandle"; - - uint32_t id = 0; - uint32_t kind = 0; - - template - void write_to_cursor(TCursor& cursor) const - { - cursor["id"] = id; - cursor["kind"] = kind; - } - - void write_slangpy_signature(SignatureBuffer& sig) const - { - sig.add("TypedHandle:"); - sig.add(kind); - } -}; -``` - -Writer-only form for types that have a bespoke functional API marshall: - -```cpp -struct MyResourceHandle { - template - void write_to_cursor(TCursor& cursor) const - { - cursor["id"] = id; - } -}; -``` - -The `write_to_cursor` implementation may be a templated cursor overload or two explicit overloads, but it must compile for both `ShaderCursor&` and `BufferElementCursor&`. If functional metadata is present and neither `slangpy_signature` nor `write_slangpy_signature(...)` exists, registration should use the C++ class name as the default signature. This signature is only a cache key; it is not used as a visible Slang type name. - -## Correction Work Completed - -The implementation has been corrected to this contract: - -- Removed `CursorWriterTraits` and all trait-specific detection. -- Removed value-aware `slang_type_name()` support; `T::slang_type_name` is the only Slang type name source. -- `register_cursor_writer()` now requires `write_to_cursor(...)` support for both `ShaderCursor&` and `BufferElementCursor&`. -- `T::slang_type_name` is optional and enables simple functional fallback metadata when present. -- `write_signature(SignatureBuffer&, const void*)` remains in the registry so static and dynamic signatures share one call path. -- Signature sources are detected in this order: - 1. `value.write_slangpy_signature(SignatureBuffer&) const` for dynamic signatures. - 2. `T::write_slangpy_signature(SignatureBuffer&)` for static function signatures. - 3. `T::slangpy_signature` for static string signatures. - 4. Default C++ class-name signature. -- The registry stores `slang_type_name` as copied registration data for functional fallback entries, not as a `const void*` callback. -- `slangpy_imports()` is static and class-owned only; it is called once at registration and its strings are copied for functional fallback entries. -- Tests now put `slang_type_name`, signatures, and imports on the registered class itself. -- Tests cover missing `T::slang_type_name` as writer-only registration and one-cursor-only writers as compile-time rejections via `CanRegisterCursorWriter`. -- Removed the legacy low-level writer wrapper APIs; the native registry now accepts only complete `register_cursor_writer()` entries. - -## Progress So Far - -Latest progress update: - -- The public cursor-writer registration contract has been tightened to require both cursor paths while keeping functional metadata optional. `CanRegisterCursorWriter` now requires `write_to_cursor(ShaderCursor&)` and `write_to_cursor(BufferElementCursor&)`; `CanRegisterFunctionalCursorWriter` additionally requires `T::slang_type_name`. -- One-cursor-only types remain useful for cursor `set()` contract tests, but they are compile-time rejected for public `register_cursor_writer()` registration. -- Native tests now cover writer-only registrations, default class-name signatures, static string signatures, static function signatures, dynamic value-aware signatures, copied imports, duplicate registration rejection, partial functional metadata rejection, and one-cursor-only compile-time rejection under the stricter contract. -- Final verification for this checkpoint passed after clang-format updates and a full `pre-commit run --all-files`. - -Completed in the current implementation slice: - -- `SignatureBuffer` moved to `src/sgl/core/signature_buffer.h` in namespace `sgl`. -- `SignatureBuilder` remains the Python-facing wrapper and now includes the native `SignatureBuffer` header. -- `src/sgl/CMakeLists.txt` includes the new header. -- Added native C++ coverage in `tests/sgl/core/test_signature_buffer.cpp`. -- `HasWriteToCursor` now checks the actual `obj.write_to_cursor(cursor)` call expression. -- `ShaderCursor::set()` now checks `HasWriteToCursor`. -- `ShaderCursor::set()` and `operator=` are non-const, so `write_to_cursor(*this)` writes through the original cursor without copying it. -- Added native C++ coverage in `tests/sgl/device/test_cursors.cpp` for cursor-specific `set()` behavior, both-overload registration, and negative concept checks. -- Introduced a combined native `CursorWriterTypeInfo` registry in `src/sgl/device/cursor_utils.h/.cpp`. -- Added `cursor_utils::register_cursor_writer()`. -- Removed legacy low-level cursor-writer registration functions; the public helper still requires both cursor writers, while internal registry entries may provide only the cursor/signature hooks they actually support. -- `src/slangpy_ext/device/cursor_utils.h` direct cursor writes now consult the combined native registry and no longer keep `WriteConverterTable::m_native_object_writer_cache`. -- Static metadata and imports are captured into the native descriptor at registration time from class-owned metadata only when the type opts into functional fallback. -- Added native C++ coverage for registry lookup, duplicate rejection, one-cursor registration rejection, and static metadata/signature/imports. -- Renamed the Python marshall API to `WriteToCursorMarshall`, `WriteToCursorMarshallInfo`, and `register_cursor_writer_marshal`. -- `WriteToCursorMarshall` now derives from `NativeValueMarshall`, so dispatch uses the native cursor-write fast path rather than Python `create_calldata()`. -- `slangpy/bindings/typeregistry.py` now falls back to native cursor-writer metadata when Python type registration has no hit. -- `NativeCallDataCache::get_value_signature()` now uses native cursor-writer signature metadata independently from functional fallback metadata; Buffer/Texture register through `register_cursor_writer()` and keep bespoke Python marshalls. -- Removed the old `has_registered_type_or_signature` Python predicate, native callback, and native predicate cache. -- `get_this` remains the legacy wrapper path for objects not owned by native signature or functional cursor-writer metadata; registered native-owned values with `get_this` fail with a clear conflict error. - -Verification run for this slice: - -```powershell -cmake --build --preset windows-msvc-debug --target slangpy_ext sgl_tests -build\windows-msvc\Debug\sgl_tests.exe --test-suite=signature_buffer,cursors -python -m pytest slangpy/tests/slangpy_tests/test_write_to_cursor_marshall.py -v -p no:cacheprovider -python -m pytest slangpy/tests/slangpy_tests/test_type_resolution.py -v -p no:cacheprovider -python -m pytest slangpy/tests/slangpy_tests/test_instances.py::test_packed_vs_unpacked_cache -v -p no:cacheprovider --device-types d3d12 -pre-commit run --files .plans/bindings.md src/sgl/device/cursor_utils.cpp src/sgl/device/cursor_utils.h src/slangpy_ext/utils/slangpy.cpp tests/sgl/device/test_cursors.cpp -pre-commit run --all-files -``` - -## Phase 1: Move SignatureBuffer To Native - -Status: complete. - -Move `SignatureBuffer` out of `src/slangpy_ext/utils/slangpy.h` and into a pure native header: - -```text -src/sgl/core/signature_buffer.h -``` - -Use namespace `sgl` for the native type: - -```cpp -namespace sgl { -class SignatureBuffer; -} -``` - -Keep `SignatureBuilder` in `sgl::slangpy` as the Python-facing object wrapper around `sgl::SignatureBuffer`. - -Expected edits: - -- Add `src/sgl/core/signature_buffer.h`. -- Move the existing implementation unchanged in behavior: - - stack-owned object - - non-copyable - - `short_vector` inline storage - - `std::string_view view() const` - - existing uint32/uint64 lower-case hex formatting -- Add the new header to `src/sgl/CMakeLists.txt`. -- Include the new header from `src/slangpy_ext/utils/slangpy.h`. -- Remove the local `SignatureBuffer` definition from the extension header. -- Update extension code to refer to `sgl::SignatureBuffer` where qualification is needed. -- Keep existing `SignatureBuilder` behavior unchanged. - -Tests to change: - -- Existing signature tests should continue to call the Python-visible `SignatureBuilder` and `get_value_signature` APIs without behavior changes. -- Any C++ files including `utils/slangpy.h` only for `SignatureBuffer` should include `sgl/core/signature_buffer.h` instead. - -Tests to add: - -- Add a C++ unit test for `SignatureBuffer` through the native C++ test system: - - appends strings and integer values - - preserves byte order for existing integer hex formatting - - returns the expected `std::string_view` - -## Phase 2: Fix The Cursor Contract - -Status: complete. - -Make each cursor check the cursor type it actually passes. The existing `write_to_cursor` implementation is already the contract; this phase should be a narrow correctness fix, not a redesign. - -Expected edits: - -- In `src/sgl/device/cursor_utils.h`, keep the existing `HasWriteToCursor` style unless a small compile fix is required. -- In `src/sgl/device/shader_cursor.h`, change `ShaderCursor::set()` to check `HasWriteToCursor`, not `BufferElementCursor`. -- Keep `BufferElementCursor::set()` checking `HasWriteToCursor`. -- `ShaderCursor::set()` should not be `const`. This avoids copying the cursor just to pass a mutable `ShaderCursor&` into `write_to_cursor`. - -```cpp -value.write_to_cursor(*this); -``` - -Tests to change: - -- Update any existing tests or examples that relied on the incorrect `ShaderCursor` concept check. - -Tests to add: - -- C++ coverage for a type that only implements `write_to_cursor(ShaderCursor&)`. -- C++ coverage for a type that only implements `write_to_cursor(BufferElementCursor&)`. -- C++ coverage for a type that implements both cursor overloads. -- A negative compile-time test with `static_assert(!HasWriteToCursor<...>)` for a type without `write_to_cursor`. - -## Phase 3: Introduce The Combined Native Registry - -Status: complete for native direct cursor-write registration. Functional-API consumption is implemented in Phase 7. - -Replace the current split native writer registries with one combined registry that owns cursor writer functions and optional functional-API metadata. - -The registry should be native-first. It should not rely on Python callbacks to determine whether a type is registered. - -Expected edits: - -- Introduce a descriptor in native code, likely near `src/sgl/device/cursor_utils.h`: - -```cpp -struct CursorWriterTypeInfo { - const std::type_info* type; - - ShaderCursorObjectWriteFunc write_shader_cursor; - BufferElementCursorObjectWriteFunc write_buffer_cursor; - - // Functional API metadata from T::slang_type_name and class-owned signature hooks. - std::string slang_type_name; - std::function write_signature; - std::vector imports; -}; -``` - -- Store one registry of `CursorWriterTypeInfo`. -- Provide exact-type lookup by `std::type_info` / `std::type_index`. -- Expose read-only access and lookup helpers for the extension. -- Remove the separate `shader_cursor_object_writer_registry()` and `buffer_element_cursor_object_writer_registry()` storage. -- Remove `WriteConverterTable::m_native_object_writer_cache`; registered native object lookup should consult the combined registry directly. -- Delete the low-level `register_shader_cursor_object_writer*` / `register_buffer_element_cursor_object_writer*` APIs so they do not remain as a second registration model. -- Duplicate registration of a registered type should fail with a clear error. -- Define whether base-type lookup is supported for nanobind-exposed derived types. If supported, document lookup precedence: exact native type first, then registered base type. - -Tests to change: - -- Existing native-object cursor writer tests should move to the combined registration API. - -Tests to add: - -- Registering one type creates one registry entry with the expected cursor writers and metadata state. -- Duplicate registration of the same type is rejected with a clear error. -- Public `register_cursor_writer()` does not accept one-cursor-only types. -- Registry lookup by exact native type succeeds. -- If base lookup is supported, derived nanobind/Python-exposed types resolve to the registered base writer. - -## Phase 4: Add `register_cursor_writer()` - -Status: complete. - -Make `register_cursor_writer()` the single public native API. - -Expected edits: - -- Add: - -```cpp -template -void register_cursor_writer(); -``` - -- Require `T` to satisfy both `HasWriteToCursor` and `HasWriteToCursor`. -- Register both cursor writer functions. -- Optionally capture `T::slang_type_name` when it is static and convertible to `std::string_view`. -- Do not support value-aware `slang_type_name()` and do not infer a Slang type from the native C++ type name. -- Register functional metadata for `register_cursor_writer()` only when `T::slang_type_name` is present. -- Direct cursor writing without SlangPy functional metadata is valid for types that keep bespoke functional marshalls. - -Functional metadata sources: - -- For functional fallback, the class must provide: - -```cpp -static constexpr std::string_view slang_type_name = "MySlangType"; -``` - -- The class may provide any one of these signature forms: - -```cpp -static constexpr std::string_view slangpy_signature = "MyTypeSignature"; -static void write_slangpy_signature(SignatureBuffer& sig); -void write_slangpy_signature(SignatureBuffer& sig) const; -``` - -- The class may provide static imports: - -```cpp -static std::vector slangpy_imports(); -``` - -Signature policy: - -- If no class-owned signature is provided, default the signature to the C++ class name. -- Imports are constant for a registered type. Do not support value-aware imports. -- Dynamic signatures are allowed through `value.write_slangpy_signature(SignatureBuffer&) const`. -- The static Slang type name must not be used as the default signature unless it is also the desired class-name signature. -- Signatures should be compact and cheap to produce; do not stringify large values or import lists on the hot cache-key path. - -Tests to change: - -- Replace direct public calls to `register_shader_cursor_object_writer()` and `register_buffer_element_cursor_object_writer()` in tests with `register_cursor_writer()`. - -Tests to add: - -- Minimal type with `write_to_cursor` support for both cursor kinds and static `slang_type_name` can register. -- Minimal type missing static `slang_type_name` registers as writer-only with no functional metadata. -- Minimal type with only one cursor overload is rejected clearly. -- Minimal type without explicit signature defaults to the C++ class-name signature. -- Type with static metadata gets a stable signature. -- Type with dynamic signature gets a value-dependent signature. -- Type with static metadata gets its declared imports. -- Registration writes correctly through both `ShaderCursor` and `BufferElementCursor` when both overloads exist. - -## Phase 5: Capture Constant Imports At Registration - -Status: complete. - -Imports are a constant list of constant strings for a registered type. Keep this mechanism simple. - -Expected shape: - -```cpp -static std::vector slangpy_imports(); -``` - -Expected edits: - -- If a type provides a static `slangpy_imports()` function, call it once during registration. -- Copy the returned strings into the native registry descriptor as owned `std::vector`. -- Do not call imports callbacks during signature generation or dispatch. -- Do not support value-aware imports. -- Imports should be string literals or other stable strings. The returned container itself does not need stable storage because registration copies it immediately. - -Tests to add: - -- Static imports are added during code generation. -- The imports function is called once at registration, not per signature generation or dispatch. -- The registry owns copied import strings after registration. - -## Phase 6: Rename CursorMarshall To WriteToCursorMarshall And Require Native Fast Path - -Status: complete. - -Rename the marshall to match the contract it represents, and require the native fast dispatch path. - -Expected edits: - -- Rename `slangpy/bindings/cursor.py` classes and APIs: - - `CursorMarshall` -> `WriteToCursorMarshall` - - `CursorMarshallInfo` -> `WriteToCursorMarshallInfo` - - `register_cursor_type` -> `register_cursor_writer_marshal`, if keeping a Python-side registration API for Python-only future work -- Update exports in `slangpy/bindings/__init__.py`. -- Update error messages and tests to use the new name. -- `WriteToCursorMarshall` must derive from `NativeValueMarshall` or use an equivalent native cached write path. -- Repeated dispatches must not call Python `create_calldata()` to write the value. -- Read-only scalar direct binding should write the original native object through the native registered `write_shader_cursor` function. -- Writable/inout use is not supported unless a separate readback design is added. - -Tests to change: - -- Rename `slangpy/tests/slangpy_tests/test_cursor_marshall.py` to `test_write_to_cursor_marshall.py`. -- Update import names and expected error text. - -Tests to add: - -- `WriteToCursorMarshall` can resolve an exact Slang type name. -- `WriteToCursorMarshall` can add declared imports during code generation. -- `WriteToCursorMarshall` direct-binds a read-only scalar value. -- `WriteToCursorMarshall` rejects writable/inout use with a clear error. -- Repeated calls use the native dispatch write path and do not invoke Python `create_calldata()` for the cursor value. - -## Phase 7: Make The Extension Consume The Native Registry - -Status: complete for direct cursor writes, signature generation, and Python fallback construction. A full nanobind-exposed functional API smoke test remains as follow-up coverage. - -The extension should use the combined native registry for both direct cursor writes and functional-API marshalling. - -Expected edits: - -- In `src/slangpy_ext/device/cursor_utils.h`, replace `write_registered_native_object()` lookup with combined-registry lookup. -- Remove the old per-`PyTypeObject*` writer cache instead of preserving it. -- Add a native lookup path that identifies whether a nanobind object is backed by a registered native cursor writer type. -- Add a Python-visible or extension-internal helper that can create `WriteToCursorMarshall` from native registry metadata when Python `PYTHON_TYPES` has no hit. -- Have that helper read the copied `CursorWriterTypeInfo::slang_type_name` string directly instead of invoking a value-aware type-name callback. -- Update `slangpy/bindings/typeregistry.py::get_or_create_type()` or the `BoundVariable` construction path so native registered cursor writer values fall back to the native registry. -- Update `NativeCallDataCache::get_value_signature()` so native registered cursor writer signature metadata participates in signature generation without requiring simple functional fallback metadata. -- Define precedence: - - registered native signature metadata owns cache signatures for registered cursor writer objects - - registered functional cursor-writer metadata owns simple fallback marshalling for registered cursor writer objects - - existing built-in native signatures such as `Texture` and `Buffer` remain unchanged for types without functional cursor-writer metadata - - Python-only registered `PYTHON_TYPES` / `PYTHON_SIGNATURES` continue to work for non-native values - - `get_this` is blocked for objects with native signature or functional cursor-writer metadata; direct-write-only registrations may still use bespoke marshalling paths -- Conflict handling should occur when creating a marshall/signature for a Python-visible object. If a native-owned cursor writer object also exposes `get_this`, fail clearly. - -Tests to change: - -- Existing tests that manually register a Python marshall for a native object should switch to one native registration call where possible. -- Tests expecting registered types with `get_this` to bypass unpacking should be removed or rewritten to expect a conflict error. - -Tests to add: - -- A nanobind-exposed native test type registered with `register_cursor_writer()` can be passed directly to a SlangPy function. -- The same native type can be written directly to `ShaderCursor`. -- The same native type can be written to `BufferElementCursor`. -- A registered cursor writer type without `get_this` does not unpack. -- A type that tries to use both `get_this` and cursor writer registration fails with a clear error. -- Python-only `PYTHON_TYPES`, `PYTHON_SIGNATURES`, and `slangpy_signature` behavior still works for non-cursor-writer values. - -## Phase 8: Remove `has_registered_type_or_signature` - -Status: complete. - -Remove the brittle compatibility path that calls from native C++ back into Python to decide whether `get_this` should run. - -Expected edits: - -- Delete `has_registered_type_or_signature()` from `slangpy/bindings/typeregistry.py`. -- Delete `_has_registered_type_or_signature()` from `slangpy/core/calldata.py`. -- Delete the native `has_registered_type_or_signature()` helper from `src/slangpy_ext/utils/slangpy.cpp`. -- Remove its native static type cache. -- Restore simple unpack policy: - - if an object has `get_this`, legacy unpacking owns it - - if an object is registered as a cursor writer, direct marshalling owns it - - using both paths on one Python-visible type is an error -- Ensure native signature generation no longer imports `slangpy.bindings.typeregistry` for this predicate. - -Tests to add: - -- A legacy `get_this` wrapper still unpacks as before. -- A registered cursor writer type bypasses legacy unpacking by native registry detection, not by Python predicate. -- Native signature generation no longer calls back into Python for the registered-type predicate. - -## Phase 9: Keep Python-Only Types On The Same Contract - -Python-only support is a follow-up, but it should mirror the native model rather than introduce a new conceptual path. - -Expected future shape: - -```python -class TextureHandle: - slang_type_name = "TextureHandle" - - def write_to_cursor(self, cursor: object) -> None: - cursor["id"] = self.id - cursor["type"] = self.type - - def write_slangpy_signature(self, sig: object) -> None: - sig.add(f"TextureHandle:{self.type}") - - @staticmethod - def slangpy_imports() -> tuple[str, ...]: - return ("my_project/texture_handle.slang",) -``` - -Registration can remain Python-side for Python-only values: - -```python -register_cursor_writer_marshal(TextureHandle) -``` - -Do not require this branch to implement Python-only `write_to_cursor` dispatch. If implemented, add an explicit Python-method dispatch path; the current native cursor writer path only handles registered native objects and normal cursor-recursive values. - -Tests to add when implemented: - -- Python-only type with `write_to_cursor` can register. -- Python-only type requires explicit static class-owned `slang_type_name` for functional API use. -- Python-only type with value-aware metadata changes call signatures correctly. -- Python-only type with `get_this` conflict is rejected. - -## Phase 10: Public API Cleanup - -After the native registry and marshall rename are in place, clean up exported names. - -Expected edits: - -- Export the new API names: - - `WriteToCursorMarshall` - - `WriteToCursorMarshallInfo` - - `register_cursor_writer_marshal` -- Remove old public cursor-writer registration names unless they are still needed internally during the PR. -- Keep compatibility aliases only if needed while the PR is draft. -- Update documentation comments and examples to show `register_cursor_writer()`. - -Tests to change: - -- Update import tests and any docs snippets that mention `CursorMarshall` or `register_cursor_type`. - -Tests to add: - -- Import smoke test for the new Python names. -- Optional deprecation/compatibility test if old aliases are temporarily retained. - -## Final Verification - -Required by repository policy: - -1. Build before running tests. -2. Run focused C++ and Python tests for the changed paths. -3. Run broader SlangPy Python tests if build time allows. -4. Run pre-commit and rerun if it modifies files. - -Suggested commands on Windows: - -```powershell -cmake --build --preset windows-msvc-debug --target slangpy_ext sgl_tests -python tools/ci.py unit-test-cpp -python -m pytest slangpy/tests/slangpy_tests/test_write_to_cursor_marshall.py -v -python -m pytest slangpy/tests/slangpy_tests/test_type_resolution.py -v -python -m pytest slangpy/tests/slangpy_tests/test_instances.py -v -pre-commit run --all-files -``` diff --git a/.plans/bindings2.md b/.plans/bindings2.md deleted file mode 100644 index 655d31b1d..000000000 --- a/.plans/bindings2.md +++ /dev/null @@ -1,326 +0,0 @@ -# Unified Cursor Writer Registry For Python-Keyed Values - -This ExecPlan is a living document. The sections Progress, Surprises and Discoveries, Decision Log, and Outcomes and Retrospective must be kept up to date as work proceeds. - -This plan follows `.agents/PLANS.md` from the repository root. - -## Purpose / Big Picture - -SlangPy already has a native cursor-writer registry that lets C++ value types describe how they are written into a `ShaderCursor` or `BufferElementCursor`. The next step is to let pure Python object types participate in that same cursor-writing system without creating a second registry in `src/slangpy_ext/` and without making the pure native `src/sgl/` code include Python headers. - -After this change, a value type can be registered from either native C++ code or Python code, and the rest of the cursor-writing system can use one lookup result and one erased writer callback shape. A user should be able to register a Python class with a Python writer callback, pass an instance to `ShaderCursor.write(...)` or a buffer cursor write path, and observe the same behavior as if the value had been unpacked into a dictionary. Existing native values such as `Buffer`, `Texture`, `Sampler`, `ShaderObject`, and `Tensor` must continue to work through the same registry. - -This plan is intentionally focused on cursor writing and value signatures. Full functional-API marshall creation for Python-keyed types, such as making all NumPy array binding native-side, is a later extension and is not required for the first working slice. - -## Progress - -- [x] (2026-05-29 11:21Z) Wrote this ExecPlan from the current design discussion and repository inspection. -- [x] (2026-05-29) Replaced the native registry key with `CursorWriterTypeKeyKind`, an opaque `type_key`, `debug_name`, and a mutation generation counter. -- [x] (2026-05-29) Updated the extension resolver so native nanobind-backed objects and pure Python objects both resolve through the same core registry. -- [x] (2026-05-29) Added Python-backed cursor-writer registration in `src/slangpy_ext/` without adding a second registry. -- [x] (2026-05-29) Added cleanup for Python-backed registry entries during `sgl::static_shutdown()`. -- [x] (2026-05-29) Added C++ and Python tests that demonstrate both native and pure Python cursor writers use the unified registry. -- [x] (2026-05-29) Built `slangpy_ext` and `sgl_tests`, ran focused C++/Python tests, and ran `pre-commit run --all-files`. - -## Surprises and Discoveries - -- Observation: The current core registry is already process-wide and native-owned in `src/sgl/device/cursor_utils.cpp`, but its key is `const std::type_info*`. - Evidence: `cursor_writer_type_info_registry()` returns a static `std::vector`, and `find_cursor_writer_type_info(const std::type_info& type)` scans `info.type`. - -- Observation: Core SGL already has a central shutdown function that is called from the Python extension module free hook. - Evidence: `src/slangpy_ext/slangpy_ext.cpp` assigns `nanobind_slangpy_ext_module.m_free = [](void*) { sgl::static_shutdown(); };`. - -- Observation: Python-backed callbacks may hold `nb::object` references inside `std::function` lambdas, so they must be removed before Python finalization. - Evidence: The design stores Python callbacks as captured nanobind objects in registry entries, and captured `nb::object` destructors release Python references. - -## Decision Log - -- Decision: Use one core registry, not one native registry plus one extension registry. - Rationale: Cursor writing has the same erased operation after lookup: a function writes a value pointer into a cursor. Splitting registries would duplicate lookup and dispatch logic while producing the same final operation. - Date/Author: 2026-05-29 / Codex - -- Decision: Core SGL will store an opaque `const void* type_key` paired with a native-defined enum `CursorWriterTypeKeyKind`. - Rationale: Core must not include Python headers or mention `PyTypeObject`, but it can safely store an opaque pointer and an enum saying whether the pointer represents `std::type_info` or a Python type. This keeps the registry unified while keeping Python-specific interpretation inside `src/slangpy_ext/`. - Date/Author: 2026-05-29 / Codex - -- Decision: Use an enum rather than an open-ended domain pointer. - Rationale: The key kinds are expected to be closed for this repository: native C++ type info and Python type. An enum is clearer and easier to validate than a domain pointer. - Date/Author: 2026-05-29 / Codex - -- Decision: Python writer callbacks are invoked without acquiring the GIL inside each callback. - Rationale: The supported call paths are Python-entered and single-threaded, so the GIL is already held when cursor writing reaches these callbacks. If a future non-Python or background-thread entry point invokes Python callbacks, that boundary should acquire the GIL rather than adding overhead to every writer callback now. - Date/Author: 2026-05-29 / Codex - -- Decision: Python-backed entries are cleared during `sgl::static_shutdown()`. - Rationale: `static_shutdown()` is already called by the SlangPy module free hook. Clearing Python-backed entries there destroys captured `nb::object` callback references while the Python runtime is still alive. - Date/Author: 2026-05-29 / Codex - -- Decision: Marshall creation for Python-keyed values is out of scope for the first implementation. - Rationale: The immediate problem is unifying cursor-writer registration and dispatch. The same registry can later grow marshall factory metadata, but this slice should not combine that with the key/lifetime refactor. - Date/Author: 2026-05-29 / Codex - -## Outcomes and Retrospective - -Implementation is complete. Core SGL now owns a single cursor-writer registry that supports native C++ keys and opaque Python type keys while keeping all Python-specific interpretation in `src/slangpy_ext/`. Native values still use the nanobind extraction path, pure Python values pass their original `PyObject*` through the erased writer callback, and Python-backed entries are removed from the registry during `sgl::static_shutdown()`. - -Focused validation so far: - - cmake --build --preset windows-msvc-debug --target slangpy_ext sgl_tests - build\windows-msvc\Debug\sgl_tests.exe --test-case="*cursor_writer*" - build\windows-msvc\Debug\sgl_tests.exe --test-case="*write_to_cursor*" - python -m pytest slangpy/tests/slangpy_tests/test_write_to_cursor_marshall.py -v -p no:cacheprovider - python -m pytest slangpy/tests/slangpy_tests/test_python_cursor_writer.py -v -p no:cacheprovider - -One early run of `sgl_tests.exe --test-case="*write_to_cursor*"` failed while it was running in parallel with another `sgl_tests.exe` process. Rerunning the same command by itself passed. - -## Context and Orientation - -A cursor is an object that points at a value inside shader-visible data. `ShaderCursor` writes data into shader objects and root parameters. `BufferElementCursor` writes data into an element of a CPU-side buffer cursor. Both cursor types expose `set(...)` and `write(...)` behavior that eventually writes scalars, vectors, matrices, structs, arrays, resources, or registered custom objects. - -The current native cursor-writer registry lives in `src/sgl/device/cursor_utils.h` and `src/sgl/device/cursor_utils.cpp`. The descriptor is named `CursorWriterTypeInfo`. It currently stores `const std::type_info* type`, erased writer functions for shader and buffer cursors, optional Slang type metadata, a signature writer, and imports. Built-in native registrations are added by `cursor_utils::register_cursor_writers()` during `sgl::static_init()` in `src/sgl/sgl.cpp`. - -The current Python extension lookup lives in `src/slangpy_ext/device/cursor_writer.h` and `src/slangpy_ext/device/cursor_writer.cpp`. It takes a `nanobind` Python object, asks nanobind whether the object is backed by one of the registered C++ types, and returns a `NativeCursorWriterValue` containing the registry entry and the extracted C++ pointer. The hot-path lookup caches by `PyTypeObject*`, but only for native nanobind-backed types. - -The current Python cursor write tables live mainly in `src/slangpy_ext/device/cursor_utils.h`. `WriteConverterTable::write_registered_native_object(...)` calls `sgl::slangpy::find_native_cursor_writer(...)`. The shader cursor binding in `src/slangpy_ext/device/shader_cursor.cpp` uses `WriteConverterTable` to implement `write_shader_cursor(...)`. - -`std::type_info` is the C++ runtime type descriptor used by `typeid(T)`. Nanobind uses it to test and extract C++ objects from Python wrappers. `PyTypeObject` is the CPython runtime type descriptor for Python classes, but this plan never exposes that type name to core SGL. Core stores Python type pointers only as opaque `const void*` values and lets the extension interpret them. - -`SignatureBuffer` is the native cache-signature builder in `src/sgl/core/signature_buffer.h`. A cursor-writer entry must be able to add a stable signature for a value so SlangPy can cache generated call data. Native values do this through C++ callbacks today; Python-backed entries may use a static signature string or call a Python signature callback. - -## Plan of Work - -Milestone 1 changes the core registry key without adding Python support. In `src/sgl/device/cursor_utils.h`, introduce: - - enum class CursorWriterTypeKeyKind { - native_type_info, - python_type, - }; - -Replace `CursorWriterTypeInfo::type` with: - - CursorWriterTypeKeyKind key_kind{CursorWriterTypeKeyKind::native_type_info}; - const void* type_key{nullptr}; - std::string debug_name; - -Keep the existing erased writer fields, `slang_type_name`, `write_signature`, and `imports`. Native registration through `register_cursor_writer()` should set `key_kind` to `native_type_info`, `type_key` to `&typeid(T)`, and `debug_name` to `detail::type_name()` or `typeid(T).name()`. - -In `src/sgl/device/cursor_utils.cpp`, update `register_cursor_writer_type(...)` to validate `type_key`, use `debug_name` in error messages, and reject duplicates by the pair `(key_kind, type_key)`. Keep `find_cursor_writer_type_info(const std::type_info& type)`, but make it search only entries whose `key_kind` is `native_type_info` and compare `*static_cast(info.type_key)` against `type`. Add: - - SGL_API void unregister_cursor_writer_types(CursorWriterTypeKeyKind key_kind); - SGL_API uint64_t cursor_writer_registry_generation(); - -`unregister_cursor_writer_types(...)` removes all entries of that key kind. `cursor_writer_registry_generation()` returns a monotonically increasing counter that changes whenever entries are added or removed. The generation lets extension caches drop stale pointers after vector reallocation or cleanup. - -Milestone 2 updates the existing extension resolver to understand the new native key representation. In `src/slangpy_ext/device/cursor_writer.cpp`, replace all `info.type` uses with native-only helper code: - - const std::type_info& native_type_info(const CursorWriterTypeInfo& info) - -This helper asserts that `info.key_kind == CursorWriterTypeKeyKind::native_type_info` and casts `info.type_key` back to `const std::type_info*`. `native_cursor_writer_pointer(...)` continues to take `const std::type_info&` because nanobind needs that exact type descriptor. - -The uncached native lookup should skip non-native entries while doing nanobind `nb_type_isinstance(...)` scans. The exact-type fast path should keep using `cursor_utils::find_cursor_writer_type_info(...)`. The cache should store the registry generation, not only registry size. On each lookup, if `cursor_utils::cursor_writer_registry_generation()` has changed, clear the cache before reading a cached pointer. - -Milestone 3 adds Python-keyed registration in the extension while still writing into the same core registry. In `src/slangpy_ext/device/cursor_writer.h`, add an extension-only registration API. The exact shape can be adjusted to match nanobind conventions, but it must accept a Python type and optional callbacks: - - void register_python_cursor_writer_type( - nb::type_object python_type, - nb::object write_shader_cursor, - nb::object write_buffer_cursor, - nb::object write_signature, - std::string slang_type_name, - std::vector imports - ); - -If a callback is `None`, leave the corresponding erased function empty. At least one cursor writer and one signature writer must exist, matching the core validation contract. A static signature string may be converted into a C++ lambda before registration so callers do not need to provide a dynamic Python signature callback. - -The Python registration function builds a `CursorWriterTypeInfo` with `key_kind = CursorWriterTypeKeyKind::python_type`, `type_key = python_type.ptr()`, and a readable `debug_name` such as `module.qualname`. The writer lambdas capture `nb::object` callbacks. They receive `const void* value`, interpret it as the original `PyObject*`, create a borrowed `nb::object`, and call the Python callback: - - callback(cursor, nb::borrow(reinterpret_cast(const_cast(value)))); - -Do not acquire the GIL inside these lambdas. Add a comment stating that Python cursor-writer callbacks are invoked only from Python-entered paths where the GIL is already held. - -Milestone 4 unifies lookup results for native and Python-keyed entries. Rename the extension result type from `NativeCursorWriterValue` to a neutral name such as `CursorWriterValue`, and rename `find_native_cursor_writer(...)` to `find_cursor_writer(...)`. If this churn is too broad, keep the old names as wrappers during the transition, but new code should use the neutral names. - -The unified resolver in `src/slangpy_ext/device/cursor_writer.cpp` should handle both key kinds. For native entries, it keeps the current nanobind behavior and returns the extracted C++ pointer. For Python entries, it should first compare exact Python type pointer: - - Py_TYPE(obj.ptr()) == reinterpret_cast(const_cast(info.type_key)) - -If exact comparison fails, it may use `PyObject_IsInstance(...)` or an explicit MRO scan so registrations on Python base classes also work. Return the original Python object pointer as `value` for Python entries. Cache both hits and misses by `PyTypeObject*` and registry generation. - -Update `src/slangpy_ext/device/cursor_utils.h` so `WriteConverterTable::write_registered_native_object(...)` becomes `write_registered_object(...)` or similarly neutral and calls the unified resolver. Its call site should not know whether the value pointer is a C++ object or a Python object; it only invokes the erased writer stored in the registry entry. - -Milestone 5 wires cleanup into shutdown. In `src/sgl/sgl.cpp`, call: - - cursor_utils::unregister_cursor_writer_types(CursorWriterTypeKeyKind::python_type); - -near the start of `sgl::static_shutdown()` after the reference-count early return and before waiting for tasks or tearing down other subsystems. This preserves process-lifetime native registrations while destroying Python-backed callbacks before interpreter finalization. If later testing shows `static_shutdown()` can run after Python finalization in some embedding scenario, add an explicit call from the existing `atexit` hook in `src/slangpy_ext/slangpy_ext.cpp` while the GIL is held, but keep the core cleanup API as the owner of registry removal. - -Milestone 6 exposes a Python-level registration function only if needed for tests and user-facing use. The existing `slangpy/bindings/cursor.py::register_cursor_writer_marshal(...)` currently registers functional fallback metadata in Python dictionaries; it does not register runtime cursor writer callbacks into the core registry. Add a separate public helper only if the implementation needs pure Python users to register cursor writers directly. A possible name is `register_cursor_writer_type(...)`, but avoid overloading it with the existing functional marshall helper unless both operations are intentionally combined. The helper should call the native extension function from Milestone 3. - -Milestone 7 adds tests and keeps existing behavior stable. Native C++ tests under `tests/sgl/device/test_cursors.cpp` should cover key-kind duplicate rejection, native lookup after the opaque key migration, and unregistering only Python-kind entries. Python tests under `slangpy/tests/slangpy_tests/` should register a small Python class with a cursor writer callback, write it into a struct-shaped cursor, and verify the resulting data matches the callback's fields. Existing tests for `WriteToCursorMarshall`, native resource cursor writing, tensors, and descriptor handles should still pass. - -## Concrete Steps - -Work from the repository root: - - cd C:\sw\slangpy - -Before editing, inspect the current relevant code: - - rg -n "CursorWriterTypeInfo|find_native_cursor_writer|register_cursor_writer_type|static_shutdown" src/sgl src/slangpy_ext slangpy tests - -Implement Milestone 1 in `src/sgl/device/cursor_utils.h` and `src/sgl/device/cursor_utils.cpp`. Keep the public API source-compatible where possible by preserving `find_cursor_writer_type_info(const std::type_info&)`. - -Implement Milestones 2 through 4 in `src/slangpy_ext/device/cursor_writer.h`, `src/slangpy_ext/device/cursor_writer.cpp`, `src/slangpy_ext/device/cursor_utils.h`, and any direct callers found by `rg "find_native_cursor_writer|NativeCursorWriterValue|get_native_cursor_writer_type_info"`. - -Implement Milestone 5 in `src/sgl/sgl.cpp`. If the extension needs a test-only cleanup function, put it in `src/slangpy_ext/device/cursor_writer.cpp` and expose it only as a private `_...` function through the nanobind module. - -Add or update tests. Prefer focused test names that state the behavior: - - register_cursor_writer_rejects_duplicate_key_kind_and_key - unregister_cursor_writer_types_removes_python_keyed_entries - test_python_cursor_writer_callback_writes_struct - -Build before running tests, as required by this repository: - - cmake --build --preset windows-msvc-debug --target slangpy_ext sgl_tests - -Run focused C++ tests. Adjust the exact doctest filters to match the final test names: - - build\windows-msvc\Debug\sgl_tests.exe --test-case="*cursor_writer*" - build\windows-msvc\Debug\sgl_tests.exe --test-case="*write_to_cursor*" - -Run focused Python tests: - - python -m pytest slangpy/tests/slangpy_tests/test_write_to_cursor_marshall.py -v -p no:cacheprovider - python -m pytest slangpy/tests/slangpy_tests/test_python_cursor_writer.py -v -p no:cacheprovider - -Run pre-commit after implementation: - - pre-commit run --all-files - -If pre-commit modifies files, inspect the diff, then rerun `pre-commit run --all-files` until it passes. - -## Validation and Acceptance - -The implementation is accepted when all of these behaviors are true. - -Existing native cursor writers still work. Passing a native `Buffer`, `Texture`, `Sampler`, `ShaderObject`, `AccelerationStructure`, or `Tensor` to the existing cursor-writing paths succeeds exactly as before. The focused C++ cursor tests and existing Python write-to-cursor marshall tests pass. - -A pure Python type can be registered without adding another registry. The registration stores a `CursorWriterTypeInfo` in the core `cursor_utils` registry with `key_kind == CursorWriterTypeKeyKind::python_type` and an opaque Python type pointer. The core registry does not include Python headers and does not mention `PyTypeObject`. - -The unified resolver returns the same result shape for native and Python values. For native values, `value` is the extracted C++ pointer. For Python values, `value` is the original Python object pointer. The writer call site invokes `info.write_shader_cursor(cursor, value)` or `info.write_buffer_cursor(cursor, value)` without branching on native versus Python. - -Python-backed callbacks are destroyed during shutdown. A test or debug assertion should show that calling `cursor_utils::unregister_cursor_writer_types(CursorWriterTypeKeyKind::python_type)` removes Python-keyed entries and advances the registry generation. Static shutdown calls this cleanup path. Native entries are not removed by this cleanup. - -The extension lookup cache cannot return stale pointers after registration or cleanup. Any lookup cache in `src/slangpy_ext/device/cursor_writer.cpp` compares a stored generation against `cursor_utils::cursor_writer_registry_generation()` and clears itself when the generation changes. - -Expected successful command summary: - - cmake --build --preset windows-msvc-debug --target slangpy_ext sgl_tests - # build completes with no errors - - build\windows-msvc\Debug\sgl_tests.exe --test-case="*cursor_writer*" - # all selected doctest cases pass - - python -m pytest slangpy/tests/slangpy_tests/test_python_cursor_writer.py -v -p no:cacheprovider - # the pure Python cursor writer test passes - - pre-commit run --all-files - # all hooks pass - -## Idempotence and Recovery - -The implementation should be safe to apply incrementally. The opaque key migration is mechanical and can be verified before adding Python entries. If a later milestone fails, keep Milestone 1 compiling and temporarily leave Python registration unexposed. - -`register_cursor_writer_type(...)` must continue to reject duplicate registrations for the same `(key_kind, type_key)` pair. Re-running `sgl::static_init()` without a matching process restart should not silently add duplicate native entries. This plan does not require changing existing native-entry lifetime, but it must not make duplicate behavior worse. - -`unregister_cursor_writer_types(CursorWriterTypeKeyKind::python_type)` must be safe to call more than once. Calling it when no Python entries exist should leave the registry unchanged except that it may advance the generation only if entries were actually removed. Prefer advancing the generation only on mutation so caches are not cleared unnecessarily. - -If a Python callback raises, preserve the existing cursor-write error wrapping style in `WriteConverterTable`: include the Python value type in the error message and keep the cursor path stack context. Do not swallow the exception. - -If a test creates Python cursor-writer registrations, make it unregister them in a `finally` block or through a test fixture so later tests do not inherit registrations. - -## Artifacts and Notes - -The central native type shape should end up close to this. The exact field order may change to match local style: - - enum class CursorWriterTypeKeyKind { - native_type_info, - python_type, - }; - - struct CursorWriterTypeInfo { - CursorWriterTypeKeyKind key_kind{CursorWriterTypeKeyKind::native_type_info}; - const void* type_key{nullptr}; - std::string debug_name; - - ShaderCursorObjectWriteFunc write_shader_cursor; - BufferElementCursorObjectWriteFunc write_buffer_cursor; - - std::string slang_type_name; - std::function write_signature; - std::vector imports; - }; - -Native registration should remain concise: - - template - void register_cursor_writer() - requires(CanRegisterCursorWriter) - { - CursorWriterTypeInfo info; - info.key_kind = CursorWriterTypeKeyKind::native_type_info; - info.type_key = &typeid(T); - info.debug_name = std::string(detail::type_name()); - ... - register_cursor_writer_type(std::move(info)); - } - -Python registration should be visibly the same registry operation: - - CursorWriterTypeInfo info; - info.key_kind = CursorWriterTypeKeyKind::python_type; - info.type_key = python_type.ptr(); - info.debug_name = python_debug_name(python_type); - info.write_shader_cursor = [callback = nb::object(write_shader_cursor)](ShaderCursor& cursor, const void* value) - { - callback(cursor, nb::borrow(reinterpret_cast(const_cast(value)))); - return true; - }; - cursor_utils::register_cursor_writer_type(std::move(info)); - -Do not place this Python lambda in core SGL. The lambda is constructed in `src/slangpy_ext/`, where `nanobind.h` and Python types are already available. Core SGL only stores and later destroys the erased `std::function`. - -## Interfaces and Dependencies - -The core native API in `src/sgl/device/cursor_utils.h` must provide: - - enum class CursorWriterTypeKeyKind; - struct CursorWriterTypeInfo; - void register_cursor_writer_type(CursorWriterTypeInfo info); - std::span cursor_writer_type_infos(); - const CursorWriterTypeInfo* find_cursor_writer_type_info(const std::type_info& type); - void unregister_cursor_writer_types(CursorWriterTypeKeyKind key_kind); - uint64_t cursor_writer_registry_generation(); - template void register_cursor_writer(); - -Core SGL must not include `nanobind.h`, `Python.h`, or mention `PyTypeObject`. - -The extension API in `src/slangpy_ext/device/cursor_writer.h` should provide neutral lookup names: - - struct CursorWriterValue { - const cursor_utils::CursorWriterTypeInfo* info; - const void* value; - }; - - std::optional find_cursor_writer(nb::handle obj); - nb::object get_cursor_writer_type_info(nb::handle obj); - void register_python_cursor_writer_type(...); - -If existing Python or C++ code still expects `_get_native_cursor_writer_type_info` or `find_native_cursor_writer`, keep compatibility wrappers temporarily, but route them through the new neutral functions. - -The Python-facing helper, if added, belongs near `slangpy/bindings/cursor.py` because that file already owns `WriteToCursorMarshall` registration concepts. Keep the naming clear so users can tell whether they are registering runtime cursor callbacks, functional marshall metadata, or both. - -## Revision Notes - -2026-05-29: Initial ExecPlan created. It captures the agreed design: one core registry, an enum key kind plus opaque key, Python callbacks stored as erased lambdas constructed in the extension, no GIL acquisition inside hot callbacks, and cleanup of Python-backed entries during `sgl::static_shutdown()`. From c0bca52f43cd31cc381df83b1202363c5359662f Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Thu, 2 Jul 2026 12:35:30 +0100 Subject: [PATCH 09/17] Re-enable tests --- samples | 2 +- slangpy/tests/device/test_module_cache.py | 1 - slangpy/tests/device/test_shader_cache.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/samples b/samples index ba1d31067..7481ce941 160000 --- a/samples +++ b/samples @@ -1 +1 @@ -Subproject commit ba1d3106724f90ecd8a1bb3eea017206dd701928 +Subproject commit 7481ce941790672481a9b4403d957d380f722a71 diff --git a/slangpy/tests/device/test_module_cache.py b/slangpy/tests/device/test_module_cache.py index cfef46dcc..ad4edc44e 100644 --- a/slangpy/tests/device/test_module_cache.py +++ b/slangpy/tests/device/test_module_cache.py @@ -8,7 +8,6 @@ @pytest.mark.parametrize("device_type", helpers.DEFAULT_DEVICE_TYPES) -@pytest.mark.skip(reason="Crash on VM") def test_module_cache(device_type: spy.DeviceType, tmpdir: str): cache_dir = tmpdir # Create device with a module cache. diff --git a/slangpy/tests/device/test_shader_cache.py b/slangpy/tests/device/test_shader_cache.py index 96b3f2be5..74cf746b4 100644 --- a/slangpy/tests/device/test_shader_cache.py +++ b/slangpy/tests/device/test_shader_cache.py @@ -8,7 +8,6 @@ @pytest.mark.parametrize("device_type", helpers.DEFAULT_DEVICE_TYPES) -@pytest.mark.skip(reason="Crash on VM") def test_shader_cache(device_type: spy.DeviceType, tmpdir: str): cache_dir = tmpdir # Create device with a shader cache. From 55335d0b17271946c56ec5a9dd129dfc3cbeb709 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Thu, 2 Jul 2026 12:42:20 +0100 Subject: [PATCH 10/17] Restore tests --- slangpy/tests/device/test_module_cache.py | 1 - slangpy/tests/device/test_shader_cache.py | 1 - 2 files changed, 2 deletions(-) diff --git a/slangpy/tests/device/test_module_cache.py b/slangpy/tests/device/test_module_cache.py index dfbe0f234..051eb638b 100644 --- a/slangpy/tests/device/test_module_cache.py +++ b/slangpy/tests/device/test_module_cache.py @@ -8,7 +8,6 @@ @pytest.mark.parametrize("device_type", helpers.DEFAULT_DEVICE_TYPES) -@pytest.mark.skip(reason="Crash on VM") def test_module_cache(device_type: spy.DeviceType, tmpdir: str): cache_dir = tmpdir # Create device with a module cache. diff --git a/slangpy/tests/device/test_shader_cache.py b/slangpy/tests/device/test_shader_cache.py index 96b3f2be5..74cf746b4 100644 --- a/slangpy/tests/device/test_shader_cache.py +++ b/slangpy/tests/device/test_shader_cache.py @@ -8,7 +8,6 @@ @pytest.mark.parametrize("device_type", helpers.DEFAULT_DEVICE_TYPES) -@pytest.mark.skip(reason="Crash on VM") def test_shader_cache(device_type: spy.DeviceType, tmpdir: str): cache_dir = tmpdir # Create device with a shader cache. From 6cb3a952fbbf60ccd1fca9553a5a2f42a453aa18 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Thu, 2 Jul 2026 12:48:07 +0100 Subject: [PATCH 11/17] Restore correct slang-rhi --- external/slang-rhi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/slang-rhi b/external/slang-rhi index d75510053..50b94e5af 160000 --- a/external/slang-rhi +++ b/external/slang-rhi @@ -1 +1 @@ -Subproject commit d755100537cd082901dd2911a0edcff40a028ec4 +Subproject commit 50b94e5af1b4d6e8ad60c3451f6765b87adc6337 From b39ccad686be54c81790d16dda34f74d2198132d Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Thu, 2 Jul 2026 17:44:17 +0100 Subject: [PATCH 12/17] Set temp dir --- tools/ci.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tools/ci.py b/tools/ci.py index 6ade55b3e..34910f1e8 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -14,6 +14,12 @@ from typing import Any, Optional, Union PROJECT_DIR = Path(__file__).resolve().parent.parent +PYTEST_BASE_TEMP_DIR = PROJECT_DIR / ".temp" / "pytest" + + +def pytest_command(test_path: str, *args: str) -> list[str]: + # Pytest clears --basetemp on startup, so keep it in a dedicated repo-local directory. + return ["pytest", test_path, *args, f"--basetemp={PYTEST_BASE_TEMP_DIR}"] def get_os(): @@ -144,7 +150,7 @@ def typing_check_python(args: Any): def unit_test_python(args: Any): env = get_python_env() os.makedirs("reports", exist_ok=True) - cmd = ["pytest", "slangpy/tests", "-vra"] + cmd = pytest_command("slangpy/tests", "-vra") if args.parallel: cmd += ["-n", "auto", "--maxprocesses=4"] run_command(cmd, env=env) @@ -152,7 +158,7 @@ def unit_test_python(args: Any): def test_examples(args: Any): env = get_python_env() - cmd = ["pytest", "samples/tests", "-vra"] + cmd = pytest_command("samples/tests", "-vra") if args.parallel: cmd += ["-n", "auto", "--maxprocesses=4"] run_command(cmd, env=env) @@ -197,7 +203,7 @@ def benchmark_python(args: Any): for device_type in device_types: print(f"Running benchmarks for device type: {device_type}") - cmd = ["pytest", "slangpy/benchmarks", "-ra", "--device-types", device_type] + cmd = pytest_command("slangpy/benchmarks", "-ra", "--device-types", device_type) if args.mongodb_connection_string: cmd += ["--benchmark-upload", args.run_id] cmd += ["--benchmark-mongodb-connection-string", args.mongodb_connection_string] From a3a9ce92f2a813e90635386ca232c4a9cd6aea46 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Fri, 3 Jul 2026 10:28:10 +0100 Subject: [PATCH 13/17] Temp disable test --- slangpy/tests/slangpy_tests/test_pytorch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/slangpy/tests/slangpy_tests/test_pytorch.py b/slangpy/tests/slangpy_tests/test_pytorch.py index 220a1c4d5..fc6bbefda 100644 --- a/slangpy/tests/slangpy_tests/test_pytorch.py +++ b/slangpy/tests/slangpy_tests/test_pytorch.py @@ -78,6 +78,8 @@ def test_basic_autograd(device_type: DeviceType): @pytest.mark.parametrize("device_type", helpers.DEFAULT_DEVICE_TYPES) def test_tensor_arguments(device_type: DeviceType): + if device_type != DeviceType.cuda: + pytest.skip("Test currently unreliable on GFX pipelines") device = helpers.get_torch_device(device_type) module = get_module(device) From 467d5a102485b2ff9c81f53f8da0508b78c7030e Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Fri, 3 Jul 2026 10:39:45 +0100 Subject: [PATCH 14/17] Make temp dir --- tools/ci.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci.py b/tools/ci.py index 34910f1e8..7734e8ee9 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -19,6 +19,7 @@ def pytest_command(test_path: str, *args: str) -> list[str]: # Pytest clears --basetemp on startup, so keep it in a dedicated repo-local directory. + PYTEST_BASE_TEMP_DIR.parent.mkdir(exist_ok=True) return ["pytest", test_path, *args, f"--basetemp={PYTEST_BASE_TEMP_DIR}"] From ddb47c18db23ecfb937c691170e324a8791674f7 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Fri, 3 Jul 2026 11:39:56 +0100 Subject: [PATCH 15/17] Proxy aware url test in samples --- samples | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples b/samples index 7481ce941..df2a1da86 160000 --- a/samples +++ b/samples @@ -1 +1 @@ -Subproject commit 7481ce941790672481a9b4403d957d380f722a71 +Subproject commit df2a1da863dec21e73168501afa8db3b491f0872 From 54257b971488aba9746295def33038e0d4a7ad8e Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Fri, 3 Jul 2026 12:19:30 +0100 Subject: [PATCH 16/17] Re-enable test to see if we can repro error --- slangpy/tests/slangpy_tests/test_pytorch.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/slangpy/tests/slangpy_tests/test_pytorch.py b/slangpy/tests/slangpy_tests/test_pytorch.py index fc6bbefda..220a1c4d5 100644 --- a/slangpy/tests/slangpy_tests/test_pytorch.py +++ b/slangpy/tests/slangpy_tests/test_pytorch.py @@ -78,8 +78,6 @@ def test_basic_autograd(device_type: DeviceType): @pytest.mark.parametrize("device_type", helpers.DEFAULT_DEVICE_TYPES) def test_tensor_arguments(device_type: DeviceType): - if device_type != DeviceType.cuda: - pytest.skip("Test currently unreliable on GFX pipelines") device = helpers.get_torch_device(device_type) module = get_module(device) From 0025565c7226cad98e58b91735a1369407bd5c21 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Fri, 3 Jul 2026 12:22:56 +0100 Subject: [PATCH 17/17] Re-enable test --- slangpy/tests/slangpy_tests/test_pytorch.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/slangpy/tests/slangpy_tests/test_pytorch.py b/slangpy/tests/slangpy_tests/test_pytorch.py index fc6bbefda..220a1c4d5 100644 --- a/slangpy/tests/slangpy_tests/test_pytorch.py +++ b/slangpy/tests/slangpy_tests/test_pytorch.py @@ -78,8 +78,6 @@ def test_basic_autograd(device_type: DeviceType): @pytest.mark.parametrize("device_type", helpers.DEFAULT_DEVICE_TYPES) def test_tensor_arguments(device_type: DeviceType): - if device_type != DeviceType.cuda: - pytest.skip("Test currently unreliable on GFX pipelines") device = helpers.get_torch_device(device_type) module = get_module(device)