Fix #12637: lower portable HLSL ReportHit to OptiX/CUDA

source/slang/hlsl.meta.slang

@@ -20103,6 +20103,16 @@ bool __reportIntersection(float tHit, uint hitKind)
     }
 }
 
+//@hidden:
+// Marker for the OptiX write side of a portable `ReportHit`. The `attributes` aggregate is
+// passed as a single operand; the CUDA varying-param legalization pass flattens it field-wise
+// into scalar attribute leaves before emission (see `legalizeOptiXReportIntersections`).
+__generic<A>
+__intrinsic_op($(kIROp_ReportOptiXIntersection))
+[require(cuda, raytracing_intersection)]
+bool __reportOptiXIntersection(float tHit, uint hitKind, A attributes);
+//@public:
+
 /// Reports a hit from an intersection shader.
 /// @param tHit Distance along the ray where the intersection occurred
 /// @param hitKind User-defined value identifying the type of hit
@@ -20112,7 +20122,7 @@ bool __reportIntersection(float tHit, uint hitKind)
 /// @category raytracing
 __generic<A>
 [ForceInline]
-[require(glsl_hlsl_spirv, raytracing_intersection)]
+[require(cuda_glsl_hlsl_spirv, raytracing_intersection)]
 bool ReportHit(float tHit, uint hitKind, A attributes)
 {
     __target_switch
@@ -20125,6 +20135,8 @@ bool ReportHit(float tHit, uint hitKind, A attributes)
         static A a;
         a = attributes;
         return __reportIntersection(tHit, hitKind);
+    case cuda:
+        return __reportOptiXIntersection(tHit, hitKind, attributes);
     }
 }
 

source/slang/slang-diagnostics.lua

@@ -5706,6 +5706,20 @@ err(
     span { loc = "location", message = "a shader-terminating intrinsic ('IgnoreHit' or 'AcceptHitAndEndSearch') is reachable from this ray entry point only through a call that could not be inlined (for example, recursion); mark the intervening function(s) '[ForceInline]' or call the intrinsic directly in the entry point so the ray payload is written back before the ray terminates." }
 )
 
+err(
+    "optix-hit-attribute-type-not-supported",
+    55216,
+    "unsupported hit attribute type for ReportHit on OptiX",
+    span { loc = "location", message = "the hit attribute type passed to 'ReportHit' cannot be lowered to OptiX attribute registers; each scalar field must fit in one 32-bit register ('float', 'bool', and 8/16/32-bit signed or unsigned integers, and vectors/arrays/matrices of those) — 'double', 'half', and 64-bit types are not supported." }
+)
+
+err(
+    "optix-hit-attribute-too-large",
+    55217,
+    "hit attribute exceeds the OptiX attribute register limit",
+    span { loc = "location", message = "the hit attribute passed to 'ReportHit' requires ~registerCount:int 32-bit attribute registers, but OptiX supports at most 8 (32 bytes)." }
+)
+
 err(
     "unable-to-auto-map-cuda-type-to-host-type",
     56001,

source/slang/slang-emit-cuda.cpp

@@ -1378,6 +1378,35 @@ bool CUDASourceEmitter::tryEmitInstExprImpl(IRInst* inst, const EmitOpInfo& inOu
             }
             return true;
         }
+    case kIROp_ReportOptiXIntersection:
+        {
+            // optixReportIntersection(tHit, hitKind, a0..aN). The CUDA legalization pass has
+            // already flattened the aggregate into scalar leaves (operands 2..N), one per attribute
+            // register. A float leaf is bit-reinterpreted with `__float_as_uint`, the exact inverse
+            // of the reader's `__int_as_float(optixGetAttribute_N())`, because CUDA lowers
+            // `IRBitCast` as a numeric C cast, not a bit cast.
+            m_writer->emit("optixReportIntersection(");
+            emitOperand(inst->getOperand(0), getInfo(EmitOp::General));
+            m_writer->emit(", ");
+            emitOperand(inst->getOperand(1), getInfo(EmitOp::General));
+            for (UInt i = 2; i < inst->getOperandCount(); ++i)
+            {
+                m_writer->emit(", ");
+                auto leaf = inst->getOperand(i);
+                if (leaf->getDataType()->getOp() == kIROp_FloatType)
+                {
+                    m_writer->emit("__float_as_uint(");
+                    emitOperand(leaf, getInfo(EmitOp::General));
+                    m_writer->emit(")");
+                }
+                else
+                {
+                    emitOperand(leaf, getInfo(EmitOp::General));
+                }
+            }
+            m_writer->emit(")");
+            return true;
+        }
     case kIROp_GetOptiXSbtDataPtr:
         {
             m_writer->emit("((");

source/slang/slang-emit.cpp

@@ -1818,6 +1818,17 @@ Result linkAndOptimizeIR(
         SLANG_PASS(simplifyIR, targetProgram, defaultIRSimplificationOptions, sink);
     }
 
+    // `ReportHit` is [ForceInline], so its `kIROp_ReportOptiXIntersection` marker now appears at
+    // each call site after `performForceInlining`. Flatten its aggregate attribute operand into
+    // scalar OptiX attribute-register leaves after the post-inline DCE above (so the dead
+    // specialized core-module `ReportHit` body is gone and each marker is diagnosed only once) but
+    // before the generic empty-type / resource legalization passes below, which do not understand
+    // the aggregate operand.
+    if (target == CodeGenTarget::CUDASource || target == CodeGenTarget::CUDAHeader)
+    {
+        SLANG_PASS(legalizeOptiXReportIntersectionsForCUDA, sink);
+    }
+
     // Report checkpointing information.
     if (codeGenContext->shouldReportCheckpointIntermediates())
     {

source/slang/slang-ir-insts-stable-names.lua

@@ -874,5 +874,6 @@ return {
 	["imageGatherOffset"] = 898,
 	["getNaturalAlignment"] = 899,
 	["Type.PtrTypeBase.SPIRVUntypedPtr"] = 900,
-	["Attr.TypeAlignment"] = 901
+	["Attr.TypeAlignment"] = 901,
+	["reportOptiXIntersection"] = 902
 }

source/slang/slang-ir-insts.lua

@@ -1657,6 +1657,13 @@ local insts = {
 	-- Operand 0: register index (int literal)
 	-- Operand 1: value to write (uint32)
 	{ setOptiXPayloadRegister = { min_operands = 2 } },
+	-- Write side of a portable `ReportHit(tHit, hitKind, attributes)` call for OptiX.
+	-- Operand 0: tHit (float). Operand 1: hitKind (uint). The remaining operands are the
+	-- aggregate's scalar attribute leaves, produced by the CUDA varying-param legalization
+	-- pass, which flattens `attributes` field-wise (one operand per OptiX attribute register)
+	-- mirroring the read side (`emitOptiXAttributeFetch`). The CUDA emitter renders this as a
+	-- single `optixReportIntersection(tHit, hitKind, a0..aN)`.
+	{ reportOptiXIntersection = { min_operands = 2 } },
 	{ GetVulkanRayTracingPayloadLocation = { min_operands = 1 } },
 	{ GetLegalizedSPIRVGlobalParamAddr = { min_operands = 1 } },
 	{

source/slang/slang-ir-legalize-varying-params.cpp

@@ -1126,6 +1126,10 @@ struct CUDAEntryPointVaryingParamLegalizeContext : EntryPointVaryingParamLegaliz
     // Maximum number of payload registers (32 registers = 128 bytes)
     static const int kMaxPayloadRegisters = 32;
 
+    // Maximum number of OptiX hit attribute registers (8 registers = 32 bytes). Shared by the read
+    // path (`getLegalizedVaryingVal`) and the `ReportHit` write path.
+    static const int kMaxHitAttributeRegisters = 8;
+
     // Track payload write-back info for inout parameters
     struct PayloadWritebackInfo
     {
@@ -2155,6 +2159,190 @@ struct CUDAEntryPointVaryingParamLegalizeContext : EntryPointVaryingParamLegaliz
         return nullptr;
     }
 
+    // An OptiX hit attribute register holds exactly 32 bits, and both this write path and the read
+    // path (`emitOptiXAttributeFetch` / the `kIROp_GetOptiXHitAttribute` emit) map one scalar leaf
+    // to one register with no sub-word packing or multi-register splitting. A leaf is supported
+    // when it survives that single-register round-trip: `float` is bit-reinterpreted
+    // (`__float_as_uint` on write, `__int_as_float` on read); an integer/bool type of 32 bits or
+    // fewer passes through the register's `unsigned int` value (a narrow type sign/zero-extends on
+    // write and truncates back to its low bits on read, symmetrically). Wider scalars (`double`,
+    // `int64_t`/`uint64_t`, `intptr`/`uintptr`) do not fit one register, and `half` has no
+    // bit-preserving read counterpart (the reader would assign the register's integer value
+    // numerically), so both are rejected rather than silently miscompiled.
+    static bool isSupportedOptiXHitAttributeLeaf(IRBasicType* basicType)
+    {
+        switch (basicType->getBaseType())
+        {
+        case BaseType::Float:
+        case BaseType::Bool:
+        case BaseType::Int8:
+        case BaseType::Int16:
+        case BaseType::Int:
+        case BaseType::UInt8:
+        case BaseType::UInt16:
+        case BaseType::UInt:
+        case BaseType::Char:
+            return true;
+        default:
+            return false;
+        }
+    }
+
+    // Flatten `value` (of `type`) into its scalar leaves, appended to `outLeaves` in the exact
+    // order the read side (`emitOptiXAttributeFetch`) consumes attribute registers: struct fields
+    // in declaration order, then array/vector/matrix elements in index order, down to each scalar
+    // leaf. This is the write-side mirror of the reader, so a value reported by `ReportHit` on
+    // OptiX round-trips through the fixed-function attribute registers when a hit shader reads it
+    // back. Returns false (so the caller can diagnose) for a type that cannot be lowered to
+    // attribute registers: an unsized array, or a scalar leaf that is not a supported 32-bit
+    // register type. Consider `struct Attributes { uint id; float weight; }`: the leaves are
+    // `{ value.id, value.weight }`, occupying attribute registers 0 and 1.
+    bool flattenOptiXHitAttributes(
+        IRInst* value,
+        IRType* type,
+        IRBuilder* builder,
+        List<IRInst*>& outLeaves)
+    {
+        // Attributes are reported by value, so a scalar attribute register never round-trips a
+        // pointer: extracting a field from a pointer value would emit `(&a).x`, and dereferencing
+        // would silently change which value is reported. Reject any pointer type rather than
+        // miscompile it.
+        if (tryGetPointedToType(builder, type))
+            return false;
+
+        if (auto structType = as<IRStructType>(type))
+        {
+            for (auto field : structType->getFields())
+            {
+                auto fieldType = field->getFieldType();
+                auto fieldVal = builder->emitFieldExtract(fieldType, value, field->getKey());
+                if (!flattenOptiXHitAttributes(fieldVal, fieldType, builder, outLeaves))
+                    return false;
+            }
+            return true;
+        }
+        else if (auto arrayType = as<IRArrayTypeBase>(type))
+        {
+            auto elementCountInst = as<IRIntLit>(arrayType->getElementCount());
+            if (!elementCountInst)
+                return false;
+            auto elementType = arrayType->getElementType();
+            for (IRIntegerValue ii = 0; ii < elementCountInst->getValue(); ++ii)
+            {
+                auto idx = builder->getIntValue(builder->getIntType(), ii);
+                auto elementVal = builder->emitElementExtract(elementType, value, idx);
+                if (!flattenOptiXHitAttributes(elementVal, elementType, builder, outLeaves))
+                    return false;
+            }
+            return true;
+        }
+        else if (auto matType = as<IRMatrixType>(type))
+        {
+            auto rowCountInst = as<IRIntLit>(matType->getRowCount());
+            auto colCountInst = as<IRIntLit>(matType->getColumnCount());
+            if (!rowCountInst || !colCountInst)
+                return false;
+            auto elementType = matType->getElementType();
+            auto rowType = builder->getVectorType(elementType, matType->getColumnCount());
+            for (IRIntegerValue row = 0; row < rowCountInst->getValue(); ++row)
+            {
+                auto rowIdx = builder->getIntValue(builder->getIntType(), row);
+                auto rowVal = builder->emitElementExtract(rowType, value, rowIdx);
+                for (IRIntegerValue col = 0; col < colCountInst->getValue(); ++col)
+                {
+                    auto colIdx = builder->getIntValue(builder->getIntType(), col);
+                    auto elementVal = builder->emitElementExtract(elementType, rowVal, colIdx);
+                    if (!flattenOptiXHitAttributes(elementVal, elementType, builder, outLeaves))
+                        return false;
+                }
+            }
+            return true;
+        }
+        else if (auto vecType = as<IRVectorType>(type))
+        {
+            auto elementCountInst = as<IRIntLit>(vecType->getElementCount());
+            if (!elementCountInst)
+                return false;
+            auto elementType = vecType->getElementType();
+            for (IRIntegerValue ii = 0; ii < elementCountInst->getValue(); ++ii)
+            {
+                auto idx = builder->getIntValue(builder->getIntType(), ii);
+                auto elementVal = builder->emitElementExtract(elementType, value, idx);
+                if (!flattenOptiXHitAttributes(elementVal, elementType, builder, outLeaves))
+                    return false;
+            }
+            return true;
+        }
+        else if (auto basicType = as<IRBasicType>(type))
+        {
+            if (!isSupportedOptiXHitAttributeLeaf(basicType))
+                return false;
+            outLeaves.add(value);
+            return true;
+        }
+
+        return false;
+    }
+
+    // Rewrite each `ReportOptiXIntersection(tHit, hitKind, attributes)` produced by the core-module
+    // `ReportHit` into a call carrying the aggregate's flattened scalar leaves, so the CUDA emitter
+    // can render a single `optixReportIntersection(tHit, hitKind, a0..aN)`. Collect the marker
+    // insts first, then rewrite: the rewrite creates a new (already-flattened) inst that must not
+    // be re-collected, and the pass runs exactly once per module.
+    void legalizeOptiXReportIntersections(IRModule* module, DiagnosticSink* sink)
+    {
+        List<IRInst*> workList;
+        for (auto globalInst : module->getGlobalInsts())
+        {
+            auto func = as<IRFunc>(globalInst);
+            if (!func)
+                continue;
+            for (auto block : func->getBlocks())
+                for (auto inst : block->getChildren())
+                    if (inst->getOp() == kIROp_ReportOptiXIntersection &&
+                        inst->getOperandCount() == 3)
+                        workList.add(inst);
+        }
+
+        for (auto inst : workList)
+        {
+            IRBuilder builder(module);
+            builder.setInsertBefore(inst);
+
+            auto tHit = inst->getOperand(0);
+            auto hitKind = inst->getOperand(1);
+            auto attrs = inst->getOperand(2);
+
+            List<IRInst*> leaves;
+            if (!flattenOptiXHitAttributes(attrs, attrs->getDataType(), &builder, leaves))
+            {
+                sink->diagnose(
+                    Diagnostics::OptixHitAttributeTypeNotSupported{.location = inst->sourceLoc});
+                continue;
+            }
+            if (leaves.getCount() > kMaxHitAttributeRegisters)
+            {
+                sink->diagnose(Diagnostics::OptixHitAttributeTooLarge{
+                    .registerCount = int(leaves.getCount()),
+                    .location = inst->sourceLoc});
+                continue;
+            }
+
+            List<IRInst*> args;
+            args.add(tHit);
+            args.add(hitKind);
+            args.addRange(leaves);
+            auto newInst = builder.emitIntrinsicInst(
+                inst->getFullType(),
+                kIROp_ReportOptiXIntersection,
+                args.getCount(),
+                args.getBuffer());
+            newInst->sourceLoc = inst->sourceLoc;
+            inst->replaceUsesWith(newInst);
+            inst->removeAndDeallocate();
+        }
+    }
+
     void beginModuleImpl() SLANG_OVERRIDE
     {
         // Because many of the varying parameters are defined
@@ -2376,7 +2564,7 @@ struct CUDAEntryPointVaryingParamLegalizeContext : EntryPointVaryingParamLegaliz
                     /*ioBaseAttributeIndex*/ ioBaseAttributeIndex,
                     /* type to fetch */ info.type,
                     /*the builder in use*/ &builder);
-                if (ioBaseAttributeIndex > 8)
+                if (ioBaseAttributeIndex > kMaxHitAttributeRegisters)
                 {
                     // A hit attribute is always a parameter, never a result, so
                     // `m_param` is set here; guard the deref in release too.
@@ -2577,6 +2765,12 @@ void legalizeEntryPointVaryingParamsForCUDA(IRModule* module, DiagnosticSink* si
     context.processModule(module, sink);
 }
 
+void legalizeOptiXReportIntersectionsForCUDA(IRModule* module, DiagnosticSink* sink)
+{
+    CUDAEntryPointVaryingParamLegalizeContext context;
+    context.legalizeOptiXReportIntersections(module, sink);
+}
+
 void depointerizeInputParams(IRFunc* entryPointFunc)
 {
     List<IRParam*> workList;

source/slang/slang-ir-legalize-varying-params.h

@@ -27,6 +27,12 @@ void legalizeEntryPointVaryingParamsForCPU(
 
 void legalizeEntryPointVaryingParamsForCUDA(IRModule* module, DiagnosticSink* sink);
 
+// Flatten the aggregate attribute operand of each portable `ReportHit` (lowered to a
+// `kIROp_ReportOptiXIntersection` marker) into scalar OptiX attribute-register leaves. Must run
+// before the generic empty-type / varying-parameter legalization passes, which do not understand
+// the aggregate operand of the marker op.
+void legalizeOptiXReportIntersectionsForCUDA(IRModule* module, DiagnosticSink* sink);
+
 void legalizeEntryPointVaryingParamsForMetal(
     IRModule* module,
     DiagnosticSink* sink,