Preserve memory budgets across suspendable execution

.github/workflows/codeql.yml

@@ -132,6 +132,12 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y build-essential cmake
 
+    - name: Check Rust guard diagnostic preservation
+      if: matrix.language == 'rust'
+      run: |
+        cargo test --locked -p regorus-mimalloc --features allocator-memory-limits mimalloc::tests::header_geometry_preserves_zeroed_aligned_reallocations -- --exact
+        cargo clippy --locked -p regorus-mimalloc --all-targets --features allocator-memory-limits -- -D warnings
+
     - name: Install Python build dependencies
       if: matrix.language == 'python'
       working-directory: ${{ matrix.working-directory }}
@@ -188,6 +194,18 @@ jobs:
       run: cargo xtask build-wasm --release
 
     - name: Perform CodeQL Analysis
+      id: analyze
       uses: github/codeql-action/analyze@24c54180a607b1449ed407dd24f251e4e9147c8d # v4.38.3
+      env:
+        CODEQL_ACTION_EXTRA_OPTIONS: ${{ matrix.language == 'rust' && '{"database":{"interpret-results":["--max-paths=5"]}}' || '{}' }}
+      with:
+        category: "/language:${{matrix.language}}"
+
+    - name: Retain Rust SARIF
+      if: matrix.language == 'rust'
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
       with:
-        category: "/language:${{matrix.language}}"
+        name: codeql-rust-sarif-${{ github.run_id }}-${{ github.run_attempt }}
+        path: ${{ steps.analyze.outputs.sarif-output }}/*.sarif
+        if-no-files-found: error
+        retention-days: 1

.github/workflows/rust-clippy.yml

@@ -47,9 +47,23 @@ jobs:
       - name: Fetch
         run: cargo fetch --locked
 
+      - name: Clear cached Clippy diagnostics
+        run: rm -f target/xtask/clippy.json
+
       - name: Run rust-clippy
+        id: clippy
         run: cargo xtask clippy --sarif rust-clippy-results.sarif
 
+      - name: Expose raw Clippy diagnostics on failure
+        if: ${{ failure() && steps.clippy.outcome == 'failure' }}
+        shell: bash
+        run: |
+          if [ -s target/xtask/clippy.json ]; then
+            cat target/xtask/clippy.json
+          else
+            echo "::warning::Clippy failed before producing a nonempty target/xtask/clippy.json"
+          fi
+
       - name: Upload analysis results to GitHub
         if: ${{ hashFiles('rust-clippy-results.sarif') != '' }}
         uses: github/codeql-action/upload-sarif@24c54180a607b1449ed407dd24f251e4e9147c8d # v3.29.11

.github/workflows/tests-debug.yml

@@ -29,5 +29,12 @@ jobs:
         shared-key: ${{ runner.os }}-regorus
     - name: Fetch dependencies
       run: cargo fetch --locked
+    - name: Check allocator features and lifecycle tests
+      run: |
+        cargo check --frozen --no-default-features --features std,arc,full-opa,rvm,allocator-memory-limits
+        cargo test --manifest-path mimalloc/Cargo.toml --features allocator-memory-limits --frozen execution_budget_
+        cargo test --manifest-path mimalloc/Cargo.toml --features allocator-memory-limits --frozen header_
+        cargo test --frozen --no-default-features --features std,arc,full-opa,rvm,allocator-memory-limits --lib native_resume_
+        cargo test --frozen --no-default-features --features std,arc,full-opa,rvm,allocator-memory-limits --lib resume_deserialization_limit_error_capture_
     - name: Run debug CI suite
       run: cargo xtask ci-debug --frozen

CHANGELOG.md

@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - disambiguate prior rule file labels ([#824](https://github.com/microsoft/regorus/pull/824))
 - keep prior rule locations in concise conflict diagnostics ([#820](https://github.com/microsoft/regorus/pull/820))
 - Rule-conflict diagnostics now retain the current source envelope and identify the previous rule by file, line, and column without nesting a second diagnostic; literal percent, CR, and LF characters in the previous file label are percent-encoded. RVM conflict result semantics are unchanged.
+- Suspendable RVM budgets remain immutable across host-await resumes, include native resume parsing and immediate result serialization, and leave malformed JSON syntax/EOF retryable; independent unbudgeted VM calls do not inherit a caller's account, and run-to-completion thread-baseline behavior remains unchanged.
 
 ### Other
 

benches/rvm_benchmark.rs

@@ -75,7 +75,7 @@ struct EvalConfig {
     memory_budget: bool,
 }
 
-const EVAL_CONFIGS: [EvalConfig; 5] = [
+const EVAL_CONFIGS: [EvalConfig; 6] = [
     EvalConfig {
         name: "regular_no_limits",
         mode: ExecutionMode::RunToCompletion,
@@ -106,6 +106,12 @@ const EVAL_CONFIGS: [EvalConfig; 5] = [
         limits: true,
         memory_budget: false,
     },
+    EvalConfig {
+        name: "suspendable_memory_budget",
+        mode: ExecutionMode::Suspendable,
+        limits: false,
+        memory_budget: true,
+    },
 ];
 
 // ---------------------------------------------------------------------------

bindings/csharp/API.md

@@ -85,7 +85,7 @@ The Regorus C# bindings provide a modern, thread-safe API for compiling and eval
 
 ## RVM Memory Budgets
 
-`Rvm.SetMemoryBudgetConfig` configures a fresh non-zero budget for each run-to-completion execution. `Rvm.ClearMemoryBudgetConfig` restores unlimited execution.
+`Rvm.SetMemoryBudgetConfig` configures a fresh non-zero budget for the next initial RVM execution in either run-to-completion or suspendable mode. `Rvm.ClearMemoryBudgetConfig` affects the next initial execution; it does not change an active suspendable continuation.
 
 ```csharp
 public readonly struct MemoryBudgetConfig
@@ -101,7 +101,7 @@ public sealed class Rvm : IDisposable
 }
 ```
 
-Ordinary `Execute` and `ExecuteEntryPoint` calls use the existing VM data and start a fresh budget for execution. Program compilation, program loading, and prior `SetDataJson`, `SetInputJson`, and `SetContextJson` calls occur before and outside the budget; use this path for static or preloaded data.
+`Execute` and `ExecuteEntryPoint` start a fresh budget for each accepted initial execution. A suspendable execution retains the same budget across `Resume` calls, including resumes on another thread. Program compilation, program loading, and prior `SetDataJson`, `SetInputJson`, and `SetContextJson` calls occur before and outside the budget; use this path for static or preloaded data.
 
 ```csharp
 using var vm = new Rvm();
@@ -117,11 +117,13 @@ catch (RegorusMemoryBudgetExceededException)
 }
 ```
 
-Native result JSON serialization and Rust `CString` allocation are included before an `Execute` or `ExecuteEntryPoint` call returns. Managed UTF-8 decoding and the managed C# `string` allocation after the native call returns are excluded.
+Native resume C-string copying and JSON parsing, VM resume work, result JSON serialization, and Rust `CString` allocation are included before an `Execute`, `ExecuteEntryPoint`, or `Resume` call returns. Managed UTF-8 decoding and the managed C# `string` allocation after the native call returns are excluded.
 
-Memory budgets require a native library built with allocator memory tracking and are supported only for run-to-completion execution. `RegorusMemoryBudgetExceededException` is thrown when a budget is exceeded. `RegorusMemoryBudgetUnsupportedException` is thrown if a configured budget is used to start or resume suspendable execution.
+Memory budgets require a native library built with allocator memory tracking. Changing or clearing the configuration while a suspendable execution is active affects only the next initial execution. `RegorusMemoryBudgetExceededException` is thrown when a budget is exceeded. Native status 11 and `RegorusMemoryBudgetUnsupportedException` remain reserved for compatibility and are not used to reject suspendable execution.
 
-Enforcement samples observed live bytes at checkpoints; it is not an allocation-time peak-memory cap. An instruction, builtin, native serialization, or `CString` allocation can overshoot before it is rejected. The reported native usage is a change in the execution thread's live-byte counter, not exact memory owned by the query. Synchronous host work affects the sample, cross-thread frees can temporarily skew it, and downward baseline ratcheting does not restore lost headroom.
+Enforcement samples currently live requested bytes at checkpoints; it is not an allocation-time peak-memory cap. An instruction, builtin, native input parse, serialization, or `CString` allocation can overshoot before it is rejected. Cross-thread frees debit the allocation's originating execution. Synchronous host work on the execution thread is included; managed allocations, host I/O, getter serialization, and other allocators are excluded.
+
+The following warm-VM baseline note applies to run-to-completion accounting. Suspendable budgets instead charge allocations by their originating execution, so retained pre-execution capacity is not charged unless it is reallocated during the run.
 
 A reused VM gets a fresh baseline, but capacities and pools retained before that baseline can make a warm execution allocate differently from a fresh one. Failed terminal execution clears retained state. Public multi-call begin/end scopes are intentionally absent because allocator counters are thread-local and abandoned or cross-thread scopes would be unsafe. The [detailed RVM memory budget documentation](../../docs/limits/memory_budget.md) is authoritative for accounting behavior.
 

bindings/csharp/README.md

@@ -124,7 +124,7 @@ of being truncated.
 
 ### Per-execution memory budget
 
-RVM run-to-completion evaluation can use an optional additional live-memory budget. Each ordinary `Execute` or `ExecuteEntryPoint` call starts with a fresh budget for execution; program compilation, program loading, and prior `SetDataJson`, `SetInputJson`, and `SetContextJson` calls occur before and outside that budget.
+RVM run-to-completion and suspendable evaluation can use an optional live-memory budget. Each accepted initial `Execute` or `ExecuteEntryPoint` call snapshots the configured budget; `Resume` continues the same budget, including when it runs on a different thread. Program compilation, program loading, and prior `SetDataJson`, `SetInputJson`, and `SetContextJson` calls occur before and outside that budget. Changing or clearing the configuration affects only the next initial execution.
 
 ```csharp
 using var vm = new Rvm();
@@ -143,13 +143,13 @@ catch (RegorusMemoryBudgetExceededException ex)
 }
 ```
 
-The native execution budget includes result JSON serialization and Rust `CString` allocation before the native call returns. Managed UTF-8 decoding and C# `string` allocation after that return are not charged.
+The native execution budget includes resume C-string copying and JSON parsing, VM resume work, result JSON serialization, and Rust `CString` allocation before the native call returns. Managed UTF-8 decoding and C# `string` allocation after that return are not charged.
 
 This is a cooperative observed-live-bytes budget, not an allocation-time peak-memory cap. A single instruction, builtin, native serialization, or `CString` allocation may exceed the limit before the next checkpoint; configure headroom for that overshoot.
 
-Accounting observes the execution thread rather than allocation ownership. Synchronous host work on that thread contributes to usage, cross-thread frees can temporarily skew observations, and a downward baseline ratchet can permanently remove headroom during an execution. A reused VM receives a new baseline, but retained capacities and pools can make it allocate differently from a fresh VM. See the [RVM memory budget documentation](../../docs/limits/memory_budget.md) for the detailed accounting model.
+Suspendable accounting tracks live requested bytes by allocation origin. The execution account is selected only during a synchronous segment; each block retains its account until freed, even on another thread. Successful reallocation charges the full replacement size to the currently selected execution. Run-to-completion accounting keeps its existing execution-thread baseline and downward ratchet. See the [RVM memory budget documentation](../../docs/limits/memory_budget.md) for the detailed accounting model.
 
-Budgets are not supported in suspendable execution mode. `ClearMemoryBudgetConfig` restores the previous unlimited per-execution behavior. Public multi-call begin/end scopes are intentionally absent because allocator counters are thread-local. Failed terminal execution clears retained state. The process-wide limit exposed by `MemoryLimits` remains a separate safeguard.
+Terminal budget or runtime failures invalidate retained results and clear execution state. Missing/unexpected resume input and malformed JSON are retryable while a valid continuation remains. Native status 11 and `RegorusMemoryBudgetUnsupportedException` remain reserved for compatibility and are not used to reject suspendable execution. The process-wide limit exposed by `MemoryLimits` remains a separate safeguard.
 
 ## RVM with Registered Host-Await Builtins
 

bindings/csharp/Regorus.Tests/RvmMemoryBudgetTests.cs

@@ -2,15 +2,21 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Diagnostics;
+using System.IO;
 using System.Linq;
+using System.Security.Cryptography;
 using System.Text.Json;
+using System.Threading;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
 namespace Regorus.Tests;
 
 [TestClass]
 public sealed class RvmMemoryBudgetTests
 {
+    public TestContext TestContext { get; set; } = null!;
+
     private const ulong TightMemoryBudgetBytes = 64 * 1024;
 
     private const string Policy = """
@@ -30,6 +36,27 @@ package limits.memory
 
     private const string PreloadedResultEntryPoint = "data.limits.memory.large_string";
 
+    private const string SuspendableHostAwaitPolicy = """
+package limits.memory
+import rego.v1
+
+result := count(__builtin_host_await(
+  __builtin_host_await(input.value, "first"),
+  "second"
+))
+""";
+
+    private const string SuspendableHostAwaitEntryPoint = "data.limits.memory.result";
+    private const ulong SuspendableMemoryBudgetBytes = 768 * 1024;
+
+    private const string NativeModulePolicy = """
+package limits.native
+
+result := 1
+""";
+
+    private const string NativeModuleEntryPoint = "data.limits.native.result";
+
     [TestMethod]
     public void Memory_budget_must_be_non_zero()
     {
@@ -86,13 +113,152 @@ public void Serialization_budget_failure_leaves_error_state()
     }
 
     [TestMethod]
-    public void Suspendable_execution_rejects_memory_budget()
+    public void Suspendable_budget_spans_compiled_host_await_thread_migration()
     {
+        var modules = new[] { new PolicyModule("memory_budget.rego", SuspendableHostAwaitPolicy) };
+        using var program = Program.CompileFromModules(
+            """{"value":"request"}""",
+            modules,
+            new[] { SuspendableHostAwaitEntryPoint });
         using var vm = new Rvm();
         vm.SetExecutionMode(ExecutionMode.Suspendable);
-        vm.SetMemoryBudgetConfig(new MemoryBudgetConfig(1024));
+        vm.LoadProgram(program);
+        vm.SetInputJson("""{"value":"request"}""");
+        vm.SetMemoryBudgetConfig(new MemoryBudgetConfig(SuspendableMemoryBudgetBytes));
+
+        Exception? executeError = null;
+        Exception? firstResumeError = null;
+        Exception? secondResumeError = null;
+        string? finalResult = null;
+        using var executeFinished = new ManualResetEventSlim();
+        using var firstResumeFinished = new ManualResetEventSlim();
+
+        var executeThread = new Thread(() =>
+        {
+            try
+            {
+                vm.Execute();
+            }
+            catch (Exception ex)
+            {
+                executeError = ex;
+            }
+            finally
+            {
+                executeFinished.Set();
+            }
+        });
+
+        var firstResumeThread = new Thread(() =>
+        {
+            try
+            {
+                if (!executeFinished.Wait(TimeSpan.FromSeconds(30)))
+                {
+                    throw new TimeoutException("Execute did not finish.");
+                }
+
+                vm.Resume("\"first\"");
+            }
+            catch (Exception ex)
+            {
+                firstResumeError = ex;
+            }
+            finally
+            {
+                firstResumeFinished.Set();
+            }
+        });
+
+        var secondResumeThread = new Thread(() =>
+        {
+            try
+            {
+                if (!firstResumeFinished.Wait(TimeSpan.FromSeconds(30)))
+                {
+                    throw new TimeoutException("First resume did not finish.");
+                }
+
+                finalResult = vm.Resume("\"second\"");
+            }
+            catch (Exception ex)
+            {
+                secondResumeError = ex;
+            }
+        });
+
+        executeThread.Start();
+        firstResumeThread.Start();
+        secondResumeThread.Start();
+        executeThread.Join();
+        firstResumeThread.Join();
+        secondResumeThread.Join();
+
+        Assert.IsNull(executeError, executeError?.ToString());
+        Assert.IsNull(firstResumeError, firstResumeError?.ToString());
+        Assert.IsNull(secondResumeError, secondResumeError?.ToString());
+        Assert.AreEqual("6", finalResult);
+    }
+
+    [TestMethod]
+    public void Suspendable_resume_parse_exhaustion_throws_typed_exception()
+    {
+        var modules = new[] { new PolicyModule("memory_budget.rego", SuspendableHostAwaitPolicy) };
+        using var program = Program.CompileFromModules(
+            """{"value":"request"}""",
+            modules,
+            new[] { SuspendableHostAwaitEntryPoint });
+        using var vm = new Rvm();
+        vm.SetExecutionMode(ExecutionMode.Suspendable);
+        vm.LoadProgram(program);
+        vm.SetInputJson("""{"value":"request"}""");
+        vm.SetMemoryBudgetConfig(new MemoryBudgetConfig(SuspendableMemoryBudgetBytes));
+
+        vm.Execute();
+        vm.ClearMemoryBudgetConfig();
+        vm.SetExecutionMode(ExecutionMode.RunToCompletion);
+        vm.Resume(JsonSerializer.Serialize(new string('x', 450 * 1024)));
+
+        Assert.ThrowsException<RegorusMemoryBudgetExceededException>(
+            () => vm.Resume(JsonSerializer.Serialize(new string('y', 450 * 1024))));
+        StringAssert.Contains(vm.GetExecutionState()!, "Error { error: MemoryBudgetExceeded");
+    }
+
+    [TestMethod]
+    public void Package_native_module_is_identified_after_native_execution()
+    {
+        if (!OperatingSystem.IsWindows())
+        {
+            Assert.Inconclusive("The native module identity assertion requires a Windows DLL.");
+            return;
+        }
+
+        var modules = new[] { new PolicyModule("native_module.rego", NativeModulePolicy) };
+        using var program = Program.CompileFromModules(
+            "{}",
+            modules,
+            new[] { NativeModuleEntryPoint });
+        using var vm = new Rvm();
+        vm.LoadProgram(program);
+
+        Assert.AreEqual("1", vm.ExecuteEntryPoint(NativeModuleEntryPoint));
+
+        using var process = Process.GetCurrentProcess();
+        var nativeModule = process.Modules
+            .Cast<ProcessModule>()
+            .FirstOrDefault(module =>
+                string.Equals(
+                    module.ModuleName,
+                    "regorus_ffi.dll",
+                    StringComparison.OrdinalIgnoreCase));
+        Assert.IsNotNull(nativeModule);
 
-        Assert.ThrowsException<RegorusMemoryBudgetUnsupportedException>(() => vm.Execute());
+        var modulePath = nativeModule!.FileName;
+        Assert.IsTrue(Path.IsPathFullyQualified(modulePath), modulePath);
+        using var moduleStream = File.OpenRead(modulePath);
+        var moduleHash = Convert.ToHexString(SHA256.HashData(moduleStream));
+        TestContext.WriteLine($"REGORUS_NATIVE_MODULE_PATH={modulePath}");
+        TestContext.WriteLine($"REGORUS_NATIVE_MODULE_SHA256={moduleHash}");
     }
 
     private static Program CreateProgram()

bindings/csharp/Regorus/NativeMethods.cs

@@ -877,7 +877,7 @@ internal enum RegorusStatus : uint
         /// </summary>
         MemoryBudgetExceeded,
         /// <summary>
-        /// An RVM memory budget was used with suspendable execution.
+        /// Reserved for native ABI compatibility.
         /// </summary>
         MemoryBudgetUnsupportedInSuspendableExecution,
     }