Fix/memory retention and orchestration priority usage

Runtime/CraftRuntime/Start-CraftOrchestrator.ps1

@@ -6,14 +6,17 @@ function Start-CraftOrchestrator {
         Generic bridge function that serializes a batch of tasks and queues them
         for fan-out execution via the C# OrchestratorService.
 
-        This is the framework-provided default. Application-specific wrappers
-        (e.g. Start-CIPPOrchestrator) can override via Orchestrator.BridgeFunction
-        in appsettings.json if they need additional logic (queue routing, dual-boot, etc.)
+        This is the framework-provided default. Applications with additional logic
+        (queue routing, dual-boot, etc.) ship their own wrapper (e.g. Start-CIPPOrchestrator)
+        and simply call that instead of this function.
 
     .PARAMETER InputObject
         Orchestrator input with the following structure:
           - OrchestratorName  (string)  — unique run identifier
           - Batch             (array)   — task objects, each with at least FunctionName
+          - Priority          (int)     — optional; queue priority bucket for the run (lower = sooner).
+                                          Defaults to the parent run's priority when queued from inside
+                                          an orchestrator run, else 4.
           - QueueFunction     (object)  — optional; called first to generate the batch dynamically
             - FunctionName    (string)  — Push-{FunctionName} is called
             - Parameters      (object)  — passed as -Item to the queue function
@@ -97,11 +100,24 @@ function Start-CraftOrchestrator {
         throw
     }
 
-    Write-Information "Craft: Queuing orchestrator '$OrchestratorName' ($TaskCount tasks$(if ($PostExecFunctionName) { ", PostExec: $PostExecFunctionName" }))"
+    # Priority resolution: explicit on the InputObject wins; otherwise inherit the enclosing run's
+    # priority (ambient, set by JobManager for orchestrator activities and post-exec jobs); otherwise
+    # the default band. Guard the explicit value — the store clamps into 0-99 buckets, so a stray
+    # negative would silently land in the critical P00 bucket.
+    $Priority = $InputObject.Priority
+    if ($null -ne $Priority) {
+        $Priority = [int]$Priority
+        if ($Priority -lt 0 -or $Priority -gt 99) { $Priority = $null }
+    }
+    if ($null -eq $Priority) {
+        $Priority = [Craft.Hosting.OperationContext]::Current.Priority ?? 4
+    }
+
+    Write-Information "Craft: Queuing orchestrator '$OrchestratorName' ($TaskCount tasks, P$Priority$(if ($PostExecFunctionName) { ", PostExec: $PostExecFunctionName" }))"
     [Craft.Services.OrchestratorBridge]::QueueOrchestrationFromFile(
         $OrchestratorName,
         $BatchPath,
-        4,
+        $Priority,
         $PostExecFunctionName,
         $PostExecParametersJson,
         $InputObject.Reference

Services/Bridges/OrchestratorBridge.cs

@@ -1,6 +1,7 @@
 using System.Collections.Concurrent;
 using Craft.Hosting;
 using Craft.Orchestration;
+using Craft.Storage;
 
 // NAMESPACE PINNED — do not change.
 // Downstream PowerShell reaches these types by fully-qualified name, e.g.
@@ -26,6 +27,10 @@ public static void QueueOrchestration(string name, string batchJson, int priorit
         string? postExecFunctionName = null, string? postExecParametersJson = null,
         string? reference = null)
     {
+        // Sanitized here as well as at run creation so the child-run registration in DrainPending
+        // records the SAME name the service ends up creating — a raw name with a table-illegal
+        // character would register a child link no live run ever matches.
+        name = TableKeys.Sanitize(name);
         var parentRunName = OperationContext.Current?.RunName;
         s_pending.Enqueue(new PendingOrchestration(name, batchJson, priority,
             postExecFunctionName, postExecParametersJson, parentRunName, reference));
@@ -46,6 +51,7 @@ public static void QueueOrchestrationFromFile(string name, string batchFilePath,
         string? postExecFunctionName = null, string? postExecParametersJson = null,
         string? reference = null)
     {
+        name = TableKeys.Sanitize(name);
         var parentRunName = OperationContext.Current?.RunName;
         s_pending.Enqueue(new PendingOrchestration(name, string.Empty, priority,
             postExecFunctionName, postExecParametersJson, parentRunName, reference, batchFilePath));

Services/Bridges/QueueBridge.cs

@@ -28,11 +28,22 @@ public static void Initialize(PowerShellRunnerService runner, JobManager jobMana
         s_queueTaskFunction = queueTaskFunction;
     }
 
+    /// <summary>Default-priority enqueue. Kept for compatibility with callers that predate priorities.</summary>
     public static void Enqueue(string cmdlet, string parametersJson)
+        => Enqueue(cmdlet, parametersJson, DefaultPriority);
+
+    /// <summary>
+    /// Enqueue with an explicit job priority. User-initiated starters (run-now scheduled tasks) pass a
+    /// high band here so they are not claimed behind the background fan-out backlog — the queue claims
+    /// strictly by priority bucket, so a starter below the backlog's band cannot run until it drains.
+    /// </summary>
+    public static void Enqueue(string cmdlet, string parametersJson, int priority)
     {
-        s_pending.Enqueue(new PendingQueueCommand(cmdlet, parametersJson));
+        s_pending.Enqueue(new PendingQueueCommand(cmdlet, parametersJson, priority));
     }
 
+    private const int DefaultPriority = 5;
+
     public static void DrainPending()
     {
         if (string.IsNullOrEmpty(s_queueTaskFunction)) return;
@@ -46,7 +57,7 @@ public static void DrainPending()
             var captured = cmd;
             s_jobManager.Enqueue(
                 name: $"Queue-{captured.Cmdlet}",
-                priority: 5,
+                priority: captured.Priority,
                 runName: $"Queue-{captured.Cmdlet}-{Guid.NewGuid():N}",
                 id: $"Queue-{Guid.NewGuid():N}",
                 work: async (ct) =>
@@ -65,5 +76,5 @@ public static void DrainPending()
         }
     }
 
-    public record PendingQueueCommand(string Cmdlet, string ParametersJson);
+    public record PendingQueueCommand(string Cmdlet, string ParametersJson, int Priority = DefaultPriority);
 }

Services/Hosting/OperationContext.cs

@@ -52,6 +52,15 @@ public sealed class Invocation
         /// <summary>Parent orchestrator run name (null for HTTP requests).</summary>
         public string? RunName { get; init; }
 
+        /// <summary>
+        /// Queue priority of the enclosing run, exposed so nested enqueues can inherit it
+        /// (Start-CIPPOrchestrator reads this to default a child run to its parent's priority).
+        /// Set only for orchestrator activity jobs and post-execution jobs — plain closure jobs
+        /// (scheduler starters, queue starters) deliberately leave it null, because their own job
+        /// priority orders the starter script, not the work it goes on to enqueue.
+        /// </summary>
+        public int? Priority { get; init; }
+
         /// <summary>Category: "HTTP", "Job", "Planner".</summary>
         public string Category { get; init; } = "Job";
 

Services/Orchestration/JobManager.cs

@@ -16,10 +16,11 @@ namespace Craft.Orchestration;
 ///   - Old completed jobs are cleaned up every 5 minutes
 ///
 /// Priority levels (lower = higher priority, callers can use any int):
-///   0-1 = Critical (system cleanup, user tasks)
-///   2-3 = High     (audit logs, webhooks)
-///   4-5 = Normal   (standards, drift, cache)
-///   6+  = Low      (alerts, DB cache, tests, extensions)
+///   0-1 = Critical (reserved: system cleanup, emergencies)
+///   2   = User-initiated (HTTP-triggered fan-outs, user scheduled tasks, run-now starters)
+///   3   = Elevated background (baseline runs)
+///   4-5 = Normal   (background fan-out default; non-HTTP queue starters)
+///   6+  = Low      (alerts, tests, extensions)
 ///
 /// How priority dispatch works:
 ///   The dispatch loop waits for both an item AND a concurrency slot.
@@ -109,8 +110,12 @@ public JobManager(ILogger<JobManager> logger, CraftSettings settings, Background
     /// <param name="work">Async work function. Receives a CancellationToken for shutdown.</param>
     /// <param name="runName">Optional run group name (e.g. "CIPPDBCacheRun") for grouping in status APIs.</param>
     /// <param name="id">Optional explicit job ID. Auto-generated if null.</param>
+    /// <param name="inheritPriority">Priority that work THIS JOB ENQUEUES should inherit, exposed to the
+    /// job via <see cref="OperationContext.Invocation.Priority"/>. Distinct from <paramref name="priority"/>:
+    /// a starter script's own queue priority orders the starter, not its fan-out. Only post-execution jobs
+    /// pass this (the run's priority); leave null everywhere else.</param>
     public string Enqueue(string name, int priority, Func<CancellationToken, Task> work,
-        string? runName = null, string? id = null)
+        string? runName = null, string? id = null, int? inheritPriority = null)
     {
         var jobId = id ?? $"{name}_{Guid.NewGuid():N}";
         var record = new JobRecord
@@ -128,7 +133,7 @@ public string Enqueue(string name, int priority, Func<CancellationToken, Task> w
 
         lock (_queueLock)
         {
-            _pendingQueue.Enqueue(new QueuedJob(record, work, null), priority);
+            _pendingQueue.Enqueue(new QueuedJob(record, work, null) { InheritPriority = inheritPriority }, priority);
         }
         _itemAvailable.Release();
 
@@ -338,10 +343,15 @@ private async Task RunJobAsync(QueuedJob job, CancellationToken ct)
 
         try
         {
-            // Set operation context for traceability — ExecuteScript reads RunName from this
+            // Set operation context for traceability — ExecuteScript reads RunName from this.
+            // Priority is the value nested enqueues should inherit: for descriptor jobs (orchestrator
+            // activities) that is the task's own queue priority; for closures it is only set when the
+            // enqueuer said so (post-exec passes the run's priority). Plain starters expose none —
+            // their job priority orders the starter script, not the work it spawns.
             var parentInvocation = new OperationContext.Invocation(job.Record.Name)
             {
                 RunName = job.Record.RunName,
+                Priority = job.Descriptor != null ? job.Record.Priority : job.InheritPriority,
                 Category = "Job"
             };
             opScope = OperationContext.Set(parentInvocation);
@@ -702,5 +712,8 @@ private sealed record QueuedJob(JobRecord Record, Func<CancellationToken, Task>?
     {
         /// <summary>Bumped by <see cref="ChangePriority"/>; entries below the live epoch are superseded.</summary>
         public int Epoch { get; init; }
+
+        /// <summary>Priority nested enqueues should inherit (closure jobs only — see Enqueue).</summary>
+        public int? InheritPriority { get; init; }
     }
 }

Services/Orchestration/OrchestratorRun.cs

@@ -5,7 +5,9 @@ public class OrchestratorRun
     public string Name { get; set; } = string.Empty;
     public string? Reference { get; set; }
     public string Status { get; set; } = "Pending";
-    public int Priority { get; set; } = 2;
+    // 4 matches what every live enqueue path actually passes when a caller sets nothing — a run row
+    // rehydrated without a stored priority must not come back HIGHER than it originally ran.
+    public int Priority { get; set; } = 4;
     public DateTime StartedUtc { get; set; }
     public DateTime? CompletedUtc { get; set; }
     public List<OrchestratorTaskItem> Tasks { get; set; } = [];