Dynamic GC based on host, API concurrency limiter, logging for 429 and configurable http worker queue timeout

Directory.Build.props

@@ -55,6 +55,14 @@
     <Authors>CyberDrain</Authors>
     <RepositoryUrl>https://github.com/CyberDrain/CRAFT</RepositoryUrl>
     <RepositoryType>git</RepositoryType>
+    <!--
+      Version stamping. Without this the assembly version defaults to 1.0.0, so the CRAFT version
+      reported in startup telemetry would be meaningless. Overridden at build time, e.g.
+      `dotnet build /p:Version=1.4.2+abc123` (CI feeds the real version). The default keeps a real,
+      non-1.0.0 marker for local builds. AssemblyInformationalVersion carries the full string
+      (including +metadata); the emitter reads that.
+    -->
+    <Version Condition="'$(Version)' == ''">0.0.0-dev</Version>
   </PropertyGroup>
 
 </Project>

Services/Configuration/CraftSettings.cs

@@ -106,4 +106,7 @@ public class CraftSettings
 
     /// <summary>Realtime SSE channel (<c>/.craft/events</c>). See <see cref="RealtimeSettings"/>.</summary>
     public RealtimeSettings Realtime { get; set; } = new();
+
+    /// <summary>Startup phone-home usage telemetry. Off by default. See <see cref="TelemetrySettings"/>.</summary>
+    public TelemetrySettings Telemetry { get; set; } = new();
 }

Services/Configuration/RateLimitSettings.cs

@@ -25,8 +25,44 @@ public class RateLimitSettings
     /// </summary>
     public int QueueLimit { get; set; }
 
+    /// <summary>
+    /// Maximum requests a single app-only API client (client-credentials caller) may have occupying
+    /// the HTTP worker system at once — counting both those queued waiting for a runspace and those
+    /// already executing, since the limiter's lease spans the whole downstream pipeline. Keyed per
+    /// client (its AppId), so one automation cannot monopolise the pool and starve the interactive UI,
+    /// which is never limited by this. Over-limit requests are rejected immediately with 429 (no
+    /// concurrency queue); the caller retries on <c>Retry-After</c>.
+    ///
+    /// <para>
+    /// 0 (default) = unlimited: the feature is off until a value is set. Distinct from
+    /// <see cref="PermitPerWindow"/>, which is a request-RATE cap; this is a simultaneous-in-flight cap.
+    /// Interactive (browser) callers are classified as UI and never counted here.
+    /// </para>
+    ///
+    /// Env override: <c>CRAFT_API_CONCURRENCY_LIMIT</c>.
+    /// </summary>
+    public int ApiConcurrencyLimit { get; set; }
+
     /// <summary>Resolved enabled state, honouring the CRAFT_RATELIMIT_ENABLED environment override.</summary>
     public bool IsEnabled =>
         Enabled
         || string.Equals(Environment.GetEnvironmentVariable("CRAFT_RATELIMIT_ENABLED"), "true", StringComparison.OrdinalIgnoreCase);
+
+    /// <summary>
+    /// Resolved per-client API concurrency cap, honouring the <c>CRAFT_API_CONCURRENCY_LIMIT</c>
+    /// environment override (which wins when it parses to a non-negative integer). 0 = unlimited/off.
+    /// </summary>
+    public int ResolvedApiConcurrencyLimit =>
+        int.TryParse(Environment.GetEnvironmentVariable("CRAFT_API_CONCURRENCY_LIMIT"),
+            System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture,
+            out var fromEnv) && fromEnv >= 0
+            ? fromEnv
+            : ApiConcurrencyLimit;
+
+    /// <summary>
+    /// Whether the rate-limiter middleware needs to run at all: either the per-client rate limiter is
+    /// enabled, or an API concurrency cap is configured. When both are off, the middleware is skipped
+    /// entirely (no per-request limiter cost).
+    /// </summary>
+    public bool RequiresLimiterMiddleware => IsEnabled || ResolvedApiConcurrencyLimit > 0;
 }

Services/Configuration/SkuProfile.cs

@@ -32,4 +32,13 @@ public class SkuProfile
 
     /// <summary>Background worker pool size to apply when this profile matches.</summary>
     public int BgPoolSize { get; set; }
+
+    /// <summary>
+    /// Optional GC heap hard limit, in MB, to apply when this profile matches. Omit or 0 = keep the
+    /// process baseline (typically the DOTNET_GCHeapHardLimit env var baked into the image for the
+    /// smallest tier). The env var is consumed by the CLR before any managed code runs, so this is
+    /// applied after the fact via <see cref="Craft.Hosting.GcHeapLimit"/> — raising the limit is
+    /// always safe; a value the heap has already outgrown is refused and logged.
+    /// </summary>
+    public int? GCHeapHardLimitMB { get; set; }
 }

Services/Configuration/TelemetrySettings.cs

@@ -0,0 +1,41 @@
+namespace Craft.Configuration;
+
+/// <summary>
+/// Startup phone-home telemetry (<c>App:Telemetry:*</c>). One usage report per process start, storm
+/// guarded so a crash loop cannot flood the ingest. See <c>StartupTelemetryService</c>.
+///
+/// <para>
+/// Off by default: nothing is sent until an operator both enables it and supplies an
+/// <see cref="AppId"/> and an <see cref="Endpoint"/>. The <c>CRAFT_TELEMETRY_OPTOUT=1</c> environment
+/// variable forces it off regardless of configuration.
+/// </para>
+/// </summary>
+public class TelemetrySettings
+{
+    /// <summary>Master switch. Off by default (privacy posture is an operator decision).</summary>
+    public bool Enabled { get; set; }
+
+    /// <summary>Ingest URL, e.g. <c>https://reporting.example.com/API/TelemetryIngest</c>. No send without it.</summary>
+    public string? Endpoint { get; set; }
+
+    /// <summary>Application id for this image (<c>cipp</c>, <c>geoipdb</c>, …). No send without it.</summary>
+    public string? AppId { get; set; }
+
+    /// <summary>Storm-guard floor: at most one report per this many hours per instance. Floored at 1.</summary>
+    public int MinIntervalHours { get; set; } = 6;
+
+    /// <summary>Outbound POST timeout in seconds. No retry — the next boot is the retry.</summary>
+    public int TimeoutSeconds { get; set; } = 10;
+
+    /// <summary>Lower bound of the jittered startup delay, in seconds.</summary>
+    public int MinStartupDelaySeconds { get; set; } = 60;
+
+    /// <summary>Upper bound of the jittered startup delay, in seconds.</summary>
+    public int MaxStartupDelaySeconds { get; set; } = 300;
+
+    /// <summary>Table holding the per-instance storm-guard state (<c>instanceId</c>, <c>lastSentUtc</c>).</summary>
+    public string GuardTable { get; set; } = "CraftTelemetryGuard";
+
+    /// <summary>Optional shared token sent as <c>X-Telemetry-Token</c> to a token-gated ingest.</summary>
+    public string? Token { get; set; }
+}

Services/Configuration/WorkerSettings.cs

@@ -75,6 +75,29 @@ public class WorkerSettings
     /// </summary>
     public int BgTimeoutSeconds { get; set; }
 
+    /// <summary>
+    /// How long an incoming HTTP request waits for a free PowerShell runspace when every worker in the
+    /// HTTP pool is already busy, before it is shed with <c>503 "Server busy, please retry"</c>.
+    ///
+    /// <para>
+    /// This is a load-shedding bound, NOT a capacity or execution knob. It does not add throughput —
+    /// under sustained saturation it only changes how long callers wait before the 503 (longer waits
+    /// hold connections and lengthen the tail). Its value is in absorbing <b>brief</b> bursts: a
+    /// request that would have got a worker a few seconds later completes instead of failing spuriously.
+    /// When 503s appear under steady load the levers are <see cref="HttpPoolSize"/> /
+    /// <see cref="MinThreads"/> / a larger host, not this.
+    /// </para>
+    ///
+    /// <para>
+    /// Distinct from <see cref="HttpTimeoutSeconds"/> (which bounds how long a request may <i>execute</i>
+    /// once it holds a worker). 0 or negative = the built-in default of 30 seconds.
+    /// </para>
+    ///
+    /// Env override: <c>CRAFT_HTTP_QUEUE_TIMEOUT</c> (seconds), which wins over this setting.
+    /// Resolved by <c>CraftHostBuilderExtensions.ResolveHttpQueueTimeout</c>.
+    /// </summary>
+    public int HttpQueueTimeoutSeconds { get; set; }
+
     /// <summary>
     /// Environment variables to inject into every PowerShell runspace.
     /// Use "{ApiBasePath}" as a placeholder — it will be replaced with the resolved API directory at startup.

Services/Hosting/CallerClassifier.cs

@@ -0,0 +1,31 @@
+namespace Craft.Hosting;
+
+/// <summary>
+/// Classifies a request as an app-only API client versus an interactive (UI) caller, from the
+/// normalised principal headers <see cref="CraftAuthMiddleware"/> writes.
+/// <para>
+/// The distinction is load-bearing for the API concurrency cap: app-only automation must not be able
+/// to monopolise the shared worker pool and starve interactive users, who are never limited by it.
+/// The rule is exactly the one the hosted app already keys off — a client-credentials caller arrives
+/// with <c>x-ms-client-principal-idp: aad</c> and its AppId (a GUID) as the principal name, whereas an
+/// interactive Entra user is normalised to <c>azureStaticWebApps</c>. Both conditions are required so a
+/// stray <c>aad</c> idp on a non-GUID principal is never misread as an API client.
+/// </para>
+/// </summary>
+public static class CallerClassifier
+{
+    /// <summary>
+    /// True when <paramref name="context"/> is an app-only API client (idp is <c>aad</c> and the
+    /// principal name parses as a GUID AppId). Depends on running after <see cref="CraftAuthMiddleware"/>.
+    /// </summary>
+    public static bool IsApiClient(HttpContext context)
+    {
+        ArgumentNullException.ThrowIfNull(context);
+
+        var idp = context.Request.Headers["x-ms-client-principal-idp"].ToString();
+        if (!string.Equals(idp, "aad", StringComparison.OrdinalIgnoreCase)) return false;
+
+        var name = context.Request.Headers["x-ms-client-principal-name"].ToString();
+        return Guid.TryParse(name, out _);
+    }
+}

Services/Hosting/CraftHostBuilderExtensions.cs

@@ -11,6 +11,7 @@
 using Craft.Services;
 using Craft.Setup;
 using Craft.Storage;
+using Craft.Telemetry;
 using Microsoft.AspNetCore.ResponseCompression;
 using Microsoft.AspNetCore.Server.Kestrel.Core;
 using Microsoft.Extensions.Logging.Console;
@@ -74,6 +75,31 @@ public static int ResolveMinThreads(CraftSettings settings)
         return Math.Max(baseline, forPools);
     }
 
+    /// <summary>The built-in HTTP worker-checkout wait when nothing overrides it.</summary>
+    public const int DefaultHttpQueueTimeoutSeconds = 30;
+
+    /// <summary>
+    /// Resolves how long an HTTP request waits for a free runspace before it is shed with 503:
+    /// the <c>CRAFT_HTTP_QUEUE_TIMEOUT</c> env var (seconds) wins, then an explicit
+    /// <c>Worker:HttpQueueTimeoutSeconds</c>, otherwise the built-in
+    /// <see cref="DefaultHttpQueueTimeoutSeconds"/>. See <see cref="WorkerSettings.HttpQueueTimeoutSeconds"/>
+    /// for why this is a load-shedding bound and not a capacity knob.
+    /// </summary>
+    public static TimeSpan ResolveHttpQueueTimeout(WorkerSettings worker)
+    {
+        ArgumentNullException.ThrowIfNull(worker);
+
+        if (int.TryParse(
+                Environment.GetEnvironmentVariable("CRAFT_HTTP_QUEUE_TIMEOUT"),
+                NumberStyles.Integer, CultureInfo.InvariantCulture, out var fromEnv) && fromEnv > 0)
+            return TimeSpan.FromSeconds(fromEnv);
+
+        if (worker.HttpQueueTimeoutSeconds > 0)
+            return TimeSpan.FromSeconds(worker.HttpQueueTimeoutSeconds);
+
+        return TimeSpan.FromSeconds(DefaultHttpQueueTimeoutSeconds);
+    }
+
     /// <summary>
     /// Kestrel limits: request timeouts, HTTP/2 tuning, and the DoS-relevant caps (body size,
     /// connection count, slow-loris minimum data rates). The caps apply regardless of the timeout.
@@ -243,6 +269,15 @@ public static IServiceCollection AddCraftServices(this IServiceCollection servic
             return new ContainerHealthMonitor(logger, health);
         });
 
+        // Startup telemetry emitter. Registered on EVERY node (not just Background) so a frontend+api
+        // node still reports; the service self-gates on role, config, and the persisted storm guard,
+        // and fires at most once per process. The roles are made injectable here for it to report
+        // host.roles, and IHttpClientFactory for the single outbound POST.
+        services.AddSingleton(roles);
+        services.AddHttpClient();
+        services.AddSingleton<StartupTelemetryService>();
+        services.AddHostedService(sp => sp.GetRequiredService<StartupTelemetryService>());
+
         return services;
     }
 
@@ -268,19 +303,27 @@ public static int ResolveRetryAfterSeconds(RateLimitLease lease, TimeSpan window
     }
 
     /// <summary>
-    /// Per-client fixed-window rate limiter so a single caller cannot exhaust the small HTTP worker
-    /// pool. Enabled by default; turn off with <c>App:RateLimit:Enabled=false</c>. Throttled requests
-    /// get a 429 carrying <c>Retry-After</c>.
+    /// The request limiter, a chain of up to two partitioned limiters sharing one 429 + Retry-After
+    /// rejection path:
+    /// <list type="bullet">
+    ///   <item><description>A per-client fixed-window <b>rate</b> limiter (on by default) so a single
+    ///   caller cannot exhaust the small HTTP worker pool; turn off with <c>App:RateLimit:Enabled=false</c>.</description></item>
+    ///   <item><description>A per-client <b>concurrency</b> cap on app-only API callers
+    ///   (<c>App:RateLimit:ApiConcurrencyLimit</c>, off by default) so one automation cannot hold every
+    ///   runspace at once and starve the interactive UI, which is never capped.</description></item>
+    /// </list>
+    /// The middleware is skipped entirely when neither is active.
     /// </summary>
     public static IServiceCollection AddCraftRateLimiter(
         this IServiceCollection services, CraftSettings settings)
     {
         ArgumentNullException.ThrowIfNull(services);
         ArgumentNullException.ThrowIfNull(settings);
 
-        if (!settings.RateLimit.IsEnabled) return services;
+        var rl = settings.RateLimit;
+        if (!rl.RequiresLimiterMiddleware) return services;
 
-        var window = TimeSpan.FromSeconds(Math.Max(1, settings.RateLimit.WindowSeconds));
+        var window = TimeSpan.FromSeconds(Math.Max(1, rl.WindowSeconds));
 
         services.AddRateLimiter(options =>
         {
@@ -291,22 +334,77 @@ public static IServiceCollection AddCraftRateLimiter(
             // honour unprompted, so emitting it is what makes the limit self-documenting.
             options.OnRejected = (context, _) =>
             {
+                var retryAfter = ResolveRetryAfterSeconds(context.Lease, window);
                 context.HttpContext.Response.Headers.RetryAfter =
-                    ResolveRetryAfterSeconds(context.Lease, window)
-                        .ToString(CultureInfo.InvariantCulture);
+                    retryAfter.ToString(CultureInfo.InvariantCulture);
+
+                // A 429 is otherwise invisible — the caller sees it, the operator does not. Log the
+                // client the limit fired for (the same partition key the limiter counts against) so a
+                // throttled integration or a runaway loop can be identified. Warning, not Error: this
+                // is an expected, client-caused outcome, but one worth surfacing. Resolved per
+                // rejection because service registration has no built provider yet; CreateLogger is
+                // cheap and the factory caches per category. Never let logging break the response.
+                try
+                {
+                    context.HttpContext.RequestServices.GetService<ILoggerFactory>()?
+                        .CreateLogger("Craft.Hosting.RateLimiter")
+                        .LogWarning(
+                            "Rate limit exceeded — 429 for {Client} on {Method} {Path}; Retry-After {RetryAfter}s",
+                            RateLimitPartitionKey.Resolve(context.HttpContext),
+                            context.HttpContext.Request.Method,
+                            context.HttpContext.Request.Path.Value,
+                            retryAfter);
+                }
+                catch { /* logging must never turn a throttle into a 500 */ }
+
                 return ValueTask.CompletedTask;
             };
 
-            options.GlobalLimiter = PartitionedRateLimiter.Create<HttpContext, string>(context =>
-                RateLimitPartition.GetFixedWindowLimiter(
-                    RateLimitPartitionKey.Resolve(context),
-                    _ => new FixedWindowRateLimiterOptions
-                    {
-                        PermitLimit = Math.Max(1, settings.RateLimit.PermitPerWindow),
-                        Window = window,
-                        QueueLimit = Math.Max(0, settings.RateLimit.QueueLimit),
-                        QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
-                    }));
+            // Built as a chain so a request must satisfy every active limiter. Order is immaterial —
+            // rejection by either sheds the request through the one OnRejected above.
+            var limiters = new List<PartitionedRateLimiter<HttpContext>>(2);
+
+            if (rl.IsEnabled)
+            {
+                // Per-client request RATE. Partition key is the authenticated principal (else address).
+                limiters.Add(PartitionedRateLimiter.Create<HttpContext, string>(context =>
+                    RateLimitPartition.GetFixedWindowLimiter(
+                        RateLimitPartitionKey.Resolve(context),
+                        _ => new FixedWindowRateLimiterOptions
+                        {
+                            PermitLimit = Math.Max(1, rl.PermitPerWindow),
+                            Window = window,
+                            QueueLimit = Math.Max(0, rl.QueueLimit),
+                            QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
+                        })));
+            }
+
+            var apiConcurrency = rl.ResolvedApiConcurrencyLimit;
+            if (apiConcurrency > 0)
+            {
+                // Per-client CONCURRENCY on app-only API callers only. The lease is held for the whole
+                // downstream pipeline, so a permit covers both the wait for a runspace and execution —
+                // that is what makes this cap "in flight + queued for a worker" rather than just one.
+                limiters.Add(PartitionedRateLimiter.Create<HttpContext, string>(context =>
+                {
+                    if (!CallerClassifier.IsApiClient(context))
+                        return RateLimitPartition.GetNoLimiter("ui");
+
+                    // Keyed on the AppId so each client's budget is its own, not shared across clients.
+                    return RateLimitPartition.GetConcurrencyLimiter(
+                        context.Request.Headers["x-ms-client-principal-name"].ToString(),
+                        _ => new ConcurrencyLimiterOptions
+                        {
+                            PermitLimit = apiConcurrency,
+                            QueueLimit = 0,   // fail fast: the excess is rejected, not parked
+                            QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
+                        });
+                }));
+            }
+
+            options.GlobalLimiter = limiters.Count == 1
+                ? limiters[0]
+                : PartitionedRateLimiter.CreateChained(limiters.ToArray());
         });
 
         return services;