Skip to content

Commit 4b479e5

Browse files
authored
Merge pull request #40 from CyberDrain/dev
Dynamic GC based on host, API concurrency limiter, logging for 429 and configurable http worker queue timeout
2 parents 8102d00 + 011960e commit 4b479e5

51 files changed

Lines changed: 2907 additions & 425 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

Directory.Build.props

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,14 @@
5555
<Authors>CyberDrain</Authors>
5656
<RepositoryUrl>https://github.com/CyberDrain/CRAFT</RepositoryUrl>
5757
<RepositoryType>git</RepositoryType>
58+
<!--
59+
Version stamping. Without this the assembly version defaults to 1.0.0, so the CRAFT version
60+
reported in startup telemetry would be meaningless. Overridden at build time, e.g.
61+
`dotnet build /p:Version=1.4.2+abc123` (CI feeds the real version). The default keeps a real,
62+
non-1.0.0 marker for local builds. AssemblyInformationalVersion carries the full string
63+
(including +metadata); the emitter reads that.
64+
-->
65+
<Version Condition="'$(Version)' == ''">0.0.0-dev</Version>
5866
</PropertyGroup>
5967

6068
</Project>

Services/Bridges/WorkerMetricsBridge.cs

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -692,6 +692,27 @@ public static int CancelRun(string runName)
692692
public static bool DeleteJob(string jobId)
693693
=> s_jobManager?.DeleteJob(jobId) ?? false;
694694

695+
/// <summary>
696+
/// Empty the durable job queue — a maintenance/reset primitive. Returns the number of queue rows
697+
/// removed, or -1 if the orchestrator is unavailable or the clear failed. In-flight work is
698+
/// unaffected and Pending tasks may be re-driven, so pair with <see cref="CancelRun"/> when the
699+
/// intent is to STOP work rather than clear a wedged or corrupted queue.
700+
/// PS usage: <c>[Craft.Services.WorkerMetricsBridge]::ClearQueue()</c>.
701+
/// </summary>
702+
public static int ClearQueue()
703+
{
704+
if (s_orchestrator is not { } orchestrator) return -1;
705+
try
706+
{
707+
return Task.Run(() => orchestrator.ClearQueueAsync(CancellationToken.None)).GetAwaiter().GetResult();
708+
}
709+
catch (Exception ex)
710+
{
711+
s_logger?.LogWarning(ex, "[WorkerMetrics] ClearQueue failed");
712+
return -1;
713+
}
714+
}
715+
695716
/// <summary>
696717
/// Change a queued job's priority. In the local buffer this re-enqueues at the new priority; for an
697718
/// unclaimed durable row it moves the row to the new priority bucket (keeping its age) and records

Services/Configuration/CraftSettings.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,4 +106,7 @@ public class CraftSettings
106106

107107
/// <summary>Realtime SSE channel (<c>/.craft/events</c>). See <see cref="RealtimeSettings"/>.</summary>
108108
public RealtimeSettings Realtime { get; set; } = new();
109+
110+
/// <summary>Startup phone-home usage telemetry. Off by default. See <see cref="TelemetrySettings"/>.</summary>
111+
public TelemetrySettings Telemetry { get; set; } = new();
109112
}

Services/Configuration/OrchestratorSettings.cs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,16 @@ public class OrchestratorSettings
1919
/// </summary>
2020
public bool BatchStatusWrites { get; set; } = true;
2121

22+
/// <summary>
23+
/// Coalesce SMALL task results (those that fit one Azure Table property) through the batched status
24+
/// writer instead of a per-task upsert on the fan-out critical path. Each result is written BEFORE
25+
/// its task's terminal marker in the same flush, so a result is always durable before the task is
26+
/// counted done (and therefore before finalize/post-execution reads it). Large results keep the
27+
/// directly-awaited chunked path. Default true; only applies when <see cref="BatchStatusWrites"/> is
28+
/// also true. Set false to fall back to the original per-task awaited result write.
29+
/// </summary>
30+
public bool BatchResultWrites { get; set; } = true;
31+
2232
/// <summary>
2333
/// When batching status writes, write the pre-invoke "Running" marker under a synchronous barrier so it
2434
/// is durable BEFORE the task invokes (batched with other concurrently-starting tasks). Preserves the

Services/Configuration/RateLimitSettings.cs

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,44 @@ public class RateLimitSettings
2525
/// </summary>
2626
public int QueueLimit { get; set; }
2727

28+
/// <summary>
29+
/// Maximum requests a single app-only API client (client-credentials caller) may have occupying
30+
/// the HTTP worker system at once — counting both those queued waiting for a runspace and those
31+
/// already executing, since the limiter's lease spans the whole downstream pipeline. Keyed per
32+
/// client (its AppId), so one automation cannot monopolise the pool and starve the interactive UI,
33+
/// which is never limited by this. Over-limit requests are rejected immediately with 429 (no
34+
/// concurrency queue); the caller retries on <c>Retry-After</c>.
35+
///
36+
/// <para>
37+
/// 0 (default) = unlimited: the feature is off until a value is set. Distinct from
38+
/// <see cref="PermitPerWindow"/>, which is a request-RATE cap; this is a simultaneous-in-flight cap.
39+
/// Interactive (browser) callers are classified as UI and never counted here.
40+
/// </para>
41+
///
42+
/// Env override: <c>CRAFT_API_CONCURRENCY_LIMIT</c>.
43+
/// </summary>
44+
public int ApiConcurrencyLimit { get; set; }
45+
2846
/// <summary>Resolved enabled state, honouring the CRAFT_RATELIMIT_ENABLED environment override.</summary>
2947
public bool IsEnabled =>
3048
Enabled
3149
|| string.Equals(Environment.GetEnvironmentVariable("CRAFT_RATELIMIT_ENABLED"), "true", StringComparison.OrdinalIgnoreCase);
50+
51+
/// <summary>
52+
/// Resolved per-client API concurrency cap, honouring the <c>CRAFT_API_CONCURRENCY_LIMIT</c>
53+
/// environment override (which wins when it parses to a non-negative integer). 0 = unlimited/off.
54+
/// </summary>
55+
public int ResolvedApiConcurrencyLimit =>
56+
int.TryParse(Environment.GetEnvironmentVariable("CRAFT_API_CONCURRENCY_LIMIT"),
57+
System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture,
58+
out var fromEnv) && fromEnv >= 0
59+
? fromEnv
60+
: ApiConcurrencyLimit;
61+
62+
/// <summary>
63+
/// Whether the rate-limiter middleware needs to run at all: either the per-client rate limiter is
64+
/// enabled, or an API concurrency cap is configured. When both are off, the middleware is skipped
65+
/// entirely (no per-request limiter cost).
66+
/// </summary>
67+
public bool RequiresLimiterMiddleware => IsEnabled || ResolvedApiConcurrencyLimit > 0;
3268
}

Services/Configuration/SkuProfile.cs

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,25 @@ public class SkuProfile
3232

3333
/// <summary>Background worker pool size to apply when this profile matches.</summary>
3434
public int BgPoolSize { get; set; }
35+
36+
/// <summary>
37+
/// Optional GC heap hard limit, in MB, to apply when this profile matches. Three-way, mirroring the
38+
/// <c>CRAFT_GC_HEAP_LIMIT_MB</c> override:
39+
/// <list type="bullet">
40+
/// <item><description>Omitted / null (or negative) = no opinion, keep the process baseline (typically
41+
/// the DOTNET_GCHeapHardLimit env var baked into the image for the smallest tier).</description></item>
42+
/// <item><description><c>0</c> = disable the cap entirely, so the GC uses the container's own memory
43+
/// allowance — for tiers with more memory than the baked limit lets them use.</description></item>
44+
/// <item><description>&gt; 0 = set that many MB.</description></item>
45+
/// </list>
46+
/// The baked env var is consumed by the CLR before any managed code runs, so this is applied after
47+
/// the fact via <see cref="Craft.Hosting.GcHeapLimit"/> — raising or removing the limit is always
48+
/// safe; a positive value the heap has already outgrown is refused and logged.
49+
/// <para>
50+
/// Per-instance override: the <c>CRAFT_GC_HEAP_LIMIT_MB</c> env var wins over this value (a positive
51+
/// value sets the cap; <c>0</c> disables the cap entirely), so an operator can hand-tune one host
52+
/// without editing the fleet-wide profile list.
53+
/// </para>
54+
/// </summary>
55+
public int? GCHeapHardLimitMB { get; set; }
3556
}
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
namespace Craft.Configuration;
2+
3+
/// <summary>
4+
/// Startup phone-home telemetry (<c>App:Telemetry:*</c>). One usage report per process start, storm
5+
/// guarded so a crash loop cannot flood the ingest. See <c>StartupTelemetryService</c>.
6+
///
7+
/// <para>
8+
/// Off by default: nothing is sent until an operator both enables it and supplies an
9+
/// <see cref="AppId"/> and an <see cref="Endpoint"/>. The <c>CRAFT_TELEMETRY_OPTOUT=1</c> environment
10+
/// variable forces it off regardless of configuration.
11+
/// </para>
12+
/// </summary>
13+
public class TelemetrySettings
14+
{
15+
/// <summary>Master switch. Off by default (privacy posture is an operator decision).</summary>
16+
public bool Enabled { get; set; }
17+
18+
/// <summary>Ingest URL, e.g. <c>https://reporting.example.com/API/TelemetryIngest</c>. No send without it.</summary>
19+
public string? Endpoint { get; set; }
20+
21+
/// <summary>Application id for this image (<c>cipp</c>, <c>geoipdb</c>, …). No send without it.</summary>
22+
public string? AppId { get; set; }
23+
24+
/// <summary>Storm-guard floor: at most one report per this many hours per instance. Floored at 1.</summary>
25+
public int MinIntervalHours { get; set; } = 6;
26+
27+
/// <summary>Outbound POST timeout in seconds. No retry — the next boot is the retry.</summary>
28+
public int TimeoutSeconds { get; set; } = 10;
29+
30+
/// <summary>Lower bound of the jittered startup delay, in seconds.</summary>
31+
public int MinStartupDelaySeconds { get; set; } = 60;
32+
33+
/// <summary>Upper bound of the jittered startup delay, in seconds.</summary>
34+
public int MaxStartupDelaySeconds { get; set; } = 300;
35+
36+
/// <summary>Table holding the per-instance storm-guard state (<c>instanceId</c>, <c>lastSentUtc</c>).</summary>
37+
public string GuardTable { get; set; } = "CraftTelemetryGuard";
38+
39+
/// <summary>Optional shared token sent as <c>X-Telemetry-Token</c> to a token-gated ingest.</summary>
40+
public string? Token { get; set; }
41+
}

Services/Configuration/WorkerSettings.cs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,23 @@ public class WorkerSettings
3535
/// </summary>
3636
public bool IgnoreSkuProfiles { get; set; }
3737

38+
/// <summary>
39+
/// A second SkuProfiles matrix, selected when the env var named by <see cref="SkuProfilesAltEnv"/> is
40+
/// present (set to a non-empty value); otherwise <see cref="SkuProfiles"/> is used. Lets a deployment
41+
/// ship one config with two sizings — e.g. a smaller per-instance matrix for instances packed onto a
42+
/// shared App Service Plan — and pick between them with a single env var. Same matching rules as
43+
/// <see cref="SkuProfiles"/>. Ignored (falls back to <see cref="SkuProfiles"/>) when empty.
44+
/// </summary>
45+
public List<SkuProfile> SkuProfilesAlt { get; set; } = [];
46+
47+
/// <summary>
48+
/// Name of the env var whose presence selects <see cref="SkuProfilesAlt"/> instead of
49+
/// <see cref="SkuProfiles"/> (e.g. "CIPP_HOSTED"). Null/empty = the second matrix is never used.
50+
/// Only presence matters — set the var (to any non-empty value) on the instances that should use the
51+
/// second matrix, and leave it unset on the rest. Configurable so each app picks its own flag.
52+
/// </summary>
53+
public string? SkuProfilesAltEnv { get; set; }
54+
3855
/// <summary>
3956
/// Minimum .NET thread-pool worker/completion threads. <b>0 (default) = derive from the pool
4057
/// sizes</b>, which is almost always what you want; set a number only to pin it.
@@ -75,6 +92,29 @@ public class WorkerSettings
7592
/// </summary>
7693
public int BgTimeoutSeconds { get; set; }
7794

95+
/// <summary>
96+
/// How long an incoming HTTP request waits for a free PowerShell runspace when every worker in the
97+
/// HTTP pool is already busy, before it is shed with <c>503 "Server busy, please retry"</c>.
98+
///
99+
/// <para>
100+
/// This is a load-shedding bound, NOT a capacity or execution knob. It does not add throughput —
101+
/// under sustained saturation it only changes how long callers wait before the 503 (longer waits
102+
/// hold connections and lengthen the tail). Its value is in absorbing <b>brief</b> bursts: a
103+
/// request that would have got a worker a few seconds later completes instead of failing spuriously.
104+
/// When 503s appear under steady load the levers are <see cref="HttpPoolSize"/> /
105+
/// <see cref="MinThreads"/> / a larger host, not this.
106+
/// </para>
107+
///
108+
/// <para>
109+
/// Distinct from <see cref="HttpTimeoutSeconds"/> (which bounds how long a request may <i>execute</i>
110+
/// once it holds a worker). 0 or negative = the built-in default of 30 seconds.
111+
/// </para>
112+
///
113+
/// Env override: <c>CRAFT_HTTP_QUEUE_TIMEOUT</c> (seconds), which wins over this setting.
114+
/// Resolved by <c>CraftHostBuilderExtensions.ResolveHttpQueueTimeout</c>.
115+
/// </summary>
116+
public int HttpQueueTimeoutSeconds { get; set; }
117+
78118
/// <summary>
79119
/// Environment variables to inject into every PowerShell runspace.
80120
/// Use "{ApiBasePath}" as a placeholder — it will be replaced with the resolved API directory at startup.
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
namespace Craft.Hosting;
2+
3+
/// <summary>
4+
/// Classifies a request as an app-only API client versus an interactive (UI) caller, from the
5+
/// normalised principal headers <see cref="CraftAuthMiddleware"/> writes.
6+
/// <para>
7+
/// The distinction is load-bearing for the API concurrency cap: app-only automation must not be able
8+
/// to monopolise the shared worker pool and starve interactive users, who are never limited by it.
9+
/// The rule is exactly the one the hosted app already keys off — a client-credentials caller arrives
10+
/// with <c>x-ms-client-principal-idp: aad</c> and its AppId (a GUID) as the principal name, whereas an
11+
/// interactive Entra user is normalised to <c>azureStaticWebApps</c>. Both conditions are required so a
12+
/// stray <c>aad</c> idp on a non-GUID principal is never misread as an API client.
13+
/// </para>
14+
/// </summary>
15+
public static class CallerClassifier
16+
{
17+
/// <summary>
18+
/// True when <paramref name="context"/> is an app-only API client (idp is <c>aad</c> and the
19+
/// principal name parses as a GUID AppId). Depends on running after <see cref="CraftAuthMiddleware"/>.
20+
/// </summary>
21+
public static bool IsApiClient(HttpContext context)
22+
{
23+
ArgumentNullException.ThrowIfNull(context);
24+
25+
var idp = context.Request.Headers["x-ms-client-principal-idp"].ToString();
26+
if (!string.Equals(idp, "aad", StringComparison.OrdinalIgnoreCase)) return false;
27+
28+
var name = context.Request.Headers["x-ms-client-principal-name"].ToString();
29+
return Guid.TryParse(name, out _);
30+
}
31+
}

0 commit comments

Comments
 (0)