diff --git a/.editorconfig b/.editorconfig index 3a2646d..6d7c665 100644 --- a/.editorconfig +++ b/.editorconfig @@ -197,6 +197,7 @@ dotnet_diagnostic.IDE0060.severity = suggestion # unused parameter # Permanently off — deliberate choices, not backlog. dotnet_diagnostic.CA1848.severity = none # LoggerMessage source-gen delegates: not adopting +dotnet_diagnostic.CA1873.severity = none # Guarded log args: companion to CA1848; IsEnabled wrapping every call is noise without source-gen dotnet_diagnostic.CA2007.severity = none # ConfigureAwait: no SynchronizationContext in ASP.NET Core # ── Analyzer rules: cleared backlog ───────────────────────────────────────────────────────────────── diff --git a/.github/workflows/dev-container.yml b/.github/workflows/dev-container.yml index 6f227d7..1ca0071 100644 --- a/.github/workflows/dev-container.yml +++ b/.github/workflows/dev-container.yml @@ -6,9 +6,9 @@ on: paths: - 'build/**' - 'docker/**' - - 'Runtime/**' - - 'Services/**' - - 'Craft.csproj' + - 'src/Craft/**' + - 'src/Craft.Configuration/**' + - 'src/Craft.Contracts/**' - 'Craft.sln' - 'tests/**' - 'Directory.Build.props' @@ -73,9 +73,9 @@ jobs: context: . file: build/Dockerfile # Always re-resolve the base image instead of reusing whatever the runner or the GHA layer - # cache already has. The Dockerfile intentionally floats on the `8.0-*` tag so .NET runtime + # cache already has. The Dockerfile intentionally floats on the `10.0-*` tag so .NET runtime # patches are picked up automatically — but that only works if the tag is actually - # re-resolved. Without this, a patched runtime (e.g. 8.0.28 -> 8.0.29, six High CVEs) can sit + # re-resolved. Without this, a patched runtime (e.g. 10.0.9 -> 10.0.10) can sit # unapplied in published images while every project-level dependency scans clean. pull: true platforms: linux/amd64,linux/arm64 @@ -87,7 +87,7 @@ jobs: COMMIT_SHA=${{ github.sha }} IMAGE_TAG=dev BUILD_CONFIGURATION=Debug - DOTNET_REGISTRY=${{ vars.DOTNET_REGISTRY || 'ghcr.io/cyberdrain' }} + DOTNET_REGISTRY=${{ vars.DOTNET_REGISTRY || 'mcr.microsoft.com' }} tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:dev labels: | org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 1cafb25..227abaa 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -9,9 +9,9 @@ on: paths: - 'build/**' - 'docker/**' - - 'Runtime/**' - - 'Services/**' - - 'Craft.csproj' + - 'src/Craft/**' + - 'src/Craft.Configuration/**' + - 'src/Craft.Contracts/**' - 'Craft.sln' - 'tests/**' - 'Directory.Build.props' @@ -22,9 +22,9 @@ on: paths: - 'build/**' - 'docker/**' - - 'Runtime/**' - - 'Services/**' - - 'Craft.csproj' + - 'src/Craft/**' + - 'src/Craft.Configuration/**' + - 'src/Craft.Contracts/**' - 'Craft.sln' - 'tests/**' - 'Directory.Build.props' @@ -105,8 +105,13 @@ jobs: - uses: actions/checkout@v4 # Build the AzureTables image the E2E stack runs (compose references it as craft:ci). + # --pull re-resolves floating 10.0-* bases; DOTNET_REGISTRY defaults to MCR until the GHCR + # CyberDrain mirror publishes .NET 10 tags (override via repo variable). - name: Build image - run: docker build -f build/Dockerfile -t craft:ci . + run: > + docker build --pull -f build/Dockerfile + --build-arg DOTNET_REGISTRY=${{ vars.DOTNET_REGISTRY || 'mcr.microsoft.com' }} + -t craft:ci . # Brings up Azurite + the SUT (combined role), asserts every subsystem PASS/FAIL, and exits # non-zero on any failure. pwsh, docker, and docker compose are preinstalled on ubuntu-latest. diff --git a/.github/workflows/release-container.yml b/.github/workflows/release-container.yml index 8a07823..0912f35 100644 --- a/.github/workflows/release-container.yml +++ b/.github/workflows/release-container.yml @@ -45,9 +45,9 @@ jobs: context: . file: build/Dockerfile # Always re-resolve the base image instead of reusing whatever the runner or the GHA layer - # cache already has. The Dockerfile intentionally floats on the `8.0-*` tag so .NET runtime + # cache already has. The Dockerfile intentionally floats on the `10.0-*` tag so .NET runtime # patches are picked up automatically — but that only works if the tag is actually - # re-resolved. Without this, a patched runtime (e.g. 8.0.28 -> 8.0.29, six High CVEs) can sit + # re-resolved. Without this, a patched runtime (e.g. 10.0.9 -> 10.0.10) can sit # unapplied in published images while every project-level dependency scans clean. pull: true push: true @@ -57,7 +57,7 @@ jobs: APP_VERSION=${{ steps.version.outputs.app_version }} COMMIT_SHA=${{ github.sha }} IMAGE_TAG=latest - DOTNET_REGISTRY=${{ vars.DOTNET_REGISTRY || 'ghcr.io/cyberdrain' }} + DOTNET_REGISTRY=${{ vars.DOTNET_REGISTRY || 'mcr.microsoft.com' }} tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:latest ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:${{ steps.version.outputs.app_version }} diff --git a/.gitignore b/.gitignore index 8e738aa..2140423 100644 --- a/.gitignore +++ b/.gitignore @@ -22,11 +22,11 @@ Thumbs.db # Logs *.log -# Local settings (contains secrets) -# CRAFT ships no appsettings.json — appsettings.example.jsonc is the annotated reference, and real -# config comes from App__* env vars or a file the downstream app supplies. Anything matching -# appsettings*.json here is therefore somebody's local override, which is exactly the file most likely -# to hold a storage connection string. Keep it out of the repo. +# Local settings +# CRAFT ships no appsettings.json — appsettings.example.jsonc is the annotated reference. +# Local secrets belong in `dotnet user-secrets` (UserSecretsId on Craft.csproj), not in a JSON file. +# Anything matching appsettings*.json here is therefore a non-secret override (or a mistake); keep it +# out of the repo either way. appsettings.json appsettings.*.json local.settings.json diff --git a/Craft.csproj b/Craft.csproj deleted file mode 100644 index be92990..0000000 --- a/Craft.csproj +++ /dev/null @@ -1,91 +0,0 @@ - - - - - net8.0 - Craft - - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Craft.sln b/Craft.sln index 68d80d6..6decc4a 100644 --- a/Craft.sln +++ b/Craft.sln @@ -3,31 +3,84 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.0.31903.59 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Craft", "Craft.csproj", "{5360BB79-F828-400C-B848-7429B0040F82}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Craft", "src\Craft\Craft.csproj", "{5360BB79-F828-400C-B848-7429B0040F82}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{BF4CE7B8-A874-4059-B557-5368C7A013EA}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Craft.Tests", "tests\Craft.Tests\Craft.Tests.csproj", "{185BC64D-BDDD-4684-BEDC-66F9BE9671E1}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Craft.Configuration", "src\Craft.Configuration\Craft.Configuration.csproj", "{C2867D09-61BF-4DED-9892-4BBB0AD9B34C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Craft.Contracts", "src\Craft.Contracts\Craft.Contracts.csproj", "{C4EDF6C2-F305-4415-9882-D703465BBE3A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {5360BB79-F828-400C-B848-7429B0040F82}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {5360BB79-F828-400C-B848-7429B0040F82}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5360BB79-F828-400C-B848-7429B0040F82}.Debug|x64.ActiveCfg = Debug|Any CPU + {5360BB79-F828-400C-B848-7429B0040F82}.Debug|x64.Build.0 = Debug|Any CPU + {5360BB79-F828-400C-B848-7429B0040F82}.Debug|x86.ActiveCfg = Debug|Any CPU + {5360BB79-F828-400C-B848-7429B0040F82}.Debug|x86.Build.0 = Debug|Any CPU {5360BB79-F828-400C-B848-7429B0040F82}.Release|Any CPU.ActiveCfg = Release|Any CPU {5360BB79-F828-400C-B848-7429B0040F82}.Release|Any CPU.Build.0 = Release|Any CPU + {5360BB79-F828-400C-B848-7429B0040F82}.Release|x64.ActiveCfg = Release|Any CPU + {5360BB79-F828-400C-B848-7429B0040F82}.Release|x64.Build.0 = Release|Any CPU + {5360BB79-F828-400C-B848-7429B0040F82}.Release|x86.ActiveCfg = Release|Any CPU + {5360BB79-F828-400C-B848-7429B0040F82}.Release|x86.Build.0 = Release|Any CPU {185BC64D-BDDD-4684-BEDC-66F9BE9671E1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {185BC64D-BDDD-4684-BEDC-66F9BE9671E1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {185BC64D-BDDD-4684-BEDC-66F9BE9671E1}.Debug|x64.ActiveCfg = Debug|Any CPU + {185BC64D-BDDD-4684-BEDC-66F9BE9671E1}.Debug|x64.Build.0 = Debug|Any CPU + {185BC64D-BDDD-4684-BEDC-66F9BE9671E1}.Debug|x86.ActiveCfg = Debug|Any CPU + {185BC64D-BDDD-4684-BEDC-66F9BE9671E1}.Debug|x86.Build.0 = Debug|Any CPU {185BC64D-BDDD-4684-BEDC-66F9BE9671E1}.Release|Any CPU.ActiveCfg = Release|Any CPU {185BC64D-BDDD-4684-BEDC-66F9BE9671E1}.Release|Any CPU.Build.0 = Release|Any CPU + {185BC64D-BDDD-4684-BEDC-66F9BE9671E1}.Release|x64.ActiveCfg = Release|Any CPU + {185BC64D-BDDD-4684-BEDC-66F9BE9671E1}.Release|x64.Build.0 = Release|Any CPU + {185BC64D-BDDD-4684-BEDC-66F9BE9671E1}.Release|x86.ActiveCfg = Release|Any CPU + {185BC64D-BDDD-4684-BEDC-66F9BE9671E1}.Release|x86.Build.0 = Release|Any CPU + {C2867D09-61BF-4DED-9892-4BBB0AD9B34C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C2867D09-61BF-4DED-9892-4BBB0AD9B34C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C2867D09-61BF-4DED-9892-4BBB0AD9B34C}.Debug|x64.ActiveCfg = Debug|Any CPU + {C2867D09-61BF-4DED-9892-4BBB0AD9B34C}.Debug|x64.Build.0 = Debug|Any CPU + {C2867D09-61BF-4DED-9892-4BBB0AD9B34C}.Debug|x86.ActiveCfg = Debug|Any CPU + {C2867D09-61BF-4DED-9892-4BBB0AD9B34C}.Debug|x86.Build.0 = Debug|Any CPU + {C2867D09-61BF-4DED-9892-4BBB0AD9B34C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C2867D09-61BF-4DED-9892-4BBB0AD9B34C}.Release|Any CPU.Build.0 = Release|Any CPU + {C2867D09-61BF-4DED-9892-4BBB0AD9B34C}.Release|x64.ActiveCfg = Release|Any CPU + {C2867D09-61BF-4DED-9892-4BBB0AD9B34C}.Release|x64.Build.0 = Release|Any CPU + {C2867D09-61BF-4DED-9892-4BBB0AD9B34C}.Release|x86.ActiveCfg = Release|Any CPU + {C2867D09-61BF-4DED-9892-4BBB0AD9B34C}.Release|x86.Build.0 = Release|Any CPU + {C4EDF6C2-F305-4415-9882-D703465BBE3A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C4EDF6C2-F305-4415-9882-D703465BBE3A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C4EDF6C2-F305-4415-9882-D703465BBE3A}.Debug|x64.ActiveCfg = Debug|Any CPU + {C4EDF6C2-F305-4415-9882-D703465BBE3A}.Debug|x64.Build.0 = Debug|Any CPU + {C4EDF6C2-F305-4415-9882-D703465BBE3A}.Debug|x86.ActiveCfg = Debug|Any CPU + {C4EDF6C2-F305-4415-9882-D703465BBE3A}.Debug|x86.Build.0 = Debug|Any CPU + {C4EDF6C2-F305-4415-9882-D703465BBE3A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C4EDF6C2-F305-4415-9882-D703465BBE3A}.Release|Any CPU.Build.0 = Release|Any CPU + {C4EDF6C2-F305-4415-9882-D703465BBE3A}.Release|x64.ActiveCfg = Release|Any CPU + {C4EDF6C2-F305-4415-9882-D703465BBE3A}.Release|x64.Build.0 = Release|Any CPU + {C4EDF6C2-F305-4415-9882-D703465BBE3A}.Release|x86.ActiveCfg = Release|Any CPU + {C4EDF6C2-F305-4415-9882-D703465BBE3A}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution + {5360BB79-F828-400C-B848-7429B0040F82} = {A1B2C3D4-E5F6-7890-ABCD-EF1234567890} {185BC64D-BDDD-4684-BEDC-66F9BE9671E1} = {BF4CE7B8-A874-4059-B557-5368C7A013EA} + {C2867D09-61BF-4DED-9892-4BBB0AD9B34C} = {A1B2C3D4-E5F6-7890-ABCD-EF1234567890} + {C4EDF6C2-F305-4415-9882-D703465BBE3A} = {A1B2C3D4-E5F6-7890-ABCD-EF1234567890} EndGlobalSection EndGlobal diff --git a/Directory.Build.props b/Directory.Build.props index 411c957..16a934a 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,11 +1,12 @@ diff --git a/README.md b/README.md index e1bd164..dc01bdf 100644 --- a/README.md +++ b/README.md @@ -8,43 +8,41 @@ Craft is a lightweight ASP.NET Core runtime that hosts PowerShell modules as HTT ``` Craft/ -├── Services/ -│ ├── Program.cs # Host startup, middleware, endpoint mapping -│ ├── Bridges/ # PowerShell-facing API surface → namespace Craft.Services (PINNED) -│ ├── Configuration/ # Settings types, one per file → Craft.Configuration -│ ├── Endpoints/ # Native C# endpoint/task contracts → Craft.Endpoints -│ ├── PowerShellHost/ # Runspace workers, pool, script repo → Craft.PowerShellHost -│ ├── Orchestration/ # Orchestrator, scheduler, jobs → Craft.Orchestration -│ ├── Storage/ # Azure Table stores, health → Craft.Storage -│ ├── Caching/ # Response cache → Craft.Caching -│ ├── Hosting/ # Logging, diagnostics, profilers → Craft.Hosting -│ ├── Auth/ # EasyAuth translation → Craft.Auth -│ ├── Realtime/ # SSE channel → Craft.Realtime -│ └── Setup/ # First-run wizard + its HTML → Craft.Setup -├── Runtime/ # PowerShell runtime bridge -├── Properties/ # ASP.NET launch profiles -├── build/ -│ ├── Dockerfile # Container image build -│ └── config/ # Runtime config templates -├── docs/ # Documentation -├── appsettings.example.jsonc # Annotated config reference (NOT loaded — see Configuration) -├── .editorconfig # Code style, enforced in CI by `dotnet format` -├── Directory.Build.props # Shared build + analyzer settings -├── global.json # Pinned SDK (keep in sync with CI and the Dockerfile) -└── Craft.csproj # Project file +├── src/Craft/ # Host project (publishes Craft.dll) +│ ├── Craft.csproj +│ ├── Properties/ # ASP.NET launch profiles (Development + user secrets) +│ ├── Runtime/ # PowerShell runtime bridge scripts +│ └── Services/ # Feature modules + Bridges (composition root in Program.cs) +│ ├── Endpoints/ # Native C# endpoint/task contracts → Craft.Endpoints +│ ├── Hosting/ # Middleware, diagnostics, HTTP endpoint maps +│ ├── PowerShellHost/ # Runspace workers, pool, script repo +│ ├── Orchestration/ # Orchestrator, scheduler, jobs +│ └── Bridges/ # PowerShell-facing API surface (namespace PINNED) +├── src/Craft.Configuration/ # Settings POCOs (leaf; folders by feature area) +├── src/Craft.Contracts/ # Pinned Craft.Services DTOs + HttpResponseContext +├── tests/Craft.Tests/ +├── perf-harness/ # Docker/k6/E2E tooling (not part of Craft.dll) +├── build/Dockerfile +├── docs/ # configuration.md, architecture.md, … +├── appsettings.example.jsonc # Annotated config reference (NOT loaded) +├── Directory.Build.props # Shared build + analyzer settings +├── global.json # Pinned SDK +└── Craft.sln ``` +See [docs/architecture.md](docs/architecture.md) for module boundaries and the PowerShell contract rules. + ## Quick Start ```bash -dotnet run +dotnet run --project src/Craft/Craft.csproj ``` ```bash docker build --pull -f build/Dockerfile -t craft . ``` -Use `--pull`. The base image tags float on the `8.0-*` minor so .NET runtime security patches are +Use `--pull`. The base image tags float on the `10.0-*` minor so .NET runtime security patches are picked up automatically, but only if the tag is re-resolved — without it Docker will happily reuse a months-old local base image. Those CVEs live in the shared runtime the base image ships, so no package bump fixes them and `dotnet list package --vulnerable` will never report them. CI sets `pull: true`. @@ -52,20 +50,24 @@ bump fixes them and `dotnet list package --vulnerable` will never report them. C ## Configuration Craft ships **no `appsettings.json`**. Every setting's default is the C# property initialiser in -[`Services/CraftSettings.cs`](Services/CraftSettings.cs), which is the single source of truth. Configure a -deployment with `App__*` environment variables, or by supplying your own `appsettings.json` (downstream app -image, or the project root for local `dotnet run`). +[`src/Craft.Configuration/CraftSettings.cs`](src/Craft.Configuration/CraftSettings.cs), which is the single source of truth. + +- **Local `dotnet run`:** put secrets and connection strings in [.NET user secrets](https://learn.microsoft.com/en-us/aspnet/core/security/app-secrets) only — not in an `appsettings.json` on disk. [`src/Craft/Properties/launchSettings.json`](src/Craft/Properties/launchSettings.json) sets `ASPNETCORE_ENVIRONMENT=Development` so CreateBuilder loads them. +- **Containers / production:** `App__*` (and other) environment variables. +- **Downstream app images:** may COPY a non-secret `appsettings.json` for structural defaults; still prefer env/Key Vault for credentials. [`appsettings.example.jsonc`](appsettings.example.jsonc) is an annotated reference listing every key and its default. It is documentation only — the `.jsonc` extension keeps it out of the build glob, and reflects that it contains comments and so is not strict JSON. -See [docs/configuration.md](docs/configuration.md) for the full reference. +See [docs/configuration.md](docs/configuration.md) for the full reference, including local user-secrets examples. ## The `Craft.Services` namespace is a public contract -Everything in [`Services/Bridges/`](Services/Bridges), plus `PowerShellRunnerService`, stays in namespace -`Craft.Services` **permanently**. Downstream PowerShell reaches these types by fully-qualified name: +Bridge facades under [`src/Craft/Services/Bridges/`](src/Craft/Services/Bridges), `PowerShellRunnerService`, +and the DTOs in [`src/Craft.Contracts/`](src/Craft.Contracts) stay in namespace `Craft.Services` +**permanently** (Facades/runner ship in `Craft.dll`; DTOs in `Craft.Contracts.dll`). Downstream +PowerShell reaches these types by fully-qualified name: ```powershell [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) @@ -73,13 +75,12 @@ Everything in [`Services/Bridges/`](Services/Bridges), plus `PowerShellRunnerSer ``` Renaming that namespace compiles cleanly and then fails at runtime inside the hosted app with -*"Unable to find type"*. Type forwarding cannot rescue it — `[TypeForwardedTo]` only works across -assemblies, and these all live in `Craft.dll`. The folder a pinned type lives in is free to change; the +*"Unable to find type"*. The folder or assembly a pinned type lives in is free to change; the namespace is not. Each pinned file carries a `NAMESPACE PINNED` header saying so. -Same rule, different reason, for `Microsoft.Azure.Functions.PowerShellWorker.HttpResponseContext`: its -namespace must match the real Azure Functions worker type because hosted-app routers match on -`PSObject.TypeNames`. +Same rule, different reason, for `Microsoft.Azure.Functions.PowerShellWorker.HttpResponseContext` +(in Craft.Contracts): its namespace must match the real Azure Functions worker type because hosted-app +routers match on `PSObject.TypeNames`. All four of its properties reach the wire — `StatusCode`, `Body`, `Headers` and `ContentType` — so a handler can redirect: @@ -111,12 +112,22 @@ suites carry most of the weight today: - **`ConfigurationReferenceTests`** — asserts every default documented in `appsettings.example.jsonc` still matches the C# default it claims to document, so the example can't rot into a lie. -The end-to-end suite (Azurite + orchestrator + scheduler + realtime + API dispatch) needs Docker: +The end-to-end suite (Azurite + orchestrator + scheduler + realtime + API dispatch) needs Docker and +PowerShell 7 (`pwsh`): ```bash -docker build -f build/Dockerfile -t craft:ci . && perf-harness/scripts/run-e2e.ps1 -SutImage craft:ci +# Build + run (from repo root). -Build pulls .NET 10 bases from MCR and tags craft:ci. +pwsh perf-harness/scripts/run-e2e.ps1 -Build + +# Or reuse an already-built image: +docker build --pull -f build/Dockerfile -t craft:ci . +pwsh perf-harness/scripts/run-e2e.ps1 -SutImage craft:ci ``` +On Apple Silicon / arm64 WSL the image is native arm64 — compose does not pin `platform`. To force +amd64 emulation, build with `docker build --platform linux/amd64 …` and add `platform: linux/amd64` +under the `sut` service in `perf-harness/docker-compose.e2e-azure.yml`. + ## Contributing Style is defined by [`.editorconfig`](.editorconfig) and enforced in CI. Before pushing: @@ -131,7 +142,3 @@ set that rule's severity explicitly in `.editorconfig` with a reason; don't disa Two justified `CA1051` suppressions exist at their declaration sites (`WorkerStats`, `CacheEntry`) — both hold `volatile` fields or `Interlocked` targets, neither of which is expressible as a property. - -One structural gotcha: `Craft.csproj` sits at the repo root, so its default `**/*.cs` glob reaches the -whole tree. Any new sibling project directory must be added to the `Compile Remove` list in -`Craft.csproj`, or its sources get compiled into `Craft.dll`. diff --git a/Services/Bridges/JobRecord.cs b/Services/Bridges/JobRecord.cs deleted file mode 100644 index 29f0771..0000000 --- a/Services/Bridges/JobRecord.cs +++ /dev/null @@ -1,22 +0,0 @@ -// NAMESPACE PINNED — do not change. -// Downstream PowerShell reaches these types by fully-qualified name, e.g. -// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) -// Renaming the namespace compiles fine and then fails at runtime in the hosted app -// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. -// The folder is free to move; the namespace is a published contract. -namespace Craft.Services; - -// ── API Models ── - -public class JobRecord -{ - public string Id { get; set; } = string.Empty; - public string Name { get; set; } = string.Empty; - public string? RunName { get; set; } - public int Priority { get; set; } - public string Status { get; set; } = "Queued"; - public DateTime QueuedUtc { get; set; } - public DateTime? StartedUtc { get; set; } - public DateTime? CompletedUtc { get; set; } - public string? LastError { get; set; } -} diff --git a/Services/Bridges/OrchestratorBridge.cs b/Services/Bridges/OrchestratorBridge.cs deleted file mode 100644 index 2717d77..0000000 --- a/Services/Bridges/OrchestratorBridge.cs +++ /dev/null @@ -1,130 +0,0 @@ -using System.Collections.Concurrent; -using Craft.Hosting; -using Craft.Orchestration; - -// NAMESPACE PINNED — do not change. -// Downstream PowerShell reaches these types by fully-qualified name, e.g. -// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) -// Renaming the namespace compiles fine and then fails at runtime in the hosted app -// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. -// The folder is free to move; the namespace is a published contract. -namespace Craft.Services; - -/// -/// Thread-safe bridge allowing PowerShell (Start-CIPPOrchestrator) to queue -/// orchestrator runs that get picked up by the C# OrchestratorService. -/// PS enqueues via QueueOrchestration(); C# drains via DrainPending(). -/// -public static class OrchestratorBridge -{ - private static OrchestratorService? s_service; - private static readonly ConcurrentQueue s_pending = new(); - - public static void Initialize(OrchestratorService service) => s_service = service; - - public static void QueueOrchestration(string name, string batchJson, int priority, - string? postExecFunctionName = null, string? postExecParametersJson = null, - string? reference = null) - { - var parentRunName = OperationContext.Current?.RunName; - s_pending.Enqueue(new PendingOrchestration(name, batchJson, priority, - postExecFunctionName, postExecParametersJson, parentRunName, reference)); - } - - /// - /// Synchronous drain — blocks until all pending orchestrations are started. - /// Safe to call from any context (no SynchronizationContext on background workers). - /// - public static void DrainPending() - { - while (s_pending.TryDequeue(out var p)) - { - try - { - if (s_service != null) - { - s_service.StartFromBatchAsync(p.Name, p.BatchJson, p.Priority, - p.PostExecFunctionName, p.PostExecParametersJson, CancellationToken.None, - p.ParentRunName, p.Reference) - .GetAwaiter().GetResult(); - - // Register as child run if parent is still active - if (!string.IsNullOrEmpty(p.ParentRunName)) - s_service.TryRegisterChildRun(p.ParentRunName, p.Name); - } - } - catch (Exception ex) - { - s_service?._logger.LogError(ex, "[Orchestrator] DrainPending failed for {Name}", p.Name); - } - } - DrainPendingPlanners(); - } - - /// - /// Async drain — preferred from async call sites (PostExec lambdas, ExecuteScript). - /// - public static async Task DrainPendingAsync() - { - while (s_pending.TryDequeue(out var p)) - { - try - { - if (s_service != null) - { - await s_service.StartFromBatchAsync(p.Name, p.BatchJson, p.Priority, - p.PostExecFunctionName, p.PostExecParametersJson, CancellationToken.None, - p.ParentRunName, p.Reference); - - // Register as child run if parent is still active - if (!string.IsNullOrEmpty(p.ParentRunName)) - s_service.TryRegisterChildRun(p.ParentRunName, p.Name); - } - } - catch (Exception ex) - { - s_service?._logger.LogError(ex, "[Orchestrator] DrainPending failed for {Name}", p.Name); - } - } - await DrainPendingPlannersAsync(); - } - - public record PendingOrchestration(string Name, string BatchJson, int Priority, - string? PostExecFunctionName, string? PostExecParametersJson, string? ParentRunName, - string? Reference = null); - - private static readonly ConcurrentQueue s_pendingPlanners = new(); - - /// - /// Queue a planner-based orchestrator run from PowerShell. The C# orchestrator - /// runs the planner script on a background worker to build the task list, then - /// dispatches tasks — same as the scheduler. Returns immediately. - /// - public static void QueuePlannerRun(string command, int priority) - { - s_pendingPlanners.Enqueue(new PendingPlannerRun(command, priority)); - } - - /// Drain queued planner runs. Called alongside DrainPending. - internal static void DrainPendingPlanners() - { - while (s_pendingPlanners.TryDequeue(out var p)) - { - if (s_service == null) continue; - // Fire-and-forget: planner runs on BG worker, dispatches tasks - _ = s_service.StartPlannerRunAsync(p.Command, p.Priority, CancellationToken.None); - } - } - - internal static Task DrainPendingPlannersAsync() - { - while (s_pendingPlanners.TryDequeue(out var p)) - { - if (s_service == null) continue; - _ = s_service.StartPlannerRunAsync(p.Command, p.Priority, CancellationToken.None); - } - return Task.CompletedTask; - } - - public record PendingPlannerRun(string Command, int Priority); -} diff --git a/Services/Bridges/QueueBridge.cs b/Services/Bridges/QueueBridge.cs deleted file mode 100644 index 65f3729..0000000 --- a/Services/Bridges/QueueBridge.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System.Collections.Concurrent; -using Craft.Orchestration; - -// NAMESPACE PINNED — do not change. -// Downstream PowerShell reaches these types by fully-qualified name, e.g. -// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) -// Renaming the namespace compiles fine and then fails at runtime in the hosted app -// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. -// The folder is free to move; the namespace is a published contract. -namespace Craft.Services; - -/// -/// Thread-safe bridge allowing PowerShell (Add-CippQueueMessage) to queue -/// background commands that get dispatched on a background worker. -/// Replaces Azure Storage Queue on CIPPNG — purely in-process. -/// -public static class QueueBridge -{ - private static PowerShellRunnerService? s_runner; - private static JobManager? s_jobManager; - private static string? s_queueTaskFunction; - private static readonly ConcurrentQueue s_pending = new(); - - public static void Initialize(PowerShellRunnerService runner, JobManager jobManager, string queueTaskFunction) - { - s_runner = runner; - s_jobManager = jobManager; - s_queueTaskFunction = queueTaskFunction; - } - - public static void Enqueue(string cmdlet, string parametersJson) - { - s_pending.Enqueue(new PendingQueueCommand(cmdlet, parametersJson)); - } - - public static void DrainPending() - { - if (string.IsNullOrEmpty(s_queueTaskFunction)) return; - - while (s_pending.TryDequeue(out var cmd)) - { - var scriptPath = s_runner?.FindScript(s_queueTaskFunction); - if (scriptPath == null || s_runner == null || s_jobManager == null) - continue; - - var captured = cmd; - s_jobManager.Enqueue( - name: $"Queue-{captured.Cmdlet}", - priority: 5, - runName: $"Queue-{captured.Cmdlet}-{Guid.NewGuid():N}", - id: $"Queue-{Guid.NewGuid():N}", - work: async (ct) => - { - var parameters = new Dictionary - { - { "Cmdlet", captured.Cmdlet }, - { "ParametersJson", captured.ParametersJson } - }; - await s_runner.ExecuteScript(scriptPath, parameters); - - // Queued commands may trigger orchestrators - await OrchestratorBridge.DrainPendingAsync(); - } - ); - } - } - - public record PendingQueueCommand(string Cmdlet, string ParametersJson); -} diff --git a/Services/Bridges/StartupInfoBridge.cs b/Services/Bridges/StartupInfoBridge.cs deleted file mode 100644 index 9e750ac..0000000 --- a/Services/Bridges/StartupInfoBridge.cs +++ /dev/null @@ -1,67 +0,0 @@ -// NAMESPACE PINNED — do not change. -// Downstream PowerShell reaches these types by fully-qualified name, e.g. -// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) -// Renaming the namespace compiles fine and then fails at runtime in the hosted app -// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. -// The folder is free to move; the namespace is a published contract. -namespace Craft.Services; - -/// -/// Static bridge exposing container startup metrics to PowerShell and HTTP endpoints. -/// Populated during pool initialization. Read-only after startup completes. -/// -/// PS usage: -/// $info = [Craft.Services.StartupInfoBridge]::GetInfo() -/// $info.HttpReadyMs # time in ms until first HTTP worker was ready -/// $info.IsFullyReady # true once all pools are done -/// $info.Phase # current phase: "Starting", "HttpReady", "Ready" -/// -public static class StartupInfoBridge -{ - private static readonly StartupStats s_stats = new(); - - /// Get the current startup statistics snapshot. - public static StartupStats GetInfo() => s_stats; - - // ── Setters (called by PowerShellWorkerPool during init) ─────────── - - internal static void SetReadinessMode(string mode) => s_stats.ReadinessMode = mode; - internal static void SetWarmupMode(string mode) => s_stats.WarmupMode = mode; - internal static void SetCpuCount(int count) => s_stats.CpuCount = count; - internal static void SetPoolConfig(int httpSize, int bgSize) - { - s_stats.HttpPoolSize = httpSize; - s_stats.BgPoolSize = bgSize; - } - internal static void SetModuleCounts(int shared, int httpOnly, int bgOnly) - { - s_stats.SharedModuleCount = shared; - s_stats.HttpOnlyModuleCount = httpOnly; - s_stats.BgOnlyModuleCount = bgOnly; - } - internal static void SetBaseWorkerDone(long ms, int functionCount) - { - s_stats.BaseWorkerMs = ms; - s_stats.BaseFunctionCount = functionCount; - s_stats.Phase = "BaseReady"; - } - internal static void SetWarmupDone(long ms) => s_stats.WarmupMs = ms; - internal static void SetHttpReady(long ms, int functionCount) - { - s_stats.HttpReadyMs = ms; - s_stats.HttpFunctionCount = functionCount; - s_stats.Phase = "HttpReady"; - } - internal static void SetHttpPoolFull(long ms) => s_stats.HttpPoolFullMs = ms; - internal static void SetBgReady(long ms, int functionCount) - { - s_stats.BgReadyMs = ms; - s_stats.BgFunctionCount = functionCount; - } - internal static void SetFullyReady(long ms) - { - s_stats.FullyReadyMs = ms; - s_stats.Phase = "Ready"; - s_stats.IsFullyReady = true; - } -} diff --git a/Services/Configuration/HealthSettings.cs b/Services/Configuration/HealthSettings.cs deleted file mode 100644 index b96466f..0000000 --- a/Services/Configuration/HealthSettings.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Craft.Configuration; - -/// -/// Role-agnostic health probe. Enabled by default at /healthz; a deployment can relocate it behind a -/// specific probe URL or turn it off entirely. Overridable via CRAFT_HEALTH_ENABLED / CRAFT_HEALTH_PATH. -/// -public class HealthSettings -{ - /// Whether the health endpoint is mapped. Default true. - public bool Enabled { get; set; } = true; - - /// Path the health endpoint is served at. Default "/healthz". A leading slash is added if missing. - public string Path { get; set; } = "/healthz"; -} diff --git a/Services/Hosting/CraftHostBuilderExtensions.cs b/Services/Hosting/CraftHostBuilderExtensions.cs deleted file mode 100644 index e4a4e05..0000000 --- a/Services/Hosting/CraftHostBuilderExtensions.cs +++ /dev/null @@ -1,303 +0,0 @@ -using System.Globalization; -using System.IO.Compression; -using System.Threading.RateLimiting; -using Craft.Auth; -using Craft.Caching; -using Craft.Configuration; -using Craft.Endpoints; -using Craft.Orchestration; -using Craft.PowerShellHost; -using Craft.Realtime; -using Craft.Services; -using Craft.Setup; -using Craft.Storage; -using Microsoft.AspNetCore.ResponseCompression; -using Microsoft.AspNetCore.Server.Kestrel.Core; -using Microsoft.Extensions.Logging.Console; -using Microsoft.Extensions.Options; - -namespace Craft.Hosting; - -/// -/// Host wiring, split out of Program.cs so startup reads as a short sequence of named steps -/// rather than several hundred lines of inline configuration. -/// -public static class CraftHostBuilderExtensions -{ - /// - /// Resolves the Kestrel request timeout in seconds: an explicit KestrelTimeoutSeconds wins, - /// otherwise it derives from Worker.HttpTimeoutSeconds, otherwise 600s. - /// - /// - /// Deriving from the worker timeout matters: if Kestrel gives up before the PowerShell worker does, - /// the caller sees a connection abort while the script keeps running and holding a runspace. - /// - public static int ResolveKestrelTimeoutSeconds(CraftSettings settings) - { - ArgumentNullException.ThrowIfNull(settings); - - var timeout = settings.KestrelTimeoutSeconds; - if (timeout > 0) return timeout; - - return settings.Worker.HttpTimeoutSeconds > 0 ? settings.Worker.HttpTimeoutSeconds : 600; - } - - /// - /// Resolves the .NET thread-pool minimum: an explicit Worker:MinThreads (or - /// CRAFT_MIN_THREADS) wins, otherwise it is derived from the worker pools. - /// - /// - /// The derived floor is HttpPoolSize + BgPoolSize + 16, never below the old - /// max(ProcessorCount * 4, 32). - /// - /// The pool term is the important part. PowerShell blocks a thread for the duration of every - /// outbound call it makes, so a pool of N workers can park N threads simultaneously; if the - /// thread-pool minimum is below that, the CLR has to inject the difference at about one thread - /// per second before the pool can reach its own concurrency. Sizing the minimum off cores alone - /// — as this did — meant a 1-core container floored at 32 and any pool above that paid the ramp - /// on every restart. - /// - /// The +16 is headroom for the runtime's own work (Kestrel, timers, the storage SDK) so those do - /// not have to contend with parked PowerShell threads for the same minimum. - /// - public static int ResolveMinThreads(CraftSettings settings) - { - ArgumentNullException.ThrowIfNull(settings); - - if (int.TryParse(Environment.GetEnvironmentVariable("CRAFT_MIN_THREADS"), out var fromEnv) && fromEnv > 0) - return fromEnv; - - if (settings.Worker.MinThreads > 0) return settings.Worker.MinThreads; - - var baseline = Math.Max(Environment.ProcessorCount * 4, 32); - var forPools = settings.Worker.HttpPoolSize + settings.Worker.BgPoolSize + 16; - return Math.Max(baseline, forPools); - } - - /// - /// Kestrel limits: request timeouts, HTTP/2 tuning, and the DoS-relevant caps (body size, - /// connection count, slow-loris minimum data rates). The caps apply regardless of the timeout. - /// - public static WebApplicationBuilder ConfigureCraftKestrel( - this WebApplicationBuilder builder, CraftSettings settings) - { - ArgumentNullException.ThrowIfNull(builder); - ArgumentNullException.ThrowIfNull(settings); - - var timeout = ResolveKestrelTimeoutSeconds(settings); - - builder.WebHost.ConfigureKestrel(options => - { - options.Limits.KeepAliveTimeout = TimeSpan.FromSeconds(timeout); - options.Limits.RequestHeadersTimeout = TimeSpan.FromSeconds(Math.Min(60, timeout)); - - // HTTP/2 — better multiplexing for the browser UI. - options.Limits.Http2.MaxStreamsPerConnection = 100; - options.Limits.Http2.HeaderTableSize = 4096; - options.Limits.Http2.MaxFrameSize = 16384; - options.Limits.Http2.MaxRequestHeaderFieldSize = 8192; - options.Limits.Http2.InitialConnectionWindowSize = 131072; - options.Limits.Http2.InitialStreamWindowSize = 98304; - - // Request body cap. Default 100 MB; 0 means unlimited. - var maxBodyMb = settings.Limits.MaxRequestBodyMB; - options.Limits.MaxRequestBodySize = maxBodyMb > 0 ? maxBodyMb * 1024L * 1024L : null; - - // Concurrent connection cap. Default 200; <= 0 hands the decision to the OS. - var maxConn = settings.Limits.MaxConcurrentConnections; - options.Limits.MaxConcurrentConnections = maxConn > 0 ? maxConn : null; - options.Limits.MaxConcurrentUpgradedConnections = maxConn > 0 ? maxConn : null; - - // Slow-loris protection. - options.Limits.MinRequestBodyDataRate = - new MinDataRate(bytesPerSecond: 240, gracePeriod: TimeSpan.FromSeconds(5)); - options.Limits.MinResponseDataRate = - new MinDataRate(bytesPerSecond: 240, gracePeriod: TimeSpan.FromSeconds(5)); - }); - - return builder; - } - - /// - /// File logging with rotation plus a timestamped console sink, both honouring the configured level - /// (App:FileLogging:LogLevel, overridable with CRAFT_LOG_LEVEL). - /// - /// - /// The resolved level. Startup logs it, and it also gates PowerShell stream capture — at Debug, - /// Write-Debug is captured; at Trace, Write-Verbose as well. - /// - public static LogLevel AddCraftLogging(this WebApplicationBuilder builder) - { - ArgumentNullException.ThrowIfNull(builder); - - var fileLoggingSettings = new FileLoggingSettings(); - builder.Configuration.GetSection("App:FileLogging").Bind(fileLoggingSettings); - var level = fileLoggingSettings.ParsedLogLevel; - - var fileLoggerProvider = new FileLoggerProvider(fileLoggingSettings, level); - builder.Logging.AddProvider(fileLoggerProvider); - LogBridge.Initialize(fileLoggerProvider); - - builder.Logging.AddSimpleConsole(options => - { - options.TimestampFormat = "yyyy-MM-ddTHH:mm:ss.fffZ "; - options.SingleLine = true; - }); - - if (level > LogLevel.Debug) - { - builder.Logging.AddFilter(l => l >= LogLevel.Information); - - // Framework logging is noise at Information and above. - builder.Logging.AddFilter("Microsoft.AspNetCore", LogLevel.Warning); - builder.Logging.AddFilter("Microsoft.Hosting", LogLevel.Warning); - builder.Logging.AddFilter("Microsoft.Extensions.Hosting", LogLevel.Warning); - } - - return level; - } - - private static readonly string[] second = new[] { "application/json", "text/json", "application/javascript", "text/javascript" }; - - /// Response compression, matching Azure Static Web Apps behaviour. - public static IServiceCollection AddCraftResponseCompression(this IServiceCollection services) - { - ArgumentNullException.ThrowIfNull(services); - - services.AddResponseCompression(options => - { - options.EnableForHttps = true; - options.Providers.Add(); - options.Providers.Add(); - options.MimeTypes = ResponseCompressionDefaults.MimeTypes.Concat( - second); - }); - - // Fastest, not Optimal: these run on the request path on a small container, where the extra - // CPU costs more than the bytes saved. Precompressed .br/.gz siblings cover the static assets. - services.Configure(o => o.Level = CompressionLevel.Fastest); - services.Configure(o => o.Level = CompressionLevel.Fastest); - - return services; - } - - /// - /// Registers the Craft service graph. Background hosted services are registered only on nodes - /// carrying the Background role — every node builds the same object graph, but only a Background - /// node actually runs the scheduler, job manager and stats sampler. - /// - public static IServiceCollection AddCraftServices(this IServiceCollection services, CraftRoles roles) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(roles); - - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - - services.AddSingleton(sp => new CacheService( - sp.GetRequiredService>(), - sp.GetRequiredService(), - roles.ResponseCacheEnabled)); - - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - // The scheduler's native-task lookup. Empty here; AddNativeEndpoints registers the - // discovered set on top when the application ships any, and last registration wins on - // resolve — so SchedulerService injects it unconditionally either way. - services.AddSingleton(NativeScheduledTasks.Empty); - services.AddSingleton(); - services.AddSingleton(); - - if (roles.Background) - { - services.AddHostedService(sp => sp.GetRequiredService()); - services.AddHostedService(sp => sp.GetRequiredService()); - services.AddHostedService(sp => sp.GetRequiredService()); - } - - services.AddSingleton(sp => - { - var health = sp.GetRequiredService>().Value.ContainerHealth; - var logger = sp.GetRequiredService().CreateLogger(); - return new ContainerHealthMonitor(logger, health); - }); - - return services; - } - - /// - /// Seconds to advertise in Retry-After on a throttled response. Prefers the limiter's own - /// estimate of when a permit next frees up, falling back to the whole window — a safe upper bound - /// for a fixed window, and the only figure available when the lease carries no metadata. - /// - /// - /// Rounded up, and floored at one second, on purpose. Truncating a 0.4s wait to - /// Retry-After: 0 tells a well-behaved client to retry immediately, which turns being - /// throttled into a hot loop — the opposite of what the header is for. - /// - public static int ResolveRetryAfterSeconds(RateLimitLease lease, TimeSpan window) - { - ArgumentNullException.ThrowIfNull(lease); - - var retryAfter = lease.TryGetMetadata(MetadataName.RetryAfter, out var metadata) - ? metadata - : window; - - return Math.Max(1, (int)Math.Ceiling(retryAfter.TotalSeconds)); - } - - /// - /// Per-client fixed-window rate limiter so a single caller cannot exhaust the small HTTP worker - /// pool. Enabled by default; turn off with App:RateLimit:Enabled=false. Throttled requests - /// get a 429 carrying Retry-After. - /// - public static IServiceCollection AddCraftRateLimiter( - this IServiceCollection services, CraftSettings settings) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(settings); - - if (!settings.RateLimit.IsEnabled) return services; - - var window = TimeSpan.FromSeconds(Math.Max(1, settings.RateLimit.WindowSeconds)); - - services.AddRateLimiter(options => - { - options.RejectionStatusCode = 429; - - // Without this the 429 carries no timing hint at all and every caller has to invent its - // own backoff. Retry-After is the one thing HTTP clients and SDKs already know how to - // honour unprompted, so emitting it is what makes the limit self-documenting. - options.OnRejected = (context, _) => - { - context.HttpContext.Response.Headers.RetryAfter = - ResolveRetryAfterSeconds(context.Lease, window) - .ToString(CultureInfo.InvariantCulture); - return ValueTask.CompletedTask; - }; - - options.GlobalLimiter = PartitionedRateLimiter.Create(context => - RateLimitPartition.GetFixedWindowLimiter( - RateLimitPartitionKey.Resolve(context), - _ => new FixedWindowRateLimiterOptions - { - PermitLimit = Math.Max(1, settings.RateLimit.PermitPerWindow), - Window = window, - QueueLimit = Math.Max(0, settings.RateLimit.QueueLimit), - QueueProcessingOrder = QueueProcessingOrder.OldestFirst, - })); - }); - - return services; - } -} diff --git a/Services/Program.cs b/Services/Program.cs deleted file mode 100644 index 713bfda..0000000 --- a/Services/Program.cs +++ /dev/null @@ -1,439 +0,0 @@ -using Craft.Auth; -using Craft.Caching; -using Craft.Configuration; -using Craft.Endpoints; -using Craft.Hosting; -using Craft.Hosting.Endpoints; -using Craft.Orchestration; -using Craft.PowerShellHost; -using Craft.Realtime; -using Craft.Services; -using Craft.Setup; -using Craft.Storage; -using Microsoft.Extensions.Options; - -var builder = WebApplication.CreateBuilder(args); - -// Bind App section to CraftSettings -builder.Services.Configure(builder.Configuration.GetSection("App")); -// Apply SkuProfiles override (host-tier pool sizing) before any consumer resolves the options -builder.Services.PostConfigure(s => SkuProfileSelector.Apply(s)); -// Also register a singleton accessor for non-DI contexts -builder.Services.AddSingleton(sp => sp.GetRequiredService>().Value); - -// Bind configuration directly for early access (avoid BuildServiceProvider warning) -var craftSettings = new CraftSettings(); -builder.Configuration.GetSection("App").Bind(craftSettings); - -// ── Deployment roles (capabilities) ──────────────────────────────────────────────────────────────── -// Resolution lives in CraftRoles so it can be unit tested without mutating process environment. -// See Services/Hosting/CraftRoles.cs for the full rule. -var roles = CraftRoles.Resolve(craftSettings); - -if (roles.None) -{ - Console.Error.WriteLine("[System] FATAL: no deployment roles enabled — set at least one of " + - "CRAFT_SERVE_FRONTEND / CRAFT_SERVE_API / CRAFT_RUN_BACKGROUND (or App:Roles:*). " + - "Roles are declared by enabling what you want; unset roles default off once any is set."); - Environment.Exit(78); // EX_CONFIG -} - -// ── Thread pool ──────────────────────────────────────────────────────────────────────────────────── -// Must happen before anything schedules work, and after config binding so the pool sizes are known. -// The default floor scales with the worker pools, not just the core count: PowerShell blocks a -// thread for the whole of every outbound call, so a pool of N workers can park N threads, and a -// minimum below that leaves the CLR injecting threads at ~1/second before the pool can reach its own -// concurrency. See CraftHostBuilderExtensions.ResolveMinThreads. -var minThreads = CraftHostBuilderExtensions.ResolveMinThreads(craftSettings); -ThreadPool.SetMinThreads(minThreads, minThreads); -var capFrontend = roles.Frontend; -var capHttp = roles.Http; -var capBackground = roles.Background; -var runPowerShell = roles.RunsPowerShell; -var cacheEnabled = roles.ResponseCacheEnabled; -var healthEnabled = roles.HealthEnabled; -var healthPath = roles.HealthPath; -var compressionEnabled = roles.CompressionEnabled; - -// Kestrel limits, logging sinks, compression, the service graph and the rate limiter all live in -// Services/Hosting/CraftHostBuilderExtensions.cs. -builder.ConfigureCraftKestrel(craftSettings); - -// The resolved level is logged at startup and also gates PowerShell stream capture. -var configuredLogLevel = builder.AddCraftLogging(); - -builder.Services.AddCraftResponseCompression(); -// ── Native C# endpoints and scheduled tasks ──────────────────────────────────────────────────────── -// Discovered before the container is built so the endpoint/task types, the central handler and any -// application service module they ship can be registered into it. Costs nothing when no assemblies -// are configured. -var nativeCatalog = NativeEndpointRegistry.Discover( - craftSettings.Endpoints, - Path.Combine(AppContext.BaseDirectory, "API"), - LoggerFactory.Create(b => b.AddSimpleConsole()).CreateLogger("Craft.Endpoints")); - -builder.Services.AddCraftServices(roles); -if (!nativeCatalog.IsEmpty) - builder.Services.AddNativeEndpoints(nativeCatalog, builder.Configuration); -builder.Services.AddCraftRateLimiter(craftSettings); - -var app = builder.Build(); - -// NOTE: the rate limiter middleware is deliberately NOT registered here. It runs after the auth -// middleware further down so it can partition on the caller's identity — see the comment there. - -// HTTP diagnostic listener — tracks DNS, TLS, socket connect, and HTTP request timing -// from ALL HttpClient instances (including those inside PowerShell's Invoke-RestMethod) -var httpDiagLogger = app.Services.GetRequiredService().CreateLogger("HttpDiag"); -var httpListener = new HttpDiagnosticListener(httpDiagLogger, slowThresholdMs: 1000); -// Must keep reference alive — GC would collect it and stop events -app.Lifetime.ApplicationStopping.Register(() => httpListener.Dispose()); - -var repo = app.Services.GetRequiredService(); -var pool = app.Services.GetRequiredService(); -var logger = app.Services.GetRequiredService>(); -var psRunner = app.Services.GetRequiredService(); -var cache = app.Services.GetRequiredService(); -var CraftSettings = app.Services.GetRequiredService(); -var realtime = app.Services.GetRequiredService(); -RealtimeBridge.Initialize(realtime); -var setupService = app.Services.GetRequiredService(); - -// AppLifecycleBridge MUST be initialized before pool.Initialize() — the PS warmup script -// runs inside pool init and calls bridge methods (IsEasyAuthConfigured, ReconcileAuthPolicy, -// RequestSetupMode). If the bridge's static state isn't populated, those calls silently -// return false because the null-conditional logger swallows the "called before Initialize" -// warning. Other bridges (Scheduler, Cache, StatsHistory) initialize later — they're only -// called from request handlers or post-warmup PS, not from warmup itself. -AppLifecycleBridge.Initialize(app.Lifetime, logger, setupService); - -// --- Container health monitoring --- -// Track restart attempts on persistent storage (/home) to detect crash loops. -// If the same instance has crashed too many times, block Kestrel so Azure provisions a new worker. -var healthMonitor = app.Services.GetRequiredService(); -if (CraftSettings.ContainerHealth.MaxRestarts > 0) -{ - healthMonitor.RecordStartupAttempt(); - if (healthMonitor.ShouldBlockStartup) - { - // Block indefinitely — Azure's warmup probe will time out (WEBSITES_CONTAINER_START_TIME_LIMIT) - // and the platform will eventually reallocate to a new worker instance. - logger.LogCritical("[Health] Startup blocked due to crash loop — waiting for Azure to provision a new worker"); - await Task.Delay(Timeout.Infinite); - } -} - -// Endpoints dictionary populated asynchronously after Kestrel starts. -// The startup middleware blocks all /API/* calls until pool.IsReady, -// so the route handler won't access this until it's fully populated. -var endpoints = new Dictionary(StringComparer.OrdinalIgnoreCase); - -// Readiness mode determines when Kestrel starts accepting connections: -// - Immediate: Kestrel starts first, init runs in background (loading page responds to Azure probes) -// - HttpReady: init runs before Kestrel, Kestrel starts once HTTP pool has a worker -// - AllReady: init runs before Kestrel, Kestrel starts once all pools are fully initialized -var readinessMode = CraftSettings.ReadinessMode?.Trim() ?? "Immediate"; - -// Safety: on B1 (single vCPU, slow-tier CPU), init can take 150-200s+ — dangerously -// close to Azure's 230s container startup timeout. Auto-downgrade to Immediate to avoid kills. -// We check both CPU count and WEBSITE_SKU because premium single-vCPU plans (e.g. P0v3) -// are fast enough to init within the timeout despite having only 1 core. -var websiteSku = Environment.GetEnvironmentVariable("WEBSITE_SKU") ?? ""; -var isSlowSingleCore = Environment.ProcessorCount <= 1 - && websiteSku.StartsWith("Basic", StringComparison.OrdinalIgnoreCase); - -if (!readinessMode.Equals("Immediate", StringComparison.OrdinalIgnoreCase) && isSlowSingleCore) -{ - logger.LogWarning("[System] ReadinessMode '{Mode}' overridden to 'Immediate' — single vCPU on Basic SKU, " + - "blocking Kestrel during init risks hitting Azure's 230s startup timeout", readinessMode); - readinessMode = "Immediate"; -} - -logger.LogInformation("[System] Readiness mode: {Mode}", readinessMode); -StartupInfoBridge.SetReadinessMode(readinessMode); - -// Announce the resolved deployment roles + derived toggles for this process. -logger.LogInformation("[System] Roles: Frontend={Frontend} Http={Http} Background={Background} | " + - "ResponseCache={Cache} Compression={Compression}", - capFrontend ? "on" : "off", capHttp ? "on" : "off", capBackground ? "on" : "off", - cacheEnabled ? "on" : "off", compressionEnabled ? "on" : "off"); - -void RunInitialization() -{ - // 1. Load scripts — parse .ps1 files, build route table - repo.LoadAll(Path.Combine(AppContext.BaseDirectory, "API")); - - // 2. Discover HTTP endpoints from loaded scripts - var discovered = psRunner.DiscoverHttpEndpoints(); - foreach (var kvp in discovered) - endpoints[kvp.Key] = kvp.Value; - - logger.LogInformation("[System] {AppName}: {Count} API endpoints discovered", CraftSettings.Name, endpoints.Count); - logger.LogInformation("[System] Pool: HTTP={Http} BG={Bg} MinThreads={MinThreads} LogLevel={LogLevel}", - CraftSettings.Worker.HttpPoolSize, - CraftSettings.Worker.BgPoolSize, - minThreads, - configuredLogLevel); - - // 3. Initialize PowerShell worker pool (loads modules, creates runspaces). - // Build only the pools this node's roles require: Http → HTTP pool, Background → BG pool. - // HttpPoolSize = 0 means this node hosts no PowerShell HTTP endpoints at all — a fully-native - // app. Skipping the pool then is not just a saving (runspace construction, ~1.6 MiB each, plus - // the PowerShell SDK's native allocations); it is the difference between paying for a - // PowerShell host and not having one. Initialize() signals readiness immediately when no pool is - // enabled, so the startup gate below does not block /API/* forever waiting for a pool that will - // never exist. - var enableHttpPool = capHttp && CraftSettings.Worker.HttpPoolSize > 0; - if (capHttp && !enableHttpPool) - logger.LogInformation("[System] HTTP worker pool disabled (Worker:HttpPoolSize=0) — PowerShell HTTP endpoints are not hosted"); - - // BgPoolSize = 0 is the same opt-out for the background side: the app's scheduled work is all - // native tasks, which run on the .NET thread pool, so BG runspaces would never be checked out. - // The disabled pool signals its ready event immediately, so the scheduler still starts. - var enableBgPool = capBackground && CraftSettings.Worker.BgPoolSize > 0; - if (capBackground && !enableBgPool) - logger.LogInformation("[System] BG worker pool disabled (Worker:BgPoolSize=0) — native scheduled tasks only"); - - pool.Initialize(enableHttp: enableHttpPool, enableBg: enableBgPool); - - // Pool is ready — clear the restart counter so we don't carry stale crash state - healthMonitor.ClearRestartCounter(); -} - -if (!runPowerShell) -{ - logger.LogWarning("[System] STATIC-ONLY (Frontend role) — PowerShell worker pool, scheduler, job manager " + - "and background services are disabled. Serving static frontend content only; /api, /API and /.auth " + - "return 404."); -} -else if (readinessMode.Equals("Immediate", StringComparison.OrdinalIgnoreCase)) -{ - // Defer init until after Kestrel is listening — Azure probe gets a fast 200, - // users see a loading page while workers initialize in the background. - app.Lifetime.ApplicationStarted.Register(() => - { - Task.Run(() => - { - try { RunInitialization(); } - catch (Exception ex) { logger.LogCritical(ex, "[System] Initialization failed"); } - }); - }); -} -else if (readinessMode.Equals("HttpReady", StringComparison.OrdinalIgnoreCase)) -{ - // Run init synchronously before Kestrel starts. pool.Initialize() signals _httpReady - // after the first HTTP worker is in the pool, but Kestrel won't start until Initialize() - // returns (which is after all pools are done). To start Kestrel at HTTP-ready, run init - // on a background thread and wait only for HTTP readiness. - var initTask = Task.Run(() => - { - try { RunInitialization(); } - catch (Exception ex) { logger.LogCritical(ex, "[System] Initialization failed"); } - }); - // Block app.Run() until HTTP pool signals ready - pool.WaitForReady(Timeout.InfiniteTimeSpan); - logger.LogInformation("[System] HTTP pool ready — starting Kestrel (BG init continues in background)"); -} -else if (readinessMode.Equals("AllReady", StringComparison.OrdinalIgnoreCase)) -{ - // Run full init synchronously before Kestrel starts — container won't respond - // to any requests until all workers (HTTP + BG) are initialized. - try { RunInitialization(); } - catch (Exception ex) { logger.LogCritical(ex, "[System] Initialization failed"); } - logger.LogInformation("[System] All pools ready — starting Kestrel"); -} -else -{ - logger.LogWarning("[System] Unknown ReadinessMode '{Mode}', falling back to Immediate", readinessMode); - app.Lifetime.ApplicationStarted.Register(() => - { - Task.Run(() => - { - try { RunInitialization(); } - catch (Exception ex) { logger.LogCritical(ex, "[System] Initialization failed"); } - }); - }); -} - -if (app.Environment.IsDevelopment()) -{ - logger.LogWarning("[Auth] Running in Development mode \u2014 unauthenticated requests will receive dev principal with roles: {Roles}", - string.Join(", ", CraftSettings.Auth.DevRoles)); -} - -// Response compression must be before static files. Skipped entirely when compression is disabled -// (App:Frontend:Compression=false / CRAFT_COMPRESSION=false) so everything is served raw/identity. -if (compressionEnabled) - app.UseResponseCompression(); -logger.LogInformation("[System] Static compression: {State}", compressionEnabled ? "enabled (precompressed .br/.gz + on-the-fly fallback)" : "DISABLED (raw/identity)"); - -// Nodes without the Http role do not short-circuit /api or auth paths: the HTTP endpoints simply aren't -// mapped (see the `if (capHttp)` blocks below), so those requests fall through to static file serving -// (a Frontend node can expose /api/me etc. from its own static dir) and finally to MapFallback (which -// 404s unmatched /api|/.auth). Nothing here intercepts them. - -// Setup mode: steers traffic to or away from the first-run wizard. Opt-in — the hosted app calls -// AppLifecycleBridge.RequestSetupMode() when it cannot self-configure. Decision table lives in -// Services/Setup/SetupGate.cs. -if (CraftSettings.Setup.Enabled) app.UseCraftSetupGate(logger); - -// Startup loading screen: while the HTTP worker pool initialises, serve a holding page to browsers -// and 503 to API callers. Only nodes with the Http role have a pool to wait on. Probes always pass. -// Decision table lives in Services/Hosting/StartupGate.cs. -app.Use(async (context, next) => -{ - if (!capHttp || pool.IsReady) - { - await next(); - return; - } - - switch (StartupGate.Decide(context.Request.Path.Value ?? "", - CraftSettings.Setup.Enabled, healthEnabled, healthPath)) - { - case StartupGateAction.PassThrough: - await next(); - return; - - case StartupGateAction.ApiUnavailable: - context.Response.StatusCode = 503; - context.Response.ContentType = "application/json"; - await context.Response.WriteAsync("""{"error":"Application is starting up. Please wait."}"""); - return; - - default: - context.Response.ContentType = "text/html; charset=utf-8"; - await context.Response.WriteAsync(SetupPages.StartupHtml); - return; - } -}); - -// Dev proxy: in Development, proxy frontend requests (including the Fast Refresh WebSocket) to -// `next dev` instead of serving precompiled files. See Services/Hosting/DevFrontendProxy.cs. -var devFrontendUrl = DevFrontendProxy.ResolveDevServerUrl( - capFrontend, app.Environment.IsDevelopment(), Environment.GetEnvironmentVariable); - -HttpClient? devProxyClient = devFrontendUrl is null - ? null - : app.UseCraftDevFrontendProxy(devFrontendUrl, logger); - -// CSP on every response, then static serving from Frontend/ (precompressed variants + cache policy). -// Only nodes with the Frontend role serve static content. See Services/Hosting/StaticFilePipeline.cs. -app.UseCraftContentSecurityPolicy(CraftSettings); - -var frontendPath = Path.Combine(AppContext.BaseDirectory, "Frontend"); -var frontendFileProvider = capFrontend - ? app.UseCraftStaticFiles(frontendPath, compressionEnabled, logger) - : null; - -// Frontend role but no directory: the node will 404 rather than serve anything, which is worth -// saying out loud — it usually means the app image forgot to COPY its build output. -if (capFrontend && frontendFileProvider is null) - logger.LogWarning("[System] Frontend directory not found: {Path}", frontendPath); - -// Auth service -var authService = app.Services.GetRequiredService(); - -// Storage readiness — only relevant to roles that use the store (http: allowedUsers; background: -// orchestrator). A frontend-only node never touches storage, so it is not resolved there (which also -// avoids requiring a connection string on a pure static origin). -var storageHealth = (capHttp || capBackground) - ? app.Services.GetRequiredService() - : null; -if (storageHealth != null) _ = storageHealth.RefreshAsync(); // prime the cache off the request path - -// Health probe (role-agnostic — mapped before the HTTP-role block so it survives every topology) -// and the realtime SSE channel. See Services/Hosting/Endpoints/. -app.MapCraftHealthEndpoint(roles, storageHealth, logger); -app.MapCraftRealtimeEndpoint(roles, CraftSettings, logger); - -// OAuth discovery documents (RFC 9728 PRM + optional RFC 8414 AS metadata) for MCP/OAuth client -// discovery, served verbatim from app settings. Anonymous by design — the setup reconcile keeps the -// well-known paths in EasyAuth's excludedPaths while the settings are present (their presence is -// the feature switch). See Services/Hosting/Endpoints/PrmEndpoint.cs. -app.MapCraftPrmEndpoint(CraftSettings, logger); - -// ── HTTP-role endpoints + middleware ────────────────────────────────────────────────────────────── -// A node without the Http role maps NONE of these, so /api and auth paths fall through to static serving -// (a Frontend node can expose them from its own static dir) and finally to MapFallback (404 for /api|/.auth). -if (capHttp) -{ - - // Answer CORS preflights for the deployment's declared-public (EasyAuth-excluded) paths — - // browser-based OAuth/MCP clients preflight their registration POST and the PowerShell - // dispatcher maps no OPTIONS verb. See Services/Hosting/CorsPreflightMiddleware.cs. - app.UseCraftPublicCorsPreflight(CraftSettings, logger); - - // Normalise the EasyAuth principal into the SWA shape the hosted PS app expects, then map the two - // auth-adjacent routes. See Services/Hosting/CraftAuthMiddleware.cs and Endpoints/AuthEndpoints.cs. - app.UseCraftAuth(CraftSettings, authService, logger); - app.MapCraftAuthEndpoints(CraftSettings, logger); - -} // end HTTP-role block (auth middleware). Bridges below run for any PS role; the - // setup/jobs/PS-dispatch routes are re-gated in a second `if (capHttp)` block further down. - -// Rate limiter middleware — only added when the limiter is registered on the service collection. -// -// Position is load-bearing, do not hoist this back to the top of the pipeline: -// * It must run AFTER UseCraftAuth. App-only callers (client-credentials API clients) arrive with -// no usable x-ms-client-principal-name — the auth middleware derives it from the token's appid. -// Limiting before that ran collapsed every API client behind a shared egress IP into one bucket. -// * It must run AFTER static file serving. A cold frontend load pulls dozens of assets, and -// counting those against the caller's budget could throttle a user for opening a page. -// Left outside the capHttp block on purpose: a frontend-only node has no auth middleware and no -// worker pool, but should still be protected, partitioned by origin address as before. -if (CraftSettings.RateLimit.IsEnabled) - app.UseRateLimiter(); - -// Concurrent request tracking for diagnostics. A holder object, not an int: the dispatch endpoint -// is registered elsewhere and a lambda cannot capture a ref local. -var activeRequests = new RequestCounter(); - -// --- Backend Process API --- -var orchestrator = app.Services.GetRequiredService(); -OrchestratorBridge.Initialize(orchestrator); -AuthBridge.Initialize(authService); -var jobManager = app.Services.GetRequiredService(); -QueueBridge.Initialize(psRunner, jobManager, CraftSettings.Orchestrator.QueueTaskFunction); -QueueStatusBridge.Initialize(jobManager, app.Services.GetRequiredService()); -WorkerMetricsBridge.Initialize(pool, app.Services.GetRequiredService(), jobManager, - app.Services.GetRequiredService().CreateLogger("Craft.Services.WorkerMetricsBridge")); -SchedulerBridge.Initialize(app.Services.GetRequiredService()); -CacheBridge.Initialize(cache); -StatsHistoryBridge.Initialize(app.Services.GetRequiredService()); -// AppLifecycleBridge.Initialize is at the TOP of the file — it must run before pool.Initialize() -// so the PS warmup script can use it (it would silently return false otherwise). - -// ── HTTP-role endpoints (continued): Setup API + Job Status + PowerShell dispatch ── -if (capHttp) -{ - - // Setup wizard API and job/run status API — both plain C#, no PowerShell involved. - // See Services/Hosting/Endpoints/. - app.MapCraftSetupEndpoints(CraftSettings); - app.MapCraftJobEndpoints(); - - // Native C# endpoints. Mapped before the PowerShell dispatcher, though ASP.NET route precedence - // would put a literal segment ahead of /API/{endpoint} regardless — which is what lets an app - // migrate one endpoint at a time with the PowerShell function still loaded as the rollback. - if (nativeCatalog.Endpoints.Count > 0) - { - var mappable = NativeEndpointRegistry.ResolveCollisions( - nativeCatalog.Endpoints, endpoints.Keys, CraftSettings.Endpoints.OnCollision, logger); - app.MapCraftNativeEndpoints(mappable, activeRequests, logger, CraftSettings.Endpoints); - } - - // Dispatch /API/{endpoint} to the discovered PowerShell function. Owns the response cache, - // stale-while-revalidate, and post-response trigger handling. - // See Services/Hosting/Endpoints/PowerShellDispatchEndpoint.cs. - app.MapCraftPowerShellDispatch(endpoints, activeRequests, logger); - -} // end HTTP-role block (setup / jobs / PowerShell dispatch) - -// Terminal fallback: proxy to the Next.js dev server in Development, otherwise serve a prerendered -// {path}.html or index.html for SPA routing. See Services/Hosting/Endpoints/FrontendFallbackEndpoint.cs. -app.MapCraftFrontendFallback( - new FrontendFallbackOptions(frontendFileProvider, devProxyClient, compressionEnabled, frontendPath), - logger); - -app.Run(); diff --git a/appsettings.example.jsonc b/appsettings.example.jsonc index b6bd88d..5d3498c 100644 --- a/appsettings.example.jsonc +++ b/appsettings.example.jsonc @@ -10,16 +10,19 @@ // JSON per the spec, and a file named `.json` that no generic tool, schema validator or linter can read // is a trap. The extension now matches the contents. // -// TO CONFIGURE A DEPLOYMENT, pick one: +// TO CONFIGURE A DEPLOYMENT, pick the right channel: // 1. Environment variables — App__Worker__HttpPoolSize=4, App__Auth__CookieName=..., and the CRAFT_* // shorthands. This is what the E2E harness and most deployments use. Highest precedence. -// 2. Your own appsettings.json, supplied by the downstream app image (COPY it to /app) or dropped in -// the project root for local `dotnet run`. +// 2. Local `dotnet run` secrets — `dotnet user-secrets set "AzureWebJobsStorage" "..."` (and other +// secret keys). User secrets load in Development only; see docs/configuration.md. Do NOT put +// connection strings or AUTH_SECRET in an appsettings.json for local work. +// 3. A downstream appsettings.json — optional non-secret structural defaults COPYed into a +// container image. Prefer env / Key Vault for credentials even there. // // EVERY VALUE BELOW IS ALREADY THE BUILT-IN DEFAULT. The defaults are the C# property initialisers in -// Services/CraftSettings.cs — that file is the single source of truth. Copy out only the keys you are -// actually changing; restating a default here just creates something that silently goes stale when the -// default moves. +// src/Craft.Configuration/CraftSettings.cs — that file is the single source of truth. +// Copy out only the keys you are actually changing; restating a default here just creates something +// that silently goes stale when the default moves. // // Full prose documentation: docs/configuration.md // ───────────────────────────────────────────────────────────────────────────────────────────────────── @@ -131,7 +134,7 @@ // Run each worker's PowerShell pipeline on one reused thread instead of a new thread per invocation. // Default true — the biggest per-request dispatch win (~50% of PS-invoke cost; see - // docs/dispatch-analysis.md). Set false only to A/B or if a module misbehaves on a long-lived thread. + // perf-harness/dispatch-analysis.md). Set false only to A/B or if a module misbehaves on a long-lived thread. // "ReuseRunspaceThread": true, // Host-tier pool sizing. When set, the first matching entry overrides @@ -237,10 +240,14 @@ }, // ── Storage backend (Azure Tables) ── - // The allowedUsers table (Auth) and orchestrator state resolve their connection as an explicit - // per-feature setting (e.g. Auth:UserStorageConnection) → the AzureWebJobsStorage env var. If - // neither is set the host FAILS TO START rather than silently falling back to the local emulator. + // Resolve order: Auth:UserStorageConnection (etc.) → AzureWebJobsStorage env var → + // App:Storage:ConnectionString (config / user secrets) → Development emulator fallback. + // Outside Development, missing config FAILS TO START rather than silently using the emulator. // "Storage": { + // // Shared connection string (same role as AzureWebJobsStorage). Local: prefer + // // dotnet user-secrets set "AzureWebJobsStorage" "…" + // // or set this key. Containers keep using the AzureWebJobsStorage env var. + // "ConnectionString": "", // // Allow the local storage emulator fallback when no connection string is configured. // // Default false (fail closed). Also enabled by CRAFT_ALLOW_DEV_STORAGE=true or // // ASPNETCORE_ENVIRONMENT=Development. @@ -373,6 +380,17 @@ "Timezone": "" }, + // Background concurrency limiter (on top of Worker.BgPoolSize). Null Base/Max use runtime defaults. + // "BackgroundLimiter": { + // "BaseConcurrency": 2, + // "MaxConcurrency": 8, + // "ScaleUpAfterSeconds": 15, + // "BurstToCeiling": false, + // "OverSubscribe": 0, + // "HttpPressureThreshold": null, + // "HttpPressureAfterSeconds": 10 + // }, + "Orchestrator": { // Prefix for Azure Tables: {Prefix}Runs, {Prefix}Tasks, {Prefix}Results "TablePrefix": "Orchestrator", @@ -380,7 +398,7 @@ "MaxRetries": 3, // Batch + coalesce per-task/run status writes off the fan-out critical path (results are never batched). // Default true. Removes the per-task Azure Table write that gates worker throughput — see - // docs/orch-analysis.md. Set false for the original per-task writes. + // perf-harness/orch-analysis.md. Set false for the original per-task writes. // "BatchStatusWrites": true, // Write the pre-invoke "Running" marker under a durable barrier (persisted before the task runs, so // AttemptCount/MaxRetries still bounds poison tasks). Default true. False = eventual (max throughput — @@ -413,7 +431,7 @@ // Bytes budget for keeping cached response bodies in memory (LRU tier over the disk cache) so a HIT // returns from RAM instead of re-reading the file. Default 64 MiB; 0 = disk-only. Big win for large - // List* responses (+157% throughput at ~150KB — see docs/cache-analysis.md). + // List* responses (+157% throughput at ~150KB — see perf-harness/cache-analysis.md). // "MaxMemoryBytes": 67108864, // Maximum cached responses held in memory diff --git a/build/Dockerfile b/build/Dockerfile index bfe1837..a0b1d0c 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -5,30 +5,30 @@ # - dev builds pass --build-arg BUILD_CONFIGURATION=Debug (default: Release) # - distroless = no shell, no package manager, non-root (APP_UID=1654) # - no SSH server; layer one on downstream if a deployment needs it -# The SDK / build stage stays on Debian; only the runtime is Azure Linux. +# The SDK / build stage stays on Ubuntu Noble (glibc); only the runtime is Azure Linux. +# .NET 10 dropped Debian bookworm images — noble is the supported full SDK/aspnet base. # -# Default the .NET base images to the GHCR mirror (CyberDrain/Containers) so -# builds survive an mcr.microsoft.com / Azure Container Registry outage. Build -# against upstream until the mirror carries the azurelinux tags: -# docker build -f build/Dockerfile --build-arg DOTNET_REGISTRY=mcr.microsoft.com -t craft:local . +# Default to upstream MCR for .NET 10 — the GHCR CyberDrain mirror does not yet carry the +# 10.0-noble / 10.0-azurelinux3.0-* tags. Flip back to ghcr.io/cyberdrain once the mirror is updated: +# docker build -f build/Dockerfile --build-arg DOTNET_REGISTRY=ghcr.io/cyberdrain -t craft:local . # # ── Base image patching ────────────────────────────────────────────────── -# The tags below deliberately float on the `8.0-*` minor, so .NET runtime security patches are picked +# The tags below deliberately float on the `10.0-*` minor, so .NET runtime security patches are picked # up automatically. That only works if the tag is actually re-resolved on each build: # # docker build --pull -f build/Dockerfile -t craft:local . # -# WITHOUT --pull, Docker reuses whatever `8.0-...` it already has locally, potentially for months. The +# WITHOUT --pull, Docker reuses whatever `10.0-...` it already has locally, potentially for months. The # CVEs this matters for live in /usr/share/dotnet/shared/Microsoft.NETCore.App/ — the shared runtime # shipped by the base image, NOT anything referenced from Craft.csproj. No package bump can fix them # and `dotnet list package --vulnerable` will never report them; only rebuilding on a newer base does. # CI passes `pull: true` for the same reason (.github/workflows/*-container.yml). # -# Do NOT pin these to an exact patch (e.g. 8.0.29) unless you also own the job that bumps the pin — +# Do NOT pin these to an exact patch (e.g. 10.0.10) unless you also own the job that bumps the pin — # a stale pin is strictly worse than a floating tag, because it looks deliberate. -ARG DOTNET_REGISTRY=ghcr.io/cyberdrain +ARG DOTNET_REGISTRY=mcr.microsoft.com -FROM ${DOTNET_REGISTRY}/dotnet/sdk:8.0-bookworm-slim AS build +FROM ${DOTNET_REGISTRY}/dotnet/sdk:10.0-noble AS build ARG BUILD_CONFIGURATION=Release # Supplied automatically by buildx (amd64 / arm64). Mapped to a .NET RID below. ARG TARGETARCH @@ -45,14 +45,19 @@ RUN case "${TARGETARCH:-amd64}" in \ # Restore inputs only, so the restore layer caches across source-only changes. Directory.Build.props # and global.json must land BEFORE restore: the props file sets properties restore evaluates, and # global.json pins the SDK — omit either and restore and publish disagree about what they're building. -COPY Craft.csproj Directory.Build.props global.json ./ -RUN dotnet restore -r "$(cat /tmp/rid)" +# The host lives under src/Craft/ and references sibling module projects; COPY their csproj files +# into the restore layer so ProjectReference restore caches across source-only changes. +COPY Directory.Build.props global.json ./ +COPY src/Craft/Craft.csproj ./src/Craft/ +COPY src/Craft.Configuration/Craft.Configuration.csproj ./src/Craft.Configuration/ +COPY src/Craft.Contracts/Craft.Contracts.csproj ./src/Craft.Contracts/ +RUN dotnet restore src/Craft/Craft.csproj -r "$(cat /tmp/rid)" COPY . . # Two things are load-bearing here: # -# Craft.csproj, named explicitly — with no argument `dotnet publish` resolves Craft.sln, which -# includes tests/Craft.Tests and publishes xunit, testhost and coverlet into the runtime image. +# src/Craft/Craft.csproj, named explicitly — with no argument `dotnet publish` resolves Craft.sln, +# which includes tests/Craft.Tests and publishes xunit, testhost and coverlet into the runtime image. # # -r --self-contained false — WITHOUT a RuntimeIdentifier, publish keeps the RID-agnostic # layout and copies EVERY platform's assets out of the NuGet packages: runtimes/win (17 MB), plus @@ -79,7 +84,7 @@ COPY . . # runtime exception inside a hosted app's script, and dropping only the compiler produces a different # exception for the same root problem. The satellite locale folders below ARE safe to delete; these # two are not. -RUN dotnet publish Craft.csproj -c ${BUILD_CONFIGURATION} \ +RUN dotnet publish src/Craft/Craft.csproj -c ${BUILD_CONFIGURATION} \ -r "$(cat /tmp/rid)" --self-contained false \ -o /app/publish && \ rm -f /app/publish/Craft.xml && \ @@ -89,9 +94,9 @@ RUN dotnet publish Craft.csproj -c ${BUILD_CONFIGURATION} \ /app/publish/pt-BR /app/publish/ru /app/publish/tr \ /app/publish/zh-Hans /app/publish/zh-Hant -# mimalloc staged from Debian (glibc) — not reliably packaged in Azure Linux's +# mimalloc staged from Ubuntu Noble (glibc) — not reliably packaged in Azure Linux's # tdnf repos, and glibc→glibc so it loads cleanly on the Azure Linux runtime. -FROM ${DOTNET_REGISTRY}/dotnet/aspnet:8.0-bookworm-slim AS mimalloc-src +FROM ${DOTNET_REGISTRY}/dotnet/aspnet:10.0-noble AS mimalloc-src ARG TARGETARCH RUN set -eux; \ apt-get update && apt-get install -y --no-install-recommends libmimalloc2.0; \ @@ -104,7 +109,7 @@ RUN set -eux; \ # distroless-extra = distroless + icu/tzdata. PowerShell needs globalization, so # the plain -distroless tag (no ICU) would break it — the -extra tag is required. -FROM ${DOTNET_REGISTRY}/dotnet/aspnet:8.0-azurelinux3.0-distroless-extra AS runtime +FROM ${DOTNET_REGISTRY}/dotnet/aspnet:10.0-azurelinux3.0-distroless-extra AS runtime # IMPORTANT: the app lives in /app, NOT under /home. Azure App Service mounts persistent storage over # /home when WEBSITES_ENABLE_APP_SERVICE_STORAGE=true, which shadows anything the image ships there — # an app under /home/app loses its binaries at runtime ("Craft.dll does not exist"). So we keep the app @@ -140,7 +145,7 @@ ENV CRAFT_VERBOSE="false" # Writable state needs no path envs: /app is app-owned (chowned COPY above), so runtime _cache/_data # land there fine; logs + restart-tracker default to the app user's home (RuntimePaths in -# CraftSettings.cs). App-owned content such as API/Config/function-permissions.json is writable in +# Craft.Configuration/Infrastructure/RuntimePaths.cs). App-owned content such as API/Config/function-permissions.json is writable in # place — downstream images that COPY additional files as root should still --chown them to APP_UID. # ── Memory & runtime tuning ── diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..e33ad83 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,94 @@ +# Craft architecture — modular monolith + +Craft is a **modular monolith**: one deployable process (`Craft.dll`) with clear module boundaries. +Configuration and the PowerShell contract DTOs are separate projects; feature modules still live in +the host until remaining type edges are inverted. + +## Layout + +| Path | Role | +|------|------| +| [`src/Craft/`](../src/Craft/) | Web host — Bridges, feature modules, Program, Runtime; publishes `Craft.dll` | +| [`src/Craft.Configuration/`](../src/Craft.Configuration/) | Settings POCOs only (dependency leaf); files grouped by feature area under `Auth/`, `Hosting/`, `PowerShell/`, … — namespace stays `Craft.Configuration` | +| [`src/Craft.Contracts/`](../src/Craft.Contracts/) | Pinned `Craft.Services` DTOs + `HttpResponseContext` | +| [`tests/`](../tests/) | Unit tests — sibling of `src/`, never compiled into `Craft.dll` | +| [`perf-harness/`](../perf-harness/) | E2E / load tooling — not part of the app project | + +`Craft.sln` and `Directory.Build.props` stay at the repo root. + +## Modules (folders under host `Services/`) + +| Folder | Namespace | Responsibility | +|--------|-----------|----------------| +| `Bridges/` | **`Craft.Services` (PINNED)** | Thin PowerShell facades over DI services | +| `PowerShellHost/` | `Craft.PowerShellHost` (+ pinned `PowerShellRunnerService` in `Craft.Services`) | Runspaces, pool, script discovery | +| `Orchestration/` | `Craft.Orchestration` | Jobs, scheduler, Durable-Functions-shaped fan-out, queue ingress | +| `Storage/` | `Craft.Storage` | Azure Tables | +| `Caching/` | `Craft.Caching` | Response cache | +| `Auth/` | `Craft.Auth` | EasyAuth / session | +| `Realtime/` | `Craft.Realtime` | SSE | +| `Setup/` | `Craft.Setup` | First-run wizard + setup-mode session flags | +| `Hosting/` (+ `Endpoints/`) | `Craft.Hosting` | Middleware, DI extensions, HTTP endpoints, metrics/startup trackers | +| `Program.cs` | (top-level) | Composition root — wires modules; owns startup order | + +## Dependency rules + +``` +Program / Hosting.Endpoints / Bridges + │ + ▼ + feature modules (Auth, Caching, Orchestration, Setup, Realtime, …) + │ + ▼ + PowerShellHost / Storage + │ + ▼ + Craft.Contracts (pinned DTOs) + │ + ▼ + Craft.Configuration +``` + +**Bridges** sit on the PowerShell edge only: they call into feature modules after `Initialize`. +Feature modules must **not** call static bridges for core logic — they use DI services instead: + +| Concern | Domain owner | PowerShell facade | +|---------|--------------|-------------------| +| Setup-mode flags | `SetupSessionState` | `AppLifecycleBridge` | +| Startup progress | `StartupProgressService` | `StartupInfoBridge` | +| Worker metrics | `WorkerMetricsService` | `WorkerMetricsBridge` | +| Orchestration ingress/drain | `OrchestratorService` | `OrchestratorBridge` | +| Queue ingress/drain | `QueueDispatchService` | `QueueBridge` | + +Composition-root `*.Initialize` calls in `Program.cs` (and `LogBridge.Initialize` during logging setup) +are the host edge — not domain cycles. + +## PowerShell public contract (do not break) + +Downstream apps resolve types by **fully-qualified name**. These must remain public under +`Craft.Services` (or the Functions mirror namespace for `HttpResponseContext`): + +- All types listed in [`PowerShellContractTests`](../tests/Craft.Tests/PowerShellContractTests.cs) +- Bridge facades live in the **host** assembly; DTOs live in **Craft.Contracts** (same namespace) +- `PowerShellRunnerService` is pinned to `Craft.Services` and still lives under the host's `PowerShellHost/` folder +- `Microsoft.Azure.Functions.PowerShellWorker.HttpResponseContext` keeps that exact type name (in Craft.Contracts) + +Folder / assembly moves are fine. **Namespace renames are a coordinated breaking change** with every hosted app. + +## Visibility + +Host wiring (endpoint mappers, setup middleware, `CraftHostBuilderExtensions`) is `internal`, with +`InternalsVisibleTo("Craft.Tests")` on the host. Host-only mutable state (`WorkerStats`, `JobRecord`, +pending queue records, `CacheEntry`) lives in feature namespaces as `internal`. `Craft.Contracts` +keeps `InternalsVisibleTo("Craft")` for DTO `internal set`ters (e.g. `StartupStats`) mutated by the +host. Types that tests assert on directly stay `public`. Bridge `Initialize` methods used only by +the composition root are `internal`. + +## Future project splits + +Still blocked on type edges (not call cycles): + +- PowerShellHost ↔ Hosting (`OperationContext`, metrics, profilers, `HandlerHeaders`) + +When those are inverted, extract Auth / Caching / Orchestration / … and optionally rename the host +folder to `Craft.Host` while keeping `AssemblyName=Craft` so Docker still runs `Craft.dll`. diff --git a/docs/configuration.md b/docs/configuration.md index 2613191..6cc52ff 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -5,12 +5,12 @@ Craft (CyberDrain Runtime for Apps, Functions, Tasks) is configured through ASP. ## Where defaults come from **Craft ships no `appsettings.json`.** Every setting's default is the C# property initialiser in -[`Services/CraftSettings.cs`](../Services/CraftSettings.cs) — that file is the single source of truth, and a +[`src/Craft.Configuration/CraftSettings.cs`](../src/Craft.Configuration/CraftSettings.cs) — that file is the single source of truth, and a deployment that sets nothing at all gets exactly those values. [`appsettings.example.jsonc`](../appsettings.example.jsonc) is an annotated reference listing every key alongside its default. It is **documentation only**: the `.jsonc` extension keeps it out of the -`appsettings*.json` content glob in `Craft.csproj`, so it is never copied to the published output or the +`appsettings*.json` content glob in `src/Craft/Craft.csproj`, so it is never copied to the published output or the container image. It carries comments, which strict JSON does not allow — hence the extension. Copy out of it only the keys you are actually changing. Restating a default in your own config creates a @@ -20,16 +20,44 @@ value that silently goes stale the day the default moves. Settings are merged in priority order (highest wins): -1. **Environment variables** — `App__Worker__BgPoolSize=8` -2. **`Properties/launchSettings.json`** — profile env vars injected by `dotnet run` / Visual Studio (local dev only, ignored in Docker/production) -3. **`appsettings.{Environment}.json`** — e.g. `appsettings.Development.json` (loaded when `ASPNETCORE_ENVIRONMENT=Development`) -4. **`appsettings.json`** — supplied by *your* app, if you supply one (always loaded, all environments) -5. **C# property defaults** in `Services/CraftSettings.cs` — the floor; always present +1. **Environment variables** — `App__Worker__BgPoolSize=8` (containers / CI / production) +2. **`src/Craft/Properties/launchSettings.json`** — profile env vars injected by `dotnet run` / Visual Studio (local only; sets `ASPNETCORE_ENVIRONMENT=Development`) +3. **.NET user secrets** — local `dotnet run` only (Development). Connection strings and other secrets go here — **not** in an `appsettings.json` on disk +4. **`appsettings.{Environment}.json` / `appsettings.json`** — optional non-secret overlays a *downstream app* may ship in its image. Do not use these for local secrets +5. **C# property defaults** in `src/Craft.Configuration/CraftSettings.cs` — the floor; always present -Both appsettings files are optional. When both are present the environment-specific file overlays onto the -base — it doesn't replace it — and values in the environment file win for the same key. Note that both are -gitignored in this repo, since a local `appsettings.json` is the file most likely to hold a storage -connection string. +The host registers settings with `AddOptions().BindConfiguration("App")`, applies SKU / +`AzureWebJobsStorage` post-configure, and `ValidateOnStart` for readiness mode and pool sizes. Nested +section objects use `= new()` so defaults apply when a section is absent from config. + +### Local development (user secrets only) + +For `dotnet run`, put every secret and connection string in the user-secrets store. Secrets never leave your +machine and are never a file in the repo (gitignored or otherwise). + +```bash +# from the repo root (src/Craft/Craft.csproj has UserSecretsId=CyberDrain.Craft) +# Azurite: Development already allows the emulator fallback — only set a connection when you need a +# real storage account (or want to be explicit): +dotnet user-secrets set "AzureWebJobsStorage" "UseDevelopmentStorage=true" --project src/Craft/Craft.csproj +# Optional App-section secrets use ":" (JSON shape), not "__": +dotnet user-secrets set "App:Auth:UserStorageConnection" "UseDevelopmentStorage=true" --project src/Craft/Craft.csproj +# EasyAuth client secret — only needed when exercising real AAD login / setup flows locally: +dotnet user-secrets set "AUTH_SECRET" "local-dev-only-change-me" --project src/Craft/Craft.csproj +dotnet user-secrets list --project src/Craft/Craft.csproj +dotnet run --project src/Craft/Craft.csproj +``` + +`WebApplication.CreateBuilder` loads user secrets only when the environment is **Development**. The checked-in +[`src/Craft/Properties/launchSettings.json`](../src/Craft/Properties/launchSettings.json) sets that for the default `dotnet run` +profile. If you override the environment to Production locally, secrets will not load — and storage will +fail closed unless `AzureWebJobsStorage` / `App:Storage:ConnectionString` is set another way. + +With `ASPNETCORE_ENVIRONMENT=Development` and no connection string at all, Craft already falls back to +`UseDevelopmentStorage=true` (Azurite). User secrets are for when you need a real account or other secrets. + +Containers and App Service keep using environment variables / Key Vault references — user secrets are a +local-dev mechanism, not a deployment one. Environment variables use `__` (double underscore) as the section separator: @@ -174,7 +202,7 @@ Controls the PowerShell runspace pools that execute all scripts. // Run each worker's PowerShell pipeline on one reused thread instead of a new thread per invocation. // Default true — the biggest single per-request dispatch win (~50% of the PS-invoke cost, +68% throughput - // on dispatch-bound load; see docs/dispatch-analysis.md), and matches the Azure Functions PS worker's + // on dispatch-bound load; see ../perf-harness/dispatch-analysis.md), and matches the Azure Functions PS worker's // persistent runspace. Safe: each worker owns one runspace and serves one request at a time. Set false // only to A/B or if a module misbehaves on a long-lived pipeline thread. "ReuseRunspaceThread": true, @@ -346,24 +374,27 @@ at load with an error naming the timer. ### Background concurrency limiter -Gates how many background/orchestrator tasks run at once, on top of the `Worker.BgPoolSize` runspaces. These -are **root-level** config keys (set at the top of `appsettings.json` or as env vars, not under `App:`). +Gates how many background/orchestrator tasks run at once, on top of the `Worker.BgPoolSize` runspaces. +Bound under `App:BackgroundLimiter` (env: `App__BackgroundLimiter__*`). Legacy root-level keys +(`BackgroundBaseConcurrency`, etc.) are still accepted via post-bind for older harness compose files. + By default it starts narrow and ramps slowly, to keep idle memory low; tune it for bursty fan-out. | Key | Default | Effect | |---|---|---| -| `BackgroundBaseConcurrency` | `clamp(cores, 2, 4)` | starting width when idle | -| `BackgroundScaleUpAfterSeconds` | `15` | how long the queue must be backed up before ramping (doubles per 10s tick) | -| `BackgroundMaxConcurrency` | `BgPoolSize` | ceiling | -| `BackgroundBurstToCeiling` | `false` | jump straight to the ceiling the moment tasks queue, skipping the ramp — **~2.7× faster fan-out** for bursts shorter than the ramp dwell (see docs/orch-analysis.md) | -| `BackgroundOverSubscribe` | `0` | admit this many tasks *above* the ceiling so they can do their pre-invoke table write and queue at the worker checkout while the pool stays full (helps only up to Azure Table write throughput) | -| `BackgroundHttpPressureThreshold` | `HttpPoolSize/2` | busy-HTTP-worker count that throttles BG to 2; `0` disables | -| `BackgroundHttpPressureAfterSeconds` | `10` | how long HTTP pressure must persist before throttling | +| `App:BackgroundLimiter:BaseConcurrency` | `clamp(cores, 2, 4)` | starting width when idle | +| `App:BackgroundLimiter:ScaleUpAfterSeconds` | `15` | how long the queue must be backed up before ramping (doubles per 10s tick) | +| `App:BackgroundLimiter:MaxConcurrency` | `BgPoolSize` | ceiling | +| `App:BackgroundLimiter:BurstToCeiling` | `false` | jump straight to the ceiling the moment tasks queue, skipping the ramp — **~2.7× faster fan-out** for bursts shorter than the ramp dwell (see ../perf-harness/orch-analysis.md) | +| `App:BackgroundLimiter:OverSubscribe` | `0` | admit this many tasks *above* the ceiling so they can do their pre-invoke table write and queue at the worker checkout while the pool stays full (helps only up to Azure Table write throughput) | +| `App:BackgroundLimiter:HttpPressureThreshold` | `HttpPoolSize/2` | busy-HTTP-worker count that throttles BG to 2; `0` disables | +| `App:BackgroundLimiter:HttpPressureAfterSeconds` | `10` | how long HTTP pressure must persist before throttling | ```jsonc -// top level of appsettings.json (NOT under "App"): -"BackgroundBurstToCeiling": true, // fill the pool immediately on a fan-out burst -"BackgroundScaleUpAfterSeconds": 5 // or: ramp sooner without going straight to ceiling +"BackgroundLimiter": { + "BurstToCeiling": true, // fill the pool immediately on a fan-out burst + "ScaleUpAfterSeconds": 5 // or: ramp sooner without going straight to ceiling +} ``` --- @@ -379,7 +410,7 @@ Fan-out/fan-in task execution with crash recovery. // Batch + coalesce per-task/run STATUS writes off the fan-out critical path, in ≤100-entity byte-budgeted // Azure Table transactions. Default true. This is the throughput fix for large fan-outs — the per-task - // table write was the ceiling (see docs/orch-analysis.md). Results are NEVER batched (their chunking / + // table write was the ceiling (see ../perf-harness/orch-analysis.md). Results are NEVER batched (their chunking / // multi-row large-payload path is untouched). Set false to fall back to per-task writes. "BatchStatusWrites": true, // Write the pre-invoke "Running" marker under a durable barrier (persisted BEFORE the task runs, batched @@ -451,7 +482,7 @@ In-memory index + disk-backed (`_cache/`) response cache for HTTP `List*` GET en // Bytes budget for the in-memory body tier (LRU over the disk cache) — a HIT returns the body from RAM // instead of re-reading + re-decoding the file. Default 64 MiB; 0 = disk-only. Gain scales with response - // size (+44% throughput for small List* responses, +157% at ~150KB; see docs/cache-analysis.md). + // size (+44% throughput for small List* responses, +157% at ~150KB; see ../perf-harness/cache-analysis.md). "MaxMemoryBytes": 67108864, // Maximum cached responses held in memory @@ -646,7 +677,7 @@ runs on. ### Realtime (SSE) Identity-gated Server-Sent Events channel at `/.craft/events`, fed in-process by -`[Craft.Services.RealtimeBridge]::Publish(...)` from PowerShell. See [realtime-bridge-plan.md](realtime-bridge-plan.md). +`[Craft.Services.RealtimeBridge]::Publish(...)` from PowerShell. **Off by default — opt in.** Set `Enabled: true` (or `CRAFT_REALTIME_ENABLED=true`, which wins over config). While off the endpoint is not mapped, `Publish` calls are no-ops, and no state or timer is held. When on, the @@ -735,7 +766,12 @@ The `allowedUsers` table works the same way as Azure Static Web Apps user invita By default, the table lives in the same storage account as the rest of the app (`AzureWebJobsStorage`). To isolate it — for example, to share a single user table across multiple Craft instances, or to keep user data in a separate storage account from operational data — set `Auth.UserStorageConnection` to a different connection string. -In `appsettings.json`: +Locally (user secrets): +```bash +dotnet user-secrets set "App:Auth:UserStorageConnection" "UseDevelopmentStorage=true" +``` + +In a downstream non-secret `appsettings.json` (prefer Key Vault / env for real credentials): ```jsonc "Auth": { "UserStorageConnection": "DefaultEndpointsProtocol=https;AccountName=myuserstorage;AccountKey=..." @@ -889,8 +925,8 @@ Craft separates its own built-in scripts from application content: 1. **Place compiled PS modules** in `API/Modules/` 2. **Place frontend build** in `Frontend/` (static files served automatically) -3. **Configure `App:` settings** — either `App__*` environment variables (preferred for containers) or your own `appsettings.json`, using [`appsettings.example.jsonc`](../appsettings.example.jsonc) as the reference. Set only what you're changing; everything else falls back to the defaults in `Services/CraftSettings.cs`. -4. **Set `AzureWebJobsStorage`** to a valid Azure Storage connection string (or `UseDevelopmentStorage=true` for Azurite) +3. **Configure `App:` settings** — `App__*` environment variables for containers; for local `dotnet run`, use `dotnet user-secrets` for secrets/connection strings (see [Local development](#local-development-user-secrets-only)). Non-secret structural defaults may live in a downstream `appsettings.json`. Use [`appsettings.example.jsonc`](../appsettings.example.jsonc) as the key reference; only set what you're changing. +4. **Set storage** — locally, Development already allows Azurite (`UseDevelopmentStorage=true`); put a real connection string in user secrets as `AzureWebJobsStorage` when needed. Deployed environments use env / Key Vault. 5. **Run:** `dotnet run` or `docker compose up` The host auto-discovers modules, builds route tables from HTTP endpoint functions, starts the scheduler, and serves both API and frontend from a single process. diff --git a/global.json b/global.json index bdc5d76..d61a8f7 100644 --- a/global.json +++ b/global.json @@ -7,7 +7,7 @@ "Bump this in lockstep with setup-dotnet in the workflows and the sdk tag in build/Dockerfile." ], "sdk": { - "version": "8.0.400", + "version": "10.0.100", "rollForward": "latestFeature", "allowPrerelease": false } diff --git a/perf-harness/docker-compose.bg.yml b/perf-harness/docker-compose.bg.yml index 6fa81c0..3f67268 100644 --- a/perf-harness/docker-compose.bg.yml +++ b/perf-harness/docker-compose.bg.yml @@ -39,13 +39,13 @@ services: - App__ReadinessMode=Immediate - App__Setup__Enabled=false - App__Orchestrator__TablePrefix=PerfBgOrch - # BackgroundTaskLimiter tuning (top-level config keys). run-orch.ps1 A/Bs the ramp behavior. + # BackgroundTaskLimiter tuning. Prefer App__BackgroundLimiter__*; root Background* keys still work via post-bind. # Defaults reproduce the app defaults on a small box (base 2, scale-up after 15s, ceiling = BG pool). - - BackgroundBaseConcurrency=${BG_BASE:-2} - - BackgroundScaleUpAfterSeconds=${BG_SCALEUP:-15} - - BackgroundMaxConcurrency=${BG_CEILING:-8} - - BackgroundBurstToCeiling=${BG_BURST:-false} - - BackgroundOverSubscribe=${BG_OVERSUB:-0} + - App__BackgroundLimiter__BaseConcurrency=${BG_BASE:-2} + - App__BackgroundLimiter__ScaleUpAfterSeconds=${BG_SCALEUP:-15} + - App__BackgroundLimiter__MaxConcurrency=${BG_CEILING:-8} + - App__BackgroundLimiter__BurstToCeiling=${BG_BURST:-false} + - App__BackgroundLimiter__OverSubscribe=${BG_OVERSUB:-0} # Batched status writer (#3). run-orch.ps1 -NoBatch sets false to A/B the per-task-write "before". - App__Orchestrator__BatchStatusWrites=${BATCH_WRITES:-true} - App__Orchestrator__DurableRunningBarrier=${DURABLE_BARRIER:-true} diff --git a/perf-harness/docker-compose.e2e-azure.yml b/perf-harness/docker-compose.e2e-azure.yml index f1b6770..d4094bf 100644 --- a/perf-harness/docker-compose.e2e-azure.yml +++ b/perf-harness/docker-compose.e2e-azure.yml @@ -14,7 +14,9 @@ services: sut: image: ${SUT_IMAGE:-craft:ci} - platform: linux/amd64 + # Native host arch (CI amd64 / local arm64). For cross-arch: build with --platform and add + # platform: linux/amd64 + # under this service, or use docker compose run --platform. container_name: craft-e2e-az-sut cpus: 2 ports: diff --git a/perf-harness/orch-analysis.md b/perf-harness/orch-analysis.md index ebd4f74..66775f7 100644 --- a/perf-harness/orch-analysis.md +++ b/perf-harness/orch-analysis.md @@ -84,9 +84,9 @@ fan-out is gated by **Azure Table write throughput, not worker throughput**. Thi ## Applied: #1 burst-to-ceiling (win) + over-subscription dial (marginal — confirms the real bottleneck) -Both are configurable (default off/0, preserving today's conservative behavior): -`BackgroundBurstToCeiling` (bool) and `BackgroundOverSubscribe` (int), alongside the existing -`BackgroundBaseConcurrency` / `BackgroundScaleUpAfterSeconds` / `BackgroundMaxConcurrency`. +Both are configurable (default off/0, preserving today's conservative behavior) under +`App:BackgroundLimiter`: `BurstToCeiling` (bool) and `OverSubscribe` (int), alongside +`BaseConcurrency` / `ScaleUpAfterSeconds` / `MaxConcurrency`. **#1 Burst-to-ceiling — clear 2.7× win.** Jumps `_currentMax` straight to the ceiling the moment tasks queue, instead of the 15 s ramp. 1000 tasks / pool 8: **12.5 s → 4.6 s**, identical to pinning baseline=ceiling, but diff --git a/perf-harness/scripts/run-e2e.ps1 b/perf-harness/scripts/run-e2e.ps1 index 24ce413..459a918 100644 --- a/perf-harness/scripts/run-e2e.ps1 +++ b/perf-harness/scripts/run-e2e.ps1 @@ -60,7 +60,12 @@ function Fetch($url, $extra = @()) { if ($Build) { Info "building $SutImage ..." - docker build -f (Join-Path $repoRoot 'build/Dockerfile') -t $SutImage $repoRoot | Out-Host + # Prefer upstream MCR until the GHCR CyberDrain mirror carries .NET 10 tags. --pull re-resolves + # floating 10.0-* bases so runtime CVEs aren't baked from a stale local cache. + $registry = if ($env:DOTNET_REGISTRY) { $env:DOTNET_REGISTRY } else { 'mcr.microsoft.com' } + docker build --pull -f (Join-Path $repoRoot 'build/Dockerfile') ` + --build-arg "DOTNET_REGISTRY=$registry" ` + -t $SutImage $repoRoot | Out-Host if ($LASTEXITCODE -ne 0) { throw "docker build failed" } } diff --git a/Services/Configuration/AuthSettings.cs b/src/Craft.Configuration/Auth/AuthSettings.cs similarity index 100% rename from Services/Configuration/AuthSettings.cs rename to src/Craft.Configuration/Auth/AuthSettings.cs diff --git a/Services/Configuration/PrmSettings.cs b/src/Craft.Configuration/Auth/PrmSettings.cs similarity index 100% rename from Services/Configuration/PrmSettings.cs rename to src/Craft.Configuration/Auth/PrmSettings.cs diff --git a/Services/Configuration/SsoSecretNames.cs b/src/Craft.Configuration/Auth/SsoSecretNames.cs similarity index 100% rename from Services/Configuration/SsoSecretNames.cs rename to src/Craft.Configuration/Auth/SsoSecretNames.cs diff --git a/Services/Configuration/CacheSettings.cs b/src/Craft.Configuration/Caching/CacheSettings.cs similarity index 80% rename from Services/Configuration/CacheSettings.cs rename to src/Craft.Configuration/Caching/CacheSettings.cs index e14462a..059ac04 100644 --- a/Services/Configuration/CacheSettings.cs +++ b/src/Craft.Configuration/Caching/CacheSettings.cs @@ -21,7 +21,7 @@ public class CacheSettings /// Budget (bytes) for keeping cached response bodies in memory (an LRU tier over the disk cache) so a /// cache HIT returns from RAM instead of re-reading + re-decoding the file every time. Default 64 MiB. /// 0 disables the in-memory tier (disk-only — every hit reads the file). The index is always in memory; - /// this only governs the hot bodies. See docs/cache-analysis.md. + /// this only governs the hot bodies. See perf-harness/cache-analysis.md. /// public long MaxMemoryBytes { get; set; } = 64L * 1024 * 1024; @@ -83,4 +83,24 @@ public class CacheSettings /// Example: { "ListTenants": 300, "ListUsers": 120 } /// public Dictionary EndpointTtl { get; set; } = new(); + + /// + /// Resolves whether the response cache is active. CRAFT_RESPONSE_CACHE wins when set; + /// otherwise if configured; otherwise + /// (typically true only when the node serves both frontend and HTTP). + /// + public bool ResolveEnabled(bool autoDefault) => + ResolveEnabled(autoDefault, Environment.GetEnvironmentVariable); + + /// + /// Same as but with an injectable environment lookup (for tests). + /// + public bool ResolveEnabled(bool autoDefault, Func env) + { + ArgumentNullException.ThrowIfNull(env); + var v = env("CRAFT_RESPONSE_CACHE"); + if (!string.IsNullOrWhiteSpace(v)) + return v.Equals("true", StringComparison.OrdinalIgnoreCase) || v == "1"; + return Enabled ?? autoDefault; + } } diff --git a/src/Craft.Configuration/Craft.Configuration.csproj b/src/Craft.Configuration/Craft.Configuration.csproj new file mode 100644 index 0000000..8c5e94a --- /dev/null +++ b/src/Craft.Configuration/Craft.Configuration.csproj @@ -0,0 +1,20 @@ + + + + + net10.0 + Craft.Configuration + Craft settings POCOs — dependency leaf for the modular monolith. + + + + + + + + + + + + + diff --git a/Services/Configuration/CraftSettings.cs b/src/Craft.Configuration/CraftSettings.cs similarity index 93% rename from Services/Configuration/CraftSettings.cs rename to src/Craft.Configuration/CraftSettings.cs index d429d8a..73241e3 100644 --- a/Services/Configuration/CraftSettings.cs +++ b/src/Craft.Configuration/CraftSettings.cs @@ -8,8 +8,9 @@ namespace Craft.Configuration; /// To onboard a new PowerShell application: /// 1. Place your compiled PS modules in API/Modules/ /// 2. Place your frontend build in Frontend/ -/// 3. Configure this section in appsettings.json -/// 4. Run the container +/// 3. Configure — App__* env vars in containers; dotnet user-secrets for local secrets; +/// optional non-secret appsettings.json in a downstream image +/// 4. Run the host / container /// public class CraftSettings { @@ -55,6 +56,9 @@ public class CraftSettings /// Orchestrator (fan-out/fan-in) configuration. public OrchestratorSettings Orchestrator { get; set; } = new(); + /// Background concurrency limiter. See . + public BackgroundLimiterSettings BackgroundLimiter { get; set; } = new(); + /// Response cache configuration. public CacheSettings Cache { get; set; } = new(); diff --git a/Services/Configuration/StatsHistorySettings.cs b/src/Craft.Configuration/Diagnostics/StatsHistorySettings.cs similarity index 100% rename from Services/Configuration/StatsHistorySettings.cs rename to src/Craft.Configuration/Diagnostics/StatsHistorySettings.cs diff --git a/Services/Configuration/ContainerHealthSettings.cs b/src/Craft.Configuration/Hosting/ContainerHealthSettings.cs similarity index 100% rename from Services/Configuration/ContainerHealthSettings.cs rename to src/Craft.Configuration/Hosting/ContainerHealthSettings.cs diff --git a/Services/Configuration/EndpointSettings.cs b/src/Craft.Configuration/Hosting/EndpointSettings.cs similarity index 100% rename from Services/Configuration/EndpointSettings.cs rename to src/Craft.Configuration/Hosting/EndpointSettings.cs diff --git a/Services/Configuration/FileLoggingSettings.cs b/src/Craft.Configuration/Hosting/FileLoggingSettings.cs similarity index 91% rename from Services/Configuration/FileLoggingSettings.cs rename to src/Craft.Configuration/Hosting/FileLoggingSettings.cs index 2a7deaf..411b47d 100644 --- a/Services/Configuration/FileLoggingSettings.cs +++ b/src/Craft.Configuration/Hosting/FileLoggingSettings.cs @@ -1,5 +1,11 @@ namespace Craft.Configuration; +/// +/// File-backed logging with size-based rotation. +/// Logs are written to {Directory}/{FilePrefix}.log and rotated to +/// {FilePrefix}.1.log, {FilePrefix}.2.log, etc. when MaxFileSizeMB is exceeded. +/// Oldest files beyond MaxFileCount are automatically deleted. +/// public class FileLoggingSettings { /// diff --git a/Services/Configuration/FrontendSettings.cs b/src/Craft.Configuration/Hosting/FrontendSettings.cs similarity index 81% rename from Services/Configuration/FrontendSettings.cs rename to src/Craft.Configuration/Hosting/FrontendSettings.cs index f4a70a4..485ed19 100644 --- a/Services/Configuration/FrontendSettings.cs +++ b/src/Craft.Configuration/Hosting/FrontendSettings.cs @@ -46,4 +46,21 @@ public class FrontendSettings /// environment variable (true/false), which takes precedence over this setting. /// public bool Compression { get; set; } = true; + + /// + /// Resolved compression state. CRAFT_COMPRESSION wins when set; otherwise + /// applies. + /// + public bool IsCompressionEnabled => ResolveCompressionEnabled(Environment.GetEnvironmentVariable); + + /// + /// Same as but with an injectable environment lookup (for tests). + /// + public bool ResolveCompressionEnabled(Func env) + { + ArgumentNullException.ThrowIfNull(env); + var v = env("CRAFT_COMPRESSION"); + if (string.IsNullOrWhiteSpace(v)) return Compression; + return v.Equals("true", StringComparison.OrdinalIgnoreCase) || v == "1"; + } } diff --git a/src/Craft.Configuration/Hosting/HealthSettings.cs b/src/Craft.Configuration/Hosting/HealthSettings.cs new file mode 100644 index 0000000..a5d43ff --- /dev/null +++ b/src/Craft.Configuration/Hosting/HealthSettings.cs @@ -0,0 +1,45 @@ +namespace Craft.Configuration; + +/// +/// Role-agnostic health probe. Enabled by default at /healthz; a deployment can relocate it behind a +/// specific probe URL or turn it off entirely. Overridable via CRAFT_HEALTH_ENABLED / CRAFT_HEALTH_PATH. +/// +public class HealthSettings +{ + /// Whether the health endpoint is mapped. Default true. + public bool Enabled { get; set; } = true; + + /// Path the health endpoint is served at. Default "/healthz". A leading slash is added if missing. + public string Path { get; set; } = "/healthz"; + + /// + /// Resolved enabled state. CRAFT_HEALTH_ENABLED wins when set; otherwise + /// applies. + /// + public bool IsEnabled => ResolveEnabled(Environment.GetEnvironmentVariable); + + /// + /// Resolved path, always normalised to start with /. CRAFT_HEALTH_PATH wins when set + /// to a non-blank value; otherwise applies. + /// + public string ResolvedPath => ResolvePath(Environment.GetEnvironmentVariable); + + /// Same as but with an injectable environment lookup (for tests). + public bool ResolveEnabled(Func env) + { + ArgumentNullException.ThrowIfNull(env); + var v = env("CRAFT_HEALTH_ENABLED"); + if (string.IsNullOrWhiteSpace(v)) return Enabled; + return v.Equals("true", StringComparison.OrdinalIgnoreCase) || v == "1"; + } + + /// Same as but with an injectable environment lookup (for tests). + public string ResolvePath(Func env) + { + ArgumentNullException.ThrowIfNull(env); + var pathEnv = env("CRAFT_HEALTH_PATH"); + var path = !string.IsNullOrWhiteSpace(pathEnv) ? pathEnv.Trim() : Path; + if (!path.StartsWith('/')) path = "/" + path; + return path; + } +} diff --git a/Services/Configuration/KestrelLimitsSettings.cs b/src/Craft.Configuration/Hosting/KestrelLimitsSettings.cs similarity index 100% rename from Services/Configuration/KestrelLimitsSettings.cs rename to src/Craft.Configuration/Hosting/KestrelLimitsSettings.cs diff --git a/Services/Configuration/RateLimitSettings.cs b/src/Craft.Configuration/Hosting/RateLimitSettings.cs similarity index 62% rename from Services/Configuration/RateLimitSettings.cs rename to src/Craft.Configuration/Hosting/RateLimitSettings.cs index 2fb3ca9..86632b8 100644 --- a/Services/Configuration/RateLimitSettings.cs +++ b/src/Craft.Configuration/Hosting/RateLimitSettings.cs @@ -9,7 +9,8 @@ public class RateLimitSettings { /// /// Enable the global rate limiter. Default true (300 requests / 10 s per client). Disable via - /// App:RateLimit:Enabled=false; the CRAFT_RATELIMIT_ENABLED=true env var can also force it on. + /// App:RateLimit:Enabled=false. The CRAFT_RATELIMIT_ENABLED environment variable + /// (true/1 or false/0) wins when set. /// public bool Enabled { get; set; } = true; @@ -25,8 +26,17 @@ public class RateLimitSettings /// public int QueueLimit { get; set; } - /// Resolved enabled state, honouring the CRAFT_RATELIMIT_ENABLED environment override. - public bool IsEnabled => - Enabled - || string.Equals(Environment.GetEnvironmentVariable("CRAFT_RATELIMIT_ENABLED"), "true", StringComparison.OrdinalIgnoreCase); + /// + /// Resolved enabled state. CRAFT_RATELIMIT_ENABLED wins when set; otherwise + /// applies. + /// + public bool IsEnabled + { + get + { + var v = Environment.GetEnvironmentVariable("CRAFT_RATELIMIT_ENABLED"); + if (string.IsNullOrWhiteSpace(v)) return Enabled; + return v.Equals("true", StringComparison.OrdinalIgnoreCase) || v == "1"; + } + } } diff --git a/Services/Configuration/RolesSettings.cs b/src/Craft.Configuration/Hosting/RolesSettings.cs similarity index 100% rename from Services/Configuration/RolesSettings.cs rename to src/Craft.Configuration/Hosting/RolesSettings.cs diff --git a/Services/Configuration/RuntimePaths.cs b/src/Craft.Configuration/Infrastructure/RuntimePaths.cs similarity index 72% rename from Services/Configuration/RuntimePaths.cs rename to src/Craft.Configuration/Infrastructure/RuntimePaths.cs index 5a68054..124ef2f 100644 --- a/Services/Configuration/RuntimePaths.cs +++ b/src/Craft.Configuration/Infrastructure/RuntimePaths.cs @@ -1,11 +1,5 @@ namespace Craft.Configuration; -/// -/// File-backed logging with size-based rotation. -/// Logs are written to {Directory}/{FilePrefix}.log and rotated to -/// {FilePrefix}.1.log, {FilePrefix}.2.log, etc. when MaxFileSizeMB is exceeded. -/// Oldest files beyond MaxFileCount are automatically deleted. -/// /// /// Default writable base directory for app-owned runtime state (logs, restart /// tracker) when no explicit path is configured. Resolves the current user's home diff --git a/src/Craft.Configuration/Orchestration/BackgroundLimiterSettings.cs b/src/Craft.Configuration/Orchestration/BackgroundLimiterSettings.cs new file mode 100644 index 0000000..6aad420 --- /dev/null +++ b/src/Craft.Configuration/Orchestration/BackgroundLimiterSettings.cs @@ -0,0 +1,46 @@ +namespace Craft.Configuration; + +/// +/// Background / orchestrator concurrency limiter — gates how many BG tasks run at once on top of +/// Worker.BgPoolSize. Bound from App:BackgroundLimiter. Legacy root-level keys +/// (BackgroundBaseConcurrency, etc.) are still overlaid in post-bind for harness compat. +/// +public class BackgroundLimiterSettings +{ + /// + /// Starting concurrency when idle. null (default) → clamp(ProcessorCount, 2, 4). + /// + public int? BaseConcurrency { get; set; } + + /// + /// Ceiling concurrency. null (default) → Worker.BgPoolSize. + /// + public int? MaxConcurrency { get; set; } + + /// + /// How long the BG queue must be backed up before ramping (seconds). Default 15. + /// + public int ScaleUpAfterSeconds { get; set; } = 15; + + /// + /// Busy-HTTP-worker count that throttles BG. null (default) → HttpPoolSize/2. + /// Set to 0 to disable HTTP-pressure throttling. + /// + public int? HttpPressureThreshold { get; set; } + + /// + /// How long HTTP pressure must persist before throttling (seconds). Default 10. + /// + public int HttpPressureAfterSeconds { get; set; } = 10; + + /// + /// Jump straight to the ceiling when tasks queue, skipping the ramp dwell. Default false. + /// + public bool BurstToCeiling { get; set; } + + /// + /// Admit this many tasks above the worker target so they can do pre-invoke work while the pool + /// stays full. Default 0 (strict pool cap). + /// + public int OverSubscribe { get; set; } +} diff --git a/Services/Configuration/OrchestratorSettings.cs b/src/Craft.Configuration/Orchestration/OrchestratorSettings.cs similarity index 97% rename from Services/Configuration/OrchestratorSettings.cs rename to src/Craft.Configuration/Orchestration/OrchestratorSettings.cs index 786132b..35162bb 100644 --- a/Services/Configuration/OrchestratorSettings.cs +++ b/src/Craft.Configuration/Orchestration/OrchestratorSettings.cs @@ -14,7 +14,7 @@ public class OrchestratorSettings /// /// Batch and coalesce per-task/run status writes through OrchestratorStatusWriter instead of writing each /// individually. Removes the per-task Azure Table write from the fan-out critical path (the throughput - /// ceiling — see docs/orch-analysis.md). Default true. Results are never batched (their chunking path is + /// ceiling — see perf-harness/orch-analysis.md). Default true. Results are never batched (their chunking path is /// untouched). Set false to fall back to the original per-task writes (for A/B). /// public bool BatchStatusWrites { get; set; } = true; diff --git a/Services/Configuration/SchedulerSettings.cs b/src/Craft.Configuration/Orchestration/SchedulerSettings.cs similarity index 100% rename from Services/Configuration/SchedulerSettings.cs rename to src/Craft.Configuration/Orchestration/SchedulerSettings.cs diff --git a/Services/Configuration/GlobalJsonPreload.cs b/src/Craft.Configuration/PowerShell/GlobalJsonPreload.cs similarity index 100% rename from Services/Configuration/GlobalJsonPreload.cs rename to src/Craft.Configuration/PowerShell/GlobalJsonPreload.cs diff --git a/Services/Configuration/ModuleInjection.cs b/src/Craft.Configuration/PowerShell/ModuleInjection.cs similarity index 100% rename from Services/Configuration/ModuleInjection.cs rename to src/Craft.Configuration/PowerShell/ModuleInjection.cs diff --git a/Services/Configuration/PermissionExtractionSettings.cs b/src/Craft.Configuration/PowerShell/PermissionExtractionSettings.cs similarity index 100% rename from Services/Configuration/PermissionExtractionSettings.cs rename to src/Craft.Configuration/PowerShell/PermissionExtractionSettings.cs diff --git a/Services/Configuration/ScriptRepoSettings.cs b/src/Craft.Configuration/PowerShell/ScriptRepoSettings.cs similarity index 100% rename from Services/Configuration/ScriptRepoSettings.cs rename to src/Craft.Configuration/PowerShell/ScriptRepoSettings.cs diff --git a/Services/Configuration/SkuProfile.cs b/src/Craft.Configuration/PowerShell/SkuProfile.cs similarity index 100% rename from Services/Configuration/SkuProfile.cs rename to src/Craft.Configuration/PowerShell/SkuProfile.cs diff --git a/Services/Configuration/WorkerSettings.cs b/src/Craft.Configuration/PowerShell/WorkerSettings.cs similarity index 99% rename from Services/Configuration/WorkerSettings.cs rename to src/Craft.Configuration/PowerShell/WorkerSettings.cs index 7ed4dba..f88ac52 100644 --- a/Services/Configuration/WorkerSettings.cs +++ b/src/Craft.Configuration/PowerShell/WorkerSettings.cs @@ -129,7 +129,7 @@ public class WorkerSettings /// /// Run each worker's PowerShell pipeline on one reused thread (PSThreadOptions.ReuseThread) instead of /// spinning a new thread per invocation. Default true. This is the single biggest per-request dispatch - /// win (thread creation was ~50% of the PS-invoke cost — see docs/dispatch-analysis.md) and matches how + /// win (thread creation was ~50% of the PS-invoke cost — see perf-harness/dispatch-analysis.md) and matches how /// the Azure Functions PowerShell worker keeps a persistent runspace. Safe because each worker owns one /// runspace and serves one request at a time. Set false only to A/B or if a module misbehaves on a /// long-lived pipeline thread. diff --git a/Services/Configuration/RealtimeSettings.cs b/src/Craft.Configuration/Realtime/RealtimeSettings.cs similarity index 85% rename from Services/Configuration/RealtimeSettings.cs rename to src/Craft.Configuration/Realtime/RealtimeSettings.cs index daa9c72..be2cbb8 100644 --- a/Services/Configuration/RealtimeSettings.cs +++ b/src/Craft.Configuration/Realtime/RealtimeSettings.cs @@ -2,15 +2,15 @@ namespace Craft.Configuration; /// /// Realtime SSE channel served at /.craft/events. Opt-in — off by default. Downstream code -/// publishes job lifecycle events through ; browsers consume them. In-memory, -/// single instance — see docs/realtime-bridge-plan.md. The limits below bound memory and the connection budget. +/// publishes job lifecycle events through Craft.Services.RealtimeBridge; browsers consume them. In-memory, +/// single instance — see docs/configuration.md#realtime-sse. The limits below bound memory and the connection budget. /// public class RealtimeSettings { /// /// Enable the realtime endpoint and bridge delivery. Default false — turn it on explicitly with /// App:Realtime:Enabled=true (delivery is then still role-gated to http/frontend nodes). While - /// off, /.craft/events is not mapped and publishes are no-ops. + /// off, /.craft/events is not mapped and Craft.Services.RealtimeBridge publishes are no-ops. /// public bool Enabled { get; set; } diff --git a/Services/Configuration/SetupSettings.cs b/src/Craft.Configuration/Setup/SetupSettings.cs similarity index 100% rename from Services/Configuration/SetupSettings.cs rename to src/Craft.Configuration/Setup/SetupSettings.cs diff --git a/Services/Configuration/StorageSettings.cs b/src/Craft.Configuration/Storage/StorageSettings.cs similarity index 67% rename from Services/Configuration/StorageSettings.cs rename to src/Craft.Configuration/Storage/StorageSettings.cs index ef99c0f..a53b819 100644 --- a/Services/Configuration/StorageSettings.cs +++ b/src/Craft.Configuration/Storage/StorageSettings.cs @@ -1,17 +1,29 @@ namespace Craft.Configuration; /// -/// Azure Storage connection policy. The Tables used for the allowedUsers authorization table and -/// orchestrator state resolve their connection string in this order: an explicit per-feature setting -/// (e.g. Auth:UserStorageConnection) → the AzureWebJobsStorage environment variable. +/// Azure Storage connection policy for the shared host store (orchestrator tables, health probes). +/// Resolution order for when no per-call override is passed: +/// AzureWebJobsStorage → → Development emulator when allowed. /// -/// If neither is configured the host does NOT silently fall back to the local storage emulator -/// (UseDevelopmentStorage=true) in production — that would point authorization and orchestrator -/// state at a non-existent emulator. The fallback is only used when explicitly opted in; otherwise +/// The allowedUsers table may use a separate account via Auth:UserStorageConnection +/// (); that override is applied only by the user-table +/// store, not here. +/// +/// If nothing is configured the host does NOT silently fall back to the local storage emulator +/// (UseDevelopmentStorage=true) in production — that would point host state at a non-existent +/// emulator. The fallback is only used when explicitly opted in; otherwise /// throws and the host fails to start. /// public class StorageSettings { + /// + /// Shared Azure Storage connection string — same role as the AzureWebJobsStorage env var. + /// Containers should keep using the env var. For local dotnet run, set it via user secrets: + /// dotnet user-secrets set "AzureWebJobsStorage" "…" (copied onto this property at startup) + /// or dotnet user-secrets set "App:Storage:ConnectionString" "…". + /// + public string ConnectionString { get; set; } = ""; + /// /// Allow the local storage emulator fallback (UseDevelopmentStorage=true) when no real /// connection string is configured. Default false (fail closed). Also enabled by the @@ -59,11 +71,14 @@ public string ResolveConnection(string? explicitConnection, string purpose) if (!string.IsNullOrWhiteSpace(explicitConnection)) return explicitConnection; var env = Environment.GetEnvironmentVariable("AzureWebJobsStorage"); if (!string.IsNullOrWhiteSpace(env)) return env; + if (!string.IsNullOrWhiteSpace(ConnectionString)) return ConnectionString; if (DevStorageAllowed) return "UseDevelopmentStorage=true"; throw new InvalidOperationException( $"No Azure Storage connection is configured for {purpose}. Set the AzureWebJobsStorage " + - "environment variable (or Auth:UserStorageConnection in appsettings). To use the local " + - "storage emulator during development, set App:Storage:AllowDevelopmentStorage=true or the " + - "CRAFT_ALLOW_DEV_STORAGE=true environment variable."); + "environment variable or App:Storage:ConnectionString, or for local Development use " + + "`dotnet user-secrets set \"AzureWebJobsStorage\" \"…\"`. For the allowedUsers table only, " + + "App:Auth:UserStorageConnection may isolate that table. To use the local storage emulator, " + + "set ASPNETCORE_ENVIRONMENT=Development, App:Storage:AllowDevelopmentStorage=true, or " + + "CRAFT_ALLOW_DEV_STORAGE=true."); } } diff --git a/Services/Bridges/CacheStats.cs b/src/Craft.Contracts/CacheStats.cs similarity index 100% rename from Services/Bridges/CacheStats.cs rename to src/Craft.Contracts/CacheStats.cs diff --git a/src/Craft.Contracts/Craft.Contracts.csproj b/src/Craft.Contracts/Craft.Contracts.csproj new file mode 100644 index 0000000..272bc85 --- /dev/null +++ b/src/Craft.Contracts/Craft.Contracts.csproj @@ -0,0 +1,17 @@ + + + + + net10.0 + Craft.Services + Craft.Contracts + Pinned Craft.Services DTOs and HttpResponseContext for PowerShell interop. + + + + + + + + + diff --git a/Services/Bridges/GenerationDetail.cs b/src/Craft.Contracts/GenerationDetail.cs similarity index 100% rename from Services/Bridges/GenerationDetail.cs rename to src/Craft.Contracts/GenerationDetail.cs diff --git a/Services/PowerShellHost/HttpResponseContext.cs b/src/Craft.Contracts/HttpResponseContext.cs similarity index 90% rename from Services/PowerShellHost/HttpResponseContext.cs rename to src/Craft.Contracts/HttpResponseContext.cs index d2ccc3c..0f36a27 100644 --- a/Services/PowerShellHost/HttpResponseContext.cs +++ b/src/Craft.Contracts/HttpResponseContext.cs @@ -4,7 +4,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker; /// /// Mirror of the Azure Functions PowerShell worker's HttpResponseContext, compiled into -/// Craft.dll. It deliberately lives in the Microsoft.Azure.Functions.PowerShellWorker +/// Craft.Contracts.dll. It deliberately lives in the Microsoft.Azure.Functions.PowerShellWorker /// namespace so its type name matches the real Functions worker exactly. /// /// This type is compiled in rather than defined at runtime via Add-Type -TypeDefinition. @@ -26,7 +26,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker; /// Hosted-app routers such as CIPP's New-CippCoreRequest pick the response object out of a /// function's pipeline output with /// $_.PSObject.TypeNames -eq 'Microsoft.Azure.Functions.PowerShellWorker.HttpResponseContext', -/// so the namespace-qualified name has to match. derives a +/// so the namespace-qualified name has to match. Craft.PowerShellHost.PowerShellWorker.Initialize derives a /// runspace-level [HttpResponseContext] PowerShell class from this type so scripts resolve /// the short name (PS classes are compiled by the PS engine, no Roslyn) while instances still carry /// this base type's name in their PSTypeNames. diff --git a/Services/Bridges/JobDetail.cs b/src/Craft.Contracts/JobDetail.cs similarity index 100% rename from Services/Bridges/JobDetail.cs rename to src/Craft.Contracts/JobDetail.cs diff --git a/Services/Bridges/JobMetrics.cs b/src/Craft.Contracts/JobMetrics.cs similarity index 100% rename from Services/Bridges/JobMetrics.cs rename to src/Craft.Contracts/JobMetrics.cs diff --git a/Services/Bridges/JobRunSummary.cs b/src/Craft.Contracts/JobRunSummary.cs similarity index 100% rename from Services/Bridges/JobRunSummary.cs rename to src/Craft.Contracts/JobRunSummary.cs diff --git a/Services/Bridges/JobSummary.cs b/src/Craft.Contracts/JobSummary.cs similarity index 100% rename from Services/Bridges/JobSummary.cs rename to src/Craft.Contracts/JobSummary.cs diff --git a/Services/Bridges/LimiterMetrics.cs b/src/Craft.Contracts/LimiterMetrics.cs similarity index 100% rename from Services/Bridges/LimiterMetrics.cs rename to src/Craft.Contracts/LimiterMetrics.cs diff --git a/Services/Bridges/LogFileInfo.cs b/src/Craft.Contracts/LogFileInfo.cs similarity index 100% rename from Services/Bridges/LogFileInfo.cs rename to src/Craft.Contracts/LogFileInfo.cs diff --git a/Services/Bridges/MemoryBreakdown.cs b/src/Craft.Contracts/MemoryBreakdown.cs similarity index 100% rename from Services/Bridges/MemoryBreakdown.cs rename to src/Craft.Contracts/MemoryBreakdown.cs diff --git a/Services/Bridges/MemoryMetrics.cs b/src/Craft.Contracts/MemoryMetrics.cs similarity index 100% rename from Services/Bridges/MemoryMetrics.cs rename to src/Craft.Contracts/MemoryMetrics.cs diff --git a/Services/Bridges/PoolMetrics.cs b/src/Craft.Contracts/PoolMetrics.cs similarity index 100% rename from Services/Bridges/PoolMetrics.cs rename to src/Craft.Contracts/PoolMetrics.cs diff --git a/Services/PowerShellHost/ScriptResult.cs b/src/Craft.Contracts/ScriptResult.cs similarity index 85% rename from Services/PowerShellHost/ScriptResult.cs rename to src/Craft.Contracts/ScriptResult.cs index 4fbfe2c..b16b145 100644 --- a/Services/PowerShellHost/ScriptResult.cs +++ b/src/Craft.Contracts/ScriptResult.cs @@ -13,7 +13,7 @@ public class ScriptResult /// /// Response headers the handler asked for, already normalised by - /// . Null when the handler set none, + /// Craft.Hosting.HandlerHeaders.FromPowerShell. Null when the handler set none, /// which is the overwhelmingly common case — a redirect's Location is the reason this /// exists. /// @@ -21,7 +21,7 @@ public class ScriptResult /// /// Content type the handler asked for. Null means - /// . + /// Craft.Hosting.HandlerHeaders.DefaultContentType. /// public string? ContentType { get; set; } } diff --git a/Services/Bridges/StartupStats.cs b/src/Craft.Contracts/StartupStats.cs similarity index 100% rename from Services/Bridges/StartupStats.cs rename to src/Craft.Contracts/StartupStats.cs diff --git a/Services/Bridges/StatsDataPoint.cs b/src/Craft.Contracts/StatsDataPoint.cs similarity index 100% rename from Services/Bridges/StatsDataPoint.cs rename to src/Craft.Contracts/StatsDataPoint.cs diff --git a/Services/Bridges/WorkerDetail.cs b/src/Craft.Contracts/WorkerDetail.cs similarity index 100% rename from Services/Bridges/WorkerDetail.cs rename to src/Craft.Contracts/WorkerDetail.cs diff --git a/Services/Bridges/WorkerMetricsSnapshot.cs b/src/Craft.Contracts/WorkerMetricsSnapshot.cs similarity index 100% rename from Services/Bridges/WorkerMetricsSnapshot.cs rename to src/Craft.Contracts/WorkerMetricsSnapshot.cs diff --git a/Services/Bridges/WorkerSummary.cs b/src/Craft.Contracts/WorkerSummary.cs similarity index 100% rename from Services/Bridges/WorkerSummary.cs rename to src/Craft.Contracts/WorkerSummary.cs diff --git a/src/Craft/Craft.csproj b/src/Craft/Craft.csproj new file mode 100644 index 0000000..4ead018 --- /dev/null +++ b/src/Craft/Craft.csproj @@ -0,0 +1,86 @@ + + + + + net10.0 + Craft + Craft + + CyberDrain.Craft + + + + + + + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + diff --git a/src/Craft/Properties/launchSettings.json b/src/Craft/Properties/launchSettings.json new file mode 100644 index 0000000..09b6c7e --- /dev/null +++ b/src/Craft/Properties/launchSettings.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "Craft": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": false, + "applicationUrl": "http://localhost:8080", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/Runtime/CraftRuntime/Invoke-CraftPostExecution.ps1 b/src/Craft/Runtime/CraftRuntime/Invoke-CraftPostExecution.ps1 similarity index 100% rename from Runtime/CraftRuntime/Invoke-CraftPostExecution.ps1 rename to src/Craft/Runtime/CraftRuntime/Invoke-CraftPostExecution.ps1 diff --git a/Runtime/CraftRuntime/Invoke-CraftQueueTask.ps1 b/src/Craft/Runtime/CraftRuntime/Invoke-CraftQueueTask.ps1 similarity index 100% rename from Runtime/CraftRuntime/Invoke-CraftQueueTask.ps1 rename to src/Craft/Runtime/CraftRuntime/Invoke-CraftQueueTask.ps1 diff --git a/Runtime/CraftRuntime/Invoke-CraftTask.ps1 b/src/Craft/Runtime/CraftRuntime/Invoke-CraftTask.ps1 similarity index 100% rename from Runtime/CraftRuntime/Invoke-CraftTask.ps1 rename to src/Craft/Runtime/CraftRuntime/Invoke-CraftTask.ps1 diff --git a/Runtime/CraftRuntime/Start-CraftOrchestrator.ps1 b/src/Craft/Runtime/CraftRuntime/Start-CraftOrchestrator.ps1 similarity index 100% rename from Runtime/CraftRuntime/Start-CraftOrchestrator.ps1 rename to src/Craft/Runtime/CraftRuntime/Start-CraftOrchestrator.ps1 diff --git a/Runtime/HTTP/Exec/Invoke-ExecBackendProcess.ps1 b/src/Craft/Runtime/HTTP/Exec/Invoke-ExecBackendProcess.ps1 similarity index 100% rename from Runtime/HTTP/Exec/Invoke-ExecBackendProcess.ps1 rename to src/Craft/Runtime/HTTP/Exec/Invoke-ExecBackendProcess.ps1 diff --git a/Services/Auth/AuthService.cs b/src/Craft/Services/Auth/AuthService.cs similarity index 94% rename from Services/Auth/AuthService.cs rename to src/Craft/Services/Auth/AuthService.cs index fc5848e..983aaf3 100644 --- a/Services/Auth/AuthService.cs +++ b/src/Craft/Services/Auth/AuthService.cs @@ -13,9 +13,8 @@ namespace Craft.Auth; public class AuthService : IDisposable { private readonly ILogger _logger; - private readonly IConfiguration _config; private readonly CraftSettings _settings; - private readonly ICraftTableStore _store; + private readonly IUserTableStore _store; // allowedUsers cache private readonly ConcurrentDictionary _allowedUsersCache = new(StringComparer.OrdinalIgnoreCase); @@ -23,10 +22,9 @@ public class AuthService : IDisposable private readonly TimeSpan _allowedUsersCacheTtl = TimeSpan.FromMinutes(5); private readonly SemaphoreSlim _allowedUsersLock = new(1, 1); - public AuthService(ILogger logger, IConfiguration config, CraftSettings settings, ICraftTableStore store) + public AuthService(ILogger logger, CraftSettings settings, IUserTableStore store) { _logger = logger; - _config = config; _settings = settings; _store = store; } @@ -35,8 +33,8 @@ public AuthService(ILogger logger, IConfiguration config, CraftSett // IsConfigured reflects whether Azure App Service EasyAuth is set up: the platform sets // WEBSITE_AUTH_CLIENT_ID when its Microsoft identity provider is configured. - public bool IsConfigured => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("WEBSITE_AUTH_CLIENT_ID")) - || !string.IsNullOrEmpty(_config.GetValue("Auth:ClientId")); + public bool IsConfigured => + !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("WEBSITE_AUTH_CLIENT_ID")); // --- User Table --- diff --git a/Services/Auth/EasyAuthPrincipal.cs b/src/Craft/Services/Auth/EasyAuthPrincipal.cs similarity index 100% rename from Services/Auth/EasyAuthPrincipal.cs rename to src/Craft/Services/Auth/EasyAuthPrincipal.cs diff --git a/Services/Bridges/AppLifecycleBridge.cs b/src/Craft/Services/Bridges/AppLifecycleBridge.cs similarity index 83% rename from Services/Bridges/AppLifecycleBridge.cs rename to src/Craft/Services/Bridges/AppLifecycleBridge.cs index 054fb47..eb783d2 100644 --- a/Services/Bridges/AppLifecycleBridge.cs +++ b/src/Craft/Services/Bridges/AppLifecycleBridge.cs @@ -16,6 +16,11 @@ namespace Craft.Services; /// [Craft.Services.AppLifecycleBridge]::IsEasyAuthConfigured() /// [Craft.Services.AppLifecycleBridge]::ReconcileAuthPolicy("CIPP warmup") /// +/// +/// Uninitialized policy: mutating setup/restart APIs throw; best-effort reads +/// (, , …) soft no-op; +/// stays soft (warmup must not fail). +/// public static class AppLifecycleBridge { private static IHostApplicationLifetime? s_lifetime; @@ -45,16 +50,7 @@ public static void RequestRestart(string reason = "Restart requested by applicat /// Check whether EasyAuth is configured. Convenience method so downstream /// apps can check without needing a direct reference to SetupService. /// - public static bool IsEasyAuthConfigured() - { - var authEnabled = Environment.GetEnvironmentVariable("WEBSITE_AUTH_ENABLED"); - return string.Equals(authEnabled, "True", StringComparison.OrdinalIgnoreCase); - } - - // --- Setup mode gating --- - private static volatile bool s_setupModeRequested; - private static volatile bool s_setupCompleted; - private static string? s_setupCompletedReason; + public static bool IsEasyAuthConfigured() => SetupService.IsEasyAuthConfigured(); /// /// Explicitly enables the Craft setup wizard. Call this from the child app @@ -63,15 +59,16 @@ public static bool IsEasyAuthConfigured() /// public static void RequestSetupMode(string reason = "Setup mode requested by application") { - s_setupModeRequested = true; - s_logger?.LogWarning("[Lifecycle] Setup mode explicitly enabled: {Reason}", reason); + var svc = s_setupService ?? throw new InvalidOperationException("AppLifecycleBridge not initialized"); + svc.RequestSetupMode(reason); } /// /// Returns true if the child app has explicitly requested setup mode. /// Used by the setup middleware to determine whether to activate the setup wizard. /// - public static bool IsSetupModeRequested() => s_setupModeRequested; + public static bool IsSetupModeRequested() => + s_setupService?.IsSetupModeRequested() ?? false; /// /// Marks setup as completed — credentials have been applied and the app is @@ -80,20 +77,19 @@ public static void RequestSetupMode(string reason = "Setup mode requested by app /// public static void MarkSetupCompleted(string reason = "Setup credentials applied") { - s_setupCompleted = true; - s_setupCompletedReason = reason; - s_logger?.LogInformation("[Lifecycle] Setup marked as completed: {Reason}", reason); + var svc = s_setupService ?? throw new InvalidOperationException("AppLifecycleBridge not initialized"); + svc.MarkSetupCompleted(reason); } /// /// Returns true if setup credentials have already been applied this session. /// - public static bool IsSetupCompleted() => s_setupCompleted; + public static bool IsSetupCompleted() => s_setupService?.IsSetupCompleted() ?? false; /// /// Returns the reason setup was completed, or null if not yet completed. /// - public static string? GetSetupCompletedReason() => s_setupCompletedReason; + public static string? GetSetupCompletedReason() => s_setupService?.GetSetupCompletedReason(); /// /// Pushes the current Setup.UnauthenticatedClientAction and Setup.ExcludedPaths into the diff --git a/Services/Bridges/AuthBridge.cs b/src/Craft/Services/Bridges/AuthBridge.cs similarity index 87% rename from Services/Bridges/AuthBridge.cs rename to src/Craft/Services/Bridges/AuthBridge.cs index 739d29d..c739afc 100644 --- a/Services/Bridges/AuthBridge.cs +++ b/src/Craft/Services/Bridges/AuthBridge.cs @@ -12,6 +12,10 @@ namespace Craft.Services; /// Static bridge so PowerShell can trigger an auth-config reload without DI. /// Call [Craft.Services.AuthBridge]::ReloadAuth() from PS after credentials change. /// +/// +/// Uninitialized policy: reload/invalidate soft no-op when Auth is not registered +/// (e.g. Frontend-only or Background-only roles). Safe for warmups that always call reload. +/// public static class AuthBridge { private static AuthService? s_service; diff --git a/Services/Bridges/CacheBridge.cs b/src/Craft/Services/Bridges/CacheBridge.cs similarity index 94% rename from Services/Bridges/CacheBridge.cs rename to src/Craft/Services/Bridges/CacheBridge.cs index e1e3a4c..b10f7e9 100644 --- a/Services/Bridges/CacheBridge.cs +++ b/src/Craft/Services/Bridges/CacheBridge.cs @@ -16,6 +16,9 @@ namespace Craft.Services; /// [Craft.Services.CacheBridge]::InvalidateAll() /// [Craft.Services.CacheBridge]::GetStats() /// +/// +/// Uninitialized policy: all APIs throw (cache is always registered when the host runs). +/// public static class CacheBridge { private static CacheService? s_cache; diff --git a/src/Craft/Services/Bridges/LogBridge.cs b/src/Craft/Services/Bridges/LogBridge.cs new file mode 100644 index 0000000..16c1446 --- /dev/null +++ b/src/Craft/Services/Bridges/LogBridge.cs @@ -0,0 +1,133 @@ +using Craft.Hosting; + +// NAMESPACE PINNED — do not change. +// Downstream PowerShell reaches these types by fully-qualified name, e.g. +// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) +// Renaming the namespace compiles fine and then fails at runtime in the hosted app +// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. +// The folder is free to move; the namespace is a published contract. +namespace Craft.Services; + +/// +/// Static bridge exposing file log access to PowerShell and HTTP endpoints. +/// Provides filtered log reading, file listing, and log management. +/// +/// PS usage: +/// $lines = [Craft.Services.LogBridge]::ReadLog() # all from current log +/// $lines = [Craft.Services.LogBridge]::ReadLog(100) # last 100 lines +/// $lines = [Craft.Services.LogBridge]::ReadLog(100, 'ERR') # last 100 error lines +/// $lines = [Craft.Services.LogBridge]::ReadLog(100, 'ERR', 'timeout') # errors containing "timeout" +/// $lines = [Craft.Services.LogBridge]::ReadLog(0, $null, $null, 'craft.1.log') # from rotated file +/// $files = [Craft.Services.LogBridge]::GetLogFiles() # list all log files +/// $path = [Craft.Services.LogBridge]::GetCurrentLogPath() # active log path +/// $dir = [Craft.Services.LogBridge]::GetLogDirectory() # log directory path +/// $lines = [Craft.Services.LogBridge]::SearchLog('timeout') # search current log +/// $lines = [Craft.Services.LogBridge]::GetErrors(50) # last 50 errors +/// $lines = [Craft.Services.LogBridge]::GetLogsBetween($from, $to) # date range +/// $lines = [Craft.Services.LogBridge]::GetLogsSince([DateTime]::UtcNow.AddHours(-1)) # last hour +/// $lines = [Craft.Services.LogBridge]::GetLogsBetween($from, $to, 'ERR') # errors in range +/// [Craft.Services.LogBridge]::ForceRotation() # manually rotate now +/// $count = [Craft.Services.LogBridge]::PurgeOldFiles(7) # delete files older than 7 days +/// +/// +/// Uninitialized policy: read/query APIs soft no-op (empty path/array); +/// mutating APIs (, ) throw. +/// +public static class LogBridge +{ + private static LogQueryService? s_service; + + public static void Initialize(LogQueryService service) => s_service = service; + + /// Get the path to the currently active log file. + public static string GetCurrentLogPath() => s_service?.GetCurrentLogPath() ?? ""; + + /// Get the log directory path. + public static string GetLogDirectory() => s_service?.GetLogDirectory() ?? ""; + + /// Get metadata about all log files in the log directory. + public static LogFileInfo[] GetLogFiles() => + s_service?.GetLogFiles() ?? Array.Empty(); + + /// + /// Read log entries with optional filtering. + /// Continuation lines (exception details starting with whitespace) are kept with their parent entry. + /// + /// Number of matching lines to return from end (0 = all). + /// Filter by log level(s): "ERR" or "ERR,CRT" (comma-separated). + /// Case-insensitive text search within log messages. + /// Specific log file name (e.g. "craft.1.log"). Null = current. + /// Include only entries at or after this UTC time. Null = no lower bound. + /// Include only entries at or before this UTC time. Null = no upper bound. + /// Case-insensitive text to exclude from results. + /// Regex pattern to match against the message portion of the line. + /// When true, return results newest-first (default: false, oldest-first). + public static string[] ReadLog(int tail = 0, string? level = null, string? search = null, + string? file = null, DateTime? from = null, DateTime? to = null, + string? exclude = null, string? regexPattern = null, bool sortNewestFirst = false) => + s_service?.ReadLog(tail, level, search, file, from, to, exclude, regexPattern, sortNewestFirst) + ?? Array.Empty(); + + /// Search the current log for lines containing the specified text. + public static string[] SearchLog(string searchText, int tail = 0) => + s_service?.SearchLog(searchText, tail) ?? Array.Empty(); + + /// Get error-level entries from the current log. + public static string[] GetErrors(int tail = 0) => + s_service?.GetErrors(tail) ?? Array.Empty(); + + /// Get warning-level entries from the current log. + public static string[] GetWarnings(int tail = 0) => + s_service?.GetWarnings(tail) ?? Array.Empty(); + + /// + /// Get log entries within a UTC date/time range, optionally filtered by level and search text. + /// PS: [Craft.Services.LogBridge]::GetLogsBetween([DateTime]'2026-05-13 08:00', [DateTime]'2026-05-13 12:00') + /// + public static string[] GetLogsBetween(DateTime from, DateTime to, string? level = null, string? search = null, string? file = null) => + s_service?.GetLogsBetween(from, to, level, search, file) ?? Array.Empty(); + + /// + /// Get log entries from a UTC start time to now. + /// PS: [Craft.Services.LogBridge]::GetLogsSince([DateTime]::UtcNow.AddHours(-1)) + /// + public static string[] GetLogsSince(DateTime from, string? level = null, string? search = null, string? file = null) => + s_service?.GetLogsSince(from, level, search, file) ?? Array.Empty(); + + /// + /// Get log entries from the last N minutes. + /// PS: [Craft.Services.LogBridge]::GetRecentLogs(30) # last 30 minutes + /// [Craft.Services.LogBridge]::GetRecentLogs(30, 'ERR') # errors in last 30 min + /// + public static string[] GetRecentLogs(int minutes, string? level = null, string? search = null, string? file = null) => + s_service?.GetRecentLogs(minutes, level, search, file) ?? Array.Empty(); + + /// + /// Search across ALL log files (current + rotated) for matching entries. + /// Returns results ordered oldest-to-newest. Useful for investigating issues across rotations. + /// PS: [Craft.Services.LogBridge]::SearchAllFiles('timeout', 'ERR') + /// + public static string[] SearchAllFiles(string? search = null, string? level = null, + DateTime? from = null, DateTime? to = null, int tail = 0, + string? exclude = null, string? regexPattern = null, bool sortNewestFirst = false) => + s_service?.SearchAllFiles(search, level, from, to, tail, exclude, regexPattern, sortNewestFirst) + ?? Array.Empty(); + + /// Manually trigger log rotation on the current file. + public static void ForceRotation() + { + var service = s_service ?? throw new InvalidOperationException("LogBridge not initialized"); + service.ForceRotation(); + } + + /// + /// Delete rotated log files older than the specified number of days. + /// The current active log file is never deleted. + /// + /// Number of files deleted. + public static int PurgeOldFiles(int olderThanDays = 7) + { + var service = s_service ?? throw new InvalidOperationException("LogBridge not initialized"); + return service.PurgeOldFiles(olderThanDays); + } +} diff --git a/src/Craft/Services/Bridges/OrchestratorBridge.cs b/src/Craft/Services/Bridges/OrchestratorBridge.cs new file mode 100644 index 0000000..9deb0ea --- /dev/null +++ b/src/Craft/Services/Bridges/OrchestratorBridge.cs @@ -0,0 +1,54 @@ +using Craft.Hosting; +using Craft.Orchestration; + +// NAMESPACE PINNED — do not change. +// Downstream PowerShell reaches these types by fully-qualified name, e.g. +// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) +// Renaming the namespace compiles fine and then fails at runtime in the hosted app +// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. +// The folder is free to move; the namespace is a published contract. +namespace Craft.Services; + +/// +/// Thread-safe bridge allowing PowerShell (Start-CIPPOrchestrator) to queue +/// orchestrator runs that get picked up by the C# OrchestratorService. +/// PS enqueues via QueueOrchestration(); C# drains via DrainPending(). +/// +/// +/// Uninitialized policy: mutating enqueue APIs throw; drain calls soft no-op +/// (empty work when the host has not wired the bridge yet). +/// +public static class OrchestratorBridge +{ + private static OrchestratorService? s_service; + + public static void Initialize(OrchestratorService service) => s_service = service; + + public static void QueueOrchestration(string name, string batchJson, int priority, + string? postExecFunctionName = null, string? postExecParametersJson = null, + string? reference = null) + { + var service = s_service ?? throw new InvalidOperationException("OrchestratorBridge not initialized"); + var parentRunName = OperationContext.Current?.RunName; + service.QueueOrchestration(name, batchJson, priority, + postExecFunctionName, postExecParametersJson, parentRunName, reference); + } + + /// + /// Synchronous drain — blocks until all pending orchestrations are started. + /// Safe to call from any context (no SynchronizationContext on background workers). + /// + public static void DrainPending() => s_service?.DrainPending(); + + /// + /// Async drain — preferred from async call sites (PostExec lambdas, ExecuteScript). + /// + public static Task DrainPendingAsync() => + s_service?.DrainPendingAsync() ?? Task.CompletedTask; + + public static void QueuePlannerRun(string command, int priority) + { + var service = s_service ?? throw new InvalidOperationException("OrchestratorBridge not initialized"); + service.QueuePlannerRun(command, priority); + } +} diff --git a/src/Craft/Services/Bridges/QueueBridge.cs b/src/Craft/Services/Bridges/QueueBridge.cs new file mode 100644 index 0000000..b30fb52 --- /dev/null +++ b/src/Craft/Services/Bridges/QueueBridge.cs @@ -0,0 +1,33 @@ +using Craft.Orchestration; + +// NAMESPACE PINNED — do not change. +// Downstream PowerShell reaches these types by fully-qualified name, e.g. +// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) +// Renaming the namespace compiles fine and then fails at runtime in the hosted app +// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. +// The folder is free to move; the namespace is a published contract. +namespace Craft.Services; + +/// +/// Thread-safe bridge allowing PowerShell (Add-CippQueueMessage) to queue +/// background commands that get dispatched on a background worker. +/// Replaces Azure Storage Queue on CIPPNG — purely in-process. +/// +/// +/// Uninitialized policy: throws; soft no-ops. +/// +public static class QueueBridge +{ + private static QueueDispatchService? s_dispatch; + + internal static void Initialize(QueueDispatchService dispatch) => s_dispatch = dispatch; + + public static void Enqueue(string cmdlet, string parametersJson) + { + var dispatch = s_dispatch ?? throw new InvalidOperationException("QueueBridge not initialized"); + dispatch.Enqueue(cmdlet, parametersJson); + } + + public static void DrainPending() => + s_dispatch?.DrainPending(); +} diff --git a/src/Craft/Services/Bridges/QueueStatusBridge.cs b/src/Craft/Services/Bridges/QueueStatusBridge.cs new file mode 100644 index 0000000..cab172d --- /dev/null +++ b/src/Craft/Services/Bridges/QueueStatusBridge.cs @@ -0,0 +1,43 @@ +using Craft.Orchestration; + +// NAMESPACE PINNED — do not change. +// Downstream PowerShell reaches these types by fully-qualified name, e.g. +// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) +// Renaming the namespace compiles fine and then fails at runtime in the hosted app +// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. +// The folder is free to move; the namespace is a published contract. +namespace Craft.Services; + +/// +/// Static bridge allowing PowerShell (Get-CIPPQueueData) to query orchestrator/job +/// progress without HTTP round-trips. Returns data in the shape the CIPP frontend expects. +/// PS usage: [Craft.Services.QueueStatusBridge]::GetRunStatus($Reference, $QueueId) +/// +/// +/// Uninitialized policy: status reads soft no-op (empty JSON / empty list); +/// is safe before Initialize (metadata bag is static). +/// +public static class QueueStatusBridge +{ + private static QueueStatusService? s_service; + + public static void Initialize(QueueStatusService service) => s_service = service; + + /// + /// Register friendly queue metadata from PowerShell (New-CippQueueEntry). + /// PS usage: [Craft.Services.QueueStatusBridge]::RegisterQueueMetadata($QueueId, $Name, $Link, $Reference) + /// + public static void RegisterQueueMetadata(string queueId, string name, string link, string reference) => + QueueStatusService.RegisterQueueMetadata(queueId, name, link, reference); + + /// + /// Get queue/run status in the format expected by the CIPP frontend. + /// Looks up by run name (Reference) or returns all recent runs. + /// Returns a JSON string matching the Get-CIPPQueueData output shape. + /// + /// Optional run reference/name to filter by (maps to RunName in JobManager) + /// Optional queue ID (same as reference in Craft context) + /// JSON array of queue status objects + public static string GetRunStatus(string? reference = null, string? queueId = null) => + s_service?.GetRunStatus(reference, queueId) ?? "[]"; +} diff --git a/Services/Bridges/RealtimeBridge.cs b/src/Craft/Services/Bridges/RealtimeBridge.cs similarity index 85% rename from Services/Bridges/RealtimeBridge.cs rename to src/Craft/Services/Bridges/RealtimeBridge.cs index acbd176..6e96625 100644 --- a/Services/Bridges/RealtimeBridge.cs +++ b/src/Craft/Services/Bridges/RealtimeBridge.cs @@ -19,9 +19,12 @@ namespace Craft.Services; /// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, "update", @{ done = 142; total = 300 }) /// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, "end", @{ done = 300; total = 300 }) /// -/// Only userId and jobId (a GUID) are required; everything else is optional. Every call is -/// best-effort and never throws back to the caller. +/// Only userId and jobId (a GUID) are required; everything else is optional. /// +/// +/// Uninitialized policy: publish is best-effort — soft no-op when Realtime is not wired, +/// and never throws back to the caller. +/// public static class RealtimeBridge { private static RealtimeService? s_service; @@ -48,7 +51,9 @@ public static void Publish(string userId, string jobId, string? mode, object? da { try { - s_service?.Publish(userId, jobId, mode, data, urlHref, urlLabel, status, message); + // Unwrap PSObject / Hashtable before the service so RealtimeService stays CLR-only. + s_service?.Publish(userId, jobId, mode, RealtimePayloadNormalizer.Normalize(data), + urlHref, urlLabel, status, message); } catch { diff --git a/src/Craft/Services/Bridges/RealtimePayloadNormalizer.cs b/src/Craft/Services/Bridges/RealtimePayloadNormalizer.cs new file mode 100644 index 0000000..43b23a7 --- /dev/null +++ b/src/Craft/Services/Bridges/RealtimePayloadNormalizer.cs @@ -0,0 +1,37 @@ +using System.Collections; +using System.Management.Automation; + +namespace Craft.Services; + +/// +/// Converts PowerShell payloads (PSObject / Hashtable / PS collections) into CLR shapes +/// can serialize. Called by +/// before publish so the service stays free of the PowerShell SDK. +/// +internal static class RealtimePayloadNormalizer +{ + public static object? Normalize(object? v) => v switch + { + null => null, + string or bool or int or long or double or float or decimal or DateTime or DateTimeOffset or Guid => v, + PSObject ps => Normalize(ps.BaseObject), + IDictionary d => NormalizeDict(d), + IEnumerable e => NormalizeList(e), + _ => v.ToString() + }; + + private static Dictionary NormalizeDict(IDictionary d) + { + var r = new Dictionary(StringComparer.Ordinal); + foreach (DictionaryEntry e in d) + r[e.Key?.ToString() ?? ""] = Normalize(e.Value); + return r; + } + + private static List NormalizeList(IEnumerable e) + { + var r = new List(); + foreach (var i in e) r.Add(Normalize(i)); + return r; + } +} diff --git a/Services/Bridges/SchedulerBridge.cs b/src/Craft/Services/Bridges/SchedulerBridge.cs similarity index 91% rename from Services/Bridges/SchedulerBridge.cs rename to src/Craft/Services/Bridges/SchedulerBridge.cs index df759b6..4c86b1b 100644 --- a/Services/Bridges/SchedulerBridge.cs +++ b/src/Craft/Services/Bridges/SchedulerBridge.cs @@ -13,6 +13,9 @@ namespace Craft.Services; /// PS usage: [Craft.Services.SchedulerBridge]::SetTimezone("America/New_York") /// [Craft.Services.SchedulerBridge]::GetTimezone() /// +/// +/// Uninitialized policy: throws; soft no-ops to "". +/// public static class SchedulerBridge { private static SchedulerService? s_service; diff --git a/src/Craft/Services/Bridges/StartupInfoBridge.cs b/src/Craft/Services/Bridges/StartupInfoBridge.cs new file mode 100644 index 0000000..272598b --- /dev/null +++ b/src/Craft/Services/Bridges/StartupInfoBridge.cs @@ -0,0 +1,33 @@ +using Craft.Hosting; + +// NAMESPACE PINNED — do not change. +// Downstream PowerShell reaches these types by fully-qualified name, e.g. +// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) +// Renaming the namespace compiles fine and then fails at runtime in the hosted app +// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. +// The folder is free to move; the namespace is a published contract. +namespace Craft.Services; + +/// +/// Static bridge exposing container startup metrics to PowerShell and HTTP endpoints. +/// Populated during pool initialization. Read-only after startup completes. +/// +/// PS usage: +/// $info = [Craft.Services.StartupInfoBridge]::GetInfo() +/// $info.HttpReadyMs # time in ms until first HTTP worker was ready +/// $info.IsFullyReady # true once all pools are done +/// $info.Phase # current phase: "Starting", "HttpReady", "Ready" +/// +/// +/// Uninitialized policy: throws (startup progress is always wired early). +/// +public static class StartupInfoBridge +{ + private static StartupProgressService? s_progress; + + internal static void Initialize(StartupProgressService progress) => s_progress = progress; + + /// Get the current startup statistics snapshot. + public static StartupStats GetInfo() => + s_progress?.Stats ?? throw new InvalidOperationException("StartupInfoBridge not initialized"); +} diff --git a/Services/Bridges/StatsHistoryBridge.cs b/src/Craft/Services/Bridges/StatsHistoryBridge.cs similarity index 91% rename from Services/Bridges/StatsHistoryBridge.cs rename to src/Craft/Services/Bridges/StatsHistoryBridge.cs index ac3a388..a93f2f6 100644 --- a/Services/Bridges/StatsHistoryBridge.cs +++ b/src/Craft/Services/Bridges/StatsHistoryBridge.cs @@ -18,6 +18,9 @@ namespace Craft.Services; /// $allJson = [Craft.Services.StatsHistoryBridge]::GetHistoryJson(10080, 500) # 7 days, max 500 points /// $count = [Craft.Services.StatsHistoryBridge]::GetCount() /// +/// +/// Uninitialized policy: reads soft no-op (empty list / 0) when Background roles are off. +/// public static class StatsHistoryBridge { private static StatsHistoryService? s_service; @@ -28,7 +31,7 @@ public static class StatsHistoryBridge WriteIndented = false, }; - public static void Initialize(StatsHistoryService service) => s_service = service; + internal static void Initialize(StatsHistoryService service) => s_service = service; /// Get history for the last N minutes, optionally downsampled. public static List GetHistory(int lastMinutes = 60, int? maxPoints = null) diff --git a/src/Craft/Services/Bridges/WorkerMetricsBridge.cs b/src/Craft/Services/Bridges/WorkerMetricsBridge.cs new file mode 100644 index 0000000..84de6d0 --- /dev/null +++ b/src/Craft/Services/Bridges/WorkerMetricsBridge.cs @@ -0,0 +1,94 @@ +using Craft.Hosting; + +// NAMESPACE PINNED — do not change. +// Downstream PowerShell reaches these types by fully-qualified name, e.g. +// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) +// Renaming the namespace compiles fine and then fails at runtime in the hosted app +// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. +// The folder is free to move; the namespace is a published contract. +namespace Craft.Services; + +/// +/// Static bridge exposing worker pool metrics and utilization data to PowerShell. +/// Domain code injects instead of calling these statics. +/// +/// PS usage: +/// $metrics = [Craft.Services.WorkerMetricsBridge]::GetSnapshot() +/// $metrics.HttpPool.BusyCount +/// $metrics.HttpPool.Workers[0].TotalInvocations +/// $metrics.BgPool.Workers +/// $metrics.Limiter.IsHttpThrottled +/// $metrics.Jobs.Running +/// +/// +/// Uninitialized policy: all APIs throw (metrics require the PowerShell graph). +/// +public static class WorkerMetricsBridge +{ + private static WorkerMetricsService? s_metrics; + + internal static void Initialize(WorkerMetricsService metrics) => s_metrics = metrics; + + private static WorkerMetricsService Require() => + s_metrics ?? throw new InvalidOperationException("WorkerMetricsBridge not initialized"); + + /// Pre-register a worker so it appears in snapshots even before first use. + internal static void RegisterWorker(int workerId, bool isHttp) => + Require().RegisterWorker(workerId, isHttp); + + /// Remove a worker's stats when it is recycled/replaced. + internal static void DeregisterWorker(int workerId) => Require().DeregisterWorker(workerId); + + /// Record that a worker was checked out (started processing). + internal static void RecordCheckout(int workerId, bool isHttp) => + Require().RecordCheckout(workerId, isHttp); + + /// Record that a worker was reclaimed (finished processing). + internal static void RecordReclaim(int workerId, bool faulted, long elapsedMs) => + Require().RecordReclaim(workerId, faulted, elapsedMs); + + /// Record the function name being executed on a worker. + internal static void RecordFunction(int workerId, string functionName) => + Require().RecordFunction(workerId, functionName); + + /// Get a full snapshot of all worker metrics. + public static WorkerMetricsSnapshot GetSnapshot() => Require().GetSnapshot(); + + /// + /// Get a detailed memory breakdown including per-generation heap sizes, LOH/POH, + /// fragmentation, pinned objects, thread info, loaded assemblies, and native memory. + /// + public static MemoryBreakdown GetMemoryBreakdown() => Require().GetMemoryBreakdown(); + + /// Get metrics for a specific pool type ("http" or "bg"). + public static PoolMetrics? GetPoolMetrics(string poolType) => Require().GetPoolMetrics(poolType); + + /// Get a summary of just the busy/available counts. + public static WorkerSummary GetSummary() => Require().GetSummary(); + + /// Get detailed job list with wait/duration times. + public static List GetJobDetails(string? runName = null, string? status = null, int limit = 100) => + Require().GetJobDetails(runName, status, limit); + + /// Get run group summaries. + public static List GetRunSummaries() => Require().GetRunSummaries(); + + /// Cancel a single queued job by ID. + public static bool CancelJob(string jobId) => Require().CancelJob(jobId); + + /// Cancel all queued jobs in a run group. + public static int CancelRun(string runName) => Require().CancelRun(runName); + + /// Delete a completed/failed/cancelled job from tracking. + public static bool DeleteJob(string jobId) => Require().DeleteJob(jobId); + + /// Change a queued job's priority (re-enqueues with new priority). + public static bool ChangePriority(string jobId, int newPriority) => + Require().ChangePriority(jobId, newPriority); + + /// + /// Force a full GC collection with LOH compaction and working-set trim. + /// Returns the MB reclaimed, or -1 if skipped due to cooldown. + /// + public static long TrimMemory() => Require().TrimMemory(); +} diff --git a/Services/Caching/CacheEntry.cs b/src/Craft/Services/Caching/CacheEntry.cs similarity index 98% rename from Services/Caching/CacheEntry.cs rename to src/Craft/Services/Caching/CacheEntry.cs index bc285c0..17331dc 100644 --- a/Services/Caching/CacheEntry.cs +++ b/src/Craft/Services/Caching/CacheEntry.cs @@ -8,7 +8,7 @@ namespace Craft.Caching; // it under the memory-tier guard. volatile is not available on properties. [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1051:Do not declare visible instance fields", Justification = "Body is volatile for lock-free reads on the cache hot path; volatile is not available on properties.")] -public class CacheEntry +internal sealed class CacheEntry { public int StatusCode { get; set; } public DateTime CachedAt { get; set; } diff --git a/Services/Hosting/CacheProfiler.cs b/src/Craft/Services/Caching/CacheProfiler.cs similarity index 99% rename from Services/Hosting/CacheProfiler.cs rename to src/Craft/Services/Caching/CacheProfiler.cs index 74a6d00..6b07b1c 100644 --- a/Services/Hosting/CacheProfiler.cs +++ b/src/Craft/Services/Caching/CacheProfiler.cs @@ -1,6 +1,6 @@ using System.Diagnostics; -namespace Craft.Hosting; +namespace Craft.Caching; /// /// Opt-in profiler for the inbound-request auth header transform + the disk-backed response cache, for perf diff --git a/Services/Caching/CacheService.cs b/src/Craft/Services/Caching/CacheService.cs similarity index 99% rename from Services/Caching/CacheService.cs rename to src/Craft/Services/Caching/CacheService.cs index b668a8c..5f5c9d1 100644 --- a/Services/Caching/CacheService.cs +++ b/src/Craft/Services/Caching/CacheService.cs @@ -4,7 +4,6 @@ using System.Text; using System.Text.Json; using Craft.Configuration; -using Craft.Hosting; using Craft.Services; namespace Craft.Caching; diff --git a/Services/Caching/CachedResponse.cs b/src/Craft/Services/Caching/CachedResponse.cs similarity index 100% rename from Services/Caching/CachedResponse.cs rename to src/Craft/Services/Caching/CachedResponse.cs diff --git a/Services/Caching/ResponseCachePolicy.cs b/src/Craft/Services/Caching/ResponseCachePolicy.cs similarity index 100% rename from Services/Caching/ResponseCachePolicy.cs rename to src/Craft/Services/Caching/ResponseCachePolicy.cs diff --git a/Services/Endpoints/CraftEndpointAttribute.cs b/src/Craft/Services/Endpoints/CraftEndpointAttribute.cs similarity index 100% rename from Services/Endpoints/CraftEndpointAttribute.cs rename to src/Craft/Services/Endpoints/CraftEndpointAttribute.cs diff --git a/Services/Endpoints/CraftJson.cs b/src/Craft/Services/Endpoints/CraftJson.cs similarity index 100% rename from Services/Endpoints/CraftJson.cs rename to src/Craft/Services/Endpoints/CraftJson.cs diff --git a/Services/Endpoints/CraftRequest.cs b/src/Craft/Services/Endpoints/CraftRequest.cs similarity index 100% rename from Services/Endpoints/CraftRequest.cs rename to src/Craft/Services/Endpoints/CraftRequest.cs diff --git a/Services/Endpoints/CraftResult.cs b/src/Craft/Services/Endpoints/CraftResult.cs similarity index 100% rename from Services/Endpoints/CraftResult.cs rename to src/Craft/Services/Endpoints/CraftResult.cs diff --git a/Services/Endpoints/ICraftEndpoint.cs b/src/Craft/Services/Endpoints/ICraftEndpoint.cs similarity index 100% rename from Services/Endpoints/ICraftEndpoint.cs rename to src/Craft/Services/Endpoints/ICraftEndpoint.cs diff --git a/Services/Endpoints/ICraftScheduledTask.cs b/src/Craft/Services/Endpoints/ICraftScheduledTask.cs similarity index 100% rename from Services/Endpoints/ICraftScheduledTask.cs rename to src/Craft/Services/Endpoints/ICraftScheduledTask.cs diff --git a/Services/Endpoints/NativeEndpointRegistry.cs b/src/Craft/Services/Endpoints/NativeEndpointRegistry.cs similarity index 100% rename from Services/Endpoints/NativeEndpointRegistry.cs rename to src/Craft/Services/Endpoints/NativeEndpointRegistry.cs diff --git a/Services/Hosting/ContainerHealthMonitor.cs b/src/Craft/Services/Hosting/ContainerHealthMonitor.cs similarity index 100% rename from Services/Hosting/ContainerHealthMonitor.cs rename to src/Craft/Services/Hosting/ContainerHealthMonitor.cs diff --git a/Services/Hosting/CorsPreflightMiddleware.cs b/src/Craft/Services/Hosting/CorsPreflightMiddleware.cs similarity index 100% rename from Services/Hosting/CorsPreflightMiddleware.cs rename to src/Craft/Services/Hosting/CorsPreflightMiddleware.cs diff --git a/Services/Hosting/CraftAuthMiddleware.cs b/src/Craft/Services/Hosting/CraftAuthMiddleware.cs similarity index 99% rename from Services/Hosting/CraftAuthMiddleware.cs rename to src/Craft/Services/Hosting/CraftAuthMiddleware.cs index fa4b4bf..73ed4c9 100644 --- a/Services/Hosting/CraftAuthMiddleware.cs +++ b/src/Craft/Services/Hosting/CraftAuthMiddleware.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using Craft.Auth; +using Craft.Caching; using Craft.Configuration; namespace Craft.Hosting; @@ -20,7 +21,7 @@ namespace Craft.Hosting; /// and authorises; it never issues or validates a token. /// /// -public static class CraftAuthMiddleware +internal static class CraftAuthMiddleware { /// Registers the principal-normalising middleware. public static WebApplication UseCraftAuth( diff --git a/src/Craft/Services/Hosting/CraftHostBuilderExtensions.cs b/src/Craft/Services/Hosting/CraftHostBuilderExtensions.cs new file mode 100644 index 0000000..f81565a --- /dev/null +++ b/src/Craft/Services/Hosting/CraftHostBuilderExtensions.cs @@ -0,0 +1,339 @@ +using System.Globalization; +using System.IO.Compression; +using System.Threading.RateLimiting; +using Craft.Auth; +using Craft.Caching; +using Craft.Configuration; +using Craft.Endpoints; +using Craft.Orchestration; +using Craft.PowerShellHost; +using Craft.Realtime; +using Craft.Services; +using Craft.Setup; +using Craft.Storage; +using Microsoft.AspNetCore.RateLimiting; +using Microsoft.AspNetCore.ResponseCompression; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.Extensions.Logging.Console; +using Microsoft.Extensions.Options; + +namespace Craft.Hosting; + +/// +/// Host wiring, split out of Program.cs so startup reads as a short sequence of named steps +/// rather than several hundred lines of inline configuration. +/// +internal static class CraftHostBuilderExtensions +{ + /// + /// Resolves the Kestrel request timeout in seconds: an explicit KestrelTimeoutSeconds wins, + /// otherwise it derives from Worker.HttpTimeoutSeconds, otherwise 600s. + /// + /// + /// Deriving from the worker timeout matters: if Kestrel gives up before the PowerShell worker does, + /// the caller sees a connection abort while the script keeps running and holding a runspace. + /// + public static int ResolveKestrelTimeoutSeconds(CraftSettings settings) + { + ArgumentNullException.ThrowIfNull(settings); + + var timeout = settings.KestrelTimeoutSeconds; + if (timeout > 0) return timeout; + + return settings.Worker.HttpTimeoutSeconds > 0 ? settings.Worker.HttpTimeoutSeconds : 600; + } + + /// + /// Resolves the .NET thread-pool minimum: an explicit Worker:MinThreads (or + /// CRAFT_MIN_THREADS) wins, otherwise it is derived from the worker pools. + /// + /// + /// The derived floor is HttpPoolSize + BgPoolSize + 16, never below the old + /// max(ProcessorCount * 4, 32). PowerShell blocks a thread for every outbound call, so a + /// pool larger than the minimum pays a one-thread-per-second injection ramp on every restart. + /// + public static int ResolveMinThreads(CraftSettings settings) + { + ArgumentNullException.ThrowIfNull(settings); + + if (int.TryParse(Environment.GetEnvironmentVariable("CRAFT_MIN_THREADS"), out var fromEnv) && fromEnv > 0) + return fromEnv; + + if (settings.Worker.MinThreads > 0) return settings.Worker.MinThreads; + + var baseline = Math.Max(Environment.ProcessorCount * 4, 32); + var forPools = settings.Worker.HttpPoolSize + settings.Worker.BgPoolSize + 16; + return Math.Max(baseline, forPools); + } + + /// + /// Kestrel limits from the Options-bound (no pre-Build dual bind). + /// Also applies from configuration (must run before Build). + /// + public static WebApplicationBuilder ConfigureCraftKestrel(this WebApplicationBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + // Thread-pool minimum must be set before traffic; bind App:Worker early for this only. + var early = new CraftSettings(); + builder.Configuration.GetSection("App").Bind(early); + var minThreads = ResolveMinThreads(early); + ThreadPool.SetMinThreads(minThreads, minThreads); + + builder.Services.AddOptions() + .Configure>((options, craft) => + { + var settings = craft.Value; + var timeout = ResolveKestrelTimeoutSeconds(settings); + + options.Limits.KeepAliveTimeout = TimeSpan.FromSeconds(timeout); + options.Limits.RequestHeadersTimeout = TimeSpan.FromSeconds(Math.Min(60, timeout)); + + options.Limits.Http2.MaxStreamsPerConnection = 100; + options.Limits.Http2.HeaderTableSize = 4096; + options.Limits.Http2.MaxFrameSize = 16384; + options.Limits.Http2.MaxRequestHeaderFieldSize = 8192; + options.Limits.Http2.InitialConnectionWindowSize = 131072; + options.Limits.Http2.InitialStreamWindowSize = 98304; + + var maxBodyMb = settings.Limits.MaxRequestBodyMB; + options.Limits.MaxRequestBodySize = maxBodyMb > 0 ? maxBodyMb * 1024L * 1024L : null; + + var maxConn = settings.Limits.MaxConcurrentConnections; + options.Limits.MaxConcurrentConnections = maxConn > 0 ? maxConn : null; + options.Limits.MaxConcurrentUpgradedConnections = maxConn > 0 ? maxConn : null; + + options.Limits.MinRequestBodyDataRate = + new MinDataRate(bytesPerSecond: 240, gracePeriod: TimeSpan.FromSeconds(5)); + options.Limits.MinResponseDataRate = + new MinDataRate(bytesPerSecond: 240, gracePeriod: TimeSpan.FromSeconds(5)); + }); + + return builder; + } + + /// + /// File logging with rotation plus a timestamped console sink, both honouring the configured level + /// (App:FileLogging:LogLevel, overridable with CRAFT_LOG_LEVEL). + /// + /// + /// Logging must work before Build(), so this binds App:FileLogging early for the + /// provider. After Build, call so rotation/format knobs + /// match the Options-bound CraftSettings.FileLogging (one logical source post-Build). + /// Directory/prefix stay as opened at construction. + /// + /// + /// The resolved level and the file provider (needed for the post-Build sync). Startup logs the + /// level, and it also gates PowerShell stream capture — at Debug, Write-Debug is captured; at + /// Trace, Write-Verbose as well. + /// + public static (LogLevel Level, FileLoggerProvider FileProvider) AddCraftLogging(this WebApplicationBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + // Temporary early bind so logging works during Build; SyncFileLoggingFromOptions aligns + // mutable knobs with CraftSettings.FileLogging after Options bind. + var fileLoggingSettings = new FileLoggingSettings(); + builder.Configuration.GetSection("App:FileLogging").Bind(fileLoggingSettings); + var level = fileLoggingSettings.ParsedLogLevel; + + var fileLoggerProvider = new FileLoggerProvider(fileLoggingSettings, level); + builder.Logging.AddProvider(fileLoggerProvider); + LogBridge.Initialize(new LogQueryService(fileLoggerProvider)); + + builder.Logging.AddSimpleConsole(options => + { + options.TimestampFormat = "yyyy-MM-ddTHH:mm:ss.fffZ "; + options.SingleLine = true; + }); + + if (level > LogLevel.Debug) + { + builder.Logging.AddFilter(l => l >= LogLevel.Information); + + builder.Logging.AddFilter("Microsoft.AspNetCore", LogLevel.Warning); + builder.Logging.AddFilter("Microsoft.Hosting", LogLevel.Warning); + builder.Logging.AddFilter("Microsoft.Extensions.Hosting", LogLevel.Warning); + } + + return (level, fileLoggerProvider); + } + + /// + /// After Build(), refresh the file logger from Options-bound + /// so AddCraftLogging's early bind does not diverge. + /// + public static void SyncFileLoggingFromOptions(this WebApplication app, FileLoggerProvider provider) + { + ArgumentNullException.ThrowIfNull(app); + ArgumentNullException.ThrowIfNull(provider); + + var settings = app.Services.GetRequiredService().FileLogging; + provider.SyncFrom(settings); + } + + private static readonly string[] second = new[] { "application/json", "text/json", "application/javascript", "text/javascript" }; + + /// Response compression, matching Azure Static Web Apps behaviour. + public static IServiceCollection AddCraftResponseCompression(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.AddResponseCompression(options => + { + options.EnableForHttps = true; + options.Providers.Add(); + options.Providers.Add(); + options.MimeTypes = ResponseCompressionDefaults.MimeTypes.Concat( + second); + }); + + services.Configure(o => o.Level = CompressionLevel.Fastest); + services.Configure(o => o.Level = CompressionLevel.Fastest); + + return services; + } + + /// + /// Registers the Craft service graph gated by deployment roles. Frontend-only nodes skip the + /// PowerShell / orchestration / auth-store graph. + /// + public static IServiceCollection AddCraftServices(this IServiceCollection services, CraftRoles roles) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(roles); + + services.AddSingleton(); + services.AddSingleton(sp => + { + var health = sp.GetRequiredService>().Value.ContainerHealth; + var logger = sp.GetRequiredService().CreateLogger(); + return new ContainerHealthMonitor(logger, health); + }); + + // Realtime is mapped for Http or Frontend; cheap when disabled. + if (roles.Http || roles.Frontend) + services.AddSingleton(); + + services.AddSingleton(sp => new CacheService( + sp.GetRequiredService>(), + sp.GetRequiredService(), + roles.ResponseCacheEnabled)); + + if (!roles.RunsPowerShell) + return services; + + // Shared host tables (orchestrator, health). Auth override must not bleed into these. + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => + { + var settings = sp.GetRequiredService(); + if (string.IsNullOrWhiteSpace(settings.Auth.UserStorageConnection)) + return sp.GetRequiredService(); + return new AzureTableStore(settings, settings.Auth.UserStorageConnection, "allowedUsers table"); + }); + services.AddSingleton(); + services.AddSingleton(); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + services.AddSingleton(); + services.AddSingleton(sp => + sp.GetRequiredService().ResolveTaskWorkAsync); + services.AddSingleton(sp => + sp.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + // Empty here; AddNativeEndpoints replaces this when the app ships native scheduled tasks. + services.AddSingleton(NativeScheduledTasks.Empty); + services.AddSingleton(); + services.AddSingleton(); + + if (roles.Http) + { + services.AddSingleton(); + } + + // Setup/AppLifecycle is used from warmup PS on any PowerShell node. + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + // Shutdown flush for PS ingress + tracked drain/planner work (Http and/or Background). + services.AddHostedService(); + + if (roles.Background) + { + services.AddHostedService(sp => sp.GetRequiredService()); + services.AddHostedService(sp => sp.GetRequiredService()); + services.AddHostedService(sp => sp.GetRequiredService()); + } + + return services; + } + + /// + /// Seconds to advertise in Retry-After on a throttled response. Prefers the limiter's own + /// estimate of when a permit next frees up, falling back to the whole window — a safe upper bound + /// for a fixed window, and the only figure available when the lease carries no metadata. + /// + public static int ResolveRetryAfterSeconds(RateLimitLease lease, TimeSpan window) + { + ArgumentNullException.ThrowIfNull(lease); + + var retryAfter = lease.TryGetMetadata(MetadataName.RetryAfter, out var metadata) + ? metadata + : window; + + return Math.Max(1, (int)Math.Ceiling(retryAfter.TotalSeconds)); + } + + /// + /// Per-client fixed-window rate limiter from Options-bound settings. Registers always; when + /// disabled the limiter is a no-op partition that never rejects. + /// + public static IServiceCollection AddCraftRateLimiter(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.AddRateLimiter(); + services.AddOptions() + .Configure>((options, craft) => + { + var settings = craft.Value; + if (!settings.RateLimit.IsEnabled) return; + + var window = TimeSpan.FromSeconds(Math.Max(1, settings.RateLimit.WindowSeconds)); + + options.RejectionStatusCode = 429; + options.OnRejected = (context, _) => + { + context.HttpContext.Response.Headers.RetryAfter = + ResolveRetryAfterSeconds(context.Lease, window) + .ToString(CultureInfo.InvariantCulture); + return ValueTask.CompletedTask; + }; + + options.GlobalLimiter = PartitionedRateLimiter.Create(context => + RateLimitPartition.GetFixedWindowLimiter( + RateLimitPartitionKey.Resolve(context), + _ => new FixedWindowRateLimiterOptions + { + PermitLimit = Math.Max(1, settings.RateLimit.PermitPerWindow), + Window = window, + QueueLimit = Math.Max(0, settings.RateLimit.QueueLimit), + QueueProcessingOrder = QueueProcessingOrder.OldestFirst, + })); + }); + + return services; + } +} diff --git a/Services/Hosting/CraftRoles.cs b/src/Craft/Services/Hosting/CraftRoles.cs similarity index 82% rename from Services/Hosting/CraftRoles.cs rename to src/Craft/Services/Hosting/CraftRoles.cs index c44f186..a4f4e39 100644 --- a/Services/Hosting/CraftRoles.cs +++ b/src/Craft/Services/Hosting/CraftRoles.cs @@ -103,23 +103,31 @@ public static CraftRoles Resolve(CraftSettings settings, Func e frontend = http = background = true; } - var cacheEnabled = (EnvFlag.Read(env, "CRAFT_RESPONSE_CACHE") ?? settings.Cache.Enabled) - ?? (frontend && http); - - var healthEnabled = EnvFlag.Read(env, "CRAFT_HEALTH_ENABLED") ?? settings.Health.Enabled; - - var healthPathEnv = env("CRAFT_HEALTH_PATH"); - var healthPath = !string.IsNullOrWhiteSpace(healthPathEnv) - ? healthPathEnv.Trim() - : settings.Health.Path; - if (!healthPath.StartsWith('/')) healthPath = "/" + healthPath; - - var compressionEnabled = EnvFlag.Read(env, "CRAFT_COMPRESSION") ?? settings.Frontend.Compression; + var cacheEnabled = settings.Cache.ResolveEnabled(frontend && http, env); + var healthEnabled = settings.Health.ResolveEnabled(env); + var healthPath = settings.Health.ResolvePath(env); + var compressionEnabled = settings.Frontend.ResolveCompressionEnabled(env); return new CraftRoles(frontend, http, background, cacheEnabled, healthEnabled, healthPath, compressionEnabled); } + /// + /// Resolve roles from IConfiguration without a full bind — + /// only the sections that drive role toggles are read. Used before builder.Build(). + /// + public static CraftRoles Resolve(IConfiguration configuration, Func? env = null) + { + ArgumentNullException.ThrowIfNull(configuration); + + var settings = new CraftSettings(); + configuration.GetSection("App:Roles").Bind(settings.Roles); + configuration.GetSection("App:Cache").Bind(settings.Cache); + configuration.GetSection("App:Health").Bind(settings.Health); + configuration.GetSection("App:Frontend").Bind(settings.Frontend); + return Resolve(settings, env ?? Environment.GetEnvironmentVariable); + } + /// Convenience overload reading the real process environment. public static CraftRoles Resolve(CraftSettings settings) => Resolve(settings, Environment.GetEnvironmentVariable); diff --git a/src/Craft/Services/Hosting/CraftSettingsConfiguration.cs b/src/Craft/Services/Hosting/CraftSettingsConfiguration.cs new file mode 100644 index 0000000..a6ed2d5 --- /dev/null +++ b/src/Craft/Services/Hosting/CraftSettingsConfiguration.cs @@ -0,0 +1,109 @@ +using Craft.Configuration; +using Microsoft.Extensions.Options; + +namespace Craft.Hosting; + +/// +/// Options-pattern registration for : bind App, apply shared +/// post-bind fixes, and fail fast on a few high-value invariants. +/// +internal static class CraftSettingsConfiguration +{ + /// + /// Registers via AddOptions + BindConfiguration("App"), + /// shared post-configure (SKU profiles, AzureWebJobsStorage fallback), and + /// ValidateOnStart. Pool-size rules are role-aware. + /// + public static OptionsBuilder AddCraftSettings( + this IServiceCollection services, + IConfiguration configuration, + CraftRoles roles) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + ArgumentNullException.ThrowIfNull(roles); + + return services + .AddOptions() + .BindConfiguration("App") + .PostConfigure(s => ApplyPostBind(s, configuration)) + .Validate(IsValidReadinessMode, + "App:ReadinessMode must be Immediate, HttpReady, or AllReady.") + .Validate(IsValidWarmupMode, + "App:Worker:WarmupMode must be BeforeReady, AfterReady, or Background.") + .Validate( + s => IsValidPoolSizes(s, roles), + "App:Worker pool sizes must be at least 1 for each pool this node's roles use " + + "(HttpPoolSize when Http is enabled, BgPoolSize when Background is enabled).") + .ValidateOnStart(); + } + + /// + /// SKU pool overrides, root AzureWebJobsStorage → Storage.ConnectionString when unset, + /// and legacy root-level background-limiter keys → . + /// + public static void ApplyPostBind(CraftSettings settings, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(settings); + ArgumentNullException.ThrowIfNull(configuration); + + SkuProfileSelector.Apply(settings); + + if (string.IsNullOrWhiteSpace(settings.Storage.ConnectionString)) + settings.Storage.ConnectionString = configuration["AzureWebJobsStorage"] ?? ""; + + ApplyLegacyBackgroundLimiterKeys(settings.BackgroundLimiter, configuration); + } + + /// + /// Overlay deprecated root-level Background* keys onto . + /// Prefer App:BackgroundLimiter:* / App__BackgroundLimiter__*; root keys remain for + /// existing harness compose files. + /// + private static void ApplyLegacyBackgroundLimiterKeys(BackgroundLimiterSettings bl, IConfiguration configuration) + { + if (int.TryParse(configuration["BackgroundBaseConcurrency"], out var baseConcurrency)) + bl.BaseConcurrency = baseConcurrency; + if (int.TryParse(configuration["BackgroundMaxConcurrency"], out var maxConcurrency)) + bl.MaxConcurrency = maxConcurrency; + if (int.TryParse(configuration["BackgroundScaleUpAfterSeconds"], out var scaleUp)) + bl.ScaleUpAfterSeconds = scaleUp; + if (int.TryParse(configuration["BackgroundHttpPressureThreshold"], out var httpThreshold)) + bl.HttpPressureThreshold = httpThreshold; + if (int.TryParse(configuration["BackgroundHttpPressureAfterSeconds"], out var httpAfter)) + bl.HttpPressureAfterSeconds = httpAfter; + if (int.TryParse(configuration["BackgroundOverSubscribe"], out var overSubscribe)) + bl.OverSubscribe = overSubscribe; + + var burst = configuration["BackgroundBurstToCeiling"]; + if (!string.IsNullOrWhiteSpace(burst)) + { + bl.BurstToCeiling = burst.Equals("true", StringComparison.OrdinalIgnoreCase) + || burst == "1"; + } + } + + private static bool IsValidPoolSizes(CraftSettings settings, CraftRoles roles) + { + if (!roles.RunsPowerShell) return true; + if (roles.Http && settings.Worker.HttpPoolSize < 1) return false; + if (roles.Background && settings.Worker.BgPoolSize < 1) return false; + return true; + } + + private static bool IsValidReadinessMode(CraftSettings settings) + { + var mode = settings.ReadinessMode?.Trim() ?? ""; + return mode.Equals("Immediate", StringComparison.OrdinalIgnoreCase) + || mode.Equals("HttpReady", StringComparison.OrdinalIgnoreCase) + || mode.Equals("AllReady", StringComparison.OrdinalIgnoreCase); + } + + private static bool IsValidWarmupMode(CraftSettings settings) + { + var mode = settings.Worker.WarmupMode?.Trim() ?? ""; + return mode.Equals("BeforeReady", StringComparison.OrdinalIgnoreCase) + || mode.Equals("AfterReady", StringComparison.OrdinalIgnoreCase) + || mode.Equals("Background", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/Services/Hosting/DevFrontendProxy.cs b/src/Craft/Services/Hosting/DevFrontendProxy.cs similarity index 99% rename from Services/Hosting/DevFrontendProxy.cs rename to src/Craft/Services/Hosting/DevFrontendProxy.cs index c5c0c71..15d8ec4 100644 --- a/Services/Hosting/DevFrontendProxy.cs +++ b/src/Craft/Services/Hosting/DevFrontendProxy.cs @@ -6,7 +6,7 @@ namespace Craft.Hosting; /// Development-only reverse proxy to the next dev server, so the frontend hot-reloads instead of /// being served from a precompiled Frontend/ directory. Has no effect in production. /// -public static class DevFrontendProxy +internal static class DevFrontendProxy { private const int PumpBufferBytes = 16 * 1024; diff --git a/Services/Hosting/DispatchProfiler.cs b/src/Craft/Services/Hosting/DispatchProfiler.cs similarity index 100% rename from Services/Hosting/DispatchProfiler.cs rename to src/Craft/Services/Hosting/DispatchProfiler.cs diff --git a/Services/Hosting/DispatchTiming.cs b/src/Craft/Services/Hosting/DispatchTiming.cs similarity index 100% rename from Services/Hosting/DispatchTiming.cs rename to src/Craft/Services/Hosting/DispatchTiming.cs diff --git a/Services/Hosting/Endpoints/AuthEndpoints.cs b/src/Craft/Services/Hosting/Endpoints/AuthEndpoints.cs similarity index 99% rename from Services/Hosting/Endpoints/AuthEndpoints.cs rename to src/Craft/Services/Hosting/Endpoints/AuthEndpoints.cs index 8c5ef41..bcde2eb 100644 --- a/Services/Hosting/Endpoints/AuthEndpoints.cs +++ b/src/Craft/Services/Hosting/Endpoints/AuthEndpoints.cs @@ -12,7 +12,7 @@ namespace Craft.Hosting.Endpoints; /// platform edge — CRAFT maps none of them. /// /// -public static class AuthEndpoints +internal static class AuthEndpoints { /// Maps /.auth/me and /api/me. public static WebApplication MapCraftAuthEndpoints( diff --git a/Services/Hosting/Endpoints/FrontendFallbackEndpoint.cs b/src/Craft/Services/Hosting/Endpoints/FrontendFallbackEndpoint.cs similarity index 99% rename from Services/Hosting/Endpoints/FrontendFallbackEndpoint.cs rename to src/Craft/Services/Hosting/Endpoints/FrontendFallbackEndpoint.cs index 64f179b..5c1fb7e 100644 --- a/Services/Hosting/Endpoints/FrontendFallbackEndpoint.cs +++ b/src/Craft/Services/Hosting/Endpoints/FrontendFallbackEndpoint.cs @@ -26,7 +26,7 @@ public sealed record FrontendFallbackOptions( /// Development, and otherwise serves a prerendered {path}.html or falls back to /// index.html for client-side routing. /// -public static class FrontendFallbackEndpoint +internal static class FrontendFallbackEndpoint { /// /// Whether a path may be answered with an HTML document. diff --git a/Services/Hosting/Endpoints/HealthEndpoint.cs b/src/Craft/Services/Hosting/Endpoints/HealthEndpoint.cs similarity index 89% rename from Services/Hosting/Endpoints/HealthEndpoint.cs rename to src/Craft/Services/Hosting/Endpoints/HealthEndpoint.cs index 67b7295..1416605 100644 --- a/Services/Hosting/Endpoints/HealthEndpoint.cs +++ b/src/Craft/Services/Hosting/Endpoints/HealthEndpoint.cs @@ -7,7 +7,7 @@ namespace Craft.Hosting.Endpoints; /// Role-agnostic liveness/readiness probe. Mapped before any role-gated block so it exists on every /// topology — a background-only worker still needs somewhere for the platform to probe. /// -public static class HealthEndpoint +internal static class HealthEndpoint { /// /// Maps the probe at when enabled, logging either way so the @@ -33,14 +33,14 @@ public static WebApplication MapCraftHealthEndpoint( return app; } - var pool = app.Services.GetRequiredService(); + var pool = app.Services.GetService(); // Always 200 while the process is up — this is liveness. Restarting a container that is merely // still warming up would turn a slow start into a crash loop. Readiness is reported in the body. app.MapGet(roles.HealthPath, () => { - var httpReady = !roles.Http || pool.IsReady; - var bgReady = !roles.Background || pool.BackgroundReady; + var httpReady = !roles.Http || (pool?.IsReady ?? false); + var bgReady = !roles.Background || (pool?.BackgroundReady ?? false); var storageReady = storageHealth is null || storageHealth.Snapshot(); return Results.Json(new diff --git a/Services/Hosting/Endpoints/JobEndpoints.cs b/src/Craft/Services/Hosting/Endpoints/JobEndpoints.cs similarity index 99% rename from Services/Hosting/Endpoints/JobEndpoints.cs rename to src/Craft/Services/Hosting/Endpoints/JobEndpoints.cs index cfaec7c..d8701ce 100644 --- a/Services/Hosting/Endpoints/JobEndpoints.cs +++ b/src/Craft/Services/Hosting/Endpoints/JobEndpoints.cs @@ -8,7 +8,7 @@ namespace Craft.Hosting.Endpoints; /// bypassing the PowerShell pool — these are polled by dashboards during a fan-out, exactly when the /// worker pool is busiest and least able to spare a runspace. /// -public static class JobEndpoints +internal static class JobEndpoints { /// Maps the /API/jobs/* and /API/runs/* routes. public static WebApplication MapCraftJobEndpoints(this WebApplication app) diff --git a/Services/Hosting/Endpoints/NativeDispatchEndpoint.cs b/src/Craft/Services/Hosting/Endpoints/NativeDispatchEndpoint.cs similarity index 100% rename from Services/Hosting/Endpoints/NativeDispatchEndpoint.cs rename to src/Craft/Services/Hosting/Endpoints/NativeDispatchEndpoint.cs diff --git a/Services/Hosting/Endpoints/PowerShellDispatchEndpoint.cs b/src/Craft/Services/Hosting/Endpoints/PowerShellDispatchEndpoint.cs similarity index 99% rename from Services/Hosting/Endpoints/PowerShellDispatchEndpoint.cs rename to src/Craft/Services/Hosting/Endpoints/PowerShellDispatchEndpoint.cs index 1467604..0a45f2e 100644 --- a/Services/Hosting/Endpoints/PowerShellDispatchEndpoint.cs +++ b/src/Craft/Services/Hosting/Endpoints/PowerShellDispatchEndpoint.cs @@ -15,7 +15,7 @@ namespace Craft.Hosting.Endpoints; /// handling that lets a short HTTP handler kick off long-running background work. /// /// -public static class PowerShellDispatchEndpoint +internal static class PowerShellDispatchEndpoint { private static readonly string[] DispatchMethods = ["GET", "POST", "PUT", "DELETE", "PATCH"]; diff --git a/Services/Hosting/Endpoints/PrmEndpoint.cs b/src/Craft/Services/Hosting/Endpoints/PrmEndpoint.cs similarity index 100% rename from Services/Hosting/Endpoints/PrmEndpoint.cs rename to src/Craft/Services/Hosting/Endpoints/PrmEndpoint.cs diff --git a/Services/Hosting/Endpoints/RealtimeEndpoint.cs b/src/Craft/Services/Hosting/Endpoints/RealtimeEndpoint.cs similarity index 99% rename from Services/Hosting/Endpoints/RealtimeEndpoint.cs rename to src/Craft/Services/Hosting/Endpoints/RealtimeEndpoint.cs index 99e5a77..31ac75d 100644 --- a/Services/Hosting/Endpoints/RealtimeEndpoint.cs +++ b/src/Craft/Services/Hosting/Endpoints/RealtimeEndpoint.cs @@ -9,7 +9,7 @@ namespace Craft.Hosting.Endpoints; /// through RealtimeBridge. Pure C# — it never occupies a PowerShell runspace, which is what /// makes it safe to hold a connection open per browser tab. /// -public static class RealtimeEndpoint +internal static class RealtimeEndpoint { /// /// Maps the SSE endpoint on nodes that face a browser, when realtime is switched on. diff --git a/Services/Hosting/Endpoints/SetupEndpoints.cs b/src/Craft/Services/Hosting/Endpoints/SetupEndpoints.cs similarity index 94% rename from Services/Hosting/Endpoints/SetupEndpoints.cs rename to src/Craft/Services/Hosting/Endpoints/SetupEndpoints.cs index 357ab2b..0f1dd77 100644 --- a/Services/Hosting/Endpoints/SetupEndpoints.cs +++ b/src/Craft/Services/Hosting/Endpoints/SetupEndpoints.cs @@ -1,7 +1,6 @@ using System.Text.Json; using Craft.Configuration; using Craft.PowerShellHost; -using Craft.Services; using Craft.Setup; namespace Craft.Hosting.Endpoints; @@ -10,7 +9,7 @@ namespace Craft.Hosting.Endpoints; /// First-run setup wizard API. Implemented directly in C# with no PowerShell involved — setup has to /// work before the worker pool is ready, and on a host that is not yet authenticated. /// -public static class SetupEndpoints +internal static class SetupEndpoints { /// /// Maps /api/setup/health (always) plus the wizard routes (only when @@ -24,10 +23,11 @@ public static WebApplication MapCraftSetupEndpoints(this WebApplication app, Cra var pool = app.Services.GetRequiredService(); var setupService = app.Services.GetRequiredService(); + var startup = app.Services.GetRequiredService(); app.MapGet("/api/setup/health", () => { - var info = StartupInfoBridge.GetInfo(); + var info = startup.Stats; return Results.Json(new { status = "ok", @@ -109,7 +109,7 @@ public static WebApplication MapCraftSetupEndpoints(this WebApplication app, Cra app.MapPost("/api/setup/configure", async (HttpContext context) => { - if (AppLifecycleBridge.IsSetupCompleted()) return SetupAlreadyCompleted(); + if (setupService.IsSetupCompleted()) return SetupAlreadyCompleted(); var root = await ReadJsonBodyAsync(context); @@ -119,7 +119,7 @@ public static WebApplication MapCraftSetupEndpoints(this WebApplication app, Cra var multiTenant = root.TryGetProperty("multiTenant", out var mt) && mt.GetBoolean(); await setupService.ConfigureAppServiceAuth(appId, clientSecret, tenantId, multiTenant); - AppLifecycleBridge.MarkSetupCompleted("EasyAuth configured via automated setup"); + setupService.MarkSetupCompleted("EasyAuth configured via automated setup"); return Results.Json(new { @@ -130,7 +130,7 @@ public static WebApplication MapCraftSetupEndpoints(this WebApplication app, Cra app.MapPost("/api/setup/manual", async (HttpContext context) => { - if (AppLifecycleBridge.IsSetupCompleted()) return SetupAlreadyCompleted(); + if (setupService.IsSetupCompleted()) return SetupAlreadyCompleted(); var root = await ReadJsonBodyAsync(context); @@ -140,7 +140,7 @@ public static WebApplication MapCraftSetupEndpoints(this WebApplication app, Cra var multiTenant = root.TryGetProperty("multiTenant", out var mt) && mt.GetBoolean(); await setupService.ConfigureManual(appId, clientSecret, tenantId, multiTenant); - AppLifecycleBridge.MarkSetupCompleted("EasyAuth configured via manual setup"); + setupService.MarkSetupCompleted("EasyAuth configured via manual setup"); return Results.Json(new { diff --git a/Services/Hosting/EnvFlag.cs b/src/Craft/Services/Hosting/EnvFlag.cs similarity index 100% rename from Services/Hosting/EnvFlag.cs rename to src/Craft/Services/Hosting/EnvFlag.cs diff --git a/Services/Hosting/FileLoggerProvider.cs b/src/Craft/Services/Hosting/FileLoggerProvider.cs similarity index 89% rename from Services/Hosting/FileLoggerProvider.cs rename to src/Craft/Services/Hosting/FileLoggerProvider.cs index ec980d1..9b82d8a 100644 --- a/Services/Hosting/FileLoggerProvider.cs +++ b/src/Craft/Services/Hosting/FileLoggerProvider.cs @@ -21,11 +21,11 @@ public sealed class FileLoggerProvider : ILoggerProvider { private readonly string _directory; private readonly string _filePrefix; - private readonly long _maxFileBytes; - private readonly int _maxFileCount; + private long _maxFileBytes; + private int _maxFileCount; private readonly LogLevel _minLevel; - private readonly string _timestampFormat; - private readonly bool _includeCategory; + private string _timestampFormat; + private bool _includeCategory; private StreamWriter? _writer; private long _currentFileSize; @@ -35,6 +35,12 @@ public sealed class FileLoggerProvider : ILoggerProvider private readonly record struct LogItem(string Line, string? ExLine, TaskCompletionSource? RotateSignal); + /// + /// Creates the provider from an early-bound so logging works + /// before Build(). Call after Build with + /// CraftSettings.FileLogging so rotation/format knobs match the Options-bound settings. + /// Directory and file prefix are fixed at construction (the active log file is already open). + /// public FileLoggerProvider(FileLoggingSettings settings, LogLevel minLevel = LogLevel.Information) { _directory = settings.ResolvedDirectory; @@ -56,6 +62,19 @@ public FileLoggerProvider(FileLoggingSettings settings, LogLevel minLevel = LogL _consumerTask = Task.Run(ConsumeLoopAsync); } + /// + /// Refresh rotation/format settings from the Options-bound + /// after Build(). Does not relocate the open log file (directory/prefix stay as constructed). + /// + public void SyncFrom(FileLoggingSettings settings) + { + ArgumentNullException.ThrowIfNull(settings); + _maxFileBytes = settings.MaxFileSizeMB * 1024L * 1024L; + _maxFileCount = settings.MaxFileCount; + _timestampFormat = settings.TimestampFormat; + _includeCategory = settings.IncludeCategory; + } + /// Path to the currently active log file. public string CurrentFilePath => Path.Combine(_directory, $"{_filePrefix}.log"); diff --git a/Services/Hosting/HandlerHeaders.cs b/src/Craft/Services/Hosting/HandlerHeaders.cs similarity index 100% rename from Services/Hosting/HandlerHeaders.cs rename to src/Craft/Services/Hosting/HandlerHeaders.cs diff --git a/Services/Hosting/HttpDiagnosticListener.cs b/src/Craft/Services/Hosting/HttpDiagnosticListener.cs similarity index 99% rename from Services/Hosting/HttpDiagnosticListener.cs rename to src/Craft/Services/Hosting/HttpDiagnosticListener.cs index ae3d225..7074f14 100644 --- a/Services/Hosting/HttpDiagnosticListener.cs +++ b/src/Craft/Services/Hosting/HttpDiagnosticListener.cs @@ -10,7 +10,7 @@ namespace Craft.Hosting; /// Tracks slow HTTP requests via DiagnosticListener (method, URL, body) and /// DNS/TLS/Socket timing via EventSource. /// -public sealed class HttpDiagnosticListener : EventListener, +internal sealed class HttpDiagnosticListener : EventListener, IObserver, IObserver> { diff --git a/Services/Bridges/LogBridge.cs b/src/Craft/Services/Hosting/LogQueryService.cs similarity index 73% rename from Services/Bridges/LogBridge.cs rename to src/Craft/Services/Hosting/LogQueryService.cs index 7b2ec94..d2b1e59 100644 --- a/Services/Bridges/LogBridge.cs +++ b/src/Craft/Services/Hosting/LogQueryService.cs @@ -1,55 +1,32 @@ -using Craft.Hosting; +using Craft.Services; -// NAMESPACE PINNED — do not change. -// Downstream PowerShell reaches these types by fully-qualified name, e.g. -// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) -// Renaming the namespace compiles fine and then fails at runtime in the hosted app -// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. -// The folder is free to move; the namespace is a published contract. -namespace Craft.Services; +namespace Craft.Hosting; /// -/// Static bridge exposing file log access to PowerShell and HTTP endpoints. -/// Provides filtered log reading, file listing, and log management. -/// -/// PS usage: -/// $lines = [Craft.Services.LogBridge]::ReadLog() # all from current log -/// $lines = [Craft.Services.LogBridge]::ReadLog(100) # last 100 lines -/// $lines = [Craft.Services.LogBridge]::ReadLog(100, 'ERR') # last 100 error lines -/// $lines = [Craft.Services.LogBridge]::ReadLog(100, 'ERR', 'timeout') # errors containing "timeout" -/// $lines = [Craft.Services.LogBridge]::ReadLog(0, $null, $null, 'craft.1.log') # from rotated file -/// $files = [Craft.Services.LogBridge]::GetLogFiles() # list all log files -/// $path = [Craft.Services.LogBridge]::GetCurrentLogPath() # active log path -/// $dir = [Craft.Services.LogBridge]::GetLogDirectory() # log directory path -/// $lines = [Craft.Services.LogBridge]::SearchLog('timeout') # search current log -/// $lines = [Craft.Services.LogBridge]::GetErrors(50) # last 50 errors -/// $lines = [Craft.Services.LogBridge]::GetLogsBetween($from, $to) # date range -/// $lines = [Craft.Services.LogBridge]::GetLogsSince([DateTime]::UtcNow.AddHours(-1)) # last hour -/// $lines = [Craft.Services.LogBridge]::GetLogsBetween($from, $to, 'ERR') # errors in range -/// [Craft.Services.LogBridge]::ForceRotation() # manually rotate now -/// $count = [Craft.Services.LogBridge]::PurgeOldFiles(7) # delete files older than 7 days +/// File-log read/filter/purge/list over . +/// Domain code and LogBridge both go through this service. /// -public static class LogBridge +public sealed class LogQueryService { - private static FileLoggerProvider? s_provider; + private readonly FileLoggerProvider _provider; - public static void Initialize(FileLoggerProvider provider) => s_provider = provider; + public LogQueryService(FileLoggerProvider provider) => _provider = provider; /// Get the path to the currently active log file. - public static string GetCurrentLogPath() => s_provider?.CurrentFilePath ?? ""; + public string GetCurrentLogPath() => _provider.CurrentFilePath ?? ""; /// Get the log directory path. - public static string GetLogDirectory() => s_provider?.LogDirectory ?? ""; + public string GetLogDirectory() => _provider.LogDirectory ?? ""; /// Get metadata about all log files in the log directory. - public static LogFileInfo[] GetLogFiles() + public LogFileInfo[] GetLogFiles() { - var dir = s_provider?.LogDirectory; + var dir = _provider.LogDirectory; if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return Array.Empty(); - var prefix = s_provider!.FilePrefix; - var currentPath = s_provider.CurrentFilePath; + var prefix = _provider.FilePrefix; + var currentPath = _provider.CurrentFilePath; return Directory.GetFiles(dir, $"{prefix}*.log") .Select(f => new FileInfo(f)) @@ -79,7 +56,7 @@ public static LogFileInfo[] GetLogFiles() /// Case-insensitive text to exclude from results. /// Regex pattern to match against the message portion of the line. /// When true, return results newest-first (default: false, oldest-first). - public static string[] ReadLog(int tail = 0, string? level = null, string? search = null, + public string[] ReadLog(int tail = 0, string? level = null, string? search = null, string? file = null, DateTime? from = null, DateTime? to = null, string? exclude = null, string? regexPattern = null, bool sortNewestFirst = false) { @@ -163,45 +140,40 @@ public static string[] ReadLog(int tail = 0, string? level = null, string? searc } /// Search the current log for lines containing the specified text. - public static string[] SearchLog(string searchText, int tail = 0) + public string[] SearchLog(string searchText, int tail = 0) => ReadLog(tail, null, searchText, null); /// Get error-level entries from the current log. - public static string[] GetErrors(int tail = 0) + public string[] GetErrors(int tail = 0) => ReadLog(tail, "ERR", null, null); /// Get warning-level entries from the current log. - public static string[] GetWarnings(int tail = 0) + public string[] GetWarnings(int tail = 0) => ReadLog(tail, "WRN", null, null); /// /// Get log entries within a UTC date/time range, optionally filtered by level and search text. - /// PS: [Craft.Services.LogBridge]::GetLogsBetween([DateTime]'2026-05-13 08:00', [DateTime]'2026-05-13 12:00') /// - public static string[] GetLogsBetween(DateTime from, DateTime to, string? level = null, string? search = null, string? file = null) + public string[] GetLogsBetween(DateTime from, DateTime to, string? level = null, string? search = null, string? file = null) => ReadLog(0, level, search, file, from, to); /// /// Get log entries from a UTC start time to now. - /// PS: [Craft.Services.LogBridge]::GetLogsSince([DateTime]::UtcNow.AddHours(-1)) /// - public static string[] GetLogsSince(DateTime from, string? level = null, string? search = null, string? file = null) + public string[] GetLogsSince(DateTime from, string? level = null, string? search = null, string? file = null) => ReadLog(0, level, search, file, from, null); /// /// Get log entries from the last N minutes. - /// PS: [Craft.Services.LogBridge]::GetRecentLogs(30) # last 30 minutes - /// [Craft.Services.LogBridge]::GetRecentLogs(30, 'ERR') # errors in last 30 min /// - public static string[] GetRecentLogs(int minutes, string? level = null, string? search = null, string? file = null) + public string[] GetRecentLogs(int minutes, string? level = null, string? search = null, string? file = null) => ReadLog(0, level, search, file, DateTime.UtcNow.AddMinutes(-minutes), null); /// /// Search across ALL log files (current + rotated) for matching entries. /// Returns results ordered oldest-to-newest. Useful for investigating issues across rotations. - /// PS: [Craft.Services.LogBridge]::SearchAllFiles('timeout', 'ERR') /// - public static string[] SearchAllFiles(string? search = null, string? level = null, + public string[] SearchAllFiles(string? search = null, string? level = null, DateTime? from = null, DateTime? to = null, int tail = 0, string? exclude = null, string? regexPattern = null, bool sortNewestFirst = false) { @@ -234,22 +206,21 @@ public static string[] SearchAllFiles(string? search = null, string? level = nul } /// Manually trigger log rotation on the current file. - public static void ForceRotation() - => s_provider?.ForceRotate(); + public void ForceRotation() => _provider.ForceRotate(); /// /// Delete rotated log files older than the specified number of days. /// The current active log file is never deleted. /// /// Number of files deleted. - public static int PurgeOldFiles(int olderThanDays = 7) + public int PurgeOldFiles(int olderThanDays = 7) { - var dir = s_provider?.LogDirectory; + var dir = _provider.LogDirectory; if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return 0; - var prefix = s_provider!.FilePrefix; - var currentPath = s_provider.CurrentFilePath; + var prefix = _provider.FilePrefix; + var currentPath = _provider.CurrentFilePath; var cutoff = DateTime.UtcNow.AddDays(-olderThanDays); var deleted = 0; @@ -269,14 +240,12 @@ public static int PurgeOldFiles(int olderThanDays = 7) return deleted; } - // ── Private helpers ─────────────────────────────────────────────── - - private static string? ResolveLogFilePath(string? fileName) + private string? ResolveLogFilePath(string? fileName) { if (string.IsNullOrEmpty(fileName)) - return s_provider?.CurrentFilePath; + return _provider.CurrentFilePath; - var dir = s_provider?.LogDirectory; + var dir = _provider.LogDirectory; if (string.IsNullOrEmpty(dir)) return null; diff --git a/Services/Hosting/OperationContext.cs b/src/Craft/Services/Hosting/OperationContext.cs similarity index 100% rename from Services/Hosting/OperationContext.cs rename to src/Craft/Services/Hosting/OperationContext.cs diff --git a/src/Craft/Services/Hosting/PendingWorkFlushHostedService.cs b/src/Craft/Services/Hosting/PendingWorkFlushHostedService.cs new file mode 100644 index 0000000..11f18fb --- /dev/null +++ b/src/Craft/Services/Hosting/PendingWorkFlushHostedService.cs @@ -0,0 +1,103 @@ +using Craft.Orchestration; +using Craft.Services; + +namespace Craft.Hosting; + +/// +/// On host shutdown: final drain of PowerShell ingress queues, then await tracked +/// fire-and-forget drain/planner/finalize work with a bounded timeout. +/// +internal sealed class PendingWorkFlushHostedService : IHostedService +{ + private static readonly TimeSpan FlushTimeout = TimeSpan.FromSeconds(30); + + private readonly PowerShellRunnerService _runner; + private readonly OrchestratorService _orchestrator; + private readonly QueueDispatchService _queueDispatch; + private readonly ILogger _logger; + + public PendingWorkFlushHostedService( + PowerShellRunnerService runner, + OrchestratorService orchestrator, + QueueDispatchService queueDispatch, + ILogger logger) + { + _runner = runner; + _orchestrator = orchestrator; + _queueDispatch = queueDispatch; + _logger = logger; + } + + public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public async Task StopAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("[Shutdown] Flushing pending orchestration/queue work"); + + try + { + await _orchestrator.DrainPendingAsync(); + } + catch (Exception ex) + { + _logger.LogError(ex, "[Shutdown] Final orchestrator DrainPending failed"); + } + + try + { + _queueDispatch.DrainPending(); + } + catch (Exception ex) + { + _logger.LogError(ex, "[Shutdown] Final queue DrainPending failed"); + } + + // Prefer a linked token that still respects host abort, but cap wait ourselves. + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(FlushTimeout); + + int runnerLeft = 0, orchLeft = 0; + try + { + runnerLeft = await _runner.FlushBackgroundDrainsAsync(FlushTimeout, timeoutCts.Token); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + runnerLeft = -1; + _logger.LogWarning("[Shutdown] Runner background-drain flush timed out after {Timeout}", FlushTimeout); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogError(ex, "[Shutdown] Runner background-drain flush failed"); + } + + try + { + orchLeft = await _orchestrator.FlushBackgroundWorkAsync(FlushTimeout, timeoutCts.Token); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + orchLeft = -1; + _logger.LogWarning("[Shutdown] Orchestrator background-work flush timed out after {Timeout}", FlushTimeout); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogError(ex, "[Shutdown] Orchestrator background-work flush failed"); + } + + if (runnerLeft > 0 || orchLeft > 0) + _logger.LogWarning( + "[Shutdown] Pending work flush incomplete — runnerLeftovers={Runner} orchLeftovers={Orch}", + runnerLeft, orchLeft); + else + _logger.LogInformation("[Shutdown] Pending work flush complete"); + } +} diff --git a/Services/Hosting/PrecompressedEncoding.cs b/src/Craft/Services/Hosting/PrecompressedEncoding.cs similarity index 100% rename from Services/Hosting/PrecompressedEncoding.cs rename to src/Craft/Services/Hosting/PrecompressedEncoding.cs diff --git a/Services/Hosting/RateLimitPartitionKey.cs b/src/Craft/Services/Hosting/RateLimitPartitionKey.cs similarity index 100% rename from Services/Hosting/RateLimitPartitionKey.cs rename to src/Craft/Services/Hosting/RateLimitPartitionKey.cs diff --git a/Services/Hosting/RequestCounter.cs b/src/Craft/Services/Hosting/RequestCounter.cs similarity index 100% rename from Services/Hosting/RequestCounter.cs rename to src/Craft/Services/Hosting/RequestCounter.cs diff --git a/Services/Hosting/ResponseTriggers.cs b/src/Craft/Services/Hosting/ResponseTriggers.cs similarity index 100% rename from Services/Hosting/ResponseTriggers.cs rename to src/Craft/Services/Hosting/ResponseTriggers.cs diff --git a/Services/Hosting/SetupModeMiddleware.cs b/src/Craft/Services/Hosting/SetupModeMiddleware.cs similarity index 94% rename from Services/Hosting/SetupModeMiddleware.cs rename to src/Craft/Services/Hosting/SetupModeMiddleware.cs index 743e9ff..36b9c35 100644 --- a/Services/Hosting/SetupModeMiddleware.cs +++ b/src/Craft/Services/Hosting/SetupModeMiddleware.cs @@ -1,4 +1,3 @@ -using Craft.Services; using Craft.Setup; namespace Craft.Hosting; @@ -11,7 +10,7 @@ namespace Craft.Hosting; /// The branch table lives in ; this only performs the chosen action. /// /// -public static class SetupModeMiddleware +internal static class SetupModeMiddleware { /// Registers the setup gate when App:Setup:Enabled. public static WebApplication UseCraftSetupGate(this WebApplication app, ILogger logger) @@ -19,12 +18,14 @@ public static WebApplication UseCraftSetupGate(this WebApplication app, ILogger ArgumentNullException.ThrowIfNull(app); ArgumentNullException.ThrowIfNull(logger); + var setupService = app.Services.GetRequiredService(); + app.Use(async (context, next) => { var action = SetupGate.Decide( context.Request.Path.Value ?? "", SetupService.IsEasyAuthConfigured(), - AppLifecycleBridge.IsSetupModeRequested()); + setupService.IsSetupModeRequested()); switch (action) { diff --git a/Services/Hosting/SkuProfileSelector.cs b/src/Craft/Services/Hosting/SkuProfileSelector.cs similarity index 100% rename from Services/Hosting/SkuProfileSelector.cs rename to src/Craft/Services/Hosting/SkuProfileSelector.cs diff --git a/Services/Hosting/StartupGate.cs b/src/Craft/Services/Hosting/StartupGate.cs similarity index 100% rename from Services/Hosting/StartupGate.cs rename to src/Craft/Services/Hosting/StartupGate.cs diff --git a/src/Craft/Services/Hosting/StartupGateMiddleware.cs b/src/Craft/Services/Hosting/StartupGateMiddleware.cs new file mode 100644 index 0000000..378a59c --- /dev/null +++ b/src/Craft/Services/Hosting/StartupGateMiddleware.cs @@ -0,0 +1,59 @@ +using Craft.PowerShellHost; +using Craft.Setup; + +namespace Craft.Hosting; + +/// +/// While the HTTP worker pool is still warming up, steers traffic to a loading page (or a 503 for +/// API callers) instead of letting requests hit an empty pool. Pass-through once the pool is ready, +/// when this node has no Http role, or for health/setup paths — see . +/// +internal static class StartupGateMiddleware +{ + /// + /// Registers the startup-loading gate. No-ops immediately when is + /// false or the pool is already ready (or absent). + /// + public static WebApplication UseCraftStartupGate( + this WebApplication app, + bool httpEnabled, + PowerShellWorkerPool? pool, + bool setupEnabled, + bool healthEnabled, + string healthPath) + { + ArgumentNullException.ThrowIfNull(app); + ArgumentNullException.ThrowIfNull(healthPath); + + app.Use(async (context, next) => + { + if (!httpEnabled || pool is null || pool.IsReady) + { + await next(); + return; + } + + switch (StartupGate.Decide(context.Request.Path.Value ?? "", + setupEnabled, healthEnabled, healthPath)) + { + case StartupGateAction.PassThrough: + await next(); + return; + + case StartupGateAction.ApiUnavailable: + context.Response.StatusCode = 503; + context.Response.ContentType = "application/json"; + await context.Response.WriteAsync( + """{"error":"Application is starting up. Please wait."}"""); + return; + + default: + context.Response.ContentType = "text/html; charset=utf-8"; + await context.Response.WriteAsync(SetupPages.StartupHtml); + return; + } + }); + + return app; + } +} diff --git a/src/Craft/Services/Hosting/StartupProgressService.cs b/src/Craft/Services/Hosting/StartupProgressService.cs new file mode 100644 index 0000000..2b540d2 --- /dev/null +++ b/src/Craft/Services/Hosting/StartupProgressService.cs @@ -0,0 +1,62 @@ +using Craft.Services; + +namespace Craft.Hosting; + +/// +/// Owns for the process. Domain code (worker pool, Program) +/// records progress here; PowerShell reads via StartupInfoBridge.GetInfo(). +/// +public sealed class StartupProgressService +{ + private readonly StartupStats _stats = new(); + + public StartupStats Stats => _stats; + + public void SetReadinessMode(string mode) => _stats.ReadinessMode = mode; + public void SetWarmupMode(string mode) => _stats.WarmupMode = mode; + public void SetCpuCount(int count) => _stats.CpuCount = count; + + public void SetPoolConfig(int httpSize, int bgSize) + { + _stats.HttpPoolSize = httpSize; + _stats.BgPoolSize = bgSize; + } + + public void SetModuleCounts(int shared, int httpOnly, int bgOnly) + { + _stats.SharedModuleCount = shared; + _stats.HttpOnlyModuleCount = httpOnly; + _stats.BgOnlyModuleCount = bgOnly; + } + + public void SetBaseWorkerDone(long ms, int functionCount) + { + _stats.BaseWorkerMs = ms; + _stats.BaseFunctionCount = functionCount; + _stats.Phase = "BaseReady"; + } + + public void SetWarmupDone(long ms) => _stats.WarmupMs = ms; + + public void SetHttpReady(long ms, int functionCount) + { + _stats.HttpReadyMs = ms; + _stats.HttpFunctionCount = functionCount; + _stats.Phase = "HttpReady"; + } + + public void SetHttpPoolFull(long ms) => _stats.HttpPoolFullMs = ms; + + public void SetBgReady(long ms, int functionCount) + { + _stats.BgReadyMs = ms; + _stats.BgFunctionCount = functionCount; + } + + public void SetFullyReady(long ms) + { + _stats.FullyReadyMs = ms; + _stats.Phase = "Ready"; + _stats.IsFullyReady = true; + } +} diff --git a/Services/Hosting/StaticCachePolicy.cs b/src/Craft/Services/Hosting/StaticCachePolicy.cs similarity index 100% rename from Services/Hosting/StaticCachePolicy.cs rename to src/Craft/Services/Hosting/StaticCachePolicy.cs diff --git a/Services/Hosting/StaticFilePipeline.cs b/src/Craft/Services/Hosting/StaticFilePipeline.cs similarity index 99% rename from Services/Hosting/StaticFilePipeline.cs rename to src/Craft/Services/Hosting/StaticFilePipeline.cs index 85e60e0..66f808f 100644 --- a/Services/Hosting/StaticFilePipeline.cs +++ b/src/Craft/Services/Hosting/StaticFilePipeline.cs @@ -8,7 +8,7 @@ namespace Craft.Hosting; /// /// Static content serving for Frontend/: CSP, precompressed variants, and cache headers. /// -public static class StaticFilePipeline +internal static class StaticFilePipeline { /// /// Applies the configured Content-Security-Policy to every response. diff --git a/Services/Hosting/StatsHistoryService.cs b/src/Craft/Services/Hosting/StatsHistoryService.cs similarity index 98% rename from Services/Hosting/StatsHistoryService.cs rename to src/Craft/Services/Hosting/StatsHistoryService.cs index 15b6f9c..f97a7a5 100644 --- a/Services/Hosting/StatsHistoryService.cs +++ b/src/Craft/Services/Hosting/StatsHistoryService.cs @@ -16,6 +16,7 @@ public class StatsHistoryService : BackgroundService { private readonly ILogger _logger; private readonly CraftSettings _settings; + private readonly WorkerMetricsService _metrics; private readonly string _dataFilePath; // Circular buffer — newest at the end @@ -44,10 +45,11 @@ public class StatsHistoryService : BackgroundService WriteIndented = false, }; - public StatsHistoryService(ILogger logger, CraftSettings settings) + public StatsHistoryService(ILogger logger, CraftSettings settings, WorkerMetricsService metrics) { _logger = logger; _settings = settings; + _metrics = metrics; var dataDir = Path.Combine(AppContext.BaseDirectory, "_data"); _dataFilePath = Path.Combine(dataDir, "stats-history.jsonl"); @@ -96,7 +98,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) /// Take a metrics snapshot and record a data point with delta computation. private StatsDataPoint CollectSample() { - var snapshot = WorkerMetricsBridge.GetSnapshot(); + var snapshot = _metrics.GetSnapshot(); var now = DateTime.UtcNow; var httpPool = snapshot.HttpPool; @@ -144,7 +146,7 @@ private StatsDataPoint CollectSample() GC1 = snapshot.Memory.GC1, GC2 = snapshot.Memory.GC2, - // CPU — delta-computed by WorkerMetricsBridge + // CPU — delta-computed on the metrics snapshot path CpuPct = snapshot.Memory.CpuPct, ContainerCpuPct = snapshot.Memory.ContainerCpuPct, OtherCpuPct = snapshot.Memory.OtherCpuPct, diff --git a/Services/Bridges/WorkerMetricsBridge.cs b/src/Craft/Services/Hosting/WorkerMetricsService.cs similarity index 69% rename from Services/Bridges/WorkerMetricsBridge.cs rename to src/Craft/Services/Hosting/WorkerMetricsService.cs index 26e720b..8272610 100644 --- a/Services/Bridges/WorkerMetricsBridge.cs +++ b/src/Craft/Services/Hosting/WorkerMetricsService.cs @@ -3,95 +3,88 @@ using System.Runtime; using Craft.Orchestration; using Craft.PowerShellHost; +using Craft.Services; -// NAMESPACE PINNED — do not change. -// Downstream PowerShell reaches these types by fully-qualified name, e.g. -// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) -// Renaming the namespace compiles fine and then fails at runtime in the hosted app -// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. -// The folder is free to move; the namespace is a published contract. -namespace Craft.Services; +namespace Craft.Hosting; /// -/// Static bridge exposing worker pool metrics and utilization data to PowerShell. -/// -/// PS usage: -/// $metrics = [Craft.Services.WorkerMetricsBridge]::GetSnapshot() -/// $metrics.HttpPool.BusyCount -/// $metrics.HttpPool.Workers[0].TotalInvocations -/// $metrics.BgPool.Workers -/// $metrics.Limiter.IsHttpThrottled -/// $metrics.Jobs.Running +/// Domain-owned worker pool metrics. PowerShell reaches the same data via +/// WorkerMetricsBridge; C# callers inject this service instead of the static bridge. /// -public static class WorkerMetricsBridge +public sealed class WorkerMetricsService { - private static PowerShellWorkerPool? s_pool; - private static BackgroundTaskLimiter? s_limiter; - private static JobManager? s_jobManager; - private static ILogger? s_logger; + + private readonly PowerShellWorkerPool _pool; + private readonly BackgroundTaskLimiter _limiter; + private readonly JobManager _jobManager; + private readonly ILogger _logger; // Per-worker tracking: workerId → stats - private static readonly ConcurrentDictionary s_workerStats = new(); - private static readonly DateTime s_startTimeUtc = DateTime.UtcNow; - private static long s_globalInvocations; + private readonly ConcurrentDictionary _workerStats = new(); + private readonly DateTime _startTimeUtc = DateTime.UtcNow; + private long _globalInvocations; // Retired-worker totals: when a worker is recycled, its accumulated counters are // moved here so pool aggregates remain monotonic across recycles. Without these, // pool.TotalInvocations would drop on recycle and StatsHistoryService deltas // (BgInvocations, BgBusyMs) would go negative. - private static long s_retiredHttpInvocations; - private static long s_retiredHttpBusyMs; - private static long s_retiredHttpFaults; - private static long s_retiredBgInvocations; - private static long s_retiredBgBusyMs; - private static long s_retiredBgFaults; - - public static void Initialize(PowerShellWorkerPool pool, BackgroundTaskLimiter limiter, JobManager jobManager, ILogger? logger = null) + private long _retiredHttpInvocations; + private long _retiredHttpBusyMs; + private long _retiredHttpFaults; + private long _retiredBgInvocations; + private long _retiredBgBusyMs; + private long _retiredBgFaults; + + public WorkerMetricsService( + ILogger logger, + PowerShellWorkerPool pool, + BackgroundTaskLimiter limiter, + JobManager jobManager) { - s_pool = pool; - s_limiter = limiter; - s_jobManager = jobManager; - s_logger = logger; + _logger = logger; + _pool = pool; + _limiter = limiter; + _jobManager = jobManager; } // ── Called by PowerShellWorkerPool/RunnerService at checkout/reclaim ── /// Pre-register a worker so it appears in snapshots even before first use. - public static void RegisterWorker(int workerId, bool isHttp) + public void RegisterWorker(int workerId, bool isHttp) { - var stats = s_workerStats.GetOrAdd(workerId, _ => new WorkerStats { WorkerId = workerId }); + var stats = _workerStats.GetOrAdd(workerId, _ => new WorkerStats { WorkerId = workerId }); stats.IsHttp = isHttp; } /// Remove a worker's stats when it is recycled/replaced. - public static void DeregisterWorker(int workerId) + public void DeregisterWorker(int workerId) { // Accumulate the retiring worker's totals into the per-pool "retired" buckets // so pool-level sums (and delta-based history) stay monotonic across recycles. - if (s_workerStats.TryRemove(workerId, out var stats)) + if (_workerStats.TryRemove(workerId, out var stats)) { var inv = Interlocked.Read(ref stats._totalInvocations); var busy = Interlocked.Read(ref stats._totalBusyMs); var faults = Interlocked.Read(ref stats._totalFaults); if (stats.IsHttp) { - Interlocked.Add(ref s_retiredHttpInvocations, inv); - Interlocked.Add(ref s_retiredHttpBusyMs, busy); - Interlocked.Add(ref s_retiredHttpFaults, faults); + Interlocked.Add(ref _retiredHttpInvocations, inv); + Interlocked.Add(ref _retiredHttpBusyMs, busy); + Interlocked.Add(ref _retiredHttpFaults, faults); } else { - Interlocked.Add(ref s_retiredBgInvocations, inv); - Interlocked.Add(ref s_retiredBgBusyMs, busy); - Interlocked.Add(ref s_retiredBgFaults, faults); + Interlocked.Add(ref _retiredBgInvocations, inv); + Interlocked.Add(ref _retiredBgBusyMs, busy); + Interlocked.Add(ref _retiredBgFaults, faults); } } } /// Record that a worker was checked out (started processing). - public static void RecordCheckout(int workerId, bool isHttp) + public void RecordCheckout(int workerId, bool isHttp) { - var stats = s_workerStats.GetOrAdd(workerId, _ => new WorkerStats { WorkerId = workerId }); + var stats = _workerStats.GetOrAdd(workerId, _ => new WorkerStats { WorkerId = workerId }); stats.IsHttp = isHttp; stats.LastCheckoutUtc = DateTime.UtcNow; stats.IsBusy = true; @@ -102,9 +95,9 @@ public static void RecordCheckout(int workerId, bool isHttp) } /// Record that a worker was reclaimed (finished processing). - public static void RecordReclaim(int workerId, bool faulted, long elapsedMs) + public void RecordReclaim(int workerId, bool faulted, long elapsedMs) { - if (!s_workerStats.TryGetValue(workerId, out var stats)) return; + if (!_workerStats.TryGetValue(workerId, out var stats)) return; stats.IsBusy = false; stats.LastReclaimUtc = DateTime.UtcNow; @@ -128,91 +121,84 @@ public static void RecordReclaim(int workerId, bool faulted, long elapsedMs) UpdateDurationStats(stats, elapsedMs); // Every 100 global invocations, attempt a memory trim (2-min cooldown) - var count = Interlocked.Increment(ref s_globalInvocations); + var count = Interlocked.Increment(ref _globalInvocations); if (count % 100 == 0) { var reclaimed = TrimMemory(); if (reclaimed >= 0) - s_logger?.LogInformation("[System] Memory trim at invocation {Count}: reclaimed ~{MB}MB {Memory}", + _logger.LogInformation("[System] Memory trim at invocation {Count}: reclaimed ~{MB}MB {Memory}", count, reclaimed, BackgroundTaskLimiter.GetMemorySnapshot()); } } /// Record the function name being executed on a worker. - public static void RecordFunction(int workerId, string functionName) + public void RecordFunction(int workerId, string functionName) { - if (!s_workerStats.TryGetValue(workerId, out var stats)) return; + if (!_workerStats.TryGetValue(workerId, out var stats)) return; stats.CurrentFunction = functionName; } // ── Public query methods ── /// Get a full snapshot of all worker metrics. - public static WorkerMetricsSnapshot GetSnapshot() + public WorkerMetricsSnapshot GetSnapshot() { var snapshot = new WorkerMetricsSnapshot { TimestampUtc = DateTime.UtcNow, - UptimeSeconds = (long)(DateTime.UtcNow - s_startTimeUtc).TotalSeconds, + UptimeSeconds = (long)(DateTime.UtcNow - _startTimeUtc).TotalSeconds, }; - if (s_pool != null) - { - var httpWorkers = new List(); - var bgWorkers = new List(); - - foreach (var (workerId, stats) in s_workerStats) - { - var detail = BuildWorkerDetail(stats); - if (stats.IsHttp) - httpWorkers.Add(detail); - else - bgWorkers.Add(detail); - } + var httpWorkers = new List(); + var bgWorkers = new List(); - snapshot.HttpPool = new PoolMetrics - { - PoolSize = s_pool.HttpPoolSize, - Available = s_pool.HttpAvailable, - BusyCount = s_pool.HttpPoolSize - s_pool.HttpAvailable, - Workers = httpWorkers, - }; + foreach (var (workerId, stats) in _workerStats) + { + var detail = BuildWorkerDetail(stats); + if (stats.IsHttp) + httpWorkers.Add(detail); + else + bgWorkers.Add(detail); + } - snapshot.BgPool = new PoolMetrics - { - PoolSize = s_pool.BgPoolSize, - Available = s_pool.BgAvailable, - BusyCount = s_pool.BgPoolSize - s_pool.BgAvailable, - Workers = bgWorkers, - }; + snapshot.HttpPool = new PoolMetrics + { + PoolSize = _pool.HttpPoolSize, + Available = _pool.HttpAvailable, + BusyCount = _pool.HttpPoolSize - _pool.HttpAvailable, + Workers = httpWorkers, + }; - // Aggregate pool-level stats (live workers + retired buckets so totals are monotonic) - AggregatePoolStats(snapshot.HttpPool, httpWorkers, - Interlocked.Read(ref s_retiredHttpInvocations), - Interlocked.Read(ref s_retiredHttpBusyMs), - Interlocked.Read(ref s_retiredHttpFaults)); - AggregatePoolStats(snapshot.BgPool, bgWorkers, - Interlocked.Read(ref s_retiredBgInvocations), - Interlocked.Read(ref s_retiredBgBusyMs), - Interlocked.Read(ref s_retiredBgFaults)); - } + snapshot.BgPool = new PoolMetrics + { + PoolSize = _pool.BgPoolSize, + Available = _pool.BgAvailable, + BusyCount = _pool.BgPoolSize - _pool.BgAvailable, + Workers = bgWorkers, + }; - if (s_limiter != null) + // Aggregate pool-level stats (live workers + retired buckets so totals are monotonic) + AggregatePoolStats(snapshot.HttpPool, httpWorkers, + Interlocked.Read(ref _retiredHttpInvocations), + Interlocked.Read(ref _retiredHttpBusyMs), + Interlocked.Read(ref _retiredHttpFaults)); + AggregatePoolStats(snapshot.BgPool, bgWorkers, + Interlocked.Read(ref _retiredBgInvocations), + Interlocked.Read(ref _retiredBgBusyMs), + Interlocked.Read(ref _retiredBgFaults)); + + snapshot.Limiter = new LimiterMetrics { - snapshot.Limiter = new LimiterMetrics - { - BaseConcurrency = s_limiter.BaseConcurrency, - CeilingConcurrency = s_limiter.CeilingConcurrency, - CurrentMax = s_limiter.CurrentMax, - Active = s_limiter.Active, - Waiting = s_limiter.Waiting, - IsHttpThrottled = s_limiter.IsHttpThrottled, - }; - } + BaseConcurrency = _limiter.BaseConcurrency, + CeilingConcurrency = _limiter.CeilingConcurrency, + CurrentMax = _limiter.CurrentMax, + Active = _limiter.Active, + Waiting = _limiter.Waiting, + IsHttpThrottled = _limiter.IsHttpThrottled, + }; - if (s_jobManager != null) { - var summary = s_jobManager.GetSummary(); + var summary = _jobManager.GetSummary(); snapshot.Jobs = new JobMetrics { Queued = summary.Queued, @@ -266,36 +252,36 @@ public static WorkerMetricsSnapshot GetSnapshot() return snapshot; } - private static DateTime s_prevCpuSampleUtc = DateTime.UtcNow; - private static TimeSpan s_prevCpuTime = Process.GetCurrentProcess().TotalProcessorTime; - private static double s_lastCpuPct; - private static readonly object s_cpuLock = new(); - private static readonly int s_processorCount = Environment.ProcessorCount; + private DateTime _prevCpuSampleUtc = DateTime.UtcNow; + private TimeSpan _prevCpuTime = Process.GetCurrentProcess().TotalProcessorTime; + private double _lastCpuPct; + private readonly object _cpuLock = new(); + private readonly int _processorCount = Environment.ProcessorCount; /// /// Compute process CPU% since the last call. Returns a value 0–100 representing /// usage across all cores (e.g. 50% on a 2-core machine = 1 core fully busy). /// Uses a minimum 500ms window to avoid divide-by-tiny-number noise. /// - private static double GetCpuPct() + private double GetCpuPct() { - lock (s_cpuLock) + lock (_cpuLock) { var now = DateTime.UtcNow; - var elapsed = now - s_prevCpuSampleUtc; + var elapsed = now - _prevCpuSampleUtc; if (elapsed.TotalMilliseconds < 500) - return s_lastCpuPct; // Too soon — return last known value + return _lastCpuPct; // Too soon — return last known value var proc = Process.GetCurrentProcess(); var cpuTime = proc.TotalProcessorTime; - var cpuDelta = cpuTime - s_prevCpuTime; + var cpuDelta = cpuTime - _prevCpuTime; - s_lastCpuPct = Math.Round(cpuDelta.TotalMilliseconds / (elapsed.TotalMilliseconds * s_processorCount) * 100, 1); - s_prevCpuSampleUtc = now; - s_prevCpuTime = cpuTime; + _lastCpuPct = Math.Round(cpuDelta.TotalMilliseconds / (elapsed.TotalMilliseconds * _processorCount) * 100, 1); + _prevCpuSampleUtc = now; + _prevCpuTime = cpuTime; - return s_lastCpuPct; + return _lastCpuPct; } } @@ -355,23 +341,23 @@ private static double GetCpuPct() return null; } - private static long s_prevContainerCpuUsec; - private static DateTime s_prevContainerCpuSampleUtc = DateTime.MinValue; - private static double s_lastContainerCpuPct; - private static readonly object s_containerCpuLock = new(); + private long _prevContainerCpuUsec; + private DateTime _prevContainerCpuSampleUtc = DateTime.MinValue; + private double _lastContainerCpuPct; + private readonly object _containerCpuLock = new(); /// /// Read total CPU usage across the container cgroup. Returns 0 on non-Linux. Same /// 500ms minimum-window guard as GetCpuPct so consecutive calls are cheap. /// - private static double GetContainerCpuPct() + private double GetContainerCpuPct() { - lock (s_containerCpuLock) + lock (_containerCpuLock) { var now = DateTime.UtcNow; - var elapsed = now - s_prevContainerCpuSampleUtc; - if (s_prevContainerCpuSampleUtc != DateTime.MinValue && elapsed.TotalMilliseconds < 500) - return s_lastContainerCpuPct; + var elapsed = now - _prevContainerCpuSampleUtc; + if (_prevContainerCpuSampleUtc != DateTime.MinValue && elapsed.TotalMilliseconds < 500) + return _lastContainerCpuPct; long? usec = null; try @@ -403,19 +389,19 @@ private static double GetContainerCpuPct() if (!usec.HasValue) return 0; - if (s_prevContainerCpuSampleUtc == DateTime.MinValue) + if (_prevContainerCpuSampleUtc == DateTime.MinValue) { - s_prevContainerCpuUsec = usec.Value; - s_prevContainerCpuSampleUtc = now; + _prevContainerCpuUsec = usec.Value; + _prevContainerCpuSampleUtc = now; return 0; } - var cpuDeltaMs = (usec.Value - s_prevContainerCpuUsec) / 1000.0; - s_lastContainerCpuPct = Math.Round( - cpuDeltaMs / (elapsed.TotalMilliseconds * s_processorCount) * 100, 1); - s_prevContainerCpuUsec = usec.Value; - s_prevContainerCpuSampleUtc = now; - return s_lastContainerCpuPct; + var cpuDeltaMs = (usec.Value - _prevContainerCpuUsec) / 1000.0; + _lastContainerCpuPct = Math.Round( + cpuDeltaMs / (elapsed.TotalMilliseconds * _processorCount) * 100, 1); + _prevContainerCpuUsec = usec.Value; + _prevContainerCpuSampleUtc = now; + return _lastContainerCpuPct; } } @@ -424,7 +410,7 @@ private static double GetContainerCpuPct() /// fragmentation, pinned objects, thread info, loaded assemblies, and native memory. /// Designed for diagnostics — more expensive than the basic MemoryMetrics in GetSnapshot(). /// - public static MemoryBreakdown GetMemoryBreakdown() + public MemoryBreakdown GetMemoryBreakdown() { var proc = Process.GetCurrentProcess(); var gcInfo = GC.GetGCMemoryInfo(GCKind.FullBlocking); @@ -489,8 +475,8 @@ _ when name.Contains("PowerShell", StringComparison.OrdinalIgnoreCase) || } // Pool memory estimates (each runspace holds modules, function table, variable table) - var httpPoolSize = s_pool?.HttpPoolSize ?? 0; - var bgPoolSize = s_pool?.BgPoolSize ?? 0; + var httpPoolSize = _pool.HttpPoolSize; + var bgPoolSize = _pool.BgPoolSize; var totalWorkers = httpPoolSize + bgPoolSize; // Native memory = RSS - committed managed @@ -506,7 +492,7 @@ _ when name.Contains("PowerShell", StringComparison.OrdinalIgnoreCase) || return new MemoryBreakdown { TimestampUtc = DateTime.UtcNow, - UptimeSeconds = (long)(DateTime.UtcNow - s_startTimeUtc).TotalSeconds, + UptimeSeconds = (long)(DateTime.UtcNow - _startTimeUtc).TotalSeconds, // Top-level sizes HeapBytes = heapBytes, @@ -544,7 +530,7 @@ _ when name.Contains("PowerShell", StringComparison.OrdinalIgnoreCase) || // Threads ThreadCount = threads.Count, ThreadStates = threadStates, - ProcessorCount = s_processorCount, + ProcessorCount = _processorCount, // Assemblies LoadedAssemblyCount = assemblies.Length, @@ -565,7 +551,7 @@ _ when name.Contains("PowerShell", StringComparison.OrdinalIgnoreCase) || } /// Get metrics for a specific pool type ("http" or "bg"). - public static PoolMetrics? GetPoolMetrics(string poolType) + public PoolMetrics? GetPoolMetrics(string poolType) { var snapshot = GetSnapshot(); return poolType.Equals("http", StringComparison.OrdinalIgnoreCase) @@ -574,56 +560,56 @@ _ when name.Contains("PowerShell", StringComparison.OrdinalIgnoreCase) || } /// Get a summary of just the busy/available counts. - public static WorkerSummary GetSummary() + public WorkerSummary GetSummary() { return new WorkerSummary { - HttpBusy = s_pool != null ? s_pool.HttpPoolSize - s_pool.HttpAvailable : 0, - HttpAvailable = s_pool?.HttpAvailable ?? 0, - HttpPoolSize = s_pool?.HttpPoolSize ?? 0, - BgBusy = s_pool != null ? s_pool.BgPoolSize - s_pool.BgAvailable : 0, - BgAvailable = s_pool?.BgAvailable ?? 0, - BgPoolSize = s_pool?.BgPoolSize ?? 0, - LimiterActive = s_limiter?.Active ?? 0, - LimiterWaiting = s_limiter?.Waiting ?? 0, - LimiterMax = s_limiter?.CurrentMax ?? 0, - IsHttpThrottled = s_limiter?.IsHttpThrottled ?? false, - JobsQueued = s_jobManager?.QueuedCount ?? 0, - JobsActive = s_jobManager?.ActiveCount ?? 0, + HttpBusy = _pool.HttpPoolSize - _pool.HttpAvailable, + HttpAvailable = _pool.HttpAvailable, + HttpPoolSize = _pool.HttpPoolSize, + BgBusy = _pool.BgPoolSize - _pool.BgAvailable, + BgAvailable = _pool.BgAvailable, + BgPoolSize = _pool.BgPoolSize, + LimiterActive = _limiter.Active, + LimiterWaiting = _limiter.Waiting, + LimiterMax = _limiter.CurrentMax, + IsHttpThrottled = _limiter.IsHttpThrottled, + JobsQueued = _jobManager.QueuedCount, + JobsActive = _jobManager.ActiveCount, }; } // ── Job management (exposed to PowerShell) ── /// Get detailed job list with wait/duration times. - public static List GetJobDetails(string? runName = null, string? status = null, int limit = 100) - => s_jobManager?.GetJobDetails(runName, status, limit) ?? new(); + public List GetJobDetails(string? runName = null, string? status = null, int limit = 100) + => _jobManager.GetJobDetails(runName, status, limit); /// Get run group summaries. - public static List GetRunSummaries() - => s_jobManager?.GetRunSummaries() ?? new(); + public List GetRunSummaries() + => _jobManager.GetRunSummaries(); /// Cancel a single queued job by ID. - public static bool CancelJob(string jobId) - => s_jobManager?.CancelJob(jobId) ?? false; + public bool CancelJob(string jobId) + => _jobManager.CancelJob(jobId); /// Cancel all queued jobs in a run group. - public static int CancelRun(string runName) - => s_jobManager?.CancelRun(runName) ?? 0; + public int CancelRun(string runName) + => _jobManager.CancelRun(runName); /// Delete a completed/failed/cancelled job from tracking. - public static bool DeleteJob(string jobId) - => s_jobManager?.DeleteJob(jobId) ?? false; + public bool DeleteJob(string jobId) + => _jobManager.DeleteJob(jobId); /// Change a queued job's priority (re-enqueues with new priority). - public static bool ChangePriority(string jobId, int newPriority) - => s_jobManager?.ChangePriority(jobId, newPriority) ?? false; + public bool ChangePriority(string jobId, int newPriority) + => _jobManager.ChangePriority(jobId, newPriority); // ── Private helpers ── - private static WorkerDetail BuildWorkerDetail(WorkerStats stats) + private WorkerDetail BuildWorkerDetail(WorkerStats stats) { - var uptimeMs = (long)(DateTime.UtcNow - s_startTimeUtc).TotalMilliseconds; + var uptimeMs = (long)(DateTime.UtcNow - _startTimeUtc).TotalMilliseconds; var utilizationPct = uptimeMs > 0 ? Math.Round(Interlocked.Read(ref stats._totalBusyMs) * 100.0 / uptimeMs, 1) : 0; @@ -687,7 +673,7 @@ private static void UpdateDurationStats(WorkerStats stats, long durationMs) while (durationMs > current && Interlocked.CompareExchange(ref stats.MaxDurationMs, durationMs, current) != current); } - private static long s_lastTrimTicks; + private long _lastTrimTicks; /// /// Force a full GC collection with LOH compaction and working-set trim. @@ -695,16 +681,16 @@ private static void UpdateDurationStats(WorkerStats stats, long durationMs) /// Has a built-in 2-minute cooldown to avoid GC thrashing. /// Returns the MB reclaimed, or -1 if skipped due to cooldown. /// - public static long TrimMemory() + public long TrimMemory() { // Cooldown: skip if last trim was less than 2 minutes ago var now = Stopwatch.GetTimestamp(); - var lastTrim = Interlocked.Read(ref s_lastTrimTicks); + var lastTrim = Interlocked.Read(ref _lastTrimTicks); if (lastTrim > 0 && Stopwatch.GetElapsedTime(lastTrim).TotalMinutes < 2) return -1; // CAS to claim the trim — only one caller wins - if (Interlocked.CompareExchange(ref s_lastTrimTicks, now, lastTrim) != lastTrim) + if (Interlocked.CompareExchange(ref _lastTrimTicks, now, lastTrim) != lastTrim) return -1; var proc = Process.GetCurrentProcess(); @@ -722,4 +708,5 @@ public static long TrimMemory() var rssAfter = proc.WorkingSet64; return (rssBefore - rssAfter) / (1024 * 1024); } + } diff --git a/Services/Bridges/WorkerStats.cs b/src/Craft/Services/Hosting/WorkerStats.cs similarity index 53% rename from Services/Bridges/WorkerStats.cs rename to src/Craft/Services/Hosting/WorkerStats.cs index f2e807a..0ae78e7 100644 --- a/Services/Bridges/WorkerStats.cs +++ b/src/Craft/Services/Hosting/WorkerStats.cs @@ -1,19 +1,11 @@ -// NAMESPACE PINNED — do not change. -// Downstream PowerShell reaches these types by fully-qualified name, e.g. -// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) -// Renaming the namespace compiles fine and then fails at runtime in the hosted app -// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. -// The folder is free to move; the namespace is a published contract. -namespace Craft.Services; - -// ── Data models ── +namespace Craft.Hosting; // Fields, not properties, by necessity: IsBusy is volatile and the _total* fields are passed to // Interlocked by ref. Neither is possible on a property. This is internal mutable state, projected -// into WorkerDetail before anything outside the bridge sees it. +// into WorkerDetail (Craft.Contracts) before anything outside the host sees it. [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1051:Do not declare visible instance fields", Justification = "IsBusy is volatile and _total* fields are Interlocked targets; neither works on a property.")] -public class WorkerStats +internal sealed class WorkerStats { public int WorkerId; public bool IsHttp; @@ -28,8 +20,8 @@ public class WorkerStats public long LastAllocBytes; // Interlocked fields - internal long _totalInvocations; - internal long _totalBusyMs; - internal long _totalFaults; - internal long _totalAllocBytes; + public long _totalInvocations; + public long _totalBusyMs; + public long _totalFaults; + public long _totalAllocBytes; } diff --git a/Services/Orchestration/BackgroundTaskLimiter.cs b/src/Craft/Services/Orchestration/BackgroundTaskLimiter.cs similarity index 95% rename from Services/Orchestration/BackgroundTaskLimiter.cs rename to src/Craft/Services/Orchestration/BackgroundTaskLimiter.cs index 1f3fbd4..3ffac53 100644 --- a/Services/Orchestration/BackgroundTaskLimiter.cs +++ b/src/Craft/Services/Orchestration/BackgroundTaskLimiter.cs @@ -83,39 +83,39 @@ public class BackgroundTaskLimiter : IDisposable public int Waiting => _waiting; public bool IsHttpThrottled => _httpThrottled; - public BackgroundTaskLimiter(ILogger logger, IConfiguration configuration, + public BackgroundTaskLimiter(ILogger logger, CraftSettings settings, PowerShellWorkerPool pool) { _logger = logger; _pool = pool; + var limiter = settings.BackgroundLimiter; + var bgPoolSize = Math.Max(1, settings.Worker.BgPoolSize); + var httpPoolSize = Math.Max(1, settings.Worker.HttpPoolSize); + // Baseline: low memory footprint when idle - BaseConcurrency = configuration.GetValue("BackgroundBaseConcurrency", - Math.Clamp(Environment.ProcessorCount, 2, 4)); + BaseConcurrency = limiter.BaseConcurrency + ?? Math.Clamp(Environment.ProcessorCount, 2, 4); // Ceiling: capped to BgPoolSize since the pool is the real bottleneck. - var bgPoolSize = Math.Max(1, settings.Worker.BgPoolSize); - CeilingConcurrency = configuration.GetValue("BackgroundMaxConcurrency", bgPoolSize); + CeilingConcurrency = limiter.MaxConcurrency ?? bgPoolSize; // How long the BG queue must be backed up before we scale up - ScaleUpAfter = TimeSpan.FromSeconds( - configuration.GetValue("BackgroundScaleUpAfterSeconds", 15)); + ScaleUpAfter = TimeSpan.FromSeconds(Math.Max(0, limiter.ScaleUpAfterSeconds)); // HTTP pressure: when this many HTTP workers are busy, throttle BG to 1 - var httpPoolSize = Math.Max(1, settings.Worker.HttpPoolSize); - HttpPressureThreshold = configuration.GetValue("BackgroundHttpPressureThreshold", - Math.Max(1, httpPoolSize / 2)); + HttpPressureThreshold = limiter.HttpPressureThreshold + ?? Math.Max(1, httpPoolSize / 2); // How long HTTP pressure must persist before throttling - HttpPressureAfter = TimeSpan.FromSeconds( - configuration.GetValue("BackgroundHttpPressureAfterSeconds", 10)); + HttpPressureAfter = TimeSpan.FromSeconds(Math.Max(0, limiter.HttpPressureAfterSeconds)); // Burst: jump straight to ceiling on the first sign of queueing (skip the ScaleUpAfter dwell). - _burstToCeiling = configuration.GetValue("BackgroundBurstToCeiling", false); + _burstToCeiling = limiter.BurstToCeiling; // Over-subscription: admit this many tasks ABOVE the worker target so they can do their pre-invoke // table writes and queue at the worker checkout while the pool stays full. 0 = off (strict pool cap). - _overSubscribe = Math.Max(0, configuration.GetValue("BackgroundOverSubscribe", 0)); + _overSubscribe = Math.Max(0, limiter.OverSubscribe); // Clamp to the ceiling. BaseConcurrency derives from ProcessorCount while CeilingConcurrency // derives from BgPoolSize, so a small pool on a big host (e.g. BgPoolSize=2 on 4 cores) started diff --git a/Services/Orchestration/JobDescriptor.cs b/src/Craft/Services/Orchestration/JobDescriptor.cs similarity index 100% rename from Services/Orchestration/JobDescriptor.cs rename to src/Craft/Services/Orchestration/JobDescriptor.cs diff --git a/Services/Orchestration/JobManager.cs b/src/Craft/Services/Orchestration/JobManager.cs similarity index 95% rename from Services/Orchestration/JobManager.cs rename to src/Craft/Services/Orchestration/JobManager.cs index 168ece7..12d38ce 100644 --- a/Services/Orchestration/JobManager.cs +++ b/src/Craft/Services/Orchestration/JobManager.cs @@ -63,9 +63,9 @@ public class JobManager : BackgroundService private readonly ConcurrentDictionary _reprioritized = new(); private long _totalProcessed; - // ── Descriptor rehydration + durability ── - private volatile JobWorkResolver? _resolver; - private volatile IJobDescriptorStateWriter? _stateWriter; + // ── Descriptor rehydration + durability (Lazy avoids ctor cycle with OrchestratorService) ── + private readonly Lazy? _resolver; + private readonly Lazy? _stateWriter; // ── Cleanup ── private readonly Timer _cleanupTimer; @@ -88,10 +88,17 @@ public class JobManager : BackgroundService public bool IsQueuedOrRunning(string jobId) => _jobs.TryGetValue(jobId, out var record) && record.Status is "Queued" or "Running"; - public JobManager(ILogger logger, CraftSettings settings, BackgroundTaskLimiter limiter) + public JobManager( + ILogger logger, + CraftSettings settings, + BackgroundTaskLimiter limiter, + Lazy? workResolver = null, + Lazy? stateWriter = null) { _logger = logger; _limiter = limiter; + _resolver = workResolver; + _stateWriter = stateWriter; MaxConcurrency = Math.Max(1, settings.Worker.BgPoolSize); _cleanupTimer = new Timer(_ => CleanupOldJobs(), null, CleanupInterval, CleanupInterval); @@ -138,20 +145,6 @@ public string Enqueue(string name, int priority, Func w return jobId; } - /// - /// Register the resolver that turns a into runnable work at dispatch - /// time. Called once at startup by . Without a resolver, descriptor - /// jobs fail fast rather than silently vanishing. - /// - public void SetWorkResolver(JobWorkResolver resolver) => _resolver = resolver; - - /// - /// Register the sink that persists operator-initiated changes to queued descriptor jobs - /// (reprioritize, cancel) so they survive a restart. Called at startup alongside - /// . Optional: without it those changes stay in-memory only. - /// - public void SetDescriptorStateWriter(IJobDescriptorStateWriter writer) => _stateWriter = writer; - /// /// Find the live queue entry for a job id. Superseded entries (an older epoch left behind by /// ) are skipped. Caller must hold . @@ -390,7 +383,7 @@ public List GetRunSummaries() .ToList(); } - public List GetJobs(string? runName = null, string? status = null, int? limit = null) + internal List GetJobs(string? runName = null, string? status = null, int? limit = null) { var query = _jobs.Values.AsEnumerable(); @@ -574,8 +567,18 @@ public bool ChangePriority(string jobId, int newPriority) /// private void NotifyStateWriter(Action action, string what, string jobName) { - var writer = _stateWriter; - if (writer == null) return; + IJobDescriptorStateWriter writer; + try + { + if (_stateWriter is null) return; + writer = _stateWriter.Value; + } + catch (Exception ex) + { + _logger.LogError(ex, "[JobManager] Failed to resolve state writer for {What} on {Name}", what, jobName); + return; + } + try { action(writer); @@ -629,7 +632,7 @@ public List GetJobDetails(string? runName = null, string? status = nu /// private async Task?> ResolveWorkAsync(JobDescriptor descriptor, CancellationToken ct) { - var resolver = _resolver + var resolver = _resolver?.Value ?? throw new InvalidOperationException( $"No JobWorkResolver registered — cannot dispatch descriptor {descriptor.RunName}/{descriptor.TaskId}"); diff --git a/src/Craft/Services/Orchestration/JobRecord.cs b/src/Craft/Services/Orchestration/JobRecord.cs new file mode 100644 index 0000000..2753bae --- /dev/null +++ b/src/Craft/Services/Orchestration/JobRecord.cs @@ -0,0 +1,18 @@ +namespace Craft.Orchestration; + +/// +/// In-memory job tracking record used by . +/// Projected to public JobDetail / API JSON before leaving the host. +/// +internal sealed class JobRecord +{ + public string Id { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public string? RunName { get; set; } + public int Priority { get; set; } + public string Status { get; set; } = "Queued"; + public DateTime QueuedUtc { get; set; } + public DateTime? StartedUtc { get; set; } + public DateTime? CompletedUtc { get; set; } + public string? LastError { get; set; } +} diff --git a/Services/Orchestration/MarkerNotPersistedException.cs b/src/Craft/Services/Orchestration/MarkerNotPersistedException.cs similarity index 100% rename from Services/Orchestration/MarkerNotPersistedException.cs rename to src/Craft/Services/Orchestration/MarkerNotPersistedException.cs diff --git a/Services/Orchestration/OrchestratorRun.cs b/src/Craft/Services/Orchestration/OrchestratorRun.cs similarity index 100% rename from Services/Orchestration/OrchestratorRun.cs rename to src/Craft/Services/Orchestration/OrchestratorRun.cs diff --git a/Services/Storage/OrchestratorRunSummary.cs b/src/Craft/Services/Orchestration/OrchestratorRunSummary.cs similarity index 95% rename from Services/Storage/OrchestratorRunSummary.cs rename to src/Craft/Services/Orchestration/OrchestratorRunSummary.cs index dd21983..9499b0d 100644 --- a/Services/Storage/OrchestratorRunSummary.cs +++ b/src/Craft/Services/Orchestration/OrchestratorRunSummary.cs @@ -1,4 +1,4 @@ -namespace Craft.Storage; +namespace Craft.Orchestration; /// /// A run's identity and parentage, read from the run row alone — no task rows. diff --git a/Services/Orchestration/OrchestratorService.cs b/src/Craft/Services/Orchestration/OrchestratorService.cs similarity index 91% rename from Services/Orchestration/OrchestratorService.cs rename to src/Craft/Services/Orchestration/OrchestratorService.cs index 78ba23a..ff85e99 100644 --- a/Services/Orchestration/OrchestratorService.cs +++ b/src/Craft/Services/Orchestration/OrchestratorService.cs @@ -4,7 +4,6 @@ using System.Text.Json.Serialization; using Craft.Configuration; using Craft.Services; -using Craft.Storage; namespace Craft.Orchestration; @@ -54,6 +53,13 @@ public class OrchestratorService : IJobDescriptorStateWriter /// private readonly ConcurrentDictionary _taskScriptPaths = new(); + // In-process ingress queues (PowerShell enqueues via OrchestratorBridge; C# drains here). + private readonly ConcurrentQueue _pending = new(); + private readonly ConcurrentQueue _pendingPlanners = new(); + + /// Fire-and-forget planner/finalize tasks still in flight (shutdown flush awaits these). + private readonly ConcurrentDictionary _backgroundWork = new(); + /// /// Get the Reference for a given run name, or null if not found/no reference set. /// @@ -95,11 +101,116 @@ public OrchestratorService( _store = store; _writer = writer; _settings = settings; + } + + /// + /// Queue an orchestrator run from PowerShell (via OrchestratorBridge). Drained by + /// after the invoking script returns. + /// + public void QueueOrchestration(string name, string batchJson, int priority, + string? postExecFunctionName = null, string? postExecParametersJson = null, + string? parentRunName = null, string? reference = null) + { + _pending.Enqueue(new PendingOrchestration(name, batchJson, priority, + postExecFunctionName, postExecParametersJson, parentRunName, reference)); + } + + /// + /// Queue a planner-based orchestrator run. The planner script builds the task list on a + /// background worker, then tasks dispatch — same path as the scheduler. + /// + public void QueuePlannerRun(string command, int priority) => + _pendingPlanners.Enqueue(new PendingPlannerRun(command, priority)); + + /// + /// Synchronous drain — blocks until all pending orchestrations are started. + /// Safe to call from any context (no SynchronizationContext on background workers). + /// Thin sync wrapper over for PowerShell bridge callers. + /// + public void DrainPending() => + DrainPendingAsync().ConfigureAwait(false).GetAwaiter().GetResult(); + + /// + /// Async drain — preferred from async call sites (PostExec lambdas, ExecuteScript). + /// + public async Task DrainPendingAsync() + { + while (_pending.TryDequeue(out var p)) + { + try + { + await StartFromBatchAsync(p.Name, p.BatchJson, p.Priority, + p.PostExecFunctionName, p.PostExecParametersJson, CancellationToken.None, + p.ParentRunName, p.Reference); + + if (!string.IsNullOrEmpty(p.ParentRunName)) + TryRegisterChildRun(p.ParentRunName, p.Name); + } + catch (Exception ex) + { + _logger.LogError(ex, "[Orchestrator] DrainPending failed for {Name}", p.Name); + } + } + DrainPendingPlanners(); + } + + private void DrainPendingPlanners() + { + while (_pendingPlanners.TryDequeue(out var p)) + { + // Fire-and-forget: planner runs on BG worker, dispatches tasks + var command = p.Command; + var priority = p.Priority; + async Task RunAsync() + { + try { await StartPlannerRunAsync(command, priority, CancellationToken.None); } + catch (Exception ex) + { + _logger.LogError(ex, "[Orchestrator] Planner run failed for {Command}", command); + } + } + TrackBackgroundWork(RunAsync()); + } + } - // The queue holds descriptors; this is how they become work again at dispatch time, and how - // operator changes to a queued task are made durable. - _jobManager.SetWorkResolver(ResolveTaskWorkAsync); - _jobManager.SetDescriptorStateWriter(this); + private void TrackBackgroundWork(Task task) + { + _backgroundWork[task] = 0; + _ = task.ContinueWith( + static (t, state) => ((ConcurrentDictionary)state!).TryRemove(t, out _), + _backgroundWork, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + } + + /// + /// Await in-flight planner/finalize work. Used on host shutdown after a final ingress drain. + /// Returns the count still incomplete after . + /// + internal async Task FlushBackgroundWorkAsync(TimeSpan timeout, CancellationToken cancellationToken) + { + var pending = _backgroundWork.Keys.ToArray(); + if (pending.Length == 0) return 0; + + var all = Task.WhenAll(pending); + var finished = await Task.WhenAny(all, Task.Delay(timeout, cancellationToken)); + if (finished == all) + { + try { await all; } + catch (Exception ex) + { + _logger.LogError(ex, "[Shutdown] Background orchestrator work faulted during flush"); + } + return 0; + } + + var leftovers = _backgroundWork.Count; + if (leftovers > 0) + _logger.LogWarning( + "[Shutdown] {Count} orchestrator background task(s) still running after {Timeout}", + leftovers, timeout); + return leftovers; } // ─── IJobDescriptorStateWriter ─── @@ -540,7 +651,11 @@ private void DispatchSingleTask(OrchestratorRun run, OrchestratorTaskItem task, /// never sees, and no run would ever finalize. Object identity — not the field values — is the one /// piece of state here that is genuinely not rehydratable. /// - private async Task?> ResolveTaskWorkAsync(JobDescriptor descriptor, CancellationToken ct) + /// + /// Rehydrates a queued descriptor into runnable work at dispatch time. Registered with + /// via DI as . + /// + internal async Task?> ResolveTaskWorkAsync(JobDescriptor descriptor, CancellationToken ct) { if (!_taskScriptPaths.TryGetValue(descriptor.RunName, out var taskPath)) { @@ -843,11 +958,12 @@ private void CheckRunCompletion(OrchestratorRun run) } // Cannot await inside lock — schedule finalization - _ = Task.Run(async () => + var finalize = Task.Run(async () => { try { await FinalizeRunAsync(run); } catch (Exception ex) { _logger.LogError(ex, "[Scheduler] FinalizeRun failed for {Name}", run.Name); } }); + TrackBackgroundWork(finalize); } } @@ -996,7 +1112,7 @@ private void DispatchPostExecution(OrchestratorRun run) await _psRunner.ExecuteScript(postExecScript, parameters); // PostExecution functions may call Start-CIPPOrchestrator (Phase 2) - await OrchestratorBridge.DrainPendingAsync(); + await DrainPendingAsync(); // Mark PostExec as Completed run.PostExecStatus = "Completed"; diff --git a/Services/Orchestration/OrchestratorStatusWriter.cs b/src/Craft/Services/Orchestration/OrchestratorStatusWriter.cs similarity index 99% rename from Services/Orchestration/OrchestratorStatusWriter.cs rename to src/Craft/Services/Orchestration/OrchestratorStatusWriter.cs index 10dd3e7..7ed45d2 100644 --- a/Services/Orchestration/OrchestratorStatusWriter.cs +++ b/src/Craft/Services/Orchestration/OrchestratorStatusWriter.cs @@ -1,13 +1,12 @@ using System.Text.Json; using Craft.Configuration; -using Craft.Storage; namespace Craft.Orchestration; /// /// Coalescing, batched, durable writer for orchestrator TASK and RUN status transitions. It removes the /// per-task Azure Table write from the fan-out critical path (that write was the throughput ceiling — see -/// docs/orch-analysis.md) by coalescing many transitions and flushing them in ≤100-entity, byte-budgeted +/// perf-harness/orch-analysis.md) by coalescing many transitions and flushing them in ≤100-entity, byte-budgeted /// transactions. /// /// Durability is preserved: diff --git a/Services/Storage/OrchestratorTableStore.cs b/src/Craft/Services/Orchestration/OrchestratorTableStore.cs similarity index 99% rename from Services/Storage/OrchestratorTableStore.cs rename to src/Craft/Services/Orchestration/OrchestratorTableStore.cs index 3a39257..d923c31 100644 --- a/Services/Storage/OrchestratorTableStore.cs +++ b/src/Craft/Services/Orchestration/OrchestratorTableStore.cs @@ -2,9 +2,9 @@ using System.Text; using System.Text.Json; using Craft.Configuration; -using Craft.Orchestration; +using Craft.Storage; -namespace Craft.Storage; +namespace Craft.Orchestration; /// /// Typed CRUD wrapper over for orchestrator persistence. Manages three diff --git a/Services/Orchestration/OrchestratorTaskItem.cs b/src/Craft/Services/Orchestration/OrchestratorTaskItem.cs similarity index 100% rename from Services/Orchestration/OrchestratorTaskItem.cs rename to src/Craft/Services/Orchestration/OrchestratorTaskItem.cs diff --git a/src/Craft/Services/Orchestration/PendingOrchestration.cs b/src/Craft/Services/Orchestration/PendingOrchestration.cs new file mode 100644 index 0000000..3baa6a3 --- /dev/null +++ b/src/Craft/Services/Orchestration/PendingOrchestration.cs @@ -0,0 +1,12 @@ +namespace Craft.Orchestration; + +/// Queued orchestration item drained by OrchestratorBridge / OrchestratorService. +internal sealed record PendingOrchestration(string Name, string BatchJson, int Priority, + string? PostExecFunctionName, string? PostExecParametersJson, string? ParentRunName, + string? Reference = null); + +/// Queued planner-based orchestrator run. +internal sealed record PendingPlannerRun(string Command, int Priority); + +/// Queued in-process background command (Add-CippQueueMessage shape). +internal sealed record PendingQueueCommand(string Cmdlet, string ParametersJson); diff --git a/src/Craft/Services/Orchestration/QueueDispatchService.cs b/src/Craft/Services/Orchestration/QueueDispatchService.cs new file mode 100644 index 0000000..baa8edb --- /dev/null +++ b/src/Craft/Services/Orchestration/QueueDispatchService.cs @@ -0,0 +1,84 @@ +using System.Collections.Concurrent; +using Craft.Configuration; +using Craft.Services; + +namespace Craft.Orchestration; + +/// +/// In-process queue for PowerShell Add-CippQueueMessage style background commands. +/// Domain code drains via this service; PowerShell enqueues through QueueBridge. +/// +public sealed class QueueDispatchService +{ + private readonly PowerShellRunnerService _runner; + private readonly JobManager _jobManager; + private readonly OrchestratorService _orchestrator; + private readonly ILogger _logger; + private readonly string _queueTaskFunction; + private readonly ConcurrentQueue _pending = new(); + + public QueueDispatchService( + PowerShellRunnerService runner, + JobManager jobManager, + OrchestratorService orchestrator, + CraftSettings settings, + ILogger logger) + { + _runner = runner; + _jobManager = jobManager; + _orchestrator = orchestrator; + _logger = logger; + _queueTaskFunction = settings.Orchestrator.QueueTaskFunction; + } + + public void Enqueue(string cmdlet, string parametersJson) => + _pending.Enqueue(new PendingQueueCommand(cmdlet, parametersJson)); + + public void DrainPending() + { + if (string.IsNullOrEmpty(_queueTaskFunction)) + { + if (!_pending.IsEmpty) + { + var dropped = 0; + while (_pending.TryDequeue(out _)) dropped++; + _logger.LogWarning( + "[Queue] Dropping {Count} pending command(s) — App:Orchestrator:QueueTaskFunction is not configured", + dropped); + } + return; + } + + while (_pending.TryDequeue(out var cmd)) + { + var scriptPath = _runner.FindScript(_queueTaskFunction); + if (scriptPath == null) + { + _logger.LogWarning( + "[Queue] Dropping command {Cmdlet} — queue task function '{Function}' was not found", + cmd.Cmdlet, _queueTaskFunction); + continue; + } + + var captured = cmd; + _jobManager.Enqueue( + name: $"Queue-{captured.Cmdlet}", + priority: 5, + runName: $"Queue-{captured.Cmdlet}-{Guid.NewGuid():N}", + id: $"Queue-{Guid.NewGuid():N}", + work: async (ct) => + { + var parameters = new Dictionary + { + { "Cmdlet", captured.Cmdlet }, + { "ParametersJson", captured.ParametersJson } + }; + await _runner.ExecuteScript(scriptPath, parameters); + + // Queued commands may trigger orchestrators + await _orchestrator.DrainPendingAsync(); + } + ); + } + } +} diff --git a/Services/Bridges/QueueStatusBridge.cs b/src/Craft/Services/Orchestration/QueueStatusService.cs similarity index 80% rename from Services/Bridges/QueueStatusBridge.cs rename to src/Craft/Services/Orchestration/QueueStatusService.cs index 008e196..856da11 100644 --- a/Services/Bridges/QueueStatusBridge.cs +++ b/src/Craft/Services/Orchestration/QueueStatusService.cs @@ -1,40 +1,41 @@ using System.Collections.Concurrent; using System.Text.Json; -using Craft.Orchestration; +using Craft.Services; -// NAMESPACE PINNED — do not change. -// Downstream PowerShell reaches these types by fully-qualified name, e.g. -// [Craft.Services.RealtimeBridge]::Publish($userId, $jobId, 'start', $data) -// Renaming the namespace compiles fine and then fails at runtime in the hosted app -// ("Unable to find type"). Type forwarding cannot help — it only works across assemblies. -// The folder is free to move; the namespace is a published contract. -namespace Craft.Services; +namespace Craft.Orchestration; /// -/// Static bridge allowing PowerShell (Get-CIPPQueueData) to query orchestrator/job -/// progress without HTTP round-trips. Returns data in the shape the CIPP frontend expects. -/// PS usage: [Craft.Services.QueueStatusBridge]::GetRunStatus($Reference, $QueueId) +/// Builds CIPP-shaped queue/run status projections from and +/// , plus PowerShell-registered display metadata. +/// Domain code and QueueStatusBridge both go through this service. /// -public static class QueueStatusBridge +public sealed class QueueStatusService { - private static JobManager? s_jobManager; - private static OrchestratorService? s_orchestratorService; + private readonly JobManager _jobManager; + private readonly OrchestratorService? _orchestratorService; /// /// Maps QueueId (GUID) or Reference to friendly display metadata (Name, Link). - /// Populated by New-CippQueueEntry in CIPPNG mode. + /// Populated by New-CippQueueEntry in CIPPNG mode. Static so registration remains + /// safe before the DI instance is wired into QueueStatusBridge. /// private static readonly ConcurrentDictionary s_queueMetadata = new(StringComparer.OrdinalIgnoreCase); - public static void Initialize(JobManager jobManager, OrchestratorService? orchestratorService = null) + private static readonly JsonSerializerOptions s_jsonOptions = new() { - s_jobManager = jobManager; - s_orchestratorService = orchestratorService; + PropertyNamingPolicy = null, + WriteIndented = false + }; + + public QueueStatusService(JobManager jobManager, OrchestratorService? orchestratorService = null) + { + _jobManager = jobManager; + _orchestratorService = orchestratorService; } /// /// Register friendly queue metadata from PowerShell (New-CippQueueEntry). - /// PS usage: [Craft.Services.QueueStatusBridge]::RegisterQueueMetadata($QueueId, $Name, $Link, $Reference) + /// Safe before the service is constructed — the metadata bag is static. /// public static void RegisterQueueMetadata(string queueId, string name, string link, string reference) { @@ -53,16 +54,14 @@ public static void RegisterQueueMetadata(string queueId, string name, string lin /// Optional run reference/name to filter by (maps to RunName in JobManager) /// Optional queue ID (same as reference in Craft context) /// JSON array of queue status objects - public static string GetRunStatus(string? reference = null, string? queueId = null) + public string GetRunStatus(string? reference = null, string? queueId = null) { - if (s_jobManager == null) return "[]"; - // PowerShell converts $null to "" when calling .NET string parameters, // so treat empty strings the same as null. var effectiveQueueId = string.IsNullOrEmpty(queueId) ? null : queueId; var effectiveReference = string.IsNullOrEmpty(reference) ? null : reference; var lookup = effectiveQueueId ?? effectiveReference; - var summaries = s_jobManager.GetRunSummaries(); + var summaries = _jobManager.GetRunSummaries(); if (!string.IsNullOrEmpty(lookup)) { @@ -72,9 +71,9 @@ public static string GetRunStatus(string? reference = null, string? queueId = nu .ToList(); // If no exact match, try matching by Reference via orchestrator service - if (matched.Count == 0 && s_orchestratorService != null) + if (matched.Count == 0 && _orchestratorService != null) { - var runName = s_orchestratorService.FindRunByReference(lookup); + var runName = _orchestratorService.FindRunByReference(lookup); if (runName != null) { matched = summaries @@ -107,7 +106,7 @@ public static string GetRunStatus(string? reference = null, string? queueId = nu var completedTasks = s.Completed + s.Failed; var total = Math.Max(s.Total, 1); var status = DeriveStatus(s); - var runReference = s_orchestratorService?.GetRunReference(s.Name) ?? s.Name; + var runReference = _orchestratorService?.GetRunReference(s.Name) ?? s.Name; // Look up friendly metadata by reference, then by run name, // then by QueueId GUID suffix (run names follow "OrchestratorName-" pattern) @@ -157,11 +156,9 @@ private static string DeriveStatus(JobRunSummary s) return "Queued"; } - private static List GetTaskDetails(string runName) + private List GetTaskDetails(string runName) { - if (s_jobManager == null) return []; - - var jobs = s_jobManager.GetJobs(runName, limit: 100); + var jobs = _jobManager.GetJobs(runName, limit: 100); return jobs.Select(j => new TaskDetail { Timestamp = (j.CompletedUtc ?? j.StartedUtc ?? j.QueuedUtc).ToString("O"), @@ -218,12 +215,6 @@ private sealed class TaskDetail public string Status { get; set; } = ""; } - private static readonly JsonSerializerOptions s_jsonOptions = new() - { - PropertyNamingPolicy = null, - WriteIndented = false - }; - private sealed class QueueMetadata { public string Name { get; set; } = ""; diff --git a/Services/Orchestration/SchedulerService.cs b/src/Craft/Services/Orchestration/SchedulerService.cs similarity index 100% rename from Services/Orchestration/SchedulerService.cs rename to src/Craft/Services/Orchestration/SchedulerService.cs diff --git a/Services/Orchestration/SchedulerTask.cs b/src/Craft/Services/Orchestration/SchedulerTask.cs similarity index 100% rename from Services/Orchestration/SchedulerTask.cs rename to src/Craft/Services/Orchestration/SchedulerTask.cs diff --git a/Services/Storage/TaskStatusWrite.cs b/src/Craft/Services/Orchestration/TaskStatusWrite.cs similarity index 95% rename from Services/Storage/TaskStatusWrite.cs rename to src/Craft/Services/Orchestration/TaskStatusWrite.cs index de0cf21..4b0c8fb 100644 --- a/Services/Storage/TaskStatusWrite.cs +++ b/src/Craft/Services/Orchestration/TaskStatusWrite.cs @@ -1,4 +1,4 @@ -namespace Craft.Storage; +namespace Craft.Orchestration; /// /// An immutable snapshot of a task's status for the coalescing batched writer. diff --git a/Services/PowerShellHost/ExportedModuleState.cs b/src/Craft/Services/PowerShellHost/ExportedModuleState.cs similarity index 100% rename from Services/PowerShellHost/ExportedModuleState.cs rename to src/Craft/Services/PowerShellHost/ExportedModuleState.cs diff --git a/Services/PowerShellHost/FunctionCategory.cs b/src/Craft/Services/PowerShellHost/FunctionCategory.cs similarity index 100% rename from Services/PowerShellHost/FunctionCategory.cs rename to src/Craft/Services/PowerShellHost/FunctionCategory.cs diff --git a/Services/PowerShellHost/FunctionEntry.cs b/src/Craft/Services/PowerShellHost/FunctionEntry.cs similarity index 100% rename from Services/PowerShellHost/FunctionEntry.cs rename to src/Craft/Services/PowerShellHost/FunctionEntry.cs diff --git a/Services/PowerShellHost/PowerShellRunnerService.cs b/src/Craft/Services/PowerShellHost/PowerShellRunnerService.cs similarity index 93% rename from Services/PowerShellHost/PowerShellRunnerService.cs rename to src/Craft/Services/PowerShellHost/PowerShellRunnerService.cs index bd1955f..7612af4 100644 --- a/Services/PowerShellHost/PowerShellRunnerService.cs +++ b/src/Craft/Services/PowerShellHost/PowerShellRunnerService.cs @@ -7,6 +7,7 @@ using System.Text.Json; using Craft.Configuration; using Craft.Hosting; +using Craft.Orchestration; using Craft.PowerShellHost; // NAMESPACE PINNED — do not change. @@ -25,6 +26,14 @@ public class PowerShellRunnerService : IDisposable private readonly WorkerSettings _workerSettings; private readonly AuthSettings _authSettings; private readonly ScriptRepoSettings _scriptsSettings; + private readonly WorkerMetricsService _metrics; + + // Lazy to avoid a ctor cycle with OrchestratorService / QueueDispatchService. + private readonly Lazy _orchestrator; + private readonly Lazy _queueDispatch; + + /// Fire-and-forget drain tasks still in flight (shutdown flush awaits these). + private readonly ConcurrentDictionary _backgroundDrains = new(); // Static JsonSerializerOptions — allocated once, reused everywhere private static readonly JsonSerializerOptions s_jsonOptions = new() { WriteIndented = false }; @@ -37,7 +46,10 @@ public PowerShellRunnerService( ILogger logger, PowerShellWorkerPool pool, ScriptRepository repo, - CraftSettings settings) + CraftSettings settings, + WorkerMetricsService metrics, + Lazy orchestrator, + Lazy queueDispatch) { _logger = logger; _pool = pool; @@ -45,6 +57,9 @@ public PowerShellRunnerService( _workerSettings = settings.Worker; _authSettings = settings.Auth; _scriptsSettings = settings.Scripts; + _metrics = metrics; + _orchestrator = orchestrator; + _queueDispatch = queueDispatch; } /// @@ -116,7 +131,7 @@ public async Task ExecuteHttpEndpoint(string endpoint, Hashtable r ["TriggerMetadata"] = triggerMetadata }; - WorkerMetricsBridge.RecordFunction(worker.Id, endpoint); + _metrics.RecordFunction(worker.Id, endpoint); // Subscribe to PS streams BEFORE invoke. worker.InvokeAsync's finally calls // Cleanup() which clears the streams, so post-invoke iteration sees nothing. @@ -162,8 +177,7 @@ public async Task ExecuteHttpEndpoint(string endpoint, Hashtable r _logger.LogInformation("[HTTP] {Function} {StatusCode} {Ms}ms", endpoint, response.StatusCode, sw.ElapsedMilliseconds); // Process any orchestrator/queue triggers queued during execution - await OrchestratorBridge.DrainPendingAsync(); - QueueBridge.DrainPending(); + await DrainPendingWorkAsync(); return response; } @@ -313,7 +327,7 @@ private async Task ExecuteHttpScriptInternal(string route, Hashtab Category = "HTTP" }; using var opScope = OperationContext.Set(invocation); - WorkerMetricsBridge.RecordFunction(worker.Id, entry.FunctionName); + _metrics.RecordFunction(worker.Id, entry.FunctionName); _logger.LogInformation("[{Pool}] {InvocationId} {Function} starting on {Worker}", poolLabel, invocation.Id, entry.FunctionName, invocation.WorkerId); @@ -383,8 +397,7 @@ private async Task ExecuteHttpScriptInternal(string route, Hashtab poolLabel, invocation.Id, entry.FunctionName, response.StatusCode, sw.ElapsedMilliseconds); // Process any orchestrator/queue triggers queued during execution - await OrchestratorBridge.DrainPendingAsync(); - QueueBridge.DrainPending(); + await DrainPendingWorkAsync(); return response; } @@ -425,23 +438,65 @@ private async Task ExecuteHttpScriptInternal(string route, Hashtab } /// - /// Drain pending orchestrator/queue triggers off the calling thread. Bridges are thread-safe - /// concurrent queues, so multiple in-flight drain calls are fine — each TryDequeue serialises. + /// Drain pending orchestrator/queue triggers off the calling thread. Ingress queues are + /// thread-safe, so multiple in-flight drain calls are fine — each TryDequeue serialises. /// private void DrainBridgesInBackground() { - _ = Task.Run(async () => + var task = Task.Run(async () => { try { - await OrchestratorBridge.DrainPendingAsync(); - QueueBridge.DrainPending(); + await DrainPendingWorkAsync(); } catch (Exception ex) { - _logger.LogError(ex, "[Scheduler] Background bridge drain failed"); + _logger.LogError(ex, "[Scheduler] Background pending-work drain failed"); } }); + TrackBackgroundDrain(task); + } + + private void TrackBackgroundDrain(Task task) + { + _backgroundDrains[task] = 0; + _ = task.ContinueWith( + static (t, state) => ((ConcurrentDictionary)state!).TryRemove(t, out _), + _backgroundDrains, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + } + + private async Task DrainPendingWorkAsync() + { + await _orchestrator.Value.DrainPendingAsync(); + _queueDispatch.Value.DrainPending(); + } + + /// + /// Await in-flight background drains started by this runner. Used on host shutdown after a + /// final ingress drain. Returns the count still incomplete after . + /// + internal async Task FlushBackgroundDrainsAsync(TimeSpan timeout, CancellationToken cancellationToken) + { + var pending = _backgroundDrains.Keys.ToArray(); + if (pending.Length == 0) return 0; + + var all = Task.WhenAll(pending); + var finished = await Task.WhenAny(all, Task.Delay(timeout, cancellationToken)); + if (finished == all) + { + await all; // observe faults + return 0; + } + + var leftovers = _backgroundDrains.Count; + if (leftovers > 0) + _logger.LogWarning( + "[Shutdown] {Count} background pending-work drain(s) still running after {Timeout}", + leftovers, timeout); + return leftovers; } /// @@ -470,7 +525,7 @@ public async Task ExecuteScript(string functionName, Dictionary? using var opScope = OperationContext.Set(invocation); // Show the job name (e.g. "CIPPDBCacheRun-Graph_tenant.com") rather than // the generic function name (e.g. "Invoke-CraftTask") in worker metrics - WorkerMetricsBridge.RecordFunction(worker.Id, parentFunction ?? functionName); + _metrics.RecordFunction(worker.Id, parentFunction ?? functionName); EventHandler? onError = null; EventHandler? onWarning = null; @@ -598,7 +653,7 @@ public async Task ExecuteScriptWithOutput(string functionName, Dictionar Category = "Planner" }; using var opScope = OperationContext.Set(invocation); - WorkerMetricsBridge.RecordFunction(worker.Id, parentFunction ?? functionName); + _metrics.RecordFunction(worker.Id, parentFunction ?? functionName); EventHandler? onError = null; EventHandler? onInfo = null; EventHandler? onDebug = null; @@ -1049,7 +1104,8 @@ private static string ConvertPsObjectToJson(object? obj) public void Dispose() { - _pool.Dispose(); + // Pool lifetime is owned by the DI container (PowerShellWorkerPool is a separate singleton). + // Disposing it here tore down workers still used by orchestrator/jobs/dispatch on runner teardown. GC.SuppressFinalize(this); } } diff --git a/Services/PowerShellHost/PowerShellWorker.cs b/src/Craft/Services/PowerShellHost/PowerShellWorker.cs similarity index 96% rename from Services/PowerShellHost/PowerShellWorker.cs rename to src/Craft/Services/PowerShellHost/PowerShellWorker.cs index 6952a1f..4a22518 100644 --- a/Services/PowerShellHost/PowerShellWorker.cs +++ b/src/Craft/Services/PowerShellHost/PowerShellWorker.cs @@ -46,7 +46,7 @@ public void Initialize(ScriptRepository repo, string apiBasePath, CraftSettings // Reuse one pipeline thread across invocations instead of spinning a new thread per BeginInvoke // (measured ~50% of the PS-invoke cost). Must be set before the runspace opens — GetGlobalVariables() - // below is the first SessionStateProxy access, which opens it. On by default; see docs/dispatch-analysis.md. + // below is the first SessionStateProxy access, which opens it. On by default; see perf-harness/dispatch-analysis.md. if (settings.Worker.ReuseRunspaceThread) _pwsh.Runspace.ThreadOptions = PSThreadOptions.ReuseThread; @@ -66,12 +66,14 @@ public void Initialize(ScriptRepository repo, string apiBasePath, CraftSettings // Common using namespaces needed by HTTP scripts RunScript("using namespace System.Net"); - // HttpResponseContext — derive the runspace class from the Craft.dll-compiled base type - // (Microsoft.Azure.Functions.PowerShellWorker.HttpResponseContext). PowerShell classes are - // compiled by the PS engine (no Roslyn), so this resolves [HttpResponseContext] by short - // name in the runtime container image, where Add-Type -TypeDefinition does NOT (no C# - // compiler). The base type's namespace-qualified name lands in $_.PSObject.TypeNames, so - // CIPP's New-CippCoreRequest response filter matches it exactly as under Azure Functions. + // HttpResponseContext — derive a runspace-level PowerShell class from the compiled base type + // in Craft.Contracts (Microsoft.Azure.Functions.PowerShellWorker.HttpResponseContext). + // PS classes are compiled by the PS engine (no Roslyn), so scripts resolve [HttpResponseContext] + // by short name while instances still carry the Functions-worker type name in PSTypeNames. + // The base type is compiled into the host image rather than Add-Type'd at runtime: Roslyn and + // ref/ are shipped (see build/Dockerfile), but this type is on every response path and must + // exist before the first script runs. See HttpResponseContext XML docs for the historical + // "no C# compiler" confusion. RunScript("class HttpResponseContext : Microsoft.Azure.Functions.PowerShellWorker.HttpResponseContext {}"); // // CRAFT_ROOT is always set — scripts use $env:CRAFT_ROOT to find the API root diff --git a/Services/PowerShellHost/PowerShellWorkerPool.cs b/src/Craft/Services/PowerShellHost/PowerShellWorkerPool.cs similarity index 92% rename from Services/PowerShellHost/PowerShellWorkerPool.cs rename to src/Craft/Services/PowerShellHost/PowerShellWorkerPool.cs index bd37e50..0a180ac 100644 --- a/Services/PowerShellHost/PowerShellWorkerPool.cs +++ b/src/Craft/Services/PowerShellHost/PowerShellWorkerPool.cs @@ -1,7 +1,7 @@ using System.Collections.Concurrent; using System.Management.Automation.Runspaces; using Craft.Configuration; -using Craft.Services; +using Craft.Hosting; namespace Craft.PowerShellHost; @@ -12,6 +12,8 @@ public class PowerShellWorkerPool : IDisposable private readonly ILogger _logger; private readonly ScriptRepository _repo; private readonly CraftSettings _settings; + private readonly StartupProgressService _startup; + private readonly Lazy _metrics; private readonly string _apiBasePath; private readonly int _httpPoolSize; private readonly int _bgPoolSize; @@ -35,11 +37,18 @@ public class PowerShellWorkerPool : IDisposable public int HttpPoolSize => _httpPoolSize; public int BgPoolSize => _bgPoolSize; - public PowerShellWorkerPool(ScriptRepository repo, ILogger logger, IConfiguration config, CraftSettings settings) + public PowerShellWorkerPool( + ScriptRepository repo, + ILogger logger, + CraftSettings settings, + StartupProgressService startup, + Lazy metrics) { _repo = repo; _logger = logger; _settings = settings; + _startup = startup; + _metrics = metrics; _apiBasePath = Path.Combine(AppContext.BaseDirectory, "API"); // 0 is meaningful and must survive: it means "this node hosts no PowerShell over HTTP", which is @@ -100,8 +109,8 @@ public void Initialize(bool enableHttp = true, bool enableBg = true) ExpandModuleExportsForDev(modulesPath); var sw = System.Diagnostics.Stopwatch.StartNew(); - StartupInfoBridge.SetCpuCount(Environment.ProcessorCount); - StartupInfoBridge.SetPoolConfig(enableHttp ? _httpPoolSize : 0, enableBg ? _bgPoolSize : 0); + _startup.SetCpuCount(Environment.ProcessorCount); + _startup.SetPoolConfig(enableHttp ? _httpPoolSize : 0, enableBg ? _bgPoolSize : 0); // Test whether thread priority control works on this OS try @@ -122,7 +131,7 @@ public void Initialize(bool enableHttp = true, bool enableBg = true) var hasWarmup = _settings.Worker.WarmupScripts.Count > 0; if (hasWarmup) _logger.LogInformation("[System] WarmupMode: {Mode} ({Count} scripts)", warmupMode, _settings.Worker.WarmupScripts.Count); - StartupInfoBridge.SetWarmupMode(warmupMode); + _startup.SetWarmupMode(warmupMode); var httpModuleList = _settings.Worker.HttpModules; var bgModuleList = _settings.Worker.BgModules; @@ -165,7 +174,7 @@ public void Initialize(bool enableHttp = true, bool enableBg = true) private void InitializeWithSharedBase(System.Diagnostics.Stopwatch sw, List sharedModules, List httpOnlyModules, List bgOnlyModules, string warmupMode) { - StartupInfoBridge.SetModuleCounts(sharedModules.Count, httpOnlyModules.Count, bgOnlyModules.Count); + _startup.SetModuleCounts(sharedModules.Count, httpOnlyModules.Count, bgOnlyModules.Count); _logger.LogInformation("[System] Shared base: {Shared} shared, {HttpOnly} HTTP-only, {BgOnly} BG-only modules", sharedModules.Count, httpOnlyModules.Count, bgOnlyModules.Count); @@ -178,7 +187,7 @@ private void InitializeWithSharedBase(System.Diagnostics.Stopwatch sw, _logger.LogInformation("[System] Base worker ready in {Ms}ms, exporting shared state", baseMs); var baseState = baseWorker.ExportModuleState(); - StartupInfoBridge.SetBaseWorkerDone(baseMs, baseState.Functions.Count); + _startup.SetBaseWorkerDone(baseMs, baseState.Functions.Count); _logger.LogInformation("[System] Base state: {FnCount} functions, {VarCount} variables, {BinCount} binary modules", baseState.Functions.Count, baseState.Variables.Count, baseState.BinaryModulePaths.Count); @@ -193,20 +202,26 @@ private void InitializeWithSharedBase(System.Diagnostics.Stopwatch sw, _httpClonedState = httpState.MergeWith(baseState); - // Run warmup before signaling ready (BeforeReady and AfterReady both run here; - // Background mode runs warmup on the first BG worker instead) - if (!warmupMode.Equals("Background", StringComparison.OrdinalIgnoreCase)) + var afterReady = IsAfterReadyWarmup(warmupMode); + var httpWarmup = !IsBackgroundWarmup(warmupMode); + + // BeforeReady: warmup before the ready signal. AfterReady: signal first, then warmup + // with the worker briefly out of the pool. Background: warmup on the BG first worker. + if (httpWarmup && !afterReady) RunWarmup(firstHttpWorker, sw); AddToHttpPool(firstHttpWorker); - // Signal HTTP ready — one worker can serve requests (warmup already complete) + // Signal HTTP ready — one worker can serve requests _httpReady.Set(); _ready.Set(); - StartupInfoBridge.SetHttpReady(sw.ElapsedMilliseconds, _httpClonedState.Functions.Count); + _startup.SetHttpReady(sw.ElapsedMilliseconds, _httpClonedState.Functions.Count); _logger.LogInformation("[System] HTTP ready: 1 worker in {Ms}ms — API accepting requests ({FnCount} functions)", sw.ElapsedMilliseconds, _httpClonedState.Functions.Count); + if (httpWarmup && afterReady) + RunWarmupFromHttpPool(sw); + // Clone remaining HTTP workers (API already serving with first worker) if (_httpPoolSize > 1) { @@ -219,7 +234,7 @@ private void InitializeWithSharedBase(System.Diagnostics.Stopwatch sw, foreach (var w in httpRemaining) AddToHttpPool(w); - StartupInfoBridge.SetHttpPoolFull(sw.ElapsedMilliseconds); + _startup.SetHttpPoolFull(sw.ElapsedMilliseconds); _logger.LogInformation("[System] HTTP pool full: {Count} workers in {Ms}ms", _httpPoolSize, sw.ElapsedMilliseconds); } @@ -233,12 +248,12 @@ private void InitializeWithSharedBase(System.Diagnostics.Stopwatch sw, _bgClonedState = bgState.MergeWith(baseState); // Background: warmup runs on the first BG worker — HTTP pool unaffected - if (warmupMode.Equals("Background", StringComparison.OrdinalIgnoreCase)) + if (IsBackgroundWarmup(warmupMode)) RunWarmup(firstBgWorker, sw); AddToBgPool(firstBgWorker); - StartupInfoBridge.SetBgReady(sw.ElapsedMilliseconds, _bgClonedState.Functions.Count); + _startup.SetBgReady(sw.ElapsedMilliseconds, _bgClonedState.Functions.Count); _logger.LogInformation("[System] BG first worker ready in {Ms}ms — {FnCount} functions", sw.ElapsedMilliseconds, _bgClonedState.Functions.Count); @@ -260,7 +275,7 @@ private void InitializeWithSharedBase(System.Diagnostics.Stopwatch sw, // Pre-register all workers in metrics bridge so they appear in snapshots before first use RegisterAllWorkers(); - StartupInfoBridge.SetFullyReady(sw.ElapsedMilliseconds); + _startup.SetFullyReady(sw.ElapsedMilliseconds); _logger.LogInformation("[System] Pool fully ready: {Http} HTTP + {Bg} BG workers in {Ms}ms (base: {BaseMs}ms)", _httpPoolSize, _bgPoolSize, sw.ElapsedMilliseconds, baseMs); } @@ -274,7 +289,8 @@ private void InitializeSimple(System.Diagnostics.Stopwatch sw, bool separateModu { // Decide where warmup runs. Normally on the HTTP first worker (unless WarmupMode=Background), but a // node without that pool runs it on the pool it does have, so warmup never silently gets skipped. - var isBgWarmup = warmupMode.Equals("Background", StringComparison.OrdinalIgnoreCase); + var isBgWarmup = IsBackgroundWarmup(warmupMode); + var afterReady = IsAfterReadyWarmup(warmupMode); var warmupOnHttp = enableHttp && (!isBgWarmup || !enableBg); var warmupOnBg = enableBg && !warmupOnHttp; @@ -287,18 +303,21 @@ private void InitializeSimple(System.Diagnostics.Stopwatch sw, bool separateModu _httpClonedState = firstWorker.ExportModuleState(); - if (warmupOnHttp) + if (warmupOnHttp && !afterReady) RunWarmup(firstWorker, sw); AddToHttpPool(firstWorker); - // Signal HTTP ready — one worker can serve requests (warmup already complete) + // Signal HTTP ready — one worker can serve requests _httpReady.Set(); _ready.Set(); var firstWorkerMs = sw.ElapsedMilliseconds; - StartupInfoBridge.SetHttpReady(firstWorkerMs, _httpClonedState.Functions.Count); + _startup.SetHttpReady(firstWorkerMs, _httpClonedState.Functions.Count); _logger.LogInformation("[System] HTTP ready: 1 worker in {Ms}ms — API accepting requests", firstWorkerMs); + if (warmupOnHttp && afterReady) + RunWarmupFromHttpPool(sw); + // Clone remaining HTTP workers (API already serving with first worker) if (_httpPoolSize > 1) { @@ -340,7 +359,7 @@ private void InitializeSimple(System.Diagnostics.Stopwatch sw, bool separateModu RunWarmup(firstBgWorker, sw); AddToBgPool(firstBgWorker); - StartupInfoBridge.SetBgReady(sw.ElapsedMilliseconds, _bgClonedState.Functions.Count); + _startup.SetBgReady(sw.ElapsedMilliseconds, _bgClonedState.Functions.Count); _logger.LogInformation("[System] First BG worker ready in {Ms}ms — BG state: {FnCount} functions, {VarCount} variables", bgSw.ElapsedMilliseconds, _bgClonedState.Functions.Count, _bgClonedState.Variables.Count); } @@ -376,7 +395,7 @@ private void InitializeSimple(System.Diagnostics.Stopwatch sw, bool separateModu RunWarmup(firstBgWorker, sw); AddToBgPool(firstBgWorker); - StartupInfoBridge.SetBgReady(sw.ElapsedMilliseconds, _bgClonedState.Functions.Count); + _startup.SetBgReady(sw.ElapsedMilliseconds, _bgClonedState.Functions.Count); _logger.LogInformation("[System] BG-only: first BG worker ready in {Ms}ms — {FnCount} functions", bgSw.ElapsedMilliseconds, _bgClonedState.Functions.Count); @@ -403,7 +422,7 @@ private void InitializeSimple(System.Diagnostics.Stopwatch sw, bool separateModu // Pre-register all workers in metrics bridge so they appear in snapshots before first use RegisterAllWorkers(); - StartupInfoBridge.SetFullyReady(sw.ElapsedMilliseconds); + _startup.SetFullyReady(sw.ElapsedMilliseconds); _logger.LogInformation("[System] Pool fully ready: {Http} HTTP + {Bg} BG workers in {Ms}ms", _httpWorkerIds.Count, _bgWorkerIds.Count, sw.ElapsedMilliseconds); } @@ -419,19 +438,43 @@ private void RunWarmup(PowerShellWorker worker, System.Diagnostics.Stopwatch sw) var warmSw = System.Diagnostics.Stopwatch.StartNew(); worker.Warmup(_settings); _logger.LogInformation("[System] Warmup completed in {Ms}ms (at {Total}ms)", warmSw.ElapsedMilliseconds, sw.ElapsedMilliseconds); - StartupInfoBridge.SetWarmupDone(warmSw.ElapsedMilliseconds); + _startup.SetWarmupDone(warmSw.ElapsedMilliseconds); } /// - /// Pre-register all pool workers with WorkerMetricsBridge so they appear + /// AfterReady: pull the first HTTP worker out of the pool, warm it, put it back. + /// Requests may already be accepted ( is set); during warmup the + /// pool has one fewer available worker. + /// + private void RunWarmupFromHttpPool(System.Diagnostics.Stopwatch sw) + { + if (_settings.Worker.WarmupScripts.Count == 0) return; + if (!_httpPool.TryTake(out var worker, 0)) + { + _logger.LogWarning("[System] AfterReady warmup skipped — no HTTP worker available in pool"); + return; + } + + try { RunWarmup(worker, sw); } + finally { _httpPool.Add(worker); } + } + + private static bool IsBackgroundWarmup(string warmupMode) => + warmupMode.Equals("Background", StringComparison.OrdinalIgnoreCase); + + private static bool IsAfterReadyWarmup(string warmupMode) => + warmupMode.Equals("AfterReady", StringComparison.OrdinalIgnoreCase); + + /// + /// Pre-register all pool workers with WorkerMetricsService so they appear /// in snapshots even before their first checkout. /// private void RegisterAllWorkers() { foreach (var id in _httpWorkerIds) - WorkerMetricsBridge.RegisterWorker(id, isHttp: true); + _metrics.Value.RegisterWorker(id, isHttp: true); foreach (var id in _bgWorkerIds) - WorkerMetricsBridge.RegisterWorker(id, isHttp: false); + _metrics.Value.RegisterWorker(id, isHttp: false); } private void AddToHttpPool(PowerShellWorker worker) @@ -464,7 +507,7 @@ private void AddToBgPool(PowerShellWorker worker) if (_httpPool.TryTake(out var w, timeout)) { w.CheckoutTimestamp = System.Diagnostics.Stopwatch.GetTimestamp(); - WorkerMetricsBridge.RecordCheckout(w.Id, isHttp: true); + _metrics.Value.RecordCheckout(w.Id, isHttp: true); // Boost thread priority so HTTP requests get CPU preference over BG work try { Thread.CurrentThread.Priority = ThreadPriority.AboveNormal; } @@ -480,7 +523,7 @@ public PowerShellWorker CheckoutBackground(CancellationToken ct) _bgReady.Wait(ct); var w = _bgPool.Take(ct); w.CheckoutTimestamp = System.Diagnostics.Stopwatch.GetTimestamp(); - WorkerMetricsBridge.RecordCheckout(w.Id, isHttp: false); + _metrics.Value.RecordCheckout(w.Id, isHttp: false); // Lower thread priority so HTTP workers get CPU preference under contention try { Thread.CurrentThread.Priority = ThreadPriority.Lowest; } @@ -499,7 +542,7 @@ public void Reclaim(PowerShellWorker worker, bool isHttp, bool faulted = false) var elapsedMs = worker.CheckoutTimestamp > 0 ? (long)System.Diagnostics.Stopwatch.GetElapsedTime(worker.CheckoutTimestamp).TotalMilliseconds : 0; - WorkerMetricsBridge.RecordReclaim(worker.Id, faulted, elapsedMs); + _metrics.Value.RecordReclaim(worker.Id, faulted, elapsedMs); worker.CheckoutTimestamp = 0; worker.InvocationCount++; @@ -532,7 +575,7 @@ public void Reclaim(PowerShellWorker worker, bool isHttp, bool faulted = false) if (needsReplace) { var oldId = worker.Id; - WorkerMetricsBridge.DeregisterWorker(oldId); + _metrics.Value.DeregisterWorker(oldId); worker.Dispose(); // Build the replacement off the calling thread so the dispatch loop is not held @@ -548,7 +591,7 @@ public void Reclaim(PowerShellWorker worker, bool isHttp, bool faulted = false) var iss = cloned != null ? BuildClonedISS(cloned) : BuildISS(isHttp: ish); var fresh = new PowerShellWorker(Interlocked.Increment(ref _nextId), iss, _logger); fresh.Initialize(_repo, _apiBasePath, _settings); - WorkerMetricsBridge.RegisterWorker(fresh.Id, ish); + _metrics.Value.RegisterWorker(fresh.Id, ish); if (ish) _httpPool.Add(fresh); else _bgPool.Add(fresh); _logger.LogInformation("[Pool] Replaced W{OldId} → W{NewId} ({Type}) (background recycle)", oldId, fresh.Id, ish ? "HTTP" : "BG"); diff --git a/Services/PowerShellHost/ScriptRepository.cs b/src/Craft/Services/PowerShellHost/ScriptRepository.cs similarity index 69% rename from Services/PowerShellHost/ScriptRepository.cs rename to src/Craft/Services/PowerShellHost/ScriptRepository.cs index 3eb6be6..471e273 100644 --- a/Services/PowerShellHost/ScriptRepository.cs +++ b/src/Craft/Services/PowerShellHost/ScriptRepository.cs @@ -1,5 +1,7 @@ +using System.Collections.Concurrent; using System.Management.Automation; using System.Management.Automation.Language; +using System.Text; using System.Text.Json; using System.Text.RegularExpressions; using Craft.Configuration; @@ -110,90 +112,164 @@ private void LoadDirectory(string dirPath, FunctionCategory category) LoadCompiledModule(psm1, category); return; } + + return; } - foreach (var ps1 in ps1Files) + // Parse files in parallel (AST parse is CPU-bound and independent per file), then merge + // into the shared dictionaries on one thread so we keep OrdinalIgnoreCase Dictionary semantics. + var extractPermissions = _settings.Scripts.PermissionExtraction.Enabled; + var parsed = new ConcurrentBag(); + + Parallel.ForEach(ps1Files, ps1 => { try { - var raw = File.ReadAllText(ps1); - var originalRaw = raw; // keep original for permission extraction + if (TryParsePs1File(ps1, category, extractPermissions, out var result)) + parsed.Add(result); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to parse {File}", ps1); + } + }); - // Strip 'using' statements - they must go in ISS/worker init, not in ScriptBlocks - var lines = raw.Split('\n'); - var bodyLines = new List(); - foreach (var line in lines) - { - var trimmed = line.TrimStart(); - if (!trimmed.StartsWith("using namespace ", StringComparison.OrdinalIgnoreCase) - && !trimmed.StartsWith("using module ", StringComparison.OrdinalIgnoreCase) - && !trimmed.StartsWith("using assembly ", StringComparison.OrdinalIgnoreCase)) - { - bodyLines.Add(line); - } - } - raw = string.Join('\n', bodyLines); + foreach (var result in parsed) + { + foreach (var entry in result.Functions) + _functions[entry.FunctionName] = entry; - // Parse with Parser.ParseInput (avoids temp file) - var ast = Parser.ParseInput(raw, out _, out ParseError[] errors); - if (errors?.Length > 0) - { - _logger.LogWarning("Parse errors in {File}: {Errors}", ps1, - string.Join("; ", errors.Take(3).Select(e => e.Message))); - } + if (result.PermissionKey is not null && result.PermissionValue is not null) + _permissions[result.PermissionKey] = result.PermissionValue; + } + } + + /// + /// Parse one .ps1 into function entries (+ optional permission metadata). Thread-safe: no shared state. + /// + private bool TryParsePs1File(string ps1, FunctionCategory category, bool extractPermissions, + out Ps1ParseResult result) + { + result = default!; + var originalRaw = File.ReadAllText(ps1); + // Strip 'using' statements — they must go in ISS/worker init, not in ScriptBlocks + var raw = StripUsingStatements(originalRaw); - var funcDefs = ast.FindAll(a => a is FunctionDefinitionAst, searchNestedScriptBlocks: false) - .Cast() - .ToList(); + var ast = Parser.ParseInput(raw, out _, out ParseError[] errors); + if (errors?.Length > 0) + { + _logger.LogWarning("Parse errors in {File}: {Errors}", ps1, + string.Join("; ", errors.Take(3).Select(e => e.Message))); + } - if (funcDefs.Count > 0) - { - foreach (var funcDef in funcDefs) - { - var sb = funcDef.Body.GetScriptBlock(); - _functions[funcDef.Name] = new FunctionEntry - { - FunctionName = funcDef.Name, - ScriptBlock = sb, - SourcePath = ps1, - Category = category - }; - } - } - else + var functions = new List(); + var funcDefs = ast.FindAll(a => a is FunctionDefinitionAst, searchNestedScriptBlocks: false) + .Cast(); + + var any = false; + foreach (var funcDef in funcDefs) + { + any = true; + functions.Add(new FunctionEntry + { + FunctionName = funcDef.Name, + ScriptBlock = funcDef.Body.GetScriptBlock(), + SourcePath = ps1, + Category = category + }); + } + + if (!any) + { + // Bare script with no function definition (e.g. timer scripts) + var name = Path.GetFileNameWithoutExtension(ps1); + functions.Add(new FunctionEntry + { + FunctionName = name, + ScriptBlock = ast.GetScriptBlock(), + SourcePath = ps1, + Category = category + }); + } + + string? permKey = null; + Dictionary? permValue = null; + if (extractPermissions) + { + permKey = Path.GetFileNameWithoutExtension(ps1); + var roleMatch = RoleRegex.Match(originalRaw); + var funcMatch = FuncRegex.Match(originalRaw); + if (roleMatch.Success || funcMatch.Success) + { + permValue = new Dictionary { - // Bare script with no function definition (e.g. timer scripts) - var name = Path.GetFileNameWithoutExtension(ps1); - _functions[name] = new FunctionEntry - { - FunctionName = name, - ScriptBlock = ast.GetScriptBlock(), - SourcePath = ps1, - Category = category - }; - } + ["Role"] = roleMatch.Success ? roleMatch.Groups[1].Value.Trim() : "", + ["Functionality"] = funcMatch.Success ? funcMatch.Groups[1].Value.Trim() : "" + }; + } + else + { + permKey = null; + } + } - // Extract .ROLE / .FUNCTIONALITY from comment-based help (if configured) - if (_settings.Scripts.PermissionExtraction.Enabled) + result = new Ps1ParseResult(functions, permKey, permValue); + return true; + } + + private readonly record struct Ps1ParseResult( + List Functions, + string? PermissionKey, + Dictionary? PermissionValue); + + /// + /// Remove top-of-file using namespace|module|assembly lines. Fast path returns the + /// original string when no using substring exists (common for most CIPP scripts). + /// + private static string StripUsingStatements(string raw) + { + if (raw.IndexOf("using ", StringComparison.OrdinalIgnoreCase) < 0) + return raw; + + StringBuilder? sb = null; + var lineStart = 0; + for (var i = 0; i <= raw.Length; i++) + { + if (i < raw.Length && raw[i] != '\n') + continue; + + var lineLen = i - lineStart; + var line = raw.AsSpan(lineStart, lineLen); + if (line.Length > 0 && line[^1] == '\r') + line = line[..^1]; + + if (IsUsingDirectiveLine(line)) + { + if (sb is null) { - var funcName = Path.GetFileNameWithoutExtension(ps1); - var roleMatch = RoleRegex.Match(originalRaw); - var funcMatch = FuncRegex.Match(originalRaw); - if (roleMatch.Success || funcMatch.Success) - { - _permissions[funcName] = new Dictionary - { - ["Role"] = roleMatch.Success ? roleMatch.Groups[1].Value.Trim() : "", - ["Functionality"] = funcMatch.Success ? funcMatch.Groups[1].Value.Trim() : "" - }; - } + sb = new StringBuilder(raw.Length); + sb.Append(raw, 0, lineStart); } } - catch (Exception ex) + else if (sb is not null) { - _logger.LogError(ex, "Failed to parse {File}", ps1); + // Keep original line text including its terminating \n (except past EOF). + var take = i < raw.Length ? lineLen + 1 : lineLen; + sb.Append(raw, lineStart, take); } + + lineStart = i + 1; } + + return sb?.ToString() ?? raw; + } + + private static bool IsUsingDirectiveLine(ReadOnlySpan line) + { + var trimmed = line.TrimStart(); + return trimmed.StartsWith("using namespace ", StringComparison.OrdinalIgnoreCase) + || trimmed.StartsWith("using module ", StringComparison.OrdinalIgnoreCase) + || trimmed.StartsWith("using assembly ", StringComparison.OrdinalIgnoreCase); } // Regex to find top-level function definitions in compiled .psm1 @@ -311,8 +387,11 @@ private void ScanPermissionsOnly(string dirPath) return; } - // Fall back to individual .ps1 files - foreach (var ps1 in Directory.GetFiles(dirPath, "*.ps1", SearchOption.AllDirectories)) + // Fall back to individual .ps1 files — parallelize like LoadDirectory. + var ps1Files = Directory.GetFiles(dirPath, "*.ps1", SearchOption.AllDirectories); + var found = new ConcurrentBag<(string Key, Dictionary Value)>(); + + Parallel.ForEach(ps1Files, ps1 => { try { @@ -322,18 +401,21 @@ private void ScanPermissionsOnly(string dirPath) var funcMatch = FuncRegex.Match(raw); if (roleMatch.Success || funcMatch.Success) { - _permissions[funcName] = new Dictionary + found.Add((funcName, new Dictionary { ["Role"] = roleMatch.Success ? roleMatch.Groups[1].Value.Trim() : "", ["Functionality"] = funcMatch.Success ? funcMatch.Groups[1].Value.Trim() : "" - }; + })); } } catch (Exception ex) { _logger.LogError(ex, "Failed to scan permissions in {File}", ps1); } - } + }); + + foreach (var (key, value) in found) + _permissions[key] = value; } public FunctionEntry? GetByRoute(string route) diff --git a/src/Craft/Services/Program.cs b/src/Craft/Services/Program.cs new file mode 100644 index 0000000..c04ba54 --- /dev/null +++ b/src/Craft/Services/Program.cs @@ -0,0 +1,309 @@ +using Craft.Auth; +using Craft.Caching; +using Craft.Configuration; +using Craft.Endpoints; +using Craft.Hosting; +using Craft.Hosting.Endpoints; +using Craft.Orchestration; +using Craft.PowerShellHost; +using Craft.Realtime; +using Craft.Services; +using Craft.Setup; +using Craft.Storage; +using Microsoft.Extensions.Options; + +var builder = WebApplication.CreateBuilder(args); + +// Roles from IConfiguration only (no full CraftSettings dual-bind). Options pipeline is the +// single CraftSettings source after Build. +var roles = CraftRoles.Resolve(builder.Configuration); + +if (roles.None) +{ + Console.Error.WriteLine("[System] FATAL: no deployment roles enabled — set at least one of " + + "CRAFT_SERVE_FRONTEND / CRAFT_SERVE_API / CRAFT_RUN_BACKGROUND (or App:Roles:*). " + + "Roles are declared by enabling what you want; unset roles default off once any is set."); + Environment.Exit(78); // EX_CONFIG +} + +var capFrontend = roles.Frontend; +var capHttp = roles.Http; +var capBackground = roles.Background; +var runPowerShell = roles.RunsPowerShell; +var cacheEnabled = roles.ResponseCacheEnabled; +var healthEnabled = roles.HealthEnabled; +var healthPath = roles.HealthPath; +var compressionEnabled = roles.CompressionEnabled; + +builder.Services.AddCraftSettings(builder.Configuration, roles); +builder.Services.AddSingleton(sp => sp.GetRequiredService>().Value); + +builder.ConfigureCraftKestrel(); + +var (configuredLogLevel, fileLoggerProvider) = builder.AddCraftLogging(); + +builder.Services.AddCraftResponseCompression(); + +// Discover native C# endpoints/tasks before Build so they can register into DI. +var endpointSettings = new EndpointSettings(); +builder.Configuration.GetSection("App:Endpoints").Bind(endpointSettings); +var nativeCatalog = NativeEndpointRegistry.Discover( + endpointSettings, + Path.Combine(AppContext.BaseDirectory, "API"), + LoggerFactory.Create(b => b.AddSimpleConsole()).CreateLogger("Craft.Endpoints")); + +builder.Services.AddCraftServices(roles); +if (!nativeCatalog.IsEmpty) + builder.Services.AddNativeEndpoints(nativeCatalog, builder.Configuration); +builder.Services.AddCraftRateLimiter(); + +var app = builder.Build(); +app.SyncFileLoggingFromOptions(fileLoggerProvider); + +var httpDiagLogger = app.Services.GetRequiredService().CreateLogger("HttpDiag"); +var httpListener = new HttpDiagnosticListener(httpDiagLogger, slowThresholdMs: 1000); +app.Lifetime.ApplicationStopping.Register(() => httpListener.Dispose()); + +var logger = app.Services.GetRequiredService>(); +var CraftSettings = app.Services.GetRequiredService(); +var startupProgress = app.Services.GetRequiredService(); +StartupInfoBridge.Initialize(startupProgress); + +var cache = app.Services.GetRequiredService(); +CacheBridge.Initialize(cache); + +if (app.Services.GetService() is { } realtime) + RealtimeBridge.Initialize(realtime); + +ScriptRepository? repo = null; +PowerShellWorkerPool? pool = null; +PowerShellRunnerService? psRunner = null; +SetupService? setupService = null; +AuthService? authService = null; +JobManager? jobManager = null; +OrchestratorService? orchestrator = null; + +if (runPowerShell) +{ + repo = app.Services.GetRequiredService(); + pool = app.Services.GetRequiredService(); + psRunner = app.Services.GetRequiredService(); + setupService = app.Services.GetRequiredService(); + AppLifecycleBridge.Initialize(app.Lifetime, logger, setupService); + + jobManager = app.Services.GetRequiredService(); + orchestrator = app.Services.GetRequiredService(); + var queueDispatch = app.Services.GetRequiredService(); + var workerMetrics = app.Services.GetRequiredService(); + WorkerMetricsBridge.Initialize(workerMetrics); + OrchestratorBridge.Initialize(orchestrator); + QueueBridge.Initialize(queueDispatch); + + authService = app.Services.GetService(); + if (authService is not null) + AuthBridge.Initialize(authService); + QueueStatusBridge.Initialize(app.Services.GetRequiredService()); + SchedulerBridge.Initialize(app.Services.GetRequiredService()); + StatsHistoryBridge.Initialize(app.Services.GetRequiredService()); +} + +var healthMonitor = app.Services.GetRequiredService(); +if (CraftSettings.ContainerHealth.MaxRestarts > 0) +{ + healthMonitor.RecordStartupAttempt(); + if (healthMonitor.ShouldBlockStartup) + { + logger.LogCritical("[Health] Startup blocked due to crash loop — waiting for Azure to provision a new worker"); + await Task.Delay(Timeout.Infinite); + } +} + +var endpoints = new Dictionary(StringComparer.OrdinalIgnoreCase); + +var readinessMode = CraftSettings.ReadinessMode?.Trim() ?? "Immediate"; + +var websiteSku = Environment.GetEnvironmentVariable("WEBSITE_SKU") ?? ""; +var isSlowSingleCore = Environment.ProcessorCount <= 1 + && websiteSku.StartsWith("Basic", StringComparison.OrdinalIgnoreCase); + +if (!readinessMode.Equals("Immediate", StringComparison.OrdinalIgnoreCase) && isSlowSingleCore) +{ + logger.LogWarning("[System] ReadinessMode '{Mode}' overridden to 'Immediate' — single vCPU on Basic SKU, " + + "blocking Kestrel during init risks hitting Azure's 230s startup timeout", readinessMode); + readinessMode = "Immediate"; +} + +logger.LogInformation("[System] Readiness mode: {Mode}", readinessMode); +startupProgress.SetReadinessMode(readinessMode); + +logger.LogInformation("[System] Roles: Frontend={Frontend} Http={Http} Background={Background} | " + + "ResponseCache={Cache} Compression={Compression}", + capFrontend ? "on" : "off", capHttp ? "on" : "off", capBackground ? "on" : "off", + cacheEnabled ? "on" : "off", compressionEnabled ? "on" : "off"); + +void RunInitialization() +{ + if (repo is null || pool is null || psRunner is null) + throw new InvalidOperationException("PowerShell services are not registered for this role."); + + repo.LoadAll(Path.Combine(AppContext.BaseDirectory, "API")); + + var discovered = psRunner.DiscoverHttpEndpoints(); + foreach (var kvp in discovered) + endpoints[kvp.Key] = kvp.Value; + + logger.LogInformation("[System] {AppName}: {Count} API endpoints discovered", CraftSettings.Name, endpoints.Count); + logger.LogInformation("[System] Pool: HTTP={Http} BG={Bg} LogLevel={LogLevel}", + CraftSettings.Worker.HttpPoolSize, + CraftSettings.Worker.BgPoolSize, + configuredLogLevel); + + // HttpPoolSize/BgPoolSize = 0 opts out of that PowerShell pool (fully-native HTTP or native + // scheduled tasks). Initialize signals readiness immediately when no pool is enabled. + var enableHttpPool = capHttp && CraftSettings.Worker.HttpPoolSize > 0; + if (capHttp && !enableHttpPool) + logger.LogInformation("[System] HTTP worker pool disabled (Worker:HttpPoolSize=0) — PowerShell HTTP endpoints are not hosted"); + + var enableBgPool = capBackground && CraftSettings.Worker.BgPoolSize > 0; + if (capBackground && !enableBgPool) + logger.LogInformation("[System] BG worker pool disabled (Worker:BgPoolSize=0) — native scheduled tasks only"); + + pool.Initialize(enableHttp: enableHttpPool, enableBg: enableBgPool); + + healthMonitor.ClearRestartCounter(); +} + +if (!runPowerShell) +{ + logger.LogWarning("[System] STATIC-ONLY (Frontend role) — PowerShell worker pool, scheduler, job manager " + + "and background services are disabled. Serving static frontend content only; /api, /API and /.auth " + + "return 404."); +} +else if (readinessMode.Equals("Immediate", StringComparison.OrdinalIgnoreCase)) +{ + app.Lifetime.ApplicationStarted.Register(() => + { + Task.Run(() => + { + try { RunInitialization(); } + catch (Exception ex) { logger.LogCritical(ex, "[System] Initialization failed"); } + }); + }); +} +else if (readinessMode.Equals("HttpReady", StringComparison.OrdinalIgnoreCase)) +{ + var initTask = Task.Run(() => + { + try { RunInitialization(); } + catch (Exception ex) + { + logger.LogCritical(ex, "[System] Initialization failed"); + throw; + } + }); + + while (pool is null || !pool.IsReady) + { + var finished = await Task.WhenAny(initTask, Task.Delay(500)).ConfigureAwait(false); + if (finished == initTask) + { + await initTask.ConfigureAwait(false); + if (pool is null || !pool.IsReady) + throw new InvalidOperationException( + "Initialization finished without signaling HTTP ready."); + break; + } + } + + logger.LogInformation("[System] HTTP pool ready — starting Kestrel (BG init continues in background)"); +} +else if (readinessMode.Equals("AllReady", StringComparison.OrdinalIgnoreCase)) +{ + try { RunInitialization(); } + catch (Exception ex) { logger.LogCritical(ex, "[System] Initialization failed"); } + logger.LogInformation("[System] All pools ready — starting Kestrel"); +} +else +{ + throw new InvalidOperationException( + $"Unexpected ReadinessMode '{readinessMode}'. Expected Immediate, HttpReady, or AllReady."); +} + +if (app.Environment.IsDevelopment()) +{ + logger.LogWarning("[Auth] Running in Development mode \u2014 unauthenticated requests will receive dev principal with roles: {Roles}", + string.Join(", ", CraftSettings.Auth.DevRoles)); +} + +if (compressionEnabled) + app.UseResponseCompression(); +logger.LogInformation("[System] Static compression: {State}", compressionEnabled ? "enabled (precompressed .br/.gz + on-the-fly fallback)" : "DISABLED (raw/identity)"); + +if (CraftSettings.Setup.Enabled && setupService is not null) + app.UseCraftSetupGate(logger); + +app.UseCraftStartupGate(capHttp, pool, CraftSettings.Setup.Enabled, healthEnabled, healthPath); + +var devFrontendUrl = DevFrontendProxy.ResolveDevServerUrl( + capFrontend, app.Environment.IsDevelopment(), Environment.GetEnvironmentVariable); + +HttpClient? devProxyClient = devFrontendUrl is null + ? null + : app.UseCraftDevFrontendProxy(devFrontendUrl, logger); + +app.UseCraftContentSecurityPolicy(CraftSettings); + +var frontendPath = Path.Combine(AppContext.BaseDirectory, "Frontend"); +var frontendFileProvider = capFrontend + ? app.UseCraftStaticFiles(frontendPath, compressionEnabled, logger) + : null; + +if (capFrontend && frontendFileProvider is null) + logger.LogWarning("[System] Frontend directory not found: {Path}", frontendPath); + +var storageHealth = (capHttp || capBackground) + ? app.Services.GetService() + : null; +if (storageHealth != null) _ = storageHealth.RefreshAsync(); + +app.MapCraftHealthEndpoint(roles, storageHealth, logger); +app.MapCraftRealtimeEndpoint(roles, CraftSettings, logger); +app.MapCraftPrmEndpoint(CraftSettings, logger); + +// Auth middleware must run before the rate limiter so authenticated principals are +// available for partitioning. Static files are already mapped above — that order is +// load-bearing: anonymous static GETs must not consume the authenticated client's budget. +if (capHttp) +{ + app.UseCraftPublicCorsPreflight(CraftSettings, logger); + if (authService is not null) + app.UseCraftAuth(CraftSettings, authService, logger); +} + +if (CraftSettings.RateLimit.IsEnabled) + app.UseRateLimiter(); + +var activeRequests = new RequestCounter(); + +if (capHttp) +{ + if (authService is not null) + app.MapCraftAuthEndpoints(CraftSettings, logger); + app.MapCraftSetupEndpoints(CraftSettings); + app.MapCraftJobEndpoints(); + + if (nativeCatalog.Endpoints.Count > 0) + { + var mappable = NativeEndpointRegistry.ResolveCollisions( + nativeCatalog.Endpoints, endpoints.Keys, CraftSettings.Endpoints.OnCollision, logger); + app.MapCraftNativeEndpoints(mappable, activeRequests, logger, CraftSettings.Endpoints); + } + + app.MapCraftPowerShellDispatch(endpoints, activeRequests, logger); +} + +app.MapCraftFrontendFallback( + new FrontendFallbackOptions(frontendFileProvider, devProxyClient, compressionEnabled, frontendPath), + logger); + +app.Run(); diff --git a/Services/Realtime/RealtimeService.cs b/src/Craft/Services/Realtime/RealtimeService.cs similarity index 96% rename from Services/Realtime/RealtimeService.cs rename to src/Craft/Services/Realtime/RealtimeService.cs index a4f9a7f..e28990b 100644 --- a/Services/Realtime/RealtimeService.cs +++ b/src/Craft/Services/Realtime/RealtimeService.cs @@ -1,7 +1,6 @@ using System.Collections; using System.Collections.Concurrent; using System.Diagnostics; -using System.Management.Automation; using System.Text; using System.Text.Json; using System.Threading.Channels; @@ -19,7 +18,7 @@ namespace Craft.Realtime; /// /// One "current message" is stored per active (userId, jobId) so a reconnecting browser resyncs /// instantly. Single instance, no backplane — the publisher and the SSE connection must be the same -/// process (combined role). See docs/realtime-bridge-plan.md. +/// process (combined role). See docs/configuration.md#realtime-sse. /// /// Opt-in: off unless App:Realtime:Enabled=true (or CRAFT_REALTIME_ENABLED=true). While off /// the endpoint is not mapped, publishes are dropped, and no state or timer is held. @@ -92,6 +91,8 @@ public Connection(int capacity) /// /// Publish a job event. and (a GUID) are /// required; everything else is optional. Best-effort and non-throwing. + /// PowerShell PSObject/Hashtable payloads should be normalised by + /// before calling; this method only reshapes CLR dictionaries/enumerables for STJ. /// public void Publish(string userId, string jobId, string? mode, object? data, string? urlHref, string? urlLabel, int? status, string? message) @@ -245,12 +246,11 @@ private static string BuildFrame(string jobId, string mode, long seq, int? statu _ => "update" }; - /// Convert PowerShell (Hashtable/PSObject) and CLR values into STJ-serializable shapes. + /// Convert CLR dictionaries/enumerables/primitives into STJ-serializable shapes. private static object? Normalize(object? v) => v switch { null => null, string or bool or int or long or double or float or decimal or DateTime or DateTimeOffset or Guid => v, - PSObject ps => Normalize(ps.BaseObject), IDictionary d => NormalizeDict(d), IEnumerable e => NormalizeList(e), _ => v.ToString() diff --git a/Services/Setup/SetupGate.cs b/src/Craft/Services/Setup/SetupGate.cs similarity index 100% rename from Services/Setup/SetupGate.cs rename to src/Craft/Services/Setup/SetupGate.cs diff --git a/Services/Setup/SetupPages.cs b/src/Craft/Services/Setup/SetupPages.cs similarity index 100% rename from Services/Setup/SetupPages.cs rename to src/Craft/Services/Setup/SetupPages.cs diff --git a/Services/Setup/SetupService.cs b/src/Craft/Services/Setup/SetupProvisioningService.cs similarity index 85% rename from Services/Setup/SetupService.cs rename to src/Craft/Services/Setup/SetupProvisioningService.cs index 35cb010..136fe35 100644 --- a/Services/Setup/SetupService.cs +++ b/src/Craft/Services/Setup/SetupProvisioningService.cs @@ -3,25 +3,22 @@ using System.Text; using System.Text.Json; using Craft.Configuration; -using Craft.Services; -using Craft.Storage; namespace Craft.Setup; /// -/// Handles the first-run bootstrap setup: -/// 1. PKCE token exchange (auth code → access token, no client secret) +/// Handles the first-run bootstrap provisioning: +/// 1. Device-code token exchange (auth code → access token, no client secret) /// 2. EasyAuth app registration creation (with secret + exemption policy) /// 3. App Service self-configuration via ARM (authsettingsV2 + env vars) /// /// All Graph and ARM calls use bearer tokens — either the user's access token /// (for Graph operations during setup) or the managed identity token (for ARM self-config). /// -public class SetupService +public class SetupProvisioningService { - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly CraftSettings _settings; - private readonly ICraftTableStore _store; private static readonly HttpClient s_httpClient; // Retry settings for policy propagation @@ -36,7 +33,7 @@ public class SetupService TimeSpan.FromSeconds(30) ]; - static SetupService() + static SetupProvisioningService() { var handler = new SocketsHttpHandler { @@ -46,20 +43,10 @@ static SetupService() s_httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Craft-Setup/1.0"); } - public SetupService(ILogger logger, CraftSettings settings, ICraftTableStore store) + public SetupProvisioningService(ILogger logger, CraftSettings settings) { _logger = logger; _settings = settings; - _store = store; - } - - /// - /// Check whether EasyAuth is fully configured by inspecting environment variables. - /// - public static bool IsEasyAuthConfigured() - { - var authEnabled = Environment.GetEnvironmentVariable("WEBSITE_AUTH_ENABLED"); - return string.Equals(authEnabled, "True", StringComparison.OrdinalIgnoreCase); } /// @@ -79,7 +66,7 @@ public string ResolveAuthAppDisplayName() /// Initiates a device code flow. Returns the user_code and verification_uri /// for the user to authenticate at microsoft.com/devicelogin. /// - public async Task StartDeviceCodeFlow(CancellationToken ct = default) + public async Task StartDeviceCodeFlow(CancellationToken ct = default) { var deviceCodeEndpoint = "https://login.microsoftonline.com/common/oauth2/v2.0/devicecode"; @@ -101,7 +88,7 @@ public async Task StartDeviceCodeFlow(CancellationToken ct = using var doc = JsonDocument.Parse(body); var root = doc.RootElement; - return new DeviceCodeResponse + return new SetupService.DeviceCodeResponse { DeviceCode = root.GetProperty("device_code").GetString()!, UserCode = root.GetProperty("user_code").GetString()!, @@ -116,7 +103,7 @@ public async Task StartDeviceCodeFlow(CancellationToken ct = /// Polls for device code flow completion. Returns the access token once the /// user has authenticated, or null if still pending. /// - public async Task PollDeviceCodeFlow(string deviceCode, CancellationToken ct = default) + public async Task PollDeviceCodeFlow(string deviceCode, CancellationToken ct = default) { var tokenEndpoint = "https://login.microsoftonline.com/common/oauth2/v2.0/token"; @@ -155,7 +142,7 @@ public async Task StartDeviceCodeFlow(CancellationToken ct = _logger.LogInformation("[Setup] Device code authentication successful for tenant {TenantId}", tenantId); - return new TokenExchangeResult + return new SetupService.TokenExchangeResult { AccessToken = accessToken, TenantId = tenantId @@ -169,7 +156,7 @@ public async Task StartDeviceCodeFlow(CancellationToken ct = /// registrations in the tenant are never searched for or reused. /// Handles app management policy exemption if the tenant blocks password creation. /// - public async Task CreateAuthAppRegistration( + public async Task CreateAuthAppRegistration( string accessToken, string tenantId, string redirectUri, bool multiTenant = false, CancellationToken ct = default) { var authHeaders = new Dictionary @@ -219,7 +206,7 @@ public async Task CreateAuthAppRegistration( _logger.LogInformation("[Setup] App registration complete: {AppId}", appId); - return new AppRegistrationResult + return new SetupService.AppRegistrationResult { AppId = appId, AppObjectId = appObjectId, @@ -600,7 +587,7 @@ await ArmRequest(HttpMethod.Put, /// public async Task ReconcileAuthPolicy(string reason, CancellationToken ct = default) { - if (!IsEasyAuthConfigured()) + if (!SetupSessionState.IsEasyAuthConfigured()) { _logger.LogInformation("[Setup] Reconcile skipped — EasyAuth not configured ({Reason})", reason); return false; @@ -779,131 +766,6 @@ public async Task ConfigureManual( await ConfigureAppServiceAuth(appId, clientSecret, tenantId, multiTenant, ct); } - // ── First User Seeding ── - - /// - /// Resolves the user table name with the same sanitization as AuthService. - /// - private string ResolveUserTableName() - { - var raw = _settings.Auth.UserTableName; - var sanitized = new string(raw.Where(char.IsLetterOrDigit).ToArray()); - if (sanitized.Length > 63) sanitized = sanitized[..63]; - if (sanitized.Length < 3) sanitized = "allowedUsers"; - return sanitized; - } - - /// - /// Checks the allowedUsers table status: whether it's reachable and whether - /// it already contains any users. - /// - public async Task CheckAllowedUsersStatus(CancellationToken ct = default) - { - try - { - var tableName = ResolveUserTableName(); - await _store.EnsureTableAsync(tableName, ct); - - var count = 0; - await foreach (var row in _store.QueryTableAsync(tableName, ct)) - { - if (!row.RowKey.StartsWith('_')) - { - count++; - if (count > 0) break; // We only need to know if any exist - } - } - - return new AllowedUsersStatus - { - Connected = true, - HasUsers = count > 0 - }; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "[Setup] Failed to check allowedUsers table"); - return new AllowedUsersStatus - { - Connected = false, - HasUsers = false, - Error = ex.Message - }; - } - } - - /// - /// Seeds the first user into the allowedUsers table with the roles from - /// Setup.FirstUserRoles (defaults to "superadmin" when unset). - /// Only works when the table is empty — refuses if users already exist. - /// Uses the same entity schema as CIPP-API's Invoke-ExecCIPPUsers. - /// - public async Task SeedFirstUser(string upn, CancellationToken ct = default) - { - if (string.IsNullOrWhiteSpace(upn)) - throw new ArgumentException("UPN (email) is required."); - - // Invariant, not current-culture: this value is an identity key compared against rows - // written by AuthService (which already lowercases invariantly). Under a Turkish locale - // the two would disagree on "I"/"i" and a seeded user would fail to match. - upn = upn.Trim().ToLowerInvariant(); - - var tableName = ResolveUserTableName(); - await _store.EnsureTableAsync(tableName, ct); - - // Guard: refuse if the table already has users - await foreach (var row in _store.QueryTableAsync(tableName, ct)) - { - if (!row.RowKey.StartsWith('_')) - throw new InvalidOperationException("The allowed users table already contains users. First-user seeding is only available on an empty table."); - } - - string[] roles = _settings.Setup.FirstUserRoles.Count > 0 - ? _settings.Setup.FirstUserRoles.ToArray() - : ["superadmin"]; - var rolesJson = JsonSerializer.Serialize(roles); - - var userRow = new StoreRow("User", upn) - { - Properties = - { - ["Roles"] = rolesJson, - ["ManualRoles"] = rolesJson, - ["AutoRoles"] = "[]", - ["Source"] = "Manual" - } - }; - - await _store.UpsertAsync(tableName, userRow, ct); - _logger.LogInformation("[Setup] Seeded first user {Upn} with roles {Roles}", upn, string.Join(",", roles)); - } - - // ── Status ── - - /// - /// Returns setup status information. - /// - public async Task GetStatus(CancellationToken ct = default) - { - var isConfigured = IsEasyAuthConfigured(); - var siteName = Environment.GetEnvironmentVariable("WEBSITE_SITE_NAME"); - var hasManagedIdentity = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("IDENTITY_ENDPOINT")); - var usersStatus = await CheckAllowedUsersStatus(ct); - - return new SetupStatus - { - IsEasyAuthConfigured = isConfigured, - IsSetupCompleted = AppLifecycleBridge.IsSetupCompleted(), - SetupCompletedReason = AppLifecycleBridge.GetSetupCompletedReason(), - IsRunningInAppService = !string.IsNullOrEmpty(siteName), - HasManagedIdentity = hasManagedIdentity, - AppName = _settings.Name, - AuthAppDisplayName = ResolveAuthAppDisplayName(), - BootstrapClientId = _settings.Setup.BootstrapClientId, - UsersStatus = usersStatus - }; - } - // ── Helper Methods ── /// @@ -1171,50 +1033,4 @@ private async Task KeyVaultRequest( } } - // ── Result Models ── - - public class TokenExchangeResult - { - public string AccessToken { get; set; } = ""; - public string TenantId { get; set; } = ""; - } - - public class AppRegistrationResult - { - public string AppId { get; set; } = ""; - public string AppObjectId { get; set; } = ""; - public string ClientSecret { get; set; } = ""; - public string TenantId { get; set; } = ""; - public string DisplayName { get; set; } = ""; - } - - public class SetupStatus - { - public bool IsEasyAuthConfigured { get; set; } - public bool IsSetupCompleted { get; set; } - public string? SetupCompletedReason { get; set; } - public bool IsRunningInAppService { get; set; } - public bool HasManagedIdentity { get; set; } - public string AppName { get; set; } = ""; - public string AuthAppDisplayName { get; set; } = ""; - public string BootstrapClientId { get; set; } = ""; - public AllowedUsersStatus UsersStatus { get; set; } = new(); - } - - public class AllowedUsersStatus - { - public bool Connected { get; set; } - public bool HasUsers { get; set; } - public string? Error { get; set; } - } - - public class DeviceCodeResponse - { - public string DeviceCode { get; set; } = ""; - public string UserCode { get; set; } = ""; - public string VerificationUri { get; set; } = ""; - public int ExpiresIn { get; set; } - public int Interval { get; set; } - public string Message { get; set; } = ""; - } } diff --git a/src/Craft/Services/Setup/SetupService.cs b/src/Craft/Services/Setup/SetupService.cs new file mode 100644 index 0000000..6405f1e --- /dev/null +++ b/src/Craft/Services/Setup/SetupService.cs @@ -0,0 +1,193 @@ +using Craft.Configuration; + +namespace Craft.Setup; + +/// +/// Facade for first-run bootstrap setup. Delegates to +/// , , and +/// while keeping the public method signatures and +/// nested DTOs used by SetupEndpoints, AppLifecycleBridge, and SetupModeMiddleware. +/// +public class SetupService +{ + private readonly SetupSessionState _session; + private readonly SetupProvisioningService _provisioning; + private readonly SetupUserBootstrap _users; + private readonly CraftSettings _settings; + + public SetupService( + SetupSessionState session, + SetupProvisioningService provisioning, + SetupUserBootstrap users, + CraftSettings settings) + { + _session = session; + _provisioning = provisioning; + _users = users; + _settings = settings; + } + + /// + /// Check whether EasyAuth is fully configured by inspecting environment variables. + /// + public static bool IsEasyAuthConfigured() => SetupSessionState.IsEasyAuthConfigured(); + + /// + /// Explicitly enables the Craft setup wizard. Called from the hosted app (via + /// AppLifecycleBridge.RequestSetupMode) when it cannot self-configure. + /// + public void RequestSetupMode(string reason = "Setup mode requested by application") => + _session.RequestSetupMode(reason); + + /// True once the hosted app has called . + public bool IsSetupModeRequested() => _session.IsSetupModeRequested(); + + /// + /// Marks setup as completed for this process — credentials applied, pending restart. + /// + public void MarkSetupCompleted(string reason = "Setup credentials applied") => + _session.MarkSetupCompleted(reason); + + /// True if setup credentials have already been applied this session. + public bool IsSetupCompleted() => _session.IsSetupCompleted(); + + /// Reason passed to , or null if not completed. + public string? GetSetupCompletedReason() => _session.GetSetupCompletedReason(); + + /// + /// Resolves the display name for the EasyAuth app registration. + /// Uses Setup.AuthAppDisplayName if set, otherwise "Craft-EasyAuth-{App.Name}". + /// + public string ResolveAuthAppDisplayName() => _provisioning.ResolveAuthAppDisplayName(); + + /// + /// Initiates a device code flow. Returns the user_code and verification_uri + /// for the user to authenticate at microsoft.com/devicelogin. + /// + public Task StartDeviceCodeFlow(CancellationToken ct = default) => + _provisioning.StartDeviceCodeFlow(ct); + + /// + /// Polls for device code flow completion. Returns the access token once the + /// user has authenticated, or null if still pending. + /// + public Task PollDeviceCodeFlow(string deviceCode, CancellationToken ct = default) => + _provisioning.PollDeviceCodeFlow(deviceCode, ct); + + /// + /// Creates a new EasyAuth app registration with a client secret. Existing + /// registrations in the tenant are never searched for or reused. + /// Handles app management policy exemption if the tenant blocks password creation. + /// + public Task CreateAuthAppRegistration( + string accessToken, string tenantId, string redirectUri, bool multiTenant = false, CancellationToken ct = default) => + _provisioning.CreateAuthAppRegistration(accessToken, tenantId, redirectUri, multiTenant, ct); + + /// + /// Configures the App Service with EasyAuth settings using the managed identity. + /// Sets environment variables and authsettingsV2 via ARM REST API. + /// + public Task ConfigureAppServiceAuth( + string appId, string clientSecret, string tenantId, bool multiTenant = false, CancellationToken ct = default) => + _provisioning.ConfigureAppServiceAuth(appId, clientSecret, tenantId, multiTenant, ct); + + /// + /// Reconciles the live authsettingsV2.globalValidation block with current Setup settings. + /// Idempotent — returns true if the live config was changed. + /// + public Task ReconcileAuthPolicy(string reason, CancellationToken ct = default) => + _provisioning.ReconcileAuthPolicy(reason, ct); + + /// + /// Saves app registration details manually (user-provided App ID, Secret, Tenant ID) + /// and configures the App Service via ARM. + /// + public Task ConfigureManual( + string appId, string clientSecret, string tenantId, bool multiTenant = false, CancellationToken ct = default) => + _provisioning.ConfigureManual(appId, clientSecret, tenantId, multiTenant, ct); + + /// + /// Checks the allowedUsers table status: whether it's reachable and whether + /// it already contains any users. + /// + public Task CheckAllowedUsersStatus(CancellationToken ct = default) => + _users.CheckAllowedUsersStatus(ct); + + /// + /// Seeds the first user into the allowedUsers table with the roles from + /// Setup.FirstUserRoles (defaults to "superadmin" when unset). + /// + public Task SeedFirstUser(string upn, CancellationToken ct = default) => + _users.SeedFirstUser(upn, ct); + + /// + /// Returns setup status information. + /// + public async Task GetStatus(CancellationToken ct = default) + { + var isConfigured = IsEasyAuthConfigured(); + var siteName = Environment.GetEnvironmentVariable("WEBSITE_SITE_NAME"); + var hasManagedIdentity = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("IDENTITY_ENDPOINT")); + var usersStatus = await CheckAllowedUsersStatus(ct); + + return new SetupStatus + { + IsEasyAuthConfigured = isConfigured, + IsSetupCompleted = IsSetupCompleted(), + SetupCompletedReason = GetSetupCompletedReason(), + IsRunningInAppService = !string.IsNullOrEmpty(siteName), + HasManagedIdentity = hasManagedIdentity, + AppName = _settings.Name, + AuthAppDisplayName = ResolveAuthAppDisplayName(), + BootstrapClientId = _settings.Setup.BootstrapClientId, + UsersStatus = usersStatus + }; + } + + // ── Result Models ── + + public class TokenExchangeResult + { + public string AccessToken { get; set; } = ""; + public string TenantId { get; set; } = ""; + } + + public class AppRegistrationResult + { + public string AppId { get; set; } = ""; + public string AppObjectId { get; set; } = ""; + public string ClientSecret { get; set; } = ""; + public string TenantId { get; set; } = ""; + public string DisplayName { get; set; } = ""; + } + + public class SetupStatus + { + public bool IsEasyAuthConfigured { get; set; } + public bool IsSetupCompleted { get; set; } + public string? SetupCompletedReason { get; set; } + public bool IsRunningInAppService { get; set; } + public bool HasManagedIdentity { get; set; } + public string AppName { get; set; } = ""; + public string AuthAppDisplayName { get; set; } = ""; + public string BootstrapClientId { get; set; } = ""; + public AllowedUsersStatus UsersStatus { get; set; } = new(); + } + + public class AllowedUsersStatus + { + public bool Connected { get; set; } + public bool HasUsers { get; set; } + public string? Error { get; set; } + } + + public class DeviceCodeResponse + { + public string DeviceCode { get; set; } = ""; + public string UserCode { get; set; } = ""; + public string VerificationUri { get; set; } = ""; + public int ExpiresIn { get; set; } + public int Interval { get; set; } + public string Message { get; set; } = ""; + } +} diff --git a/src/Craft/Services/Setup/SetupSessionState.cs b/src/Craft/Services/Setup/SetupSessionState.cs new file mode 100644 index 0000000..930a882 --- /dev/null +++ b/src/Craft/Services/Setup/SetupSessionState.cs @@ -0,0 +1,57 @@ +namespace Craft.Setup; + +/// +/// Session-scoped setup-mode flags and the static EasyAuth env check. +/// PowerShell reaches these via AppLifecycleBridge; C# callers use +/// (facade) or this type directly. +/// +public class SetupSessionState +{ + private readonly ILogger _logger; + private volatile bool _setupModeRequested; + private volatile bool _setupCompleted; + private string? _setupCompletedReason; + + public SetupSessionState(ILogger logger) + { + _logger = logger; + } + + /// + /// Check whether EasyAuth is fully configured by inspecting environment variables. + /// + public static bool IsEasyAuthConfigured() + { + var authEnabled = Environment.GetEnvironmentVariable("WEBSITE_AUTH_ENABLED"); + return string.Equals(authEnabled, "True", StringComparison.OrdinalIgnoreCase); + } + + /// + /// Explicitly enables the Craft setup wizard. Called from the hosted app (via + /// AppLifecycleBridge.RequestSetupMode) when it cannot self-configure. + /// + public void RequestSetupMode(string reason = "Setup mode requested by application") + { + _setupModeRequested = true; + _logger.LogWarning("[Lifecycle] Setup mode explicitly enabled: {Reason}", reason); + } + + /// True once the hosted app has called . + public bool IsSetupModeRequested() => _setupModeRequested; + + /// + /// Marks setup as completed for this process — credentials applied, pending restart. + /// + public void MarkSetupCompleted(string reason = "Setup credentials applied") + { + _setupCompleted = true; + _setupCompletedReason = reason; + _logger.LogInformation("[Lifecycle] Setup marked as completed: {Reason}", reason); + } + + /// True if setup credentials have already been applied this session. + public bool IsSetupCompleted() => _setupCompleted; + + /// Reason passed to , or null if not completed. + public string? GetSetupCompletedReason() => _setupCompletedReason; +} diff --git a/src/Craft/Services/Setup/SetupUserBootstrap.cs b/src/Craft/Services/Setup/SetupUserBootstrap.cs new file mode 100644 index 0000000..a42a887 --- /dev/null +++ b/src/Craft/Services/Setup/SetupUserBootstrap.cs @@ -0,0 +1,119 @@ +using System.Text.Json; +using Craft.Configuration; +using Craft.Storage; + +namespace Craft.Setup; + +/// +/// First-user seeding and allowedUsers table probes for the setup wizard. +/// +public class SetupUserBootstrap +{ + private readonly ILogger _logger; + private readonly CraftSettings _settings; + private readonly IUserTableStore _store; + + public SetupUserBootstrap(ILogger logger, CraftSettings settings, IUserTableStore store) + { + _logger = logger; + _settings = settings; + _store = store; + } + + /// + /// Resolves the user table name with the same sanitization as AuthService. + /// + private string ResolveUserTableName() + { + var raw = _settings.Auth.UserTableName; + var sanitized = new string(raw.Where(char.IsLetterOrDigit).ToArray()); + if (sanitized.Length > 63) sanitized = sanitized[..63]; + if (sanitized.Length < 3) sanitized = "allowedUsers"; + return sanitized; + } + + /// + /// Checks the allowedUsers table status: whether it's reachable and whether + /// it already contains any users. + /// + public async Task CheckAllowedUsersStatus(CancellationToken ct = default) + { + try + { + var tableName = ResolveUserTableName(); + await _store.EnsureTableAsync(tableName, ct); + + var count = 0; + await foreach (var row in _store.QueryTableAsync(tableName, ct)) + { + if (!row.RowKey.StartsWith('_')) + { + count++; + if (count > 0) break; // We only need to know if any exist + } + } + + return new SetupService.AllowedUsersStatus + { + Connected = true, + HasUsers = count > 0 + }; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "[Setup] Failed to check allowedUsers table"); + return new SetupService.AllowedUsersStatus + { + Connected = false, + HasUsers = false, + Error = ex.Message + }; + } + } + + /// + /// Seeds the first user into the allowedUsers table with the roles from + /// Setup.FirstUserRoles (defaults to "superadmin" when unset). + /// Only works when the table is empty — refuses if users already exist. + /// Uses the same entity schema as CIPP-API's Invoke-ExecCIPPUsers. + /// + public async Task SeedFirstUser(string upn, CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(upn)) + throw new ArgumentException("UPN (email) is required."); + + // Invariant, not current-culture: this value is an identity key compared against rows + // written by AuthService (which already lowercases invariantly). Under a Turkish locale + // the two would disagree on "I"/"i" and a seeded user would fail to match. + upn = upn.Trim().ToLowerInvariant(); + + var tableName = ResolveUserTableName(); + await _store.EnsureTableAsync(tableName, ct); + + // Guard: refuse if the table already has users + await foreach (var row in _store.QueryTableAsync(tableName, ct)) + { + if (!row.RowKey.StartsWith('_')) + throw new InvalidOperationException("The allowed users table already contains users. First-user seeding is only available on an empty table."); + } + + string[] roles = _settings.Setup.FirstUserRoles.Count > 0 + ? _settings.Setup.FirstUserRoles.ToArray() + : ["superadmin"]; + var rolesJson = JsonSerializer.Serialize(roles); + + var userRow = new StoreRow("User", upn) + { + Properties = + { + ["Roles"] = rolesJson, + ["ManualRoles"] = rolesJson, + ["AutoRoles"] = "[]", + ["Source"] = "Manual" + } + }; + + await _store.UpsertAsync(tableName, userRow, ct); + _logger.LogInformation("[Setup] Seeded first user {Upn} with roles {Roles}", upn, string.Join(",", roles)); + } +} diff --git a/Services/Setup/index.html b/src/Craft/Services/Setup/index.html similarity index 100% rename from Services/Setup/index.html rename to src/Craft/Services/Setup/index.html diff --git a/Services/Setup/startup.html b/src/Craft/Services/Setup/startup.html similarity index 100% rename from Services/Setup/startup.html rename to src/Craft/Services/Setup/startup.html diff --git a/Services/Storage/AzureTableStore.cs b/src/Craft/Services/Storage/AzureTableStore.cs similarity index 91% rename from Services/Storage/AzureTableStore.cs rename to src/Craft/Services/Storage/AzureTableStore.cs index 2e53ea0..d50d9a1 100644 --- a/Services/Storage/AzureTableStore.cs +++ b/src/Craft/Services/Storage/AzureTableStore.cs @@ -11,7 +11,7 @@ namespace Craft.Storage; /// references Azure.Data.Tables. All Azure-specific concerns — 100-entity transaction batches, /// the ~4 MB transaction cap, OData filter escaping, and 404 handling — are contained here. /// -public sealed class AzureTableStore : ICraftTableStore +public sealed class AzureTableStore : ICraftTableStore, IUserTableStore { private readonly Lazy _connectionString; private readonly ConcurrentDictionary _clients = new(StringComparer.OrdinalIgnoreCase); @@ -33,13 +33,26 @@ public sealed class AzureTableStore : ICraftTableStore "PartitionKey", "RowKey", "Timestamp", "odata.etag" }; + /// + /// Shared host store — orchestrator tables and health probes. Never applies + /// Auth:UserStorageConnection; that override is only for . + /// public AzureTableStore(CraftSettings settings) + : this(settings, explicitConnection: null, purpose: "table storage") + { + } + + /// + /// Store bound to an explicit connection (e.g. allowedUsers isolation). Empty/null + /// falls through to the shared resolution chain. + /// + public AzureTableStore(CraftSettings settings, string? explicitConnection, string purpose) { // Resolved lazily so constructing the store on a role that never touches storage does not - // require a connection string — it is only resolved on first actual use. Prefers the explicit - // RBAC-table override for backward compatibility; else the shared AzureWebJobsStorage connection. + // require a connection string — it is only resolved on first actual use. + var overrideConnection = string.IsNullOrWhiteSpace(explicitConnection) ? null : explicitConnection; _connectionString = new Lazy(() => - settings.Storage.ResolveConnection(settings.Auth.UserStorageConnection, "table storage")); + settings.Storage.ResolveConnection(overrideConnection, purpose)); _clientOptions = BuildClientOptions(settings.Storage); } diff --git a/Services/Storage/ICraftTableStore.cs b/src/Craft/Services/Storage/ICraftTableStore.cs similarity index 100% rename from Services/Storage/ICraftTableStore.cs rename to src/Craft/Services/Storage/ICraftTableStore.cs diff --git a/src/Craft/Services/Storage/IUserTableStore.cs b/src/Craft/Services/Storage/IUserTableStore.cs new file mode 100644 index 0000000..88a5328 --- /dev/null +++ b/src/Craft/Services/Storage/IUserTableStore.cs @@ -0,0 +1,9 @@ +namespace Craft.Storage; + +/// +/// Table store for the allowedUsers authorization table. Resolves via +/// when set; otherwise shares the +/// host's connection (AzureWebJobsStorage / +/// App:Storage:ConnectionString). Orchestrator and other host tables never use the auth override. +/// +public interface IUserTableStore : ICraftTableStore; diff --git a/Services/Storage/StorageHealthMonitor.cs b/src/Craft/Services/Storage/StorageHealthMonitor.cs similarity index 100% rename from Services/Storage/StorageHealthMonitor.cs rename to src/Craft/Services/Storage/StorageHealthMonitor.cs diff --git a/Services/Storage/StoreRow.cs b/src/Craft/Services/Storage/StoreRow.cs similarity index 100% rename from Services/Storage/StoreRow.cs rename to src/Craft/Services/Storage/StoreRow.cs diff --git a/tests/Craft.Tests/ConfigurationReferenceTests.cs b/tests/Craft.Tests/ConfigurationReferenceTests.cs index 8b032cc..ffa9866 100644 --- a/tests/Craft.Tests/ConfigurationReferenceTests.cs +++ b/tests/Craft.Tests/ConfigurationReferenceTests.cs @@ -71,6 +71,7 @@ public static TheoryData DocumentedDefaults() { "App:Scheduler:ConfigFile", settings.Scheduler.ConfigFile }, { "App:Scheduler:CheckIntervalSeconds", settings.Scheduler.CheckIntervalSeconds }, { "App:Scheduler:ApplyTZOffset", settings.Scheduler.ApplyTZOffset }, + { "App:Scheduler:Timezone", settings.Scheduler.Timezone }, { "App:Orchestrator:TablePrefix", settings.Orchestrator.TablePrefix }, { "App:Orchestrator:MaxRetries", settings.Orchestrator.MaxRetries }, { "App:Cache:MaxEntries", settings.Cache.MaxEntries }, @@ -97,9 +98,36 @@ public void DocumentedValue_MatchesTheCSharpDefault(string key, object expected) "(or stop listing this key in it), because right now it misleads anyone copying from it."); } + /// + /// Every uncommented leaf under App in the example must appear in + /// . Without this, a new key can be uncommented in the example + /// and never checked against the C# default. + /// + [Fact] + public void EveryUncommentedAppLeaf_IsCoveredByDocumentedDefaults() + { + var covered = DocumentedDefaults().Select(row => (string)row[0]!).ToHashSet(StringComparer.OrdinalIgnoreCase); + + // Keys with a non-null Value are leaves; intermediate sections have null Value. + var leaves = LoadExample() + .GetSection("App") + .AsEnumerable(makePathsRelative: false) + .Where(kv => kv.Value is not null) + .Select(kv => kv.Key) + .ToList(); + + Assert.NotEmpty(leaves); + + var missing = leaves.Where(k => !covered.Contains(k)).OrderBy(k => k, StringComparer.Ordinal).ToList(); + Assert.True(missing.Count == 0, + "Uncommented App keys in appsettings.example.jsonc are missing from DocumentedDefaults(): " + + string.Join(", ", missing) + + ". Add each to DocumentedDefaults() so its value is checked against the C# default."); + } + /// /// The example must never become load-bearing. Craft ships no appsettings.json, so a deployment - /// that sets nothing gets the C# defaults — this asserts a couple of representative ones directly, + /// that sets nothing gets the C# defaults — this asserts representative ones directly, /// independent of any file. /// [Fact] @@ -107,11 +135,34 @@ public void Defaults_AreUsableWithNoConfigurationFileAtAll() { var settings = new CraftSettings(); + Assert.Equal("Craft", settings.Name); + Assert.Equal("Immediate", settings.ReadinessMode); + Assert.Equal(2, settings.Worker.HttpPoolSize); Assert.Equal(4, settings.Worker.BgPoolSize); + Assert.Equal("AfterReady", settings.Worker.WarmupMode); + Assert.True(settings.Worker.ReuseRunspaceThread); + Assert.Equal("craft-session", settings.Auth.CookieName); Assert.False(settings.Realtime.Enabled); Assert.False(settings.Setup.Enabled); + Assert.True(settings.Health.Enabled); + Assert.Equal("/healthz", settings.Health.Path); + + Assert.True(settings.RateLimit.Enabled); + Assert.Equal(300, settings.RateLimit.PermitPerWindow); + Assert.Equal(10, settings.RateLimit.WindowSeconds); + + Assert.True(settings.Orchestrator.BatchStatusWrites); + Assert.Equal("Orchestrator", settings.Orchestrator.TablePrefix); + Assert.Equal(3, settings.Orchestrator.MaxRetries); + + Assert.True(settings.Frontend.Compression); + Assert.Equal(30, settings.Storage.MaxConnectionsPerServer); + Assert.Equal(15, settings.BackgroundLimiter.ScaleUpAfterSeconds); + + Assert.Equal(1000, settings.Cache.MaxEntries); + Assert.Equal(600, settings.Cache.DefaultTtlSeconds); } } diff --git a/tests/Craft.Tests/Craft.Tests.csproj b/tests/Craft.Tests/Craft.Tests.csproj index 87caf02..196c94c 100644 --- a/tests/Craft.Tests/Craft.Tests.csproj +++ b/tests/Craft.Tests/Craft.Tests.csproj @@ -2,7 +2,7 @@ - net8.0 + net10.0 false true @@ -25,7 +25,7 @@ - + - + diff --git a/tests/Craft.Tests/JobDescriptorRehydrationTests.cs b/tests/Craft.Tests/JobDescriptorRehydrationTests.cs index e192f54..206e512 100644 --- a/tests/Craft.Tests/JobDescriptorRehydrationTests.cs +++ b/tests/Craft.Tests/JobDescriptorRehydrationTests.cs @@ -1,9 +1,9 @@ using System.Runtime.CompilerServices; using Craft.Configuration; +using Craft.Hosting; using Craft.Orchestration; using Craft.PowerShellHost; using Craft.Storage; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; namespace Craft.Tests; @@ -88,15 +88,16 @@ public Task DeletePartitionAsync(string table, string partitionKey, Cancellation } } - private static (JobManager Jobs, CountingStore Store, OrchestratorTableStore Orch) NewHarness() + private static (JobManager Jobs, CountingStore Store, OrchestratorTableStore Orch) NewHarness( + Lazy? workResolver = null) { var settings = new CraftSettings(); settings.Worker.BgPoolSize = 8; - var config = new ConfigurationBuilder().AddInMemoryCollection([]).Build(); var repo = new ScriptRepository(NullLogger.Instance, settings); - var pool = new PowerShellWorkerPool(repo, NullLogger.Instance, config, settings); - var limiter = new BackgroundTaskLimiter(NullLogger.Instance, config, settings, pool); - var jobs = new JobManager(NullLogger.Instance, settings, limiter); + var pool = new PowerShellWorkerPool(repo, NullLogger.Instance, settings, new StartupProgressService(), + new Lazy(() => null!)); + var limiter = new BackgroundTaskLimiter(NullLogger.Instance, settings, pool); + var jobs = new JobManager(NullLogger.Instance, settings, limiter, workResolver: workResolver); var store = new CountingStore(); var orch = new OrchestratorTableStore(NullLogger.Instance, settings, store); return (jobs, store, orch); @@ -104,38 +105,27 @@ private static (JobManager Jobs, CountingStore Store, OrchestratorTableStore Orc private static Task Pump(JobManager jobs) => Task.Run(() => jobs.StartAsync(CancellationToken.None)); - private static async Task WaitUntil(Func condition, int timeoutMs = 5000) - { - var deadline = Environment.TickCount64 + timeoutMs; - while (Environment.TickCount64 < deadline) - { - if (condition()) return true; - await Task.Delay(10); - } - return condition(); - } - /// The descriptor reaches the resolver intact — that is the whole contract of the queue. [Fact] public async Task Dispatch_HandsTheDescriptorToTheResolver() { - var (jobs, _, _) = NewHarness(); var seen = new List(); var done = 0; - - jobs.SetWorkResolver((d, _) => + JobWorkResolver? resolver = null; + var (jobs, _, _) = NewHarness(new Lazy(() => resolver!)); + resolver = (d, _) => { lock (seen) seen.Add(d); return Task.FromResult?>( _ => { Interlocked.Increment(ref done); return Task.CompletedTask; }); - }); + }; for (var i = 0; i < 25; i++) jobs.Enqueue(new JobDescriptor("CIPPDBCacheRun", $"Graph_tenant{i:D3}", 5), $"CIPPDBCacheRun-Graph_tenant{i:D3}"); _ = Pump(jobs); - Assert.True(await WaitUntil(() => Volatile.Read(ref done) == 25), $"only {done}/25 ran"); - await Task.WhenAny(jobs.StopAsync(CancellationToken.None), Task.Delay(5000)); + Assert.True(await TestWait.WaitUntil(() => Volatile.Read(ref done) == 25), $"only {done}/25 ran"); + await TestWait.StopWithin(jobs.StopAsync(CancellationToken.None)); lock (seen) { @@ -152,23 +142,23 @@ public async Task Dispatch_HandsTheDescriptorToTheResolver() [Fact] public async Task StaleDescriptor_IsSkipped_AndDispatchContinues() { - var (jobs, _, _) = NewHarness(); var ran = 0; - - jobs.SetWorkResolver((d, _) => Task.FromResult?>( + JobWorkResolver? resolver = null; + var (jobs, _, _) = NewHarness(new Lazy(() => resolver!)); + resolver = (d, _) => Task.FromResult?>( d.TaskId == "gone" ? null // stale — resolver declines - : _ => { Interlocked.Increment(ref ran); return Task.CompletedTask; })); + : _ => { Interlocked.Increment(ref ran); return Task.CompletedTask; }); jobs.Enqueue(new JobDescriptor("run", "gone", 0), "run-gone"); for (var i = 0; i < 10; i++) jobs.Enqueue(new JobDescriptor("run", $"live{i}", 5), $"run-live{i}"); _ = Pump(jobs); - Assert.True(await WaitUntil(() => Volatile.Read(ref ran) == 10), $"only {ran}/10 ran after a stale descriptor"); + Assert.True(await TestWait.WaitUntil(() => Volatile.Read(ref ran) == 10), $"only {ran}/10 ran after a stale descriptor"); var stale = jobs.GetJobs(status: "Skipped"); - await Task.WhenAny(jobs.StopAsync(CancellationToken.None), Task.Delay(5000)); + await TestWait.StopWithin(jobs.StopAsync(CancellationToken.None)); Assert.Single(stale); Assert.Equal("run-gone", stale[0].Name); @@ -182,10 +172,10 @@ public async Task DescriptorWithNoResolver_FailsTheJob_RatherThanDisappearing() jobs.Enqueue(new JobDescriptor("run", "task", 0), "run-task"); _ = Pump(jobs); - Assert.True(await WaitUntil(() => jobs.GetJobs(status: "Failed").Count == 1)); + Assert.True(await TestWait.WaitUntil(() => jobs.GetJobs(status: "Failed").Count == 1)); var failed = jobs.GetJobs(status: "Failed").Single(); - await Task.WhenAny(jobs.StopAsync(CancellationToken.None), Task.Delay(5000)); + await TestWait.StopWithin(jobs.StopAsync(CancellationToken.None)); Assert.Contains("resolver", failed.LastError, StringComparison.OrdinalIgnoreCase); } @@ -264,7 +254,7 @@ public async Task IsQueuedOrRunning_False_OnceTheJobHasCompleted() work: _ => { ran.TrySetResult(); return Task.CompletedTask; }); await ran.Task; - Assert.True(await WaitUntil(() => !jobs.IsQueuedOrRunning("run-b-task-1")), + Assert.True(await TestWait.WaitUntil(() => !jobs.IsQueuedOrRunning("run-b-task-1")), "a completed job still reports as queued/running — the re-drive would never fire"); } diff --git a/tests/Craft.Tests/JobDispatchStallTests.cs b/tests/Craft.Tests/JobDispatchStallTests.cs index 5e37be7..c54adf6 100644 --- a/tests/Craft.Tests/JobDispatchStallTests.cs +++ b/tests/Craft.Tests/JobDispatchStallTests.cs @@ -1,7 +1,7 @@ using Craft.Configuration; +using Craft.Hosting; using Craft.Orchestration; using Craft.PowerShellHost; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; namespace Craft.Tests; @@ -33,10 +33,10 @@ private static (JobManager Jobs, BackgroundTaskLimiter Limiter) NewPair(int bgPo { var settings = new CraftSettings(); settings.Worker.BgPoolSize = bgPoolSize; - var config = new ConfigurationBuilder().AddInMemoryCollection([]).Build(); var repo = new ScriptRepository(NullLogger.Instance, settings); - var pool = new PowerShellWorkerPool(repo, NullLogger.Instance, config, settings); - var limiter = new BackgroundTaskLimiter(NullLogger.Instance, config, settings, pool); + var pool = new PowerShellWorkerPool(repo, NullLogger.Instance, settings, new StartupProgressService(), + new Lazy(() => null!)); + var limiter = new BackgroundTaskLimiter(NullLogger.Instance, settings, pool); return (new JobManager(NullLogger.Instance, settings, limiter), limiter); } @@ -52,19 +52,8 @@ private static (JobManager Jobs, BackgroundTaskLimiter Limiter) NewPair(int bgPo private static Task StartPump(JobManager jobs) => Task.Run(() => jobs.StartAsync(CancellationToken.None)); /// Stop without ever hanging the suite, whatever state the loop is in. - private static async Task StopPump(JobManager jobs) => - await Task.WhenAny(jobs.StopAsync(CancellationToken.None), Task.Delay(5000)); - - private static async Task WaitUntil(Func condition, int timeoutMs = 5000) - { - var deadline = Environment.TickCount64 + timeoutMs; - while (Environment.TickCount64 < deadline) - { - if (condition()) return true; - await Task.Delay(10); - } - return condition(); - } + private static Task StopPump(JobManager jobs) => + TestWait.StopWithin(jobs.StopAsync(CancellationToken.None)); /// /// THE GUARANTEE. A job that blocks its thread must not stop other jobs from being dispatched while @@ -97,9 +86,9 @@ public async Task DispatchContinues_WhileOneJobBlocksItsThread() runName: "CIPPDBCacheRun"); _ = StartPump(jobs); - Assert.True(reached.Wait(5000), "the blocking job never started"); + Assert.True(reached.Wait(30_000), "the blocking job never started"); - var drained = await WaitUntil(() => Volatile.Read(ref ran) == 20); + var drained = await TestWait.WaitUntil(() => Volatile.Read(ref ran) == 20); // Snapshot the stall signature while the blocker is still holding its thread. var stillBlocked = !blocked.IsSet; @@ -142,7 +131,7 @@ public async Task LimiterWaiting_ReportsBacklog_WhenTheLimiterIsSaturated() _ = StartPump(jobs); // Saturated + backlog ⇒ the loop must be parked in AcquireAsync and say so. - var reported = await WaitUntil(() => limiter.Waiting > 0 && jobs.QueuedCount > 0); + var reported = await TestWait.WaitUntil(() => limiter.Waiting > 0 && jobs.QueuedCount > 0); var waiting = limiter.Waiting; var queued = jobs.QueuedCount; var active = limiter.Active; @@ -178,7 +167,7 @@ public async Task Dispatch_NeverExceedsTheLimiterCeiling() }); _ = StartPump(jobs); - await WaitUntil(() => Volatile.Read(ref concurrent) >= limiter.CurrentMax); + await TestWait.WaitUntil(() => Volatile.Read(ref concurrent) >= limiter.CurrentMax); await Task.Delay(300); var observedPeak = Volatile.Read(ref peak); diff --git a/tests/Craft.Tests/JobDurabilityTests.cs b/tests/Craft.Tests/JobDurabilityTests.cs index 450ba42..055eb03 100644 --- a/tests/Craft.Tests/JobDurabilityTests.cs +++ b/tests/Craft.Tests/JobDurabilityTests.cs @@ -1,9 +1,9 @@ using Craft.Auth; using Craft.Configuration; +using Craft.Hosting; using Craft.Orchestration; using Craft.PowerShellHost; using Craft.Storage; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; namespace Craft.Tests; @@ -15,7 +15,7 @@ namespace Craft.Tests; /// public class JobDurabilityTests { - private sealed class FakeStore : ICraftTableStore + private sealed class FakeStore : IUserTableStore { private readonly Dictionary> _tables = new(); @@ -84,15 +84,15 @@ private static OrchestratorTableStore NewStore(out FakeStore backing) return new OrchestratorTableStore(NullLogger.Instance, new CraftSettings(), backing); } - private static JobManager NewJobManager() + private static JobManager NewJobManager(Lazy? stateWriter = null) { var settings = new CraftSettings(); settings.Worker.BgPoolSize = 8; - var config = new ConfigurationBuilder().AddInMemoryCollection([]).Build(); var repo = new ScriptRepository(NullLogger.Instance, settings); - var pool = new PowerShellWorkerPool(repo, NullLogger.Instance, config, settings); - var limiter = new BackgroundTaskLimiter(NullLogger.Instance, config, settings, pool); - return new JobManager(NullLogger.Instance, settings, limiter); + var pool = new PowerShellWorkerPool(repo, NullLogger.Instance, settings, new StartupProgressService(), + new Lazy(() => null!)); + var limiter = new BackgroundTaskLimiter(NullLogger.Instance, settings, pool); + return new JobManager(NullLogger.Instance, settings, limiter, stateWriter: stateWriter); } /// Records what the JobManager reports, standing in for OrchestratorService. @@ -173,9 +173,8 @@ await store.WriteTaskStatusBatchAsync( [Fact] public void ChangePriority_ReportsTheDescriptor_ForDurablePersistence() { - var jobs = NewJobManager(); var sink = new RecordingSink(); - jobs.SetDescriptorStateWriter(sink); + var jobs = NewJobManager(new Lazy(() => sink)); var id = jobs.Enqueue(new JobDescriptor("run", "task1", 5), "run-task1"); Assert.True(jobs.ChangePriority(id, 0)); @@ -189,9 +188,8 @@ public void ChangePriority_ReportsTheDescriptor_ForDurablePersistence() [Fact] public void CancelJob_ReportsTheDescriptor_SoRecoveryDoesNotReQueueIt() { - var jobs = NewJobManager(); var sink = new RecordingSink(); - jobs.SetDescriptorStateWriter(sink); + var jobs = NewJobManager(new Lazy(() => sink)); var id = jobs.Enqueue(new JobDescriptor("run", "task1", 5), "run-task1"); Assert.True(jobs.CancelJob(id)); @@ -203,9 +201,8 @@ public void CancelJob_ReportsTheDescriptor_SoRecoveryDoesNotReQueueIt() [Fact] public void CancelRun_ReportsEveryQueuedDescriptor() { - var jobs = NewJobManager(); var sink = new RecordingSink(); - jobs.SetDescriptorStateWriter(sink); + var jobs = NewJobManager(new Lazy(() => sink)); for (var i = 0; i < 25; i++) jobs.Enqueue(new JobDescriptor("run", $"task{i}", 5), $"run-task{i}"); @@ -224,9 +221,8 @@ public void CancelRun_ReportsEveryQueuedDescriptor() [Fact] public void ClosureJobs_AreNotReported_HavingNothingToPersist() { - var jobs = NewJobManager(); var sink = new RecordingSink(); - jobs.SetDescriptorStateWriter(sink); + var jobs = NewJobManager(new Lazy(() => sink)); var id = jobs.Enqueue("Start-CIPPDBCache", 5, _ => Task.CompletedTask); Assert.True(jobs.ChangePriority(id, 0)); @@ -240,8 +236,7 @@ public void ClosureJobs_AreNotReported_HavingNothingToPersist() [Fact] public void SinkFailure_DoesNotFailTheOperatorAction() { - var jobs = NewJobManager(); - jobs.SetDescriptorStateWriter(new ThrowingSink()); + var jobs = NewJobManager(new Lazy(() => new ThrowingSink())); var id = jobs.Enqueue(new JobDescriptor("run", "task1", 5), "run-task1"); @@ -266,8 +261,7 @@ private sealed class ThrowingSink : IJobDescriptorStateWriter [Fact] public void AuthService_Dispose_DoesNotThrow() { - var config = new ConfigurationBuilder().AddInMemoryCollection([]).Build(); - var auth = new AuthService(NullLogger.Instance, config, new CraftSettings(), new FakeStore()); + var auth = new AuthService(NullLogger.Instance, new CraftSettings(), new FakeStore()); auth.Dispose(); // must not throw — this is what broke host shutdown auth.Dispose(); // and must stay safe if the container disposes twice diff --git a/tests/Craft.Tests/JobQueueRetentionTests.cs b/tests/Craft.Tests/JobQueueRetentionTests.cs index 678919c..7fd2d2b 100644 --- a/tests/Craft.Tests/JobQueueRetentionTests.cs +++ b/tests/Craft.Tests/JobQueueRetentionTests.cs @@ -1,7 +1,7 @@ using Craft.Configuration; +using Craft.Hosting; using Craft.Orchestration; using Craft.PowerShellHost; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; namespace Craft.Tests; @@ -44,10 +44,10 @@ private static JobManager NewJobManager(int bgPoolSize = 8) { var settings = new CraftSettings(); settings.Worker.BgPoolSize = bgPoolSize; - var config = new ConfigurationBuilder().AddInMemoryCollection([]).Build(); var repo = new ScriptRepository(NullLogger.Instance, settings); - var pool = new PowerShellWorkerPool(repo, NullLogger.Instance, config, settings); - var limiter = new BackgroundTaskLimiter(NullLogger.Instance, config, settings, pool); + var pool = new PowerShellWorkerPool(repo, NullLogger.Instance, settings, new StartupProgressService(), + new Lazy(() => null!)); + var limiter = new BackgroundTaskLimiter(NullLogger.Instance, settings, pool); return new JobManager(NullLogger.Instance, settings, limiter); } diff --git a/tests/Craft.Tests/OrchestratorResultStreamingTests.cs b/tests/Craft.Tests/OrchestratorResultStreamingTests.cs index f731d04..233b967 100644 --- a/tests/Craft.Tests/OrchestratorResultStreamingTests.cs +++ b/tests/Craft.Tests/OrchestratorResultStreamingTests.cs @@ -1,6 +1,7 @@ using System.Runtime.CompilerServices; using System.Text.Json; using Craft.Configuration; +using Craft.Orchestration; using Craft.Storage; using Microsoft.Extensions.Logging.Abstractions; diff --git a/tests/Craft.Tests/PowerShellContractTests.cs b/tests/Craft.Tests/PowerShellContractTests.cs index 6c06bb9..2ed543d 100644 --- a/tests/Craft.Tests/PowerShellContractTests.cs +++ b/tests/Craft.Tests/PowerShellContractTests.cs @@ -1,5 +1,3 @@ -using System.Reflection; - namespace Craft.Tests; /// @@ -8,9 +6,12 @@ namespace Craft.Tests; /// Hosted apps call these directly — [Craft.Services.RealtimeBridge]::Publish(...), /// [Craft.Services.OrchestratorBridge]::QueueOrchestration(...). A namespace rename, a class /// rename or a visibility change compiles perfectly and then fails at runtime inside the hosted app -/// with "Unable to find type", which is the worst possible place to discover it. Type forwarding -/// cannot soften the blow: [TypeForwardedTo] only works across assemblies, and all of these -/// live in Craft.dll. +/// with "Unable to find type", which is the worst possible place to discover it. +/// +/// +/// After the Contracts extraction, bridge facades and PowerShellRunnerService live in the +/// host Craft assembly while DTOs live in Craft.Contracts (same Craft.Services +/// namespace). PowerShell resolves by FQN across loaded assemblies; these tests do the same. /// /// /// If a test here fails, the fix is almost always to revert the rename — not to update the list. @@ -20,21 +21,18 @@ namespace Craft.Tests; /// public class PowerShellContractTests { - private static readonly Assembly s_craft = typeof(Craft.Services.RealtimeBridge).Assembly; private static readonly string[] ContractSurface = { - // Bridges + // Bridges + runner (host assembly) "AppLifecycleBridge", "AuthBridge", "CacheBridge", "LogBridge", "OrchestratorBridge", "QueueBridge", "QueueStatusBridge", "RealtimeBridge", "SchedulerBridge", "StartupInfoBridge", "StatsHistoryBridge", "WorkerMetricsBridge", "PowerShellRunnerService", - // DTOs a bridge can hand to PowerShell, directly or nested + // DTOs (Craft.Contracts assembly, same namespace) "CacheStats", "LogFileInfo", "StartupStats", "StatsDataPoint", "ScriptResult", - "JobRecord", "JobSummary", "JobRunSummary", "JobDetail", - "WorkerStats", "WorkerMetricsSnapshot", "PoolMetrics", "WorkerDetail", "LimiterMetrics", + "JobSummary", "JobRunSummary", "JobDetail", + "WorkerMetricsSnapshot", "PoolMetrics", "WorkerDetail", "LimiterMetrics", "JobMetrics", "MemoryMetrics", "WorkerSummary", "MemoryBreakdown", "GenerationDetail", - // Nested public records on the bridges — the queued-item types the background side drains. - "PendingOrchestration", "PendingPlannerRun", "PendingQueueCommand", }; /// @@ -58,11 +56,49 @@ public class PowerShellContractTests "Craft.Services.PowerShellRunnerService", }; + private static Type? FindContractType(string fullName) + { + // Touch both assemblies so they are loaded before we scan. + _ = typeof(Craft.Services.RealtimeBridge); + _ = typeof(Craft.Services.ScriptResult); + + foreach (var asm in AppDomain.CurrentDomain.GetAssemblies()) + { + var name = asm.GetName().Name; + if (name is not ("Craft" or "Craft.Contracts")) + continue; + var type = asm.GetType(fullName, throwOnError: false); + if (type is not null) + return type; + } + + return null; + } + + private static IEnumerable EnumerateCraftServicesTypes() + { + _ = typeof(Craft.Services.RealtimeBridge); + _ = typeof(Craft.Services.ScriptResult); + + foreach (var asm in AppDomain.CurrentDomain.GetAssemblies()) + { + var name = asm.GetName().Name; + if (name is not ("Craft" or "Craft.Contracts")) + continue; + + foreach (var type in asm.GetExportedTypes()) + { + if (type.Namespace == "Craft.Services") + yield return type; + } + } + } + [Theory] [MemberData(nameof(ContractTypeNames))] public void ContractType_ExistsAndIsPublic(string fullName) { - var type = s_craft.GetType(fullName, throwOnError: false); + var type = FindContractType(fullName); Assert.True(type is not null, $"[{fullName}] is referenced from PowerShell but no longer exists under that name. " + @@ -82,11 +118,11 @@ public void ContractType_ExistsAndIsPublic(string fullName) public void HttpResponseContext_KeepsFunctionsWorkerTypeName() { const string expected = "Microsoft.Azure.Functions.PowerShellWorker.HttpResponseContext"; - var type = s_craft.GetType(expected, throwOnError: false); + var type = typeof(Microsoft.Azure.Functions.PowerShellWorker.HttpResponseContext); - Assert.True(type is not null, - $"{expected} must keep this exact name — hosted-app routers match on PSObject.TypeNames."); - Assert.True(type!.IsPublic); + Assert.Equal(expected, type.FullName); + Assert.True(type.IsPublic); + Assert.Equal("Craft.Contracts", type.Assembly.GetName().Name); } /// @@ -100,9 +136,9 @@ public void CraftServicesNamespace_ContainsOnlyTheContractSurface() { var allowed = new HashSet(ContractSurface); - var actual = s_craft.GetExportedTypes() - .Where(t => t.Namespace == "Craft.Services") + var actual = EnumerateCraftServicesTypes() .Select(t => t.Name) + .Distinct() .ToList(); var unexpected = actual.Where(n => !allowed.Contains(n)).OrderBy(n => n).ToList(); diff --git a/tests/Craft.Tests/ScriptRepositoryLoadTests.cs b/tests/Craft.Tests/ScriptRepositoryLoadTests.cs new file mode 100644 index 0000000..4304da6 --- /dev/null +++ b/tests/Craft.Tests/ScriptRepositoryLoadTests.cs @@ -0,0 +1,88 @@ +using Craft.Configuration; +using Craft.PowerShellHost; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Craft.Tests; + +/// +/// Startup script indexing — must discover routes from +/// multi-file module trees and survive parallel parse of many .ps1 files. +/// +public class ScriptRepositoryLoadTests +{ + [Fact] + public void LoadAll_IndexesFunctionsAndRoutes_FromParallelPs1Parse() + { + var root = Path.Combine(Path.GetTempPath(), "craft-scriptrepo-" + Guid.NewGuid().ToString("N")); + var module = Path.Combine(root, "Modules", "TestHttp"); + Directory.CreateDirectory(module); + + try + { + // Mix: bare using line (must be stripped), function defs, bare script. + File.WriteAllText(Path.Combine(module, "Invoke-Alpha.ps1"), + "using namespace System.Net\nfunction Invoke-Alpha { param($Request) 'a' }\n"); + File.WriteAllText(Path.Combine(module, "Invoke-Beta.ps1"), + "function Invoke-Beta { param($Request) 'b' }\n"); + File.WriteAllText(Path.Combine(module, "gamma.ps1"), + "# bare timer-style script\n'g'\n"); + + // Enough files to exercise Parallel.ForEach meaningfully. + for (var i = 0; i < 32; i++) + { + File.WriteAllText(Path.Combine(module, $"Invoke-Item{i:D2}.ps1"), + $"function Invoke-Item{i:D2} {{ param($Request) '{i}' }}\n"); + } + + var settings = new CraftSettings(); + settings.Scripts.HttpModules = ["TestHttp"]; + settings.Scripts.BackgroundScriptDirs = []; + + var repo = new ScriptRepository(NullLogger.Instance, settings); + repo.LoadAll(root); + + Assert.True(repo.HttpRoutes.ContainsKey("Alpha")); + Assert.True(repo.HttpRoutes.ContainsKey("Beta")); + Assert.Equal("Invoke-Alpha", repo.HttpRoutes["Alpha"]); + Assert.NotNull(repo.GetByRoute("Alpha")); + Assert.NotNull(repo.GetByName("Invoke-Beta")); + Assert.NotNull(repo.GetByName("gamma")); + + for (var i = 0; i < 32; i++) + Assert.True(repo.HttpRoutes.ContainsKey($"Item{i:D2}"), $"missing route Item{i:D2}"); + } + finally + { + try { Directory.Delete(root, recursive: true); } catch { /* best-effort */ } + } + } + + [Fact] + public void LoadAll_StripsUsingDirectives_WithoutLosingFunctionBody() + { + var root = Path.Combine(Path.GetTempPath(), "craft-scriptrepo-" + Guid.NewGuid().ToString("N")); + var module = Path.Combine(root, "Modules", "TestHttp"); + Directory.CreateDirectory(module); + + try + { + File.WriteAllText(Path.Combine(module, "Invoke-WithUsing.ps1"), + "using module Foo\r\nusing assembly Bar\r\nfunction Invoke-WithUsing { 'ok' }\r\n"); + + var settings = new CraftSettings(); + settings.Scripts.HttpModules = ["TestHttp"]; + settings.Scripts.BackgroundScriptDirs = []; + + var repo = new ScriptRepository(NullLogger.Instance, settings); + repo.LoadAll(root); + + var entry = repo.GetByRoute("WithUsing"); + Assert.NotNull(entry); + Assert.Equal("Invoke-WithUsing", entry!.FunctionName); + } + finally + { + try { Directory.Delete(root, recursive: true); } catch { /* best-effort */ } + } + } +} diff --git a/tests/Craft.Tests/SetupWizardStatusTests.cs b/tests/Craft.Tests/SetupWizardStatusTests.cs index 9d47225..ca68240 100644 --- a/tests/Craft.Tests/SetupWizardStatusTests.cs +++ b/tests/Craft.Tests/SetupWizardStatusTests.cs @@ -14,8 +14,8 @@ namespace Craft.Tests; /// public class SetupWizardStatusTests { - private static SetupService NewService(ICraftTableStore store, CraftSettings? settings = null) => - new(NullLogger.Instance, settings ?? new CraftSettings(), store); + private static SetupUserBootstrap NewService(IUserTableStore store, CraftSettings? settings = null) => + new(NullLogger.Instance, settings ?? new CraftSettings(), store); // ── The probe ─────────────────────────────────────────────────────────────────────────────── @@ -231,11 +231,11 @@ public async Task SeedingThenProbing_ReportsTheUserSoStepTwoUnlocks() // ── Fake ──────────────────────────────────────────────────────────────────────────────────── /// - /// In-memory . Only the four members the setup path touches do + /// In-memory . Only the four members the setup path touches do /// anything; the rest throw so an accidental new dependency shows up as a failing test rather /// than a silent no-op. /// - private sealed class FakeStore : ICraftTableStore + private sealed class FakeStore : IUserTableStore { public List Rows { get; } = []; public List EnsuredTables { get; } = []; diff --git a/tests/Craft.Tests/StatusWriterDurabilityTests.cs b/tests/Craft.Tests/StatusWriterDurabilityTests.cs index a7da1a3..4534c1d 100644 --- a/tests/Craft.Tests/StatusWriterDurabilityTests.cs +++ b/tests/Craft.Tests/StatusWriterDurabilityTests.cs @@ -245,9 +245,8 @@ public async Task WritesThatFail_AreRetried_NotLost() backing.FailBatches = false; // storage recovers - var deadline = Environment.TickCount64 + 5000; - while (Environment.TickCount64 < deadline && backing.Rows("OrchestratorTasks").Count == 0) - await Task.Delay(20); + Assert.True(await TestWait.WaitUntil(() => backing.Rows("OrchestratorTasks").Count > 0), + "retried write never landed after storage recovered"); var rows = backing.Rows("OrchestratorTasks"); Assert.Single(rows); @@ -268,9 +267,8 @@ public async Task Retry_DoesNotOverwrite_NewerStateForTheSameTask() writer.QueueTask("run", new OrchestratorTaskItem { Id = "t1", Status = "Completed" }); backing.FailBatches = false; - var deadline = Environment.TickCount64 + 5000; - while (Environment.TickCount64 < deadline && backing.Rows("OrchestratorTasks").Count == 0) - await Task.Delay(20); + Assert.True(await TestWait.WaitUntil(() => backing.Rows("OrchestratorTasks").Count > 0), + "newer Completed state never landed after storage recovered"); var rows = backing.Rows("OrchestratorTasks"); Assert.Single(rows); diff --git a/tests/Craft.Tests/TestWait.cs b/tests/Craft.Tests/TestWait.cs new file mode 100644 index 0000000..98ac56b --- /dev/null +++ b/tests/Craft.Tests/TestWait.cs @@ -0,0 +1,26 @@ +namespace Craft.Tests; + +/// +/// Shared polling helpers for async host / pump tests. Defaults are deliberately generous — +/// CI and loaded hosts routinely exceed the old 5s local-machine budgets. +/// +internal static class TestWait +{ + /// Poll until is true, or the timeout elapses. + public static async Task WaitUntil(Func condition, int timeoutMs = 30_000) + { + var deadline = Environment.TickCount64 + timeoutMs; + while (Environment.TickCount64 < deadline) + { + if (condition()) return true; + await Task.Delay(10); + } + return condition(); + } + + /// + /// Await a stop/shutdown task without hanging the suite when the pump is wedged. + /// + public static Task StopWithin(Task stopTask, int timeoutMs = 15_000) => + Task.WhenAny(stopTask, Task.Delay(timeoutMs)); +} diff --git a/tests/Craft.Tests/ZeroHttpPoolTests.cs b/tests/Craft.Tests/ZeroHttpPoolTests.cs index 15f2e1b..c84527e 100644 --- a/tests/Craft.Tests/ZeroHttpPoolTests.cs +++ b/tests/Craft.Tests/ZeroHttpPoolTests.cs @@ -1,6 +1,6 @@ using Craft.Configuration; +using Craft.Hosting; using Craft.PowerShellHost; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; namespace Craft.Tests; @@ -22,9 +22,13 @@ private static PowerShellWorkerPool Pool(int httpPoolSize) settings.Worker.HttpPoolSize = httpPoolSize; settings.Worker.BgPoolSize = 4; - var config = new ConfigurationBuilder().Build(); var repo = new ScriptRepository(NullLogger.Instance, settings); - return new PowerShellWorkerPool(repo, NullLogger.Instance, config, settings); + return new PowerShellWorkerPool( + repo, + NullLogger.Instance, + settings, + new StartupProgressService(), + new Lazy(() => throw new InvalidOperationException("metrics unused"))); } [Fact]