Recover more of an application and report what cannot be recovered

.github/workflows/dotnet-build.yml

@@ -176,6 +176,57 @@ jobs:
       - name: Test (${{ matrix.project }})
         run: dotnet test ${{ matrix.project }} --no-build --configuration Release --framework net10.0
 
+  # Every Screenplay specification builds its compilation from source strings, which is what makes them hermetic -
+  # and what puts a whole class of defect out of their reach by construction. A compilation built that way has no
+  # intermediate folder, no source generator output on disk and no referenced package declaring an event, so a
+  # generator that mis-attributes one of those produces a wrong document that every specification still passes. Two
+  # shipped that way. This reads a real project through MSBuild and holds the document it generates to the compiler
+  # the language ships - it has to come back with nothing to say, warnings included.
+  screenplay-end-to-end:
+    needs: [dotnet-build]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
+
+      - name: Setup .Net
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: |
+            ${{ env.DOTNET8_VERSION }}
+            ${{ env.DOTNET9_VERSION }}
+            ${{ env.DOTNET10_VERSION }}
+
+      - name: Cache NuGet packages
+        uses: actions/cache@v6
+        with:
+          path: ~/.nuget/packages
+          key: nuget-${{ runner.os }}-${{ hashFiles('**/*.csproj', '**/Directory.*.props', '**/Directory.*.targets') }}
+          restore-keys: |
+            nuget-${{ runner.os }}-
+
+      - name: Download build output
+        uses: actions/download-artifact@v4
+        with:
+          name: build-output
+
+      - name: Extract build output
+        run: tar -xzf build-output.tar.gz
+
+      # The build output artifact carries bin folders only, and reading a project through MSBuild needs the assets
+      # file that restore writes to obj.
+      - name: Restore the application to generate from
+        run: dotnet restore TestApps/AspNetCore/AspNetCore.csproj
+
+      - name: Generate and read back the document of a real application
+        run: |
+          dotnet Source/DotNET/Screenplay.EndToEnd/bin/Release/net10.0/Cratis.Arc.Screenplay.EndToEnd.dll \
+            TestApps/AspNetCore/AspNetCore.csproj \
+            "${RUNNER_TEMP}/AspNetCore.play"
+
   proxy-specs:
     needs: [dotnet-build]
     runs-on: ubuntu-latest

Arc.slnx

@@ -31,6 +31,7 @@
         <Project Path="Source\DotNET\OpenApi\OpenApi.csproj" />
         <Project Path="Source\DotNET\OpenApi.Specs\OpenApi.Specs.csproj" />
         <Project Path="Source\DotNET\Screenplay\Screenplay.csproj" />
+        <Project Path="Source\DotNET\Screenplay.EndToEnd\Screenplay.EndToEnd.csproj" />
         <Project Path="Source\DotNET\Screenplay.Specs\Screenplay.Specs.csproj" />
     </Folder>
     <Folder Name="/Source/DotNET/Tools/">

Directory.Packages.props

@@ -67,6 +67,12 @@
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="5.6.0" />
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Analyzer.Testing.XUnit" Version="1.1.2" />
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.CodeFix.Testing.XUnit" Version="1.1.2" />
+    <!-- Reading a real project into a compilation, for the end to end check on Screenplay generation.
+         Microsoft.Build.Framework is declared above and is referenced with ExcludeAssets="runtime", so that MSBuild
+         itself always comes from the SDK that MSBuildLocator registers rather than from an output folder. -->
+    <PackageVersion Include="Microsoft.CodeAnalysis.Workspaces.MSBuild" Version="5.6.0" />
+    <PackageVersion Include="Microsoft.Build.Locator" Version="1.11.2" />
+    <PackageVersion Include="Microsoft.NET.StringTools" Version="18.8.2" />
     <!-- Override vulnerable/incompatible transitive dependencies from analyzer testing packages -->
     <PackageVersion Include="Microsoft.CodeAnalysis.Workspaces.Common" Version="5.6.0" />
     <PackageVersion Include="System.Formats.Asn1" Version="10.0.10" />

Documentation/generating-a-screenplay.md

@@ -35,7 +35,7 @@ The generator is used through the Cratis CLI, which ships separately from Arc:
 cratis screenplay generate ./MyApp/MyApp.csproj --file MyApp.play
 ```
 
-Point it at a project or a solution. Anything the generator could not express is reported on the way out, and a run that reports an error exits non-zero — so a broken build never quietly produces a document that looks fine.
+Point it at a project or a solution. Anything the generator could not express is reported on the way out, and a run that reports an error exits non-zero — so a document nothing trustworthy went into never quietly looks fine. What the CLI has to do before it hands the source over is covered in [what the generator expects of its host](#what-the-generator-expects-of-its-host).
 
 Because output is reproducible, a CI step can regenerate and fail when the committed `.play` no longer matches the source.
 
@@ -48,7 +48,49 @@ Warning SP0019: The query 'Raw' returns 'IActionResult', which says how the resu
 transported rather than what it is, so the query was left out (Library.Messaging.Feed)
 ```
 
-Diagnostics come in three severities. **Information** means something is worth knowing but the document is complete. **Warning** means something was left out. **Error** means the document should not be trusted at all — the most common cause being source that did not compile.
+Diagnostics come in three severities. **Information** means something is worth knowing but the document is complete. **Warning** means something was left out. **Error** means the document should not be trusted at all — either because the generator produced something the language rejects, or because nothing at all was recovered from source the compiler accepted.
+
+## What the generator expects of its host
+
+The generator takes a Roslyn `Compilation` and nothing else. It never opens a project file or loads a workspace, which is what lets it be driven from a CLI, from a specification, or from an editor. The other side of that bargain is that **assembling the compilation the way a real build would is the caller's job** — the generator reads what it is handed and cannot tell a missing type from a type that was never written.
+
+The part hosts get wrong is source generators. `MSBuildWorkspace.GetCompilationAsync()` does not run them, and neither does any loading mode that stops at the compile items the project file lists. An Arc application leans on generation heavily — `[LoggerMessage]` partial classes, strongly-typed resource designer classes, the proxy and metrics generators — so a compilation loaded that way is missing every type those emit, and every reference to one becomes an unresolved-symbol error. This is the common case rather than an edge case.
+
+A host should run the project's generators before handing the compilation over:
+
+```csharp
+var driver = CSharpGeneratorDriver.Create(
+    generators: project.AnalyzerReferences
+        .SelectMany(reference => reference.GetGenerators(LanguageNames.CSharp))
+        .Select(GeneratorExtensions.AsSourceGenerator),
+    parseOptions: (CSharpParseOptions)project.ParseOptions!);
+
+driver.RunGeneratorsAndUpdateCompilation(compilation, out var generated, out _);
+```
+
+Then generate from `generated` rather than from `compilation`.
+
+### What happens when it does not
+
+Nothing is hidden, and nothing correct is thrown away. A compilation carrying errors is still analyzed, and `SP0024` says what happened — how many errors there were, the first of them, and how many artifacts came through anyway:
+
+```text
+Warning SP0024: The source did not compile - 607 error(s), the first being 'The name
+'AccountsMessages' does not exist in the current context'. 341 artifact(s) were recovered
+anyway, 341 of them from a declaration no error sits inside, so the document describes those
+exactly as the source states them - a missing type named like 'SomethingMessages' or a
+designer class usually means the compilation was handed over without the compile items a
+build generates (Accounts)
+```
+
+Its **severity is decided rather than fixed**, because "the source did not compile" covers two outcomes that could not be further apart:
+
+- **Warning** when at least one artifact was recovered from a declaration no compilation error sits inside. Those artifacts are described exactly as their source states them whatever failed elsewhere, so the run is successful and the document is worth keeping. A compilation missing its generated symbols lands here — the errors sit in code that declares no artifact, and the model is unaffected.
+- **Error** when none were, either because nothing was recovered at all or because every declaration something came out of is one the compiler could not make sense of. There is then no part of the document a reader could trust, and a host following the contract exits non-zero.
+
+A count is used rather than a proportion deliberately: any threshold would make the same recovery pass for a large application and fail for a small one, and zero is the only number that means recovery was *prevented* rather than merely dented.
+
+Either way the document is written out, so what was recovered can be read.
 
 ## The generator checks its own output
 
@@ -65,7 +107,9 @@ and the document is returned as it stands so the line can be read (Library)
 
 The document is still written out, so you can open it at the reported line and see what happened. If you hit this, it is a bug worth [reporting](https://github.com/Cratis/Arc/issues) — include the line, and the C# declaration it came from.
 
-Source that did not compile (`SP0024`) suppresses `SP0034`. A model recovered from symbols the compiler never accepted describes an application that does not exist, so a poor document made from it is a consequence of the broken build rather than a second, separate defect. Fix the build and generate again.
+Source that did not compile (`SP0024`) suppresses `SP0034` **when it is reported as an error** — a model recovered from symbols the compiler never accepted describes an application that does not exist, so a poor document made from it is a consequence of the broken build rather than a second, separate defect. Fix the build and generate again.
+
+As a warning it suppresses nothing. That severity says the model stands, and a document built from a model that stands is exactly what the check exists for — suppressing it there would hand back a `.play` the language rejects with nothing wrong reported.
 
 ## Screens
 

Source/DotNET/Screenplay.EndToEnd/Program.cs

@@ -0,0 +1,80 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Cratis.Arc.Screenplay;
+using Cratis.Arc.Screenplay.EndToEnd;
+using Cratis.Screenplay;
+
+if (args.Length < 2)
+{
+    Console.WriteLine("Usage:");
+    Console.WriteLine("  Cratis.Arc.Screenplay.EndToEnd <project> <output-file>");
+
+    return 2;
+}
+
+var project = Path.GetFullPath(args[0]);
+var output = Path.GetFullPath(args[1]);
+
+Console.WriteLine($"Generating the Screenplay document of '{project}'");
+
+var failures = new List<string>();
+var compilation = await ProjectCompilation.Of(project, failures);
+
+foreach (var failure in failures)
+{
+    Console.WriteLine($"  workspace: {failure}");
+}
+
+if (compilation is null)
+{
+    Console.WriteLine($"'{project}' yielded no compilation, so there is nothing to generate from");
+
+    return 1;
+}
+
+var generated = new ScreenplayGenerator().Generate(compilation, new ScreenplayOptions());
+
+Directory.CreateDirectory(Path.GetDirectoryName(output)!);
+await File.WriteAllTextAsync(output, generated.Source);
+
+Console.WriteLine($"Wrote {generated.Source.Split('\n').Length} lines to '{output}'");
+
+foreach (var group in generated.Diagnostics.GroupBy(_ => _.Code).OrderBy(_ => _.Key, StringComparer.Ordinal))
+{
+    Console.WriteLine($"  {group.Key} x{group.Count()}");
+}
+
+var errors = generated.Diagnostics.Where(_ => _.Severity == ScreenplayDiagnosticSeverity.Error).ToList();
+foreach (var error in errors)
+{
+    Console.WriteLine($"  generation error {error.Code}: {error.Message}");
+}
+
+var compiled = new ScreenplayCompiler().Compile(generated.Source);
+var rejected = compiled.Diagnostics.ToList();
+foreach (var diagnostic in rejected)
+{
+    Console.WriteLine($"  document {diagnostic.Severity} on line {diagnostic.Location.Line}: {diagnostic.Message}");
+}
+
+// The document has to compile clean, warnings included. A warning the language reports is the generator writing a
+// document that refers to something it never introduces, which is a defect here rather than in the application - and
+// it is precisely the class of defect no specification built from source strings can reach.
+if (rejected.Count > 0)
+{
+    Console.WriteLine($"The generated document did not read back clean - {rejected.Count} diagnostic(s)");
+
+    return 1;
+}
+
+if (errors.Count > 0)
+{
+    Console.WriteLine($"Generation reported {errors.Count} error(s)");
+
+    return 1;
+}
+
+Console.WriteLine("The generated document reads back clean");
+
+return 0;

Source/DotNET/Screenplay.EndToEnd/ProjectCompilation.cs

@@ -0,0 +1,61 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Runtime.CompilerServices;
+using Microsoft.Build.Locator;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.MSBuild;
+
+namespace Cratis.Arc.Screenplay.EndToEnd;
+
+/// <summary>
+/// Reads a real project file into the compilation the generator is handed.
+/// </summary>
+/// <remarks>
+/// The generator itself never loads a workspace - it takes a <see cref="Compilation"/> and nothing else - which is
+/// what lets every specification build one from source strings. That seam is exactly what this check exists to get
+/// behind: a compilation built from strings has no intermediate folder, no source generator output on disk and no
+/// referenced package that declares an event, so a defect that only shows in one of those is invisible to a
+/// specification by construction. Two shipped that way.
+/// </remarks>
+public static class ProjectCompilation
+{
+    /// <summary>
+    /// Loads a project and everything it references.
+    /// </summary>
+    /// <param name="path">The full path of the project file.</param>
+    /// <param name="failures">Everything the workspace reported while loading, in the order it reported them.</param>
+    /// <returns>The <see cref="Compilation"/>, or <see langword="null"/> when the project yielded none.</returns>
+    /// <remarks>
+    /// Registering the SDK has to happen before any MSBuild type is touched, which is why the member touching one is
+    /// marked as not inlinable - the JIT would otherwise resolve those types while registration is still running.
+    /// </remarks>
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static async Task<Compilation?> Of(string path, ICollection<string> failures)
+    {
+        if (!MSBuildLocator.IsRegistered)
+        {
+            MSBuildLocator.RegisterDefaults();
+        }
+
+        return await Load(path, failures);
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    static async Task<Compilation?> Load(string path, ICollection<string> failures)
+    {
+        var reported = new Lock();
+        using var workspace = MSBuildWorkspace.Create();
+        using var subscription = workspace.RegisterWorkspaceFailedHandler(args =>
+        {
+            lock (reported)
+            {
+                failures.Add(args.Diagnostic.Message);
+            }
+        });
+
+        var project = await workspace.OpenProjectAsync(path);
+
+        return await project.GetCompilationAsync();
+    }
+}

Source/DotNET/Screenplay.EndToEnd/Screenplay.EndToEnd.csproj

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<Project Sdk="Microsoft.NET.Sdk">
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <AssemblyName>Cratis.Arc.Screenplay.EndToEnd</AssemblyName>
+        <RootNamespace>Cratis.Arc.Screenplay.EndToEnd</RootNamespace>
+        <!-- Set as the plural so that the multi targeting the repository defaults to in Release is really replaced
+             rather than only overridden for the inner build, which would restore this for frameworks the workspace
+             packages do not support. This runs on one framework because it is a check rather than something shipped. -->
+        <TargetFrameworks>net10.0</TargetFrameworks>
+        <IsPackable>false</IsPackable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="../Screenplay/Screenplay.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <!-- Reading a real project rather than source strings is the whole point of this check. -->
+        <PackageReference Include="Microsoft.Build.Locator" />
+        <PackageReference Include="Microsoft.CodeAnalysis.Workspaces.MSBuild" />
+        <!-- Without the C# language services a loaded project silently produces no compilation at all. -->
+        <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" />
+        <!-- MSBuild itself has to come from the SDK that MSBuildLocator registers, never from the output folder. -->
+        <PackageReference Include="Microsoft.Build.Framework" ExcludeAssets="runtime" PrivateAssets="all" />
+        <PackageReference Include="Microsoft.NET.StringTools" ExcludeAssets="runtime" PrivateAssets="all" />
+    </ItemGroup>
+</Project>