Recover more of an application, describe it across projects, and report what cannot be recovered

.github/workflows/dotnet-build.yml

@@ -176,6 +176,57 @@ jobs:
       - name: Test (${{ matrix.project }})
         run: dotnet test ${{ matrix.project }} --no-build --configuration Release --framework net10.0
 
+  # Every Screenplay specification builds its compilation from source strings, which is what makes them hermetic -
+  # and what puts a whole class of defect out of their reach by construction. A compilation built that way has no
+  # intermediate folder, no source generator output on disk and no referenced package declaring an event, so a
+  # generator that mis-attributes one of those produces a wrong document that every specification still passes. Two
+  # shipped that way. This reads a real project through MSBuild and holds the document it generates to the compiler
+  # the language ships - it has to come back with nothing to say, warnings included.
+  screenplay-end-to-end:
+    needs: [dotnet-build]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
+
+      - name: Setup .Net
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: |
+            ${{ env.DOTNET8_VERSION }}
+            ${{ env.DOTNET9_VERSION }}
+            ${{ env.DOTNET10_VERSION }}
+
+      - name: Cache NuGet packages
+        uses: actions/cache@v6
+        with:
+          path: ~/.nuget/packages
+          key: nuget-${{ runner.os }}-${{ hashFiles('**/*.csproj', '**/Directory.*.props', '**/Directory.*.targets') }}
+          restore-keys: |
+            nuget-${{ runner.os }}-
+
+      - name: Download build output
+        uses: actions/download-artifact@v4
+        with:
+          name: build-output
+
+      - name: Extract build output
+        run: tar -xzf build-output.tar.gz
+
+      # The build output artifact carries bin folders only, and reading a project through MSBuild needs the assets
+      # file that restore writes to obj.
+      - name: Restore the application to generate from
+        run: dotnet restore TestApps/AspNetCore/AspNetCore.csproj
+
+      - name: Generate and read back the document of a real application
+        run: |
+          dotnet Source/DotNET/Screenplay.EndToEnd/bin/Release/net10.0/Cratis.Arc.Screenplay.EndToEnd.dll \
+            TestApps/AspNetCore/AspNetCore.csproj \
+            "${RUNNER_TEMP}/AspNetCore.play"
+
   proxy-specs:
     needs: [dotnet-build]
     runs-on: ubuntu-latest

Arc.slnx

@@ -31,6 +31,7 @@
         <Project Path="Source\DotNET\OpenApi\OpenApi.csproj" />
         <Project Path="Source\DotNET\OpenApi.Specs\OpenApi.Specs.csproj" />
         <Project Path="Source\DotNET\Screenplay\Screenplay.csproj" />
+        <Project Path="Source\DotNET\Screenplay.EndToEnd\Screenplay.EndToEnd.csproj" />
         <Project Path="Source\DotNET\Screenplay.Specs\Screenplay.Specs.csproj" />
     </Folder>
     <Folder Name="/Source/DotNET/Tools/">

Directory.Packages.props

@@ -67,6 +67,12 @@
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="5.6.0" />
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Analyzer.Testing.XUnit" Version="1.1.2" />
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.CodeFix.Testing.XUnit" Version="1.1.2" />
+    <!-- Reading a real project into a compilation, for the end to end check on Screenplay generation.
+         Microsoft.Build.Framework is declared above and is referenced with ExcludeAssets="runtime", so that MSBuild
+         itself always comes from the SDK that MSBuildLocator registers rather than from an output folder. -->
+    <PackageVersion Include="Microsoft.CodeAnalysis.Workspaces.MSBuild" Version="5.6.0" />
+    <PackageVersion Include="Microsoft.Build.Locator" Version="1.11.2" />
+    <PackageVersion Include="Microsoft.NET.StringTools" Version="18.8.2" />
     <!-- Override vulnerable/incompatible transitive dependencies from analyzer testing packages -->
     <PackageVersion Include="Microsoft.CodeAnalysis.Workspaces.Common" Version="5.6.0" />
     <PackageVersion Include="System.Formats.Asn1" Version="10.0.10" />

Source/DotNET/Screenplay.EndToEnd/Program.cs

@@ -0,0 +1,85 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Cratis.Arc.Screenplay;
+using Cratis.Arc.Screenplay.EndToEnd;
+using Cratis.Screenplay;
+
+if (args.Length < 2)
+{
+    Console.WriteLine("Usage:");
+    Console.WriteLine("  Cratis.Arc.Screenplay.EndToEnd <project-or-solution> <output-file>");
+
+    return 2;
+}
+
+var project = Path.GetFullPath(args[0]);
+var output = Path.GetFullPath(args[1]);
+
+Console.WriteLine($"Generating the Screenplay document of '{project}'");
+
+var failures = new List<string>();
+var compilations = await ProjectCompilation.Of(project, failures);
+
+foreach (var failure in failures)
+{
+    Console.WriteLine($"  workspace: {failure}");
+}
+
+if (compilations.Count == 0)
+{
+    Console.WriteLine($"'{project}' yielded no compilation, so there is nothing to generate from");
+
+    return 1;
+}
+
+foreach (var loaded in compilations)
+{
+    Console.WriteLine($"  project: {loaded.AssemblyName}");
+}
+
+var generated = new ScreenplayGenerator().Generate(compilations, new ScreenplayOptions());
+
+Directory.CreateDirectory(Path.GetDirectoryName(output)!);
+await File.WriteAllTextAsync(output, generated.Source);
+
+Console.WriteLine($"Wrote {generated.Source.Split('\n').Length} lines to '{output}'");
+
+foreach (var group in generated.Diagnostics.GroupBy(_ => _.Code).OrderBy(_ => _.Key, StringComparer.Ordinal))
+{
+    Console.WriteLine($"  {group.Key} x{group.Count()}");
+}
+
+var errors = generated.Diagnostics.Where(_ => _.Severity == ScreenplayDiagnosticSeverity.Error).ToList();
+foreach (var error in errors)
+{
+    Console.WriteLine($"  generation error {error.Code}: {error.Message}");
+}
+
+var compiled = new ScreenplayCompiler().Compile(generated.Source);
+var rejected = compiled.Diagnostics.ToList();
+foreach (var diagnostic in rejected)
+{
+    Console.WriteLine($"  document {diagnostic.Severity} on line {diagnostic.Location.Line}: {diagnostic.Message}");
+}
+
+// The document has to compile clean, warnings included. A warning the language reports is the generator writing a
+// document that refers to something it never introduces, which is a defect here rather than in the application - and
+// it is precisely the class of defect no specification built from source strings can reach.
+if (rejected.Count > 0)
+{
+    Console.WriteLine($"The generated document did not read back clean - {rejected.Count} diagnostic(s)");
+
+    return 1;
+}
+
+if (errors.Count > 0)
+{
+    Console.WriteLine($"Generation reported {errors.Count} error(s)");
+
+    return 1;
+}
+
+Console.WriteLine("The generated document reads back clean");
+
+return 0;

Source/DotNET/Screenplay.EndToEnd/ProjectCompilation.cs

@@ -0,0 +1,130 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Runtime.CompilerServices;
+using Microsoft.Build.Locator;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.MSBuild;
+
+namespace Cratis.Arc.Screenplay.EndToEnd;
+
+/// <summary>
+/// Reads a real project, or a real solution, into the compilations the generator is handed.
+/// </summary>
+/// <remarks>
+/// The generator itself never loads a workspace - it takes compilations and nothing else - which is what lets every
+/// specification build them from source strings. That seam is exactly what this check exists to get behind: a
+/// compilation built from strings has no intermediate folder, no source generator output on disk and no referenced
+/// package that declares an event, so a defect that only shows in one of those is invisible to a specification by
+/// construction. Two shipped that way.
+/// <para>
+/// A solution is read as every project it holds that is not a specification project, which is what an application
+/// written as several projects really is. Whether a project is one is decided by its name, the convention a solution
+/// is laid out by.
+/// </para>
+/// </remarks>
+public static class ProjectCompilation
+{
+    static readonly string[] _specifications = ["Specs", "Tests", "Specs.AppHost"];
+    static readonly string[] _solutions = [".sln", ".slnx", ".slnf"];
+
+    /// <summary>
+    /// Loads a project or a solution and everything it references.
+    /// </summary>
+    /// <param name="path">The full path of the project or solution file.</param>
+    /// <param name="failures">Everything the workspace reported while loading, in the order it reported them.</param>
+    /// <returns>The compilations, empty when nothing yielded one.</returns>
+    /// <remarks>
+    /// Registering the SDK has to happen before any MSBuild type is touched, which is why the members touching one
+    /// are marked as not inlinable - the JIT would otherwise resolve those types while registration is still running.
+    /// </remarks>
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static async Task<IReadOnlyList<Compilation>> Of(string path, ICollection<string> failures)
+    {
+        if (!MSBuildLocator.IsRegistered)
+        {
+            MSBuildLocator.RegisterDefaults();
+        }
+
+        return IsSolution(path) ? await LoadSolution(path, failures) : await LoadProject(path, failures);
+    }
+
+    /// <summary>
+    /// Determines whether a path names a solution rather than a project.
+    /// </summary>
+    /// <param name="path">The path to check.</param>
+    /// <returns>True when it names a solution.</returns>
+    static bool IsSolution(string path) =>
+        Array.Exists(_solutions, extension => string.Equals(Path.GetExtension(path), extension, StringComparison.OrdinalIgnoreCase));
+
+    /// <summary>
+    /// Loads a single project.
+    /// </summary>
+    /// <param name="path">The full path of the project file.</param>
+    /// <param name="failures">Everything the workspace reported while loading.</param>
+    /// <returns>The compilation, empty when the project yielded none.</returns>
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    static async Task<IReadOnlyList<Compilation>> LoadProject(string path, ICollection<string> failures)
+    {
+        var reported = new Lock();
+        using var workspace = MSBuildWorkspace.Create();
+        using var subscription = workspace.RegisterWorkspaceFailedHandler(args => Report(args, reported, failures));
+
+        var project = await workspace.OpenProjectAsync(path);
+
+        return await project.GetCompilationAsync() is { } compilation ? [compilation] : [];
+    }
+
+    /// <summary>
+    /// Loads every project of a solution that holds part of the application.
+    /// </summary>
+    /// <param name="path">The full path of the solution file.</param>
+    /// <param name="failures">Everything the workspace reported while loading.</param>
+    /// <returns>The compilations, ordered by the name of the project each one came from.</returns>
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    static async Task<IReadOnlyList<Compilation>> LoadSolution(string path, ICollection<string> failures)
+    {
+        var reported = new Lock();
+        using var workspace = MSBuildWorkspace.Create();
+        using var subscription = workspace.RegisterWorkspaceFailedHandler(args => Report(args, reported, failures));
+
+        var solution = await workspace.OpenSolutionAsync(path);
+        var compilations = new List<Compilation>();
+
+        foreach (var project in solution.Projects
+            .Where(_ => !IsSpecifications(_.Name))
+            .OrderBy(_ => _.Name, StringComparer.Ordinal))
+        {
+            if (await project.GetCompilationAsync() is { } compilation)
+            {
+                compilations.Add(compilation);
+            }
+        }
+
+        return compilations;
+    }
+
+    /// <summary>
+    /// Determines whether a project holds specifications rather than part of the application.
+    /// </summary>
+    /// <param name="name">The name of the project.</param>
+    /// <returns>True when it holds specifications.</returns>
+    static bool IsSpecifications(string name) =>
+        Array.Exists(_specifications, suffix =>
+            string.Equals(name, suffix, StringComparison.Ordinal) ||
+            name.EndsWith($".{suffix}", StringComparison.Ordinal));
+
+    /// <summary>
+    /// Records what the workspace reported while loading.
+    /// </summary>
+    /// <param name="args">What was reported.</param>
+    /// <param name="reported">The lock guarding the collection.</param>
+    /// <param name="failures">The failures collected so far.</param>
+    static void Report(WorkspaceDiagnosticEventArgs args, Lock reported, ICollection<string> failures)
+    {
+        lock (reported)
+        {
+            failures.Add(args.Diagnostic.Message);
+        }
+    }
+}

Source/DotNET/Screenplay.EndToEnd/Screenplay.EndToEnd.csproj

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<Project Sdk="Microsoft.NET.Sdk">
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <AssemblyName>Cratis.Arc.Screenplay.EndToEnd</AssemblyName>
+        <RootNamespace>Cratis.Arc.Screenplay.EndToEnd</RootNamespace>
+        <!-- Set as the plural so that the multi targeting the repository defaults to in Release is really replaced
+             rather than only overridden for the inner build, which would restore this for frameworks the workspace
+             packages do not support. This runs on one framework because it is a check rather than something shipped. -->
+        <TargetFrameworks>net10.0</TargetFrameworks>
+        <IsPackable>false</IsPackable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="../Screenplay/Screenplay.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <!-- Reading a real project rather than source strings is the whole point of this check. -->
+        <PackageReference Include="Microsoft.Build.Locator" />
+        <PackageReference Include="Microsoft.CodeAnalysis.Workspaces.MSBuild" />
+        <!-- Without the C# language services a loaded project silently produces no compilation at all. -->
+        <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" />
+        <!-- MSBuild itself has to come from the SDK that MSBuildLocator registers, never from the output folder. -->
+        <PackageReference Include="Microsoft.Build.Framework" ExcludeAssets="runtime" PrivateAssets="all" />
+        <PackageReference Include="Microsoft.NET.StringTools" ExcludeAssets="runtime" PrivateAssets="all" />
+    </ItemGroup>
+</Project>