Import external events, ignore build output in slices, and read separators in names

.github/workflows/dotnet-build.yml

@@ -176,6 +176,57 @@ jobs:
       - name: Test (${{ matrix.project }})
         run: dotnet test ${{ matrix.project }} --no-build --configuration Release --framework net10.0
 
+  # Every Screenplay specification builds its compilation from source strings, which is what makes them hermetic -
+  # and what puts a whole class of defect out of their reach by construction. A compilation built that way has no
+  # intermediate folder, no source generator output on disk and no referenced package declaring an event, so a
+  # generator that mis-attributes one of those produces a wrong document that every specification still passes. Two
+  # shipped that way. This reads a real project through MSBuild and holds the document it generates to the compiler
+  # the language ships - it has to come back with nothing to say, warnings included.
+  screenplay-end-to-end:
+    needs: [dotnet-build]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
+
+      - name: Setup .Net
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: |
+            ${{ env.DOTNET8_VERSION }}
+            ${{ env.DOTNET9_VERSION }}
+            ${{ env.DOTNET10_VERSION }}
+
+      - name: Cache NuGet packages
+        uses: actions/cache@v6
+        with:
+          path: ~/.nuget/packages
+          key: nuget-${{ runner.os }}-${{ hashFiles('**/*.csproj', '**/Directory.*.props', '**/Directory.*.targets') }}
+          restore-keys: |
+            nuget-${{ runner.os }}-
+
+      - name: Download build output
+        uses: actions/download-artifact@v4
+        with:
+          name: build-output
+
+      - name: Extract build output
+        run: tar -xzf build-output.tar.gz
+
+      # The build output artifact carries bin folders only, and reading a project through MSBuild needs the assets
+      # file that restore writes to obj.
+      - name: Restore the application to generate from
+        run: dotnet restore TestApps/AspNetCore/AspNetCore.csproj
+
+      - name: Generate and read back the document of a real application
+        run: |
+          dotnet Source/DotNET/Screenplay.EndToEnd/bin/Release/net10.0/Cratis.Arc.Screenplay.EndToEnd.dll \
+            TestApps/AspNetCore/AspNetCore.csproj \
+            "${RUNNER_TEMP}/AspNetCore.play"
+
   proxy-specs:
     needs: [dotnet-build]
     runs-on: ubuntu-latest

Arc.slnx

@@ -31,6 +31,7 @@
         <Project Path="Source\DotNET\OpenApi\OpenApi.csproj" />
         <Project Path="Source\DotNET\OpenApi.Specs\OpenApi.Specs.csproj" />
         <Project Path="Source\DotNET\Screenplay\Screenplay.csproj" />
+        <Project Path="Source\DotNET\Screenplay.EndToEnd\Screenplay.EndToEnd.csproj" />
         <Project Path="Source\DotNET\Screenplay.Specs\Screenplay.Specs.csproj" />
     </Folder>
     <Folder Name="/Source/DotNET/Tools/">

Directory.Packages.props

@@ -67,6 +67,12 @@
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="5.6.0" />
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Analyzer.Testing.XUnit" Version="1.1.2" />
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.CodeFix.Testing.XUnit" Version="1.1.2" />
+    <!-- Reading a real project into a compilation, for the end to end check on Screenplay generation.
+         Microsoft.Build.Framework is declared above and is referenced with ExcludeAssets="runtime", so that MSBuild
+         itself always comes from the SDK that MSBuildLocator registers rather than from an output folder. -->
+    <PackageVersion Include="Microsoft.CodeAnalysis.Workspaces.MSBuild" Version="5.6.0" />
+    <PackageVersion Include="Microsoft.Build.Locator" Version="1.11.2" />
+    <PackageVersion Include="Microsoft.NET.StringTools" Version="18.8.2" />
     <!-- Override vulnerable/incompatible transitive dependencies from analyzer testing packages -->
     <PackageVersion Include="Microsoft.CodeAnalysis.Workspaces.Common" Version="5.6.0" />
     <PackageVersion Include="System.Formats.Asn1" Version="10.0.10" />

Source/DotNET/Screenplay.EndToEnd/Program.cs

@@ -0,0 +1,80 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Cratis.Arc.Screenplay;
+using Cratis.Arc.Screenplay.EndToEnd;
+using Cratis.Screenplay;
+
+if (args.Length < 2)
+{
+    Console.WriteLine("Usage:");
+    Console.WriteLine("  Cratis.Arc.Screenplay.EndToEnd <project> <output-file>");
+
+    return 2;
+}
+
+var project = Path.GetFullPath(args[0]);
+var output = Path.GetFullPath(args[1]);
+
+Console.WriteLine($"Generating the Screenplay document of '{project}'");
+
+var failures = new List<string>();
+var compilation = await ProjectCompilation.Of(project, failures);
+
+foreach (var failure in failures)
+{
+    Console.WriteLine($"  workspace: {failure}");
+}
+
+if (compilation is null)
+{
+    Console.WriteLine($"'{project}' yielded no compilation, so there is nothing to generate from");
+
+    return 1;
+}
+
+var generated = new ScreenplayGenerator().Generate(compilation, new ScreenplayOptions());
+
+Directory.CreateDirectory(Path.GetDirectoryName(output)!);
+await File.WriteAllTextAsync(output, generated.Source);
+
+Console.WriteLine($"Wrote {generated.Source.Split('\n').Length} lines to '{output}'");
+
+foreach (var group in generated.Diagnostics.GroupBy(_ => _.Code).OrderBy(_ => _.Key, StringComparer.Ordinal))
+{
+    Console.WriteLine($"  {group.Key} x{group.Count()}");
+}
+
+var errors = generated.Diagnostics.Where(_ => _.Severity == ScreenplayDiagnosticSeverity.Error).ToList();
+foreach (var error in errors)
+{
+    Console.WriteLine($"  generation error {error.Code}: {error.Message}");
+}
+
+var compiled = new ScreenplayCompiler().Compile(generated.Source);
+var rejected = compiled.Diagnostics.ToList();
+foreach (var diagnostic in rejected)
+{
+    Console.WriteLine($"  document {diagnostic.Severity} on line {diagnostic.Location.Line}: {diagnostic.Message}");
+}
+
+// The document has to compile clean, warnings included. A warning the language reports is the generator writing a
+// document that refers to something it never introduces, which is a defect here rather than in the application - and
+// it is precisely the class of defect no specification built from source strings can reach.
+if (rejected.Count > 0)
+{
+    Console.WriteLine($"The generated document did not read back clean - {rejected.Count} diagnostic(s)");
+
+    return 1;
+}
+
+if (errors.Count > 0)
+{
+    Console.WriteLine($"Generation reported {errors.Count} error(s)");
+
+    return 1;
+}
+
+Console.WriteLine("The generated document reads back clean");
+
+return 0;

Source/DotNET/Screenplay.EndToEnd/ProjectCompilation.cs

@@ -0,0 +1,61 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Runtime.CompilerServices;
+using Microsoft.Build.Locator;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.MSBuild;
+
+namespace Cratis.Arc.Screenplay.EndToEnd;
+
+/// <summary>
+/// Reads a real project file into the compilation the generator is handed.
+/// </summary>
+/// <remarks>
+/// The generator itself never loads a workspace - it takes a <see cref="Compilation"/> and nothing else - which is
+/// what lets every specification build one from source strings. That seam is exactly what this check exists to get
+/// behind: a compilation built from strings has no intermediate folder, no source generator output on disk and no
+/// referenced package that declares an event, so a defect that only shows in one of those is invisible to a
+/// specification by construction. Two shipped that way.
+/// </remarks>
+public static class ProjectCompilation
+{
+    /// <summary>
+    /// Loads a project and everything it references.
+    /// </summary>
+    /// <param name="path">The full path of the project file.</param>
+    /// <param name="failures">Everything the workspace reported while loading, in the order it reported them.</param>
+    /// <returns>The <see cref="Compilation"/>, or <see langword="null"/> when the project yielded none.</returns>
+    /// <remarks>
+    /// Registering the SDK has to happen before any MSBuild type is touched, which is why the member touching one is
+    /// marked as not inlinable - the JIT would otherwise resolve those types while registration is still running.
+    /// </remarks>
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static async Task<Compilation?> Of(string path, ICollection<string> failures)
+    {
+        if (!MSBuildLocator.IsRegistered)
+        {
+            MSBuildLocator.RegisterDefaults();
+        }
+
+        return await Load(path, failures);
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    static async Task<Compilation?> Load(string path, ICollection<string> failures)
+    {
+        var reported = new Lock();
+        using var workspace = MSBuildWorkspace.Create();
+        using var subscription = workspace.RegisterWorkspaceFailedHandler(args =>
+        {
+            lock (reported)
+            {
+                failures.Add(args.Diagnostic.Message);
+            }
+        });
+
+        var project = await workspace.OpenProjectAsync(path);
+
+        return await project.GetCompilationAsync();
+    }
+}

Source/DotNET/Screenplay.EndToEnd/Screenplay.EndToEnd.csproj

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<Project Sdk="Microsoft.NET.Sdk">
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <AssemblyName>Cratis.Arc.Screenplay.EndToEnd</AssemblyName>
+        <RootNamespace>Cratis.Arc.Screenplay.EndToEnd</RootNamespace>
+        <!-- Set as the plural so that the multi targeting the repository defaults to in Release is really replaced
+             rather than only overridden for the inner build, which would restore this for frameworks the workspace
+             packages do not support. This runs on one framework because it is a check rather than something shipped. -->
+        <TargetFrameworks>net10.0</TargetFrameworks>
+        <IsPackable>false</IsPackable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="../Screenplay/Screenplay.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <!-- Reading a real project rather than source strings is the whole point of this check. -->
+        <PackageReference Include="Microsoft.Build.Locator" />
+        <PackageReference Include="Microsoft.CodeAnalysis.Workspaces.MSBuild" />
+        <!-- Without the C# language services a loaded project silently produces no compilation at all. -->
+        <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" />
+        <!-- MSBuild itself has to come from the SDK that MSBuildLocator registers, never from the output folder. -->
+        <PackageReference Include="Microsoft.Build.Framework" ExcludeAssets="runtime" PrivateAssets="all" />
+        <PackageReference Include="Microsoft.NET.StringTools" ExcludeAssets="runtime" PrivateAssets="all" />
+    </ItemGroup>
+</Project>

Source/DotNET/Screenplay.Specs/Analyzed.cs

@@ -54,6 +54,46 @@ public static class Analyzed
     public static ApplicationModelAnalysis Source(params (string Path, string Text)[] sources) =>
         Source(DeclaredUserInterfaceFiles.None, sources);
 
+    /// <summary>
+    /// Compiles source referencing a package and recovers the model it describes.
+    /// </summary>
+    /// <param name="package">The package the source references.</param>
+    /// <param name="sources">The source files, keyed by the path each one is compiled as.</param>
+    /// <returns>The <see cref="ApplicationModelAnalysis"/>.</returns>
+    public static ApplicationModelAnalysis SourceReferencing(MetadataReference package, params (string Path, string Text)[] sources) =>
+        new ApplicationModelAnalyzer(DeclaredUserInterfaceFiles.None)
+            .Analyze(Compile([package], sources), new ScreenplayOptions().WithDefaults(AssemblyName));
+
+    /// <summary>
+    /// Compiles source into the assembly image a referenced package really is.
+    /// </summary>
+    /// <param name="name">The name of the assembly.</param>
+    /// <param name="text">The source.</param>
+    /// <returns>The <see cref="MetadataReference"/>.</returns>
+    /// <exception cref="InvalidOperationException">Thrown when the source of the package does not compile.</exception>
+    /// <remarks>
+    /// The image is emitted rather than referenced as a compilation, because a package the application depends on is
+    /// metadata with no syntax tree behind it - which is the whole reason nothing in the compilation declares what it
+    /// holds.
+    /// </remarks>
+    public static MetadataReference Package(string name, string text)
+    {
+        var compilation = CSharpCompilation.Create(
+            name,
+            [CSharpSyntaxTree.ParseText(text, new CSharpParseOptions(documentationMode: DocumentationMode.Parse))],
+            _references,
+            new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, nullableContextOptions: NullableContextOptions.Enable));
+
+        using var stream = new MemoryStream();
+        var emitted = compilation.Emit(stream);
+        if (!emitted.Success)
+        {
+            throw new InvalidOperationException($"The package '{name}' did not compile - {emitted.Diagnostics.First(_ => _.Severity == DiagnosticSeverity.Error)}");
+        }
+
+        return MetadataReference.CreateFromImage(stream.ToArray());
+    }
+
     /// <summary>
     /// Compiles source and recovers the model it describes.
     /// </summary>
@@ -87,23 +127,48 @@ public static ApplicationModelAnalysis Source(IUserInterfaceFiles files, params
     /// </summary>
     /// <param name="sources">The source files, keyed by the path each one is compiled as.</param>
     /// <returns>The <see cref="Compilation"/>.</returns>
-    public static Compilation Compile(params (string Path, string Text)[] sources) =>
+    public static Compilation Compile(params (string Path, string Text)[] sources) => Compile([], sources);
+
+    /// <summary>
+    /// Compiles source referencing further packages into a compilation.
+    /// </summary>
+    /// <param name="packages">The packages the source references beyond the platform.</param>
+    /// <param name="sources">The source files, keyed by the path each one is compiled as.</param>
+    /// <returns>The <see cref="Compilation"/>.</returns>
+    public static Compilation Compile(IEnumerable<MetadataReference> packages, params (string Path, string Text)[] sources) =>
         CSharpCompilation.Create(
             AssemblyName,
             sources.Append(Root).Select(_ => CSharpSyntaxTree.ParseText(
                 _.Text,
                 new CSharpParseOptions(documentationMode: DocumentationMode.Parse),
                 path: _.Path)),
-            _references,
+            _references.Concat(packages),
             new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, nullableContextOptions: NullableContextOptions.Enable));
 
     /// <summary>
     /// Gets everything the compiler itself reported, so that a specification never asserts against broken source.
     /// </summary>
     /// <param name="sources">The source files, keyed by the path each one is compiled as.</param>
     /// <returns>The errors, empty when the source compiles.</returns>
-    public static IEnumerable<string> ErrorsIn(params (string Path, string Text)[] sources) =>
-        Compile(sources)
+    public static IEnumerable<string> ErrorsIn(params (string Path, string Text)[] sources) => ErrorsIn([], sources);
+
+    /// <summary>
+    /// Gets everything the compiler itself reported for source referencing a package.
+    /// </summary>
+    /// <param name="package">The package the source references.</param>
+    /// <param name="sources">The source files, keyed by the path each one is compiled as.</param>
+    /// <returns>The errors, empty when the source compiles.</returns>
+    public static IEnumerable<string> ErrorsIn(MetadataReference package, params (string Path, string Text)[] sources) =>
+        ErrorsIn([package], sources);
+
+    /// <summary>
+    /// Gets everything the compiler itself reported for source referencing further packages.
+    /// </summary>
+    /// <param name="packages">The packages the source references beyond the platform.</param>
+    /// <param name="sources">The source files, keyed by the path each one is compiled as.</param>
+    /// <returns>The errors, empty when the source compiles.</returns>
+    public static IEnumerable<string> ErrorsIn(IEnumerable<MetadataReference> packages, params (string Path, string Text)[] sources) =>
+        Compile(packages, sources)
             .GetDiagnostics()
             .Where(_ => _.Severity == DiagnosticSeverity.Error)
             .Select(_ => _.ToString());