Add Aspire AppHost for local inner-loop development

.gitignore

@@ -257,3 +257,19 @@ paket-files/
 *.binlog
 
 *.lscache
+
+# Aspire inner-loop
+.azurite/
+samples/*/bin/sample/
+bin/index/
+
+
+# Aspire CLI / tooling state
+.agents/
+.playwright/
+.modules/
+
+
+# HtmlGenerator drops Errors.txt into its working directory on startup.
+src/SourceBrowser/src/HtmlGenerator/Errors.txt
+

Directory.Packages.props

@@ -1,3 +1,13 @@
 <Project>
   <Import Project="src\SourceBrowser\src\Directory.Packages.props" />
-</Project>
+  <ItemGroup>
+    <PackageVersion Include="Aspire.Hosting.Azure.Storage" Version="13.3.2" />
+    <PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="10.5.0" />
+    <PackageVersion Include="Microsoft.Extensions.ServiceDiscovery" Version="10.5.0" />
+    <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.15.3" />
+    <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.15.3" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.15.2" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.15.1" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.15.1" />
+  </ItemGroup>
+</Project>

NuGet.config

@@ -1,11 +1,20 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
 <configuration>
   <packageSources>
     <clear />
     <add key="dotnet-public" value="https://dnceng.pkgs.visualstudio.com/public/_packaging/dotnet-public/nuget/v3/index.json" />
+    <add key="https://api.nuget.org/v3/index.json" value="https://api.nuget.org/v3/index.json" />
   </packageSources>
   <disabledPackageSources>
     <clear />
   </disabledPackageSources>
-</configuration>
+  <packageSourceMapping>
+    <packageSource key="https://api.nuget.org/v3/index.json">
+      <package pattern="*" />
+    </packageSource>
+    <packageSource key="dotnet-public">
+      <package pattern="*" />
+    </packageSource>
+  </packageSourceMapping>
+</configuration>

README.md

@@ -2,6 +2,7 @@
 This repo contains the code for building http://source.dot.net
 
 ## Documentation
+- [Local inner loop (Aspire)](docs/inner-loop.md) - **Recommended for development.** Runs the full pipeline (sample build → BinLogToSln → HtmlGenerator → blob → SourceIndexServer) locally against a tiny sample library.
 - [Source Selection Algorithm](docs/source-selection-algorithm.md) - How the indexer chooses the best implementation when multiple builds exist for the same assembly
 
 ## Build Status
@@ -10,17 +11,27 @@ This repo contains the code for building http://source.dot.net
 ## What Is It?
 This repo uses https://github.com/KirillOsenkov/SourceBrowser (with a few additions here https://github.com/dotnet/SourceBrowser/tree/source-indexer) to index the dotnet sources and produce a navigatable and searchable website containing the full source code. This includes code from the runtime, winforms, wpf, aspnetcore, and msbuild, among others. For a full list see here https://github.com/dotnet/source-indexer/blob/main/src/index/repositories.props.
 
-## Build Prerequsites
-The build requires .NET 8.0 and Visual Studio 2022 to build.
+## Local development (Aspire)
+The repository ships with an [Aspire](https://aspire.dev) AppHost (`src/source-indexer.AppHost/`) that emulates the full production pipeline locally against a tiny sample library — **this is the recommended way to work on the repo**.
+
+```pwsh
+aspire start
+```
+
+The dashboard URL (with login token) is printed to the console. From there you can run the `bootstrap-all` step to build the sample, run `BinLogToSln`, run `HtmlGenerator`, upload to Azurite, and start the web app. See [docs/inner-loop.md](docs/inner-loop.md) for the full resource graph and walkthroughs.
+
+You can also open the repo in **Visual Studio** or **VS Code** to set breakpoints and debug individual components (e.g., `HtmlGenerator`, `SourceIndexServer`, `BinLogToSln`) while the rest of the pipeline runs under the AppHost.
+
+## Building the production index (Windows-only)
+The official pipeline build only works on Windows because `HtmlGenerator` is a .NET Framework executable. For local dev prefer the Aspire flow above; use this path only if you need to reproduce the exact pipeline build.
+
+**Prerequisites:** .NET 8.0 and Visual Studio 2022.
 
-## Build
-The build will only work on windows because the source indexer executable is a .net framework executable.
 1. `git clone https://github.com/dotnet/source-indexer.git`
 2. For each *.sln file `dotnet restore`
 3. Find VS 2022 msbuild.exe on your machine, typically found at `C:\Program Files (x86)\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\MSBuild.exe`
 4. `msbuild build.proj`
 
-## Running the built index
 After the build is finished the index will exist in `bin\index` and can be run by running `dotnet Microsoft.SourceBrowser.SourceIndexServer.dll` in that folder. The index will be served on `http://localhost:5000`
 
 ## Deployment

aspire.config.json

@@ -0,0 +1,5 @@
+{
+  "appHost": {
+    "language": "csharp"
+  }
+}

docs/inner-loop.md

@@ -0,0 +1,112 @@
+# Local inner loop (Aspire)
+
+`src/source-indexer.AppHost/` is an [Aspire](https://aspire.dev) AppHost that
+emulates the production indexing pipeline end-to-end against a tiny sample
+library (`samples/MiniRuntime/`). It lets you exercise the full
+binlog → HtmlGenerator → blob → SourceIndexServer chain locally without
+pushing to ADO.
+
+## Prerequisites
+
+- **.NET SDK 10** (pinned in root `global.json`).
+- **Docker / Podman** running locally (Azurite emulators run as containers).
+- **Aspire CLI** — see <https://aspire.dev/docs/getting-started>.
+- **Azure CLI** (`az`) on PATH — used by the `publish-index` step.
+
+Then from the repo root:
+
+```pwsh
+aspire start
+```
+
+The dashboard URL (with login token) is printed to the console.
+
+## Resource graph
+
+Auto-started (running as soon as the AppHost is up):
+
+| Resource | Type | Purpose |
+|---|---|---|
+| `stage1Storage` | Azurite emulator | Models `netsourceindexstage1` (V2 upstream upload destination). |
+| `prodStorage`   | Azurite emulator | Models `netsourceindexprod` (final HTML index destination). |
+| `stage1`        | Blob container under `stage1Storage` | Where `upload-stage1` drops the `.tar.gz`. |
+| `index-local`   | Blob container under `prodStorage`   | Where the generated HTML index is uploaded. The web app reads from here. |
+| `web`           | `SourceIndexServer` project          | ASP.NET Core app. `SOURCE_BROWSER_INDEX_PROXY_URL` is wired to `index-local` so it serves the indexed HTML out of Azurite. |
+
+Explicit-start (stopped by default — click **Start** in the dashboard):
+
+| Resource | Type | What it does |
+|---|---|---|
+| `step1-sample-build`    | Executable: `dotnet build /bl:` | Builds `samples/MiniRuntime` and produces `samples/MiniRuntime/bin/sample/msbuild.binlog`. |
+| `step2-upload-stage1`   | Project: `UploadIndexStage1`    | Tars+gzips the sample folder + binlog and uploads it to `stage1`. Real V2 upstream tool — fully dogfooded. |
+| `step3-htmlgenerator`   | Project: `HtmlGenerator`        | Runs `HtmlGenerator` on the binlog from `step1-sample-build` to produce static HTML under `bin/index/`. |
+| `step4-publish-index`   | Executable: `az storage blob upload-batch` | Uploads `bin/index/index/` to the `index-local` container in `prodStorage`. |
+
+## One-click bootstrap
+
+The `prodStorage` resource exposes a custom **Bootstrap full pipeline**
+command (internal id `bootstrap-all`) that runs the four pipeline
+resources in order (`step1-sample-build` → `step2-upload-stage1` →
+`step3-htmlgenerator` → `step4-publish-index`) and waits for each one to
+finish. This is the easy first-run path:
+
+1. `aspire start`
+2. Open the dashboard.
+3. On the `prodStorage` row, click the **⋯** button in the Actions
+   column and choose **Bootstrap full pipeline**.
+4. Wait for it to finish (watch the logs).
+5. Open the `web` URL — you should see `MiniRuntime`'s indexed HTML.
+
+![Bootstrap full pipeline menu on prodStorage](images/bootstrap-menu.png)
+
+Re-running any individual resource regenerates just that stage.
+
+## How this maps to prod
+
+| Prod | Local inner loop |
+|---|---|
+| V2 upstream repo publishes `.tar.gz` to `netsourceindexstage1/stage1/<repo>/<ts>.tar.gz` (`UploadIndexStage1`) | `step2-upload-stage1` resource → `stage1Storage/stage1` |
+| `HtmlGenerator.exe` reads binlog + writes HTML to `bin/index/` (`src/index/index.proj`) | `step3-htmlgenerator` resource |
+| `AzureFileCopy@6` uploads `bin/index/index/*` to `netsourceindexprod/index-<GUID>/` (`azure-pipelines.yml`) | `step4-publish-index` resource (`az storage blob upload-batch`) → `prodStorage/index-local` |
+| App Service slot setting `SOURCE_BROWSER_INDEX_PROXY_URL` flipped to the new container (`deployment/deploy-storage-proxy.ps1`) | `web` resource started with `SOURCE_BROWSER_INDEX_PROXY_URL` env var pointing at `prodStorage/index-local` |
+
+See `docs/handoff/03-indexing-pipeline.md` and
+`docs/handoff/05-azure-pipeline.md` for the full prod mechanics.
+
+## Auth: Azurite vs prod
+
+Production uses managed identity (`TokenCredential`) to talk to Azure
+Storage. Azurite only speaks shared-key / connection-string auth. To keep
+both paths working without a fork, `SourceIndexServer/Models/AzureBlobFileSystem.cs`
+and `UploadIndexStage1/Program.cs` check for `AZURE_STORAGE_CONNECTION_STRING`
+in the environment:
+
+- If set → use `BlobServiceClient(connectionString)` (Azurite-friendly).
+- If not set → use the original `BlobServiceClient(uri, TokenCredential)`
+  code path (prod-friendly).
+
+Aspire injects the connection string automatically via `WithReference(...)`
+on the blob resources in the AppHost. Prod is unaffected.
+
+## Persistence
+
+The two Azurite emulators use `ContainerLifetime.Persistent` and bind-mount
+their data directories to `.azurite/stage1` and `.azurite/prod` at the repo
+root. Blobs uploaded during a session survive `aspire start` restarts and
+container recreations — to wipe state, just delete the `.azurite/` folder.
+The folder is gitignored.
+
+## Debugging individual components
+
+The AppHost runs each stage as its own resource, but the underlying projects are normal .NET projects you can debug directly in **Visual Studio** or **VS Code**:
+
+- **`HtmlGenerator`** (`src/SourceBrowser/src/HtmlGenerator/`) — set breakpoints, then either start it from the IDE pointing at an existing binlog under `samples/MiniRuntime/bin/sample/`, or attach to the `step3-htmlgenerator` process after kicking it off from the dashboard.
+- **`SourceIndexServer`** (`src/SourceBrowser/src/SourceIndexServer/`) — F5 from the IDE for fast inner-loop on the web UI. To debug against Azurite data, set `AZURE_STORAGE_CONNECTION_STRING` and `SOURCE_BROWSER_INDEX_PROXY_URL` from the running AppHost (visible on the `web` resource's env vars panel in the dashboard) in your launch profile.
+- **`UploadIndexStage1`** (`src/UploadIndexStage1/`) — same pattern: copy the env vars off the AppHost's `step2-upload-stage1` resource and run from the IDE.
+- **`BinLogToSln`** (`src/SourceBrowser/src/BinLogToSln/`) — runnable directly against any `.binlog` produced by `step1-sample-build`.
+
+The Aspire dashboard's per-resource "Environment" tab is the source of truth for the env vars Aspire injects — copy those into your `launchSettings.json` to reproduce the AppHost environment under a debugger.
+
+## Open follow-ups
+
+- **End-to-end OTel validation**: ServiceDefaults are wired into `SourceIndexServer` and `UploadIndexStage1`, but flowing traces across `HtmlGenerator` (net472) into the dashboard hasn't been validated yet.

samples/MiniRuntime/MiniArray.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace MiniRuntime;
+
+/// <summary>
+/// A minimal heap-backed array wrapper used to demonstrate generics and
+/// indexer documentation in the indexed HTML output.
+/// </summary>
+/// <typeparam name="T">The element type stored in the array.</typeparam>
+public sealed class MiniArray<T> : IReadOnlyList<T>
+{
+    private readonly T[] _items;
+
+    /// <summary>
+    /// Initializes a new <see cref="MiniArray{T}"/> with the given length.
+    /// </summary>
+    /// <param name="length">The number of elements the array will hold.</param>
+    /// <exception cref="ArgumentOutOfRangeException">
+    /// Thrown if <paramref name="length"/> is negative.
+    /// </exception>
+    public MiniArray(int length)
+    {
+        if (length < 0)
+        {
+            throw new ArgumentOutOfRangeException(nameof(length));
+        }
+
+        _items = new T[length];
+    }
+
+    /// <summary>
+    /// Gets or sets the element at the given index.
+    /// </summary>
+    /// <param name="index">The zero-based index.</param>
+    public T this[int index]
+    {
+        get => _items[index];
+        set => _items[index] = value;
+    }
+
+    /// <summary>
+    /// Gets the number of elements in the array.
+    /// </summary>
+    public int Count => _items.Length;
+
+    /// <inheritdoc />
+    public IEnumerator<T> GetEnumerator()
+    {
+        for (int i = 0; i < _items.Length; i++)
+        {
+            yield return _items[i];
+        }
+    }
+
+    IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+
+    internal T[] UnsafeBuffer => _items;
+}

samples/MiniRuntime/MiniRuntime.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net9.0</TargetFramework>
+    <RootNamespace>MiniRuntime</RootNamespace>
+    <AssemblyName>MiniRuntime</AssemblyName>
+    <GenerateDocumentationFile>true</GenerateDocumentationFile>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <NoWarn>$(NoWarn);CS1591</NoWarn>
+    <!-- This is a tiny fixture used only by the Aspire inner-loop AppHost.
+         It is not packaged or published anywhere. -->
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+</Project>

samples/MiniRuntime/MiniRuntimeInfo.cs

@@ -0,0 +1,33 @@
+namespace MiniRuntime;
+
+/// <summary>
+/// Provides static information about the MiniRuntime fixture library.
+/// </summary>
+/// <remarks>
+/// This type exists so the source indexer has something with XML doc
+/// comments, public surface, and internal helpers to render.
+/// </remarks>
+public static class MiniRuntimeInfo
+{
+    /// <summary>
+    /// Gets the human-readable name of this runtime fixture.
+    /// </summary>
+    public static string Name => "MiniRuntime";
+
+    /// <summary>
+    /// Gets the version string for this runtime fixture.
+    /// </summary>
+    public static string Version => "0.1.0-inner-loop";
+
+    /// <summary>
+    /// Returns a banner suitable for logging at startup.
+    /// </summary>
+    /// <returns>A formatted banner string.</returns>
+    public static string GetBanner() => Banners.Format(Name, Version);
+
+    internal static class Banners
+    {
+        internal static string Format(string name, string version)
+            => $"=== {name} v{version} ===";
+    }
+}

samples/MiniRuntime/MiniSpan.cs

@@ -0,0 +1,61 @@
+using System;
+
+namespace MiniRuntime;
+
+/// <summary>
+/// A trivial slice-like view over a <see cref="MiniArray{T}"/>.
+/// </summary>
+/// <typeparam name="T">The element type.</typeparam>
+public readonly struct MiniSpan<T>
+{
+    private readonly MiniArray<T> _source;
+    private readonly int _start;
+    private readonly int _length;
+
+    /// <summary>
+    /// Initializes a new <see cref="MiniSpan{T}"/> over the given array slice.
+    /// </summary>
+    /// <param name="source">The backing array.</param>
+    /// <param name="start">The inclusive start index.</param>
+    /// <param name="length">The slice length.</param>
+    public MiniSpan(MiniArray<T> source, int start, int length)
+    {
+        ArgumentNullException.ThrowIfNull(source);
+
+        if ((uint)start > (uint)source.Count)
+        {
+            throw new ArgumentOutOfRangeException(nameof(start));
+        }
+
+        if ((uint)length > (uint)(source.Count - start))
+        {
+            throw new ArgumentOutOfRangeException(nameof(length));
+        }
+
+        _source = source;
+        _start = start;
+        _length = length;
+    }
+
+    /// <summary>Gets the length of the span.</summary>
+    public int Length => _length;
+
+    /// <summary>Gets a reference to the element at the given offset.</summary>
+    /// <param name="index">The zero-based offset within the span.</param>
+    public T this[int index]
+    {
+        get
+        {
+            if ((uint)index >= (uint)_length)
+            {
+                throw new IndexOutOfRangeException();
+            }
+
+            return _source[_start + index];
+        }
+    }
+
+    /// <summary>Slices this span further.</summary>
+    public MiniSpan<T> Slice(int start, int length) =>
+        new MiniSpan<T>(_source, _start + start, length);
+}