Add CI flaky-test diagnostics

.github/workflows/dotnet.yml

@@ -182,7 +182,41 @@ jobs:
         run: dotnet build tests/CodeIndex.Tests/CodeIndex.Tests.csproj --configuration Release --framework ${{ matrix.test-framework }} --no-restore
 
       - name: Test
-        run: dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj --configuration Release --framework ${{ matrix.test-framework }} --no-build --nologo --logger "trx;LogFileName=test_results.trx" --results-directory ./TestResults
+        shell: pwsh
+        run: |
+          $testArgs = @(
+            "test",
+            "tests/CodeIndex.Tests/CodeIndex.Tests.csproj",
+            "--configuration", "Release",
+            "--framework", "${{ matrix.test-framework }}",
+            "--no-build",
+            "--nologo",
+            "--settings", "tests/CodeIndex.Tests/CodeIndex.Tests.runsettings",
+            "--blame-crash",
+            "--blame-hang",
+            "--blame-hang-timeout", "5m",
+            "--logger", "trx;LogFileName=test_results.trx",
+            "--results-directory", "./TestResults"
+          )
+
+          dotnet @testArgs
+          $firstExitCode = $LASTEXITCODE
+          if ($firstExitCode -eq 0) {
+            exit 0
+          }
+
+          Write-Warning "Initial test run failed with exit code $firstExitCode. Rerunning once to classify possible flakiness."
+          dotnet @testArgs
+          $retryExitCode = $LASTEXITCODE
+          if ($retryExitCode -eq 0) {
+            New-Item -ItemType Directory -Force -Path ./TestResults | Out-Null
+            "Initial test run failed, but the single retry passed. Treat this run as flaky and inspect TRX/blame artifacts." |
+              Set-Content -Encoding UTF8 ./TestResults/flaky-retry.txt
+            Write-Warning "Tests passed on retry; uploaded TestResults include flaky-retry.txt."
+            exit 0
+          }
+
+          exit $retryExitCode
 
       - name: Summarize TRX telemetry
         if: always()
@@ -194,7 +228,14 @@ jobs:
         with:
           name: TestResults-${{ matrix.os }}-${{ matrix.test-framework }}
           if-no-files-found: warn
-          path: TestResults/**/*.trx
+          path: |
+            TestResults/**/*.trx
+            TestResults/**/*.txt
+            TestResults/**/*.xml
+            TestResults/**/*.dmp
+            TestResults/**/*.dump
+            TestResults/**/*Sequence*.xml
+            TestResults/**/*.hangdump
 
       - name: Publish
         if: matrix.os == 'ubuntu-latest' && matrix.test-framework == 'net8.0'

DEVELOPER_GUIDE.md

@@ -7,6 +7,7 @@
 ```bash
 dotnet build
 dotnet test
+dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj --settings tests/CodeIndex.Tests/CodeIndex.Tests.runsettings --blame-crash --blame-hang --blame-hang-timeout 5m
 dotnet run --project src/CodeIndex -- <command> [options]
 ```
 
@@ -32,7 +33,11 @@ multi-targets `net8.0;net9.0`, and CI runs the test suite on both frameworks
 across Linux, Windows, and macOS. Use a .NET SDK that can restore and run both
 target frameworks when validating the full CI-equivalent test matrix.
 
-For test suite structure, shared helpers, and test-writing conventions, see [TESTING_GUIDE.md](TESTING_GUIDE.md).
+CI uses `tests/CodeIndex.Tests/CodeIndex.Tests.runsettings` plus VSTest blame
+crash/hang collection and a bounded one-time retry to distinguish repeatable
+failures from pass-on-retry flakes. For test suite structure, shared helpers,
+state-isolation rules, timeout diagnostics, and test-writing conventions, see
+[TESTING_GUIDE.md](TESTING_GUIDE.md).
 
 ## CI / Artifact Distribution
 

TESTING_GUIDE.md

@@ -11,6 +11,7 @@ If you change test code, test helpers, test execution flow, or testing conventio
 ```bash
 dotnet test
 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj
+dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj --settings tests/CodeIndex.Tests/CodeIndex.Tests.runsettings --blame-crash --blame-hang --blame-hang-timeout 5m
 dotnet test --filter "FullyQualifiedName~GitHelperTests"
 ```
 
@@ -25,6 +26,7 @@ Use the full suite by default. Use targeted filters only while iterating locally
 - These test-only packages are separate from the production dependency rule in `src/CodeIndex`, which still allows only `Microsoft.Data.Sqlite` at runtime.
 - `FsCheck.Xunit` is reserved for property-based tests that assert universal invariants (never-throws contracts, idempotence, "output is parseable by downstream consumer") across randomly generated inputs. Use it to complement, not replace, the example-based `[Fact]` / `[Theory]` tests — pick FsCheck when the property is a universally quantified claim, and an example test when a specific concrete case is the contract.
 - Test parallelism: enabled by default across independent test classes. Tests that touch process-global state such as SQLite pool resets, environment variables, or current-directory overrides must use an explicit non-parallel collection, and tests that swap `Console.Out` / `Console.Error` must lock on `TestConsoleLock.Gate`.
+- CI runs the test project through `tests/CodeIndex.Tests/CodeIndex.Tests.runsettings`, enables VSTest blame crash and hang collection, applies a 30-minute session timeout plus 60-second xUnit long-running diagnostics, and reruns the suite once after an initial failure. If the retry passes, CI uploads `TestResults/flaky-retry.txt` with the TRX and blame artifacts so the run is treated as suspect instead of silently trusted.
 
 ## Test Layout
 
@@ -85,6 +87,16 @@ The test project mirrors the production areas closely.
 - When a production comment or error string is bilingual, preserve that expectation in tests where it matters.
 - If a behavior change is user-visible, update tests, `CHANGELOG.md`, and any affected docs together.
 
+### Shared state and parallelism audit
+
+Use the inventory below before adding or moving a test class:
+
+- SQLite pool resets, direct `SqliteConnection.ClearAllPools()` calls, process current-directory changes, or process-global environment variable mutation: put the class in the `SQLite pool sensitive` non-parallel collection.
+- Environment variables: use `EnvironmentVariableScope.Capture(...)` so setup failures and assertion failures restore the original values through one cleanup path.
+- `Console.Out` or `Console.Error` replacement: lock `TestConsoleLock.Gate` around the whole capture/swap window.
+- Temporary repositories and files: create them through `TestProjectHelper` when practical, and do not depend on user-level git config.
+- Long-running or performance-oriented tests: keep them skipped by default or give them broad deterministic budgets; if CI reports them in xUnit long-running diagnostics, first check runner load before tightening thresholds.
+
 ## Shared Helpers
 
 ### `TestProjectHelper`
@@ -198,6 +210,7 @@ Check the following:
 ```bash
 dotnet test
 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj
+dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj --settings tests/CodeIndex.Tests/CodeIndex.Tests.runsettings --blame-crash --blame-hang --blame-hang-timeout 5m
 dotnet test --filter "FullyQualifiedName~GitHelperTests"
 ```
 
@@ -212,6 +225,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests"
 - これらの test-only package は `src/CodeIndex` の本番依存ルールとは別であり、runtime 側は引き続き `Microsoft.Data.Sqlite` のみを許容する。
 - `FsCheck.Xunit` はランダム生成入力に対する普遍的不変条件（never-throws、idempotence、"出力が downstream consumer で parse 可能" 等）を表明する property-based テスト専用です。例ベースの `[Fact]` / `[Theory]` を置き換えるのではなく補完するもので、普遍量化された主張なら FsCheck、特定の具体ケースが契約なら例ベースという形で使い分けてください。
 - テスト並列実行: 独立したテストクラス間ではデフォルトで有効です。SQLite pool の解放、環境変数の変更、カレントディレクトリの上書きのような process-global 状態を触るテストは、明示的な non-parallel collection に入れてください。`Console.Out` / `Console.Error` を差し替えるテストは `TestConsoleLock.Gate` で lock してください。
+- CI は `tests/CodeIndex.Tests/CodeIndex.Tests.runsettings` 経由でテストプロジェクトを実行し、VSTest の blame crash / hang 収集、30分のセッションタイムアウト、60秒の xUnit long-running 診断を有効にします。初回失敗時は suite を1回だけ再実行し、再実行で成功した場合は TRX / blame artifact と一緒に `TestResults/flaky-retry.txt` を upload して、その実行を疑わしい flaky run として扱います。
 
 ## テスト構成
 
@@ -271,6 +285,16 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests"
 - 本番コードのコメントやエラー文字列が英日併記前提なら、重要な箇所ではその期待もテストに反映する。
 - ユーザーに見える挙動を変えたら、テストに加えて `CHANGELOG.md` と関連ドキュメントも同じ変更に含める。
 
+### 共有状態と並列実行の監査
+
+テストクラスを追加または移動する前に、次の一覧を確認してください。
+
+- SQLite pool reset、`SqliteConnection.ClearAllPools()` の直接呼び出し、プロセスの current directory 変更、process-global な環境変数変更: クラスを non-parallel な `SQLite pool sensitive` collection に入れる。
+- 環境変数: `EnvironmentVariableScope.Capture(...)` を使い、setup failure や assertion failure でも単一の cleanup 経路で元の値に戻す。
+- `Console.Out` / `Console.Error` の差し替え: capture / swap 期間全体を `TestConsoleLock.Gate` で lock する。
+- 一時 repo / file: 可能な限り `TestProjectHelper` 経由で作り、user-level の git config に依存しない。
+- 長時間または performance 系テスト: デフォルト skip にするか、決定的で十分広い budget を与える。CI の xUnit long-running 診断に出た場合は、閾値を締める前に runner 負荷を確認する。
+
 ## 共通ヘルパー
 
 ### `TestProjectHelper`

changelog.d/unreleased/1610.fixed.md

@@ -0,0 +1,17 @@
+---
+category: fixed
+issues:
+  - 1610
+affected:
+  - .github/workflows/dotnet.yml
+  - TESTING_GUIDE.md
+  - DEVELOPER_GUIDE.md
+---
+
+## English
+
+- **CI flaky-test classification (#1610)** — Added a bounded one-time test retry in CI and a `flaky-retry.txt` artifact marker when the retry passes after an initial failure.
+
+## 日本語
+
+- **CI の flaky test 分類 (#1610)** — CI テストに1回限りの bounded retry を追加し、初回失敗後の再実行で成功した場合に `flaky-retry.txt` artifact marker を残すようにしました。

changelog.d/unreleased/1612.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+issues:
+  - 1612
+affected:
+  - TESTING_GUIDE.md
+  - DEVELOPER_GUIDE.md
+---
+
+## English
+
+- **Test parallelism audit guidance (#1612)** — Documented the shared-state inventory for test classes that need serialized execution or explicit cleanup guards.
+
+## 日本語
+
+- **テスト並列実行の監査ガイド (#1612)** — 直列実行または明示的な cleanup guard が必要なテストクラス向けに、共有状態の inventory を文書化しました。

changelog.d/unreleased/1821.fixed.md

@@ -0,0 +1,18 @@
+---
+category: fixed
+issues:
+  - 1821
+affected:
+  - .github/workflows/dotnet.yml
+  - tests/CodeIndex.Tests/CodeIndex.Tests.runsettings
+  - TESTING_GUIDE.md
+  - DEVELOPER_GUIDE.md
+---
+
+## English
+
+- **CI test timeout and blame diagnostics (#1821)** — Added test runsettings with a session timeout and xUnit long-running diagnostics, enabled VSTest crash/hang blame capture in CI, and uploaded blame artifacts with TRX results.
+
+## 日本語
+
+- **CI テストの timeout / blame 診断 (#1821)** — セッションタイムアウトと xUnit long-running 診断を含む test runsettings を追加し、CI で VSTest の crash/hang blame 収集を有効化して、TRX と一緒に blame artifact をアップロードするようにしました。

tests/CodeIndex.Tests/CiWorkflowTests.cs

@@ -0,0 +1,66 @@
+using System.Xml.Linq;
+
+namespace CodeIndex.Tests;
+
+public class CiWorkflowTests
+{
+    [Fact]
+    public void DotnetWorkflow_RunsTestsWithRunsettingsBlameRetryAndArtifacts()
+    {
+        var workflow = File.ReadAllText(Path.Combine(GetRepositoryRoot(), ".github", "workflows", "dotnet.yml"));
+
+        Assert.Contains("--settings\", \"tests/CodeIndex.Tests/CodeIndex.Tests.runsettings", workflow);
+        Assert.Contains("--blame-crash", workflow);
+        Assert.Contains("--blame-hang", workflow);
+        Assert.Contains("--blame-hang-timeout\", \"5m", workflow);
+        Assert.Contains("Rerunning once to classify possible flakiness.", workflow);
+        Assert.Contains("flaky-retry.txt", workflow);
+        Assert.Contains("TestResults/**/*.trx", workflow);
+        Assert.Contains("TestResults/**/*Sequence*.xml", workflow);
+        Assert.Contains("TestResults/**/*.dmp", workflow);
+        Assert.Contains("TestResults/**/*.dump", workflow);
+    }
+
+    [Fact]
+    public void Runsettings_DefinesSessionTimeoutAndXunitLongRunningDiagnostics()
+    {
+        var path = Path.Combine(GetRepositoryRoot(), "tests", "CodeIndex.Tests", "CodeIndex.Tests.runsettings");
+        var document = XDocument.Load(path);
+
+        Assert.Equal(
+            "1800000",
+            document.Root?.Element("RunConfiguration")?.Element("TestSessionTimeout")?.Value);
+        Assert.Equal(
+            "60",
+            document.Root?.Element("xUnit")?.Element("LongRunningTestSeconds")?.Value);
+        Assert.Equal(
+            "./TestResults",
+            document.Root?.Element("RunConfiguration")?.Element("ResultsDirectory")?.Value);
+    }
+
+    [Fact]
+    public void TestingGuide_DocumentsSharedStateParallelismInventory()
+    {
+        var guide = File.ReadAllText(Path.Combine(GetRepositoryRoot(), "TESTING_GUIDE.md"));
+
+        Assert.Contains("Shared state and parallelism audit", guide);
+        Assert.Contains("SQLite pool sensitive", guide);
+        Assert.Contains("EnvironmentVariableScope.Capture", guide);
+        Assert.Contains("TestConsoleLock.Gate", guide);
+        Assert.Contains("TestProjectHelper", guide);
+        Assert.Contains("共有状態と並列実行の監査", guide);
+    }
+
+    private static string GetRepositoryRoot()
+    {
+        var dir = new DirectoryInfo(AppContext.BaseDirectory);
+        while (dir != null)
+        {
+            if (File.Exists(Path.Combine(dir.FullName, "CodeIndex.sln")))
+                return dir.FullName;
+            dir = dir.Parent;
+        }
+
+        throw new InvalidOperationException("Could not locate repository root / リポジトリルートを特定できませんでした");
+    }
+}

tests/CodeIndex.Tests/CodeIndex.Tests.runsettings

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <RunConfiguration>
+    <ResultsDirectory>./TestResults</ResultsDirectory>
+    <TestSessionTimeout>1800000</TestSessionTimeout>
+  </RunConfiguration>
+  <xUnit>
+    <LongRunningTestSeconds>60</LongRunningTestSeconds>
+  </xUnit>
+</RunSettings>