Merge pull request #5078 from Widthdom/codex/split-large-types-20260812

Decompose large indexing and query orchestration

Author: Widthdom

TESTING_GUIDE.md

@@ -0,0 +1,14 @@
+---
+category: fixed
+affected:
+  - src/CodeIndex/Cli/QueryCommandRunner.Batch.cs
+  - src/CodeIndex/Cli/QueryCommandRunner.BatchParallelExecution.cs
+---
+
+## English
+
+- **Parallel batch output limits no longer discard accepted, undispatched input records** — when `batch --parallel` reaches `--max-output-chars`, accepted nonblank input records that were not dispatched are returned to the shared bounded input pump in source order. The next batch invocation reparses those records with fresh line numbers and counters, ahead of input the pump already buffered, while commands that started remain owned by the first invocation.
+
+## 日本語
+
+- **parallel batch の出力上限で、受理済みかつ未 dispatch の input record が失われなくなりました** — `batch --parallel` が `--max-output-chars` に達した場合、受理済みでも未 dispatch の nonblank input record を source 順のまま共有の bounded input pump に戻します。次の batch invocation は、すでに pump が buffer した入力より先にそれらの record を再 parse し、新しい line number と counter を割り当てます。開始済みの command は最初の invocation の所有のままです。

changelog.d/unreleased/+batch-output-limit-replay.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+affected:
+  - src/CodeIndex/Database/CSharpBaseListParser.cs
+  - src/CodeIndex/Database/DbContext.ConnectionFunctions.cs
+  - src/CodeIndex/Database/DbReader.CSharpResolution.cs
+  - src/CodeIndex/Database/DbWriter.CSharpMetadataTargets.cs
+---
+
+## English
+
+- **C# inheritance resolution now shares one grammar-aware base-list parser** — exact caller and metadata-target resolution no longer confuse generic `where` constraints with base types, and nested generics, tuples, arrays, alias qualifiers, and primary/base-constructor syntax are split consistently.
+
+## 日本語
+
+- **C# 継承解決が grammar-aware な base-list parser を共有するようになりました** — exact caller と metadata-target の解決で generic `where` constraint を基底型と誤認せず、nested generic、tuple、array、alias qualifier、primary / base constructor 構文を一貫して分割します。

changelog.d/unreleased/+csharp-base-list-parser.fixed.md

@@ -0,0 +1,15 @@
+---
+category: fixed
+affected:
+  - src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs
+  - src/CodeIndex/Cli/IndexCommandRunner.FullScan.ResultConsumer.cs
+  - tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs
+---
+
+## English
+
+- **Full-scan extraction now preserves shared state and worker-resource ownership** — a later C# workspace drift no longer drops text-index mutations already committed for earlier files, and a fatal parallel worker result defers shared queue, cache, hook, and cancellation cleanup until every peer worker has stopped.
+
+## 日本語
+
+- **full-scan extraction が共有 state と worker resource の ownership を保持するようになりました** — 後段の C# workspace drift が先行 file で commit 済みの text-index mutation を失わず、parallel worker の fatal result では全 peer worker の停止まで共有 queue、cache、hook、cancellation の cleanup を延期します。

changelog.d/unreleased/+full-scan-extraction-state.fixed.md

@@ -0,0 +1,15 @@
+---
+category: fixed
+affected:
+  - src/CodeIndex/Database/DbReader.cs
+  - src/CodeIndex/Database/DbSymbolReader.UnusedSymbols.Candidates.cs
+  - src/CodeIndex/Database/DbSymbolReader.UnusedSymbols.cs
+---
+
+## English
+
+- **C# unused-symbol queries now remain safe on read-only legacy databases** — partial-type use filtering is applied before paging without issuing per-candidate database queries, while databases whose legacy chunk or symbol tables lack the required columns retain the existing degraded fallback instead of failing during SQL preparation.
+
+## 日本語
+
+- **C# の unused-symbol query が read-only legacy database でも安全に動作するようになりました** — partial type の使用判定を candidate ごとの database query なしで paging 前に適用し、必要な column が legacy chunk / symbol table にない database では SQL prepare で失敗せず、従来の degraded fallback を維持します。

changelog.d/unreleased/+unused-legacy-schema.fixed.md

@@ -93,94 +93,113 @@ or NotSupportedException
         }
     }
 
-    private static bool ShouldUseFullScanFtsBulkLoad(
-        bool rebuild,
-        bool startedWithNoIndexedFiles,
-        int extractionWorkItemCount,
-        FilePurgePlan staleFilePurgePlan,
-        bool scanHadErrors,
-        ReadableFileByteTracker readableFileBytes,
-        ReusableIndexedFileStatsSnapshot reusableIndexedFileStats,
-        IReadOnlyList<FullScanFileTarget> fileTargets,
-        IReadOnlyList<int>? extractionFileIndexes,
-        Action throwIfCancelled)
+    private sealed partial class FullScanPreWriteSession
     {
-        if (rebuild || startedWithNoIndexedFiles)
-            return true;
-        if (extractionWorkItemCount == 0 && staleFilePurgePlan.Count == 0)
-            return false;
-
-        var dirtyBytes = staleFilePurgePlan.DeletedBytes;
-        var persistedSizeExcessBytes = 0L;
-        var byteEstimateComplete = !scanHadErrors
-            && staleFilePurgePlan.ByteEstimateComplete
-            && readableFileBytes.EstimateComplete;
-
-        void AddDirtyFileBytes(int fileIndex)
+        internal void DecideFtsBulkLoad()
         {
-            throwIfCancelled();
-            try
+            var request = Request;
+            var options = request.Core.Options;
+            var baseline = request.Baseline;
+            var runtime = request.Runtime;
+            var scan = State.Scan;
+            var csharp = State.CSharp;
+            var selection = State.Selection;
+            if (options.Rebuild || baseline.StartedWithNoIndexedFiles)
             {
-                var target = fileTargets[fileIndex];
-                var info = new FileInfo(target.FilePath);
-                if (!info.Exists || info.Length < 0)
+                selection.UseFtsBulkLoad = true;
+                return;
+            }
+
+            if (selection.ExtractionWorkItemCount == 0
+                && scan.StaleFilePurgePlan.Count == 0)
+            {
+                selection.UseFtsBulkLoad = false;
+                return;
+            }
+
+            var dirtyBytes = scan.StaleFilePurgePlan.DeletedBytes;
+            var persistedSizeExcessBytes = 0L;
+            var byteEstimateComplete = !baseline.ScanHadErrors
+                && scan.StaleFilePurgePlan.ByteEstimateComplete
+                && selection.ReadableFileBytes.EstimateComplete;
+
+            void AddDirtyFileBytes(int fileIndex)
+            {
+                ThrowIfFullScanCancelled();
+                try
+                {
+                    var target = runtime.FileTargets[fileIndex];
+                    var info = new FileInfo(target.FilePath);
+                    if (!info.Exists || info.Length < 0)
+                    {
+                        byteEstimateComplete = false;
+                        return;
+                    }
+
+                    selection.ReadableFileBytes.Remember(
+                        fileIndex,
+                        info.Length);
+                    var persistedSize = csharp.ReusableIndexedFileStats!
+                        .GetPersistedSize(target.IndexPath);
+                    if (!FtsBulkLoadTriggerGuard.TryAccumulateDirtyFileBytes(
+                            dirtyBytes,
+                            persistedSizeExcessBytes,
+                            info.Length,
+                            persistedSize,
+                            out dirtyBytes,
+                            out persistedSizeExcessBytes))
+                    {
+                        byteEstimateComplete = false;
+                    }
+                }
+                catch (Exception ex) when (
+                    ex is IOException
+                        or UnauthorizedAccessException
+                        or NotSupportedException
+                        or ArgumentException)
                 {
                     byteEstimateComplete = false;
-                    return;
                 }
+            }
 
-                readableFileBytes.Remember(fileIndex, info.Length);
-                var persistedSize = reusableIndexedFileStats.GetPersistedSize(target.IndexPath);
-                if (!FtsBulkLoadTriggerGuard.TryAccumulateDirtyFileBytes(
-                        dirtyBytes,
-                        persistedSizeExcessBytes,
-                        info.Length,
-                        persistedSize,
-                        out dirtyBytes,
-                        out persistedSizeExcessBytes))
+            if (selection.ExtractionFileIndexes != null)
+            {
+                foreach (var fileIndex in selection.ExtractionFileIndexes)
+                    AddDirtyFileBytes(fileIndex);
+            }
+            else
+            {
+                for (var fileIndex = 0;
+                     fileIndex < runtime.FileTargets.Length;
+                     fileIndex++)
                 {
-                    byteEstimateComplete = false;
+                    AddDirtyFileBytes(fileIndex);
                 }
             }
-            catch (Exception ex) when (
-                ex is IOException
-                    or UnauthorizedAccessException
-                    or NotSupportedException
-                    or ArgumentException)
+
+            byteEstimateComplete &=
+                selection.ReadableFileBytes.EstimateComplete;
+            var totalBytes = selection.ReadableFileBytes.KnownBytes;
+            if (!selection.ReadableFileBytes.EstimateComplete
+                || totalBytes
+                > long.MaxValue - scan.StaleFilePurgePlan.DeletedBytes)
             {
                 byteEstimateComplete = false;
             }
-        }
+            else
+            {
+                totalBytes += scan.StaleFilePurgePlan.DeletedBytes;
+            }
 
-        if (extractionFileIndexes != null)
-        {
-            foreach (var fileIndex in extractionFileIndexes)
-                AddDirtyFileBytes(fileIndex);
-        }
-        else
-        {
-            for (var fileIndex = 0; fileIndex < fileTargets.Count; fileIndex++)
-                AddDirtyFileBytes(fileIndex);
-        }
+            if (totalBytes > long.MaxValue - persistedSizeExcessBytes)
+                byteEstimateComplete = false;
+            else
+                totalBytes += persistedSizeExcessBytes;
 
-        byteEstimateComplete &= readableFileBytes.EstimateComplete;
-        var totalBytes = readableFileBytes.KnownBytes;
-        if (!readableFileBytes.EstimateComplete
-            || totalBytes > long.MaxValue - staleFilePurgePlan.DeletedBytes)
-        {
-            byteEstimateComplete = false;
-        }
-        else
-        {
-            totalBytes += staleFilePurgePlan.DeletedBytes;
+            selection.UseFtsBulkLoad = byteEstimateComplete
+                && FtsBulkLoadTriggerGuard.ShouldUseForDirtyBytes(
+                    dirtyBytes,
+                    totalBytes);
         }
-
-        if (totalBytes > long.MaxValue - persistedSizeExcessBytes)
-            byteEstimateComplete = false;
-        else
-            totalBytes += persistedSizeExcessBytes;
-
-        return byteEstimateComplete
-            && FtsBulkLoadTriggerGuard.ShouldUseForDirtyBytes(dirtyBytes, totalBytes);
     }
 }