Address adversarial review findings (#5100)

Author: Widthdom

changelog.d/unreleased/5100.changed.md

@@ -5,6 +5,7 @@ issues:
 affected:
   - src/CodeIndex/Cli/ConsoleUi.cs
   - src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs
+  - src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs
   - src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs
   - src/CodeIndex/Database/UnknownExtensionClassifier.cs
   - src/CodeIndex/Models/QueryResults.cs

src/CodeIndex/Cli/ExportImportCommandRunner.ArchiveScope.cs

@@ -329,7 +329,11 @@ OR key IN (
                    'unknown_extension_file_path_limit',
                    'unknown_extension_extension_counts_json',
                    'unknown_extension_category_counts_json',
-                   'unknown_extension_groups_json');
+                   'unknown_extension_groups_json',
+                   'unknown_extension_group_count',
+                   'unknown_extension_groups_truncated',
+                   'unknown_extension_group_limit',
+                   'unknown_extension_group_omitted_count');
 
             INSERT INTO codeindex_meta(key, value)
             VALUES (@indexCompletenessKey, 'incomplete')

src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs

@@ -512,6 +512,8 @@ void RecordDryRunScanErrors(IEnumerable<FileIndexer.ScanError> scanErrors)
         var unknownExtensionGroupOmittedCount = Math.Max(
             0,
             unknownExtensionClassification.GroupCount - unknownExtensionGroups.Count);
+        var unknownExtensionFileCountLowerBound = candidatePathsTruncated
+            || (authoritativeFullScan && dryScanMetadata.HadErrors);
         var unknownExtensionWarning = unknownExtensionClassification.ActionableFileCount > 0
             ? $"{unknownExtensionClassification.ActionableFileCount} file(s) were excluded because no language mapping or extractor was available. {UnknownExtensionClassifier.Guidance}"
             : null;
@@ -553,7 +555,7 @@ void RecordDryRunScanErrors(IEnumerable<FileIndexer.ScanError> scanErrors)
                 UnknownExtensionGroupLimit = UnknownExtensionClassifier.MaxCompletionGroups,
                 UnknownExtensionGroupOmittedCount = unknownExtensionGroupOmittedCount,
                 UnknownExtensionDiagnosticsScope = authoritativeFullScan ? "workspace" : "candidate_scope",
-                UnknownExtensionFileCountLowerBound = candidatePathsTruncated,
+                UnknownExtensionFileCountLowerBound = unknownExtensionFileCountLowerBound,
                 UnknownExtensionGuidance = unknownExtensionTotal > 0
                     ? UnknownExtensionClassifier.Guidance
                     : null,
@@ -593,7 +595,7 @@ void RecordDryRunScanErrors(IEnumerable<FileIndexer.ScanError> scanErrors)
             CommandOutputWriter.WriteLine($"Dry run: {dryFileCount} indexable files inspected{lowerBound}");
             if (unknownExtensionTotal > 0)
             {
-                CommandOutputWriter.WriteLine($"  unknown extensions {unknownExtensionTotal,6}{(candidatePathsTruncated ? " (lower bound)" : string.Empty)}");
+                CommandOutputWriter.WriteLine($"  unknown extensions {unknownExtensionTotal,6}{(unknownExtensionFileCountLowerBound ? " (lower bound)" : string.Empty)}");
                 foreach (var group in unknownExtensionGroups)
                     CommandOutputWriter.WriteLine($"    {group.Extension}: {ConsoleUi.FormatNumber(group.Count)} ({group.RecommendedAction})");
                 if (unknownExtensionGroupOmittedCount > 0)

src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs

@@ -155,7 +155,7 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output)
                 UnknownExtensionGroupLimit = UnknownExtensionClassifier.MaxCompletionGroups,
                 UnknownExtensionGroupOmittedCount = unknownExtensionGroupOmittedCount,
                 UnknownExtensionDiagnosticsScope = "workspace",
-                UnknownExtensionFileCountLowerBound = false,
+                UnknownExtensionFileCountLowerBound = output.ScanHadErrors,
                 UnknownExtensionGuidance = output.ScanResult.UnknownExtensionFiles.Count > 0
                     ? UnknownExtensionClassifier.Guidance
                     : null,
@@ -242,7 +242,8 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output)
             {
                 CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine(
                     "Unknown extensions",
-                    ConsoleUi.FormatNumber(output.ScanResult.UnknownExtensionFiles.Count),
+                    ConsoleUi.FormatNumber(output.ScanResult.UnknownExtensionFiles.Count)
+                        + (output.ScanHadErrors ? " (lower bound)" : string.Empty),
                     indent: "  "));
                 foreach (var group in unknownExtensionGroups)
                 {

src/CodeIndex/Cli/IndexCommandRunner.FullScan.SnapshotFailure.cs

@@ -21,6 +21,7 @@ private sealed class FullScanSnapshotFailureContext
         internal string? PriorFoldFingerprint { get; init; }
         internal required List<IndexMemorySampleJsonResult> MemorySamples { get; init; }
         internal required IReadOnlyDictionary<string, int> LanguageCounts { get; init; }
+        internal required IReadOnlyList<string> UnknownExtensionFiles { get; init; }
         internal int FilesCount { get; init; }
         internal int Skipped { get; init; }
         internal int DanglingSymlinkCount { get; init; }
@@ -83,13 +84,39 @@ private static int WriteFullScanSnapshotFailure(
             || (persistedCSharpFiles && failure.PriorMetadataTargetCsharpMatchesCurrent);
         var foldReady = (failure.PriorReadiness & DbContext.FoldReadyFlag) != 0;
         var memoryTimeline = BuildMemoryTimeline(failure.MemorySamples);
+        var unknownExtensionClassification = UnknownExtensionClassifier.Classify(
+            failure.UnknownExtensionFiles);
+        var unknownExtensionGroups = unknownExtensionClassification.Groups
+            .Take(UnknownExtensionClassifier.MaxCompletionGroups)
+            .ToList();
+        var unknownExtensionGroupOmittedCount = Math.Max(
+            0,
+            unknownExtensionClassification.GroupCount - unknownExtensionGroups.Count);
+        var warningCount = failure.Warnings;
+        if (unknownExtensionClassification.ActionableFileCount > 0)
+        {
+            var warning = $"{unknownExtensionClassification.ActionableFileCount} file(s) were excluded because no language mapping or extractor was available. {UnknownExtensionClassifier.Guidance}";
+            failure.WarningList.Add(new CliJsonMessage("<unknown_extensions>", warning));
+            warningCount++;
+        }
 
         if (failure.Options.Json)
         {
             CommandOutputWriter.WriteLine(JsonSerializer.Serialize(new IndexFullScanJsonResult
             {
                 Status = "partial",
                 Mode = failure.Options.Rebuild ? "rebuild" : "incremental",
+                UnknownExtensionFileCount = failure.UnknownExtensionFiles.Count,
+                UnknownExtensionGroups = unknownExtensionGroups.Count > 0 ? unknownExtensionGroups : null,
+                UnknownExtensionGroupCount = unknownExtensionClassification.GroupCount,
+                UnknownExtensionGroupsTruncated = unknownExtensionGroupOmittedCount > 0,
+                UnknownExtensionGroupLimit = UnknownExtensionClassifier.MaxCompletionGroups,
+                UnknownExtensionGroupOmittedCount = unknownExtensionGroupOmittedCount,
+                UnknownExtensionDiagnosticsScope = "workspace",
+                UnknownExtensionFileCountLowerBound = true,
+                UnknownExtensionGuidance = failure.UnknownExtensionFiles.Count > 0
+                    ? UnknownExtensionClassifier.Guidance
+                    : null,
                 Summary = new IndexFullScanSummaryJsonResult
                 {
                     FilesTotal = totalFiles,
@@ -100,7 +127,7 @@ private static int WriteFullScanSnapshotFailure(
                     FilesSkipped = failure.Skipped,
                     FilesPurged = 0,
                     DanglingSymlinksSkipped = failure.DanglingSymlinkCount,
-                    Warnings = failure.Warnings,
+                    Warnings = warningCount,
                     Errors = failure.Errors,
                     SymbolsDroppedByKindFilter = failure.SymbolsDroppedByKindFilter,
                 },

src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs

@@ -456,6 +456,7 @@ bool FullScanJavaScriptTypeScriptConfigChanged()
                     PriorFoldFingerprint = priorFoldFingerprint,
                     MemorySamples = memorySamples,
                     LanguageCounts = languageCounts,
+                    UnknownExtensionFiles = scanResult.UnknownExtensionFiles,
                     FilesCount = files.Count,
                     Skipped = preWriteSelection.Skipped,
                     DanglingSymlinkCount = scanResult.DanglingSymlinks.Count,

tests/CodeIndex.Tests/ExportImportCommandRunnerTests.cs

@@ -2461,6 +2461,10 @@ private static void AssertPartialArchiveStatus(string dbPath)
             !status.RootElement.TryGetProperty("head_freshness", out var headFreshness)
             || headFreshness.GetProperty("state").GetString() != "head_current");
         Assert.False(status.RootElement.TryGetProperty("unknown_extension_file_count", out _));
+        Assert.False(status.RootElement.TryGetProperty("unknown_extension_group_count", out _));
+        Assert.False(status.RootElement.TryGetProperty("unknown_extension_groups_truncated", out _));
+        Assert.False(status.RootElement.TryGetProperty("unknown_extension_group_limit", out _));
+        Assert.False(status.RootElement.TryGetProperty("unknown_extension_group_omitted_count", out _));
         Assert.True(
             !status.RootElement.TryGetProperty("last_index_run", out var lastIndexRun)
             || lastIndexRun.ValueKind == JsonValueKind.Null);

tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs

@@ -1459,7 +1459,7 @@ public void Run_DryRun_FullScan_ReportsProjectedPurgesAndUnknownExtensionsWithou
     }
 
     [Fact]
-    public void Run_DryRun_FullScan_ReportsUnreadableDirectory()
+    public void Run_DryRun_FullScan_ReportsUnreadableDirectory_Issue5100()
     {
         if (OperatingSystem.IsWindows())
             return;
@@ -1470,18 +1470,23 @@ public void Run_DryRun_FullScan_ReportsUnreadableDirectory()
         {
             Directory.CreateDirectory(secretDir);
             File.WriteAllText(Path.Combine(secretDir, "a.cs"), "public class A { }\n");
+            File.WriteAllText(Path.Combine(projectRoot, "notes.unknownext"), "unknown language\n");
             SetUnixPermissions(secretDir, UnixFileMode.None);
 
             var (exitCode, json) = RunAndCaptureJson([projectRoot, "--dry-run", "--json"]);
 
             Assert.Equal(CommandExitCodes.Success, exitCode);
             Assert.Equal("dry_run", json.GetProperty("status").GetString());
             Assert.Equal(0, json.GetProperty("files_total").GetInt32());
+            Assert.Equal(1, json.GetProperty("unknown_extension_file_count").GetInt32());
+            Assert.True(json.GetProperty("unknown_extension_file_count_lower_bound").GetBoolean());
             Assert.Equal("secret", json.GetProperty("errors")[0].GetProperty("file").GetString());
             Assert.Equal("Could not scan directory due to permissions.", json.GetProperty("errors")[0].GetProperty("message").GetString());
 
-            var (humanExitCode, _, stderr) = RunAndCaptureStreams([projectRoot, "--dry-run"]);
+            var (humanExitCode, stdout, stderr) = RunAndCaptureStreams([projectRoot, "--dry-run"]);
             Assert.Equal(CommandExitCodes.Success, humanExitCode);
+            Assert.Contains("unknown extensions", stdout);
+            Assert.Contains("(lower bound)", stdout);
             Assert.Contains("secret", stderr);
             Assert.Contains("Could not scan directory due to permissions.", stderr);
         }

tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs

@@ -3120,7 +3120,7 @@ public void Run_FullScan_RebuildFailureRetainsPersistedReadinessForReclassifiedL
     }
 
     [Fact]
-    public void Run_FullScan_PartialDiscoveryRetainsKnownLanguageReadinessFailures()
+    public void Run_FullScan_PartialDiscoveryRetainsKnownLanguageReadinessFailures_Issue5100()
     {
         if (OperatingSystem.IsWindows())
             return;
@@ -3137,6 +3137,7 @@ public void Run_FullScan_PartialDiscoveryRetainsKnownLanguageReadinessFailures()
             File.WriteAllText(
                 Path.Combine(projectRoot, "Contract.cs"),
                 "public interface IContract { }\n");
+            File.WriteAllText(Path.Combine(projectRoot, "notes.unknownext"), "unknown language\n");
             Directory.CreateDirectory(unreadableDirectory);
             File.WriteAllText(Path.Combine(unreadableDirectory, "blocked.py"), "print('blocked')\n");
             originalMode = File.GetUnixFileMode(unreadableDirectory);
@@ -3152,6 +3153,9 @@ public void Run_FullScan_PartialDiscoveryRetainsKnownLanguageReadinessFailures()
 
             Assert.Equal(CommandExitCodes.PartialResult, exitCode);
             Assert.True(json.GetProperty("summary").GetProperty("errors").GetInt32() >= 2);
+            Assert.True(
+                json.GetProperty("unknown_extension_file_count_lower_bound").GetBoolean(),
+                json.GetRawText());
             Assert.False(
                 json.GetProperty("sql_graph_contract_ready").GetBoolean(),
                 json.GetRawText());