Restore extractor docs after workspace merge #1709

Author: Widthdom

DEVELOPER_GUIDE.md

@@ -3059,6 +3059,54 @@ Cloud セッションは開発ループの中で `dotnet build` にフォール
 - ドキュメント（README, CHANGELOG）は前半英語、後半日本語の構成。
 - 不要な本番パッケージは入れない。test-only package は、テストハーネスの改善に明確に寄与し、`tests/CodeIndex.Tests/` に閉じる限り許容されるが、本番依存ルールを緩めるものではない。
 
+## Custom Language Extraction
+
+Downstream users can add lightweight language support without rebuilding
+`cdidx`:
+
+- extension aliases are read from `~/.config/cdidx/langmap.yaml` and the first
+  workspace ancestor `.cdidx-langmap.yaml`; workspace entries override user
+  entries;
+- regex-backed symbol patterns are read from `.cdidx/patterns/*.yaml` and
+  `~/.config/cdidx/patterns/*.yaml`;
+- `cdidx test-extractor --language <lang> --file <path> --json` runs symbol
+  extraction without building an index, and `--expect-symbols <json>` compares
+  the extracted JSON to a fixture.
+
+Minimal examples:
+
+```yaml
+# .cdidx-langmap.yaml
+entries:
+  - extension: ".kts.in"
+    language: "kotlin"
+```
+
+```yaml
+# .cdidx/patterns/toydsl.yaml
+language: "toydsl"
+extensions:
+  - extension: ".toy"
+patterns:
+  - kind: "class"
+    regex: "^entity (?<name>\\w+)"
+```
+
+Each configured regex should expose a named `name` capture. If it does not,
+`cdidx` uses the full match text as the symbol name. Invalid sidecar files are
+ignored so a broken local experiment does not prevent indexing.
+
+## カスタム言語抽出
+
+下流ユーザーは `cdidx` を再ビルドせずに軽量な言語対応を追加できます。
+
+- 拡張子 alias は `~/.config/cdidx/langmap.yaml` と、最初に見つかった workspace
+  祖先の `.cdidx-langmap.yaml` から読み込まれ、workspace 側が user 側を上書きします。
+- regex ベースのシンボルパターンは `.cdidx/patterns/*.yaml` と
+  `~/.config/cdidx/patterns/*.yaml` から読み込まれます。
+- `cdidx test-extractor --language <lang> --file <path> --json` は index を作らずに
+  symbol extraction だけを実行し、`--expect-symbols <json>` で fixture JSON と比較できます。
+
 ## SQLite reader のデバッグ
 
 `Database/DbDebug.cs` は `ExecuteTrackedReader` / `TrackedRead` の最後に流れた SQL、パラメーター、行ごとの状態を記録し、ループ途中で `SqliteException` が発生した場合に再現に十分な文脈を stderr へダンプする。インデックス済みのソースバイトが想定外の経路に漏れないよう、ダンプ経路はゲート制御されている:

README.md

@@ -58,6 +58,12 @@ cdidx validate
 cdidx mcp
 ```
 
+Custom language loops can stay out of tree: put extension aliases in
+`.cdidx-langmap.yaml`, put regex symbol patterns in `.cdidx/patterns/*.yaml`,
+and run `cdidx test-extractor --language <lang> --file <path> --json` to test
+an extractor fixture without building a full index. See
+[Custom Language Extraction](DEVELOPER_GUIDE.md#custom-language-extraction).
+
 After the first command, use these cues and follow-up commands:
 
 | Situation | What to expect or run |
@@ -300,6 +306,12 @@ cdidx validate
 cdidx mcp
 ```
 
+カスタム言語の開発ループは out-of-tree で回せます。拡張子 alias は
+`.cdidx-langmap.yaml`、regex シンボルパターンは `.cdidx/patterns/*.yaml` に置き、
+`cdidx test-extractor --language <lang> --file <path> --json` で full index を作らずに
+extractor fixture を確認できます。詳細は
+[Custom Language Extraction](DEVELOPER_GUIDE.md#custom-language-extraction) を参照してください。
+
 初回実行後は、次の見方と追加コマンドをよく使います。
 
 | 状況 | 見るもの / 使うもの |

src/CodeIndex/Cli/CdidxConfigFile.cs

@@ -349,7 +349,7 @@ internal static int RunShow(string[] args, JsonSerializerOptions jsonOptions)
         var json = args.Contains("--json", StringComparer.Ordinal);
         args = args.Where(a => a != "--json").ToArray();
         if (args.Length > 0)
-            return CommandErrorWriter.Write("config show does not accept positional arguments.", CommandExitCodes.UsageError, "run `cdidx config show` from the workspace whose config should be shown.");
+            return CommandErrorWriter.WriteJsonOrHuman(json, jsonOptions, "config show does not accept positional arguments.", CommandExitCodes.UsageError, "run `cdidx config show` from the workspace whose config should be shown.");
 
         var path = FindConfigFile(Environment.CurrentDirectory);
         var active = ActiveWorkspace.Load();

src/CodeIndex/Cli/ProgramRunner.cs

@@ -263,6 +263,7 @@ internal static int Run(
                     "workspace" => WorkspaceCommandRunner.Run(subArgs, jsonOptions),
                     "db" => DbCommandRunner.RunIntegrityCheck(subArgs, jsonOptions),
                     "report" => ReportCommandRunner.Run(subArgs, jsonOptions, appVersion),
+                    "test-extractor" => RunTestExtractor(subArgs, jsonOptions),
                     _ when IsProjectPathArg(commandName)
                         => IndexCommandRunner.Run(args, jsonOptions),
                     _ => ShowError(args, $"Unknown command: {commandName}")
@@ -332,6 +333,74 @@ private static bool IsWindowsDrivePath(string arg) =>
         && arg[1] == ':'
         && ((arg[0] >= 'A' && arg[0] <= 'Z') || (arg[0] >= 'a' && arg[0] <= 'z'));
 
+    private static int RunTestExtractor(string[] args, JsonSerializerOptions jsonOptions)
+    {
+        string? language = null;
+        string? file = null;
+        string? expect = null;
+        var json = false;
+        for (var i = 0; i < args.Length; i++)
+        {
+            var arg = args[i];
+            if (TryConsumeInlineOrNext(args, ref i, arg, "--language", out var value))
+                language = value;
+            else if (TryConsumeInlineOrNext(args, ref i, arg, "--file", out value))
+                file = value;
+            else if (TryConsumeInlineOrNext(args, ref i, arg, "--expect-symbols", out value) || TryConsumeInlineOrNext(args, ref i, arg, "--expect", out value))
+                expect = value;
+            else if (arg == "--json")
+                json = true;
+            else
+                return CommandErrorWriter.Write($"Unknown test-extractor argument: {arg}", CommandExitCodes.InvalidArgument, "use --language <lang> --file <path> [--expect-symbols <json>] [--json].");
+        }
+
+        if (string.IsNullOrWhiteSpace(language) || string.IsNullOrWhiteSpace(file))
+            return CommandErrorWriter.Write("test-extractor requires --language and --file.", CommandExitCodes.InvalidArgument, "use --language <lang> --file <path> [--expect-symbols <json>] [--json].");
+        if (!File.Exists(file))
+            return CommandErrorWriter.Write($"File not found: {file}", CommandExitCodes.NotFound);
+
+        var source = File.ReadAllText(file);
+        var symbols = Indexer.SymbolExtractor.Extract(1, language, source, file);
+        if (expect != null)
+        {
+            var expected = File.ReadAllText(expect);
+            var actual = JsonSerializer.Serialize(symbols);
+            if (!JsonEquivalent(expected, actual))
+            {
+                Console.Error.WriteLine("Expected symbols did not match extracted symbols.");
+                Console.Error.WriteLine(actual);
+                return CommandExitCodes.InvalidArgument;
+            }
+        }
+
+        if (json || expect == null)
+            Console.WriteLine(JsonSerializer.Serialize(symbols));
+        return CommandExitCodes.Success;
+    }
+
+    private static bool TryConsumeInlineOrNext(string[] args, ref int index, string arg, string flag, out string value)
+    {
+        value = string.Empty;
+        if (arg.StartsWith(flag + "=", StringComparison.Ordinal))
+        {
+            value = arg[(flag.Length + 1)..];
+            return true;
+        }
+
+        if (arg != flag || index + 1 >= args.Length)
+            return false;
+
+        value = args[++index];
+        return true;
+    }
+
+    private static bool JsonEquivalent(string expected, string actual)
+    {
+        using var expectedDoc = JsonDocument.Parse(expected);
+        using var actualDoc = JsonDocument.Parse(actual);
+        return JsonSerializer.Serialize(expectedDoc.RootElement) == JsonSerializer.Serialize(actualDoc.RootElement);
+    }
+
     internal static void EnsureRedirectedStdoutUsesUtf8()
     {
         if (!Console.IsOutputRedirected || Console.Out is StringWriter || Console.Out.GetType().Assembly != typeof(Console).Assembly)

tests/CodeIndex.Tests/WorkspaceCommandRunnerTests.cs

@@ -78,6 +78,17 @@ public void ConfigErrors_HonorJsonFlag()
         }
     }
 
+    [Fact]
+    public void ConfigShowErrors_HonorJsonFlag()
+    {
+        var (exitCode, stdout, stderr) = ConsoleCapture.Capture(() => CdidxConfigFile.RunShow(["extra", "--json"], _jsonOptions));
+
+        Assert.Equal(CommandExitCodes.UsageError, exitCode);
+        Assert.Contains("\"status\":\"error\"", stdout);
+        Assert.Contains("config show does not accept positional arguments", stdout);
+        Assert.DoesNotContain("config show does not accept positional arguments", stderr);
+    }
+
     [Fact]
     public void ConfigShow_PrintsPrecedence()
     {