Merge pull request #2737 from Widthdom/codex/fix-issue1664-1824

Fix Elixir pipe and defimpl indexing

Author: Widthdom

DEVELOPER_GUIDE.md

@@ -223,6 +223,7 @@ Do not add mutable static caches, shared `StringBuilder` instances, reused `Matc
 | `procedure` | Procedure declarations in languages such as Fortran | Callable definition |
 | `program` | Program block declarations in languages such as Fortran | Definition target and container |
 | `protocol` | Protocol declarations in languages that distinguish protocols from interfaces | Definition target and container |
+| `protocol_impl` | Elixir `defimpl` protocol implementation declarations | Definition target and container for implementation blocks |
 | `reference` | Secondary extracted symbolic references, such as HTML classes or metadata keys | Search/filter symbol |
 | `rule` | CSS/SCSS rule container context used by nested references | Container context |
 | `route` | Razor route directives | Context/search symbol |

changelog.d/unreleased/1664.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+issues:
+  - 1664
+affected:
+  - src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs
+  - tests/CodeIndex.Tests/ReferenceExtractorTests.cs
+---
+
+## English
+
+- **Elixir pipe chains now preserve call targets (#1664)** — `|>` calls to both `Module.function(...)` and local `function(...)` targets are indexed as call references.
+
+## 日本語
+
+- **Elixir pipe chain の呼び出し先を保持するようになりました (#1664)** — `|>` による `Module.function(...)` とローカル `function(...)` の両方を call reference として index します。

changelog.d/unreleased/1824.fixed.md

@@ -0,0 +1,20 @@
+---
+category: fixed
+issues:
+  - 1824
+affected:
+  - DEVELOPER_GUIDE.md
+  - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs
+  - src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs
+  - src/CodeIndex/Models/SymbolKindCatalog.cs
+  - tests/CodeIndex.Tests/SymbolExtractorTests.cs
+  - tests/CodeIndex.Tests/ReferenceExtractorTests.cs
+---
+
+## English
+
+- **Elixir protocol implementations are now indexed (#1824)** — `defimpl Protocol, for: Type` blocks produce `protocol_impl` symbols and type references for both the protocol and implemented type names.
+
+## 日本語
+
+- **Elixir protocol implementation を index するようになりました (#1824)** — `defimpl Protocol, for: Type` block は `protocol_impl` symbol と、protocol/type 双方への type reference を生成します。

src/CodeIndex/Database/DbContext.cs

@@ -1568,6 +1568,7 @@ value  TEXT
         EnforceRequiredFileIdConstraints();
         EnforceReferenceLineSetNullConstraint();
         EnsureReferenceLinesContextKey();
+        EnsureKindCheckConstraintsCurrent();
 
         // Indexes / インデックス
         Execute("CREATE INDEX IF NOT EXISTS idx_files_lang     ON files(lang)");
@@ -1952,6 +1953,104 @@ private bool ReferenceLinesHasContextUniqueKey()
         return false;
     }
 
+    private void EnsureKindCheckConstraintsCurrent()
+    {
+        var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds);
+        var referenceKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.ReferenceKinds);
+        var symbolsCreateSql =
+            $"""
+            CREATE TABLE symbols (
+                id              INTEGER PRIMARY KEY AUTOINCREMENT,
+                file_id         INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+                kind            TEXT CHECK (kind IN ({symbolKindCheck})),
+                sub_kind        TEXT,
+                name            TEXT,
+                line            INTEGER,
+                start_line      INTEGER,
+                start_column    INTEGER,
+                end_line        INTEGER,
+                body_start_line INTEGER,
+                body_end_line   INTEGER,
+                signature       TEXT,
+                container_kind  TEXT CHECK (container_kind IS NULL OR container_kind IN ({symbolKindCheck})),
+                container_name  TEXT,
+                container_qualified_name TEXT,
+                family_key      TEXT,
+                visibility      TEXT,
+                return_type     TEXT,
+                is_metadata_target INTEGER,
+                name_folded     TEXT
+            )
+            """;
+        const string symbolsColumns = "id, file_id, kind, sub_kind, name, line, start_line, start_column, end_line, body_start_line, body_end_line, signature, container_kind, container_name, container_qualified_name, family_key, visibility, return_type, is_metadata_target, name_folded";
+        var symbolReferencesCreateSql =
+            $"""
+            CREATE TABLE symbol_references (
+                id              INTEGER PRIMARY KEY AUTOINCREMENT,
+                file_id         INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+                symbol_name     TEXT,
+                reference_kind  TEXT CHECK (reference_kind IN ({referenceKindCheck})),
+                line            INTEGER,
+                column_number   INTEGER,
+                context         TEXT,
+                reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL,
+                container_kind  TEXT CHECK (container_kind IS NULL OR container_kind IN ({symbolKindCheck})),
+                container_name  TEXT,
+                symbol_name_folded TEXT,
+                container_name_folded TEXT,
+                is_self_reference INTEGER NOT NULL DEFAULT 0,
+                is_mutual_recursion INTEGER NOT NULL DEFAULT 0
+            )
+            """;
+        const string symbolReferencesColumns = "id, file_id, symbol_name, reference_kind, line, column_number, context, reference_line_id, container_kind, container_name, symbol_name_folded, container_name_folded, is_self_reference, is_mutual_recursion";
+
+        var foreignKeys = ReadPragmaLong("foreign_keys");
+        Execute("PRAGMA foreign_keys=OFF");
+        try
+        {
+            if (!TableCheckContainsAll("symbols", SymbolKindCatalog.SymbolKinds))
+                RebuildTableWithCurrentKindChecks("symbols", "_symbols_kind_check", symbolsCreateSql, symbolsColumns);
+
+            if (!TableCheckContainsAll("symbol_references", SymbolKindCatalog.SymbolKinds.Concat(SymbolKindCatalog.ReferenceKinds)))
+                RebuildTableWithCurrentKindChecks("symbol_references", "_symbol_references_kind_check", symbolReferencesCreateSql, symbolReferencesColumns);
+        }
+        finally
+        {
+            Execute($"PRAGMA foreign_keys={foreignKeys}");
+        }
+    }
+
+    private bool TableCheckContainsAll(string tableName, IEnumerable<string> allowedValues)
+    {
+        var createSql = GetTableCreateSql(tableName);
+        if (createSql == null)
+            return true;
+
+        if (!createSql.Contains("CHECK", StringComparison.OrdinalIgnoreCase))
+            return true;
+
+        return allowedValues.All(value => createSql.Contains($"'{value.Replace("'", "''")}'", StringComparison.Ordinal));
+    }
+
+    private string? GetTableCreateSql(string tableName)
+    {
+        using var cmd = _connection.CreateCommand();
+        if (_activeMigrationTransaction != null)
+            cmd.Transaction = _activeMigrationTransaction;
+        cmd.CommandText = "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = @table";
+        cmd.Parameters.AddWithValue("@table", tableName);
+        return cmd.ExecuteScalar() as string;
+    }
+
+    private void RebuildTableWithCurrentKindChecks(string tableName, string oldTableName, string createSql, string columns)
+    {
+        Execute($"DROP TABLE IF EXISTS {oldTableName}");
+        Execute($"ALTER TABLE {tableName} RENAME TO {oldTableName}");
+        Execute(createSql);
+        Execute($"INSERT INTO {tableName} ({columns}) SELECT {columns} FROM {oldTableName}");
+        Execute($"DROP TABLE {oldTableName}");
+    }
+
     private void RebuildTableWithRequiredFileId(string tableName, string createSql, string columns)
     {
         if (ColumnIsNotNull(tableName, "file_id"))

src/CodeIndex/Indexer/References/Languages/ElixirReferenceExtractor.cs

@@ -1,9 +1,37 @@
 using CodeIndex.Models;
+using System.Text.RegularExpressions;
 
 namespace CodeIndex.Indexer;
 
 internal static class ElixirReferenceExtractor
 {
+    private static readonly Regex PipeCallRegex = new(
+        @"\|>\s*(?:(?:[A-Z_]\w*(?:\.[A-Z_]\w*)*)\.)?(?<name>[a-z_]\w*[!?]?)\s*(?=\(|$)",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant);
+
+    private static readonly Regex DefimplRegex = new(
+        @"^\s*defimpl\s+(?<protocol>[\w.]+)\s*,\s*for:\s*(?<types>\[[^\]]+\]|[\w.{}]+)",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant);
+
+    private static readonly Regex DefimplTypeRegex = new(
+        @"[\w.{}]+",
+        RegexOptions.Compiled | RegexOptions.CultureInvariant);
+
+    private static readonly HashSet<string> IgnoredPipeCallNames = new(StringComparer.Ordinal)
+    {
+        "and",
+        "catch",
+        "do",
+        "else",
+        "end",
+        "fn",
+        "in",
+        "not",
+        "or",
+        "rescue",
+        "when",
+    };
+
     public static void EmitTypePositionReferences(
         string preparedLine,
         List<ReferenceRecord> references,
@@ -13,6 +41,8 @@ public static void EmitTypePositionReferences(
         int lineNumber,
         SymbolRecord? container)
     {
+        EmitDefimplReferences(preparedLine, references, seen, fileId, context, lineNumber, container);
+
         LanguageReferenceExtractionSupport.EmitTypePositionReferences(
             "elixir",
             preparedLine,
@@ -26,11 +56,64 @@ public static void EmitTypePositionReferences(
             container);
     }
 
+    private static void EmitDefimplReferences(
+        string preparedLine,
+        List<ReferenceRecord> references,
+        HashSet<string> seen,
+        long fileId,
+        string context,
+        int lineNumber,
+        SymbolRecord? container)
+    {
+        var match = DefimplRegex.Match(preparedLine);
+        if (!match.Success)
+            return;
+
+        AddDefimplGroupReference(match.Groups["protocol"]);
+
+        var typesGroup = match.Groups["types"];
+        foreach (Match typeMatch in DefimplTypeRegex.Matches(typesGroup.Value))
+            AddDefimplTypeReference(typeMatch.Groups[0], typesGroup.Index + typeMatch.Index);
+
+        void AddDefimplGroupReference(Group group)
+            => AddDefimplTypeReference(group, group.Index);
+
+        void AddDefimplTypeReference(Group group, int column)
+        {
+            var name = group.Value.Trim();
+            if (name.Length == 0)
+                return;
+
+            var key = $"type_reference:{name}:{lineNumber}:{column}";
+            if (!seen.Add(key))
+                return;
+
+            references.Add(new ReferenceRecord
+            {
+                FileId = fileId,
+                SymbolName = name,
+                ReferenceKind = "type_reference",
+                Line = lineNumber,
+                Column = column,
+                Context = context.Trim(),
+                ContainerKind = container?.Kind,
+                ContainerName = container?.Name,
+            });
+        }
+    }
+
     public static void EmitAdditionalCallReferences(
         string preparedLine,
         Action<string, int> addCallLikeReference,
         IReadOnlySet<string>? definitionNames)
     {
+        foreach (Match match in PipeCallRegex.Matches(preparedLine))
+        {
+            var name = match.Groups["name"].Value;
+            if (!IgnoredPipeCallNames.Contains(name))
+                addCallLikeReference(name, match.Groups["name"].Index);
+        }
+
         LanguageReferenceExtractionSupport.EmitAdditionalCallReferences(
             "elixir",
             preparedLine,

src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs

@@ -1743,6 +1743,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult
             new("function", new Regex(@"^\s*(?:def|defp|defmacro|defguardp?)\s+(?<name>\w+)", RegexOptions.Compiled), BodyStyle.ElixirEnd),
             new("class",    new Regex(@"^\s*defmodule\s+(?<name>[\w.]+)", RegexOptions.Compiled), BodyStyle.ElixirEnd),
             new("interface", new Regex(@"^\s*defprotocol\s+(?<name>[\w.]+)", RegexOptions.Compiled), BodyStyle.ElixirEnd),
+            new("protocol_impl", new Regex(@"^\s*defimpl\s+(?<name>[\w.]+(?:\s*,\s*for:\s*(?:\[[^\]]+\]|[\w.{}]+))?)", RegexOptions.Compiled), BodyStyle.ElixirEnd),
             new("import",   new Regex(@"^\s*(?:import|alias|use|require)\s+(?<name>[\w.]+)", RegexOptions.Compiled), BodyStyle.None),
         ],
         ["dart"] =
@@ -2115,7 +2116,7 @@ public static IReadOnlyCollection<string> GetSupportedLanguages()
 
     private static readonly HashSet<string> ContainerKinds =
     [
-        "class", "struct", "interface", "protocol", "namespace", "enum", "object", "heading", "specialization", "class_hook"
+        "class", "struct", "interface", "protocol", "protocol_impl", "namespace", "enum", "object", "heading", "specialization", "class_hook"
     ];
 
     private static bool IsRustDirectTraitBodyMember(List<SymbolRecord> symbols, int candidateLine)

src/CodeIndex/Models/SymbolKindCatalog.cs

@@ -42,6 +42,7 @@ public static class SymbolKindCatalog
         "procedure",
         "program",
         "protocol",
+        "protocol_impl",
         "reference",
         "rule",
         "route",