Fix stale reference cleanup (#2700)

* Fix dangling reference line cleanup (#1781)

* Batch stale file deletes (#1826)

* Clean stale symbol references (#1785)

Author: Widthdom

changelog.d/unreleased/1781.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+issues:
+  - 1781
+affected:
+  - src/CodeIndex/Database/DbContext.cs
+  - tests/CodeIndex.Tests/DatabaseTests.cs
+---
+
+## English
+
+- **Reference context links are cleared instead of dangling after reference-line deletion (#1781)** — `symbol_references.reference_line_id` now uses `ON DELETE SET NULL`, and existing indexes are migrated by nulling already-dangling line-context pointers before rebuilding the table constraint.
+
+## 日本語
+
+- **reference-line 削除後の参照コンテキストリンクが dangling ではなく NULL 化されるようになりました (#1781)** — `symbol_references.reference_line_id` は `ON DELETE SET NULL` を使い、既存 index は既に dangling になっている line-context pointer を NULL 化してからテーブル制約を再構築します。

changelog.d/unreleased/1785.fixed.md

@@ -0,0 +1,16 @@
+---
+category: fixed
+issues:
+  - 1785
+affected:
+  - src/CodeIndex/Database/DbWriter.cs
+  - tests/CodeIndex.Tests/DatabaseTests.cs
+---
+
+## English
+
+- **Stale file purges remove cross-file references to symbols defined only by the purged files (#1785)** — cleanup now drops phantom symbol edges when a deleted file was the only remaining definition for the referenced name.
+
+## 日本語
+
+- **stale file purge が purge 対象ファイルにしか定義が残っていないシンボルへの cross-file reference を削除するようになりました (#1785)** — 削除済みファイルが参照名の唯一の定義だった場合、phantom symbol edge を cleanup で取り除きます。

changelog.d/unreleased/1826.fixed.md

@@ -0,0 +1,15 @@
+---
+category: fixed
+issues:
+  - 1826
+affected:
+  - src/CodeIndex/Database/DbWriter.cs
+---
+
+## English
+
+- **Stale file purges delete file rows in chunked batches (#1826)** — purge paths now issue chunked `DELETE ... IN (...)` statements instead of one delete statement per stale file.
+
+## 日本語
+
+- **stale file purge が file row を chunked batch で削除するようになりました (#1826)** — purge 経路は stale file ごとに個別の DELETE を発行せず、chunked `DELETE ... IN (...)` を使います。

src/CodeIndex/Database/DbContext.cs

@@ -1354,7 +1354,7 @@ reference_kind  TEXT CHECK (reference_kind IN (" + referenceKindCheck + @")),
                 line            INTEGER,
                 column_number   INTEGER,
                 context         TEXT,
-                reference_line_id INTEGER REFERENCES reference_lines(id),
+                reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL,
                 container_kind  TEXT CHECK (container_kind IS NULL OR container_kind IN (" + symbolKindCheck + @")),
                 container_name  TEXT
             )");
@@ -1403,7 +1403,7 @@ value  TEXT
         EnsureColumn(
             "symbol_references",
             "reference_line_id",
-            rebuildsSymbolReferences ? "INTEGER" : "INTEGER REFERENCES reference_lines(id)");
+            rebuildsSymbolReferences ? "INTEGER" : "INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL");
         // #86: Unicode-aware folded name columns for `--exact` name matching across all
         // `--exact` command variants. Populated by the writer via NameFold.Fold; NULL on
         // legacy rows until a full reindex, in which case the reader falls back to the
@@ -1415,6 +1415,7 @@ value  TEXT
         EnsureColumn("symbol_references", "is_self_reference", "INTEGER NOT NULL DEFAULT 0");
         EnsureColumn("symbol_references", "is_mutual_recursion", "INTEGER NOT NULL DEFAULT 0");
         EnforceRequiredFileIdConstraints();
+        EnforceReferenceLineSetNullConstraint();
         EnsureReferenceLinesContextKey();
 
         // Indexes / インデックス
@@ -1513,6 +1514,7 @@ CREATE TRIGGER IF NOT EXISTS fts_chunks_au AFTER UPDATE ON chunks BEGIN
 
     private void EnforceRequiredFileIdConstraints()
     {
+        var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds);
         Execute("PRAGMA foreign_keys=OFF");
         var legacyAlterTable = ExecuteScalar("PRAGMA legacy_alter_table");
         Execute("PRAGMA legacy_alter_table=ON");
@@ -1534,11 +1536,11 @@ file_id     INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
                 "id, file_id, chunk_index, start_line, end_line, content");
             RebuildTableWithRequiredFileId(
                 "symbols",
-                """
+                $"""
                 CREATE TABLE symbols (
                     id              INTEGER PRIMARY KEY AUTOINCREMENT,
                     file_id         INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
-                    kind            TEXT,
+                    kind            TEXT CHECK (kind IN ({symbolKindCheck})),
                     sub_kind        TEXT,
                     name            TEXT,
                     line            INTEGER,
@@ -1548,7 +1550,7 @@ file_id         INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
                     body_start_line INTEGER,
                     body_end_line   INTEGER,
                     signature       TEXT,
-                    container_kind  TEXT,
+                    container_kind  TEXT CHECK (container_kind IS NULL OR container_kind IN ({symbolKindCheck})),
                     container_name  TEXT,
                     container_qualified_name TEXT,
                     family_key      TEXT,
@@ -1588,6 +1590,8 @@ private void RebuildReferenceLineTablesWithRequiredFileId()
             return;
         }
 
+        var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds);
+        var referenceKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.ReferenceKinds);
         const string referenceLinesCreateSql =
             """
             CREATE TABLE reference_lines (
@@ -1599,18 +1603,18 @@ file_id     INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
             )
             """;
         const string referenceLinesColumns = "id, file_id, line, context";
-        const string symbolReferencesCreateSql =
-            """
+        var symbolReferencesCreateSql =
+            $"""
             CREATE TABLE symbol_references (
                 id              INTEGER PRIMARY KEY AUTOINCREMENT,
                 file_id         INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
                 symbol_name     TEXT,
-                reference_kind  TEXT,
+                reference_kind  TEXT CHECK (reference_kind IN ({referenceKindCheck})),
                 line            INTEGER,
                 column_number   INTEGER,
                 context         TEXT,
-                reference_line_id INTEGER REFERENCES reference_lines(id),
-                container_kind  TEXT,
+                reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL,
+                container_kind  TEXT CHECK (container_kind IS NULL OR container_kind IN ({symbolKindCheck})),
                 container_name  TEXT,
                 symbol_name_folded TEXT,
                 container_name_folded TEXT,
@@ -1636,11 +1640,81 @@ is_mutual_recursion INTEGER NOT NULL DEFAULT 0
         Execute($"DROP TABLE {oldReferenceLines}");
     }
 
+    private void EnforceReferenceLineSetNullConstraint()
+    {
+        if (SymbolReferencesReferenceLineDeletesSetNull())
+            return;
+
+        var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds);
+        var referenceKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.ReferenceKinds);
+        var symbolReferencesCreateSql =
+            $"""
+            CREATE TABLE symbol_references (
+                id              INTEGER PRIMARY KEY AUTOINCREMENT,
+                file_id         INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+                symbol_name     TEXT,
+                reference_kind  TEXT CHECK (reference_kind IN ({referenceKindCheck})),
+                line            INTEGER,
+                column_number   INTEGER,
+                context         TEXT,
+                reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL,
+                container_kind  TEXT CHECK (container_kind IS NULL OR container_kind IN ({symbolKindCheck})),
+                container_name  TEXT,
+                symbol_name_folded TEXT,
+                container_name_folded TEXT,
+                is_self_reference INTEGER NOT NULL DEFAULT 0,
+                is_mutual_recursion INTEGER NOT NULL DEFAULT 0
+            )
+            """;
+        const string symbolReferencesColumns = "id, file_id, symbol_name, reference_kind, line, column_number, context, reference_line_id, container_kind, container_name, symbol_name_folded, container_name_folded, is_self_reference, is_mutual_recursion";
+        const string oldSymbolReferences = "_symbol_references_reference_line_delete";
+
+        Execute($"DROP TABLE IF EXISTS {oldSymbolReferences}");
+        Execute(@"
+            UPDATE symbol_references
+            SET reference_line_id = NULL
+            WHERE reference_line_id IS NOT NULL
+              AND NOT EXISTS (
+                  SELECT 1
+                  FROM reference_lines
+                  WHERE reference_lines.id = symbol_references.reference_line_id
+              )");
+        Execute($"ALTER TABLE symbol_references RENAME TO {oldSymbolReferences}");
+        Execute(symbolReferencesCreateSql);
+        Execute($"INSERT INTO symbol_references ({symbolReferencesColumns}) SELECT {symbolReferencesColumns} FROM {oldSymbolReferences}");
+        Execute($"DROP TABLE {oldSymbolReferences}");
+    }
+
+    private bool SymbolReferencesReferenceLineDeletesSetNull()
+    {
+        using var cmd = _connection.CreateCommand();
+        if (_activeMigrationTransaction != null)
+            cmd.Transaction = _activeMigrationTransaction;
+        cmd.CommandText = "PRAGMA foreign_key_list('symbol_references')";
+
+        using var reader = cmd.ExecuteTrackedReader();
+        while (reader.TrackedRead())
+        {
+            var table = reader.GetString(2);
+            var from = reader.GetString(3);
+            var onDelete = reader.GetString(6);
+            if (string.Equals(table, "reference_lines", StringComparison.OrdinalIgnoreCase)
+                && string.Equals(from, "reference_line_id", StringComparison.OrdinalIgnoreCase))
+            {
+                return string.Equals(onDelete, "SET NULL", StringComparison.OrdinalIgnoreCase);
+            }
+        }
+
+        return false;
+    }
+
     private void EnsureReferenceLinesContextKey()
     {
         if (ReferenceLinesHasContextUniqueKey())
             return;
 
+        var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds);
+        var referenceKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.ReferenceKinds);
         const string referenceLinesCreateSql =
             """
             CREATE TABLE reference_lines (
@@ -1652,18 +1726,18 @@ file_id     INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
             )
             """;
         const string referenceLinesColumns = "id, file_id, line, context";
-        const string symbolReferencesCreateSql =
-            """
+        var symbolReferencesCreateSql =
+            $"""
             CREATE TABLE symbol_references (
                 id              INTEGER PRIMARY KEY AUTOINCREMENT,
                 file_id         INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
                 symbol_name     TEXT,
-                reference_kind  TEXT,
+                reference_kind  TEXT CHECK (reference_kind IN ({referenceKindCheck})),
                 line            INTEGER,
                 column_number   INTEGER,
                 context         TEXT,
-                reference_line_id INTEGER REFERENCES reference_lines(id),
-                container_kind  TEXT,
+                reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL,
+                container_kind  TEXT CHECK (container_kind IS NULL OR container_kind IN ({symbolKindCheck})),
                 container_name  TEXT,
                 symbol_name_folded TEXT,
                 container_name_folded TEXT,
@@ -1969,14 +2043,14 @@ file_id         INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
                 line            INTEGER,
                 column_number   INTEGER,
                 context         TEXT,
-                reference_line_id INTEGER REFERENCES reference_lines(id),
+                reference_line_id INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL,
                 container_kind  TEXT,
                 container_name  TEXT,
                 is_self_reference INTEGER NOT NULL DEFAULT 0,
                 is_mutual_recursion INTEGER NOT NULL DEFAULT 0
             )"));
         yield return ("EnsureColumn symbol_references.reference_line_id",
-            () => EnsureColumn("symbol_references", "reference_line_id", "INTEGER REFERENCES reference_lines(id)"));
+            () => EnsureColumn("symbol_references", "reference_line_id", "INTEGER REFERENCES reference_lines(id) ON DELETE SET NULL"));
         yield return ("EnsureColumn symbol_references.is_self_reference",
             () => EnsureColumn("symbol_references", "is_self_reference", "INTEGER NOT NULL DEFAULT 0"));
         yield return ("EnsureColumn symbol_references.is_mutual_recursion",

src/CodeIndex/Database/DbWriter.cs

@@ -28,6 +28,7 @@ public class DbWriter
     private readonly SemaphoreSlim _transactionGate = new(1, 1);
     private readonly AsyncLocal<Guid?> _currentTransactionGateToken = new();
     private const int BatchSize = 500;
+    private const int DeleteFilesBatchSize = 500;
     private const int MaxSqlVariables = 999;
     private const int SqliteConstraintErrorCode = 19;
     private int _rowSkipSavepointCounter;
@@ -617,22 +618,7 @@ public int PurgeStaleFilesSharingChecksum(string projectRoot, string retainedRel
             ReleaseCommand(cmd);
         }
 
-        if (staleIds.Count == 0)
-            return 0;
-
-        using var txn = !IsInTransaction() ? BeginTransaction() : null;
-        using var deleteCmd = _conn.CreateCommand();
-        deleteCmd.CommandText = "DELETE FROM files WHERE id = @id";
-        var pId = deleteCmd.Parameters.Add("@id", SqliteType.Integer);
-        deleteCmd.Prepare();
-        foreach (var id in staleIds)
-        {
-            pId.Value = id;
-            deleteCmd.ExecuteNonQuery();
-        }
-        txn?.Commit();
-
-        return staleIds.Count;
+        return DeleteStaleFileIds(staleIds);
     }
 
     /// <summary>
@@ -677,18 +663,76 @@ private int DeleteStaleFileIds(IReadOnlyCollection<long> staleIds)
             return 0;
 
         using var txn = !IsInTransaction() ? BeginTransaction() : null;
+        DeleteFilesByIdBatched(staleIds);
+        txn?.Commit();
+
+        return staleIds.Count;
+    }
+
+    private void DeleteFilesByIdBatched(IEnumerable<long> fileIds, int batchSize = DeleteFilesBatchSize)
+    {
+        var batch = new List<long>(batchSize);
+        foreach (var id in fileIds)
+        {
+            batch.Add(id);
+            if (batch.Count == batchSize)
+            {
+                DeleteFileIdBatch(batch);
+                batch.Clear();
+            }
+        }
+
+        if (batch.Count > 0)
+            DeleteFileIdBatch(batch);
+    }
+
+    private void DeleteFileIdBatch(IReadOnlyList<long> fileIds)
+    {
+        DeleteCrossFileReferencesToSymbolsDefinedOnlyByFiles(fileIds);
+
         using var deleteCmd = _conn.CreateCommand();
-        deleteCmd.CommandText = "DELETE FROM files WHERE id = @id";
-        var pId = deleteCmd.Parameters.Add("@id", SqliteType.Integer);
-        deleteCmd.Prepare();
-        foreach (var id in staleIds)
+        var parameters = new List<string>(fileIds.Count);
+        for (var i = 0; i < fileIds.Count; i++)
         {
-            pId.Value = id;
-            deleteCmd.ExecuteNonQuery();
+            var parameterName = $"@id{i}";
+            parameters.Add(parameterName);
+            deleteCmd.Parameters.Add(parameterName, SqliteType.Integer).Value = fileIds[i];
         }
-        txn?.Commit();
 
-        return staleIds.Count;
+        deleteCmd.CommandText = $"DELETE FROM files WHERE id IN ({string.Join(", ", parameters)})";
+        deleteCmd.ExecuteNonQuery();
+    }
+
+    private void DeleteCrossFileReferencesToSymbolsDefinedOnlyByFiles(IReadOnlyList<long> fileIds)
+    {
+        using var deleteCmd = _conn.CreateCommand();
+        var parameters = new List<string>(fileIds.Count);
+        for (var i = 0; i < fileIds.Count; i++)
+        {
+            var parameterName = $"@id{i}";
+            parameters.Add(parameterName);
+            deleteCmd.Parameters.Add(parameterName, SqliteType.Integer).Value = fileIds[i];
+        }
+
+        var idList = string.Join(", ", parameters);
+        deleteCmd.CommandText = $@"
+            DELETE FROM symbol_references
+            WHERE file_id NOT IN ({idList})
+              AND symbol_name IS NOT NULL
+              AND symbol_name <> ''
+              AND EXISTS (
+                  SELECT 1
+                  FROM symbols deleted_symbols
+                  WHERE deleted_symbols.file_id IN ({idList})
+                    AND deleted_symbols.name = symbol_references.symbol_name
+              )
+              AND NOT EXISTS (
+                  SELECT 1
+                  FROM symbols retained_symbols
+                  WHERE retained_symbols.file_id NOT IN ({idList})
+                    AND retained_symbols.name = symbol_references.symbol_name
+              )";
+        deleteCmd.ExecuteNonQuery();
     }
 
     private static string GetRelativeDirectory(string relativePath)