Raise the index file size cap to 500KB, and say when a file is skipped

src/config/symbolIndexer.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect, vi, beforeAll } from 'vitest';
 import { SymbolIndexer } from './symbolIndexer.js';
 import { workspace } from 'vscode';
+import { logger } from '../system/logger.js';
 import { getAnalyzer, setGrammarsPath } from '../parsing/registry.js';
 import { grammarsDir, hasGrammars } from '../parsing/grammarsTestSupport.js';
 
@@ -261,6 +262,47 @@ describe('SymbolIndexer', () => {
       expect(removeFileSpy).toHaveBeenCalledWith('src/gone.ts');
     });
 
+    it('skips a file over the size limit and says so', async () => {
+      // The skip used to be silent, so a file invisible to find_references gave
+      // the user nothing to go on — the same shape as the scan truncation in
+      // #40, smaller. Raising the limit only moves the cliff; naming it is what
+      // makes falling off it observable.
+      const body = `export function tiny() { return 1; }\n` + 'x'.repeat(600 * 1024);
+      vi.spyOn(workspace, 'findFiles').mockResolvedValue([{ fsPath: '/mock-workspace/src/huge.ts' }] as never);
+      vi.spyOn(workspace.fs, 'stat').mockResolvedValue({ type: 1, size: body.length, mtime: Date.now() } as never);
+      vi.spyOn(workspace.fs, 'readFile').mockResolvedValue(Buffer.from(body) as never);
+      const warn = vi.spyOn(logger, 'warn').mockImplementation(() => {});
+
+      const indexer = new SymbolIndexer(null);
+      await indexer.initialize(['**/*.ts']);
+
+      expect(indexer.getGraph().symbolCount()).toBe(0);
+      const msg = warn.mock.calls.map((c) => String(c[0])).join('\n');
+      expect(msg).toContain('src/huge.ts');
+      expect(msg).toContain('500 KB');
+
+      vi.restoreAllMocks();
+    });
+
+    it('indexes a file that would have exceeded the OLD 100 KB limit', async () => {
+      // The reason for raising it. This repo's p99 is 41 KB, but generated
+      // TypeScript users legitimately want indexed — GraphQL types, protobuf
+      // output, OpenAPI clients — routinely passes 100 KB.
+      const body = `export function realSymbol() { return 1; }\n` + '// pad\n'.repeat(30 * 1024);
+      expect(body.length).toBeGreaterThan(100 * 1024);
+      expect(body.length).toBeLessThan(500 * 1024);
+      vi.spyOn(workspace, 'findFiles').mockResolvedValue([{ fsPath: '/mock-workspace/src/generated.ts' }] as never);
+      vi.spyOn(workspace.fs, 'stat').mockResolvedValue({ type: 1, size: body.length, mtime: Date.now() } as never);
+      vi.spyOn(workspace.fs, 'readFile').mockResolvedValue(Buffer.from(body) as never);
+
+      const indexer = new SymbolIndexer(null);
+      await indexer.initialize(['**/*.ts']);
+
+      expect(indexer.getGraph().symbolCount()).toBeGreaterThan(0);
+
+      vi.restoreAllMocks();
+    });
+
     it('respects maxSymbolsPerFile when capping large files', async () => {
       // Generate a file with 10 exported functions and cap to 3.
       const body = Array.from({ length: 10 }, (_, i) => `export function fn${i}() { return ${i}; }\n`).join('\n');

src/config/symbolIndexer.ts

@@ -26,7 +26,22 @@ import {
 } from './indexExcludes.js';
 
 const CACHE_FILE = 'cache/symbol-graph.json';
-const MAX_FILE_SIZE = 100 * 1024; // 100KB
+/**
+ * Largest file the symbol indexer will parse.
+ *
+ * Was 100 KB, which is low for real source: this repo's own p99 is 41 KB, but
+ * generated TypeScript that users legitimately want indexed — GraphQL types,
+ * protobuf output, OpenAPI clients, large const tables — routinely passes it.
+ * The pathological files a cap exists to stop (minified bundles, vendored
+ * trees) mostly sit in directories INDEX_EXCLUDE_DIRS already removes, so the
+ * cap was doing less protective work than its size suggested.
+ *
+ * Measured with tree-sitter, roughly linear: 136 KB → 81ms, 272 KB → 130ms,
+ * 680 KB → 289ms. 500 KB costs ~215ms worst case against a full index, which
+ * is affordable. Past that, skipping is genuinely right — a multi-MB `.ts` is
+ * almost certainly generated or minified and its symbols would be noise.
+ */
+const MAX_FILE_SIZE = 500 * 1024; // 500KB
 const MAX_JSON_SIZE = 50 * 1024 * 1024; // 50MB persistence limit
 
 /** Outcome of `replaySymbolsToEmbeddingIndex` — used to log real numbers (and
@@ -153,7 +168,19 @@ export class SymbolIndexer implements Disposable {
         if (restored && this.graph.getFileHash(relativePath) === hash) return;
         const bytes = await workspace.fs.readFile(uri);
         const content = Buffer.from(bytes).toString('utf-8');
-        if (content.length > MAX_FILE_SIZE) return;
+        if (content.length > MAX_FILE_SIZE) {
+          // Say so. The skip used to be silent, so a user whose file was
+          // invisible to find_references had no way to discover why — the same
+          // shape as the scan truncation in #40, just smaller. Raising the cap
+          // only moves the cliff; naming it is what makes falling off it
+          // observable.
+          logger.warn(
+            `[SideCar] Not indexing ${relativePath} — ${Math.round(content.length / 1024)} KB exceeds the ` +
+              `${MAX_FILE_SIZE / 1024} KB limit. Its symbols will be absent from find_references, ` +
+              `analyze_impact and PKI retrieval.`,
+          );
+          return;
+        }
         await this.indexFile(relativePath, content, hash);
         parsed++;
       }),