scanner.ts

import { listFiles } from "../core-shared.js"
import { ShoferIgnoreController } from "../core-shared.js"
import { stat } from "fs/promises"
import * as path from "path"
import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../engine/shared/get-relative-path.js"
import { getWorkspacePathForContext } from "../core-shared.js"
import { scannerExtensions } from "../engine/shared/supported-extensions.js"
import { CodeBlock, ICodeParser, IEmbedder, IVectorStore, IDirectoryScanner } from "../engine/interfaces/index.js"
import { createHash } from "crypto"
import { v5 as uuidv5 } from "uuid"
import pLimit from "p-limit"
import { Mutex } from "async-mutex"
import { CacheManager } from "../cache-manager"
import { t } from "../i18n.js"
import {
	QDRANT_CODE_BLOCK_NAMESPACE,
	MAX_FILE_SIZE_BYTES,
	MAX_LIST_FILES_LIMIT_CODE_INDEX,
	MAX_BATCH_BYTES,
	MAX_BATCH_RETRIES,
	INITIAL_RETRY_DELAY_MS,
	PARSING_CONCURRENCY,
	BATCH_PROCESSING_CONCURRENCY,
	MAX_PENDING_BATCHES,
} from "../engine/constants/index.js"
import { BATCH_SEGMENT_THRESHOLD } from "../engine/constants/index.js"
import { isPathInIgnoredDirectory } from "../core-shared.js"
import type { IIgnoreFilter } from "../engine/shared/git-ignore-filter.js"
import { sanitizeErrorMessage } from "../engine/shared/validation-helpers.js"
import { codeIndexLog } from "../logging.js"
import { incCodeIndexError, recordSegmentDedup, setting } from "../plugin-runtime.js"
import fs from "fs/promises"
export class DirectoryScanner implements IDirectoryScanner {
	private readonly batchSegmentThreshold: number
	private shoferIgnoreController: ShoferIgnoreController | undefined

	constructor(
		private readonly embedder: IEmbedder,
		private readonly qdrantClient: IVectorStore,
		private readonly codeParser: ICodeParser,
		private readonly cacheManager: CacheManager,
		private readonly ignoreInstance: IIgnoreFilter,
		batchSegmentThreshold?: number,
		shoferIgnoreController?: ShoferIgnoreController,
	) {
		this.shoferIgnoreController = shoferIgnoreController
		// Get the configurable batch size from VSCode settings, fallback to default
		// If not provided in constructor, try to get from VSCode settings
		if (batchSegmentThreshold !== undefined) {
			this.batchSegmentThreshold = batchSegmentThreshold
		} else {
			this.batchSegmentThreshold = setting("embeddingBatchSize", BATCH_SEGMENT_THRESHOLD)
		}
	}

	/**
	 * Recursively scans a directory for code blocks in supported files.
	 * @param directoryPath The directory to scan
	 * @param shoferIgnoreController Optional ShoferIgnoreController instance for filtering
	 * @param context VS Code ExtensionContext for cache storage
	 * @param onError Optional error handler callback
	 * @returns Promise<{codeBlocks: CodeBlock[], stats: {processed: number, skipped: number}}> Array of parsed code blocks and processing stats
	 */
	public async scanDirectory(
		directory: string,
		onError?: (error: Error) => void,
		onBlocksIndexed?: (indexedCount: number) => void,
		onFileParsed?: (fileBlockCount: number) => void,
		signal?: AbortSignal,
	): Promise<{ stats: { processed: number; skipped: number }; totalBlockCount: number }> {
		const directoryPath = directory
		// Capture workspace context at scan start
		const scanWorkspace = getWorkspacePathForContext(directoryPath)

		// Get all files recursively (handles .gitignore automatically)
		const [allPaths, _] = await listFiles(directoryPath, true, MAX_LIST_FILES_LIMIT_CODE_INDEX)

		// Filter out directories (marked with trailing '/')
		const filePaths = allPaths.filter((p) => !p.endsWith("/"))

		// Use cached ShoferIgnoreController or create one if not provided
		if (!this.shoferIgnoreController) {
			this.shoferIgnoreController = new ShoferIgnoreController(directoryPath)
			await this.shoferIgnoreController.initialize()
		}

		// Filter paths using .shoferignore
		const allowedPaths = this.shoferIgnoreController.filterPaths(filePaths)

		// Filter by supported extensions, ignore patterns, and excluded directories
		const supportedPaths = allowedPaths.filter((filePath) => {
			const ext = path.extname(filePath).toLowerCase()
			const relativeFilePath = generateRelativeFilePath(filePath, scanWorkspace)

			// Check if file is in an ignored directory using the shared helper
			// Use relative path to avoid matching parent directories outside the workspace
			if (isPathInIgnoredDirectory(relativeFilePath)) {
				return false
			}

			return scannerExtensions.includes(ext) && !this.ignoreInstance.ignores(relativeFilePath)
		})

		// Initialize tracking variables
		const processedFiles = new Set<string>()
		let processedCount = 0
		let skippedCount = 0

		// Initialize parallel processing tools
		const parseLimiter = pLimit(PARSING_CONCURRENCY) // Concurrency for file parsing
		const batchLimiter = pLimit(BATCH_PROCESSING_CONCURRENCY) // Concurrency for batch processing
		const mutex = new Mutex()

		// Shared batch accumulators (protected by mutex)
		let currentBatchBlocks: CodeBlock[] = []
		let currentBatchTexts: string[] = []
		let currentBatchBytes = 0
		let currentBatchFileInfos: {
			filePath: string
			fileHash: string
			isNew: boolean
			mtimeMs: number
			size: number
		}[] = []
		const activeBatchPromises = new Set<Promise<void>>()
		let pendingBatchCount = 0

		// Initialize block counter
		let totalBlockCount = 0

		// Process all files in parallel with concurrency control
		const parsePromises = supportedPaths.map((filePath) =>
			parseLimiter(async () => {
				// Check abort signal before processing each file
				if (signal?.aborted) return

				try {
					// Phase 1 fast-path: stat() only — skip read+hash when mtime+size match
					const stats = await stat(filePath)
					if (stats.size > MAX_FILE_SIZE_BYTES) {
						skippedCount++ // Skip large files
						return
					}

					const cached = this.cacheManager.getEntry(filePath)

					// Compare mtimes at integer-millisecond resolution. The scanner
					// reads Node's fractional `stats.mtimeMs` (e.g. ...553.164) while
					// the file watcher writes VS Code's integer `fileStat.mtime`
					// (...553); without flooring, every file the watcher last touched
					// would spuriously fail this fast-path and get re-read + re-hashed.
					if (
						cached &&
						Math.floor(cached.mtimeMs) === Math.floor(stats.mtimeMs) &&
						cached.size === stats.size
					) {
						// File unchanged — no read, no hash, no parse
						processedFiles.add(filePath)
						skippedCount++
						return
					}

					// Read file content
					const content = await fs.readFile(filePath, "utf-8")

					// Calculate current hash
					const currentFileHash = createHash("sha256").update(content).digest("hex")
					processedFiles.add(filePath)

					// Check against cache — hash match means content unchanged
					const isNewFile = !cached
					if (cached && cached.hash === currentFileHash) {
						// mtime changed but content identical (e.g. touch, rebase, rsync -t)
						this.cacheManager.updateEntry(filePath, {
							hash: currentFileHash,
							mtimeMs: stats.mtimeMs,
							size: stats.size,
							segmentHashes: cached.segmentHashes ?? [],
						})
						skippedCount++
						return
					}

					// File is new or changed - parse it using the injected parser function
					const blocks = await this.codeParser.parseFile(filePath, { content, fileHash: currentFileHash })
					const fileBlockCount = blocks.length
					onFileParsed?.(fileBlockCount)
					processedCount++

					// Process embeddings if configured
					if (this.embedder && this.qdrantClient && blocks.length > 0) {
						// Add to batch accumulators
						let addedBlocksFromFile = false
						for (const block of blocks) {
							const trimmedContent = block.content.trim()
							if (trimmedContent) {
								const release = await mutex.acquire()
								try {
									currentBatchBlocks.push(block)
									currentBatchTexts.push(trimmedContent)
									currentBatchBytes += Buffer.byteLength(trimmedContent, "utf8")
									addedBlocksFromFile = true

									// Check if batch threshold is met
									// Check abort signal before dispatching batch
									if (signal?.aborted) {
										throw new DOMException("Indexing aborted", "AbortError")
									}

									if (
										currentBatchBlocks.length >= this.batchSegmentThreshold ||
										currentBatchBytes >= MAX_BATCH_BYTES
									) {
										// Wait if we've reached the maximum pending batches
										while (pendingBatchCount >= MAX_PENDING_BATCHES) {
											if (signal?.aborted) {
												throw new DOMException("Indexing aborted", "AbortError")
											}
											await Promise.race(activeBatchPromises)
										}

										// Copy current batch data and clear accumulators
										const batchBlocks = [...currentBatchBlocks]
										const batchTexts = [...currentBatchTexts]
										const batchFileInfos = [...currentBatchFileInfos]
										currentBatchBlocks = []
										currentBatchTexts = []
										currentBatchBytes = 0
										currentBatchFileInfos = []

										// Increment pending batch count
										pendingBatchCount++

										// Queue batch processing
										const batchPromise = batchLimiter(() =>
											this.processBatch(
												batchBlocks,
												batchTexts,
												batchFileInfos,
												scanWorkspace,
												onError,
												onBlocksIndexed,
												signal,
											),
										)
										activeBatchPromises.add(batchPromise)

										// Clean up completed promises to prevent memory accumulation
										batchPromise.finally(() => {
											activeBatchPromises.delete(batchPromise)
											pendingBatchCount--
										})
									}
								} finally {
									release()
								}
							}
						}

						// Add file info once per file (outside the block loop)
						if (addedBlocksFromFile) {
							const release = await mutex.acquire()
							try {
								totalBlockCount += fileBlockCount
								currentBatchFileInfos.push({
									filePath,
									fileHash: currentFileHash,
									isNew: isNewFile,
									mtimeMs: stats.mtimeMs,
									size: stats.size,
								})
							} finally {
								release()
							}
						}
					} else {
						// Only update cache if not being processed in a batch
						this.cacheManager.updateEntry(filePath, {
							hash: currentFileHash,
							mtimeMs: stats.mtimeMs,
							size: stats.size,
							segmentHashes: [],
						})
					}
				} catch (error) {
					// Re-throw AbortError — it's not a file processing error, just a user-initiated stop
					if (error instanceof DOMException && error.name === "AbortError") {
						throw error
					}
					codeIndexLog.error(`Error processing file ${filePath} in workspace ${scanWorkspace}:`, error)
					incCodeIndexError("scanDirectory:processFile")
					if (onError) {
						onError(
							error instanceof Error
								? new Error(`${error.message} (Workspace: ${scanWorkspace}, File: ${filePath})`)
								: new Error(
										t("embeddings:scanner.unknownErrorProcessingFile", { filePath }) +
											` (Workspace: ${scanWorkspace})`,
									),
						)
					}
				}
			}),
		)

		// Wait for all parsing to complete
		await Promise.all(parsePromises)

		// Check abort signal before processing remaining batch
		if (signal?.aborted) {
			return {
				stats: {
					processed: processedCount,
					skipped: skippedCount,
				},
				totalBlockCount,
			}
		}

		// Process any remaining items in batch
		if (currentBatchBlocks.length > 0) {
			const release = await mutex.acquire()
			try {
				// Copy current batch data and clear accumulators
				const batchBlocks = [...currentBatchBlocks]
				const batchTexts = [...currentBatchTexts]
				const batchFileInfos = [...currentBatchFileInfos]
				currentBatchBlocks = []
				currentBatchTexts = []
				currentBatchBytes = 0
				currentBatchFileInfos = []

				// Increment pending batch count for final batch
				pendingBatchCount++

				// Queue final batch processing
				const batchPromise = batchLimiter(() =>
					this.processBatch(
						batchBlocks,
						batchTexts,
						batchFileInfos,
						scanWorkspace,
						onError,
						onBlocksIndexed,
						signal,
					),
				)
				activeBatchPromises.add(batchPromise)

				// Clean up completed promises to prevent memory accumulation
				batchPromise.finally(() => {
					activeBatchPromises.delete(batchPromise)
					pendingBatchCount--
				})
			} finally {
				release()
			}
		}

		// Wait for all batch processing to complete
		await Promise.all(activeBatchPromises)

		// Check abort signal before handling deleted files
		if (signal?.aborted) {
			return {
				stats: {
					processed: processedCount,
					skipped: skippedCount,
				},
				totalBlockCount,
			}
		}

		// Handle deleted files
		const oldPaths = this.cacheManager.getAllPaths()
		for (const cachedFilePath of oldPaths) {
			if (!processedFiles.has(cachedFilePath)) {
				// File was deleted or is no longer supported/indexed
				if (this.qdrantClient) {
					try {
						await this.qdrantClient.deletePointsByFilePath(cachedFilePath)
						await this.cacheManager.deleteHash(cachedFilePath)
					} catch (error: any) {
						const errorStatus = error?.status || error?.response?.status || error?.statusCode
						const errorMessage = error instanceof Error ? error.message : String(error)

						codeIndexLog.error(
							`[DirectoryScanner] Failed to delete points for ${cachedFilePath} in workspace ${scanWorkspace}:`,
							error,
						)

						incCodeIndexError("scanDirectory:deleteRemovedFiles")
						if (onError) {
							// Report error to error handler
							onError(
								error instanceof Error
									? new Error(
											`${error.message} (Workspace: ${scanWorkspace}, File: ${cachedFilePath})`,
										)
									: new Error(
											t("embeddings:scanner.unknownErrorDeletingPoints", {
												filePath: cachedFilePath,
											}) + ` (Workspace: ${scanWorkspace})`,
										),
							)
						}
						// Log error and continue processing instead of re-throwing
						codeIndexLog.error(`Failed to delete points for removed file: ${cachedFilePath}`, error)
					}
				}
			}
		}

		return {
			stats: {
				processed: processedCount,
				skipped: skippedCount,
			},
			totalBlockCount,
		}
	}

	private async processBatch(
		batchBlocks: CodeBlock[],
		batchTexts: string[],
		batchFileInfos: { filePath: string; fileHash: string; isNew: boolean; mtimeMs: number; size: number }[],
		scanWorkspace: string,
		onError?: (error: Error) => void,
		onBlocksIndexed?: (indexedCount: number) => void,
		signal?: AbortSignal,
	): Promise<void> {
		if (batchBlocks.length === 0) return

		// ── Per-segment dedup (same approach as FileWatcher._executeBatchUpsertOperations) ──
		// Build a map of filePath → Set of previously-indexed segmentHashes from the cache.
		// Blocks whose segmentHash already exists in Qdrant (same file, same block) are
		// skipped — no embedding call, no upsert. Only new/changed blocks are embedded.
		const prevHashesByFile = new Map<string, Set<string>>()
		for (const info of batchFileInfos) {
			if (info.isNew) continue // new files have no previous segments
			const cached = this.cacheManager.getEntry(info.filePath)
			if (cached?.segmentHashes) {
				prevHashesByFile.set(info.filePath, new Set(cached.segmentHashes))
			}
		}

		let reusedCount = 0
		const newBlocks: CodeBlock[] = []
		const newTexts: string[] = []
		const staleSegmentIds: string[] = []

		for (let i = 0; i < batchBlocks.length; i++) {
			const block = batchBlocks[i]!
			const prevSet = prevHashesByFile.get(block.file_path)
			if (prevSet?.has(block.segmentHash)) {
				reusedCount++
				// Remove from prevSet so we know it's not stale
				prevSet.delete(block.segmentHash)
			} else {
				newBlocks.push(block)
				newTexts.push(batchTexts[i]!)
			}
		}

		// Stale segments: were in the cache for this file but not present in the
		// current parse — they were removed from the source file. Delete them from
		// Qdrant by their deterministic point ID.
		for (const [filePath, prevSet] of prevHashesByFile) {
			for (const hash of prevSet) {
				staleSegmentIds.push(uuidv5(hash, QDRANT_CODE_BLOCK_NAMESPACE))
			}
		}

		if (staleSegmentIds.length > 0) {
			try {
				await this.qdrantClient.deletePointsByIds(staleSegmentIds)
			} catch (deleteError: any) {
				const errorMessage = deleteError instanceof Error ? deleteError.message : String(deleteError)
				codeIndexLog.error(
					`[DirectoryScanner] Failed to delete stale segment points in workspace ${scanWorkspace}:`,
					deleteError,
				)
				incCodeIndexError("processBatch:deletePointsByIds")
				// Re-throw so the batch retry loop can try again
				throw new Error(
					`Failed to delete ${staleSegmentIds.length} stale segments. Workspace: ${scanWorkspace}. ${errorMessage}`,
					{ cause: deleteError },
				)
			}
		}

		// If every block in the batch turned out to be a reuse, skip the embedder
		// call entirely — just update cache entries with the new file hashes.
		if (newBlocks.length === 0 && reusedCount > 0) {
			recordSegmentDedup({ reused: reusedCount, embedded: 0, deleted: staleSegmentIds.length })
			const blocksByFile = new Map<string, string[]>()
			for (const block of batchBlocks) {
				const existing = blocksByFile.get(block.file_path)
				if (existing) {
					existing.push(block.segmentHash)
				} else {
					blocksByFile.set(block.file_path, [block.segmentHash])
				}
			}
			for (const fileInfo of batchFileInfos) {
				this.cacheManager.updateEntry(fileInfo.filePath, {
					hash: fileInfo.fileHash,
					mtimeMs: fileInfo.mtimeMs,
					size: fileInfo.size,
					segmentHashes: blocksByFile.get(fileInfo.filePath) ?? [],
				})
			}
			onBlocksIndexed?.(reusedCount)
			return
		}

		let attempts = 0
		let success = false
		let lastError: Error | null = null

		while (attempts < MAX_BATCH_RETRIES && !success) {
			// Bail out early between retries when the caller has cancelled.
			if (signal?.aborted) return

			attempts++
			try {
				// Create embeddings for new/changed blocks only
				const { embeddings } = await this.embedder.createEmbeddings(newTexts, undefined, signal)

				// Prepare points for Qdrant (new blocks only)
				const points = newBlocks.map((block, index) => {
					const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, scanWorkspace)
					const pointId = uuidv5(block.segmentHash, QDRANT_CODE_BLOCK_NAMESPACE)

					return {
						id: pointId,
						vector: embeddings[index]!,
						payload: {
							filePath: generateRelativeFilePath(normalizedAbsolutePath, scanWorkspace),
							codeChunk: block.content,
							startLine: block.start_line,
							endLine: block.end_line,
							segmentHash: block.segmentHash,
						},
					}
				})

				// Upsert points to Qdrant
				await this.qdrantClient.upsertPoints(points)
				onBlocksIndexed?.(newBlocks.length + reusedCount)

				// Fire a single aggregated telemetry event per batch
				recordSegmentDedup({ reused: reusedCount, embedded: newBlocks.length, deleted: staleSegmentIds.length })

				// Update cache entries for all files in this batch (including
				// reused blocks so their segmentHashes are preserved).
				const blocksByFile = new Map<string, string[]>()
				for (const block of batchBlocks) {
					const existing = blocksByFile.get(block.file_path)
					if (existing) {
						existing.push(block.segmentHash)
					} else {
						blocksByFile.set(block.file_path, [block.segmentHash])
					}
				}
				for (const fileInfo of batchFileInfos) {
					this.cacheManager.updateEntry(fileInfo.filePath, {
						hash: fileInfo.fileHash,
						mtimeMs: fileInfo.mtimeMs,
						size: fileInfo.size,
						segmentHashes: blocksByFile.get(fileInfo.filePath) ?? [],
					})
				}
				success = true
			} catch (error) {
				lastError = error as Error
				codeIndexLog.error(
					`[DirectoryScanner] Error processing batch (attempt ${attempts}) in workspace ${scanWorkspace}:`,
					error,
				)
				incCodeIndexError("processBatch:retry")
				if (attempts < MAX_BATCH_RETRIES) {
					const delay = INITIAL_RETRY_DELAY_MS * Math.pow(2, attempts - 1)
					await new Promise((resolve) => setTimeout(resolve, delay))
				}
			}
		}

		if (!success && lastError) {
			codeIndexLog.error(`[DirectoryScanner] Failed to process batch after ${MAX_BATCH_RETRIES} attempts`)
			if (onError) {
				// Preserve the original error message from embedders which now have detailed i18n messages
				const errorMessage = lastError.message || "Unknown error"

				// For other errors, provide context
				onError(
					new Error(
						t("embeddings:scanner.failedToProcessBatchWithError", {
							maxRetries: MAX_BATCH_RETRIES,
							errorMessage,
						}),
					),
				)
			}
		}
	}

	/**
	 * Scans a specific set of files (Phase 2 — git-aware narrowing).
	 * Reuses the same per-file pipeline as scanDirectory (stat → cache check →
	 * parse → embed → upsert) but operates on an explicit list of paths instead
	 * of walking the directory tree.
	 *
	 * Only files matching scannerExtensions and passing .shoferignore/.gitignore
	 * filters are processed.
	 *
	 * @param workspacePath — absolute workspace root
	 * @param paths — absolute file paths to scan
	 */
	public async scanSpecificFiles(
		workspacePath: string,
		paths: string[],
		onError?: (error: Error) => void,
		onBlocksIndexed?: (indexedCount: number) => void,
		onFileParsed?: (fileBlockCount: number) => void,
		signal?: AbortSignal,
	): Promise<{ stats: { processed: number; skipped: number }; totalBlockCount: number }> {
		// Ensure ShoferIgnoreController is cached
		if (!this.shoferIgnoreController) {
			this.shoferIgnoreController = new ShoferIgnoreController(workspacePath)
			await this.shoferIgnoreController.initialize()
		}

		// Filter by supported extensions, .shoferignore, .gitignore, and ignored directories
		const supportedPaths = paths.filter((filePath) => {
			const ext = path.extname(filePath).toLowerCase()
			const relativeFilePath = generateRelativeFilePath(filePath, workspacePath)

			if (isPathInIgnoredDirectory(relativeFilePath)) return false
			return scannerExtensions.includes(ext) && !this.ignoreInstance.ignores(relativeFilePath)
		})

		// Filter by .shoferignore
		const allowedPaths = this.shoferIgnoreController.filterPaths(supportedPaths)

		let processedCount = 0
		let skippedCount = 0
		let totalBlockCount = 0

		const parseLimiter = pLimit(PARSING_CONCURRENCY)
		const batchLimiter = pLimit(BATCH_PROCESSING_CONCURRENCY)
		const mutex = new Mutex()

		let currentBatchBlocks: CodeBlock[] = []
		let currentBatchTexts: string[] = []
		let currentBatchBytes = 0
		let currentBatchFileInfos: {
			filePath: string
			fileHash: string
			isNew: boolean
			mtimeMs: number
			size: number
		}[] = []
		const activeBatchPromises = new Set<Promise<void>>()
		let pendingBatchCount = 0

		const parsePromises = allowedPaths.map((filePath) =>
			parseLimiter(async () => {
				if (signal?.aborted) return

				try {
					const stats = await stat(filePath)
					if (stats.size > MAX_FILE_SIZE_BYTES) {
						skippedCount++
						return
					}

					const cached = this.cacheManager.getEntry(filePath)
					// Compare mtimes at integer-millisecond resolution. The scanner
					// reads Node's fractional `stats.mtimeMs` (e.g. ...553.164) while
					// the file watcher writes VS Code's integer `fileStat.mtime`
					// (...553); without flooring, every file the watcher last touched
					// would spuriously fail this fast-path and get re-read + re-hashed.
					if (
						cached &&
						Math.floor(cached.mtimeMs) === Math.floor(stats.mtimeMs) &&
						cached.size === stats.size
					) {
						skippedCount++
						return
					}

					const content = await fs.readFile(filePath, "utf-8")

					const currentFileHash = createHash("sha256").update(content).digest("hex")
					const isNewFile = !cached

					if (cached && cached.hash === currentFileHash) {
						this.cacheManager.updateEntry(filePath, {
							hash: currentFileHash,
							mtimeMs: stats.mtimeMs,
							size: stats.size,
							segmentHashes: cached.segmentHashes ?? [],
						})
						skippedCount++
						return
					}

					const blocks = await this.codeParser.parseFile(filePath, { content, fileHash: currentFileHash })
					const fileBlockCount = blocks.length
					onFileParsed?.(fileBlockCount)
					processedCount++

					if (this.embedder && this.qdrantClient && blocks.length > 0) {
						let addedBlocksFromFile = false
						for (const block of blocks) {
							const trimmedContent = block.content.trim()
							if (trimmedContent) {
								const release = await mutex.acquire()
								try {
									currentBatchBlocks.push(block)
									currentBatchTexts.push(trimmedContent)
									currentBatchBytes += Buffer.byteLength(trimmedContent, "utf8")
									addedBlocksFromFile = true

									if (signal?.aborted) throw new DOMException("Indexing aborted", "AbortError")

									if (
										currentBatchBlocks.length >= this.batchSegmentThreshold ||
										currentBatchBytes >= MAX_BATCH_BYTES
									) {
										while (pendingBatchCount >= MAX_PENDING_BATCHES) {
											if (signal?.aborted)
												throw new DOMException("Indexing aborted", "AbortError")
											await Promise.race(activeBatchPromises)
										}

										const batchBlocks = [...currentBatchBlocks]
										const batchTexts = [...currentBatchTexts]
										const batchFileInfos = [...currentBatchFileInfos]
										currentBatchBlocks = []
										currentBatchTexts = []
										currentBatchBytes = 0
										currentBatchFileInfos = []
										pendingBatchCount++

										const batchPromise = batchLimiter(() =>
											this.processBatch(
												batchBlocks,
												batchTexts,
												batchFileInfos,
												workspacePath,
												onError,
												onBlocksIndexed,
												signal,
											),
										)
										activeBatchPromises.add(batchPromise)
										batchPromise.finally(() => {
											activeBatchPromises.delete(batchPromise)
											pendingBatchCount--
										})
									}
								} finally {
									release()
								}
							}
						}

						if (addedBlocksFromFile) {
							const release = await mutex.acquire()
							try {
								totalBlockCount += fileBlockCount
								currentBatchFileInfos.push({
									filePath,
									fileHash: currentFileHash,
									isNew: isNewFile,
									mtimeMs: stats.mtimeMs,
									size: stats.size,
								})
							} finally {
								release()
							}
						}
					} else {
						this.cacheManager.updateEntry(filePath, {
							hash: currentFileHash,
							mtimeMs: stats.mtimeMs,
							size: stats.size,
							segmentHashes: [],
						})
					}
				} catch (error) {
					if (error instanceof DOMException && error.name === "AbortError") throw error
					codeIndexLog.error(`Error processing file ${filePath}:`, error)
					if (onError) {
						onError(error instanceof Error ? error : new Error(String(error)))
					}
				}
			}),
		)

		await Promise.all(parsePromises)

		if (signal?.aborted) {
			return { stats: { processed: processedCount, skipped: skippedCount }, totalBlockCount }
		}

		// Drain remaining batch
		if (currentBatchBlocks.length > 0) {
			const release = await mutex.acquire()
			try {
				const batchBlocks = [...currentBatchBlocks]
				const batchTexts = [...currentBatchTexts]
				const batchFileInfos = [...currentBatchFileInfos]
				currentBatchBlocks = []
				currentBatchTexts = []
				currentBatchBytes = 0
				currentBatchFileInfos = []
				pendingBatchCount++

				const batchPromise = batchLimiter(() =>
					this.processBatch(
						batchBlocks,
						batchTexts,
						batchFileInfos,
						workspacePath,
						onError,
						onBlocksIndexed,
						signal,
					),
				)
				activeBatchPromises.add(batchPromise)
				batchPromise.finally(() => {
					activeBatchPromises.delete(batchPromise)
					pendingBatchCount--
				})
			} finally {
				release()
			}
		}

		await Promise.all(activeBatchPromises)

		return {
			stats: { processed: processedCount, skipped: skippedCount },
			totalBlockCount,
		}
	}

	/**
	 * Deletes points and cache entries for a specific set of deleted files
	 * (Phase 2 — git-aware narrowing).
	 *
	 * @param paths — absolute file paths to delete from Qdrant and cache
	 */
	public async deleteSpecificFiles(paths: string[]): Promise<void> {
		for (const filePath of paths) {
			if (this.qdrantClient) {
				try {
					await this.qdrantClient.deletePointsByFilePath(filePath)
				} catch {
					// Best-effort — log but don't throw
					codeIndexLog.error(`[DirectoryScanner] Failed to delete points for deleted file: ${filePath}`)
				}
			}
			this.cacheManager.deleteHash(filePath)
		}
	}
}