diff --git a/src/lore/ingest/chunker.py b/src/lore/ingest/chunker.py index 2caa118..75bc026 100644 --- a/src/lore/ingest/chunker.py +++ b/src/lore/ingest/chunker.py @@ -228,9 +228,14 @@ def _split_large_chunks( def chunk_document( - file_path: Path, *, min_words: int = 100, max_words: int = 1000 + file_path: Path, + *, + text: str | None = None, + min_words: int = 100, + max_words: int = 1000, ) -> list[DocumentChunk]: - text = file_path.read_text(encoding="utf-8", errors="replace") + if text is None: + text = file_path.read_text(encoding="utf-8", errors="replace") _, body = _strip_frontmatter(text) if not body.strip(): diff --git a/src/lore/ingest/doc.py b/src/lore/ingest/doc.py index 164181e..a0f9c71 100644 --- a/src/lore/ingest/doc.py +++ b/src/lore/ingest/doc.py @@ -14,9 +14,9 @@ log = logging.getLogger("lore.ingest") -def extract_doc_chunks(provider: LLMProvider, file_path: Path): +def extract_doc_chunks(provider: LLMProvider, file_path: Path, text: str | None = None): """Chunk file, LLM extract, validate keys. Yields (ext, tags, chunk).""" - chunks = chunk_document(file_path) + chunks = chunk_document(file_path, text=text) for chunk in chunks: extractions = provider.extract_from_chunk( chunk.text, chunk.heading, chunk.source_file diff --git a/src/lore/ingest/doc_repo.py b/src/lore/ingest/doc_repo.py index 8172812..b3250d2 100644 --- a/src/lore/ingest/doc_repo.py +++ b/src/lore/ingest/doc_repo.py @@ -66,10 +66,13 @@ def process_file( ) -> list[ParsedFile]: raw = file_path.read_bytes() content_hash = compute_content_hash(raw) + text = raw.decode("utf-8", errors="replace") rel = file_path.relative_to(repo_path).as_posix() parsed_files: list[ParsedFile] = [] - for ext, tags, _chunk in extract_doc_chunks(self._provider, file_path): + for ext, tags, _chunk in extract_doc_chunks( + self._provider, file_path, text=text + ): parsed_files.append( ParsedFile( key=ext.key, diff --git a/tests/unit/test_ingest_chunker.py b/tests/unit/test_ingest_chunker.py index 4d85f24..deb8f97 100644 --- a/tests/unit/test_ingest_chunker.py +++ b/tests/unit/test_ingest_chunker.py @@ -176,6 +176,16 @@ def test_chunk_document_end_to_end(tmp_path): assert chunks[0].word_count > 0 +def test_chunk_document_uses_supplied_text(tmp_path): + doc = tmp_path / "not-on-disk.md" + text = "# Supplied content\n" + "word " * 100 + + chunks = chunk_document(doc, text=text) + + assert chunks[0].heading == "Supplied content" + assert chunks[0].text.startswith("word") + + def test_chunk_document_empty_file(tmp_path): doc = tmp_path / "empty.md" doc.write_text("") diff --git a/tests/unit/test_ingest_doc_repo.py b/tests/unit/test_ingest_doc_repo.py index 5b1cbf6..49dd03d 100644 --- a/tests/unit/test_ingest_doc_repo.py +++ b/tests/unit/test_ingest_doc_repo.py @@ -125,6 +125,40 @@ def test_process_file_returns_parsed_files(tmp_path): assert result[0].locked is False +def test_process_file_reads_file_once(tmp_path, monkeypatch): + doc = _write_doc(tmp_path / "guide.md") + extractions = [ + DocChunkExtraction( + key="guide:test:intro", + summary="Test content.", + ) + ] + ingester = DocRepoIngester(_MockProvider(extractions)) + + original_read_bytes = Path.read_bytes + original_read_text = Path.read_text + read_bytes_count = 0 + + def count_read_bytes(path, *args, **kwargs): + nonlocal read_bytes_count + if path == doc: + read_bytes_count += 1 + return original_read_bytes(path, *args, **kwargs) + + def fail_read_text(path, *args, **kwargs): + if path == doc: + raise AssertionError("process_file should pass decoded text to chunker") + return original_read_text(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_bytes", count_read_bytes) + monkeypatch.setattr(Path, "read_text", fail_read_text) + + result = ingester.process_file(doc, tmp_path) + + assert len(result) == 1 + assert read_bytes_count == 1 + + def test_process_file_multiple_extractions(tmp_path): doc = _write_doc(tmp_path / "multi.md") extractions = [