diff --git a/.changeset/bounded-physical-page-reads.md b/.changeset/bounded-physical-page-reads.md new file mode 100644 index 0000000..03fd1b4 --- /dev/null +++ b/.changeset/bounded-physical-page-reads.md @@ -0,0 +1,5 @@ +--- +'@ontos-ai/knowhere-sdk': minor +--- + +Add bounded physical-page reads and complete snapshot summaries for local and stored parsed snapshots. Select chunks before rich normalization and asset URL hydration, preserve physical pages and chunk identity, and expose a body-free document index with explicit embedded ownership for progressive readers. Add `knowledge.getDocumentSummary(reference)` to load the complete verified metadata index independently of rich chunk construction, table/image files, navigation bodies, and asset URL generation; remote-only references require a prepared or synchronized snapshot. diff --git a/.changeset/initial-directory-read.md b/.changeset/initial-directory-read.md new file mode 100644 index 0000000..34078ba --- /dev/null +++ b/.changeset/initial-directory-read.md @@ -0,0 +1,5 @@ +--- +'@ontos-ai/knowhere-sdk': minor +--- + +Add revision- and namespace-fenced initial directory reads without downloading document assets or triggering retrieval enhancement. diff --git a/.changeset/readable-snapshot-image-origins.md b/.changeset/readable-snapshot-image-origins.md new file mode 100644 index 0000000..40e76d3 --- /dev/null +++ b/.changeset/readable-snapshot-image-origins.md @@ -0,0 +1,7 @@ +--- +'@ontos-ai/knowhere-sdk': minor +--- + +Add durable readable snapshot preparation and authenticated, pinned image access before full image replication completes. Prefer certified local image replicas and keep archive checkpoints, completeness validation, and full-copy commits independent from the readable marker. + +Batch selected image URL capabilities through the immutable snapshot API, avoiding one API authorization request per page. Expose `getParsedDocumentObjectUrls` for authorized application resource readers without reading image bodies. diff --git a/README.md b/README.md index f3f574b..5e59c8a 100644 --- a/README.md +++ b/README.md @@ -300,6 +300,14 @@ if (!documentId) { console.log(documentId); +// Read all initially recognized chapters without waiting for Fine or copying assets. +const directory = await client.documents.getStructure(documentId, { + namespace: 'support-center', +}); +if (directory.directoryVersion === 1) { + console.log(directory.sections); // physical pageNums are 1-based +} + // Agentic mode (LLM navigation + answer synthesis) const response = await client.retrieval.query({ namespace: 'support-center', @@ -395,6 +403,16 @@ if (chunks.chunks[0]) { console.log(archived.status); ``` +Progressive parsing delivers `job.document.directory_ready` once logically per +parsed revision, independently of baseline and final completion. Retries retain +the same `event_id`; consumers should apply them idempotently and pass the +event's `job_result_id` and `namespace` to `getStructure`. Version zero means +the directory is pending. Version one includes every recognized descendant, +including an empty directory when no hierarchy was found. A structure read +does not trigger Fine or download page images. Raw webhook types are exported +as `DocumentDirectoryReadyEvent`; SDK responses use camelCase and parse the +directory timestamp into a `Date`. + ### Local Knowledge Tools The SDK can also run exact inspection tools over parsed results. Local import @@ -468,6 +486,121 @@ temporary disk space for ZIP integrity validation. See the [streaming snapshot guide](docs/streamed-parsed-document-sync.md) for the adapter contract, retry semantics, and memory validation harness. +For immutable snapshots that support object delivery, establish readability before +copying every image: + +```typescript +const prepared = await knowledge.prepareReadableSnapshot({ + jobId: 'job_123', + revisionKey: 'snapshot_baseline_123', // Always the snapshot ID, not the Job Result ID. + documentId: 'doc_123', + signal, +}); + +// Persist the selected revision in your durable workflow and publish your ready state. +const page = await knowledge.readChunks({ + documentId: prepared.documentId, + revisionKey: prepared.revisionKey, + page: 1, +}); + +// Resume image replication in a subsequent durable queue invocation. +const copied = await knowledge.syncParsedDocument({ + jobId: prepared.snapshot.jobId, + revisionKey: prepared.revisionKey, + signal, +}); +// Re-enqueue the same revision while copied.completed is false. +``` + +`prepareReadableSnapshot` validates and stores every essential inventory object, +including `manifest.json`, `chunks.json`, `doc_nav.json`, and `source.pdf`. It checks +the pinned object inventory against document references and writes a separate +`.knowhere-sdk/readable.json` descriptor. Interrupted core transfers reuse valid +persisted objects on retry. Preparation does not write a full-copy commit, advance +archive checkpoints, or change the current displayed snapshot. Readers must pass +the selected `revisionKey` until the application publishes its own snapshot pointer. + +`readChunks` supplies local essential-object URLs after the readable barrier. Image +URLs use fresh capabilities through the authenticated immutable snapshot while +replication is incomplete; local image URLs are preferred after a full-copy commit +certifies the same document, revision, and snapshot checksum. Conflicting commit +identities are rejected without reading every image body to resolve metadata URLs. +Applications can also call `getParsedDocumentObjectUrl({ documentId, revisionKey, path, signal })` +or `getParsedDocumentObjectUrls({ documentId, revisionKey, paths, signal })` directly. +The plural method returns a readonly record keyed by requested paths, with `null` +for unavailable objects. It deduplicates paths and authorizes partial-copy image +URLs in sequential batches of at most 200 through the authenticated snapshot API. +`readChunks` batches only assets referenced by its selected chunks; URL resolution +does not download image bodies. Legacy full copies use their existing local URLs. +Applications can use `readParsedDocumentObject(...)` for bytes. Object reads prefer stored bytes and +verify origin image length and SHA-256 before returning uncopied bytes. Origin +reads do not copy images or certify replication. Capability URLs expire and must +be refreshed rather than stored as permanent identities. The application must +authorize its own document owner before exposing bytes or URLs to its readers. + +Use `knowledge.readChunks({ documentId, revisionKey, includeAssetUrls: false })` +when the application resolves stored asset references through its own authorized +resource reader. URL generation is enabled by default. + +Use `knowledge.getDocumentSummary(reference)` to build document navigation before +loading any selected chunk body: + +```typescript +const summary = await knowledge.getDocumentSummary({ documentId, revisionKey }); +console.log(summary.chunks); // Complete ordered metadata index, including unloaded chunks +``` + +The method reads only cached `manifest.json` and `chunks.json` after verifying a +stored commit or readable snapshot identity. It does not construct rich chunks, +read image/table files or navigation bodies, generate asset URLs, or scan remote +chunks. A local cache ID (including a string reference) is also supported. A +remote-only reference or unavailable stored revision rejects with `ValidationError`; +prepare or synchronize the snapshot first. Job IDs alone do not locate a complete +stored snapshot. + +For a complete local or stored snapshot, `readChunks` also returns +`documentSummary`: the resolved revision, full chunk/type counts, trusted manifest +page count (or `null`), sorted original physical page numbers, presentation flags, +and an ordered `chunks` index containing IDs, positions, types, section/source +paths, physical page numbers, and optional file paths. Index entries contain no +body text or asset URLs. The summary covers the complete snapshot even when the +returned chunk batch is filtered or paginated. Remote-only chunk windows omit it. +Descriptors also expose optional `embeddedChunkIds` for explicit `embeds` +connections, preserving chapter ownership for unloaded image/table entries. +Unversioned local caches report `revisionKey: null`; a caller-supplied revision +does not certify their contents. Stored snapshots report their verified revision. + +Use `sourcePageNumber` to read one original physical page without hydrating URLs +for other pages: + +```typescript +const page = await knowledge.readChunks({ + documentId: prepared.documentId, + revisionKey: prepared.revisionKey, + sourcePageNumber: 137, + page: 1, + pageSize: 20, +}); +``` + +`sourcePageNumber` must be a positive safe integer. It filters before display +pagination and URL resolution, retaining original chunk IDs and global positions. +Multi-page chunks retain their content while their page-asset list is narrowed +to the requested physical page. `totalChunks` and `totalPages` describe matching +chunks and display batches; `documentSummary.pageNumbers` preserves the original +physical page domain, including sparse pages. The stored JSON is still read and +lightly scanned in full; only selected chunks receive rich normalization and URL +hydration. Physical-page reads require a complete local or stored snapshot and +throw `ValidationError` for remote-only documents; prepare or synchronize the +pinned snapshot first. Ordinary remote reads keep their existing behavior. + +Only `syncParsedDocument(...).completed` and its existing full commit certify the +complete archive copy and integrity checks. An origin or replication failure does +not remove the readable descriptor. Historical snapshots without immutable object +delivery return the API's unsupported/not-found error from preparation; callers +can retain the existing full-archive synchronization path for those snapshots. + If a search result only has a published `documentId`, read-oriented helpers can accept that remote identifier directly. `readChunks` supports display paging; remote fallback requests asset URLs from the Knowhere API, and storage hits use diff --git a/scripts/publish-beta-release.mjs b/scripts/publish-beta-release.mjs index 4edc8bb..33c129d 100644 --- a/scripts/publish-beta-release.mjs +++ b/scripts/publish-beta-release.mjs @@ -31,6 +31,9 @@ if (!version.includes(`-${distTag}.`)) { throw new Error(`Beta publish expected ${version} to include prerelease tag ${distTag}`); } +// Embed the assigned snapshot version rather than the workflow's earlier build version. +releaseUtils.runCommand('pnpm', ['run', 'build:all']); + // npm provenance is disabled while this repository is private. // Re-enable `--provenance` when the source repository becomes public. releaseUtils.publishWorkspacePackages({ tag: distTag }); diff --git a/src/__tests__/beta-release-contract.test.ts b/src/__tests__/beta-release-contract.test.ts index c8aefe4..4e6667b 100644 --- a/src/__tests__/beta-release-contract.test.ts +++ b/src/__tests__/beta-release-contract.test.ts @@ -1,5 +1,5 @@ -import { spawnSync } from 'node:child_process'; -import { mkdtemp, mkdir, readFile, writeFile, copyFile, rm } from 'node:fs/promises'; +import { spawnSync, type SpawnSyncReturns } from 'node:child_process'; +import { mkdtemp, mkdir, readFile, writeFile, copyFile, rm, symlink } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { delimiter, join } from 'node:path'; import { expect, it } from 'vitest'; @@ -31,6 +31,8 @@ if (COMMAND === 'git' && args.join(' ') === 'rev-parse HEAD') { manifest.version = '2.6.1-beta.20990101000000'; writeFileSync('package.json', JSON.stringify(manifest)); } +} else if (COMMAND === 'pnpm' && args[0] === 'run' && args[1] === 'build:all') { + // The separate package contract exercises actual build and pack bytes. } else if (COMMAND === 'pnpm' && args[0] === 'list') { console.log(JSON.stringify([{ path: process.cwd() }])); } else if (COMMAND === 'npm' && args[0] === 'view') { @@ -79,3 +81,118 @@ if (COMMAND === 'git' && args.join(' ') === 'rev-parse HEAD') { await rm(directory, { recursive: true, force: true }); } }); + +it('publishes CommonJS and ESM bytes with the newly assigned beta package version', async (): Promise => { + const directory: string = await mkdtemp(join(tmpdir(), 'knowhere-beta-package-contract-')); + const bin: string = join(directory, 'bin'); + const scripts: string = join(directory, 'scripts'); + const source: string = join(directory, 'src'); + const temporary: string = join(directory, 'temporary'); + const extracted: string = join(directory, 'extracted'); + const capturePath: string = join(directory, 'published-package.json'); + const expectedVersion: string = '2.8.0-beta.20990101000000'; + try { + await Promise.all( + [bin, scripts, source, temporary, extracted].map( + (name: string): Promise => mkdir(name), + ), + ); + await Promise.all([ + ...['publish-beta-release.mjs', 'release-utils.mjs'].map( + (name: string): Promise => + copyFile(join(process.cwd(), 'scripts', name), join(scripts, name)), + ), + copyFile(join(process.cwd(), 'package.json'), join(directory, 'package.json')), + copyFile(join(process.cwd(), 'tsconfig.json'), join(directory, 'tsconfig.json')), + copyFile(join(process.cwd(), 'src', 'version.ts'), join(source, 'version.ts')), + writeFile(join(source, 'index.ts'), "export { VERSION } from './version.js';\n"), + symlink(join(process.cwd(), 'node_modules'), join(directory, 'node_modules'), 'dir'), + ]); + // Match the workflow's build before Changesets assigns its snapshot version. + const initialBuild: SpawnSyncReturns = spawnSync('pnpm', ['run', 'build'], { + cwd: directory, + encoding: 'utf8', + }); + expect(initialBuild.status, initialBuild.stdout + initialBuild.stderr).toBe(0); + + const commandBody: string = ` +import { execFileSync } from 'node:child_process'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +const args = process.argv.slice(2); +const originalEnvironment = { ...process.env, PATH: process.env.RELEASE_ORIGINAL_PATH }; +if (COMMAND === 'git' && args.join(' ') === 'rev-parse HEAD') { + console.log('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'); +} else if (COMMAND === 'pnpm' && args[0] === 'exec' && args[1] === 'changeset') { + if (args[2] === 'version') { + const manifest = JSON.parse(readFileSync('package.json', 'utf8')); + manifest.version = '${expectedVersion}'; + writeFileSync('package.json', JSON.stringify(manifest)); + } +} else if (COMMAND === 'pnpm' && args[0] === 'run' && args[1] === 'build:all') { + execFileSync('pnpm', ['run', 'build'], { stdio: 'inherit', env: originalEnvironment }); +} else if (COMMAND === 'pnpm' && args[0] === 'list') { + console.log(JSON.stringify([{ path: process.cwd() }])); +} else if (COMMAND === 'pnpm' && args[0] === 'pack') { + execFileSync('pnpm', args, { stdio: 'inherit', env: originalEnvironment }); +} else if (COMMAND === 'npm' && args[0] === 'view') { + process.exit(1); +} else if (COMMAND === 'npm' && args[0] === 'publish') { + const extracted = process.env.RELEASE_EXTRACTED_PATH; + execFileSync('tar', ['-xzf', args[1], '-C', extracted]); + const packageDirectory = join(extracted, 'package'); + const manifest = JSON.parse(readFileSync(join(packageDirectory, 'package.json'), 'utf8')); + const loadCommonJs = createRequire(import.meta.url); + const commonJs = loadCommonJs(join(packageDirectory, 'dist', 'index.js')); + const module = await import(pathToFileURL(join(packageDirectory, 'dist', 'index.mjs')).href); + writeFileSync(process.env.RELEASE_CAPTURE_PATH, JSON.stringify({ + packageVersion: manifest.version, + commonJsVersion: commonJs.VERSION, + moduleVersion: module.VERSION, + })); +} else if (COMMAND === 'gh' && args[0] === 'release' && args[1] === 'view') { + process.exit(0); +} else { + throw new Error('Unexpected release boundary command: ' + COMMAND + ' ' + args.join(' ')); +} +`; + await Promise.all( + ['git', 'pnpm', 'npm', 'gh'].map( + (name: string): Promise => + writeFile( + join(bin, name), + `#!/usr/bin/env node\nconst COMMAND = '${name}';\n${commandBody}`, + { mode: 0o755 }, + ), + ), + ); + const result: SpawnSyncReturns = spawnSync( + process.execPath, + [join(scripts, 'publish-beta-release.mjs')], + { + cwd: directory, + encoding: 'utf8', + env: { + ...process.env, + PATH: `${bin}${delimiter}${process.env.PATH ?? ''}`, + TMPDIR: temporary, + BETA_DIST_TAG: 'beta', + RELEASE_ORIGINAL_PATH: process.env.PATH, + RELEASE_EXTRACTED_PATH: extracted, + RELEASE_CAPTURE_PATH: capturePath, + }, + }, + ); + expect(result.status, result.stdout + result.stderr).toBe(0); + const captured: unknown = JSON.parse(await readFile(capturePath, 'utf8')); + expect(captured).toEqual({ + packageVersion: expectedVersion, + commonJsVersion: expectedVersion, + moduleVersion: expectedVersion, + }); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}, 30_000); diff --git a/src/__tests__/package-version-contract.test.ts b/src/__tests__/package-version-contract.test.ts new file mode 100644 index 0000000..475f6da --- /dev/null +++ b/src/__tests__/package-version-contract.test.ts @@ -0,0 +1,17 @@ +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { expect, it } from 'vitest'; +import { VERSION } from '../index.js'; +import { NODE_SDK_DOCUMENT_METADATA_DEFAULTS } from '../lib/document-metadata.js'; + +it('reports the package version in public exports and default document metadata', async (): Promise => { + const manifest: unknown = JSON.parse(await readFile(join(process.cwd(), 'package.json'), 'utf8')); + if (typeof manifest !== 'object' || manifest === null || !('version' in manifest)) { + throw new Error('Expected the SDK package manifest to declare its version'); + } + + expect({ + exportedVersion: VERSION, + clientVersion: NODE_SDK_DOCUMENT_METADATA_DEFAULTS.clientVersion, + }).toEqual({ exportedVersion: manifest.version, clientVersion: manifest.version }); +}); diff --git a/src/index.ts b/src/index.ts index 02f58a6..c14c59f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,8 +20,17 @@ export type { JobResult, JobStatus, JobError, + JobSnapshot, + JobSnapshotObject, + JobSnapshotDelivery, + JobSnapshotObjectAccess, + JobSnapshotObjectAccessBatch, // Document types Document, + DocumentDirectoryParams, + DocumentDirectoryReadyEvent, + DocumentDirectoryResponse, + DocumentDirectorySection, DocumentChunk, DocumentChunkGetParams, DocumentChunkListParams, @@ -69,6 +78,7 @@ export type { KnowhereAssetStorageWriteResult, ParsedDocumentCommit, ParsedDocumentCommitSource, + ParsedDocumentReadableSnapshot, ParsedDocumentObject, ParsedDocumentObjectHead, ParsedDocumentObjectParams, @@ -116,6 +126,8 @@ export type { KnowledgeReadParams, KnowledgeReadChunk, KnowledgeReadResponse, + KnowledgeChunkDescriptor, + KnowledgeDocumentSummary, KnowledgeGrepParams, KnowledgeGrepMatch, KnowledgeGrepResponse, @@ -126,6 +138,10 @@ export type { KnowledgeStartupRecoveryResponse, KnowledgeSyncParsedDocumentParams, KnowledgeSyncParsedDocumentResponse, + KnowledgePrepareReadableSnapshotParams, + KnowledgePrepareReadableSnapshotResponse, + KnowledgeReadParsedDocumentObjectParams, + KnowledgeReadParsedDocumentObjectUrlsParams, } from './types/index.js'; export { storeParseResultAssets } from './storage/asset-storage.js'; diff --git a/src/knowledge/__tests__/knowledge.test.ts b/src/knowledge/__tests__/knowledge.test.ts index 157b1f7..9f97ea9 100644 --- a/src/knowledge/__tests__/knowledge.test.ts +++ b/src/knowledge/__tests__/knowledge.test.ts @@ -5,6 +5,7 @@ import JSZip from 'jszip'; import { describe, it, expect, vi, afterEach } from 'vitest'; import { Knowledge } from '../knowledge.js'; +import { ValidationError } from '../../errors/index.js'; import type { Knowhere } from '../../client.js'; import type { Chunk, @@ -37,6 +38,164 @@ describe('Knowledge', () => { tempDirectories.length = 0; }); + it('rejects remote-only physical page reads without requesting unselected asset URLs', async (): Promise => { + const cacheDirectory: string = await createTempDirectory(); + const { client, documentsListChunks } = createClient(createParseResult()); + const knowledge: Knowledge = new Knowledge(client, { cacheDirectory }); + await expect( + knowledge.readChunks({ documentId: 'doc_remote', sourcePageNumber: 5 }), + ).rejects.toThrow(/complete local or stored parsed snapshot/); + expect(documentsListChunks).not.toHaveBeenCalled(); + const normalRead = await knowledge.readChunks({ documentId: 'doc_remote', page: 1 }); + expect(normalRead.documentSummary).toBeUndefined(); + expect(documentsListChunks).toHaveBeenCalledTimes(1); + }); + + it.each(['doc_remote', { documentId: 'doc_remote' }, { jobId: 'job-remote' }])( + 'rejects a remote-only document summary without scanning or importing: %j', + async (reference): Promise => { + const cacheDirectory: string = await createTempDirectory(); + const { client, documentsListChunks, jobsGet, jobsLoad } = createClient(createParseResult()); + const knowledge: Knowledge = new Knowledge(client, { cacheDirectory }); + await expect(knowledge.getDocumentSummary(reference)).rejects.toBeInstanceOf(ValidationError); + expect(documentsListChunks).not.toHaveBeenCalled(); + expect(jobsGet).not.toHaveBeenCalled(); + expect(jobsLoad).not.toHaveBeenCalled(); + }, + ); + + it('reads local summary metadata without table files or rich result construction', async (): Promise => { + const cacheDirectory: string = await createTempDirectory(); + const result: ParseResult = createParseResult(); + result.chunks[0].metadata = { + page_nums: [1], + connect_to: [{ target: 'chunk-table', relation: 'embeds' }], + }; + result.chunks[1].metadata = { page_nums: [5] }; + result.manifest.processing = { pageCount: 9 }; + const { client, documentsListChunks, jobsLoad } = createClient(result); + const knowledge: Knowledge = new Knowledge(client, { cacheDirectory }); + await knowledge.parseToLocalCache({ + url: 'https://example.com/report.md', + localDocumentId: 'local-summary', + }); + await rm(path.join(cacheDirectory, 'documents/local-summary/tables'), { recursive: true }); + await writeFile( + path.join(cacheDirectory, 'documents/local-summary/doc_nav.json'), + 'malformed navigation', + ); + const freshKnowledge: Knowledge = new Knowledge(client, { cacheDirectory }); + const summary = await freshKnowledge.getDocumentSummary({ + localDocumentId: 'local-summary', + revisionKey: 'foreign-requested-snapshot', + }); + expect(summary).toMatchObject({ + revisionKey: null, + chunkCount: 3, + typeCounts: { text: 2, image: 0, table: 1, page: 0 }, + pageCount: 9, + pageNumbers: [1, 5], + hasNativePageContent: true, + }); + expect(summary.chunks[0]?.embeddedChunkIds).toEqual(['chunk-table']); + expect(summary.chunks[1]).toMatchObject({ + chunkId: 'chunk-table', + filePath: 'tables/revenue.html', + }); + expect(JSON.stringify(summary)).not.toContain('Alpha revenue'); + expect(JSON.stringify(summary)).not.toContain('https://assets'); + expect(documentsListChunks).not.toHaveBeenCalled(); + expect(jobsLoad).not.toHaveBeenCalled(); + expect(await freshKnowledge.getDocumentSummary('local-summary')).toEqual(summary); + }); + + it('rejects malformed local chunk metadata without importing a remote replacement', async (): Promise => { + const cacheDirectory: string = await createTempDirectory(); + const { client, jobsLoad } = createClient(createParseResult()); + const knowledge: Knowledge = new Knowledge(client, { cacheDirectory }); + await knowledge.parseToLocalCache({ + url: 'https://example.com/report.md', + localDocumentId: 'local-summary', + }); + await writeFile( + path.join(cacheDirectory, 'documents/local-summary/chunks.json'), + JSON.stringify({ chunks: [null] }), + ); + await expect(knowledge.getDocumentSummary('local-summary')).rejects.toBeInstanceOf( + ValidationError, + ); + expect(jobsLoad).not.toHaveBeenCalled(); + }); + + it('returns full local snapshot descriptors while paging sparse physical-page matches', async (): Promise => { + const cacheDirectory: string = await createTempDirectory(); + const result: ParseResult = createParseResult(); + result.chunks = result.chunks.map((chunk, index) => ({ + ...chunk, + metadata: { ...chunk.metadata, page_nums: [index === 0 ? 1 : 5] }, + })); + result.manifest.processing = { pageCount: 9 }; + const { client } = createClient(result); + const knowledge: Knowledge = new Knowledge(client, { cacheDirectory }); + await knowledge.parseToLocalCache({ + url: 'https://example.com/report.md', + localDocumentId: 'physical-local', + }); + const read = await knowledge.readChunks({ + localDocumentId: 'physical-local', + sourcePageNumber: 5, + page: 2, + pageSize: 1, + }); + expect(read.chunks.map((chunk) => [chunk.chunkId, chunk.position])).toEqual([ + ['chunk-margin', 3], + ]); + expect(read.totalChunks).toBe(2); + expect(read.page).toBe(2); + expect(read.totalPages).toBe(2); + expect(read.documentSummary).toMatchObject({ + revisionKey: null, + chunkCount: 3, + pageNumbers: [1, 5], + pageCount: 9, + hasNativePageContent: true, + }); + expect(read.documentSummary?.chunks.map((chunk) => chunk.chunkId)).toEqual([ + 'chunk-intro', + 'chunk-table', + 'chunk-margin', + ]); + }); + + it('does not certify a requested snapshot revision for an unversioned local cache', async (): Promise => { + const knowledge: Knowledge = await createKnowledgeWithCachedResult(); + const read = await knowledge.readChunks({ + localDocumentId: 'local-report', + revisionKey: 'foreign-requested-snapshot', + limit: 1, + }); + expect(read.document.jobId).toBe('job-1'); + expect(read.chunks[0]?.chunkId).toBe('chunk-intro'); + expect(read.documentSummary?.revisionKey).toBeNull(); + }); + + it.each([ + { page: 1, pageSize: 2 }, + { sectionPath: 'Intro', limit: 5 }, + ])( + 'forwards explicit metadata-only reads to the documents API: %j', + async (window): Promise => { + const cacheDirectory = await createTempDirectory(); + const { client, documentsListChunks } = createClient(createParseResult()); + const knowledge = new Knowledge(client, { cacheDirectory }); + await knowledge.readChunks({ documentId: 'doc_remote', ...window, includeAssetUrls: false }); + expect(documentsListChunks).toHaveBeenCalledWith( + 'doc_remote', + expect.objectContaining({ includeAssetUrls: false }), + ); + }, + ); + it('should parse through the SDK and store a local document copy', async () => { const cacheDirectory = await createTempDirectory(); const parseResult = createParseResult(); diff --git a/src/knowledge/__tests__/readable-snapshots.test.ts b/src/knowledge/__tests__/readable-snapshots.test.ts new file mode 100644 index 0000000..a9eeb1d --- /dev/null +++ b/src/knowledge/__tests__/readable-snapshots.test.ts @@ -0,0 +1,1552 @@ +import { createHash } from 'crypto'; +import { createServer, type Server } from 'http'; +import JSZip from 'jszip'; +import { afterEach, describe, expect, it } from 'vitest'; +import { Knowhere } from '../../client.js'; +import { PermissionDeniedError, ValidationError } from '../../errors/index.js'; +import type { JobSnapshot, JobSnapshotObjectAccessBatch } from '../../types/job.js'; +import type { + ParsedDocumentObject, + ParsedDocumentStorage, + ParsedDocumentSyncProgress, + ParsedDocumentWriteObjectResult, +} from '../../types/storage.js'; + +interface InventoryObject { + readonly path: string; + readonly size_bytes: number; + readonly checksum_sha256: string; + readonly content_type: string; + readonly kind: 'essential' | 'image'; +} + +interface ReadableFixture { + readonly client: Knowhere; + readonly knowledge: Knowhere['knowledge']; + readonly storage: ParsedDocumentStorage; + readonly objects: Map; + readonly origins: Map; + readonly inventory: InventoryObject[]; + readonly requests: string[]; + readonly batchRequests: string[][]; + readonly progress: Map; + readonly snapshot: JobSnapshot; + readonly setOriginSnapshot: (snapshotId: string) => void; + readonly setCapabilityExpiration: (expiresAt: string) => void; + readonly denyImageAccess: () => void; + readonly setBatchResponsePaths: (paths: readonly string[]) => void; + readonly setBatchIdentity: (identity: Readonly>) => void; +} + +const servers: Server[] = []; + +afterEach(async (): Promise => { + await Promise.all( + servers.splice(0).map( + (server: Server): Promise => + new Promise((resolve, reject): void => { + server.closeAllConnections(); + server.close((error): void => (error ? reject(error) : resolve())); + }), + ), + ); +}); + +async function createFixture(omitArchivedImage: boolean = false): Promise { + const objects: Map = new Map(); + const progress: Map = new Map(); + const storage: ParsedDocumentStorage = { + readObject(params): Promise { + const body: Uint8Array | undefined = objects.get(`${params.revisionKey}/${params.path}`); + return Promise.resolve(body ? { ...params, body } : null); + }, + writeObject(params): Promise { + objects.set(`${params.revisionKey}/${params.path}`, params.body); + return Promise.resolve(params); + }, + readSyncProgress(params): Promise { + return Promise.resolve(progress.get(params.revisionKey) ?? null); + }, + writeSyncProgress(params): Promise { + progress.set(params.revisionKey, params); + return Promise.resolve(); + }, + }; + const origins: Map = new Map([ + [ + 'manifest.json', + Buffer.from( + JSON.stringify({ + job_id: 'job_readable', + job_result_id: 'result_shared', + source_file_name: 'manual.pdf', + files: { source: 'source.pdf', navigation: 'doc_nav.json' }, + statistics: { total_chunks: 1 }, + }), + ), + ], + [ + 'chunks.json', + Buffer.from( + JSON.stringify({ + chunks: [ + { + chunk_id: 'page_1', + type: 'page', + content: 'Pinned baseline text', + path: 'manual.pdf/Page 1', + metadata: { + page_nums: [1], + summary: 'Pinned baseline text', + page_assets: [{ artifact_ref: 'images/page-1.jpg' }], + }, + }, + ], + }), + ), + ], + ['doc_nav.json', Buffer.from(JSON.stringify({ sections: [] }))], + ['source.pdf', Buffer.from('%PDF-pinned-original')], + ['tables/table-1.html', Buffer.from('
Essential table
')], + ['images/page-1.jpg', Buffer.from('pinned-image-bytes')], + ]); + const inventory: InventoryObject[] = [...origins].map( + ([path, body]): InventoryObject => ({ + path, + size_bytes: body.byteLength, + checksum_sha256: createHash('sha256').update(body).digest('hex'), + content_type: path.endsWith('.jpg') ? 'image/jpeg' : 'application/octet-stream', + kind: path.startsWith('images/') ? 'image' : 'essential', + }), + ); + const zip: JSZip = new JSZip(); + for (const [path, body] of origins) + if (!omitArchivedImage || !path.startsWith('images/')) zip.file(path, body); + const archive: Uint8Array = await zip.generateAsync({ type: 'uint8array' }); + const checksum: string = createHash('sha256').update(archive).digest('hex'); + const requests: string[] = []; + const batchRequests: string[][] = []; + let batchResponsePaths: readonly string[] | undefined; + let batchIdentity: Readonly> = {}; + let originSnapshotId: string = 'snapshot_baseline'; + let capability: number = 0; + let capabilityExpiration: string = '2030-01-01T00:00:00Z'; + let hasDeniedImageAccess: boolean = false; + let baseURL: string = ''; + const server: Server = createServer((request, response): void => { + const url: URL = new URL(request.url ?? '/', baseURL); + requests.push(`${request.method ?? 'GET'} ${url.pathname}${url.search}`); + const identity = { + snapshot_id: originSnapshotId, + job_id: 'job_readable', + job_result_id: 'result_shared', + document_id: 'doc_readable', + checksum, + }; + if (url.pathname.startsWith('/v2/')) { + if (request.headers.authorization !== 'Bearer test-key') { + response.writeHead(403).end(); + return; + } + response.setHeader('content-type', 'application/json'); + if (url.pathname.endsWith('/readable')) { + response.end(JSON.stringify({ ...identity, objects: inventory })); + return; + } + if (url.pathname.endsWith('/objects/access') && request.method === 'POST') { + const chunks: Buffer[] = []; + request.on('data', (chunk: Buffer): void => { + chunks.push(chunk); + }); + request.on('end', (): void => { + const body: unknown = JSON.parse(Buffer.concat(chunks).toString('utf8')); + if ( + !body || + typeof body !== 'object' || + !('paths' in body) || + !Array.isArray(body.paths) || + !body.paths.every((path: unknown): path is string => typeof path === 'string') + ) { + response.writeHead(400).end(); + return; + } + batchRequests.push(body.paths); + if ( + hasDeniedImageAccess && + body.paths.some((path): boolean => path.startsWith('images/')) + ) { + response + .writeHead(403) + .end(JSON.stringify({ error: { message: 'Owner access revoked' } })); + return; + } + if ( + body.paths.some( + (path): boolean => !inventory.some((object): boolean => object.path === path), + ) + ) { + response.writeHead(404).end(); + return; + } + response.end( + JSON.stringify({ + ...identity, + ...batchIdentity, + objects: (batchResponsePaths ?? body.paths).map((path) => ({ + path, + download_url: `${baseURL}/origin/${encodeURIComponent(path)}?token=${++capability}`, + expires_at: capabilityExpiration, + })), + }), + ); + }); + return; + } + if (url.pathname.endsWith('/objects')) { + const path: string = url.searchParams.get('path') ?? ''; + if (hasDeniedImageAccess && path.startsWith('images/')) { + response + .writeHead(403) + .end(JSON.stringify({ error: { message: 'Owner access revoked' } })); + return; + } + if (!inventory.some((object): boolean => object.path === path)) { + response.writeHead(404).end(); + return; + } + capability += 1; + response.end( + JSON.stringify({ + ...identity, + path, + download_url: `${baseURL}/origin/${encodeURIComponent(path)}?token=${capability}`, + expires_at: capabilityExpiration, + }), + ); + return; + } + if (url.pathname === '/v2/jobs/job_readable') { + response.end( + JSON.stringify({ + job_id: 'job_readable', + status: 'done', + source_type: 'file', + created_at: '2026-10-28T00:00:00Z', + document_id: 'doc_readable', + current_snapshot_id: 'snapshot_baseline', + snapshots: [ + { + ...identity, + snapshot_id: 'snapshot_baseline', + stage: 'baseline', + result_url: `${baseURL}/result.zip`, + result_url_expires_at: '2030-01-01T00:00:00Z', + }, + ], + }), + ); + return; + } + if (url.pathname === '/v2/documents/doc_readable/chunks') { + response.end( + JSON.stringify({ + document_id: 'doc_readable', + job_id: 'job_readable', + job_result_id: 'result_shared', + namespace: 'default', + chunks: [], + pagination: { page: 1, page_size: 1, total: 0, total_pages: 0 }, + }), + ); + return; + } + response.writeHead(404).end(); + return; + } + if (url.pathname === '/result.zip') { + response.writeHead(200, { 'content-length': archive.byteLength, etag: 'pinned-archive' }); + response.end(archive); + return; + } + const body: Uint8Array | undefined = origins.get( + decodeURIComponent(url.pathname.slice('/origin/'.length)), + ); + response.writeHead(body ? 200 : 404); + response.end(body); + }); + servers.push(server); + await new Promise((resolve): void => { + server.listen(0, '127.0.0.1', resolve); + }); + const address: ReturnType = server.address(); + if (!address || typeof address === 'string') throw new Error('Missing fixture origin port'); + baseURL = `http://127.0.0.1:${address.port}`; + const client: Knowhere = new Knowhere({ baseURL, apiKey: 'test-key', maxRetries: 0 }); + const snapshot: JobSnapshot = { + snapshotId: 'snapshot_baseline', + stage: 'baseline', + jobId: 'job_readable', + jobResultId: 'result_shared', + documentId: 'doc_readable', + checksum, + resultUrl: `${baseURL}/result.zip`, + resultUrlExpiresAt: new Date('2030-01-01'), + }; + return { + client, + knowledge: client.knowledge.withParsedStorage({ storage, limits: { maxEntriesPerSync: 1 } }), + storage, + objects, + origins, + inventory, + requests, + batchRequests, + progress, + snapshot, + setOriginSnapshot(snapshotId): void { + originSnapshotId = snapshotId; + }, + setCapabilityExpiration(expiresAt): void { + capabilityExpiration = expiresAt; + }, + denyImageAccess(): void { + hasDeniedImageAccess = true; + }, + setBatchResponsePaths(paths): void { + batchResponsePaths = paths; + }, + setBatchIdentity(identity): void { + batchIdentity = identity; + }, + }; +} + +const reference = { jobId: 'job_readable', revisionKey: 'snapshot_baseline' } as const; +const imageReference = { + documentId: 'doc_readable', + revisionKey: 'snapshot_baseline', + path: 'images/page-1.jpg', +} as const; + +function createFullCommit(fixture: ReadableFixture): Record { + return { + version: 1, + documentId: fixture.snapshot.documentId, + revisionKey: fixture.snapshot.snapshotId, + snapshotId: fixture.snapshot.snapshotId, + jobResultId: fixture.snapshot.jobResultId, + checksum: fixture.snapshot.checksum, + stage: fixture.snapshot.stage, + source: 'resultZip', + committedAt: new Date().toISOString(), + }; +} + +function updateOriginObject(fixture: ReadableFixture, path: string, body: Uint8Array): void { + fixture.origins.set(path, body); + const object: InventoryObject = { + path, + size_bytes: body.byteLength, + checksum_sha256: createHash('sha256').update(body).digest('hex'), + content_type: path.endsWith('.jpg') ? 'image/jpeg' : 'application/json', + kind: path.startsWith('images/') ? 'image' : 'essential', + }; + const index: number = fixture.inventory.findIndex( + (candidate): boolean => candidate.path === path, + ); + if (index < 0) fixture.inventory.push(object); + else fixture.inventory[index] = object; +} + +function addPageImages(fixture: ReadableFixture, count: number): readonly string[] { + const paths: string[] = Array.from( + { length: count }, + (_, index): string => `images/page-${index + 1}.jpg`, + ); + for (const path of paths) updateOriginObject(fixture, path, Buffer.from(path)); + updateOriginObject( + fixture, + 'chunks.json', + Buffer.from( + JSON.stringify({ + chunks: paths.map((path, index) => ({ + chunk_id: `page_${index + 1}`, + type: 'page', + content: `Page ${index + 1}`, + path: `manual.pdf/Page ${index + 1}`, + metadata: { + page_nums: [index + 1], + page_assets: [{ artifact_ref: path }, { artifact_ref: path }], + }, + })), + }), + ), + ); + return paths; +} + +describe('readable snapshot storage boundary', (): void => { + it.each([{ committedAt: '2026-10-08T00:00:00Z' }, { committedAt: 'legacy-import' }, {}])( + 'preserves legacy marker reads, outline, grep, and summaries without source: %j', + async (legacyTimestamp): Promise => { + const fixture: ReadableFixture = await createFixture(); + const legacyReference = { documentId: 'doc_test', revisionKey: 'retained-revision' } as const; + const payloads: Readonly> = { + '.knowhere-sdk/commit.json': { version: 1, ...legacyReference, ...legacyTimestamp }, + 'manifest.json': { sourceFileName: 'Construction.docx' }, + 'chunks.json': { + chunks: [ + { + chunk_id: 'table-1', + type: 'table', + path: 'tables/result.html', + content: 'tables/result.html', + file_path: 'tables/result.html', + metadata: { page_nums: [2] }, + }, + { + chunk_id: 'text-1', + type: 'text', + path: 'Construction.docx/Introduction', + content: 'Introduction', + metadata: {}, + }, + ], + }, + }; + for (const [path, payload] of Object.entries(payloads)) + fixture.objects.set(`retained-revision/${path}`, Buffer.from(JSON.stringify(payload))); + fixture.storage.getObjectUrl = (): Promise => + Promise.resolve('https://blob.example/retained/tables/result.html'); + const read = await fixture.knowledge.readChunks(legacyReference); + expect(read.chunks).toMatchObject([ + { + chunkId: 'table-1', + content: 'tables/result.html', + assetUrl: 'https://blob.example/retained/tables/result.html', + }, + { chunkId: 'text-1', content: 'Introduction' }, + ]); + expect((await fixture.knowledge.getDocumentOutline(legacyReference)).totalChunks).toBe(2); + expect( + (await fixture.knowledge.grepChunks({ ...legacyReference, pattern: 'Introduction' })) + .matches[0]?.chunkId, + ).toBe('text-1'); + expect(await fixture.knowledge.getDocumentSummary(legacyReference)).toMatchObject({ + revisionKey: 'retained-revision', + chunkCount: 2, + pageNumbers: [2], + }); + expect(fixture.requests).toEqual([]); + }, + ); + + it('returns all 188 metadata descriptors without normalizing bodies or loading asset URLs', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + addPageImages(fixture, 188); + updateOriginObject( + fixture, + 'manifest.json', + Buffer.from( + JSON.stringify({ + job_id: 'job_readable', + job_result_id: 'result_shared', + source_file_name: 'manual.pdf', + processing: { page_count: 188 }, + }), + ), + ); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.requests.length = 0; + const reads: string[] = []; + const readObject = fixture.storage.readObject.bind(fixture.storage); + fixture.storage.readObject = (params): Promise => { + reads.push(params.path); + if ( + params.path === 'doc_nav.json' || + params.path.startsWith('tables/') || + params.path.startsWith('images/') + ) + throw new Error(`Unexpected rich-object read: ${params.path}`); + return readObject(params); + }; + fixture.storage.getObjectUrl = (): Promise => { + throw new Error('Metadata must not resolve asset URLs'); + }; + const summary = await fixture.knowledge.getDocumentSummary(imageReference); + expect(summary).toMatchObject({ + revisionKey: 'snapshot_baseline', + chunkCount: 188, + typeCounts: { text: 0, image: 0, table: 0, page: 188 }, + pageCount: 188, + pageNumbers: Array.from({ length: 188 }, (_, index): number => index + 1), + hasPageAssets: true, + hasNativePageContent: false, + }); + expect(summary.chunks).toHaveLength(188); + expect(summary.chunks[187]).toEqual({ + chunkId: 'page_188', + position: 188, + chunkType: 'page', + sourceChunkPath: 'manual.pdf/Page 188', + sectionPath: 'Page 188', + pageNumbers: [188], + }); + expect(reads).toEqual([ + '.knowhere-sdk/commit.json', + '.knowhere-sdk/readable.json', + 'manifest.json', + 'chunks.json', + ]); + expect(fixture.requests).toEqual([]); + expect(fixture.batchRequests).toEqual([]); + }); + + it.each(['connect_to', 'connectTo'])( + 'preserves complete explicit ownership from %s without loading bodies', + async (field): Promise => { + const fixture: ReadableFixture = await createFixture(); + updateOriginObject( + fixture, + 'chunks.json', + Buffer.from( + JSON.stringify({ + chunks: [ + { + chunk_id: 'owner', + type: 'text', + path: 'manual.pdf/Chapter', + metadata: { + page_nums: [1], + [field]: [ + { target: 'figure', relation: 'embeds' }, + { target: 'figure', relation: 'embeds' }, + { target: 'unowned', relation: 'related' }, + ], + }, + }, + { + chunk_id: 'figure', + type: 'image', + path: 'images/page-1.jpg', + file_path: 'images/page-1.jpg', + metadata: { page_nums: [1] }, + }, + ], + }), + ), + ); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.requests.length = 0; + const summary = await fixture.knowledge.getDocumentSummary(imageReference); + expect(summary.chunks[0]?.embeddedChunkIds).toEqual(['figure']); + expect(summary.chunks[1]).toMatchObject({ chunkId: 'figure', filePath: 'images/page-1.jpg' }); + expect(summary.chunks[1]).not.toHaveProperty('embeddedChunkIds'); + expect(summary.hasNativePageContent).toBe(true); + expect(fixture.requests).toEqual([]); + }, + ); + + it('reads legacy full-copy metadata using the committed revision without immutable delivery calls', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + for (const path of ['manifest.json', 'chunks.json']) { + fixture.objects.set(`snapshot_baseline/${path}`, fixture.origins.get(path)!); + } + fixture.objects.set( + 'snapshot_baseline/.knowhere-sdk/commit.json', + Buffer.from( + JSON.stringify({ + version: 1, + documentId: 'doc_readable', + revisionKey: 'snapshot_baseline', + source: 'resultZip', + committedAt: '2026-10-28T00:00:00.000Z', + }), + ), + ); + fixture.objects.set( + 'current/.knowhere-sdk/current-snapshot.json', + Buffer.from(JSON.stringify({ snapshotId: 'snapshot_baseline' })), + ); + const summary = await fixture.knowledge.getDocumentSummary('doc_readable'); + expect(summary).toMatchObject({ revisionKey: 'snapshot_baseline', chunkCount: 1 }); + expect(fixture.requests).toEqual([]); + }); + + it.each(['snapshotId', 'documentId', 'revisionKey'])( + 'rejects a conflicting readable %s before returning metadata', + async (field): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + const descriptor: Record = JSON.parse( + Buffer.from(fixture.objects.get('snapshot_baseline/.knowhere-sdk/readable.json')!).toString( + 'utf8', + ), + ) as Record; + fixture.objects.set( + 'snapshot_baseline/.knowhere-sdk/readable.json', + Buffer.from(JSON.stringify({ ...descriptor, [field]: 'foreign' })), + ); + fixture.requests.length = 0; + await expect(fixture.knowledge.getDocumentSummary(imageReference)).rejects.toBeInstanceOf( + ValidationError, + ); + expect(fixture.requests).toEqual([]); + }, + ); + + it.each([ + { path: 'chunks.json', payload: { chunks: null } }, + { path: 'chunks.json', payload: { chunks: [null] } }, + { path: 'chunks.json', payload: { chunks: [{ type: 'text', path: 'manual.pdf/Chapter' }] } }, + { path: 'manifest.json', payload: null }, + ])('rejects malformed summary core $path: $payload', async ({ path, payload }): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.objects.set(`snapshot_baseline/${path}`, Buffer.from(JSON.stringify(payload))); + fixture.requests.length = 0; + await expect(fixture.knowledge.getDocumentSummary(imageReference)).rejects.toBeInstanceOf( + ValidationError, + ); + expect(fixture.requests).toEqual([]); + }); + + it('rejects an unavailable pin without reading the current remote document', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.requests.length = 0; + await expect( + fixture.knowledge.getDocumentSummary({ documentId: 'doc_readable', revisionKey: 'foreign' }), + ).rejects.toBeInstanceOf(ValidationError); + expect(fixture.requests).toEqual([]); + }); + + it.each([ + { version: 2 }, + { stage: 'invalid-stage' }, + { snapshotId: 'foreign' }, + { jobResultId: 'foreign' }, + ])('rejects an invalid full-copy metadata certificate: %j', async (override): Promise => { + const fixture: ReadableFixture = await createFixture(); + for (const path of ['manifest.json', 'chunks.json']) + fixture.objects.set(`snapshot_baseline/${path}`, fixture.origins.get(path)!); + fixture.objects.set( + 'snapshot_baseline/.knowhere-sdk/commit.json', + Buffer.from(JSON.stringify({ ...createFullCommit(fixture), ...override })), + ); + await expect(fixture.knowledge.getDocumentSummary(imageReference)).rejects.toBeInstanceOf( + ValidationError, + ); + expect(fixture.requests).toEqual([]); + }); + + it('accepts an opaque full-copy checksum without inferring an archive hash contract', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + for (const path of ['manifest.json', 'chunks.json']) + fixture.objects.set(`snapshot_baseline/${path}`, fixture.origins.get(path)!); + fixture.objects.set( + 'snapshot_baseline/.knowhere-sdk/commit.json', + Buffer.from(JSON.stringify({ ...createFullCommit(fixture), checksum: 'snapshot-checksum' })), + ); + expect(await fixture.knowledge.getDocumentSummary(imageReference)).toMatchObject({ + revisionKey: 'snapshot_baseline', + chunkCount: 1, + pageNumbers: [1], + }); + expect(fixture.requests).toEqual([]); + }); + + it('rejects valid metadata bytes that no longer match the readable snapshot inventory', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.objects.set( + 'snapshot_baseline/chunks.json', + Buffer.from( + JSON.stringify({ + chunks: [{ chunk_id: 'foreign-page', type: 'page', path: 'manual.pdf/Page 1' }], + }), + ), + ); + fixture.requests.length = 0; + await expect(fixture.knowledge.getDocumentSummary(imageReference)).rejects.toBeInstanceOf( + ValidationError, + ); + expect(fixture.requests).toEqual([]); + }); + + it('reads physical page 137 before hydrating URLs and returns a complete body-free index', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const paths: readonly string[] = addPageImages(fixture, 188); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.requests.length = 0; + const read = await fixture.knowledge.readChunks({ + ...imageReference, + sourcePageNumber: 137, + page: 1, + pageSize: 1, + }); + expect(read.chunks.map((chunk) => [chunk.chunkId, chunk.position])).toEqual([ + ['page_137', 137], + ]); + expect(fixture.batchRequests).toEqual([[paths[136]]]); + expect(fixture.requests).toEqual([ + 'POST /v2/jobs/job_readable/snapshots/snapshot_baseline/objects/access', + ]); + expect(read.totalChunks).toBe(1); + expect(read.documentSummary).toMatchObject({ + revisionKey: 'snapshot_baseline', + chunkCount: 188, + typeCounts: { text: 0, image: 0, table: 0, page: 188 }, + pageCount: null, + pageNumbers: Array.from({ length: 188 }, (_, index) => index + 1), + hasPageAssets: true, + hasNativePageContent: false, + }); + expect(read.documentSummary?.chunks[136]).toEqual({ + chunkId: 'page_137', + position: 137, + chunkType: 'page', + sectionPath: 'Page 137', + sourceChunkPath: 'manual.pdf/Page 137', + pageNumbers: [137], + }); + }); + + it('uses sparse asset page aliases and narrows multi-page assets before URL collection', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const paths: readonly string[] = addPageImages(fixture, 5); + updateOriginObject( + fixture, + 'chunks.json', + Buffer.from( + JSON.stringify({ + chunks: [ + { + chunk_id: 'native', + type: 'text', + content: 'Native text', + path: 'manual.pdf/Intro', + metadata: { page_nums: [1] }, + }, + { + chunk_id: 'multi', + type: 'page', + content: 'Body must stay out of descriptors', + path: 'manual.pdf/Images', + metadata: { + page_assets: [ + { page_number: 1, artifact_ref: paths[0], content_type: 'image/jpeg' }, + { pageNum: 5, artifact_ref: paths[4], content_type: 'image/jpeg' }, + ], + }, + }, + ], + }), + ), + ); + await fixture.knowledge.prepareReadableSnapshot(reference); + const first = await fixture.knowledge.readChunks({ + ...imageReference, + pageSize: 1, + includeAssetUrls: false, + }); + expect(first.documentSummary).toMatchObject({ + pageNumbers: [1, 5], + hasPageAssets: true, + hasNativePageContent: true, + }); + const read = await fixture.knowledge.readChunks({ + ...imageReference, + sourcePageNumber: 5, + pageSize: 1, + }); + expect(read.chunks.map((chunk) => [chunk.chunkId, chunk.position])).toEqual([['multi', 2]]); + expect(read.chunks[0]?.metadata.pageAssets).toHaveLength(1); + expect(fixture.batchRequests).toEqual([[paths[4]]]); + }); + + it.each([0, -1, 1.5, Number.MAX_SAFE_INTEGER + 1, Number.NaN])( + 'rejects invalid physical page %s', + async (sourcePageNumber): Promise => { + const fixture: ReadableFixture = await createFixture(); + await expect( + fixture.knowledge.readChunks({ ...imageReference, sourcePageNumber }), + ).rejects.toBeInstanceOf(ValidationError); + expect(fixture.requests).toEqual([]); + }, + ); + + it.each([{ pageNums: [5] }, { page_nums: [5] }, { pageNumbers: [5] }, { pageNumber: 5 }])( + 'preserves physical pages from metadata aliases %j', + async (metadata): Promise => { + const fixture: ReadableFixture = await createFixture(); + updateOriginObject( + fixture, + 'chunks.json', + Buffer.from( + JSON.stringify({ + chunks: [ + { + chunk_id: 'physical-five', + type: 'text', + content: 'Fifth source page', + path: 'manual.pdf/Five', + metadata, + }, + ], + }), + ), + ); + await fixture.knowledge.prepareReadableSnapshot(reference); + const read = await fixture.knowledge.readChunks({ ...imageReference, sourcePageNumber: 5 }); + expect(read.chunks[0]?.pageNumbers).toEqual([5]); + expect(read.documentSummary?.pageNumbers).toEqual([5]); + expect(read.chunks[0]?.position).toBe(1); + }, + ); + + it('does not guess physical pages for unlabeled multi-page assets', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const paths: readonly string[] = addPageImages(fixture, 5); + updateOriginObject( + fixture, + 'chunks.json', + Buffer.from( + JSON.stringify({ + chunks: [ + { + chunk_id: 'ambiguous', + type: 'page', + content: 'Pages one and five', + path: 'manual.pdf/Several', + metadata: { + page_nums: [1, 5], + page_assets: [{ artifact_ref: paths[0] }, { artifact_ref: paths[4] }], + }, + }, + ], + }), + ), + ); + await fixture.knowledge.prepareReadableSnapshot(reference); + const read = await fixture.knowledge.readChunks({ ...imageReference, sourcePageNumber: 5 }); + expect(read.chunks[0]?.metadata.pageAssets).toEqual([]); + expect(fixture.batchRequests).toEqual([]); + expect(read.documentSummary?.hasPageAssets).toBe(true); + }); + + it.each(['connect_to', 'connectTo'])( + 'keeps complete explicit ownership from %s when the owning body is not selected', + async (field): Promise => { + const fixture: ReadableFixture = await createFixture(); + addPageImages(fixture, 5); + updateOriginObject( + fixture, + 'chunks.json', + Buffer.from( + JSON.stringify({ + chunks: [ + { + chunk_id: 'unloaded-body', + type: 'text', + content: 'Unloaded owning body', + path: 'manual.pdf/Chapter', + metadata: { + page_nums: [1], + [field]: [ + { target: 'figure-five', relation: 'embeds' }, + { target: 'related-table', relation: 'related' }, + { target: 'figure-five', relation: 'embeds' }, + { target: 42, relation: 'embeds' }, + ], + }, + }, + { + chunk_id: 'figure-five', + type: 'image', + content: 'Selected figure', + path: 'images/page-5.jpg', + file_path: 'images/page-5.jpg', + metadata: { page_nums: [5] }, + }, + ], + }), + ), + ); + await fixture.knowledge.prepareReadableSnapshot(reference); + const read = await fixture.knowledge.readChunks({ + ...imageReference, + sourcePageNumber: 5, + includeAssetUrls: false, + }); + expect(read.chunks.map((chunk) => chunk.chunkId)).toEqual(['figure-five']); + expect(read.documentSummary?.chunks[0]).toEqual({ + chunkId: 'unloaded-body', + position: 1, + chunkType: 'text', + sectionPath: 'Chapter', + sourceChunkPath: 'manual.pdf/Chapter', + pageNumbers: [1], + embeddedChunkIds: ['figure-five'], + }); + expect(read.documentSummary?.chunks[1]).not.toHaveProperty('embeddedChunkIds'); + expect(fixture.batchRequests).toEqual([]); + }, + ); + + it('authorizes 188 selected page images in one batch without single capabilities or image bodies', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const paths: readonly string[] = addPageImages(fixture, 188); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.objects.set( + 'snapshot_baseline/images/page-1.jpg', + Buffer.from('damaged partial replica'), + ); + fixture.storage.getObjectUrl = (params): Promise => + Promise.resolve(`https://replica.example/${params.path}`); + const imageReads: string[] = []; + const readObject = fixture.storage.readObject.bind(fixture.storage); + fixture.storage.readObject = (params): Promise => { + if (params.path.startsWith('images/')) imageReads.push(params.path); + return readObject(params); + }; + fixture.requests.length = 0; + const read = await fixture.knowledge.readChunks({ ...imageReference, pageSize: 200 }); + expect(read.chunks).toHaveLength(188); + expect(read.chunks[187]?.metadata.pageAssets).toMatchObject([ + { + artifactRef: paths[187], + }, + { + artifactRef: paths[187], + }, + ]); + expect(read.chunks[187]?.metadata.pageAssets).toHaveProperty( + '0.assetUrl', + expect.stringContaining('/origin/images%2Fpage-188.jpg'), + ); + expect(read.chunks[187]?.metadata.pageAssets).toHaveProperty( + '1.assetUrl', + expect.stringContaining('/origin/images%2Fpage-188.jpg'), + ); + expect(fixture.batchRequests).toEqual([paths]); + expect(fixture.requests).toEqual([ + 'POST /v2/jobs/job_readable/snapshots/snapshot_baseline/objects/access', + ]); + expect(imageReads).toEqual([]); + const beforeMetadataOnly: number = fixture.requests.length; + await fixture.knowledge.readChunks({ + ...imageReference, + pageSize: 200, + includeAssetUrls: false, + }); + expect(fixture.requests).toHaveLength(beforeMetadataOnly); + }); + + it('resolves only selected page asset references for the requested page', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const paths: readonly string[] = addPageImages(fixture, 188); + await fixture.knowledge.prepareReadableSnapshot(reference); + await fixture.knowledge.readChunks({ ...imageReference, page: 2, pageSize: 100 }); + expect(fixture.batchRequests).toEqual([paths.slice(100)]); + }); + + it.each([ + { paths: [] }, + { paths: ['images/page-1.jpg', 'images/page-1.jpg'] }, + { paths: ['images/page-1.jpg', 'images/foreign.jpg'] }, + { paths: ['images/foreign.jpg'] }, + ])( + 'rejects incomplete or conflicting batch membership $paths', + async ({ paths }): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.setBatchResponsePaths(paths); + await expect(fixture.knowledge.readChunks(imageReference)).rejects.toBeInstanceOf( + ValidationError, + ); + }, + ); + + it.each(['snapshot_id', 'job_id', 'job_result_id', 'document_id', 'checksum'])( + 'rejects missing and conflicting %s in batch capabilities', + async (field: string): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + for (const value of [null, 'foreign']) { + fixture.setBatchIdentity({ [field]: value }); + await expect(fixture.knowledge.readChunks(imageReference)).rejects.toBeInstanceOf( + ValidationError, + ); + } + }, + ); + + it('deduplicates public URL requests and splits more than 200 image paths into ordered batches', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const paths: readonly string[] = addPageImages(fixture, 401); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.requests.length = 0; + const urls = await fixture.knowledge.getParsedDocumentObjectUrls({ + ...imageReference, + paths: [...paths, paths[0], 'images/unlisted.jpg'], + }); + expect(Object.keys(urls)).toHaveLength(402); + expect(urls[paths[400]]).toContain('/origin/images%2Fpage-401.jpg'); + expect(urls['images/unlisted.jpg']).toBeNull(); + expect(fixture.batchRequests).toEqual([ + paths.slice(0, 200), + paths.slice(200, 400), + paths.slice(400), + ]); + expect(fixture.requests).toHaveLength(3); + }); + + it('loads the readable descriptor and full certificate once for all URL paths without reading image bytes', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const paths: readonly string[] = addPageImages(fixture, 3); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.objects.set( + 'snapshot_baseline/.knowhere-sdk/commit.json', + Buffer.from(JSON.stringify(createFullCommit(fixture))), + ); + fixture.storage.getObjectUrl = (params): Promise => + Promise.resolve(`https://replica.example/${params.path}`); + const reads: string[] = []; + const readObject = fixture.storage.readObject.bind(fixture.storage); + fixture.storage.readObject = (params): Promise => { + reads.push(params.path); + return readObject(params); + }; + const requestCount: number = fixture.requests.length; + expect( + await fixture.knowledge.getParsedDocumentObjectUrls({ ...imageReference, paths }), + ).toEqual(Object.fromEntries(paths.map((path) => [path, `https://replica.example/${path}`]))); + expect(reads).toEqual(['.knowhere-sdk/readable.json', '.knowhere-sdk/commit.json']); + expect(fixture.requests).toHaveLength(requestCount); + }); + + it('resolves legacy full-copy URLs without requiring immutable delivery endpoints', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const paths: readonly string[] = ['images/page-1.jpg', 'source.pdf']; + fixture.storage.getObjectUrl = (params): Promise => + Promise.resolve(`https://replica.example/${params.path}`); + expect( + await fixture.knowledge.getParsedDocumentObjectUrls({ ...imageReference, paths }), + ).toEqual(Object.fromEntries(paths.map((path) => [path, null]))); + fixture.objects.set( + 'snapshot_baseline/.knowhere-sdk/commit.json', + Buffer.from( + JSON.stringify({ + version: 1, + documentId: imageReference.documentId, + revisionKey: imageReference.revisionKey, + source: 'resultZip', + committedAt: new Date().toISOString(), + }), + ), + ); + expect( + await fixture.knowledge.getParsedDocumentObjectUrls({ ...imageReference, paths }), + ).toEqual(Object.fromEntries(paths.map((path) => [path, `https://replica.example/${path}`]))); + expect(fixture.requests).toEqual([]); + }); + + it('rejects a conflicting full-copy pin before returning batch image URLs', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.objects.set( + 'snapshot_baseline/.knowhere-sdk/commit.json', + Buffer.from( + JSON.stringify({ + ...createFullCommit(fixture), + checksum: '0'.repeat(64), + }), + ), + ); + fixture.storage.getObjectUrl = (params): Promise => + Promise.resolve(`https://replica.example/${params.path}`); + const requestCount: number = fixture.requests.length; + await expect( + fixture.knowledge.getParsedDocumentObjectUrls({ + ...imageReference, + paths: [imageReference.path], + }), + ).rejects.toBeInstanceOf(ValidationError); + expect(fixture.requests).toHaveLength(requestCount); + }); + + it('stops after an aborted capability batch before requesting another shard', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const paths: readonly string[] = addPageImages(fixture, 201); + await fixture.knowledge.prepareReadableSnapshot(reference); + const controller: AbortController = new AbortController(); + const getAccess = fixture.client.jobs.getSnapshotObjectsAccess.bind(fixture.client.jobs); + fixture.client.jobs.getSnapshotObjectsAccess = async ( + ...params + ): Promise => { + const access = await getAccess(...params); + controller.abort(new Error('Cancelled after first batch')); + return access; + }; + await expect( + fixture.knowledge.getParsedDocumentObjectUrls({ + ...imageReference, + paths, + signal: controller.signal, + }), + ).rejects.toThrow('Cancelled after first batch'); + expect(fixture.batchRequests).toEqual([paths.slice(0, 200)]); + }); + + it('reads pinned metadata without image capability requests when asset URLs are disabled', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.denyImageAccess(); + const requestCount: number = fixture.requests.length; + const read = await fixture.knowledge.readChunks({ ...imageReference, includeAssetUrls: false }); + expect(read.chunks[0]?.content).toBe('Pinned baseline text'); + expect(read.chunks[0]?.metadata.pageAssets).toEqual([{ artifactRef: 'images/page-1.jpg' }]); + expect(fixture.requests).toHaveLength(requestCount); + }); + + it.each([ + '![one](images/first figure.jpg) and ![two](images/second figure.jpg)', + '[images/first figure.jpg] and [images/second figure.jpg]', + ])( + 'keeps same-line image references with spaces separate: %s', + async (content: string): Promise => { + const fixture: ReadableFixture = await createFixture(); + updateOriginObject(fixture, 'images/first figure.jpg', Buffer.from('first')); + updateOriginObject(fixture, 'images/second figure.jpg', Buffer.from('second')); + updateOriginObject( + fixture, + 'chunks.json', + Buffer.from( + JSON.stringify({ + chunks: [ + { + chunk_id: 'page_1', + type: 'page', + content, + path: 'manual.pdf/Page 1', + metadata: { + page_assets: [ + { artifact_ref: 'images/first figure.jpg' }, + { artifact_ref: 'images/second figure.jpg' }, + ], + }, + }, + ], + }), + ), + ); + await fixture.knowledge.prepareReadableSnapshot(reference); + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/readable.json')).toBe(true); + expect(fixture.requests.some((request): boolean => request.includes('/origin/images'))).toBe( + false, + ); + }, + ); + + it('still rejects an undeclared structured image reference alongside valid Markdown links', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + updateOriginObject( + fixture, + 'chunks.json', + Buffer.from( + JSON.stringify({ + chunks: [ + { + type: 'page', + content: '![one](images/page-1.jpg)', + metadata: { page_assets: [{ artifact_ref: 'images/unlisted.jpg' }] }, + }, + ], + }), + ), + ); + await expect(fixture.knowledge.prepareReadableSnapshot(reference)).rejects.toThrow( + /images\/unlisted.jpg/, + ); + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/readable.json')).toBe(false); + }); + + it('keeps damaged uncommitted image URLs on the pinned origin without reading image bodies', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.objects.set('snapshot_baseline/images/page-1.jpg', Buffer.from('damaged replica')); + fixture.storage.getObjectUrl = (params): Promise => + Promise.resolve(`https://replica.example/${params.path}`); + const readObject = fixture.storage.readObject.bind(fixture.storage); + const bodyReads: string[] = []; + fixture.storage.readObject = (params): Promise => { + if (params.path.startsWith('images/')) bodyReads.push(params.path); + return readObject(params); + }; + const imageUrl = await fixture.knowledge.getParsedDocumentObjectUrl(imageReference); + expect(imageUrl).toMatch(/\/origin\/images%2Fpage-1.jpg\?token=/); + expect( + await fixture.knowledge.getParsedDocumentObjectUrl({ ...imageReference, path: 'source.pdf' }), + ).toBe('https://replica.example/source.pdf'); + expect(bodyReads).toEqual([]); + expect(fixture.requests.some((request): boolean => request.includes('/origin/images'))).toBe( + false, + ); + await expect(fixture.knowledge.readParsedDocumentObject(imageReference)).rejects.toThrow( + /replica integrity/i, + ); + }); + + it.each(['documentId', 'revisionKey', 'snapshotId', 'jobResultId', 'checksum', 'stage'])( + 'rejects a full replica certificate with conflicting %s', + async (field: string): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + const commit = createFullCommit(fixture); + commit[field] = field === 'stage' ? 'final' : 'different'; + fixture.objects.set( + 'snapshot_baseline/.knowhere-sdk/commit.json', + Buffer.from(JSON.stringify(commit)), + ); + fixture.storage.getObjectUrl = (): Promise => + Promise.resolve('https://replica.example/page-1.jpg'); + await expect( + fixture.knowledge.getParsedDocumentObjectUrl(imageReference), + ).rejects.toBeInstanceOf(ValidationError); + }, + ); + + it('publishes readable core without image copies or a full-sync commit', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const prepared = await fixture.knowledge.prepareReadableSnapshot(reference); + expect(prepared.snapshot).toEqual(fixture.snapshot); + expect(prepared.readable.snapshotId).toBe('snapshot_baseline'); + expect(fixture.objects.has('snapshot_baseline/source.pdf')).toBe(true); + expect(fixture.objects.has('snapshot_baseline/tables/table-1.html')).toBe(true); + expect(fixture.objects.has('snapshot_baseline/images/page-1.jpg')).toBe(false); + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/commit.json')).toBe(false); + expect(fixture.progress.size).toBe(0); + expect(fixture.requests.some((request): boolean => request.includes('/result.zip'))).toBe( + false, + ); + expect(JSON.stringify(prepared.readable)).not.toContain('downloadUrl'); + expect(JSON.stringify(prepared.readable)).not.toContain('resultUrl'); + const read = await fixture.knowledge.readChunks(imageReference); + expect(read.chunks[0]?.content).toBe('Pinned baseline text'); + expect(read.document.sourceFileName).toBe('manual.pdf'); + expect(read.document.jobId).toBe('job_readable'); + expect((await fixture.knowledge.getDocumentOutline(imageReference)).totalChunks).toBe(1); + expect( + (await fixture.knowledge.grepChunks({ ...imageReference, pattern: 'baseline' })).matches, + ).toHaveLength(1); + expect(fixture.requests.some((request): boolean => request.includes('/documents/'))).toBe( + false, + ); + }); + + it('refreshes pinned image access on every origin read and prefers a replica once present', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + const first = await fixture.knowledge.readParsedDocumentObject(imageReference); + expect(Buffer.from(first?.body ?? []).toString()).toBe('pinned-image-bytes'); + expect(first?.contentType).toBe('image/jpeg'); + expect(fixture.objects.has('snapshot_baseline/images/page-1.jpg')).toBe(false); + await fixture.knowledge.readParsedDocumentObject(imageReference); + const imageReads: string[] = fixture.requests.filter((request): boolean => + request.includes('/origin/images%2Fpage-1.jpg'), + ); + expect(imageReads).toHaveLength(2); + expect(new Set(imageReads).size).toBe(2); + fixture.objects.set('snapshot_baseline/images/page-1.jpg', Buffer.from('pinned-image-bytes')); + const requestCount: number = fixture.requests.length; + await fixture.knowledge.readParsedDocumentObject(imageReference); + expect(fixture.requests).toHaveLength(requestCount); + await expect( + fixture.knowledge.readParsedDocumentObject({ ...imageReference, path: '../secret' }), + ).rejects.toThrow(/path/i); + }); + + it('resumes interrupted core writes without re-downloading valid durable objects', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const write = fixture.storage.writeObject.bind(fixture.storage); + fixture.storage.writeObject = (params): Promise => { + if (params.path === 'source.pdf') return Promise.reject(new Error('Storage interrupted')); + return write(params); + }; + await expect(fixture.knowledge.prepareReadableSnapshot(reference)).rejects.toThrow( + 'Storage interrupted', + ); + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/readable.json')).toBe(false); + fixture.storage.writeObject = write; + const restarted = fixture.client.knowledge.withParsedStorage({ storage: fixture.storage }); + await restarted.prepareReadableSnapshot(reference); + expect( + fixture.requests.filter((request): boolean => request.includes('/origin/manifest.json')), + ).toHaveLength(1); + const requestsBeforeRepeat: number = fixture.requests.length; + await restarted.prepareReadableSnapshot(reference); + expect(fixture.requests).toHaveLength(requestsBeforeRepeat + 1); + }); + + it('rejects missing image references and corrupt essential bytes before readability', async (): Promise => { + const missing: ReadableFixture = await createFixture(); + missing.inventory.splice( + missing.inventory.findIndex((object): boolean => object.kind === 'image'), + 1, + ); + await expect(missing.knowledge.prepareReadableSnapshot(reference)).rejects.toThrow( + /image|reference/i, + ); + expect(missing.objects.has('snapshot_baseline/.knowhere-sdk/readable.json')).toBe(false); + const corrupt: ReadableFixture = await createFixture(); + corrupt.origins.set('source.pdf', Buffer.from('wrong-source')); + await expect(corrupt.knowledge.prepareReadableSnapshot(reference)).rejects.toThrow( + /integrity|checksum|length/i, + ); + expect(corrupt.objects.has('snapshot_baseline/.knowhere-sdk/readable.json')).toBe(false); + }); + + it('rejects a changed origin snapshot even when its upstream result ID is unchanged', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.setOriginSnapshot('snapshot_final'); + await expect(fixture.knowledge.readParsedDocumentObject(imageReference)).rejects.toThrow( + /identity|snapshot/i, + ); + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/commit.json')).toBe(false); + }); + + it('keeps the existing archive checkpoint and full-copy commit barrier', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + let completed: boolean = false; + for (let segment: number = 0; segment < 12 && !completed; segment += 1) { + completed = (await fixture.knowledge.syncParsedDocument(reference)).completed; + if (!completed) + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/commit.json')).toBe(false); + } + expect(completed).toBe(true); + expect(fixture.progress.get('snapshot_baseline')?.status).toBe('completed'); + expect(fixture.objects.has('snapshot_baseline/images/page-1.jpg')).toBe(true); + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/readable.json')).toBe(true); + }); + + it('repairs missing core after restart before returning an existing readable descriptor', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.objects.delete('snapshot_baseline/source.pdf'); + await fixture.client.knowledge + .withParsedStorage({ storage: fixture.storage }) + .prepareReadableSnapshot(reference); + expect(fixture.objects.has('snapshot_baseline/source.pdf')).toBe(true); + }); + + it('does not certify a storage sink that acknowledged corrupt essential bytes', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const write = fixture.storage.writeObject.bind(fixture.storage); + fixture.storage.writeObject = (params): Promise => + write(params.path === 'source.pdf' ? { ...params, body: Buffer.from('corrupt') } : params); + await expect(fixture.knowledge.prepareReadableSnapshot(reference)).rejects.toThrow( + /integrity/i, + ); + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/readable.json')).toBe(false); + }); + + it('does not expose uncommitted local objects or undeclared origin paths', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + fixture.objects.set('snapshot_baseline/images/page-1.jpg', Buffer.from('partial')); + expect(await fixture.knowledge.readParsedDocumentObject(imageReference)).toBeNull(); + fixture.objects.delete('snapshot_baseline/images/page-1.jpg'); + await fixture.knowledge.prepareReadableSnapshot(reference); + const requests: number = fixture.requests.length; + expect( + await fixture.knowledge.readParsedDocumentObject({ + ...imageReference, + path: 'images/unknown.jpg', + }), + ).toBeNull(); + expect(fixture.requests).toHaveLength(requests); + }); + + it('waits for an active core write to settle on cancellation and leaves replication untouched', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const controller: AbortController = new AbortController(); + let markStarted: () => void = (): void => {}; + let release: () => void = (): void => {}; + const started: Promise = new Promise((resolve): void => { + markStarted = resolve; + }); + const gate: Promise = new Promise((resolve): void => { + release = resolve; + }); + const write = fixture.storage.writeObject.bind(fixture.storage); + fixture.storage.writeObject = async (params): Promise => { + if (params.path === 'source.pdf') { + markStarted(); + await gate; + } + return write(params); + }; + let hasSettled: boolean = false; + const pending = fixture.knowledge.prepareReadableSnapshot({ + ...reference, + signal: controller.signal, + }); + void pending.then( + (): void => { + hasSettled = true; + }, + (): void => { + hasSettled = true; + }, + ); + await started; + controller.abort(new Error('Stop core transfer')); + await Promise.resolve(); + expect(hasSettled).toBe(false); + release(); + await expect(pending).rejects.toThrow('Stop core transfer'); + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/readable.json')).toBe(false); + expect(fixture.progress.size).toBe(0); + await fixture.knowledge.prepareReadableSnapshot(reference); + }); + + it('preserves full-copy rejection for a missing referenced archive image while core stays readable', async (): Promise => { + const fixture: ReadableFixture = await createFixture(true); + await fixture.knowledge.prepareReadableSnapshot(reference); + const knowledge = fixture.client.knowledge.withParsedStorage({ storage: fixture.storage }); + await expect(knowledge.syncParsedDocument(reference)).rejects.toThrow( + /Required parsed result file missing: images\/page-1.jpg/, + ); + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/commit.json')).toBe(false); + expect(fixture.progress.get('snapshot_baseline')?.status).not.toBe('completed'); + expect((await knowledge.readChunks(imageReference)).chunks[0]?.content).toBe( + 'Pinned baseline text', + ); + expect( + Buffer.from( + (await knowledge.readParsedDocumentObject(imageReference))?.body ?? [], + ).toString(), + ).toBe('pinned-image-bytes'); + }); + + it('persists only stable inventory fields even when delivery includes an expiring capability', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + Object.assign(fixture.inventory[0], { + download_url: 'https://origin.example/secret?expires=1', + }); + const result = await fixture.knowledge.prepareReadableSnapshot(reference); + expect(JSON.stringify(result.readable)).not.toContain('origin.example'); + expect( + Buffer.from( + fixture.objects.get('snapshot_baseline/.knowhere-sdk/readable.json') ?? [], + ).toString(), + ).not.toContain('origin.example'); + }); + + it('requires pinned origins for navigation resource paths', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + const body: Uint8Array = Buffer.from( + JSON.stringify({ + sections: [], + resources: { images: [{ path: 'images/unlisted.jpg' }] }, + }), + ); + fixture.origins.set('doc_nav.json', body); + const index: number = fixture.inventory.findIndex( + (object): boolean => object.path === 'doc_nav.json', + ); + fixture.inventory[index] = { + ...fixture.inventory[index], + size_bytes: body.byteLength, + checksum_sha256: createHash('sha256').update(body).digest('hex'), + }; + await expect(fixture.knowledge.prepareReadableSnapshot(reference)).rejects.toThrow( + /reference|origin/i, + ); + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/readable.json')).toBe(false); + }); + + it('rejects corrupt image origin bytes without certifying replication or damaging readable text', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.origins.set('images/page-1.jpg', Buffer.from('corrupt-image')); + await expect(fixture.knowledge.readParsedDocumentObject(imageReference)).rejects.toThrow( + /integrity/i, + ); + expect(fixture.progress.size).toBe(0); + expect((await fixture.knowledge.readChunks(imageReference)).chunks[0]?.content).toBe( + 'Pinned baseline text', + ); + }); + + it('hydrates uncopied page images with a fresh readable URL and prefers a certified replica URL', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + const read = await fixture.knowledge.readChunks(imageReference); + const pageAssets = read.chunks[0]?.metadata.pageAssets as { readonly assetUrl?: string }[]; + expect(pageAssets[0]?.assetUrl).toMatch(/^http:\/\//); + const response: Response = await fetch(pageAssets[0]?.assetUrl ?? ''); + expect(await response.text()).toBe('pinned-image-bytes'); + expect(fixture.objects.has('snapshot_baseline/images/page-1.jpg')).toBe(false); + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/commit.json')).toBe(false); + const refreshed: string | null = + await fixture.knowledge.getParsedDocumentObjectUrl(imageReference); + expect(refreshed).not.toBe(pageAssets[0]?.assetUrl); + fixture.objects.set('snapshot_baseline/images/page-1.jpg', Buffer.from('pinned-image-bytes')); + fixture.objects.set( + 'snapshot_baseline/.knowhere-sdk/commit.json', + Buffer.from(JSON.stringify(createFullCommit(fixture))), + ); + fixture.storage.getObjectUrl = (params): Promise => + Promise.resolve( + params.path === imageReference.path ? 'https://replica.example/page-1.jpg' : null, + ); + const requestCount: number = fixture.requests.length; + const readObject = fixture.storage.readObject.bind(fixture.storage); + const imageBodyReads: string[] = []; + fixture.storage.readObject = (params): Promise => { + if (params.path.startsWith('images/')) imageBodyReads.push(params.path); + return readObject(params); + }; + const localRead = await fixture.knowledge.readChunks(imageReference); + expect(localRead.chunks[0]?.metadata.pageAssets).toMatchObject([ + { assetUrl: 'https://replica.example/page-1.jpg' }, + ]); + expect(fixture.requests).toHaveLength(requestCount); + expect(imageBodyReads).toEqual([]); + }); + + it('rejects wrong-version image capabilities during readChunks URL hydration', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.setOriginSnapshot('snapshot_final'); + await expect(fixture.knowledge.readChunks(imageReference)).rejects.toBeInstanceOf( + ValidationError, + ); + }); + + it.each(['2000-01-01T00:00:00Z', '2030-99-01T00:00:00Z'])( + 'rejects expired or malformed image capabilities (%s)', + async (expiresAt: string): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.setCapabilityExpiration(expiresAt); + await expect( + fixture.knowledge.getParsedDocumentObjectUrl(imageReference), + ).rejects.toBeInstanceOf(ValidationError); + }, + ); + + it('propagates a typed authorization failure without replacing the readable snapshot', async (): Promise => { + const fixture: ReadableFixture = await createFixture(); + await fixture.knowledge.prepareReadableSnapshot(reference); + fixture.denyImageAccess(); + await expect(fixture.knowledge.readChunks(imageReference)).rejects.toBeInstanceOf( + PermissionDeniedError, + ); + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/readable.json')).toBe(true); + expect(fixture.objects.has('snapshot_baseline/.knowhere-sdk/commit.json')).toBe(false); + }); +}); diff --git a/src/knowledge/index.ts b/src/knowledge/index.ts index 4ed7014..d649785 100644 --- a/src/knowledge/index.ts +++ b/src/knowledge/index.ts @@ -27,6 +27,10 @@ export type { KnowledgeStartupRecoveryResponse, KnowledgeSyncParsedDocumentParams, KnowledgeSyncParsedDocumentResponse, + KnowledgePrepareReadableSnapshotParams, + KnowledgePrepareReadableSnapshotResponse, + KnowledgeReadParsedDocumentObjectParams, + KnowledgeReadParsedDocumentObjectUrlsParams, LocalKnowledgeDocument, LocalKnowledgeParseResponse, } from './types.js'; diff --git a/src/knowledge/knowledge.ts b/src/knowledge/knowledge.ts index 9cbd4f2..5b79ba4 100644 --- a/src/knowledge/knowledge.ts +++ b/src/knowledge/knowledge.ts @@ -25,6 +25,7 @@ import { } from '../storage/streamed-parsed-document-sync.js'; import { LocalKnowledgeStore } from './local-store.js'; import { serializeSnapshotPublication } from './serialize-snapshot-publication.js'; +import { ReadableParsedDocumentSnapshots } from '../storage/readable-parsed-document-snapshots.js'; import type { IndexedKnowledgeChunk, KnowledgeAsyncCacheResult, @@ -46,6 +47,8 @@ import type { KnowledgeReadChunk, KnowledgeReadParams, KnowledgeReadResponse, + KnowledgeChunkDescriptor, + KnowledgeDocumentSummary, KnowledgeSearchParams, KnowledgeSearchReference, KnowledgeSearchResponse, @@ -54,14 +57,20 @@ import type { KnowledgeStartupRecoveryResponse, KnowledgeSyncParsedDocumentParams, KnowledgeSyncParsedDocumentResponse, + KnowledgePrepareReadableSnapshotParams, + KnowledgePrepareReadableSnapshotResponse, + KnowledgeReadParsedDocumentObjectParams, + KnowledgeReadParsedDocumentObjectUrlsParams, KnowledgeChunkType, LocalKnowledgeDocument, LocalKnowledgeParseResponse, } from './types.js'; import type { - ParsedDocumentCommit, ParsedDocumentStorageConfig, ParsedDocumentStorageLimits, + ParsedDocumentObject, + ParsedDocumentCommit, + ParsedDocumentReadableSnapshot, } from '../types/storage.js'; const DEFAULT_READ_LIMIT = 12; @@ -238,6 +247,72 @@ export class Knowledge { return response; } + /** Establish a durable readable core without advancing archive replication or display pointers. */ + async prepareReadableSnapshot( + params: KnowledgePrepareReadableSnapshotParams, + ): Promise { + params.signal?.throwIfAborted(); + const storage = this.requireParsedStorageConfig().storage; + if (!params.jobId.trim() || !params.revisionKey.trim()) + throw new ValidationError('A job ID and immutable snapshot revisionKey are required.'); + const job: JobResult = await this.client.jobs.get(params.jobId, { signal: params.signal }); + params.signal?.throwIfAborted(); + const snapshot: JobSnapshot | undefined = job.snapshots?.find( + (candidate: JobSnapshot): boolean => candidate.snapshotId === params.revisionKey, + ); + if ( + !job.isDone || + !snapshot || + snapshot.jobId !== params.jobId || + (params.documentId !== undefined && snapshot.documentId !== params.documentId) || + (job.documentId !== undefined && snapshot.documentId !== job.documentId) + ) + throw new ValidationError( + 'Requested immutable readable snapshot is unavailable or differs from the request.', + ); + const readable: ParsedDocumentReadableSnapshot = await new ReadableParsedDocumentSnapshots( + this.client.jobs, + storage, + ).prepare(snapshot, params.signal); + params.signal?.throwIfAborted(); + return { + documentId: snapshot.documentId, + revisionKey: snapshot.snapshotId, + snapshot, + readable, + }; + } + + /** Read a stored object, or an uncopied image through its fresh pinned origin capability. */ + async readParsedDocumentObject( + params: KnowledgeReadParsedDocumentObjectParams, + ): Promise { + return new ReadableParsedDocumentSnapshots( + this.client.jobs, + this.requireParsedStorageConfig().storage, + ).readObject(params); + } + + /** Resolve a local URL or a fresh immutable image capability without copying the image. */ + async getParsedDocumentObjectUrl( + params: KnowledgeReadParsedDocumentObjectParams, + ): Promise { + return new ReadableParsedDocumentSnapshots( + this.client.jobs, + this.requireParsedStorageConfig().storage, + ).getObjectUrl(params); + } + + /** Resolve exact requested paths in batches; absent objects have null URLs. No image bodies are read. */ + async getParsedDocumentObjectUrls( + params: KnowledgeReadParsedDocumentObjectUrlsParams, + ): Promise>> { + return new ReadableParsedDocumentSnapshots( + this.client.jobs, + this.requireParsedStorageConfig().storage, + ).getObjectUrls(params); + } + private async performParsedDocumentSync( params: KnowledgeSyncParsedDocumentParams, ): Promise { @@ -532,6 +607,39 @@ export class Knowledge { return this.store.listDocuments(); } + /** Read a complete metadata index without constructing chunk bodies or resolving asset URLs. */ + async getDocumentSummary( + reference: KnowledgeDocumentReference | string, + ): Promise { + const normalized: KnowledgeDocumentReference = normalizeDocumentReference(reference); + if (normalized.localDocumentId && !normalized.documentId) { + const document: LocalKnowledgeDocument | undefined = await this.store.getDocument( + normalized.localDocumentId, + ); + if (document) { + const metadata = await this.store.loadResultMetadata(normalized.localDocumentId); + return parseDocumentMetadata(metadata.manifest, metadata.chunks, null).documentSummary; + } + } + const documentId: string | undefined = + normalized.documentId ?? + (normalized.localDocumentId && looksLikeRemoteDocumentId(normalized.localDocumentId) + ? normalized.localDocumentId + : undefined); + const revisionKey: string | undefined = + normalized.revisionKey ?? + normalized.jobId ?? + (documentId ? await this.resolveStoredSnapshotKey(documentId) : undefined); + if (documentId && revisionKey) { + const metadata: StoredParsedDocumentMetadata | null = + await this.readCommittedParsedStorageMetadata(documentId, revisionKey); + if (metadata) return metadata.documentSummary; + } + throw new ValidationError( + 'Document summary requires a complete local or committed/readable stored parsed snapshot; prepare or synchronize the pinned snapshot first', + ); + } + async getDocumentOutline( reference: string | KnowledgeDocumentReference, ): Promise { @@ -584,18 +692,43 @@ export class Knowledge { const { document, result } = await this.loadReadableResult(params); const limit = clampLimit(params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT); - const chunks = indexChunks(result).filter((chunk) => matchesReadScope(chunk, params)); - const selected = selectReadWindow(chunks, params, limit); + const records: readonly Record[] = result.chunks.map((chunk) => ({ + ...chunk, + })); + const documentSummary: KnowledgeDocumentSummary = summarizeStoredRecords( + records, + result.manifest, + null, + ); + const chunks = documentSummary.chunks.filter((chunk) => matchesDescriptorScope(chunk, params)); + const selectedDescriptors = selectReadDescriptors(chunks, params, limit); + const selected = selectedDescriptors.flatMap((descriptor) => { + const source = result.chunks[descriptor.position - 1]; + if (source === undefined) return []; + return indexChunks({ ...result, chunks: [source] }).map((chunk) => + narrowReadChunkPage({ ...chunk, position: descriptor.position }, params.sourcePageNumber), + ); + }); const lastSelected = selected[selected.length - 1]; const nextChunk = lastSelected && selected.length < chunks.length - ? chunks[chunks.indexOf(lastSelected) + 1]?.position + ? chunks[chunks.findIndex((chunk) => chunk.position === lastSelected.position) + 1] + ?.position : undefined; return { document, + documentSummary, chunks: selected.map(toReadChunk), nextChunk, + ...(params.sourcePageNumber === undefined + ? {} + : { + page: getReadPageParams(params).page, + pageSize: getReadPageParams(params).pageSize, + totalChunks: chunks.length, + totalPages: Math.max(1, Math.ceil(chunks.length / getReadPageParams(params).pageSize)), + }), }; } @@ -749,6 +882,11 @@ export class Knowledge { throw new ValidationError('Requested parsed snapshot is not committed locally'); } + if (params.sourcePageNumber !== undefined) { + throw new ValidationError( + 'sourcePageNumber requires a complete local or stored parsed snapshot; prepare or synchronize the pinned snapshot before reading a physical page', + ); + } if (hasScanReadFilters(params)) { return this.scanRemoteDocumentChunks(documentId, params); } @@ -758,7 +896,7 @@ export class Knowledge { page: pageParams.page, pageSize: pageParams.pageSize, chunkType: params.chunkType, - includeAssetUrls: true, + includeAssetUrls: params.includeAssetUrls ?? true, }); const revisionKey = getRemoteRevisionKey(remotePage); if (params.revisionKey && params.revisionKey !== revisionKey) @@ -813,7 +951,7 @@ export class Knowledge { page: 1, pageSize: limits.remotePageSize, chunkType: params.chunkType, - includeAssetUrls: true, + includeAssetUrls: params.includeAssetUrls ?? true, }); const revisionKey = getRemoteRevisionKey(firstPage); if (params.revisionKey && params.revisionKey !== revisionKey) @@ -850,7 +988,7 @@ export class Knowledge { page, pageSize: limits.remotePageSize, chunkType: params.chunkType, - includeAssetUrls: true, + includeAssetUrls: params.includeAssetUrls ?? true, }); } @@ -1095,30 +1233,29 @@ export class Knowledge { const storedResult = await this.readCommittedParsedStorageResult( params.documentId, params.revisionKey, + params.params, ); if (!storedResult) return null; const pageParams = getReadPageParams(params.params); - const scopedChunks = storedResult.chunks - .filter((chunk) => !params.params.chunkType || chunk.chunkType === params.params.chunkType) - .filter((chunk) => matchesReadScope(chunk, params.params)); - const selected = hasScanReadFilters(params.params) - ? selectReadWindow( - scopedChunks, - params.params, - clampLimit(params.params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT), - ) - : scopedChunks.slice( - (pageParams.page - 1) * pageParams.pageSize, - pageParams.page * pageParams.pageSize, - ); - const visibleChunks = await applyStoredAssetUrlsToReadChunks({ - storageConfig: this.requireParsedStorageConfig(), - documentId: params.documentId, - revisionKey: params.revisionKey, - chunks: selected.map(toReadChunk), - }); - const totalChunks = scopedChunks.length; + const selected = storedResult.chunks; + const chunks: KnowledgeReadChunk[] = selected.map(toReadChunk); + const visibleChunks = + params.params.includeAssetUrls === false + ? chunks + : await applyStoredAssetUrlsToReadChunks({ + chunks, + resolveObjectUrls: ( + paths: readonly string[], + ): Promise>> => + this.getParsedDocumentObjectUrls({ + documentId: params.documentId, + revisionKey: params.revisionKey, + paths, + }), + }); + const totalChunks = storedResult.scopedChunkCount ?? selected.length; return { + documentSummary: storedResult.documentSummary, document: createStoredKnowledgeDocument({ documentId: params.documentId, storedResult, @@ -1129,7 +1266,7 @@ export class Knowledge { pageSize: hasScanReadFilters(params.params) ? undefined : pageParams.pageSize, totalChunks, totalPages: Math.max(1, Math.ceil(totalChunks / pageParams.pageSize)), - nextChunk: getNextChunkPosition(scopedChunks, selected), + nextChunk: storedResult.nextChunk, }; } @@ -1228,7 +1365,61 @@ export class Knowledge { private async readCommittedParsedStorageResult( documentId: string, revisionKey: string, + readParams?: KnowledgeReadParams, ): Promise { + const metadata: StoredParsedDocumentMetadata | null = + await this.readCommittedParsedStorageMetadata(documentId, revisionKey, 'read'); + if (!metadata) return null; + const { manifest, records, documentSummary, docNav } = metadata; + const scoped = documentSummary.chunks.filter((chunk) => + matchesDescriptorScope(chunk, readParams), + ); + const page = getReadPageParams(readParams ?? {}); + const selected = + readParams === undefined + ? documentSummary.chunks + : hasScanReadFilters(readParams) + ? selectReadDescriptors( + scoped, + readParams, + clampLimit(readParams.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT), + ) + : scoped.slice((page.page - 1) * page.pageSize, page.page * page.pageSize); + const chunks = selected.flatMap((descriptor) => { + const record = records[descriptor.position - 1]; + if (record === undefined) return []; + const parsed = parseStoredResultChunks({ + payload: [record], + sourceFileName: manifest.sourceFileName, + }); + return parsed.map((chunk) => + narrowReadChunkPage( + { ...chunk, position: descriptor.position }, + readParams?.sourcePageNumber, + ), + ); + }); + return { + storedAt: metadata.storedAt, + manifest, + chunks, + docNav: docNav?.sections ? { sections: docNav.sections } : undefined, + typeCounts: documentSummary.typeCounts, + documentSummary, + scopedChunkCount: scoped.length, + nextChunk: + selected.length > 0 + ? scoped[scoped.findIndex((chunk) => chunk.position === selected.at(-1)?.position) + 1] + ?.position + : undefined, + }; + } + + private async readCommittedParsedStorageMetadata( + documentId: string, + revisionKey: string, + mode: 'summary' | 'read' = 'summary', + ): Promise { const parsedStorageConfig = this.parsedStorageConfig; if (!parsedStorageConfig) { return null; @@ -1239,9 +1430,13 @@ export class Knowledge { documentId, revisionKey, }); - if (!commit) { - return null; - } + const readable: ParsedDocumentReadableSnapshot | null = commit + ? null + : await new ReadableParsedDocumentSnapshots( + this.client.jobs, + parsedStorageConfig.storage, + ).readDescriptor({ documentId, revisionKey }); + if (!commit && !readable) return null; const [manifestObject, chunksObject, docNavObject] = await Promise.all([ parsedStorageConfig.storage.readObject({ @@ -1254,30 +1449,32 @@ export class Knowledge { revisionKey, path: 'chunks.json', }), - parsedStorageConfig.storage.readObject({ - documentId, - revisionKey, - path: 'doc_nav.json', - }), + mode === 'read' + ? parsedStorageConfig.storage.readObject({ documentId, revisionKey, path: 'doc_nav.json' }) + : Promise.resolve(null), ]); if (!manifestObject || !chunksObject) { return null; } - const manifest = readJsonObject(manifestObject); - const chunks = parseStoredResultChunks({ - payload: readJsonObject(chunksObject), - sourceFileName: manifest.sourceFileName, - }); + const manifestPayload: unknown = readJsonObject(manifestObject); + if (mode === 'summary') + validateStoredMetadataCertificate(commit, readable, manifestPayload, [ + manifestObject, + chunksObject, + ]); const docNav = docNavObject - ? readJsonObject<{ readonly sections?: DocNavSection[] }>(docNavObject) + ? keysToCamel<{ readonly sections?: DocNavSection[] }>(readJsonObject(docNavObject)) : undefined; return { - commit, - manifest, - chunks, + storedAt: commit?.committedAt ?? readable?.readableAt ?? '', docNav: docNav?.sections ? { sections: docNav.sections } : undefined, - typeCounts: countIndexedTypes(chunks), + ...parseDocumentMetadata( + manifestPayload, + readJsonObject(chunksObject), + revisionKey, + mode === 'summary', + ), }; } @@ -1619,7 +1816,10 @@ interface ContinuationCursor { } interface StoredParsedDocumentResult { - readonly commit: ParsedDocumentCommit; + readonly documentSummary?: KnowledgeDocumentSummary; + readonly scopedChunkCount?: number; + readonly nextChunk?: number; + readonly storedAt: string; readonly manifest: Manifest; readonly chunks: IndexedKnowledgeChunk[]; readonly docNav?: { @@ -1628,6 +1828,19 @@ interface StoredParsedDocumentResult { readonly typeCounts: Record; } +interface ParsedDocumentMetadata { + readonly manifest: Manifest; + readonly records: readonly Record[]; + readonly documentSummary: KnowledgeDocumentSummary; +} + +interface StoredParsedDocumentMetadata extends ParsedDocumentMetadata { + readonly storedAt: string; + readonly docNav?: { + readonly sections: DocNavSection[]; + }; +} + const defaultParsedStorageScheduler = { schedule(task: () => Promise): void { void task(); @@ -1653,6 +1866,12 @@ function getParsedStorageLimits( } function validateReadParams(params: KnowledgeReadParams): void { + if ( + params.sourcePageNumber !== undefined && + (!Number.isSafeInteger(params.sourcePageNumber) || params.sourcePageNumber < 1) + ) { + throw new ValidationError('sourcePageNumber must be a positive safe integer'); + } const usesPagedDisplay = params.page !== undefined || params.pageSize !== undefined; if (!usesPagedDisplay) { return; @@ -1829,11 +2048,12 @@ function createStoredKnowledgeDocument(params: { jobId: params.storedResult.manifest.jobId || params.revisionKey, documentId: params.documentId, sourceFileName: params.storedResult.manifest.sourceFileName, - chunkCount: params.storedResult.chunks.length, + chunkCount: + params.storedResult.documentSummary?.chunkCount ?? params.storedResult.chunks.length, typeCounts: params.storedResult.typeCounts, resultDirectoryPath: `parsed-storage:${params.documentId}`, - createdAt: new Date(params.storedResult.commit.committedAt), - updatedAt: new Date(params.storedResult.commit.committedAt), + createdAt: new Date(params.storedResult.storedAt), + updatedAt: new Date(params.storedResult.storedAt), }; } @@ -1872,6 +2092,248 @@ function toIndexedRemoteChunk(chunk: DocumentChunk): IndexedKnowledgeChunk { return toIndexedReadChunk(readChunk); } +function readRawMetadata(record: Record): Record { + return isRecord(record.metadata) ? record.metadata : {}; +} + +function readRawPageAssets(metadata: Record): readonly Record[] { + const values: unknown = metadata.pageAssets ?? metadata.page_assets; + return Array.isArray(values) ? values.filter(isRecord) : []; +} + +function readAssetPageNumber(asset: Record): number | undefined { + const page: unknown = asset.pageNum ?? asset.pageNumber ?? asset.page_num ?? asset.page_number; + return typeof page === 'number' && Number.isSafeInteger(page) && page > 0 ? page : undefined; +} + +function hasRawPageAssetReference(asset: Record): boolean { + return Boolean(readRecordString(asset, 'artifactRef', 'artifact_ref', 'assetUrl', 'asset_url')); +} + +function readEmbeddedChunkIds(metadata: Record): readonly string[] { + const connections: readonly unknown[] = [metadata.connectTo, metadata.connect_to].flatMap( + (value) => toUnknownArray(value) ?? [], + ); + return [ + ...new Set( + connections.flatMap((connection): readonly string[] => + isRecord(connection) && + connection.relation === 'embeds' && + typeof connection.target === 'string' && + connection.target.length > 0 + ? [connection.target] + : [], + ), + ), + ]; +} + +function readPhysicalPageNumbers(metadata: Record): number[] { + const candidates: unknown[] = [ + metadata.pageNumber, + metadata.pageNum, + metadata.page_number, + metadata.page_num, + ]; + for (const value of [metadata.pageNumbers, metadata.pageNums, metadata.page_nums]) { + const values: readonly unknown[] | null = toUnknownArray(value); + if (values !== null) candidates.push(...values); + } + candidates.push(...readRawPageAssets(metadata).map(readAssetPageNumber)); + return [ + ...new Set( + candidates.filter( + (value): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value > 0, + ), + ), + ].sort((left, right) => left - right); +} + +function summarizeStoredRecords( + records: readonly Record[], + manifest: Manifest, + revisionKey: string | null, +): KnowledgeDocumentSummary { + const typeCounts: Record = { text: 0, image: 0, table: 0, page: 0 }; + let hasPageAssets: boolean = false; + let hasNativePageContent: boolean = false; + const chunks: KnowledgeChunkDescriptor[] = records.map((record, index) => { + const metadata: Record = readRawMetadata(record); + const rawType: string | undefined = readRecordString(record, 'type', 'chunkType', 'chunk_type'); + const chunkType: KnowledgeChunkType = isKnowledgeChunkType(rawType) ? rawType : 'text'; + const sourceChunkPath: string = + readRecordString(record, 'path', 'sourceChunkPath', 'source_chunk_path') ?? ''; + const filePath: string | undefined = + readRecordString(record, 'filePath', 'file_path') ?? + readRecordString(metadata, 'filePath', 'file_path'); + const hasAssets: boolean = readRawPageAssets(metadata).some(hasRawPageAssetReference); + const embeddedChunkIds: readonly string[] = readEmbeddedChunkIds(metadata); + typeCounts[chunkType] += 1; + hasPageAssets ||= hasAssets; + hasNativePageContent ||= (chunkType === 'page' || chunkType === 'text') && !hasAssets; + return { + chunkId: readRecordString(record, 'chunkId', 'chunk_id') ?? '', + position: index + 1, + chunkType, + sourceChunkPath, + sectionPath: normalizeSectionPath(sourceChunkPath, manifest.sourceFileName), + pageNumbers: readPhysicalPageNumbers(metadata), + ...(filePath === undefined ? {} : { filePath }), + ...(embeddedChunkIds.length === 0 ? {} : { embeddedChunkIds }), + }; + }); + const pageCount: number | undefined = [ + manifest.processing?.pageCount, + manifest.statistics?.totalPages, + ].find( + (value): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value > 0, + ); + return { + revisionKey, + chunkCount: chunks.length, + typeCounts, + pageCount: pageCount ?? null, + pageNumbers: [...new Set(chunks.flatMap((chunk) => chunk.pageNumbers))].sort( + (left, right) => left - right, + ), + hasPageAssets, + hasNativePageContent, + chunks, + }; +} + +function parseDocumentMetadata( + manifestPayload: unknown, + chunksPayload: unknown, + revisionKey: string | null, + shouldValidate: boolean = true, +): ParsedDocumentMetadata { + if ( + shouldValidate && + (!isRecord(manifestPayload) || + !readRecordString(manifestPayload, 'sourceFileName', 'source_file_name')) + ) + throw new ValidationError('Parsed snapshot manifest is missing a source file name'); + const values: readonly unknown[] | null = toUnknownArray( + isRecord(chunksPayload) ? chunksPayload.chunks : chunksPayload, + ); + if ( + shouldValidate && + (values === null || + !values.every( + (record): record is Record => + isRecord(record) && Boolean(readRecordString(record, 'chunkId', 'chunk_id')), + )) + ) + throw new ValidationError('Parsed snapshot chunks must be an array of chunk records with IDs'); + const manifest: Manifest = keysToCamel(manifestPayload); + const records: readonly Record[] = extractStoredChunkRecords(chunksPayload); + return { + manifest, + records, + documentSummary: summarizeStoredRecords(records, manifest, revisionKey), + }; +} + +function validateStoredMetadataCertificate( + commit: ParsedDocumentCommit | null, + readable: ParsedDocumentReadableSnapshot | null, + manifestPayload: unknown, + objects: readonly ParsedDocumentObject[], +): void { + if ( + commit && + (commit.version !== 1 || + (commit.snapshotId !== undefined && commit.snapshotId !== commit.revisionKey) || + (commit.stage !== undefined && commit.stage !== 'baseline' && commit.stage !== 'final')) + ) + throw new ValidationError('Invalid full-copy parsed snapshot metadata certificate'); + if (!isRecord(manifestPayload)) + throw new ValidationError('Invalid parsed snapshot manifest metadata'); + const jobResultId: string | undefined = readRecordString( + manifestPayload, + 'jobResultId', + 'job_result_id', + ); + if ( + (commit?.jobResultId !== undefined && + jobResultId !== undefined && + commit.jobResultId !== jobResultId) || + (readable && + (readRecordString(manifestPayload, 'jobId', 'job_id') !== readable.jobId || + (jobResultId !== undefined && jobResultId !== readable.jobResultId))) + ) + throw new ValidationError('Parsed snapshot manifest metadata differs from its pinned identity'); + if (!readable) return; + for (const object of objects) { + const entry = readable.objects.find((candidate): boolean => candidate.path === object.path); + if ( + !entry || + object.body.byteLength !== entry.sizeBytes || + createHash('sha256').update(object.body).digest('hex') !== entry.checksumSha256 + ) + throw new ValidationError(`Parsed snapshot metadata integrity check failed: ${object.path}`); + } +} + +function matchesDescriptorScope( + chunk: KnowledgeChunkDescriptor, + params: KnowledgeReadParams | undefined, +): boolean { + if (params === undefined) return true; + return ( + (!params.chunkId || chunk.chunkId === params.chunkId) && + (!params.chunkType || chunk.chunkType === params.chunkType) && + (!params.sectionPath || isInSection(chunk.sectionPath, params.sectionPath)) && + (params.sourcePageNumber === undefined || chunk.pageNumbers.includes(params.sourcePageNumber)) + ); +} + +function selectReadDescriptors( + chunks: readonly KnowledgeChunkDescriptor[], + params: KnowledgeReadParams, + limit: number, +): readonly KnowledgeChunkDescriptor[] { + if (params.sourcePageNumber !== undefined && !hasScanReadFilters(params)) { + const page = getReadPageParams(params); + return chunks.slice((page.page - 1) * page.pageSize, page.page * page.pageSize); + } + if (params.chunkId) return chunks.slice(0, limit); + return chunks + .filter( + (chunk) => + chunk.position >= (params.startChunk ?? 1) && + chunk.position <= (params.endChunk ?? Number.MAX_SAFE_INTEGER), + ) + .slice(0, limit); +} + +function narrowReadChunkPage( + chunk: IndexedKnowledgeChunk, + sourcePageNumber: number | undefined, +): IndexedKnowledgeChunk { + if (sourcePageNumber === undefined) return chunk; + const metadata: Record = { ...chunk.metadata }; + const pageNumbers: number[] = chunk.pageNumbers ?? readPhysicalPageNumbers(chunk.metadata); + if (Array.isArray(metadata.pageAssets) || Array.isArray(metadata.page_assets)) { + metadata.pageAssets = readRawPageAssets(metadata).filter( + (asset) => + readAssetPageNumber(asset) === sourcePageNumber || + (readAssetPageNumber(asset) === undefined && + pageNumbers.length === 1 && + pageNumbers[0] === sourcePageNumber), + ); + delete metadata.page_assets; + } + return { + ...chunk, + metadata, + pageNumbers, + }; +} + function parseStoredResultChunks(params: { readonly payload: unknown; readonly sourceFileName: string; @@ -2000,51 +2462,49 @@ function matchesRemoteReadScope(chunk: DocumentChunk, params: KnowledgeReadParam return matchesReadScope(toIndexedRemoteChunk(chunk), params); } -function getNextChunkPosition( - chunks: readonly IndexedKnowledgeChunk[], - selected: readonly IndexedKnowledgeChunk[], -): number | undefined { - const lastSelected = selected[selected.length - 1]; - if (!lastSelected || selected.length >= chunks.length) { - return undefined; - } - return chunks[chunks.indexOf(lastSelected) + 1]?.position; -} - async function applyStoredAssetUrlsToReadChunks(params: { - readonly storageConfig: ParsedDocumentStorageConfig; - readonly documentId: string; - readonly revisionKey: string; readonly chunks: readonly KnowledgeReadChunk[]; + readonly resolveObjectUrls: ( + paths: readonly string[], + ) => Promise>>; }): Promise { + const paths: Set = new Set(); + for (const chunk of params.chunks) { + if (chunk.filePath) paths.add(chunk.filePath); + for (const asset of getStoredPageAssets(chunk.metadata) ?? []) { + if (!isRecord(asset)) continue; + const artifactRef: string | undefined = readRecordString( + asset, + 'artifactRef', + 'artifact_ref', + ); + if (artifactRef) paths.add(artifactRef); + } + } + const urls: Readonly> = await params.resolveObjectUrls([...paths]); return Promise.all( params.chunks.map((chunk) => applyStoredAssetUrlsToReadChunk({ - ...params, chunk, + resolveObjectUrl: (path: string): Promise => + Promise.resolve(urls[path] ?? null), }), ), ); } async function applyStoredAssetUrlsToReadChunk(params: { - readonly storageConfig: ParsedDocumentStorageConfig; - readonly documentId: string; - readonly revisionKey: string; readonly chunk: KnowledgeReadChunk; + readonly resolveObjectUrl: (path: string) => Promise; }): Promise { const metadata = cloneMetadata(params.chunk.metadata); const storedAssetUrl = await getStoredObjectUrl({ - storageConfig: params.storageConfig, - documentId: params.documentId, - revisionKey: params.revisionKey, path: params.chunk.filePath, + resolveObjectUrl: params.resolveObjectUrl, }); const pageAssets = await applyStoredPageAssetUrls({ - storageConfig: params.storageConfig, - documentId: params.documentId, - revisionKey: params.revisionKey, metadata, + resolveObjectUrl: params.resolveObjectUrl, }); if (pageAssets) { metadata.pageAssets = pageAssets; @@ -2059,10 +2519,8 @@ async function applyStoredAssetUrlsToReadChunk(params: { } async function applyStoredPageAssetUrls(params: { - readonly storageConfig: ParsedDocumentStorageConfig; - readonly documentId: string; - readonly revisionKey: string; readonly metadata: Record; + readonly resolveObjectUrl: (path: string) => Promise; }): Promise { const pageAssets = getStoredPageAssets(params.metadata); if (!pageAssets) { @@ -2081,10 +2539,8 @@ async function applyStoredPageAssetUrls(params: { } const storedAssetUrl = await getStoredObjectUrl({ - storageConfig: params.storageConfig, - documentId: params.documentId, - revisionKey: params.revisionKey, path: artifactRef, + resolveObjectUrl: params.resolveObjectUrl, }); if (storedAssetUrl) { return { @@ -2129,22 +2585,14 @@ function normalizeStoredPageAsset(pageAsset: unknown): unknown { } async function getStoredObjectUrl(params: { - readonly storageConfig: ParsedDocumentStorageConfig; - readonly documentId: string; - readonly revisionKey: string; readonly path: string | undefined; + readonly resolveObjectUrl: (path: string) => Promise; }): Promise { if (!params.path) { return undefined; } - return ( - (await params.storageConfig.storage.getObjectUrl?.({ - documentId: params.documentId, - revisionKey: params.revisionKey, - path: params.path, - })) ?? undefined - ); + return (await params.resolveObjectUrl(params.path)) ?? undefined; } function countRemoteChunkTypes( @@ -2505,22 +2953,6 @@ function matchesReadScope(chunk: IndexedKnowledgeChunk, params: KnowledgeReadPar return true; } -function selectReadWindow( - chunks: IndexedKnowledgeChunk[], - params: KnowledgeReadParams, - limit: number, -): IndexedKnowledgeChunk[] { - if (params.chunkId) { - return chunks.slice(0, limit); - } - - const startChunk = params.startChunk ?? chunks[0]?.position ?? 1; - const endChunk = params.endChunk ?? Number.MAX_SAFE_INTEGER; - return chunks - .filter((chunk) => chunk.position >= startChunk && chunk.position <= endChunk) - .slice(0, limit); -} - function toReadChunk(chunk: IndexedKnowledgeChunk): KnowledgeReadChunk { return { position: chunk.position, diff --git a/src/knowledge/local-store.ts b/src/knowledge/local-store.ts index 6b30242..38e788c 100644 --- a/src/knowledge/local-store.ts +++ b/src/knowledge/local-store.ts @@ -216,6 +216,21 @@ export class LocalKnowledgeStore { return { document, result }; } + async loadResultMetadata(localDocumentId: string): Promise<{ + readonly manifest: unknown; + readonly chunks: unknown; + }> { + const document: LocalKnowledgeDocument | undefined = await this.getDocument(localDocumentId); + if (!document) throw new Error(`Local Knowhere document not found: ${localDocumentId}`); + const [manifest, chunks]: readonly string[] = await Promise.all( + ['manifest.json', 'chunks.json'].map( + (fileName): Promise => + fs.readFile(path.join(document.resultDirectoryPath, fileName), 'utf8'), + ), + ); + return { manifest: JSON.parse(manifest) as unknown, chunks: JSON.parse(chunks) as unknown }; + } + private getResultDirectoryPath(localDocumentId: string): string { const documentsDirectory = path.resolve(this.cacheDirectory, 'documents'); const resultDirectoryPath = path.resolve(documentsDirectory, localDocumentId); diff --git a/src/knowledge/types.ts b/src/knowledge/types.ts index 776e887..83d9d46 100644 --- a/src/knowledge/types.ts +++ b/src/knowledge/types.ts @@ -6,6 +6,9 @@ import type { KnowhereAssetStorageOptions, ParsedDocumentCommit, ParsedDocumentStorageConfig, + ParsedDocumentReadableSnapshot, + ParsedDocumentObjectParams, + ParsedDocumentRevisionParams, } from '../types/storage.js'; export type KnowledgeChunkType = DocumentChunkType; @@ -134,6 +137,30 @@ export interface KnowledgeSyncParsedDocumentResponse { commit?: ParsedDocumentCommit; } +export interface KnowledgePrepareReadableSnapshotParams { + readonly jobId: string; + /** Immutable snapshot ID, including when baseline and final share a Job Result ID. */ + readonly revisionKey: string; + readonly documentId?: string; + readonly signal?: AbortSignal; +} + +export interface KnowledgePrepareReadableSnapshotResponse { + readonly documentId: string; + readonly revisionKey: string; + readonly snapshot: JobSnapshot; + readonly readable: ParsedDocumentReadableSnapshot; +} + +export interface KnowledgeReadParsedDocumentObjectParams extends ParsedDocumentObjectParams { + readonly signal?: AbortSignal; +} + +export interface KnowledgeReadParsedDocumentObjectUrlsParams extends ParsedDocumentRevisionParams { + readonly paths: readonly string[]; + readonly signal?: AbortSignal; +} + export interface KnowledgeSection { sectionPath: string; sectionTitle: string; @@ -157,6 +184,10 @@ export interface KnowledgeOutline { } export interface KnowledgeReadParams extends KnowledgeDocumentReference { + /** Read chunks on one original physical page from a complete local or stored snapshot. */ + sourcePageNumber?: number; + /** Generate asset URLs while reading metadata (default true). */ + includeAssetUrls?: boolean; page?: number; pageSize?: number; sectionPath?: string; @@ -185,6 +216,8 @@ export interface KnowledgeReadChunk { } export interface KnowledgeReadResponse { + /** Complete snapshot metadata; omitted when only a remote chunk window is available. */ + documentSummary?: KnowledgeDocumentSummary; document: LocalKnowledgeDocument; chunks: KnowledgeReadChunk[]; nextChunk?: number; @@ -194,6 +227,29 @@ export interface KnowledgeReadResponse { totalPages?: number; } +export interface KnowledgeChunkDescriptor { + readonly chunkId: string; + readonly position: number; + readonly chunkType: KnowledgeChunkType; + readonly sectionPath: string; + readonly sourceChunkPath: string; + readonly pageNumbers: number[]; + readonly filePath?: string; + /** Parser chunk identifiers explicitly embedded by this chunk, without loading either body. */ + readonly embeddedChunkIds?: readonly string[]; +} + +export interface KnowledgeDocumentSummary { + readonly revisionKey: string | null; + readonly chunkCount: number; + readonly typeCounts: Record; + readonly pageCount: number | null; + readonly pageNumbers: number[]; + readonly hasPageAssets: boolean; + readonly hasNativePageContent: boolean; + readonly chunks: readonly KnowledgeChunkDescriptor[]; +} + export interface KnowledgeGrepParams extends KnowledgeDocumentReference { pattern: string; continuationCursor?: string; diff --git a/src/resources/__tests__/directory-wire.test.ts b/src/resources/__tests__/directory-wire.test.ts new file mode 100644 index 0000000..26a1c14 --- /dev/null +++ b/src/resources/__tests__/directory-wire.test.ts @@ -0,0 +1,69 @@ +import { createServer, type Server } from 'node:http'; +import { once } from 'node:events'; +import { expect, it } from 'vitest'; +import { Knowhere } from '../../index.js'; + +it('decodes directory wire fields and dates while sending revision and namespace fences', async () => { + let receivedUrl: string | undefined; + let authorization: string | undefined; + const server: Server = createServer((request, response): void => { + receivedUrl = request.url; + authorization = request.headers.authorization; + response.writeHead(200, { 'Content-Type': 'application/json' }); + response.end( + JSON.stringify({ + document_id: 'doc-one', + job_id: 'job-one', + job_result_id: 'result-one', + namespace: 'project/one', + directory_version: 1, + directory_committed_at: '2026-10-29T17:00:00+00:00', + sections: [ + { + section_id: 'section-one', + parent_section_id: null, + section_path: 'Chapter', + section_title: null, + section_level: 1, + sort_order: 0, + page_nums: [1, 2], + }, + ], + }), + ); + }); + try { + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const address = server.address(); + if (address === null || typeof address === 'string') throw new Error('Expected a TCP address'); + const client = new Knowhere({ + baseURL: `http://127.0.0.1:${address.port}`, + apiKey: 'test-key', + maxRetries: 0, + }); + const directory = await client.documents.getStructure('doc-one', { + jobResultId: 'result-one', + namespace: 'project/one', + }); + expect(receivedUrl).toBe( + '/v2/documents/doc-one/structure?job_result_id=result-one&namespace=project%2Fone', + ); + expect(authorization).toBe('Bearer test-key'); + expect(directory.directoryCommittedAt).toEqual(new Date('2026-10-29T17:00:00Z')); + expect(directory.sections[0]).toEqual({ + sectionId: 'section-one', + parentSectionId: null, + sectionPath: 'Chapter', + sectionTitle: null, + sectionLevel: 1, + sortOrder: 0, + pageNums: [1, 2], + }); + } finally { + await new Promise((resolve, reject): void => { + server.close((err): void => (err ? reject(err) : resolve())); + server.closeAllConnections(); + }); + } +}); diff --git a/src/resources/__tests__/documents.test.ts b/src/resources/__tests__/documents.test.ts index f135760..04f0543 100644 --- a/src/resources/__tests__/documents.test.ts +++ b/src/resources/__tests__/documents.test.ts @@ -102,6 +102,69 @@ describe('Documents Resource', () => { expect(document.documentId).toBe('doc-123'); }); + it('reads the initial directory pinned to a revision and namespace without fetching assets', async () => { + const directory = { + documentId: 'doc-123', + jobId: 'job-123', + jobResultId: 'result-123', + namespace: 'support-center', + directoryVersion: 1, + directoryCommittedAt: new Date('2026-10-29T17:00:00Z'), + sections: [ + { + sectionId: 'sec-123', + parentSectionId: null, + sectionPath: 'Chapter 1', + sectionTitle: 'Chapter 1', + sectionLevel: 1, + sortOrder: 0, + pageNums: [1, 2], + }, + ], + }; + mockHttpClient.get.mockResolvedValue(directory); + + const response = await documents.getStructure('doc-123', { + jobResultId: 'result-123', + namespace: 'support-center', + }); + + expect(response).toEqual(directory); + expect(mockHttpClient.get).toHaveBeenCalledExactlyOnceWith('/v2/documents/doc-123/structure', { + params: { job_result_id: 'result-123', namespace: 'support-center' }, + }); + expect(mockHttpClient.post).not.toHaveBeenCalled(); + }); + + it('preserves pending directory state and omits unspecified structure options', async () => { + mockHttpClient.get.mockResolvedValue({ + documentId: 'doc-123', + jobId: 'job-123', + jobResultId: 'result-123', + namespace: 'support-center', + directoryVersion: 0, + directoryCommittedAt: null, + sections: [], + }); + + const response = await documents.getStructure('doc-123'); + + expect(response.directoryVersion).toBe(0); + expect(response.directoryCommittedAt).toBeNull(); + expect(mockHttpClient.get).toHaveBeenCalledExactlyOnceWith( + '/v2/documents/doc-123/structure', + undefined, + ); + }); + + it('propagates a stale or inaccessible directory response', async () => { + const error = new Error('not found'); + mockHttpClient.get.mockRejectedValue(error); + await expect(documents.getStructure('doc-404', { jobResultId: 'old-result' })).rejects.toBe( + error, + ); + }); + it('should list current document chunks with optional filters', async () => { mockHttpClient.get.mockResolvedValue({ documentId: 'doc-123', diff --git a/src/resources/documents.ts b/src/resources/documents.ts index 64263ff..8051d4e 100644 --- a/src/resources/documents.ts +++ b/src/resources/documents.ts @@ -1,6 +1,8 @@ import { BaseResource } from './base.js'; import type { Document, + DocumentDirectoryParams, + DocumentDirectoryResponse, DocumentChunkGetParams, DocumentChunkListParams, DocumentChunkListResponse, @@ -39,6 +41,20 @@ export class Documents extends BaseResource { return this.httpClient.get(this.endpoint(`/documents/${documentId}`)); } + /** Read the initial chapter directory without downloading assets or triggering Fine. */ + async getStructure( + documentId: string, + params?: DocumentDirectoryParams, + ): Promise { + const queryParams: Record = {}; + if (params?.jobResultId !== undefined) queryParams.job_result_id = params.jobResultId; + if (params?.namespace !== undefined) queryParams.namespace = params.namespace; + return this.httpClient.get( + this.endpoint(`/documents/${encodeURIComponent(documentId)}/structure`), + Object.keys(queryParams).length > 0 ? { params: queryParams } : undefined, + ); + } + /** * List current-revision chunks for one canonical document. */ diff --git a/src/resources/jobs.ts b/src/resources/jobs.ts index 33ea2e3..a59a937 100644 --- a/src/resources/jobs.ts +++ b/src/resources/jobs.ts @@ -1,5 +1,12 @@ import { BaseResource } from './base.js'; -import type { Job, JobResult, JobEnhancementRetryResult } from '../types/job.js'; +import type { + Job, + JobResult, + JobEnhancementRetryResult, + JobSnapshotDelivery, + JobSnapshotObjectAccess, + JobSnapshotObjectAccessBatch, +} from '../types/job.js'; import type { CreateJobParams, UploadParams, WaitOptions, LoadOptions } from '../types/params.js'; import type { ParseResult } from '../types/result.js'; import { @@ -42,12 +49,69 @@ export class Jobs extends BaseResource { /** * Get job status */ - async get(jobId: string): Promise { - const jobResult = await this.httpClient.get(this.endpoint(`/jobs/${jobId}`)); + async get(jobId: string, options?: { readonly signal?: AbortSignal }): Promise { + const jobResult = options + ? await this.httpClient.get(this.endpoint(`/jobs/${jobId}`), { + signal: options.signal, + }) + : await this.httpClient.get(this.endpoint(`/jobs/${jobId}`)); enrichJobResult(jobResult); return jobResult; } + /** Obtain the immutable inventory; historical snapshots may not support object delivery. */ + async getSnapshotDelivery( + jobId: string, + snapshotId: string, + options?: { readonly signal?: AbortSignal }, + ): Promise { + return this.httpClient.get( + this.endpoint( + `/jobs/${encodeURIComponent(jobId)}/snapshots/${encodeURIComponent(snapshotId)}/readable`, + ), + { signal: options?.signal }, + ); + } + + /** Resolve a fresh authorized capability for one exact immutable inventory member. */ + async getSnapshotObjectAccess( + jobId: string, + snapshotId: string, + path: string, + options?: { readonly signal?: AbortSignal }, + ): Promise { + return this.httpClient.get( + this.endpoint( + `/jobs/${encodeURIComponent(jobId)}/snapshots/${encodeURIComponent(snapshotId)}/objects`, + ), + { params: { path }, signal: options?.signal }, + ); + } + + /** Resolve fresh authorized capabilities for up to 200 unique immutable inventory members. */ + async getSnapshotObjectsAccess( + jobId: string, + snapshotId: string, + paths: readonly string[], + options?: { readonly signal?: AbortSignal }, + ): Promise { + return this.httpClient.post( + this.endpoint( + `/jobs/${encodeURIComponent(jobId)}/snapshots/${encodeURIComponent(snapshotId)}/objects/access`, + ), + { paths }, + { signal: options?.signal }, + ); + } + + /** Download capability bytes without forwarding API authentication to the object origin. */ + async downloadSnapshotObject( + access: JobSnapshotObjectAccess, + options?: { readonly signal?: AbortSignal }, + ): Promise { + return this.httpClient.download(access.downloadUrl, { signal: options?.signal }); + } + /** Renew unfinished enhancement without creating another ingestion. */ async retryEnhancement(jobId: string): Promise { return this.httpClient.post( diff --git a/src/storage/readable-parsed-document-snapshots.ts b/src/storage/readable-parsed-document-snapshots.ts new file mode 100644 index 0000000..792018a --- /dev/null +++ b/src/storage/readable-parsed-document-snapshots.ts @@ -0,0 +1,533 @@ +import { createHash } from 'crypto'; +import { ValidationError } from '../errors/index.js'; +import type { Jobs } from '../resources/jobs.js'; +import type { + JobSnapshot, + JobSnapshotDelivery, + JobSnapshotObject, + JobSnapshotObjectAccess, + JobSnapshotObjectAccessBatch, +} from '../types/job.js'; +import type { + ParsedDocumentObject, + ParsedDocumentObjectParams, + ParsedDocumentRevisionParams, + ParsedDocumentReadableSnapshot, + ParsedDocumentStorage, +} from '../types/storage.js'; +import { + normalizeResultPath, + parsedDocumentCommitPath, + readParsedDocumentCommit, +} from './parsed-document-storage.js'; + +const readablePath: string = '.knowhere-sdk/readable.json'; +const requiredCorePaths: readonly string[] = [ + 'manifest.json', + 'chunks.json', + 'doc_nav.json', + 'source.pdf', +]; +const checksumPattern: RegExp = /^[a-f0-9]{64}$/; +const maxObjectAccessPaths: number = 200; + +/** Owns the separate readable barrier; never writes a full commit or archive progress. */ +export class ReadableParsedDocumentSnapshots { + constructor( + private readonly jobs: Jobs, + private readonly storage: ParsedDocumentStorage, + ) {} + + async prepare( + snapshot: JobSnapshot, + signal?: AbortSignal, + ): Promise { + signal?.throwIfAborted(); + const existing: ParsedDocumentReadableSnapshot | null = await this.readDescriptor({ + documentId: snapshot.documentId, + revisionKey: snapshot.snapshotId, + }); + if (existing) validateIdentity(snapshot, existing); + const received: JobSnapshotDelivery = + existing ?? + (await this.jobs.getSnapshotDelivery(snapshot.jobId, snapshot.snapshotId, { signal })); + validateIdentity(snapshot, received); + validateInventory(received.objects); + const delivery: JobSnapshotDelivery = { + ...received, + objects: received.objects.map( + (object: JobSnapshotObject): JobSnapshotObject => ({ + path: object.path, + sizeBytes: object.sizeBytes, + checksumSha256: object.checksumSha256, + contentType: object.contentType, + kind: object.kind, + }), + ), + }; + for (const object of delivery.objects) { + signal?.throwIfAborted(); + if (object.kind === 'image') continue; + const params: ParsedDocumentObjectParams = { + documentId: snapshot.documentId, + revisionKey: snapshot.snapshotId, + path: object.path, + }; + const stored: ParsedDocumentObject | null = await this.storage.readObject(params); + if (stored && matchesIntegrity(stored.body, object)) continue; + const body: Uint8Array = await this.downloadObject(delivery, object, signal); + signal?.throwIfAborted(); + await this.storage.writeObject({ ...params, body, contentType: object.contentType }); + } + await this.validateCore(delivery); + signal?.throwIfAborted(); + if (existing) return existing; + const readable: ParsedDocumentReadableSnapshot = { + version: 1, + documentId: snapshot.documentId, + revisionKey: snapshot.snapshotId, + snapshotId: snapshot.snapshotId, + jobId: snapshot.jobId, + jobResultId: snapshot.jobResultId, + checksum: snapshot.checksum, + stage: snapshot.stage, + objects: delivery.objects, + readableAt: new Date().toISOString(), + }; + await this.storage.writeObject({ + documentId: snapshot.documentId, + revisionKey: snapshot.snapshotId, + path: readablePath, + contentType: 'application/json; charset=utf-8', + body: Buffer.from(JSON.stringify(readable)), + }); + return readable; + } + + async readDescriptor(params: { + readonly documentId: string; + readonly revisionKey: string; + }): Promise { + const object: ParsedDocumentObject | null = await this.storage.readObject({ + ...params, + path: readablePath, + }); + if (!object) return null; + const payload: unknown = JSON.parse(Buffer.from(object.body).toString('utf8')); + if ( + !isRecord(payload) || + payload.version !== 1 || + payload.documentId !== params.documentId || + payload.revisionKey !== params.revisionKey || + payload.snapshotId !== params.revisionKey || + typeof payload.jobId !== 'string' || + !payload.jobId || + typeof payload.jobResultId !== 'string' || + !payload.jobResultId || + typeof payload.checksum !== 'string' || + !checksumPattern.test(payload.checksum) || + (payload.stage !== 'baseline' && payload.stage !== 'final') || + typeof payload.readableAt !== 'string' || + !Number.isFinite(Date.parse(payload.readableAt)) + ) + throw new ValidationError('Invalid readable parsed snapshot identity.'); + validateInventory(payload.objects); + return payload as unknown as ParsedDocumentReadableSnapshot; + } + + async readObject( + params: ParsedDocumentObjectParams & { readonly signal?: AbortSignal }, + ): Promise { + params.signal?.throwIfAborted(); + const path: string = validateObjectPath(params.path); + const descriptor: ParsedDocumentReadableSnapshot | null = await this.readDescriptor(params); + if (!descriptor && !(await readParsedDocumentCommit({ ...params, storage: this.storage }))) + return null; + const object: JobSnapshotObject | undefined = descriptor?.objects.find( + (candidate: JobSnapshotObject): boolean => candidate.path === path, + ); + if (descriptor && !object) return null; + const stored: ParsedDocumentObject | null = await this.storage.readObject({ ...params, path }); + params.signal?.throwIfAborted(); + if (stored) { + if (object && !matchesIntegrity(stored.body, object)) + throw new ValidationError(`Parsed snapshot replica integrity check failed: ${path}`); + return { + ...stored, + contentType: object?.contentType ?? stored.contentType, + contentLength: stored.body.byteLength, + }; + } + if (!descriptor || !object || object.kind !== 'image') return null; + const body: Uint8Array = await this.downloadObject(descriptor, object, params.signal); + params.signal?.throwIfAborted(); + return { + documentId: params.documentId, + revisionKey: params.revisionKey, + path, + body, + contentType: object.contentType, + contentLength: body.byteLength, + }; + } + + async getObjectUrl( + params: ParsedDocumentObjectParams & { readonly signal?: AbortSignal }, + ): Promise { + params.signal?.throwIfAborted(); + const path: string = validateObjectPath(params.path); + const descriptor: ParsedDocumentReadableSnapshot | null = await this.readDescriptor(params); + if (!descriptor && !(await readParsedDocumentCommit({ ...params, storage: this.storage }))) + return null; + const object: JobSnapshotObject | undefined = descriptor?.objects.find( + (candidate: JobSnapshotObject): boolean => candidate.path === path, + ); + if (descriptor && !object) return null; + if ( + descriptor && + object?.kind === 'image' && + !(await this.hasCertifiedReplica(descriptor, params.signal)) + ) + return (await this.resolveObjectAccess(descriptor, object, params.signal)).downloadUrl; + const localUrl: string | null | undefined = await this.storage.getObjectUrl?.({ + ...params, + path, + }); + params.signal?.throwIfAborted(); + if (localUrl) return localUrl; + if (!descriptor || !object || object.kind !== 'image') return null; + return (await this.resolveObjectAccess(descriptor, object, params.signal)).downloadUrl; + } + + async getObjectUrls( + params: ParsedDocumentRevisionParams & { + readonly paths: readonly string[]; + readonly signal?: AbortSignal; + }, + ): Promise>> { + params.signal?.throwIfAborted(); + const requestedPaths: Map = new Map( + params.paths.map((path: string): [string, string] => [path, validateObjectPath(path)]), + ); + if (requestedPaths.size === 0) return {}; + const paths: Set = new Set(requestedPaths.values()); + const urls: Map = new Map(); + const descriptor: ParsedDocumentReadableSnapshot | null = await this.readDescriptor(params); + params.signal?.throwIfAborted(); + const hasFullCopy: boolean = descriptor + ? await this.hasCertifiedReplica(descriptor, params.signal) + : (await readParsedDocumentCommit({ ...params, storage: this.storage })) !== null; + params.signal?.throwIfAborted(); + const inventory: Map = new Map( + descriptor?.objects.map((object): [string, JobSnapshotObject] => [object.path, object]) ?? [], + ); + const resolved = await Promise.all( + [...paths].map( + async ( + path, + ): Promise<{ + readonly path: string; + readonly url: string | null; + readonly needsOrigin: boolean; + }> => { + params.signal?.throwIfAborted(); + const object: JobSnapshotObject | undefined = inventory.get(path); + if (descriptor ? !object : !hasFullCopy) return { path, url: null, needsOrigin: false }; + if (hasFullCopy || object?.kind !== 'image') { + const localUrl: string | null | undefined = await this.storage.getObjectUrl?.({ + documentId: params.documentId, + revisionKey: params.revisionKey, + path, + }); + params.signal?.throwIfAborted(); + if (localUrl) return { path, url: localUrl, needsOrigin: false }; + } + params.signal?.throwIfAborted(); + return { path, url: null, needsOrigin: Boolean(descriptor && object?.kind === 'image') }; + }, + ), + ); + for (const entry of resolved) urls.set(entry.path, entry.url); + const originPaths: readonly string[] = resolved + .filter((entry) => entry.needsOrigin) + .map((entry) => entry.path); + if (descriptor) { + for (let offset: number = 0; offset < originPaths.length; offset += maxObjectAccessPaths) { + const batchPaths: readonly string[] = originPaths.slice( + offset, + offset + maxObjectAccessPaths, + ); + const batchUrls: ReadonlyMap = await this.resolveObjectUrls( + descriptor, + batchPaths, + params.signal, + ); + for (const [path, url] of batchUrls) urls.set(path, url); + } + } + params.signal?.throwIfAborted(); + return Object.fromEntries( + [...requestedPaths].map(([requested, path]): [string, string | null] => [ + requested, + urls.get(path) ?? null, + ]), + ); + } + + private async resolveObjectUrls( + identity: JobSnapshotDelivery, + paths: readonly string[], + signal?: AbortSignal, + ): Promise> { + signal?.throwIfAborted(); + const access: JobSnapshotObjectAccessBatch = await this.jobs.getSnapshotObjectsAccess( + identity.jobId, + identity.snapshotId, + paths, + { signal }, + ); + if (!isRecord(access)) throw new ValidationError('Invalid pinned snapshot capability batch.'); + validateIdentity(identity, access); + if (!Array.isArray(access.objects) || access.objects.length !== paths.length) + throw new ValidationError('Incomplete pinned snapshot capability batch.'); + const requested: Set = new Set(paths); + const urls: Map = new Map(); + for (const object of access.objects) { + if ( + !isRecord(object) || + typeof object.path !== 'string' || + !requested.has(object.path) || + urls.has(object.path) + ) + throw new ValidationError('Unexpected or duplicate pinned snapshot capability path.'); + validateObjectCapability(object.path, object); + urls.set(object.path, object.downloadUrl); + } + signal?.throwIfAborted(); + return urls; + } + + private async hasCertifiedReplica( + descriptor: ParsedDocumentReadableSnapshot, + signal?: AbortSignal, + ): Promise { + const stored: ParsedDocumentObject | null = await this.storage.readObject({ + documentId: descriptor.documentId, + revisionKey: descriptor.revisionKey, + path: parsedDocumentCommitPath, + }); + signal?.throwIfAborted(); + if (!stored) return false; + const commit: unknown = JSON.parse(Buffer.from(stored.body).toString('utf8')); + if ( + !isRecord(commit) || + commit.version !== 1 || + commit.documentId !== descriptor.documentId || + commit.revisionKey !== descriptor.revisionKey || + (commit.source !== 'resultZip' && commit.source !== 'remoteReconstruction') || + typeof commit.committedAt !== 'string' || + !Number.isFinite(Date.parse(commit.committedAt)) + ) + throw new ValidationError('Invalid full-copy parsed snapshot certificate.'); + let hasPinnedIdentity: boolean = true; + for (const field of ['snapshotId', 'jobResultId', 'checksum', 'stage'] as const) { + if (commit[field] === undefined) { + hasPinnedIdentity = false; + continue; + } + if (commit[field] !== descriptor[field]) + throw new ValidationError(`Full-copy parsed snapshot certificate differs at ${field}.`); + } + return hasPinnedIdentity; + } + + private async downloadObject( + identity: JobSnapshotDelivery, + object: JobSnapshotObject, + signal?: AbortSignal, + ): Promise { + const access: JobSnapshotObjectAccess = await this.resolveObjectAccess( + identity, + object, + signal, + ); + const body: Uint8Array = await this.jobs.downloadSnapshotObject(access, { signal }); + signal?.throwIfAborted(); + if (!matchesIntegrity(body, object)) + throw new ValidationError(`Parsed snapshot object integrity check failed: ${object.path}`); + return body; + } + + private async resolveObjectAccess( + identity: JobSnapshotDelivery, + object: JobSnapshotObject, + signal?: AbortSignal, + ): Promise { + signal?.throwIfAborted(); + const access: JobSnapshotObjectAccess = await this.jobs.getSnapshotObjectAccess( + identity.jobId, + identity.snapshotId, + object.path, + { signal }, + ); + validateIdentity(identity, access); + validateObjectCapability(object.path, access); + signal?.throwIfAborted(); + return access; + } + + private async validateCore(delivery: JobSnapshotDelivery): Promise { + const payloads: Map> = new Map(); + for (const entry of delivery.objects) { + if (entry.kind === 'image') continue; + const object: ParsedDocumentObject | null = await this.storage.readObject({ + documentId: delivery.documentId, + revisionKey: delivery.snapshotId, + path: entry.path, + }); + if (!object || !matchesIntegrity(object.body, entry)) + throw new ValidationError( + `Persisted readable snapshot core integrity check failed: ${entry.path}`, + ); + if (!['manifest.json', 'chunks.json', 'doc_nav.json'].includes(entry.path)) continue; + const payload: unknown = JSON.parse(Buffer.from(object.body).toString('utf8')); + if (!isRecord(payload)) + throw new ValidationError(`Invalid readable snapshot core: ${entry.path}`); + payloads.set(entry.path, payload); + } + const manifest: Record = payloads.get('manifest.json')!; + if ( + (manifest.job_id ?? manifest.jobId) !== delivery.jobId || + ((manifest.job_result_id ?? manifest.jobResultId) !== undefined && + (manifest.job_result_id ?? manifest.jobResultId) !== delivery.jobResultId) + ) + throw new ValidationError( + 'Readable snapshot manifest identity differs from its pinned origin.', + ); + const chunks: unknown = payloads.get('chunks.json')?.chunks; + if (!Array.isArray(chunks) || chunks.some((chunk: unknown): boolean => !isRecord(chunk))) + throw new ValidationError('Invalid readable snapshot chunks.json.'); + const navigation: unknown = payloads.get('doc_nav.json')?.sections; + if (!Array.isArray(navigation)) + throw new ValidationError('Invalid readable snapshot navigation.'); + const references: Set = new Set(); + if (isRecord(manifest.files)) + for (const path of Object.values(manifest.files)) + if (typeof path === 'string') references.add(validateObjectPath(path)); + collectReferences(chunks, references); + collectReferences(payloads.get('doc_nav.json'), references); + for (const chunk of chunks as Record[]) { + const kind: unknown = chunk.type ?? chunk.chunk_type ?? chunk.chunkType; + if ((kind === 'image' || kind === 'table') && typeof chunk.path === 'string') + references.add(validateObjectPath(chunk.path)); + } + const paths: Set = new Set(delivery.objects.map((object): string => object.path)); + for (const path of references) + if (!paths.has(path)) + throw new ValidationError( + `Readable snapshot image or essential reference has no pinned origin: ${path}`, + ); + } +} + +function validateObjectCapability( + path: string, + access: unknown, +): asserts access is Pick { + if ( + !isRecord(access) || + access.path !== path || + typeof access.downloadUrl !== 'string' || + !/^https?:\/\//i.test(access.downloadUrl) || + !(access.expiresAt instanceof Date) || + !Number.isFinite(access.expiresAt.getTime()) || + access.expiresAt.getTime() <= Date.now() + ) + throw new ValidationError('Invalid pinned snapshot object capability.'); +} + +function validateIdentity( + expected: Omit, + actual: Omit, +): void { + for (const field of ['snapshotId', 'jobId', 'jobResultId', 'documentId', 'checksum'] as const) + if (typeof actual[field] !== 'string' || !actual[field] || actual[field] !== expected[field]) + throw new ValidationError(`Parsed snapshot origin identity differs at ${field}.`); +} + +function validateInventory(value: unknown): asserts value is readonly JobSnapshotObject[] { + if (!Array.isArray(value)) + throw new ValidationError('Invalid readable snapshot object inventory.'); + const objects: Map = new Map(); + for (const candidate of value as unknown[]) { + if ( + !isRecord(candidate) || + typeof candidate.path !== 'string' || + !Number.isSafeInteger(candidate.sizeBytes) || + (candidate.sizeBytes as number) < 0 || + typeof candidate.checksumSha256 !== 'string' || + !checksumPattern.test(candidate.checksumSha256) || + typeof candidate.contentType !== 'string' || + !candidate.contentType || + (candidate.kind !== 'essential' && candidate.kind !== 'image') + ) + throw new ValidationError('Invalid readable snapshot object inventory member.'); + const path: string = validateObjectPath(candidate.path); + if (path !== candidate.path || objects.has(path)) + throw new ValidationError(`Duplicate or noncanonical readable snapshot object path: ${path}`); + if (candidate.kind === 'image' && !isImagePath(path)) + throw new ValidationError(`Non-image readable snapshot object cannot skip copying: ${path}`); + objects.set(path, candidate as unknown as JobSnapshotObject); + } + for (const path of requiredCorePaths) + if (objects.get(path)?.kind !== 'essential') + throw new ValidationError(`Readable snapshot essential object is missing: ${path}`); +} + +function validateObjectPath(path: string): string { + const normalized: string = normalizeResultPath(path); + if ( + normalized === '.knowhere-sdk' || + normalized.startsWith('.knowhere-sdk/') || + normalized === 'result.zip' + ) + throw new ValidationError(`Reserved parsed snapshot object path: ${path}`); + return normalized; +} + +function matchesIntegrity(body: Uint8Array, object: JobSnapshotObject): boolean { + return ( + body.byteLength === object.sizeBytes && + createHash('sha256').update(body).digest('hex') === object.checksumSha256 + ); +} + +function collectReferences(value: unknown, references: Set): void { + if (Array.isArray(value)) { + for (const item of value as unknown[]) collectReferences(item, references); + return; + } + if (!isRecord(value)) return; + for (const [key, child] of Object.entries(value)) { + if (typeof child === 'string') { + if (['file_path', 'filePath', 'artifact_ref', 'artifactRef'].includes(key)) + references.add(validateObjectPath(child)); + if (key === 'path' && /^(?:images|tables|page_pdfs)\//.test(child)) + references.add(validateObjectPath(child)); + if (['ref', 'content', 'summary'].includes(key)) { + for (const match of child.matchAll(/\[((?:images|tables)\/[^\]\r\n]+)\]/g)) + references.add(validateObjectPath(match[1])); + for (const match of child.matchAll(/\]\(((?:images|tables)\/[^)\r\n]+)\)/g)) + references.add(validateObjectPath(match[1])); + } + } else collectReferences(child, references); + } +} + +function isImagePath(path: string): boolean { + return /\.(?:png|jpe?g|gif|webp|svg|avif|bmp|tiff?)$/i.test(path); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} diff --git a/src/types/document.ts b/src/types/document.ts index 86df509..6b16c1d 100644 --- a/src/types/document.ts +++ b/src/types/document.ts @@ -60,6 +60,49 @@ export interface DocumentListResponse { pagination: DocumentListPagination; } +/** A revision-fenced read of the initial directory; this never triggers refinement. */ +export interface DocumentDirectoryParams { + readonly jobResultId?: string; + readonly namespace?: string; +} + +/** One recognized chapter, including descendants available before Fine. */ +export interface DocumentDirectorySection { + readonly sectionId: string; + readonly parentSectionId: string | null; + readonly sectionPath: string; + readonly sectionTitle: string | null; + readonly sectionLevel: number; + readonly sortOrder: number; + /** Physical source pages, starting at one. */ + readonly pageNums: readonly number[]; +} + +/** Immutable initial directory. Version zero explicitly means not ready yet. */ +export interface DocumentDirectoryResponse { + readonly documentId: string; + readonly jobId: string; + readonly jobResultId: string; + readonly namespace: string; + readonly directoryVersion: 0 | 1; + readonly directoryCommittedAt: Date | null; + readonly sections: readonly DocumentDirectorySection[]; +} + +/** Raw webhook JSON, delivered independently of parse completion and Fine. */ +export interface DocumentDirectoryReadyEvent { + readonly event: 'job.document.directory_ready'; + readonly event_id: string; + readonly status: 'completed'; + readonly job_id: string; + readonly document_id: string; + readonly job_result_id: string; + readonly namespace: string; + readonly directory_version: 1; + readonly directory_committed_at: string; + readonly completed_at: string; +} + /** * Document chunk types supported by document chunk endpoints. */ diff --git a/src/types/job.ts b/src/types/job.ts index 12480df..be29ee1 100644 --- a/src/types/job.ts +++ b/src/types/job.ts @@ -115,6 +115,36 @@ export interface JobSnapshot { readonly resultUrlExpiresAt: Date; } +/** Immutable object inventory published with a snapshot, without expiring capabilities. */ +export interface JobSnapshotObject { + readonly path: string; + readonly sizeBytes: number; + readonly checksumSha256: string; + readonly contentType: string; + readonly kind: 'essential' | 'image'; +} + +export interface JobSnapshotDelivery { + readonly snapshotId: string; + readonly jobId: string; + readonly jobResultId: string; + readonly documentId: string; + readonly checksum: string; + readonly objects: readonly JobSnapshotObject[]; +} + +/** Fresh capability resolved through the authenticated immutable snapshot route. */ +export interface JobSnapshotObjectAccess extends Omit { + readonly path: string; + readonly downloadUrl: string; + readonly expiresAt: Date; +} + +/** Capabilities for an exact set of immutable inventory members, sharing one snapshot pin. */ +export interface JobSnapshotObjectAccessBatch extends Omit { + readonly objects: readonly Pick[]; +} + export interface JobEnhancementRetryResult { readonly jobId: string; readonly enhancement: JobEnhancement; diff --git a/src/types/storage.ts b/src/types/storage.ts index 6cd0c4a..ce8dcf7 100644 --- a/src/types/storage.ts +++ b/src/types/storage.ts @@ -1,4 +1,5 @@ import type { ParseResult } from './result.js'; +import type { JobSnapshotDelivery } from './job.js'; export type KnowhereAssetStorageBody = Uint8Array; @@ -98,6 +99,14 @@ export interface ParsedDocumentCommit { readonly committedAt: string; } +/** Readability is independent of the full-copy commit and archive checkpoints. */ +export interface ParsedDocumentReadableSnapshot extends JobSnapshotDelivery { + readonly version: 1; + readonly revisionKey: string; + readonly stage: 'baseline' | 'final'; + readonly readableAt: string; +} + export interface ParsedDocumentSyncProgress { readonly documentId: string; readonly revisionKey: string; diff --git a/src/version.ts b/src/version.ts index 5515303..53ea336 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1,3 @@ -export const VERSION = '2.1.3'; +import packageJson from '../package.json'; + +export const VERSION: string = packageJson.version;