diff --git a/apps/commons-api/migrations/versioned/031_fast_artifact_provenance.sql b/apps/commons-api/migrations/versioned/031_fast_artifact_provenance.sql new file mode 100644 index 00000000..ac0228ee --- /dev/null +++ b/apps/commons-api/migrations/versioned/031_fast_artifact_provenance.sql @@ -0,0 +1,6 @@ +-- Artifact provenance resolves derivations through the source artifact id +-- stored in metadata. Keep that lookup indexed and exclude deleted revisions +-- so opening the provenance tab remains fast as a Library grows. +CREATE INDEX IF NOT EXISTS idx_library_item_revision_source + ON library_item ((metadata->>'sourceFileId')) + WHERE deleted_at IS NULL; diff --git a/apps/commons-api/models/schema.ts b/apps/commons-api/models/schema.ts index 9c01f103..76e29dd1 100644 --- a/apps/commons-api/models/schema.ts +++ b/apps/commons-api/models/schema.ts @@ -753,6 +753,9 @@ export const libraryItem = pgTable( table.ownerUserId, table.sha256, ), + revisionSourceIdx: index('idx_library_item_revision_source') + .on(sql`(${table.metadata}->>'sourceFileId')`) + .where(sql`${table.deletedAt} is null`), }), ); @@ -2397,7 +2400,7 @@ export const provenanceEvent = pgTable( 'provenance_event', { eventId: uuid('event_id') - .default(sql`uuid_generate_v4()`) + .default(sql`gen_random_uuid()`) .primaryKey(), traceId: uuid('trace_id') .notNull() diff --git a/apps/commons-api/src/files/library.controller.ts b/apps/commons-api/src/files/library.controller.ts index 8b027f5f..24b02e22 100644 --- a/apps/commons-api/src/files/library.controller.ts +++ b/apps/commons-api/src/files/library.controller.ts @@ -81,8 +81,18 @@ export class LibraryController { @Get(':itemId/provenance') @RateLimit({ limit: 120, windowMs: 60_000, keyStrategy: 'user' }) - provenance(@Req() req: Request, @Param('itemId') itemId: string) { - return this.library.provenance(itemId, principalFrom(req)); + provenance( + @Req() req: Request, + @Param('itemId') itemId: string, + @Query('eventLimit') eventLimit?: string, + ) { + const parsedEventLimit = + eventLimit === undefined ? undefined : Number(eventLimit); + return this.library.provenance( + itemId, + principalFrom(req), + Number.isFinite(parsedEventLimit) ? parsedEventLimit : undefined, + ); } @Patch(':itemId') diff --git a/apps/commons-api/src/files/library.service.spec.ts b/apps/commons-api/src/files/library.service.spec.ts index b3d69030..0d036bf8 100644 --- a/apps/commons-api/src/files/library.service.spec.ts +++ b/apps/commons-api/src/files/library.service.spec.ts @@ -236,3 +236,140 @@ describe('LibraryService agent discovery', () => { expect(context).toEqual({ ownerId: 'user-1', workspaceId: undefined }); }); }); + +describe('LibraryService artifact provenance', () => { + const item = { + itemId: '11111111-1111-4111-8111-111111111111', + ownerUserId: 'user-1', + workspaceId: null, + sourceAgentId: 'agent-1', + sourceSessionId: null, + kind: 'document', + name: 'Fast provenance.docx', + description: null, + mimeType: + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + sizeBytes: 1024, + sha256: 'a'.repeat(64), + source: 'agent_generated', + status: 'ready', + visibility: 'private', + textPreview: 'Fast provenance', + extractedTextChars: 15, + extractionError: null, + metadata: {}, + isFavorite: false, + deletedAt: null, + createdAt: new Date('2026-08-31T00:00:00.000Z'), + updatedAt: new Date('2026-08-31T00:00:00.000Z'), + }; + + function harness() { + const eventRows = Array.from({ length: 40 }, (_, index) => { + const sequence = 50 - index; + return { + eventId: `event-${sequence}`, + traceId: '22222222-2222-4222-8222-222222222222', + sessionId: null, + sequence, + category: 'tool', + eventType: 'tool.completed', + name: `Event ${sequence}`, + phase: null, + status: 'completed', + spanId: null, + parentSpanId: null, + summary: null, + payload: null, + result: null, + contentHash: null, + inputTokens: null, + outputTokens: null, + cachedTokens: null, + costUsd: null, + durationMs: 1, + eaaAction: null, + metadata: {}, + startedAt: new Date( + `2026-08-31T00:00:${String(sequence).padStart(2, '0')}.000Z`, + ), + endedAt: null, + createdAt: new Date('2026-08-31T00:01:00.000Z'), + }; + }); + const values = jest.fn().mockResolvedValue(undefined); + const db = { + query: { + libraryItem: { + findFirst: jest.fn().mockResolvedValue(item), + findMany: jest.fn().mockResolvedValue([]), + }, + libraryLink: { + findMany: jest.fn().mockResolvedValue([ + { + scopeType: 'provenance_trace', + scopeId: '22222222-2222-4222-8222-222222222222', + }, + ]), + }, + provenanceRun: { + findMany: jest.fn().mockResolvedValue([ + { + traceId: '22222222-2222-4222-8222-222222222222', + status: 'completed', + captureMode: 'metadata', + eventCount: 75, + droppedEventCount: 0, + startedAt: new Date('2026-08-31T00:00:00.000Z'), + anchorStatus: 'not_requested', + }, + ]), + }, + provenanceEvent: { findMany: jest.fn().mockResolvedValue(eventRows) }, + libraryAuditEvent: { findMany: jest.fn().mockResolvedValue([]) }, + libraryGrant: { findMany: jest.fn().mockResolvedValue([]) }, + libraryShareLink: { findMany: jest.fn().mockResolvedValue([]) }, + }, + insert: jest.fn().mockReturnValue({ values }), + } as any; + return { + db, + service: new LibraryService(db, {} as any, {} as any, {} as any), + }; + } + + it('returns only the newest 40 events in chronological display order', async () => { + const { db, service } = harness(); + + const record = (await service.provenance('item-1', { + principalId: 'user-1', + principalType: 'user', + })) as any; + + expect(db.query.provenanceEvent.findMany).toHaveBeenCalledWith( + expect.objectContaining({ limit: 40 }), + ); + expect(record.actions).toHaveLength(40); + expect(record.actions[0].sequence).toBe(11); + expect(record.actions.at(-1).sequence).toBe(50); + expect(record.disclosure).toMatchObject({ + eventsIncluded: true, + eventsReturned: 40, + eventsTruncated: true, + }); + }); + + it('can return compact provenance without querying event rows', async () => { + const { db, service } = harness(); + + const record = (await service.provenance( + 'item-1', + { principalId: 'user-1', principalType: 'user' }, + 0, + )) as any; + + expect(db.query.provenanceEvent.findMany).not.toHaveBeenCalled(); + expect(record.actions).toEqual([]); + expect(record.disclosure.eventsIncluded).toBe(false); + }); +}); diff --git a/apps/commons-api/src/files/library.service.ts b/apps/commons-api/src/files/library.service.ts index 7b5f5c34..4e3bfcd0 100644 --- a/apps/commons-api/src/files/library.service.ts +++ b/apps/commons-api/src/files/library.service.ts @@ -235,17 +235,24 @@ export class LibraryService { }; } - async provenance(itemId: string, principal: LibraryPrincipal) { + async provenance( + itemId: string, + principal: LibraryPrincipal, + requestedEventLimit = 40, + ) { const item = await this.getAccessible(itemId, principal); - const result = await this.buildArtifactProvenance(item, true); - await this.audit(itemId, principal, 'provenance_viewed'); + const eventLimit = clamp(Math.floor(requestedEventLimit), 0, 100); + const [result] = await Promise.all([ + this.buildArtifactProvenance(item, eventLimit), + this.audit(itemId, principal, 'provenance_viewed'), + ]); return result; } /** A safe EAA-oriented projection for compact UI, export, and sharing. */ private async buildArtifactProvenance( item: typeof schema.libraryItem.$inferSelect, - includeEvents: boolean, + eventLimit?: number, ) { const links = await this.db.query.libraryLink.findMany({ where: (table) => eq(table.itemId, item.itemId), @@ -265,38 +272,48 @@ export class LibraryService { }) : []; const traceIds = runs.map((run) => run.traceId); - const [events, audits, grants, shares] = await Promise.all([ - includeEvents && traceIds.length - ? this.db.query.provenanceEvent.findMany({ - where: (table) => inArray(table.traceId, traceIds), - orderBy: (table) => [asc(table.startedAt), asc(table.sequence)], - }) - : Promise.resolve([]), - this.db.query.libraryAuditEvent.findMany({ - where: (table) => eq(table.itemId, item.itemId), - orderBy: (table) => asc(table.createdAt), - }), - this.db.query.libraryGrant.findMany({ - where: (table) => eq(table.itemId, item.itemId), - }), - this.db.query.libraryShareLink.findMany({ - where: (table) => eq(table.itemId, item.itemId), - }), - ]); const sourceFileId = stringValue( item.metadata?.sourceFileId ?? item.metadata?.revisionOf, ); - const source = sourceFileId - ? await this.db.query.libraryItem.findFirst({ - where: (table) => eq(table.itemId, sourceFileId), - }) - : undefined; - const revisions = await this.db.query.libraryItem.findMany({ - where: (table) => - sql`${table.metadata}->>'sourceFileId' = ${item.itemId}`, - orderBy: (table) => asc(table.createdAt), - limit: 25, - }); + const [events, audits, grants, shares, source, revisions] = + await Promise.all([ + eventLimit !== 0 && traceIds.length + ? this.db.query.provenanceEvent.findMany({ + where: (table) => inArray(table.traceId, traceIds), + orderBy: (table) => + eventLimit === undefined + ? [asc(table.startedAt), asc(table.sequence)] + : [desc(table.startedAt), desc(table.sequence)], + ...(eventLimit === undefined ? {} : { limit: eventLimit }), + }) + : Promise.resolve([]), + this.db.query.libraryAuditEvent.findMany({ + where: (table) => eq(table.itemId, item.itemId), + orderBy: (table) => asc(table.createdAt), + }), + this.db.query.libraryGrant.findMany({ + where: (table) => eq(table.itemId, item.itemId), + }), + this.db.query.libraryShareLink.findMany({ + where: (table) => eq(table.itemId, item.itemId), + }), + sourceFileId + ? this.db.query.libraryItem.findFirst({ + where: (table) => eq(table.itemId, sourceFileId), + }) + : Promise.resolve(undefined), + this.db.query.libraryItem.findMany({ + where: (table) => + and( + isNull(table.deletedAt), + sql`${table.metadata}->>'sourceFileId' = ${item.itemId}`, + ), + orderBy: (table) => asc(table.createdAt), + limit: 25, + }), + ]); + const orderedEvents = + eventLimit === undefined ? events : [...events].reverse(); const metadataProvenance = recordValue(item.metadata?.provenance); const contentHash = `sha256:${item.sha256}`; @@ -346,26 +363,27 @@ export class LibraryService { anchorProvider: run.anchorProvider, anchorRef: run.anchorRef, })), - actions: includeEvents - ? events.map((event) => ({ - id: - stringValue((event.eaaAction as any)?.id) ?? - `urn:agentcommons:event:${event.traceId}:${event.sequence}`, - traceId: event.traceId, - sequence: event.sequence, - category: event.category, - eventType: event.eventType, - name: event.name, - summary: event.summary, - status: event.status, - performedBy: stringValue((event.eaaAction as any)?.performedBy), - contentHash: event.contentHash, - startedAt: event.startedAt, - endedAt: event.endedAt, - durationMs: event.durationMs, - lineage: recordValue(event.metadata)?.lineage, - })) - : [], + actions: + eventLimit !== 0 + ? orderedEvents.map((event) => ({ + id: + stringValue((event.eaaAction as any)?.id) ?? + `urn:agentcommons:event:${event.traceId}:${event.sequence}`, + traceId: event.traceId, + sequence: event.sequence, + category: event.category, + eventType: event.eventType, + name: event.name, + summary: event.summary, + status: event.status, + performedBy: stringValue((event.eaaAction as any)?.performedBy), + contentHash: event.contentHash, + startedAt: event.startedAt, + endedAt: event.endedAt, + durationMs: event.durationMs, + lineage: recordValue(event.metadata)?.lineage, + })) + : [], derivation: { source: source ? { @@ -437,7 +455,12 @@ export class LibraryService { traceId: stringValue(event.metadata?.traceId), })), disclosure: { - eventsIncluded: includeEvents, + eventsIncluded: eventLimit !== 0, + eventsReturned: orderedEvents.length, + eventsTruncated: + eventLimit !== undefined && + runs.reduce((total, run) => total + Number(run.eventCount ?? 0), 0) > + orderedEvents.length, privateReasoningIncluded: false, credentialsIncluded: false, }, @@ -795,9 +818,10 @@ export class LibraryService { ? this.files.createShareDownloadUrl(item.itemId) : Promise.resolve(undefined), disclosure.provenance - ? this.buildArtifactProvenance(item, disclosure.events).then( - redactSharedProvenance, - ) + ? this.buildArtifactProvenance( + item, + disclosure.events ? undefined : 0, + ).then(redactSharedProvenance) : Promise.resolve(undefined), ]); const download = settledValue(downloadResult); diff --git a/apps/commons-app/components/artifacts/artifact-surface.tsx b/apps/commons-app/components/artifacts/artifact-surface.tsx index 566bee5b..9c0f3540 100644 --- a/apps/commons-app/components/artifacts/artifact-surface.tsx +++ b/apps/commons-app/components/artifacts/artifact-surface.tsx @@ -55,6 +55,7 @@ export function ArtifactSurface({ ); const [provenanceLoading, setProvenanceLoading] = useState(false); const [provenanceError, setProvenanceError] = useState(""); + const provenanceAbortRef = useRef(null); const load = useCallback(async () => { setLoading(true); @@ -81,10 +82,18 @@ export function ArtifactSurface({ }, [artifact.fileId]); useEffect(() => { + provenanceAbortRef.current?.abort("artifact-changed"); + provenanceAbortRef.current = null; setPreview(null); setProvenance(null); + setProvenanceLoading(false); + setProvenanceError(""); setView("preview"); - load(); + void load(); + return () => { + provenanceAbortRef.current?.abort("artifact-changed"); + provenanceAbortRef.current = null; + }; }, [load]); useEffect(() => { @@ -113,39 +122,55 @@ export function ArtifactSurface({ } } - useEffect(() => { - if (view !== "provenance" || provenance || provenanceLoading) return; - let cancelled = false; + const loadProvenance = useCallback(async () => { + if (provenance || provenanceAbortRef.current) return; + const controller = new AbortController(); + provenanceAbortRef.current = controller; + const timeoutId = window.setTimeout( + () => controller.abort("timeout"), + 8_000, + ); setProvenanceLoading(true); setProvenanceError(""); - fetch(`/api/library/${encodeURIComponent(artifact.fileId)}/provenance`, { - cache: "no-store", - }) - .then(async (response) => { - const data = await response.json().catch(() => null); - if (!response.ok) { - throw new Error( - data?.message || data?.error || "Could not load provenance", - ); - } - if (!cancelled) setProvenance(data?.data ?? data); - }) - .catch((cause) => { - if (!cancelled) { - setProvenanceError( - cause instanceof Error - ? cause.message - : "Could not load provenance", - ); - } - }) - .finally(() => { - if (!cancelled) setProvenanceLoading(false); - }); - return () => { - cancelled = true; - }; - }, [artifact.fileId, provenance, provenanceLoading, view]); + try { + const response = await fetch( + `/api/library/${encodeURIComponent(artifact.fileId)}/provenance?eventLimit=40`, + { cache: "no-store", signal: controller.signal }, + ); + const data = await response.json().catch(() => null); + if (!response.ok) { + throw new Error( + data?.message || data?.error || "Could not load provenance", + ); + } + setProvenance(data?.data ?? data); + } catch (cause) { + if (controller.signal.reason === "artifact-changed") return; + setProvenanceError( + controller.signal.reason === "timeout" + ? "Provenance took too long to load. Please try again." + : cause instanceof Error + ? cause.message + : "Could not load provenance", + ); + } finally { + window.clearTimeout(timeoutId); + if (provenanceAbortRef.current === controller) { + provenanceAbortRef.current = null; + setProvenanceLoading(false); + } + } + }, [artifact.fileId, provenance]); + + useEffect(() => { + if (view === "provenance") void loadProvenance(); + }, [loadProvenance, view]); + + useEffect(() => { + if (!preview || provenance || provenanceError) return; + const timeoutId = window.setTimeout(() => void loadProvenance(), 150); + return () => window.clearTimeout(timeoutId); + }, [loadProvenance, preview, provenance, provenanceError]); const resolved = useMemo( () => ({ @@ -203,7 +228,10 @@ export function ArtifactSurface({
setView("provenance")} + onClick={() => { + setView("provenance"); + void loadProvenance(); + }} > Pr @@ -282,7 +310,10 @@ export function ArtifactSurface({ ) : null} setView("provenance")} + onClick={() => { + setView("provenance"); + void loadProvenance(); + }} icon={BadgeCheck} > Provenance @@ -320,7 +351,11 @@ export function ArtifactSurface({ provenanceLoading ? ( ) : provenanceError ? ( - + void loadProvenance()} + /> ) : provenance ? ( ) : null @@ -686,9 +721,13 @@ function SurfaceTab({ function CenteredMessage({ message, loading = false, + actionLabel, + onAction, }: { message: string; loading?: boolean; + actionLabel?: string; + onAction?: () => void; }) { return (
@@ -696,7 +735,16 @@ function CenteredMessage({ {loading ? ( ) : null} - {message} +

{message}

+ {actionLabel && onAction ? ( + + ) : null}
); diff --git a/apps/commons-app/components/provenance/artifact-provenance.tsx b/apps/commons-app/components/provenance/artifact-provenance.tsx index e407ec5f..cf1f274a 100644 --- a/apps/commons-app/components/provenance/artifact-provenance.tsx +++ b/apps/commons-app/components/provenance/artifact-provenance.tsx @@ -104,6 +104,8 @@ export type ArtifactProvenanceRecord = { }>; disclosure: { eventsIncluded: boolean; + eventsReturned?: number; + eventsTruncated?: boolean; privateReasoningIncluded: false; credentialsIncluded: false; }; @@ -296,7 +298,18 @@ export function ArtifactProvenance({ ) : null}
- Trail +
+ Trail + {record.disclosure.eventsTruncated ? ( + + Latest{" "} + {record.disclosure.eventsReturned ?? record.actions.length} + {record.capture.eventCount + ? ` of ${record.capture.eventCount}` + : ""} + + ) : null} +
{actionRows.length ? ( actionRows.slice(-40).map((action) => (