diff --git a/.changeset/project-templates-first-class.md b/.changeset/project-templates-first-class.md new file mode 100644 index 0000000000..a80bb1352a --- /dev/null +++ b/.changeset/project-templates-first-class.md @@ -0,0 +1,18 @@ +--- +"@milaboratories/pl-middle-layer": minor +--- + +Make project templates first-class entities. + +A template is now stored in the user's root rather than only exported to a file: the +immutable `template-v1` document lives in an ephemeral resource's data blob, its label and +timestamps in KV, and the whole shelf is listed and watched like the project list. +`saveProjectAsTemplate`, `renameTemplate`, `deleteTemplate`, `getTemplateData`, +`resolveTemplate`, `createProjectFromTemplate` and `shareTemplate` are the new surface. + +A template can also travel: `EnvelopePayload` is now a discriminated union +(`{ kind: "projects" }` | `{ kind: "template" }`) and `EnvelopeData.schemaVersion` is 2. +A template share is read-only and writes no `acceptance/{login}` receipt — the recipient +gets the document on their own shelf and decides when to apply it, so there is no +acceptance to report back. A client that does not recognise a payload kind hides that +share rather than mis-rendering it. diff --git a/lib/node/pl-middle-layer/src/middle_layer/index.ts b/lib/node/pl-middle-layer/src/middle_layer/index.ts index b06e189d56..ec428e8dbc 100644 --- a/lib/node/pl-middle-layer/src/middle_layer/index.ts +++ b/lib/node/pl-middle-layer/src/middle_layer/index.ts @@ -4,3 +4,12 @@ export * from "./driver_kit"; export * from "./ops"; export { ProjectsField } from "./project_list"; export type { OutgoingShare, PendingShare } from "./sharing_list"; +export { TemplatesField } from "./template_list"; +export type { + CreateProjectFromTemplateOutcome, + SaveProjectAsTemplateOutcome, + ShareTemplateOutcome, + StoredTemplateData, + TemplateId, + TemplateListEntry, +} from "./template_list"; diff --git a/lib/node/pl-middle-layer/src/middle_layer/middle_layer.ts b/lib/node/pl-middle-layer/src/middle_layer/middle_layer.ts index 88485a990f..1b4f0b0930 100644 --- a/lib/node/pl-middle-layer/src/middle_layer/middle_layer.ts +++ b/lib/node/pl-middle-layer/src/middle_layer/middle_layer.ts @@ -20,6 +20,22 @@ import { ProjectsField, ProjectsResourceType, } from "./project_list"; +import type { + CreateProjectFromTemplateOutcome, + SaveProjectAsTemplateOutcome, + ShareTemplateOutcome, + StoredTemplateData, + TemplateId, + TemplateListEntry, +} from "./template_list"; +import { + createTemplateList, + decodeStoredTemplateData, + TemplateLabelKey, + TemplatesField, + TemplatesResourceType, +} from "./template_list"; +import { createTemplate, deleteTemplate, renameTemplate } from "../mutator/template"; import { createProject, duplicateProject, @@ -31,7 +47,7 @@ import type { ProjectTemplateV1 } from "@milaboratories/pl-model-common"; import { extractConfig, ensureError } from "@platforma-sdk/model"; import type { TemplateApplyProblem, TemplateApplyReport } from "../model/template_apply"; import { TemplateEntryRejected, kindMismatch } from "../model/template_apply"; -import type { BlockPackProvider } from "../model/template_resolve"; +import type { BlockPackProvider, TemplateResolveOutcome } from "../model/template_resolve"; import { resolveTemplateEntries } from "../model/template_resolve"; import type { PreparedTemplateEntry } from "../mutator/template_construct"; import { applyTemplateEntries } from "../mutator/template_construct"; @@ -45,6 +61,7 @@ import { canGrantToEveryone, canImpersonate, decodeEnvelopeData, + envelopeProjectMap, isAcceptanceField, SharingOutboxField, SharingOutboxResourceType, @@ -56,9 +73,13 @@ import { type ProjectFieldUuid, type ShareId, type ShareProjectsOptions, + type ShareTemplateOptions, } from "../model/sharing_model"; +import type { TemplateShareProblem } from "../model/template_share"; +import { unshareableTemplateEntries } from "../model/template_share"; import { buildShareEnvelope, + buildTemplateShareEnvelope, copyEnvelopeProjectsIntoList, envelopeProjectFieldUuid, isEnvelopeProjectField, @@ -151,16 +172,20 @@ export class MiddleLayer { public readonly driverKit: DriverKit, public readonly signer: Signer, private readonly projectListResourceId: SignedResourceId, + private readonly templateListResourceId: SignedResourceId, private readonly sharingOutboxResourceId: SignedResourceId, private readonly sharingStateResourceId: SignedResourceId, private readonly openedProjectsList: WatchableValue, private readonly projectListTree: SynchronizedTreeState, + private readonly templateListTree: SynchronizedTreeState, private readonly sharingOutboxTree: SynchronizedTreeState, private readonly sharingStateTree: SynchronizedTreeState, private readonly pendingSharesTree: SynchronizedTreeState, public readonly blockRegistryProvider: V2RegistryProvider, /** Contains a reactive list of projects along with their meta information. */ public readonly projectList: ComputableStableDefined, + /** Contains a reactive list of stored templates along with their labels and provenance. */ + public readonly templateList: ComputableStableDefined, /** Reactive view of the donor's outbox — the shares this user has created. * v1: API only, no UI. */ public outgoingShares: Computable, @@ -417,15 +442,37 @@ export class MiddleLayer { provider: BlockPackProvider, options: { allowUnstable?: boolean; author?: AuthorMarker } = {}, ): Promise { - const resolution = await resolveTemplateEntries(document, provider, { + const preparation = await this.prepareTemplateEntries(document, provider, { allowUnstable: options.allowUnstable ?? false, }); - if (resolution.problems.length > 0) return { added: [], problems: resolution.problems }; + if (preparation.problems.length > 0) return { added: [], problems: preparation.problems }; + return await this.applyPreparedEntries(id, document, preparation.prepared, options.author); + } + + /** + * Stages 1 and 2 of an apply — resolve every entry to a block pack, then prepare every + * block — for a document that may not have a project yet. + * + * Neither stage creates anything, which is what lets a caller run them before it decides to + * create a project at all: {@link createProjectFromTemplate} does exactly that, so an + * unapplicable template leaves no empty project behind. + */ + private async prepareTemplateEntries( + document: ProjectTemplateV1, + provider: BlockPackProvider, + options: { allowUnstable: boolean }, + ): Promise<{ + prepared: Map; + problems: TemplateApplyProblem[]; + }> { + const prepared = new Map(); + + const resolution = await resolveTemplateEntries(document, provider, options); + if (resolution.problems.length > 0) return { prepared, problems: [...resolution.problems] }; // One map, not one per field: resolution reports by entry id, so everything this loop // needs about an entry is looked up the same way. const byEntryId = new Map(document.blocks.map((entry) => [entry.id, entry])); - const prepared = new Map(); const problems: TemplateApplyProblem[] = []; for (const entry of resolution.resolved) { @@ -482,8 +529,21 @@ export class MiddleLayer { } } - if (problems.length > 0) return { added: [], problems }; + return { prepared, problems }; + } + /** + * Stage 3 of an apply — create the blocks in one transaction, all or nothing. + * + * An entry it cannot create throws, the transaction is never committed, and the project keeps + * none of the blocks the apply had placed. + */ + private async applyPreparedEntries( + id: ProjectId, + document: ProjectTemplateV1, + prepared: Map, + author?: AuthorMarker, + ): Promise { const rid = await this.resolveProjectId(id); let added: AppliedEntry[] = []; try { @@ -491,7 +551,7 @@ export class MiddleLayer { this.env.projectHelper, this.pl, rid, - options.author, + author, (mut) => { added = applyTemplateEntries({ document, @@ -516,6 +576,159 @@ export class MiddleLayer { return { added, problems: [] }; } + // + // Template List Manipulation + // + + private readonly templateIdCache = new LRUCache({ max: 1024 }); + + /** + * Saves a project as a template: a snapshot of its blocks and their params, no data. + * + * The document the export produced is what gets stored; the YAML it also rendered is a + * file format, and a stored template is rendered to it only on download. + * + * A block that cannot be expressed as a template entry stores nothing at all, and every + * such block is reported — fixing an unexportable project takes one pass, not one per block. + * + * @param projectId project to snapshot + * @param label label for the template; defaults to the project's own label + */ + public async saveProjectAsTemplate( + projectId: ProjectId, + label?: string, + ): Promise { + const outcome = await this.exportProjectAsTemplate(projectId); + if (!outcome.ok) return { ok: false, problems: outcome.problems }; + + const rid = await this.resolveProjectId(projectId); + let tpl: ResourceRef; + await this.pl.withWriteTx("MLSaveProjectAsTemplate", async (tx) => { + const meta = await tx.getKValueJson(rid, ProjectMetaKey); + tpl = createTemplate(tx, this.templateListResourceId, label ?? meta.label, { + schemaVersion: 1, + document: outcome.document, + sourceProjectLabel: meta.label, + }); + await tx.commit(); + }); + await this.templateListTree.refreshState(); + + const signedRid = await tpl!.globalId; + const templateId = resourceIdToString(signedRid) as TemplateId; + this.templateIdCache.set(templateId, signedRid); + return { ok: true, templateId }; + } + + /** Changes a template's label. The stored document is immutable and stays untouched — + * improving a template means saving a new one. */ + public async renameTemplate(id: TemplateId, label: string): Promise { + const rid = await this.resolveTemplateId(id); + await this.pl.withWriteTx("MLRenameTemplate", async (tx) => { + renameTemplate(tx, rid, label); + await tx.commit(); + }); + await this.templateListTree.refreshState(); + } + + /** Permanently deletes a template from the template list. */ + public async deleteTemplate(id: TemplateId): Promise { + await this.pl.withWriteTx("MLRemoveTemplate", async (tx) => { + await deleteTemplate(tx, this.templateListResourceId, id); + await tx.commit(); + }); + this.templateIdCache.delete(id); + await this.templateListTree.refreshState(); + } + + /** Reads a stored template: its document plus what was true when it was taken. */ + public async getTemplateData(id: TemplateId): Promise { + const rid = await this.resolveTemplateId(id); + return await this.pl.withReadTx("MLGetTemplate", async (tx) => { + const rd = await tx.getResourceData(rid, false); + if (rd.data === undefined) throw new Error(`Template ${id} carries no document.`); + return decodeStoredTemplateData(rd.data); + }); + } + + /** + * Where each entry of a stored template would get its block from, and which entries have + * nowhere to get one — resolution creates nothing, so this is the preview a UI shows before + * offering Apply. {@link createProjectFromTemplate} runs the same stage itself. + */ + public async resolveTemplate( + id: TemplateId, + provider: BlockPackProvider, + options: { allowUnstable?: boolean } = {}, + ): Promise { + const stored = await this.getTemplateData(id); + return await resolveTemplateEntries(stored.document, provider, { + allowUnstable: options.allowUnstable ?? false, + }); + } + + /** + * Creates one project holding every block the stored template lists, in the template's order. + * + * Resolution and preparation run before the project exists, so a template with an entry + * nothing can supply a block for leaves no empty project in the list. The one write that + * follows is all or nothing, and an entry it rejects takes the project with it. + * + * @param id template to apply + * @param label label for the new project + * @param provider where each entry's block comes from + * @param options `allowUnstable` widens resolution to pre-release implementations + */ + public async createProjectFromTemplate( + id: TemplateId, + label: string, + provider: BlockPackProvider, + options: { allowUnstable?: boolean; author?: AuthorMarker } = {}, + ): Promise { + const stored = await this.getTemplateData(id); + + const preparation = await this.prepareTemplateEntries(stored.document, provider, { + allowUnstable: options.allowUnstable ?? false, + }); + if (preparation.problems.length > 0) return { ok: false, problems: preparation.problems }; + + const projectId = await this.createProject({ label }); + const report = await this.applyPreparedEntries( + projectId, + stored.document, + preparation.prepared, + options.author, + ); + if (report.problems.length > 0) { + // The apply is one transaction, so the project holds none of the blocks: it is the empty + // project this call created moments ago and nothing else, and leaving it in the list would + // show the user a project they never asked for. + await this.deleteProject(projectId); + return { ok: false, problems: report.problems }; + } + return { ok: true, projectId, added: report.added }; + } + + /** Resolves a TemplateId to a signed SignedResourceId. + * Uses LRU cache with TX-scan fallback. */ + private async resolveTemplateId(templateId: TemplateId): Promise { + const cached = this.templateIdCache.get(templateId); + if (cached !== undefined) return cached; + + // Cache miss — scan template list fields to find the matching resource + const rid = await this.pl.withReadTx("ResolveTemplateId", async (tx) => { + const data = await tx.getResourceData(this.templateListResourceId, true); + for (const f of data.fields) { + if (isNullSignedResourceId(f.value)) continue; + if (resourceIdToString(f.value) === (templateId as string)) return f.value; + } + throw new Error(`Template ${templateId} not found in template list.`); + }); + + this.templateIdCache.set(templateId, rid); + return rid; + } + /** Permanently deletes project from the project list, this will result in * destruction of all attached objects, like files, analysis results etc. */ public async deleteProject(id: ProjectId): Promise { @@ -738,23 +951,116 @@ export class MiddleLayer { expiresAt, }); - // Grant in the same transaction (writable: the cross-color accept rule demands a writable - // grant on the envelope). Atomic with the create. - const envelopeGid = await envelope.globalId; - if (everyone) { - // One everyone-grant: empty/ignored target, ANY_AUTHORISED. The backend rewrites - // the target to the everyone-user; gated by role + permission ceiling. - tx.grantAccess(envelopeGid, "", { writable: true }, GrantType.ANY_AUTHORISED); - } else { - for (const recipient of options.recipients) { - tx.grantAccess(envelopeGid, recipient, { writable: true }); - } - } + // Grant in the same transaction, atomic with the create. + await this.grantShareEnvelope(tx, envelope, everyone, everyone ? [] : options.recipients, { + writable: true, + }); + + await tx.commit(); + }); + + await this.sharingOutboxTree.refreshState(); + } + + /** + * Grants one freshly built envelope inside the transaction that created it: a single make-public + * grant for an everyone-share (empty/ignored target, ANY_AUTHORISED — the backend rewrites the + * target to the everyone-user, gated by role + permission ceiling), or one grant per named + * recipient. + * + * `writable` is not a preference. A project pack needs a writable grant because accepting copies + * the snapshots out of the envelope, and the cross-color attach rule permits that only to a + * writable grant holder. A template share copies nothing — the document sits in the envelope's + * own immutable data — so it is granted read-only, and must be: a writable everyone-grant would + * hand every user on the server write access to the envelope. + */ + private async grantShareEnvelope( + tx: PlTransaction, + envelope: ResourceRef, + everyone: boolean, + recipients: string[], + permissions: { writable: boolean }, + ): Promise { + const gid = await envelope.globalId; + if (everyone) tx.grantAccess(gid, "", permissions, GrantType.ANY_AUTHORISED); + else for (const r of recipients) tx.grantAccess(gid, r, permissions); + } + + /** + * Shares one stored template. The envelope carries the document itself, so there is no project + * snapshot and no resource for the recipient to copy out — which is why the grant is read-only. + * The cost of that is the donor's receipt: nobody can write an acceptance onto a read-only + * envelope, so a template share never reports who accepted it. + * + * A template holding a block installed from a folder on this machine is refused rather than sent, + * with every offending entry named. {@link checkTemplateShareable} answers the same question + * without attempting the share, so a UI can state it on the template itself. + * + * @param id template to share + * @param options recipients XOR everyone, plus the title recipients see + */ + public async shareTemplate( + id: TemplateId, + options: ShareTemplateOptions, + ): Promise { + const loaded = await this.loadShareableTemplate(id); + if (!loaded.ok) return { ok: false, problems: loaded.problems }; + const everyone = "everyone" in options; + const sender = this.currentUserLogin ?? ""; + // Targeted share: sharedAt + ttl. Share-with-everybody: never expires (null). + const expiresAt = everyone ? null : Date.now() + this.env.ops.envelopeTtlMs; + + let shareId: ShareId | undefined; + await this.pl.withWriteTx("MLShareTemplate", async (tx) => { + const { envelope, data } = buildTemplateShareEnvelope( + tx, + this.sharingOutboxResourceId, + loaded.template, + { sender, title: options.title, expiresAt }, + ); + shareId = data.shareId; + await this.grantShareEnvelope(tx, envelope, everyone, everyone ? [] : options.recipients, { + writable: false, + }); await tx.commit(); }); await this.sharingOutboxTree.refreshState(); + return { ok: true, shareId: shareId! }; + } + + /** + * Every entry of a stored template that stands in the way of sharing it, empty for a template + * that can be shared. Reads the template and nothing else, so a UI can state the refusal on the + * template itself instead of only when the user tries to share it. + */ + public async checkTemplateShareable(id: TemplateId): Promise { + const stored = await this.getTemplateData(id); + return unshareableTemplateEntries(stored.document); + } + + /** The document and the label of a template that may be shared, or every entry that stops it. + * The label is what the recipient's own list will show, so it travels with the document. */ + private async loadShareableTemplate( + id: TemplateId, + ): Promise< + | { ok: true; template: { document: ProjectTemplateV1; label: string } } + | { ok: false; problems: readonly TemplateShareProblem[] } + > { + const rid = await this.resolveTemplateId(id); + const template = await this.pl.withReadTx("MLReadTemplateForShare", async (tx) => { + const rd = await tx.getResourceData(rid, false); + if (rd.data === undefined) throw new Error(`Template ${id} carries no document.`); + return { + document: decodeStoredTemplateData(rd.data).document, + label: await tx.getKValueJson(rid, TemplateLabelKey), + }; + }); + + const problems = unshareableTemplateEntries(template.document); + if (problems.length > 0) return { ok: false, problems }; + return { ok: true, template }; } /** @@ -772,6 +1078,12 @@ export class MiddleLayer { * existing snapshot (and its timestamp), `remove` drops the project from the pack. A project not * in the map defaults to `keep`. Omit the whole map for the legacy auto behavior (live sources * updated, gone ones kept) — the everyone-refresh path relies on that. + * + * `opts.templateId` is required for, and only used by, a share that carries a template: a stored + * template is immutable, so an improved one is a different template and the share cannot re-read + * the one it started from — the caller names the new target. Every other option means the same + * thing for both kinds of share. Sharing the named template must be permitted (see + * {@link checkTemplateShareable}) or this throws. */ public async changeShare( shareId: ShareId, @@ -780,8 +1092,18 @@ export class MiddleLayer { everyone?: boolean; title?: string; projectActions?: Record; + templateId?: TemplateId; } = {}, ): Promise { + // Read outside the write tx: it is two round-trips of its own, and the refusal it can produce + // must be raised before anything is torn down. + const target = + opts.templateId === undefined ? undefined : await this.loadShareableTemplate(opts.templateId); + if (target !== undefined && !target.ok) + throw new Error( + `changeShare: template ${opts.templateId} cannot be shared: ${describeShareProblems(target.problems)}`, + ); + await this.pl.withWriteTx("MLChangeShare", async (tx) => { const old = await this.resolveOutboxEnvelope(tx, shareId); if (old === undefined) @@ -791,6 +1113,38 @@ export class MiddleLayer { const grants = await tx.listGrants(old.rid); // A targeted share may be upgraded to everyone; an everyone-share can't be narrowed back. const everyone = grants.some((g) => isEveryoneUserLogin(g.user)) || opts.everyone === true; + const priorRecipients = grants + .filter((g) => !isEveryoneUserLogin(g.user) && g.user !== self) + .map((g) => g.user); + + if (old.data.payload.kind === "template") { + if (target === undefined) + throw new Error( + `changeShare: share ${shareId} carries a template, so it needs an explicit target ` + + "template — a stored template never changes, so an improved one is a different template.", + ); + const recipients = everyone ? [] : (opts.recipients ?? priorRecipients); + + // Same shareId, same outbox field name — detach the old field before rebuilding, or they collide. + tx.removeField(field(this.sharingOutboxResourceId, old.fieldName)); + const { envelope } = buildTemplateShareEnvelope( + tx, + this.sharingOutboxResourceId, + target.template, + { + sender: self, + title: opts.title === undefined ? old.data.title : opts.title.trim(), + expiresAt: everyone ? null : Date.now() + this.env.ops.envelopeTtlMs, + shareId, // SAME shareId — the essence of change + }, + ); + + // Nothing to transfer: a read-only grant cannot write an acceptance, so a template share + // never accumulated one. + await this.grantShareEnvelope(tx, envelope, everyone, recipients, { writable: false }); + await tx.commit(); + return; + } // Read the old envelope's project snapshots (uuid -> rid) and accept/reject records. const oldRd = await tx.getResourceData(old.rid, true); @@ -812,9 +1166,6 @@ export class MiddleLayer { const decidedLogins = acceptances.map((a) => a.login); // Everyone-shares ignore recipients; targeted shares keep decided users plus the edited set. - const priorRecipients = grants - .filter((g) => !isEveryoneUserLogin(g.user) && g.user !== self) - .map((g) => g.user); const recipients = everyone ? [] : Array.from(new Set([...(opts.recipients ?? priorRecipients), ...decidedLogins])); @@ -831,8 +1182,9 @@ export class MiddleLayer { // projectActions map, fall back to the legacy auto behavior: update a live source, keep a gone one. const actions = opts.projectActions; const sources: EnvelopeProjectSource[] = []; - for (const uuid of Object.keys(old.data.projects) as ProjectFieldUuid[]) { - const { label, source, updatedAt } = old.data.projects[uuid]; + const oldProjects = envelopeProjectMap(old.data); + for (const uuid of Object.keys(oldProjects) as ProjectFieldUuid[]) { + const { label, source, updatedAt } = oldProjects[uuid]; const liveRid = liveProjects.get(source); const action = actions @@ -874,9 +1226,7 @@ export class MiddleLayer { writeEnvelopeAcceptance(tx, envelope, login, acc.action, acc.timestamp); } - const gid = await envelope.globalId; - if (everyone) tx.grantAccess(gid, "", { writable: true }, GrantType.ANY_AUTHORISED); - else for (const r of recipients) tx.grantAccess(gid, r, { writable: true }); + await this.grantShareEnvelope(tx, envelope, everyone, recipients, { writable: true }); await tx.commit(); }); @@ -909,7 +1259,8 @@ export class MiddleLayer { const rd = await tx.getResourceData(f.value, false); if (rd.data === undefined) continue; const data = decodeEnvelopeData(rd.data); - if (Object.values(data.projects).some((p) => wanted.has(p.source))) + if (data === undefined) continue; + if (Object.values(envelopeProjectMap(data)).some((p) => wanted.has(p.source))) out.push({ fieldName: f.name, rid: f.value, shareId: data.shareId }); } return out; @@ -961,6 +1312,7 @@ export class MiddleLayer { const rd = await tx.getResourceData(f.value, false); if (rd.data === undefined) continue; const data = decodeEnvelopeData(rd.data); + if (data === undefined) continue; if (data.shareId === shareId) return { fieldName: f.name, rid: f.value, data }; } return undefined; @@ -986,22 +1338,31 @@ export class MiddleLayer { } /** - * Accepts one or more pending shares: duplicates each share's projects into this user's - * project list, records the decision per share, and (read-write share) writes the donor-visible - * acceptance onto the envelope. Per-share failures (e.g. an expiry race) are collected, not - * short-circuited — the rest still get accepted. Accept-all = pass every current pending shareId. + * Accepts one or more pending shares. What accepting does depends on what the share carries: a + * pack of projects is duplicated into this user's project list, while a template is added to this + * user's own template list and builds nothing — the recipient decides later whether to apply it. + * Either way the decision is recorded per share, and a read-write share also gets the + * donor-visible acceptance written onto its envelope. Per-share failures (e.g. an expiry race) + * are collected, not short-circuited — the rest still get accepted. Accept-all = pass every + * current pending shareId. * * `rename` resolves label collisions (same callback contract as {@link duplicateProject}), but - * the source lives in the envelope tree, so accept calls the low-level mutator directly. + * the source lives in the envelope tree, so accept calls the low-level mutator directly. It does + * not apply to a template share, whose label is not required to be unique. */ public async acceptShare( shareIds: ShareId[], rename?: (previousLabel: string, existingLabels: string[]) => string, - ): Promise<{ accepted: ProjectId[]; failed: { shareId: ShareId; error: string }[] }> { + ): Promise<{ + accepted: ProjectId[]; + acceptedTemplates: TemplateId[]; + failed: { shareId: ShareId; error: string }[]; + }> { const live = await this.resolveLiveEnvelopes(); const login = this.currentUserLogin; const accepted: ProjectId[] = []; + const acceptedTemplates: TemplateId[] = []; const failed: { shareId: ShareId; error: string }[] = []; for (const shareId of shareIds) { @@ -1012,6 +1373,36 @@ export class MiddleLayer { } try { const now = Date.now(); + const payload = envelope.data.payload; + + if (payload.kind === "template") { + const rid = await this.pl.withWriteTx("MLAcceptTemplateShare", async (tx) => { + // The template lands on this user's own shelf, keeping who sent it as its provenance. + const tpl = createTemplate(tx, this.templateListResourceId, payload.label, { + schemaVersion: 1, + document: payload.document, + sender: payload.from, + }); + + writeSharingDecision(tx, this.sharingStateResourceId, shareId, { + decision: "accepted", + timestamp: now, + envelopeSharedAt: envelope.data.sharedAt, + acceptedProjects: [], // a template share creates no project + }); + + // No acceptance/{login} on the envelope: the grant is read-only, so the write would be + // refused by the backend, and the donor deliberately gave up that receipt. + await tx.commit(); + return await tpl.globalId; + }); + + const templateId = resourceIdToString(rid) as TemplateId; + this.templateIdCache.set(templateId, rid); + acceptedTemplates.push(templateId); + continue; + } + const createdRids = await this.pl.withWriteTx("MLAcceptShare", async (tx) => { const created = await copyEnvelopeProjectsIntoList( tx, @@ -1045,8 +1436,12 @@ export class MiddleLayer { } } - await Promise.all([this.projectListTree.refreshState(), this.sharingStateTree.refreshState()]); - return { accepted, failed }; + await Promise.all([ + this.projectListTree.refreshState(), + this.templateListTree.refreshState(), + this.sharingStateTree.refreshState(), + ]); + return { accepted, acceptedTemplates, failed }; } /** Records rejection of a pending share; it never surfaces again. */ @@ -1104,6 +1499,7 @@ export class MiddleLayer { const rd = await tx.getResourceData(f.value, false); if (rd.data === undefined) continue; const envData = decodeEnvelopeData(rd.data); + if (envData === undefined) continue; if (envData.expiresAt === null) continue; // never expires if (envData.expiresAt <= now) toDelete.push({ fieldName: f.name }); } @@ -1172,6 +1568,7 @@ export class MiddleLayer { // this.env.quickJs; await Promise.all([ this.projectListTree.terminate(), + this.templateListTree.terminate(), this.sharingOutboxTree.terminate(), this.sharingStateTree.terminate(), this.pendingSharesTree.terminate(), @@ -1218,7 +1615,7 @@ export class MiddleLayer { ) ops.defaultTreeOptions.traversalMode = getDebugFlags().treeTraversalMode; - const { projects, sharingOutbox, sharingState } = await pl.withWriteTx( + const { projects, templates, sharingOutbox, sharingState } = await pl.withWriteTx( "MLInitialization", async (tx) => { // Lazily create each clientRoot-attached singleton resource. Returns the existing @@ -1240,6 +1637,7 @@ export class MiddleLayer { }; const projectsR = await lazyInit(ProjectsField, ProjectsResourceType); + const templatesR = await lazyInit(TemplatesField, TemplatesResourceType); const outboxR = await lazyInit(SharingOutboxField, SharingOutboxResourceType); const stateR = await lazyInit(SharingStateField, SharingStateResourceType); @@ -1247,6 +1645,7 @@ export class MiddleLayer { return { projects: projectsR.existing ?? (await projectsR.ref!.globalId), + templates: templatesR.existing ?? (await templatesR.ref!.globalId), sharingState: stateR.existing ?? (await stateR.ref!.globalId), sharingOutbox: outboxR.existing ?? (await outboxR.ref!.globalId), }; @@ -1312,6 +1711,7 @@ export class MiddleLayer { const openedProjects = new WatchableValue([]); const projectListTC = await createProjectList(pl, projects, openedProjects, env); + const templateListTC = await createTemplateList(pl, templates, env); // Project sharing trees and reactive views. const outgoingTC = await createOutgoingShares(pl, sharingOutbox, env); @@ -1329,18 +1729,31 @@ export class MiddleLayer { driverKit, driverKit.signer, projects, + templates, sharingOutbox, sharingState, openedProjects, projectListTC.tree, + templateListTC.tree, outgoingTC.tree, sharingStateTree, pendingSharesTree, v2RegistryProvider, projectListTC.computable, + templateListTC.computable, outgoingTC.computable, pendingShares, liveEnvelopes, ); } } + +// +// Internals +// + +/** Refusal reasons as one line, each naming the entry it belongs to, so a throw that escapes to a + * log still says which block stands in the way. */ +function describeShareProblems(problems: readonly TemplateShareProblem[]): string { + return problems.map((p) => `${p.entryId}: ${p.error}`).join("; "); +} diff --git a/lib/node/pl-middle-layer/src/middle_layer/sharing_list.ts b/lib/node/pl-middle-layer/src/middle_layer/sharing_list.ts index 2dded0f19f..16fa3cf9e7 100644 --- a/lib/node/pl-middle-layer/src/middle_layer/sharing_list.ts +++ b/lib/node/pl-middle-layer/src/middle_layer/sharing_list.ts @@ -15,11 +15,14 @@ import type { EnvelopeAcceptance, EnvelopeData, EnvelopeMode, + EnvelopePayloadKind, ShareId, } from "../model/sharing_model"; import { AcceptanceFieldPrefix, asShareId, + envelopeProjectMap, + normalizeEnvelopeData, SharedEnvelopeResourceType, SharingOutboxResourceType, SharingStateResourceType, @@ -32,19 +35,28 @@ export interface OutgoingShare { expiresAt?: number; // EnvelopeData.expiresAt; null maps to undefined = never expires mode: EnvelopeMode; title: string; // display name shown to recipients; defaults to the first project's name + /** What the share carries — a pack of projects, or one template document. */ + payloadKind: EnvelopePayloadKind; /** One entry per project in the pack, so the change UI can offer a per-project decision. * `projectId` is the donor's source project id; `updatedAt` is when this project's - * snapshot was last (re)taken. */ + * snapshot was last (re)taken. Empty for a template share. */ projects: { projectId: ProjectId; label: string; updatedAt: number }[]; + /** The shared template, for a template share; absent for a project share. */ + template?: { label: string; blockCount: number }; /** Full recipient logins, from `ListGrants` on the envelope; `["*"]` for everyone-shares. */ recipients: string[]; + /** Whether {@link responses} can ever be populated for this share. A template share is granted + * read-only, so no recipient can record a reply on the envelope and the donor never learns who + * accepted — a view must say so rather than render an empty response list as "nobody yet". */ + responsesAvailable: boolean; /** Per recipient who has responded: their decision and when, from acceptance/{login}. */ responses: Record; } -/** Per-project view for the donor's change UI, from {@link EnvelopeData.projects}. */ +/** Per-project view for the donor's change UI, from the envelope's `projects` payload; empty + * for a payload that carries no project. */ function envelopeProjects(data: EnvelopeData): OutgoingShare["projects"] { - return Object.values(data.projects).map((p) => ({ + return Object.values(envelopeProjectMap(data)).map((p) => ({ projectId: p.source, label: p.label, updatedAt: p.updatedAt, @@ -57,6 +69,9 @@ export interface PendingShare { sender: string; // EnvelopeData.sender, display only title: string; // display name shown to recipients; defaults to the first project's name mode: EnvelopeMode; // v1 renders only "copy" entries + /** What the offer carries, so the recipient is told what they are being offered — projects to + * copy, or a template for their own shelf. */ + payloadKind: EnvelopePayloadKind; grantedAt: number; } @@ -108,8 +123,8 @@ export function createOutgoingSharesComputable( if (envelope === undefined) continue; if (!resourceTypesEqual(envelope.resourceType, SharedEnvelopeResourceType)) continue; - const data = envelope.getDataAsJson(); - if (data === undefined) continue; + const data = normalizeEnvelopeData(envelope.getDataAsJson()); + if (data === undefined) continue; // unknown version or payload kind — not ours to show const responses: OutgoingShare["responses"] = {}; for (const f of envelope.listDynamicFields()) { @@ -127,7 +142,17 @@ export function createOutgoingSharesComputable( ...(data.expiresAt !== null ? { expiresAt: data.expiresAt } : {}), mode: data.mode, title: data.title, + payloadKind: data.payload.kind, projects: envelopeProjects(data), + ...(data.payload.kind === "template" + ? { + template: { + label: data.payload.label, + blockCount: data.payload.document.blocks.length, + }, + } + : {}), + responsesAvailable: data.payload.kind !== "template", responses, envelopeRid: envelope.id, }); @@ -255,7 +280,9 @@ export function createLiveEnvelopesComputable( for (const envelope of roots) { if (envelope === undefined) continue; if (!resourceTypesEqual(envelope.resourceType, SharedEnvelopeResourceType)) continue; - const data = envelope.getDataAsJson(); + const data = normalizeEnvelopeData(envelope.getDataAsJson()); + // Same recognise-or-hide rule as the pending view, and for the same envelope: hiding an + // offer while accept could still resolve it would only move the problem. if (data === undefined) continue; result.push({ rid: envelope.id, data }); } @@ -292,7 +319,9 @@ export function createPendingSharesComputable( for (const envelope of roots) { if (envelope === undefined) continue; if (!resourceTypesEqual(envelope.resourceType, SharedEnvelopeResourceType)) continue; - const data = envelope.getDataAsJson(); + const data = normalizeEnvelopeData(envelope.getDataAsJson()); + // An envelope whose schemaVersion or payload kind this build does not know is hidden, not + // offered: there is nothing useful to do with a share we cannot read. if (data === undefined) continue; if (handled.has(data.shareId)) continue; // already accepted or rejected if (currentUserLogin !== null && data.sender === currentUserLogin) continue; // own share @@ -303,6 +332,7 @@ export function createPendingSharesComputable( sender: data.sender, title: data.title, mode: data.mode, + payloadKind: data.payload.kind, grantedAt: data.sharedAt, }); } diff --git a/lib/node/pl-middle-layer/src/middle_layer/template_list.ts b/lib/node/pl-middle-layer/src/middle_layer/template_list.ts new file mode 100644 index 0000000000..ab9bc35688 --- /dev/null +++ b/lib/node/pl-middle-layer/src/middle_layer/template_list.ts @@ -0,0 +1,177 @@ +import type { PruningFunction } from "@milaboratories/pl-tree"; +import { SynchronizedTreeState } from "@milaboratories/pl-tree"; +import type { + Filter, + PlClient, + PlTransaction, + ResourceType, + SignedResourceId, +} from "@milaboratories/pl-client"; +import { + field, + isNullSignedResourceId, + resourceIdToString, + resourceTypesEqual, + treeFilter, +} from "@milaboratories/pl-client"; +import type { TreeAndComputableU } from "./types"; +import { Computable } from "@milaboratories/computable"; +import type { MiddleLayerEnvironment } from "./middle_layer"; +import { notEmpty } from "@milaboratories/ts-helpers"; +import type { Branded, ProjectTemplateV1 } from "@milaboratories/pl-model-common"; +import type { TemplateExportProblem } from "../model/template_export"; +import type { TemplateShareProblem } from "../model/template_share"; +import type { ShareId } from "../model/sharing_model"; +import type { AppliedEntry, TemplateApplyProblem } from "../model/template_apply"; +import type { ProjectId } from "../model/project_model"; + +export const TemplatesField = "templates"; +export const TemplatesResourceType: ResourceType = { name: "Templates", version: "1" }; +export const TemplateResourceType: ResourceType = { name: "UserTemplate", version: "1" }; + +/** Mutable: the only part of a stored template a rename may touch. */ +export const TemplateLabelKey = "TemplateLabel"; +export const TemplateCreatedTimestamp = "TemplateCreated"; + +/** + * Unique template identifier in middle layer, the stringified signed resource id of the + * `UserTemplate`. Branded so it cannot be confused with a {@link ProjectId} — both are + * stringified resource ids and every template method takes one of them. + */ +export type TemplateId = Branded; + +/** + * Immutable `data` on a UserTemplate: the document plus what was true when it was taken. + * + * The document lives here and never in KV: KV is listed and fully re-read on every poll, + * while resource data syncs incrementally, and a template document is a multi-kilobyte + * value that never changes. + */ +export interface StoredTemplateData { + schemaVersion: 1; + document: ProjectTemplateV1; + /** Provenance, display only; absent for a template that arrived as a share. */ + sourceProjectLabel?: string; + /** Login of the sender, when it arrived as a share. */ + sender?: string; +} + +/** Decodes the immutable `data` blob of a `UserTemplate` read through a transaction. The + * single raw-decode site; the tree side reads the same JSON with `getDataAsJson`. */ +export function decodeStoredTemplateData(data: Uint8Array): StoredTemplateData { + return JSON.parse(Buffer.from(data).toString("utf-8")) as StoredTemplateData; +} + +/** One template as the template list surfaces it. */ +export interface TemplateListEntry { + /** Unique template identifier in middle layer. Use to operate with the given template. */ + id: TemplateId; + /** The mutable label, the only part a rename changes. */ + label: string; + created: Date; + /** Number of blocks the stored document lists — derived, not stored. */ + blockCount: number; + sourceProjectLabel?: string; + sender?: string; +} + +/** What saving a project as a template yields: the stored template, or every block in the way. */ +export type SaveProjectAsTemplateOutcome = + | { readonly ok: true; readonly templateId: TemplateId } + | { readonly ok: false; readonly problems: readonly TemplateExportProblem[] }; + +/** What sharing a stored template yields: the share's logical id, or every entry in the way. */ +export type ShareTemplateOutcome = + | { readonly ok: true; readonly shareId: ShareId } + | { readonly ok: false; readonly problems: readonly TemplateShareProblem[] }; + +/** + * What applying a stored template yields. + * + * `ok: false` carries no project id because no project was created: nothing is written + * until every entry has an installable block. + */ +export type CreateProjectFromTemplateOutcome = + | { + readonly ok: true; + readonly projectId: ProjectId; + readonly added: readonly AppliedEntry[]; + } + | { readonly ok: false; readonly problems: readonly TemplateApplyProblem[] }; + +/** + * Resolves the templates-list resource on the transaction's client root, lazily creating (and + * locking) an empty one when the {@link TemplatesField} is not yet populated. Returns its signed + * id. Used when writing into a root that may have no templates list yet, e.g. a template landing + * in a recipient's root. + */ +export async function ensureTemplateListRid(tx: PlTransaction): Promise { + const templatesField = field(tx.clientRoot, TemplatesField); + tx.createField(templatesField, "Dynamic"); + const fData = await tx.getField(templatesField); + if (isNullSignedResourceId(fData.value)) { + const ref = tx.createEphemeral(TemplatesResourceType); + tx.lock(ref); + tx.setField(templatesField, ref); + return await ref.globalId; + } + return fData.value; +} + +export const TemplatesListTreePruningFunction: PruningFunction = (resource) => { + if (!resourceTypesEqual(resource.type, TemplatesResourceType)) return []; + return resource.fields; +}; + +export const templatesListFieldFilter: Filter = treeFilter.resourceTypeEq( + TemplatesResourceType.name, +); + +export async function createTemplateList( + pl: PlClient, + rid: SignedResourceId, + env: MiddleLayerEnvironment, +): Promise> { + const tree = await SynchronizedTreeState.init( + pl, + rid, + { + ...env.ops.defaultTreeOptions, + pruning: TemplatesListTreePruningFunction, + fieldFilter: templatesListFieldFilter, + }, + env.logger, + ); + + const c = Computable.make((ctx) => { + const node = ctx.accessor(tree.entry()).node(); + if (node === undefined) return undefined; + const result: TemplateListEntry[] = []; + + // Templates list resource keeps templates assigned to fields. Each field name is a UUID + for (const field of node.listDynamicFields()) { + const tpl = node.traverse(field); + if (tpl === undefined) continue; + const data = tpl.getDataAsJson(); + // A template whose data has not synced yet is not an entry with unknown content — + // it is an entry we cannot describe at all, so it stays out of the list until it has. + if (data === undefined) continue; + const label = notEmpty(tpl.getKeyValueAsJson(TemplateLabelKey)); + const created = notEmpty(tpl.getKeyValueAsJson(TemplateCreatedTimestamp)); + result.push({ + id: resourceIdToString(tpl.id) as TemplateId, + label, + created: new Date(created), + blockCount: data.document.blocks.length, + ...(data.sourceProjectLabel !== undefined + ? { sourceProjectLabel: data.sourceProjectLabel } + : {}), + ...(data.sender !== undefined ? { sender: data.sender } : {}), + }); + } + result.sort((a, b) => b.created.valueOf() - a.created.valueOf()); + return result; + }).withStableType(); + + return { computable: c, tree }; +} diff --git a/lib/node/pl-middle-layer/src/middle_layer/templates.test.ts b/lib/node/pl-middle-layer/src/middle_layer/templates.test.ts new file mode 100644 index 0000000000..5aeb44c946 --- /dev/null +++ b/lib/node/pl-middle-layer/src/middle_layer/templates.test.ts @@ -0,0 +1,301 @@ +import { expect, test } from "vitest"; +import * as tp from "node:timers/promises"; +import type { ResourceRef, SignedResourceId } from "@milaboratories/pl-client"; +import { resourceIdToString } from "@milaboratories/pl-client"; +import type { + BlockKindSelectorReference, + BlockPackLocationReference, + ProjectTemplateV1, + ProjectTemplateV1Entry, +} from "@milaboratories/pl-model-common"; +import { PROJECT_TEMPLATE_SCHEMA_V1 } from "@milaboratories/pl-model-common"; +import type { BlockPackSpec } from "@milaboratories/pl-model-middle-layer"; +import type { BlockPackProvider } from "../model/template_resolve"; +import { withMl } from "../test/with_ml"; +import { createTemplate } from "../mutator/template"; +import type { MiddleLayer } from "./middle_layer"; +import type { StoredTemplateData, TemplateId, TemplateListEntry } from "./template_list"; +import { ensureTemplateListRid } from "./template_list"; + +/** + * The stored-template entity against a live backend: rename, apply, share, accept. + * + * Every test here stores its template directly through the mutator rather than by saving a + * project, so the document under test is the one the test wrote — a `file:` entry, an entry + * nothing can resolve — none of which a real project would produce. What a real project + * produces is covered by the round trip in `drivers-ml-blocks-integration`, which has block + * packs on disk to build one from. + * + * Needs a backend, like every `withMl` test in this package, and no gate: `PL_ADDRESS` is + * either configured or the client fails to connect. + */ + +const KIND = "@platforma-open/milaboratories.demo.kind@^1.0.0" as BlockKindSelectorReference; + +/** A block installed from a folder on the author's own machine. */ +const LOCAL_FOLDER = "file:///Users/dev/blocks/demo/block" as BlockPackLocationReference; + +/** A legacy registry block, which predates kinds — so it cannot be written to a template. */ +const KindlessBlock: BlockPackSpec = { + type: "from-registry-v1", + registryUrl: "https://block.registry.platforma.bio/releases", + id: { organization: "milaboratory", name: "enter-numbers", version: "1.1.1" }, +}; + +test("a rename changes the label and leaves the stored document byte-identical", async () => { + await withMl(async (ml) => { + const document = documentOf(entry("a"), entry("b")); + const stored = await storeTemplate(ml, "First name", { + schemaVersion: 1, + document, + sourceProjectLabel: "Source project", + }); + const dataBefore = await rawTemplateData(ml, stored.rid); + + await ml.renameTemplate(stored.id, "Second name"); + + // The label is the only mutable part: the document rides in the immutable `data` blob, + // which has no setter — improving a template means saving a new one. + expect(await rawTemplateData(ml, stored.rid)).toStrictEqual(dataBefore); + expect((await ml.getTemplateData(stored.id)).document).toStrictEqual(document); + + const list = await awaitTemplateList(ml, (l) => l.some((t) => t.label === "Second name")); + expect(list).toHaveLength(1); + expect(list[0]).toMatchObject({ + id: stored.id, + label: "Second name", + blockCount: 2, + sourceProjectLabel: "Source project", + }); + }); +}); + +test("an entry nothing can resolve creates no project, and every entry is reported", async () => { + await withMl(async (ml) => { + const stored = await storeTemplate(ml, "Nothing implements these", { + schemaVersion: 1, + document: documentOf(entry("a"), entry("b")), + }); + + const outcome = await ml.createProjectFromTemplate( + stored.id, + "From a template", + resolvesNothing(), + ); + + if (outcome.ok) throw new Error("a template nothing can resolve must not apply"); + // Resolution runs before the project exists, which is what makes this a statement about + // the template rather than about a half-built project. + expect(outcome.problems.map((p) => p.entryId)).toStrictEqual(["a", "b"]); + expect(await ml.projectList.awaitStableValue()).toStrictEqual([]); + }); +}); + +test("a template holding a block from a folder on this machine is refused rather than shared", async () => { + await withMl(async (ml) => { + const stored = await storeTemplate(ml, "Built here", { + schemaVersion: 1, + document: documentOf(entry("a"), entry("b", LOCAL_FOLDER)), + }); + + // Asked of a template that is merely being displayed, so the refusal can be stated on the + // template itself instead of only once the user has tried to send it. + expect((await ml.checkTemplateShareable(stored.id)).map((p) => p.entryId)).toStrictEqual(["b"]); + + const outcome = await ml.shareTemplate(stored.id, { + recipients: ["colleague"], + title: "Built here", + }); + + if (outcome.ok) throw new Error("a template with a file: entry must not be shareable"); + expect(outcome.problems.map((p) => p.entryId)).toStrictEqual(["b"]); + // Refused before anything was written: no envelope, so nothing to revoke. + expect((await ml.outgoingShares.getValue()) ?? []).toStrictEqual([]); + }); +}); + +test("a project holding a block that cannot be written out produces no template", async () => { + await withMl(async (ml) => { + const projectId = await ml.createProject({ label: "Two legacy blocks" }); + await ml.openProject(projectId); + const project = ml.getOpenedProject(projectId); + + const first = await project.addBlock("Block 1", KindlessBlock); + const second = await project.addBlock("Block 2", KindlessBlock); + + const outcome = await ml.saveProjectAsTemplate(projectId); + + if (outcome.ok) throw new Error("a project with an unexportable block must store no template"); + // Every offending block at once, not the first one: fixing an unexportable project takes + // one pass, not one pass per block. + expect(outcome.problems.map((p) => p.blockId).sort()).toStrictEqual([first, second].sort()); + expect(await ml.templateList.awaitStableValue()).toStrictEqual([]); + }); +}); + +test("an accepted template share lands on the acceptor's shelf and builds nothing", async () => { + await withMl(async (ml) => { + const stored = await storeTemplate(ml, "A pipeline", { + schemaVersion: 1, + document: documentOf(entry("a")), + sourceProjectLabel: "Source project", + }); + + const shared = await ml.shareTemplate(stored.id, { everyone: true, title: "A pipeline" }); + if (!shared.ok) throw new Error(`share refused: ${JSON.stringify(shared.problems)}`); + + const outcome = await ml.acceptShare([shared.shareId]); + + expect(outcome.failed).toStrictEqual([]); + expect(outcome.acceptedTemplates).toHaveLength(1); + // Nothing is built until the recipient applies it — which is what makes an all-or-nothing + // apply survivable for them: there is always something left to retry from. + expect(outcome.accepted).toStrictEqual([]); + expect(await ml.projectList.awaitStableValue()).toStrictEqual([]); + + const list = await awaitTemplateList(ml, (l) => l.length === 2); + const accepted = list.find((t) => t.id === outcome.acceptedTemplates[0])!; + expect(accepted.label).toBe("A pipeline"); + // Who sent it, kept as the accepted template's provenance; the donor's own source project + // is not part of the payload and does not travel. + expect(accepted.sender).toBe(ml.currentUserLogin ?? ""); + expect(accepted.sourceProjectLabel).toBeUndefined(); + }); +}); + +test("a changed template share keeps its id, and whoever already responded is not re-prompted", async () => { + await withMl(async (ml) => { + const first = await storeTemplate(ml, "First", { + schemaVersion: 1, + document: documentOf(entry("a")), + }); + const second = await storeTemplate(ml, "Second", { + schemaVersion: 1, + document: documentOf(entry("a"), entry("b")), + }); + + const shared = await ml.shareTemplate(first.id, { everyone: true, title: "First" }); + if (!shared.ok) throw new Error(`share refused: ${JSON.stringify(shared.problems)}`); + + // Someone responds to the share, which is what the replace below must not undo. + const accept = await ml.acceptShare([shared.shareId]); + expect(accept.acceptedTemplates).toHaveLength(1); + + // A stored template never changes, so an improved one is a different template — the + // replace names it rather than re-reading the one the share started from. + await ml.changeShare(shared.shareId, { templateId: second.id, title: "Second" }); + + const outgoing = (await ml.outgoingShares.getValue()) ?? []; + expect(outgoing.map((s) => s.shareId)).toStrictEqual([shared.shareId]); + expect(outgoing[0]).toMatchObject({ + payloadKind: "template", + title: "Second", + template: { label: "Second", blockCount: 2 }, + // A template share is granted read-only, so no recipient can ever write a reply on the + // envelope — a view has to say that rather than render an empty list as "nobody yet". + responsesAvailable: false, + }); + expect(outgoing[0].projects).toStrictEqual([]); + + // The decision the accept recorded is keyed on the shareId, which the change preserved, so + // the replaced share is not offered again. A replace that minted a new id would show up + // here as a fresh offer. + const pending = await settledPendingShareIds(ml); + expect(pending).not.toContain(shared.shareId); + }); +}); + +// +// Internals +// + +const entry = (id: string, location?: BlockPackLocationReference): ProjectTemplateV1Entry => ({ + id, + kind: KIND, + params: {}, + ...(location !== undefined ? { location } : {}), +}); + +const documentOf = (...blocks: ProjectTemplateV1Entry[]): ProjectTemplateV1 => ({ + schema: PROJECT_TEMPLATE_SCHEMA_V1, + blocks, +}); + +/** A stored template, plus the resource id needed to read its raw `data` blob back. */ +type StoredTemplate = { id: TemplateId; rid: SignedResourceId }; + +/** + * Stores one template through the mutator, on the same templates list the middle layer reads. + * + * The middle layer has no way to store an arbitrary document — it only saves a project — so a + * test that needs a specific document writes it here, exactly as `saveProjectAsTemplate` and + * the accept path do. + */ +async function storeTemplate( + ml: MiddleLayer, + label: string, + data: StoredTemplateData, +): Promise { + let tpl: ResourceRef; + await ml.pl.withWriteTx("TestStoreTemplate", async (tx) => { + const listRid = await ensureTemplateListRid(tx); + tpl = createTemplate(tx, listRid, label, data); + await tx.commit(); + }); + const rid = await tpl!.globalId; + return { id: resourceIdToString(rid) as TemplateId, rid }; +} + +/** The template's immutable `data` blob, as bytes — the form a rename must not touch. */ +async function rawTemplateData(ml: MiddleLayer, rid: SignedResourceId): Promise { + return await ml.pl.withReadTx("TestReadTemplateData", async (tx) => { + const rd = await tx.getResourceData(rid, false); + if (rd.data === undefined) throw new Error("template carries no document"); + return Buffer.from(rd.data); + }); +} + +/** + * The template list once it satisfies `predicate`. + * + * A template written by the mutator lands in the list through the tree's own poll, with no + * refresh to await, so a test that stored one waits for it rather than reading once. + */ +async function awaitTemplateList( + ml: MiddleLayer, + predicate: (list: TemplateListEntry[]) => boolean, + timeoutMs = 15_000, +): Promise { + const abortSignal = AbortSignal.timeout(timeoutMs); + while (true) { + const list = await ml.templateList.getValue(); + if (list !== undefined && predicate(list)) return list; + await ml.templateList.awaitChange(abortSignal); + } +} + +/** + * The shareIds currently offered to this user, read after discovery has had a poll to run. + * + * Discovery of a just-granted envelope is a poll behind, so reading the view once would say + * "not offered" about a share that simply had not been seen yet. + */ +async function settledPendingShareIds(ml: MiddleLayer): Promise { + await tp.setTimeout(2_000); + return ((await ml.pendingShares.getValue()) ?? []).map((s) => s.shareId); +} + +/** + * A provider that finds nothing, for a document whose entries resolve through their kind. + * + * `no-implementation` rather than a thrown error: an entry whose kind exists but which + * nothing implements is the reachable case, and it is a problem about that entry rather + * than a failure of the apply. + */ +function resolvesNothing(): BlockPackProvider { + return { + byKind: () => Promise.resolve({ ok: false, reason: "no-implementation" }), + byExactVersion: () => Promise.resolve({ ok: false, reason: "no-such-block-version" }), + byLocation: () => Promise.resolve({ ok: false, reason: "not-found" }), + }; +} diff --git a/lib/node/pl-middle-layer/src/model/index.ts b/lib/node/pl-middle-layer/src/model/index.ts index c3b65b0ae6..d0e13e9404 100644 --- a/lib/node/pl-middle-layer/src/model/index.ts +++ b/lib/node/pl-middle-layer/src/model/index.ts @@ -35,3 +35,17 @@ export { type TemplateApplyProblem, type TemplateApplyReport, } from "./template_apply"; + +// The template export path. A caller renders a stored template back to a file, and reports +// every block that stood in the way of storing one, so the outcome type and the stringifier +// are as public as the `MiddleLayer.saveProjectAsTemplate` that produces them. +export { + stringifyProjectTemplateV1, + locationOf, + type ProjectTemplateExportOutcome, +} from "./template_serializer"; +export type { TemplateExportProblem } from "./template_export"; + +// The template share path. Whether a template may be shared at all is a question a UI asks about +// a template it is merely displaying, so the check and its problem type are public. +export { unshareableTemplateEntries, type TemplateShareProblem } from "./template_share"; diff --git a/lib/node/pl-middle-layer/src/model/sharing_model.test.ts b/lib/node/pl-middle-layer/src/model/sharing_model.test.ts index 8de9f224c0..9ca289f347 100644 --- a/lib/node/pl-middle-layer/src/model/sharing_model.test.ts +++ b/lib/node/pl-middle-layer/src/model/sharing_model.test.ts @@ -1,6 +1,13 @@ import { test, expect } from "vitest"; import { Role } from "@milaboratories/pl-client"; -import { canGrantToEveryone, canImpersonate } from "./sharing_model"; +import { + canGrantToEveryone, + canImpersonate, + decodeEnvelopeData, + EnvelopeSchemaVersionCurrent, + envelopeProjectMap, + normalizeEnvelopeData, +} from "./sharing_model"; // canImpersonate is the admin gate for "open another user's root". It must be strictly // stricter than canGrantToEveryone: a regular USER may share their own projects but must @@ -20,3 +27,110 @@ test("canGrantToEveryone and canImpersonate does not include USER", () => { expect(canGrantToEveryone(Role.USER)).toBe(false); expect(canImpersonate(Role.USER)).toBe(false); }); + +// +// Envelope decode — the recognise-or-hide gate every reader of a share passes through. +// +// Pure by construction: the gate is a function of the blob, and the three sites that +// discover envelopes (the pending view, the live-envelope view the accept flow reads, +// and the donor's own outbox) all skip an envelope this returns `undefined` for. So an +// envelope that does not decode cannot be offered, accepted, or listed. + +/** A v1 envelope, exactly as one written before the payload discriminant existed: the + * project map sits at the top level and there is no `payload` field. */ +const v1Envelope = { + schemaVersion: 1, + shareId: "5d1a6f6c-2b6e-4a3f-9c2d-8f0e1b7a4c55", + sharedAt: 1_700_000_000_000, + expiresAt: null, + mode: "copy", + sender: "donor", + title: "Two projects", + projects: { + "9c7e4d10-2b83-4f6a-91d5-7e0c3a8b5f42": { + label: "Project 1", + source: "42", + updatedAt: 1_700_000_000_000, + }, + }, +}; + +const blob = (envelope: unknown) => Buffer.from(JSON.stringify(envelope), "utf-8"); + +test("a v1 envelope still decodes, and reads as a share of projects", () => { + const data = decodeEnvelopeData(blob(v1Envelope)); + + // Upcast on read: past the decode nothing knows two shapes ever existed. + expect(data?.schemaVersion).toBe(EnvelopeSchemaVersionCurrent); + expect(data?.payload).toStrictEqual({ kind: "projects", projects: v1Envelope.projects }); + expect(envelopeProjectMap(data!)).toStrictEqual(v1Envelope.projects); + + // Everything a view renders survives the upcast unchanged. + expect(data).toMatchObject({ + shareId: v1Envelope.shareId, + sharedAt: v1Envelope.sharedAt, + expiresAt: null, + mode: "copy", + sender: "donor", + title: "Two projects", + }); +}); + +test("a current envelope carrying a template decodes as one", () => { + const data = decodeEnvelopeData( + blob({ + ...v1Envelope, + schemaVersion: EnvelopeSchemaVersionCurrent, + projects: undefined, + payload: { + kind: "template", + document: { schema: "template-v1", blocks: [] }, + label: "A pipeline", + from: "donor", + }, + }), + ); + + expect(data?.payload).toStrictEqual({ + kind: "template", + document: { schema: "template-v1", blocks: [] }, + label: "A pipeline", + from: "donor", + }); + // A template payload carries no project snapshot, so a project-shaped reader sees nothing + // rather than something it would then try to copy. + expect(envelopeProjectMap(data!)).toStrictEqual({}); +}); + +test("an envelope whose payload kind this build does not know does not decode at all", () => { + // The whole point of the discriminant: a share this build cannot act on is hidden rather + // than offered, and the decode is where that is decided — once, for every reader. + expect( + decodeEnvelopeData( + blob({ + ...v1Envelope, + schemaVersion: EnvelopeSchemaVersionCurrent, + projects: undefined, + payload: { kind: "workspace", whatever: true }, + }), + ), + ).toBeUndefined(); +}); + +test("an envelope from a newer schema does not decode either", () => { + expect( + decodeEnvelopeData(blob({ ...v1Envelope, schemaVersion: EnvelopeSchemaVersionCurrent + 1 })), + ).toBeUndefined(); +}); + +test("a v1 envelope with no project map at all does not decode", () => { + // There is no payload to reconstruct: a v1 blob without `projects` describes nothing, + // which is not the same as describing an empty pack. + expect(decodeEnvelopeData(blob({ ...v1Envelope, projects: undefined }))).toBeUndefined(); +}); + +test("anything that is not an envelope object does not decode", () => { + expect(normalizeEnvelopeData(null)).toBeUndefined(); + expect(normalizeEnvelopeData("an envelope")).toBeUndefined(); + expect(normalizeEnvelopeData(42)).toBeUndefined(); +}); diff --git a/lib/node/pl-middle-layer/src/model/sharing_model.ts b/lib/node/pl-middle-layer/src/model/sharing_model.ts index 20f85d6408..e31a365933 100644 --- a/lib/node/pl-middle-layer/src/model/sharing_model.ts +++ b/lib/node/pl-middle-layer/src/model/sharing_model.ts @@ -1,6 +1,6 @@ import type { ResourceType, Role } from "@milaboratories/pl-client"; import { Role as RoleEnum } from "@milaboratories/pl-client"; -import type { Branded, ProjectId } from "@milaboratories/pl-model-common"; +import type { Branded, ProjectId, ProjectTemplateV1 } from "@milaboratories/pl-model-common"; import { randomUUID } from "node:crypto"; /** @@ -84,23 +84,60 @@ export function canImpersonate(role: Role | null): boolean { } } -/** One project's snapshot inside an envelope, keyed by {@link ProjectFieldUuid} in {@link EnvelopeData.projects}. */ +/** One project's snapshot inside an envelope, keyed by {@link ProjectFieldUuid} in a + * `projects` {@link EnvelopePayload}. */ export interface EnvelopeProject { label: string; // carried so the pending-share UI renders without traversing into the project source: ProjectId; // donor's source projectId; supersedes a prior share and matches the snapshot to its live source on change updatedAt: number; // ms epoch of the last (re)snapshot } -/** Immutable `data` on a SharedEnvelope, set at createEphemeral, never mutated. */ +/** + * What a share carries. The discriminant is what a reader checks before anything else: a + * client that does not know a kind hides the share instead of offering something it cannot + * act on. + * + * `projects` snapshots ride as `project/{uuid}` fields on the envelope and this map only + * describes them; a `template` payload has no fields at all — the document is right here. + */ +export type EnvelopePayload = + | { kind: "projects"; projects: Record } + | { + kind: "template"; + document: ProjectTemplateV1; + /** Label to give the template on the recipient's own shelf. */ + label: string; + /** Donor login, kept on the accepted template as its provenance. */ + from: string; + }; + +export type EnvelopePayloadKind = EnvelopePayload["kind"]; + +/** Version written into every new envelope. Bumped from 1 when the payload became discriminated. */ +export const EnvelopeSchemaVersionCurrent = 2; + +/** + * Immutable `data` on a SharedEnvelope, set at createEphemeral, never mutated. + * + * Always the current version in memory: a v1 envelope (project map at the top level, no + * `payload` field) is upcast on read by {@link normalizeEnvelopeData}, so no reader past the + * decode has to know that two shapes ever existed. + */ export interface EnvelopeData { - schemaVersion: 1; + schemaVersion: typeof EnvelopeSchemaVersionCurrent; shareId: ShareId; // donor-generated UUID; logical share identity, stable across changes sharedAt: number; // ms epoch; this instance's creation time — distinguishes instances of one shareId expiresAt: number | null; // ms epoch; sharedAt + ttl (default 14 days) for a targeted share; null for share-with-everybody (never expires) mode: EnvelopeMode; // what the acceptor's app should do with the contents sender: string; // donor login (informational; backend granted_by is authoritative) title: string; // display name shown to recipients; defaults to the first project's name - projects: Record; // contained projects, keyed by project field uuid + payload: EnvelopePayload; // what the share carries +} + +/** The project map of a projects-payload envelope, or `{}` for any other payload — the one + * place a project-shaped reader turns a payload into the map it expects. */ +export function envelopeProjectMap(data: EnvelopeData): Record { + return data.payload.kind === "projects" ? data.payload.projects : {}; } /** Dynamic field on SharingState, one per handled share, keyed by shareId. */ @@ -110,7 +147,7 @@ export interface SharingDecision { decision: "accepted" | "rejected"; timestamp: number; // ms epoch — when the acceptor acted envelopeSharedAt: number; // the acted-on envelope instance's sharedAt — pins which instance was handled (paired with the shareId key; the resource id is never stored) - acceptedProjects: string[]; // ids of the projects created in the acceptor's list ([] for a rejected share) + acceptedProjects: string[]; // ids of the projects created in the acceptor's list ([] for a rejected share, and for a template share, which creates none) } /** Dynamic field on SharedEnvelope, one per recipient who accepted or rejected, keyed @@ -134,10 +171,45 @@ export interface EnvelopeAcceptance { * Single owner of the raw-data → {@link EnvelopeData} decode. The envelope's immutable `data` * blob is UTF-8 JSON set once at createEphemeral; every site that reads it from a raw resource * `data` byte buffer (the basic-resource read path) goes through here. The reactive tree-node - * path uses `node.getDataAsJson()`, which decodes the same JSON. + * path decodes the same JSON with `getDataAsJson` and normalizes it with + * {@link normalizeEnvelopeData} — both paths must, so neither sees the raw v1 shape. + * + * `undefined` for an envelope this build cannot act on; see {@link normalizeEnvelopeData}. */ -export function decodeEnvelopeData(data: Uint8Array): EnvelopeData { - return JSON.parse(Buffer.from(data).toString("utf-8")) as EnvelopeData; +export function decodeEnvelopeData(data: Uint8Array): EnvelopeData | undefined { + return normalizeEnvelopeData(JSON.parse(Buffer.from(data).toString("utf-8"))); +} + +/** + * Brings a decoded envelope blob to the current shape, or reports that this build cannot act + * on it by returning `undefined` — an unknown `schemaVersion` or an unknown payload kind. A + * caller hides such a share rather than offering the recipient something it cannot handle. + * + * A v1 envelope carried its project map at the top level and had no `payload` field; it reads + * here as a `projects` payload, so envelopes written before the discriminant existed keep + * working unchanged. + */ +export function normalizeEnvelopeData(raw: unknown): EnvelopeData | undefined { + if (typeof raw !== "object" || raw === null) return undefined; + const e = raw as RawEnvelopeData; + if (e.schemaVersion !== 1 && e.schemaVersion !== EnvelopeSchemaVersionCurrent) return undefined; + + const payload = + e.payload ?? + (e.projects !== undefined ? ({ kind: "projects", projects: e.projects } as const) : undefined); + if (payload === undefined) return undefined; + if (!KnownPayloadKinds.has(payload.kind)) return undefined; + + return { + schemaVersion: EnvelopeSchemaVersionCurrent, + shareId: e.shareId, + sharedAt: e.sharedAt, + expiresAt: e.expiresAt, + mode: e.mode, + sender: e.sender, + title: e.title, + payload, + }; } /** @@ -166,3 +238,38 @@ export type ShareProjectsOptions = title: string; mode: EnvelopeMode; }; + +/** + * Options for {@link MiddleLayer.shareTemplate}. + * + * Recipients XOR everyone, exactly as {@link ShareProjectsOptions}, minus the mode: a template + * share is always granted read-only, because the recipient copies no resource out of the + * envelope — the document is in the envelope's own data. + */ +export type ShareTemplateOptions = + | { + recipients: string[]; // recipient logins + title: string; // display name shown to recipients; defaults to the template's label + } + | { + everyone: true; // share with all users on the server + title: string; + }; + +// +// Internals +// + +/** Every payload kind this build can act on; anything else is hidden rather than offered. */ +const KnownPayloadKinds = new Set(["projects", "template"]); + +/** + * The envelope blob as it comes off the wire, before {@link normalizeEnvelopeData} decides + * whether this build can act on it: the version is any number, the payload may be missing, + * and `projects` is the v1 top-level project map. + */ +type RawEnvelopeData = Omit & { + schemaVersion: number; + payload?: EnvelopePayload; + projects?: Record; +}; diff --git a/lib/node/pl-middle-layer/src/model/template_share.test.ts b/lib/node/pl-middle-layer/src/model/template_share.test.ts new file mode 100644 index 0000000000..a449250825 --- /dev/null +++ b/lib/node/pl-middle-layer/src/model/template_share.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, test } from "vitest"; +import type { + BlockKindSelectorReference, + BlockPackLocationReference, + ProjectTemplateV1, + ProjectTemplateV1Entry, +} from "@milaboratories/pl-model-common"; +import { PROJECT_TEMPLATE_SCHEMA_V1 } from "@milaboratories/pl-model-common"; +import { unshareableTemplateEntries } from "./template_share"; + +/** + * Whether a stored template may travel to another machine. + * + * A function of the document alone, which is why it is asked both when a share is attempted + * and when a template is merely displayed — the refusal has to be visible on the template + * itself, not only after the user tries. + */ + +const KIND = "@platforma-open/milaboratories.demo.kind@^1.0.0" as BlockKindSelectorReference; + +const entry = (id: string, location?: string): ProjectTemplateV1Entry => ({ + id, + kind: KIND, + params: {}, + ...(location !== undefined ? { location: location as BlockPackLocationReference } : {}), +}); + +const documentOf = (...blocks: ProjectTemplateV1Entry[]): ProjectTemplateV1 => ({ + schema: PROJECT_TEMPLATE_SCHEMA_V1, + blocks, +}); + +describe("unshareableTemplateEntries", () => { + test("a template whose entries name no place at all can be shared", () => { + // The common case: every entry resolves through its kind, which means the same thing + // on the recipient's machine as it does here. + expect(unshareableTemplateEntries(documentOf(entry("a"), entry("b")))).toStrictEqual([]); + }); + + test("an entry installed from a folder on this machine refuses the share, and says which folder", () => { + const problems = unshareableTemplateEntries( + documentOf(entry("a"), entry("b", "file:///Users/dev/blocks/demo/block")), + ); + + expect(problems.map((p) => p.entryId)).toStrictEqual(["b"]); + expect(problems[0].error).toContain("file:///Users/dev/blocks/demo/block"); + }); + + test("every offending entry is reported, not only the first", () => { + // A UI names each block once instead of sending its user round the loop per entry. + const problems = unshareableTemplateEntries( + documentOf( + entry("a", "file:///blocks/a"), + entry("b"), + entry("c", "FILE:///blocks/c"), // the scheme is case-insensitive + ), + ); + + expect(problems.map((p) => p.entryId)).toStrictEqual(["a", "c"]); + }); + + test("a location whose scheme cannot be read is refused too", () => { + // Nothing can resolve it anywhere, here included — so the reason differs from the + // `file:` one, and the message says so rather than blaming the recipient's machine. + const problems = unshareableTemplateEntries(documentOf(entry("a", "/blocks/a"))); + + expect(problems.map((p) => p.entryId)).toStrictEqual(["a"]); + expect(problems[0].error).toContain("nothing can resolve"); + }); + + test("a location naming a place both machines can reach is not refused", () => { + // The rule is about a place that means something different elsewhere, not about + // locations as such. + expect( + unshareableTemplateEntries(documentOf(entry("a", "https://blocks.example.org/demo"))), + ).toStrictEqual([]); + }); +}); diff --git a/lib/node/pl-middle-layer/src/model/template_share.ts b/lib/node/pl-middle-layer/src/model/template_share.ts new file mode 100644 index 0000000000..ddab7b0fca --- /dev/null +++ b/lib/node/pl-middle-layer/src/model/template_share.ts @@ -0,0 +1,52 @@ +import type { ProjectTemplateV1 } from "@milaboratories/pl-model-common"; +import { parseBlockPackLocation } from "@milaboratories/pl-model-common"; + +/** One template entry standing in the way of sharing the template, and why. */ +export type TemplateShareProblem = { + /** The template-local id of the entry the problem belongs to; on an exported template it is + * the block's project-local uuid. */ + readonly entryId: string; + readonly error: string; +}; + +/** + * Every entry of a template that cannot travel to another machine, or an empty list for a + * template that can be shared. + * + * An entry's `location` names a place rather than a name, and a `file:` place is a folder on + * the author's own disk: a recipient resolving it finds nothing, or worse finds something + * else. Such a template stays perfectly usable where it was made, so it is stored and applied + * as normal — only sharing it is refused. + * + * A location whose scheme cannot be read is refused for the same reason: nothing can resolve + * it anywhere, here included. + * + * Every offending entry is reported, not only the first, so a UI can name each block instead + * of sending its user round the loop once per entry. + */ +export function unshareableTemplateEntries( + document: ProjectTemplateV1, +): readonly TemplateShareProblem[] { + const problems: TemplateShareProblem[] = []; + for (const entry of document.blocks) { + if (entry.location === undefined) continue; + let scheme: string; + try { + scheme = parseBlockPackLocation(entry.location).scheme; + } catch (e) { + problems.push({ + entryId: entry.id, + error: `Block is installed from a location nothing can resolve: ${e instanceof Error ? e.message : String(e)}`, + }); + continue; + } + if (scheme === "file") + problems.push({ + entryId: entry.id, + error: + `Block is installed from ${entry.location}, a folder on this machine — it resolves ` + + "to nothing on the recipient's, so this template cannot be shared", + }); + } + return problems; +} diff --git a/lib/node/pl-middle-layer/src/mutator/sharing.ts b/lib/node/pl-middle-layer/src/mutator/sharing.ts index 8cc48f1b4d..81d8ce50f2 100644 --- a/lib/node/pl-middle-layer/src/mutator/sharing.ts +++ b/lib/node/pl-middle-layer/src/mutator/sharing.ts @@ -2,7 +2,7 @@ import type { PlTransaction, ResourceRef, SignedResourceId } from "@milaboratori import { field, isNotNullSignedResourceId, resourceIdToString } from "@milaboratories/pl-client"; import { randomUUID } from "node:crypto"; import type { ProjectMeta } from "@milaboratories/pl-model-middle-layer"; -import type { ProjectId } from "@milaboratories/pl-model-common"; +import type { ProjectId, ProjectTemplateV1 } from "@milaboratories/pl-model-common"; import { ProjectMetaKey } from "../model/project_model"; import { duplicateProject } from "./project"; import type { @@ -15,6 +15,7 @@ import type { SharingDecision, } from "../model/sharing_model"; import { + EnvelopeSchemaVersionCurrent, SharedEnvelopeResourceType, acceptanceField, decisionField, @@ -101,14 +102,14 @@ export async function buildShareEnvelope( } const data: EnvelopeData = { - schemaVersion: 1, + schemaVersion: EnvelopeSchemaVersionCurrent, shareId, sharedAt, expiresAt: params.expiresAt, mode: params.mode, sender: params.sender, title: params.title, - projects, + payload: { kind: "projects", projects }, }; // Immutable data set once at creation, never altered. @@ -127,6 +128,57 @@ export async function buildShareEnvelope( return { envelope, data }; } +/** + * Builds one {@link SharedEnvelopeResourceType} carrying a template document on the donor side, + * and attaches it under `{shareId}` on the donor's outbox. The caller issues the grant — always + * read-only — and commits, keeping create + grant atomic. + * + * There is nothing to snapshot and no input field to seal: the document is the whole payload and + * rides in the envelope's immutable `data`, which is also why the recipient needs no write access + * (it copies no resource out of the envelope). + * + * @returns the new envelope resource and the generated `EnvelopeData`. + */ +export function buildTemplateShareEnvelope( + tx: PlTransaction, + outboxRid: SignedResourceId, + template: { document: ProjectTemplateV1; label: string }, + params: { + sender: string; + title: string; + /** ms epoch; sharedAt + ttl for a targeted share, null for share-with-everybody. */ + expiresAt: number | null; + /** Existing shareId for a change; a fresh one is minted when omitted. */ + shareId?: ShareId; + sharedAt?: number; + }, +): { envelope: ResourceRef; data: EnvelopeData } { + const data: EnvelopeData = { + schemaVersion: EnvelopeSchemaVersionCurrent, + shareId: params.shareId ?? newShareId(), + sharedAt: params.sharedAt ?? Date.now(), + expiresAt: params.expiresAt, + mode: "read-only", + sender: params.sender, + title: params.title, + payload: { + kind: "template", + document: template.document, + label: template.label, + from: params.sender, + }, + }; + + // Immutable data set once at creation, never altered. + const envelope = tx.createEphemeral(SharedEnvelopeResourceType, JSON.stringify(data)); + + // Attach to the outbox under {shareId} in the same transaction so the held-resource rule + // keeps the ephemeral envelope alive. + tx.createField(field(outboxRid, data.shareId), "Dynamic", envelope); + + return { envelope, data }; +} + /** * Records a response onto the envelope as a dynamic `acceptance/{login}` field: the acceptor * writing their own decision (their writable grant permits it), or the donor transferring an diff --git a/lib/node/pl-middle-layer/src/mutator/template.ts b/lib/node/pl-middle-layer/src/mutator/template.ts new file mode 100644 index 0000000000..a69fd679c9 --- /dev/null +++ b/lib/node/pl-middle-layer/src/mutator/template.ts @@ -0,0 +1,75 @@ +import type { PlTransaction, ResourceRef, SignedResourceId } from "@milaboratories/pl-client"; +import { field, isNullSignedResourceId, resourceIdToString } from "@milaboratories/pl-client"; +import { randomUUID } from "node:crypto"; +import type { StoredTemplateData, TemplateId } from "../middle_layer/template_list"; +import { + TemplateCreatedTimestamp, + TemplateLabelKey, + TemplateResourceType, +} from "../middle_layer/template_list"; + +/** + * Creates one `UserTemplate` inside the given write transaction and attaches it to the + * templates list under a freshly minted uuid field. + * + * Create and attach are the same transaction on purpose: an ephemeral resource nothing + * holds is collectable, so the list field is what keeps the template alive. + * + * The document rides in the immutable `data` blob, set once here and never altered; only + * the label and the creation timestamp go to KV, and only the label is ever written again. + * + * @returns the new template resource; the caller reads its `globalId` after the commit. + */ +export function createTemplate( + tx: PlTransaction, + listRid: SignedResourceId, + label: string, + data: StoredTemplateData, +): ResourceRef { + const tpl = tx.createEphemeral(TemplateResourceType, JSON.stringify(data)); + tx.lock(tpl); + tx.setKValue(tpl, TemplateLabelKey, JSON.stringify(label)); + tx.setKValue(tpl, TemplateCreatedTimestamp, String(Date.now())); + tx.createField(field(listRid, randomUUID()), "Dynamic", tpl); + return tpl; +} + +/** Renames a stored template. Touches the label KV entry and nothing else, so the stored + * document stays byte-identical. */ +export function renameTemplate(tx: PlTransaction, rid: SignedResourceId, label: string): void { + tx.setKValue(rid, TemplateLabelKey, JSON.stringify(label)); +} + +/** + * Detaches a template from the templates list, which is what destroys it — the list field is + * the only thing holding the ephemeral resource. + * + * The field name is a uuid unrelated to the template id, so the field carrying the template + * is found by value, the same way a project is removed from the project list. + */ +export async function deleteTemplate( + tx: PlTransaction, + listRid: SignedResourceId, + id: TemplateId, +): Promise { + const fieldName = await findTemplateField(tx, listRid, id); + if (fieldName === undefined) throw new Error(`Template ${id} not found in template list.`); + tx.removeField(field(listRid, fieldName)); +} + +// +// Internals +// + +async function findTemplateField( + tx: PlTransaction, + listRid: SignedResourceId, + id: TemplateId, +): Promise { + const data = await tx.getResourceData(listRid, true); + for (const f of data.fields) { + if (isNullSignedResourceId(f.value)) continue; + if (resourceIdToString(f.value) === (id as string)) return f.name; + } + return undefined; +} diff --git a/lib/node/pl-middle-layer/src/test/with_ml.ts b/lib/node/pl-middle-layer/src/test/with_ml.ts new file mode 100644 index 0000000000..dbb5485ee5 --- /dev/null +++ b/lib/node/pl-middle-layer/src/test/with_ml.ts @@ -0,0 +1,38 @@ +import path from "path"; +import { randomUUID } from "node:crypto"; +import type { PlClient } from "@milaboratories/pl-client"; +import { TestHelpers } from "@milaboratories/pl-client"; +import { MiddleLayer } from "../middle_layer/middle_layer"; + +/** + * A live {@link MiddleLayer} over a temporary root, closed again when the body returns. + * + * Needs a backend: the client comes from `PL_ADDRESS` (plus `PL_TEST_USER` / + * `PL_TEST_PASSWORD` where the server requires auth), so a test using this fails at + * connect time when none is configured. + */ +export async function withMl( + cb: (ml: MiddleLayer, workFolder: string) => Promise, +): Promise { + const workFolder = path.resolve(`work/${randomUUID()}`); + + await TestHelpers.withTempRoot(async (pl: PlClient) => { + const ml = await MiddleLayer.init(pl, workFolder, { + defaultTreeOptions: { pollingInterval: 250, stopPollingDelay: 500 }, + devBlockUpdateRecheckInterval: 300, + localSecret: MiddleLayer.generateLocalSecret(), + localProjections: [], + openFileDialogCallback: () => { + throw new Error("Not implemented."); + }, + }); + ml.addRuntimeCapability("requiresUIAPIVersion", 1); + ml.addRuntimeCapability("requiresUIAPIVersion", 2); + ml.addRuntimeCapability("requiresUIAPIVersion", 3); + try { + await cb(ml, workFolder); + } finally { + await ml.close(); + } + }); +} diff --git a/tests/drivers-ml-blocks-integration/src/template-round-trip.test.ts b/tests/drivers-ml-blocks-integration/src/template-round-trip.test.ts index b8bd547137..4d8d9e4e0b 100644 --- a/tests/drivers-ml-blocks-integration/src/template-round-trip.test.ts +++ b/tests/drivers-ml-blocks-integration/src/template-round-trip.test.ts @@ -209,6 +209,94 @@ test("v3: a reference to a deleted block survives the export unexamined", async }); }); +/** + * The stored template is the same round trip, with the file replaced by an object on the + * server: save the project, then apply what was saved. + * + * What this adds over the trip above is the storage in the middle — the document goes into a + * template's immutable `data` blob and comes back out of it — and the two calls a user + * actually reaches: `saveProjectAsTemplate` and `createProjectFromTemplate`. The apply side + * differs in one way that matters: resolution runs before the project exists, so a template + * nothing can build leaves no empty project behind. + */ +test("v3: a project saved as a template applies back as an equivalent project", async ({ + expect, +}) => { + await withMl(async (ml) => { + const sourceId = await ml.createProject({ label: "Source" }); + await ml.openProject(sourceId); + const source = ml.getOpenedProject(sourceId); + + const numbersId = await source.addBlock("Numbers", enterNumbersSpec); + const sumId = await source.addBlock("Sum", sumNumbersSpec); + + // Everything here is inside the params contract, so the applied project must hold it + // verbatim. What falls outside the contract is lost by design, and is pinned by the + // trip above rather than a second time here. + await source.mutateBlockStorage(numbersId, { + operation: "update-block-data", + value: { numbers: [1, 2, 3], labels: [], description: "" }, + }); + await source.mutateBlockStorage(sumId, { + operation: "update-block-data", + value: { sources: [createPlRef(numbersId, "numbers")] }, + }); + await settled(source, sumId); + + // --- Save --------------------------------------------------------------- + + const saved = await ml.saveProjectAsTemplate(sourceId); + if (!saved.ok) throw new Error(`save failed: ${JSON.stringify(saved.problems)}`); + + const stored = await ml.getTemplateData(saved.templateId); + // Entry ids are the source project's own block ids, in structure order — which is the + // instantiation order the apply below has to reproduce. + expect(stored.document.blocks.map((entry) => entry.id)).toStrictEqual([numbersId, sumId]); + expect(stored.document.blocks[0].params).toStrictEqual({ numbers: [1, 2, 3] }); + expect(stored.document.blocks[1].params).toStrictEqual({ + sources: [createPlRef(numbersId, "numbers")], + }); + // Provenance, and the label the template gets by default: the project it was taken from. + expect(stored.sourceProjectLabel).toBe("Source"); + + const listed = (await ml.templateList.awaitStableValue()).find( + (t) => t.id === saved.templateId, + ); + expect(listed).toMatchObject({ label: "Source", blockCount: 2, sourceProjectLabel: "Source" }); + + // --- Apply -------------------------------------------------------------- + + const applied = await ml.createProjectFromTemplate( + saved.templateId, + "Target", + localPacksOnly(), + ); + if (!applied.ok) throw new Error(`apply failed: ${JSON.stringify(applied.problems)}`); + + expect(applied.added.map((entry) => entry.templateLocalId)).toStrictEqual([numbersId, sumId]); + + // 1. The applied project describes the same pipeline, in the same order. + const reExported = await ml.exportProjectAsTemplate(applied.projectId); + if (!reExported.ok) throw new Error(`re-export failed: ${JSON.stringify(reExported.problems)}`); + expect(canonical(reExported.document)).toStrictEqual(canonical(stored.document)); + + // 2. And every block starts with the params its source block held, with the one field + // that must differ: the reference now names the block the apply created. + await ml.openProject(applied.projectId); + const target = ml.getOpenedProject(applied.projectId); + const appliedId = new Map(applied.added.map((entry) => [entry.templateLocalId, entry.blockId])); + + expect(blockData(await settled(target, appliedId.get(numbersId)!))).toStrictEqual({ + numbers: [1, 2, 3], + labels: [], + description: "", + }); + expect(blockData(await settled(target, appliedId.get(sumId)!))).toStrictEqual({ + sources: [createPlRef(appliedId.get(numbersId)!, "numbers")], + }); + }); +}); + /** A block's current data, as the model sees it. */ function blockData(state: Awaited>): unknown { return deriveDataFromStorage(state.blockStorage);