diff --git a/bundle/README.md b/bundle/README.md index d981476..ef6108f 100644 --- a/bundle/README.md +++ b/bundle/README.md @@ -108,6 +108,29 @@ Repeated targets at different locations remain separate diagnostics. target inventories. Authored index fields without a source snapshot have a path and target but no source object. +`Stats.UnresolvedDeleted` and `Stats.UnresolvedOmitted` are the subsets the +wired store could classify (SPEC §2c, `anyblockjson.ClassifyUnresolvedTarget`): +tombstones, and rows the space still holds that this export did not write. +What is in neither is absent — the one class that is a loss. `index.json` +states the same subsets as `unresolved.deleted` and `unresolved.omitted`, +and a space icon whose image object is a tombstone is dropped with a warning +rather than declared, following the document-level icon rule. + +`Stats.UnresolvedTypes` and `Stats.UnresolvedTypeReferences` name the types +the written documents reference by derived id that no written type document +carries (SPEC §2c): a type the source space never held, left by an old +import. `index.json` states the same list as `unresolved.types`. A bundled +key never appears; an uninstalled type travels as a type document carrying +`uninstalled: true`, so it is not one of these either. + +`Inspect` is `Validate` with the whole verdict kept: a `Report` of issues +graded `error`, `warning` or `info`, each with a stable code and the index +field it is about. On the full surface a declared target is admitted — a +deleted one as info, an omitted or absent one as a warning — while an +undeclared dangling target stays an error; `InspectAuthoring` and +`ValidateAuthoring` refuse every dangling target. `Validate` is `Inspect` +refusing exactly on the error-severity issues, with the same wording. + Codec `unresolved_target` warnings put the complete source location in `Path`, for example `/blocks//object_id`, `/blocks//text/marks/0/param`, or `/properties//1`. @@ -119,12 +142,12 @@ or `/properties//1`; the message names the missing target Index references with a known source use the same object prefix and `SourcePath` in the report; otherwise they use `index.json#` followed by the index pointer. -## Known integration issue: icons in 1-to-1 spaces +## Icons by content cid (1-to-1 spaces, participants) -In 1-to-1 spaces, `spaceIcon` can reference a raw file CID with no corresponding -file object in the space's object store. Export preserves that CID in -`index.json` at `/icon/file`, but the bundle's target check expects an exported -object and reports `unresolved_target`. This is a known mismatch between the -stored icon reference and the bundle's object-reference model; it does not by -itself mean an object was deleted or a file export failed. Resolving these raw -icon CIDs is not yet implemented. +In 1-to-1 spaces `spaceIcon`, and on participant objects `iconImage`, hold a +raw content CID with no file object behind it. Export writes that as the +icon's `cid` variant (SPEC §2b), decided by the CID's codec, and neither the +composer nor `Validate` treats it as an object the bundle should carry. A +bundle written before the variant existed carries the same CID under +`icon.file`; that spelling is read as the `cid` variant for compatibility and +names no object either. Import writes the CID back to the stored slot. diff --git a/bundle/compose.go b/bundle/compose.go index a3bc139..69ebe2d 100644 --- a/bundle/compose.go +++ b/bundle/compose.go @@ -137,8 +137,27 @@ type Stats struct { // its normal state. The slots are the ones bundle.Validate refuses on, // so an export states exactly what a later validation would find. UnresolvedTargets []string + // UnresolvedDeleted and UnresolvedOmitted are the subsets of + // UnresolvedTargets the wired store could classify (§2c, + // anyblockjson.ClassifyUnresolvedTarget): tombstones, and rows the space + // still holds that this export did not write. What is in neither is + // absent — the one class that is a loss. Sorted. index.json states the + // same subsets (Index.Unresolved.Deleted / Omitted). + UnresolvedDeleted []string + UnresolvedOmitted []string // UnresolvedReferences retains each missing index target with source location. UnresolvedReferences []anyblockjson.ObjectReference + // UnresolvedTypes are the derived type ids the written documents name + // — in a type_internal_key, a template_for, an object_types entry — that + // no written type document carries (§2c): an object or template an old + // import created with a type it never made. Bundled keys are never + // here; every reader carries the shipped table. Sorted. index.json + // states the same list (Index.Unresolved.Types), and a reader imports + // such an object as a Page and warns. + UnresolvedTypes []string + // UnresolvedTypeReferences retains each such reference with the slot + // (Path) and the written document id (ObjectID) it came from. + UnresolvedTypeReferences []anyblockjson.ObjectReference // RefusedOptions names the vocabularies the dictionary cannot state and // why — one `key: reason` line each, sorted. The writer refuses a // vocabulary on a property whose format does not admit one (§2a), and @@ -244,7 +263,11 @@ type Composer struct { // function Marshal used to write them, rather than a second opinion that // could disagree about a type's derived id. They answer the one question // no document can: whether an id this index names is carried here. - documentIds map[string]bool + documentIds map[string]bool + // typeRefs are the derived type ids the written documents name, by + // slot and document — the validator's own census (derivedTypeUses), + // taken here so the index can state what no document carries (§2c). + typeRefs map[derivedTypeUse]struct{} indexReferenceSources map[anyblockjson.ObjectReference]struct{} widgetViewLabels map[string]map[string]string @@ -291,6 +314,7 @@ func NewComposer(opts anyblockjson.Options, spaceName string) (*Composer, error) used: map[string]bool{}, declared: map[string]map[declaredProperty]bool{}, documentIds: map[string]bool{}, + typeRefs: map[derivedTypeUse]struct{}{}, spaceSettings: spaceSettingsCandidates{ names: map[string]struct{}{}, descriptions: map[string]struct{}{}, @@ -498,11 +522,11 @@ func (c *Composer) observeRelation(sbType model.SmartBlockType, base *model.Smar // Close, so whatever the composition needs to know about a document it has // to take from the bytes it is about to write (design §1.1). func (c *Composer) ObserveWritten(sbType model.SmartBlockType, base *model.SmartBlockSnapshotBase, doc []byte) error { + var envelope bundleDocumentEnvelope + if err := json.Unmarshal(doc, &envelope); err != nil { + return fmt.Errorf("read document envelope: %w", err) + } if sbType == model.SmartBlockType_FileObject || sbType == model.SmartBlockType_File { - var envelope bundleDocumentEnvelope - if err := json.Unmarshal(doc, &envelope); err != nil { - return fmt.Errorf("read file envelope: %w", err) - } if c.opts.IncludeFileRemote && envelope.FileRemote == nil { return fmt.Errorf("observe written file: IncludeFileRemote requires file_remote; use the same Options for Marshal and NewComposer") } @@ -546,6 +570,12 @@ func (c *Composer) ObserveWritten(sbType model.SmartBlockType, base *model.Smart if id := anyblockjson.FoldDocumentId(c.opts, sbType, base.GetDetails().GetFields()["id"].GetStringValue(), base.GetKey()); id != "" { c.documentIds[id] = true + // the derived type ids this document names, by slot — the same + // census the validator takes, so what the index declares and what + // the validator would refuse cannot disagree (§2c) + for _, use := range derivedTypeUses(id, envelope) { + c.typeRefs[use] = struct{}{} + } } for key := range used { c.used[key] = true @@ -1062,6 +1092,18 @@ func (c *Composer) Finish() (index, properties []byte, stats Stats, err error) { idx.Name = spaceSettings.Name idx.Description = spaceSettings.Description idx.Icon = spaceSettings.Icon + // the space icon follows the document-level icon rule (§2b, §9): an + // icon is optional, so one whose image object the space DELETED is + // dropped and warned rather than carried as a declared target, and the + // index falls through to whatever icon channel is left + if iconID := idx.IconImageId(); iconID != "" && anyblockjson.DroppedDeletedIconRef(c.opts, iconID) { + idx.Icon = nil + if c.opts.OnWarning != nil { + c.opts.OnWarning(anyblockjson.Issue{Path: "/icon/file", Message: fmt.Sprintf( + "space icon %q names an image object the space deleted and is dropped — an icon is optional, "+ + "and the index falls through to whatever icon channel is left", iconID)}) + } + } idx.Homepage = spaceSettings.Homepage // the caller's name is the fallback for a space whose document has none if idx.Name == "" { @@ -1101,10 +1143,26 @@ func (c *Composer) Finish() (index, properties []byte, stats Stats, err error) { // question "is this export incomplete" is answered where the bundle is // described, rather than by a reader discovering silence. unresolvedTargets := c.unresolvedIndexTargets(&idx) - if len(orphans) > 0 || len(unresolvedTargets) > 0 { + unresolvedDeleted, unresolvedOmitted := c.classifyUnresolvedTargets(unresolvedTargets) + // a relation document is never written (§2f), so a property's target + // types reach the bundle only through its dictionary entry — a slot the + // validator censuses, and one the document walk above cannot see + for _, def := range dict.Properties { + for _, target := range def.ObjectTypes { + spelling := anyblockjson.TypeKeySpelling(target) + if anyblockjson.IsDerivedTypeId(spelling) { + c.typeRefs[derivedTypeUse{ref: spelling, slot: "object_types", source: anyblockjson.PropertiesFileName}] = struct{}{} + } + } + } + unresolvedTypes, unresolvedTypeRefs := c.unresolvedTypeReferences() + if len(orphans) > 0 || len(unresolvedTargets) > 0 || len(unresolvedTypes) > 0 { idx.Unresolved = &anyblockjson.Unresolved{ Properties: orphans, Targets: unresolvedTargets, + Deleted: unresolvedDeleted, + Omitted: unresolvedOmitted, + Types: unresolvedTypes, } } idxData, err := anyblockjson.MarshalIndex(&idx, c.opts) @@ -1130,6 +1188,10 @@ func (c *Composer) Finish() (index, properties []byte, stats Stats, err error) { } stats.RefusedOptions = refusedOptions stats.UnresolvedTargets = unresolvedTargets + stats.UnresolvedDeleted = unresolvedDeleted + stats.UnresolvedOmitted = unresolvedOmitted + stats.UnresolvedTypes = unresolvedTypes + stats.UnresolvedTypeReferences = unresolvedTypeRefs stats.UnresolvedReferences = c.unresolvedIndexReferences(&idx) return idxData, dictData, stats, nil } @@ -1160,6 +1222,56 @@ func (c *Composer) unresolvedIndexTargets(idx *anyblockjson.Index) []string { return out } +// unresolvedTypeReferences names the derived type ids the written documents +// reference that no written type document carries (§2c), as a sorted set +// and as the references themselves with the slot and the document each +// came from. Bundled keys never appear: derivedTypeUses skips them, since +// every reader carries the shipped table. Called with the mutex held. +func (c *Composer) unresolvedTypeReferences() ([]string, []anyblockjson.ObjectReference) { + seen := map[string]bool{} + var ids []string + var refs []anyblockjson.ObjectReference + for use := range c.typeRefs { + if c.documentIds[use.ref] { + continue + } + if !seen[use.ref] { + seen[use.ref] = true + ids = append(ids, use.ref) + } + refs = append(refs, anyblockjson.ObjectReference{TargetObjectID: use.ref, Path: "/" + use.slot, ObjectID: use.source}) + } + sort.Strings(ids) + sort.Slice(refs, func(i, j int) bool { + a, b := refs[i], refs[j] + if a.TargetObjectID != b.TargetObjectID { + return a.TargetObjectID < b.TargetObjectID + } + if a.ObjectID != b.ObjectID { + return a.ObjectID < b.ObjectID + } + return a.Path < b.Path + }) + return ids, refs +} + +// classifyUnresolvedTargets asks the wired store WHY each unresolved target +// dangles (§2c, anyblockjson.ClassifyUnresolvedTarget) and splits the +// tombstones from the rows the space still holds. Both come back sorted, +// which they are already, since the input is. With no store wired nothing +// is classified and every target reads as absent, the fail-safe direction. +func (c *Composer) classifyUnresolvedTargets(targets []string) (deleted, omitted []string) { + for _, id := range targets { + switch anyblockjson.ClassifyUnresolvedTarget(c.opts, id) { + case anyblockjson.UnresolvedDeleted: + deleted = append(deleted, id) + case anyblockjson.UnresolvedOmitted: + omitted = append(omitted, id) + } + } + return deleted, omitted +} + // hasSemanticState distinguishes a genuinely empty composition from one in // which every source document was deliberately omitted because index.json or // properties.json carries it instead. spaceName is only a fallback label, so diff --git a/bundle/composeclassify_test.go b/bundle/composeclassify_test.go new file mode 100644 index 0000000..1db91d0 --- /dev/null +++ b/bundle/composeclassify_test.go @@ -0,0 +1,260 @@ +package bundle + +// composeclassify_test.go pins the composer's account of WHY an index +// target dangles (§2c, Index.Unresolved.Deleted / Omitted). The store draws +// the line — a deleted object keeps a tombstone row so a link to it can be +// told apart from a link to an object that never loaded — and the composer +// reads it through the two capabilities the exporter already wires. + +import ( + "encoding/json" + "testing" + "testing/fstest" + + "github.com/gogo/protobuf/types" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-block/codec/anyblockjson" + "github.com/anyproto/any-block/format/v1/model" + "github.com/anyproto/any-block/internal/testfixtures" +) + +func composeCid(seed string) string { + sum, err := mh.Sum([]byte(seed), mh.SHA2_256, -1) + if err != nil { + panic(err) + } + return cid.NewCidV1(cid.DagCBOR, sum).String() +} + +// classifyingStore is the pair storeresolver implements: every id in the +// map has a row, and the flag says whether that row is a tombstone. +type classifyingStore map[string]bool + +func (s classifyingStore) ObjectName(string) (string, bool) { return "", false } +func (s classifyingStore) ObjectExists(id string) (exists, known bool) { + _, ok := s[id] + return ok, true +} +func (s classifyingStore) ObjectDeleted(id string) (deleted, known bool) { + return s[id], true +} + +// How this can fail: put every dangling target in one class (the audited +// space's 41 tombstoned homepages read as losses, or its unsynced widget +// targets read as by-design); classify the written page; or forget the +// stats copy the exporter's report reads. +func TestComposer_TheIndexSaysWhyEachTargetIsUnresolved(t *testing.T) { + written, tomb, live, absent := composeCid("written"), composeCid("tomb"), composeCid("live"), composeCid("absent") + store := classifyingStore{written: false, tomb: true, live: false} + c := newComposer(t, anyblockjson.Options{ResolveObjectNames: store}, "Corpus") + + omitted, issues := c.Observe(model.SmartBlockType_Workspace, + spaceObservation(map[string]*types.Value{"name": strVal("Corpus"), "homepage": strVal(tomb)})) + require.True(t, omitted) + require.Empty(t, issues) + widgets, err := anyblockjson.WidgetsSnapshot(&anyblockjson.Index{Widgets: []anyblockjson.Widget{ + {Target: written}, {Target: live}, {Target: absent}, + }}) + require.NoError(t, err) + omitted, issues = c.Observe(model.SmartBlockType_Widget, widgets) + require.True(t, omitted) + require.Empty(t, issues) + writtenPage(t, c, written) + + indexData, _, stats, err := c.Finish() + require.NoError(t, err) + assert.ElementsMatch(t, []string{tomb, live, absent}, stats.UnresolvedTargets) + assert.Equal(t, []string{tomb}, stats.UnresolvedDeleted) + assert.Equal(t, []string{live}, stats.UnresolvedOmitted) + + idx, err := anyblockjson.UnmarshalIndex(indexData, anyblockjson.Options{}) + require.NoError(t, err) + require.NotNil(t, idx.Unresolved) + assert.Equal(t, []string{tomb}, idx.Unresolved.Deleted) + assert.Equal(t, []string{live}, idx.Unresolved.Omitted) + assert.ElementsMatch(t, []string{tomb, live, absent}, idx.Unresolved.Targets) +} + +// With no store wired the composer cannot say why, and says nothing it +// cannot know: every dangling target stays undeclared in both subsets, so a +// reader treats it as absent — the fail-safe direction. +func TestComposer_WithoutAStoreNoTargetIsClassified(t *testing.T) { + tomb := composeCid("tomb") + c := newComposer(t, anyblockjson.Options{}, "Corpus") + omitted, _ := c.Observe(model.SmartBlockType_Workspace, + spaceObservation(map[string]*types.Value{"name": strVal("Corpus"), "homepage": strVal(tomb)})) + require.True(t, omitted) + + _, _, stats, err := c.Finish() + require.NoError(t, err) + assert.Equal(t, []string{tomb}, stats.UnresolvedTargets) + assert.Empty(t, stats.UnresolvedDeleted) + assert.Empty(t, stats.UnresolvedOmitted) +} + +// A space icon whose image object is a tombstone follows the document-level +// icon rule (§2b, §9): an icon is optional, so a deleted one is dropped and +// warned, not carried as a declared target. An icon the space still holds +// is declared like any other target. +func TestComposer_ADeletedSpaceIconIsDroppedNotDeclared(t *testing.T) { + tomb, live := composeCid("tomb"), composeCid("live") + store := classifyingStore{tomb: true, live: false} + + t.Run("deleted: dropped, warned, not declared", func(t *testing.T) { + var warnings []anyblockjson.Issue + opts := anyblockjson.Options{ResolveObjectNames: store, OnWarning: func(i anyblockjson.Issue) { warnings = append(warnings, i) }} + c := newComposer(t, opts, "Corpus") + omitted, _ := c.Observe(model.SmartBlockType_Workspace, + spaceObservation(map[string]*types.Value{"name": strVal("Corpus"), "iconImage": strVal(tomb)})) + require.True(t, omitted) + + indexData, _, stats, err := c.Finish() + require.NoError(t, err) + idx, err := anyblockjson.UnmarshalIndex(indexData, anyblockjson.Options{}) + require.NoError(t, err) + assert.Nil(t, idx.Icon, "the index falls through to whatever icon channel is left") + assert.Empty(t, stats.UnresolvedTargets) + require.Len(t, warnings, 1) + assert.Equal(t, "/icon/file", warnings[0].Path) + assert.Contains(t, warnings[0].Message, tomb) + }) + + t.Run("omitted: kept and declared", func(t *testing.T) { + c := newComposer(t, anyblockjson.Options{ResolveObjectNames: store}, "Corpus") + omitted, _ := c.Observe(model.SmartBlockType_Workspace, + spaceObservation(map[string]*types.Value{"name": strVal("Corpus"), "iconImage": strVal(live)})) + require.True(t, omitted) + + indexData, _, stats, err := c.Finish() + require.NoError(t, err) + idx, err := anyblockjson.UnmarshalIndex(indexData, anyblockjson.Options{}) + require.NoError(t, err) + assert.Equal(t, live, idx.IconImageId()) + assert.Equal(t, []string{live}, stats.UnresolvedOmitted) + }) +} + +// A space icon addressed by content cid — a 1-to-1 space's icon, fetched by +// content id with no file object behind it — is written in the `cid` variant +// (§2b), and the composer neither declares it unresolved nor drops it: it +// names no object the bundle could carry. +func TestComposer_AContentCidSpaceIconIsNeitherDeclaredNorDropped(t *testing.T) { + var warnings []anyblockjson.Issue + opts := anyblockjson.Options{ResolveObjectNames: classifyingStore{}, OnWarning: func(i anyblockjson.Issue) { warnings = append(warnings, i) }} + c := newComposer(t, opts, "Corpus") + omitted, _ := c.Observe(model.SmartBlockType_Workspace, + spaceObservation(map[string]*types.Value{"name": strVal("Corpus"), "iconImage": strVal(testfixtures.ContentID)})) + require.True(t, omitted) + + indexData, _, stats, err := c.Finish() + require.NoError(t, err) + idx, err := anyblockjson.UnmarshalIndex(indexData, anyblockjson.Options{}) + require.NoError(t, err) + require.NotNil(t, idx.Icon) + assert.Equal(t, "cid", idx.Icon.Format) + assert.Equal(t, testfixtures.ContentID, idx.Icon.Cid) + assert.Empty(t, stats.UnresolvedTargets) + assert.Empty(t, warnings) +} + +// A document may name a type no document carries — by `type_internal_key`, +// by `template_for` — when an old import created it with a type it never +// made (five such objects in the corpus, from three importers). The +// composer states them in the index as derived ids, so the reader imports +// them as Pages with a warning instead of refusing the space (§2c). A type +// the bundle DOES carry, uninstalled or not, is never listed. +func TestComposer_TheIndexStatesTheTypesNoDocumentCarries(t *testing.T) { + c := newComposer(t, anyblockjson.Options{ResolveProperties: composerTypeVocabulary{}}, "Corpus") + page := &model.SmartBlockSnapshotBase{Details: detFields(map[string]*types.Value{"id": strVal(testfixtures.ObjectID)})} + require.NoError(t, c.ObserveWritten(model.SmartBlockType_Page, page, + []byte(`{"formatVersion":"2.0","id":"`+testfixtures.ObjectID+`","type":"gone","type_internal_key":"gone"}`))) + templateId := composeCid("template") + template := &model.SmartBlockSnapshotBase{Details: detFields(map[string]*types.Value{"id": strVal(templateId)})} + require.NoError(t, c.ObserveWritten(model.SmartBlockType_Template, template, + []byte(`{"formatVersion":"2.0","kind":"template","id":"`+templateId+`","type":"Template","type_internal_key":"template","template_for":"type-gone2"}`))) + typeSnap := &model.SmartBlockSnapshotBase{Key: "wine", + Details: detFields(map[string]*types.Value{"id": strVal(testfixtures.ObjectIDAlt), "uniqueKey": strVal("ot-wine")})} + require.NoError(t, c.ObserveWritten(model.SmartBlockType_STType, typeSnap, + []byte(`{"formatVersion":"2.0","id":"type-wine","kind":"object_type","internal_key":"wine","type_internal_key":"objectType"}`))) + + indexData, _, stats, err := c.Finish() + require.NoError(t, err) + assert.Equal(t, []string{"type-gone", "type-gone2"}, stats.UnresolvedTypes) + idx, err := anyblockjson.UnmarshalIndex(indexData, anyblockjson.Options{}) + require.NoError(t, err) + require.NotNil(t, idx.Unresolved) + assert.Equal(t, []string{"type-gone", "type-gone2"}, idx.Unresolved.Types, + "the wine type is carried; template and objectType are bundled; gone and gone2 are the losses") +} + +// A relation's stored target list may hold the app's own `_missing_object` +// sentinel — an old importer wrote it where a type it could not resolve +// belonged. A document's `object_types` already drops a stored sentinel +// silently when the store can be asked (§9); the dictionary entry the +// composer writes for the same relation must drop it the same way, or an +// export ships a target no reader can resolve: three real spaces failed +// on exactly this. +func TestComposer_TheDictionaryDropsTheMissingObjectSentinelFromTargets(t *testing.T) { + opts := anyblockjson.Options{ + ResolveObjectNames: classifyingStore{}, + ResolveProperties: composerPropertyResolver{def: anyblockjson.PropertyDefinition{ + Key: "related_ritual", Name: "Related ritual", Format: model.RelationFormat_object, + ObjectTypes: []string{"ritual", "_missing_object"}, + }}, + } + composer := newComposer(t, opts, "Rituals") + page := &model.SmartBlockSnapshotBase{Details: &types.Struct{Fields: map[string]*types.Value{"id": strVal("ritual-page")}}} + omitted, issues := composer.Observe(model.SmartBlockType_Page, page) + require.False(t, omitted) + require.Empty(t, issues) + require.NoError(t, composer.ObserveWritten(model.SmartBlockType_Page, page, + []byte(`{"formatVersion":"2.0","id":"ritual-page","properties":{"related_ritual":["ritual-object"]}}`))) + + _, properties, _, err := composer.Finish() + require.NoError(t, err) + var wire struct { + Properties []struct { + ObjectTypes []string `json:"object_types"` + } `json:"properties"` + } + require.NoError(t, json.Unmarshal(properties, &wire)) + require.Len(t, wire.Properties, 1) + assert.Equal(t, []string{"type-ritual"}, wire.Properties[0].ObjectTypes) + assert.NotContains(t, string(properties), "_missing_object") +} + +// A relation document is never written, so a property's target types reach +// the bundle ONLY through its dictionary entry — a slot the validator +// censuses and the composer did not, so the composer shipped a bundle its +// own validator refuses. The dictionary joins the type census. +func TestComposer_TheDictionaryTargetsJoinTheTypeCensus(t *testing.T) { + opts := anyblockjson.Options{ + ResolveProperties: composerPropertyResolver{def: anyblockjson.PropertyDefinition{ + Key: "related_ritual", Name: "Related ritual", Format: model.RelationFormat_object, + ObjectTypes: []string{"ritual"}, + }}, + } + c := newComposer(t, opts, "Rituals") + page := &model.SmartBlockSnapshotBase{Details: detFields(map[string]*types.Value{"id": strVal("ritual-page")})} + omitted, issues := c.Observe(model.SmartBlockType_Page, page) + require.False(t, omitted) + require.Empty(t, issues) + require.NoError(t, c.ObserveWritten(model.SmartBlockType_Page, page, + []byte(`{"formatVersion":"2.0","id":"ritual-page","properties":{"related_ritual":["ritual-object"]}}`))) + + index, properties, stats, err := c.Finish() + require.NoError(t, err) + assert.Equal(t, []string{"type-ritual"}, stats.UnresolvedTypes, + "no ritual type document was written, and the dictionary names it") + + // and the bundle the composer just produced passes its own validator + require.NoError(t, Validate(fstest.MapFS{ + "objects/ritual-page.anyblock.json": {Data: []byte(`{"formatVersion":"2.0","id":"ritual-page","properties":{"related_ritual":["ritual-object"]}}`)}, + anyblockjson.IndexFileName: {Data: index}, + anyblockjson.PropertiesFileName: {Data: properties}, + })) +} diff --git a/bundle/composenoderivedtypeids_test.go b/bundle/composenoderivedtypeids_test.go index 91177b4..2693eb7 100644 --- a/bundle/composenoderivedtypeids_test.go +++ b/bundle/composenoderivedtypeids_test.go @@ -410,15 +410,25 @@ func TestComposeNoDerivedTypeIds_ADanglingTemplateTargetIsStillReported(t *testi path, ok := plan.DocPath(noDerivedTemplateId) require.True(t, ok) - err = Validate(fstest.MapFS{ + fsys := fstest.MapFS{ path: {Data: data}, anyblockjson.IndexFileName: {Data: index}, anyblockjson.PropertiesFileName: {Data: dict}, - }) - require.Error(t, err) - assert.Contains(t, err.Error(), `template_for references type "type-ghost"`, + } + // the composer declared the type it could not carry (unresolved.types, + // §2c), so the full surface admits the bundle and states the loss — + // which is still a report, and the same report a mode-on export would + // never have produced + require.NoError(t, Validate(fsys)) + report, err := Inspect(fsys) + require.NoError(t, err) + require.Len(t, report.Issues, 1) + assert.Equal(t, anyblockjson.IssueCodeUnresolvedType, report.Issues[0].Code) + assert.Contains(t, report.Issues[0].Message, `template_for references type "type-ghost"`, "the bundle names the type document it is missing — 39 of these across the corpus, "+ "and a mode-on export would have reported none of them") + require.ErrorContains(t, ValidateAuthoring(fsys), `template_for references type "type-ghost"`, + "and an author's dangling target is still refused") } // THE SECOND FINDING, from the side that survives. The dictionary's diff --git a/bundle/composespace_test.go b/bundle/composespace_test.go index 0ee7fc3..40f67b2 100644 --- a/bundle/composespace_test.go +++ b/bundle/composespace_test.go @@ -171,7 +171,7 @@ func TestEveryLiftedFieldConflictIsOrderIndependentAndAtomic(t *testing.T) { {"name", map[string]*types.Value{"name": strVal("Zulu")}, map[string]*types.Value{"name": strVal("Alpha")}, `name=["Alpha", "Zulu"]`}, {"description", map[string]*types.Value{"description": strVal("Zulu")}, map[string]*types.Value{"description": strVal("Alpha")}, `description=["Alpha", "Zulu"]`}, {"homepage", map[string]*types.Value{"homepage": strVal("widgets")}, map[string]*types.Value{"homepage": strVal("graph")}, `homepage=["_graph", "_widgets"]`}, - {"icon", map[string]*types.Value{"iconEmoji": strVal("🧭")}, map[string]*types.Value{"iconEmoji": strVal("🔥")}, `icon=[{"format":"emoji","emoji":"🔥","file":"","name":"","color":null}, {"format":"emoji","emoji":"🧭","file":"","name":"","color":null}]`}, + {"icon", map[string]*types.Value{"iconEmoji": strVal("🧭")}, map[string]*types.Value{"iconEmoji": strVal("🔥")}, `icon=[{"format":"emoji","emoji":"🔥","file":"","cid":"","name":"","color":null}, {"format":"emoji","emoji":"🧭","file":"","cid":"","name":"","color":null}]`}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -206,7 +206,7 @@ func TestMultiFieldConflictErrorAndNilArtifactsAreDeterministic(t *testing.T) { "name": strVal("Alpha"), "description": strVal("Alpha"), "homepage": strVal("graph"), "iconEmoji": strVal("🔥"), }) - want := `conflicting observed space settings: description=["Alpha", "Zulu"]; homepage=["_graph", "_widgets"]; icon=[{"format":"emoji","emoji":"🔥","file":"","name":"","color":null}, {"format":"emoji","emoji":"🧭","file":"","name":"","color":null}]; name=["Alpha", "Zulu"]` + want := `conflicting observed space settings: description=["Alpha", "Zulu"]; homepage=["_graph", "_widgets"]; icon=[{"format":"emoji","emoji":"🔥","file":"","cid":"","name":"","color":null}, {"format":"emoji","emoji":"🧭","file":"","cid":"","name":"","color":null}]; name=["Alpha", "Zulu"]` for i, observations := range [][]*model.SmartBlockSnapshotBase{{one, two}, {two, one}} { result := composeSpaceObservations(t, "Alpha", false, observations...) require.EqualError(t, result.err, want, "permutation %d", i) diff --git a/bundle/convert/README.md b/bundle/convert/README.md new file mode 100644 index 0000000..dc42f99 --- /dev/null +++ b/bundle/convert/README.md @@ -0,0 +1,44 @@ +# Bundle conversion + +`Bundle(fsys, Options)` validates the full AnyBlock v2 bundle format and returns +native v1 archive entries. `Authoring(fsys, Options)` retains the stricter +authoring-only entry point. Neither function writes files or invokes processes. +Use a confined directory filesystem (`os.OpenRoot`) or a ZIP reader, with +`index.json` at the filesystem root. + +`Options.Encoding` defaults to `pb`; `json` selects protobuf-envelope JSON. +The `profile` entry is always binary protobuf. `SpaceID` supplies the destination +space for folded participant references. `OnWarning` receives fidelity warnings. + +`Result.Entries` contains snapshots and the profile in memory. `Result.Files` +maps native attachment paths to their source paths; callers must stream those +files from the input filesystem, keeping it open until copying is complete. +`SourceNetworkID` identifies the export's network when present. + +`Bundle` validates through `bundle.Inspect`: a declared dangling index target +(SPEC §2c) is admitted, and every non-error issue is forwarded to `OnWarning` +as a `severity: message` line. `Result.Unresolved` splits the declared targets +into `Deleted`, `Omitted` and `Absent`, spelled as the index spells them, so an +importer can keep a deleted id and tombstone it while the other two take the +sentinel it has always written. Homepage and widget targets are converted +verbatim either way. A declared missing type (`unresolved.types`) is +normalized: the object is imported as a Page, a template as one for Page, +each with a warning naming the document and the type, and +`Result.Unresolved.Types` lists them. + +Full conversion preserves stored property and option keys, installed built-in +and custom definitions, type/template settings, participant references, file +metadata and encryption keys, views, widgets, and space settings. Unknown +property formats are not invented. Uninstalled properties are restored live. +Option legends use the codec's target-liveness semantics, falling back to name +resolution in a fresh space; duplicate names can be ambiguous. Files without +bundled bytes require source-network access to become available. + +The CLI's default directory conversion remains authoring-only. Full conversion: + +```sh +anyblock to-v1 -full -in export-directory -out native.zip -zip -space-id DESTINATION_SPACE +``` + +Output must be a new path outside the input directory. Both directory and ZIP +output stream attachments instead of holding their bytes in memory. diff --git a/bundle/convert/convert.go b/bundle/convert/convert.go new file mode 100644 index 0000000..f7f20e7 --- /dev/null +++ b/bundle/convert/convert.go @@ -0,0 +1,518 @@ +// Package convert adapts AnyBlock v2 bundles to native v1 archives. +package convert + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "path/filepath" + "sort" + "strings" + + "github.com/gogo/protobuf/types" + + anyblockbundle "github.com/anyproto/any-block/bundle" + ab "github.com/anyproto/any-block/codec/anyblockjson" + "github.com/anyproto/any-block/codec/anyblockjson/domain" + "github.com/anyproto/any-block/codec/anyblockjson/vocabulary" + "github.com/anyproto/any-block/format/v1/model" +) + +// Options controls v1 encoding and destination-specific reference conversion. +type Options struct { + Encoding string // "pb" (default) or "json" + SpaceID string + OnWarning func(string) +} + +// Result is a complete native archive, including its binary root profile. +// Entries use slash-separated paths relative to the archive root. +type Result struct { + Entries map[string][]byte + Documents int + // Files maps native archive paths to source blob paths. Callers stream these from the input filesystem. + Files map[string]string + SourceNetworkID string + // Unresolved is what the index declared it could not carry, by class + // (SPEC §2c), spelled as the index spells it. An importer keeps a + // Deleted id and tombstones it; Omitted and Absent take the sentinel + // the importer has always written, and each earns a report entry. + Unresolved UnresolvedTargets +} + +// UnresolvedTargets splits an index's declared dangling targets into the +// three classes of SPEC §2c. Each list is sorted; the three are disjoint +// and together are exactly `unresolved.targets`. +type UnresolvedTargets struct { + Deleted []string + Omitted []string + Absent []string + // Types are the derived type ids the documents name that no type + // document carries and the source space never held (SPEC §2c). Each + // object of such a type is imported as a Page, and each template for + // one as a template for Page, with a warning. + Types []string +} + +func unresolvedTargets(u *ab.Unresolved) UnresolvedTargets { + var out UnresolvedTargets + if u == nil { + return out + } + classified := map[string]bool{} + for _, id := range u.Deleted { + classified[id] = true + } + for _, id := range u.Omitted { + classified[id] = true + } + out.Deleted = append([]string(nil), u.Deleted...) + out.Omitted = append([]string(nil), u.Omitted...) + out.Types = append([]string(nil), u.Types...) + for _, id := range u.Targets { + if !classified[id] { + out.Absent = append(out.Absent, id) + } + } + sort.Strings(out.Deleted) + sort.Strings(out.Omitted) + sort.Strings(out.Absent) + sort.Strings(out.Types) + return out +} + +// normalizeMissingTypes gives an object whose type the bundle declares it +// cannot carry (SPEC §2c, unresolved.types) the Page type, and a template +// for such a type the Page target, each with a warning. The source space +// never held the type — an old import created the object with a type it +// never made — so restoring the key verbatim would reproduce an object no +// type search finds and the app can only guess a layout for; a Page is +// what the user can work with. A type document is never touched. +func normalizeMissingTypes(name string, kind model.SmartBlockType, snapshot *model.SmartBlockSnapshotBase, missing map[string]bool, warn func(string)) { + if len(missing) == 0 || kind == model.SmartBlockType_STType || snapshot == nil { + return + } + for i, objectType := range snapshot.ObjectTypes { + key := strings.TrimPrefix(objectType, "ot-") + if !missing["type-"+key] { + continue + } + snapshot.ObjectTypes[i] = "ot-page" + if kind == model.SmartBlockType_Template && i == 1 { + if snapshot.Details != nil { + // re-derived from the Page target by completeBundleSnapshot + delete(snapshot.Details.Fields, "targetObjectType") + } + warn(fmt.Sprintf("%s: template targets type %q, which no document carries and the source space never held; imported as a template for Page", name, key)) + continue + } + warn(fmt.Sprintf("%s: type %q has no declaration in the bundle and the source space never held it; imported as a Page", name, key)) + } +} + +// Authoring converts a validated v2 authoring bundle to a native v1 archive. +// It does not write files. Exported space backups are outside this subset. +func Authoring(fsys fs.FS, options Options) (*Result, error) { + return convertBundle(fsys, options, true) +} + +// Bundle converts the complete v2 format, including installed definitions, +// participants, file bindings and stored identities. It also accepts authored bundles. +func Bundle(fsys fs.FS, options Options) (*Result, error) { return convertBundle(fsys, options, false) } + +func convertBundle(fsys fs.FS, options Options, authoring bool) (*Result, error) { + encoding, spaceID := options.Encoding, options.SpaceID + if encoding == "" { + encoding = "pb" + } + if encoding != "pb" && encoding != "json" { + return nil, fmt.Errorf("unknown encoding %q: use pb or json", encoding) + } + if spaceID != "" { + if err := domain.ValidateSpaceId(spaceID); err != nil { + return nil, fmt.Errorf("invalid space id: %w", err) + } + } + warn := func(message string) { + if options.OnWarning != nil { + options.OnWarning(message) + } + } + inspect := anyblockbundle.Inspect + if authoring { + inspect = anyblockbundle.InspectAuthoring + } + report, err := inspect(fsys) + if err != nil { + return nil, err + } + if err := report.Err(); err != nil { + return nil, err + } + // what the bundle states about itself and is still valid for: the + // declared dangling targets, graded (§2c). Forwarded as lines so a + // caller with only a string sink still shows the loss. + for _, issue := range report.Issues { + if issue.Severity != anyblockbundle.SeverityError { + warn(fmt.Sprintf("%s: %s", issue.Severity, issue.Message)) + } + } + indexData, err := fs.ReadFile(fsys, ab.IndexFileName) + if err != nil { + return nil, err + } + if authoring { + if err := ab.ValidateAuthoringIndex(indexData); err != nil { + return nil, err + } + } + initialIndex, err := ab.UnmarshalIndex(indexData, ab.Options{}) + if err != nil { + return nil, err + } + dictionaryPath := ab.PropertiesFileName + blobs := map[string]bool{} + if initialIndex.Manifest != nil { + if initialIndex.Manifest.Properties != "" { + dictionaryPath = initialIndex.Manifest.Properties + } + for _, name := range initialIndex.Manifest.Files { + blobs[name] = true + } + } + dictionaryData, err := fs.ReadFile(fsys, dictionaryPath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + if authoring && len(dictionaryData) > 0 { + if err := ab.ValidateAuthoringPropertyDictionary(dictionaryData); err != nil { + return nil, err + } + } + documents := map[string][]byte{} + typeIDs := map[string]string{} + err = fs.WalkDir(fsys, ".", func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + // Do not follow links even inside the root: a bundle should have exactly + // the same members when moved between filesystems or packaged as a ZIP. + if entry.Type()&fs.ModeSymlink != 0 { + return fmt.Errorf("symlink in bundle: %s", name) + } + if entry.IsDir() || filepath.Ext(name) != ".json" || name == ab.IndexFileName || name == dictionaryPath || blobs[name] { + return nil + } + data, err := fs.ReadFile(fsys, name) + if err != nil { + return err + } + if authoring { + if err := ab.ValidateAuthoring(data); err != nil { + return fmt.Errorf("%s: %w", name, err) + } + } + documents[name] = data + var header struct { + Kind string `json:"kind"` + ID string `json:"id"` + Key string `json:"internal_key"` + } + if err := json.Unmarshal(data, &header); err != nil { + return err + } + if header.Kind == "object_type" { + typeIDs[header.Key] = header.ID + } + return nil + }) + if err != nil { + return nil, err + } + plan, err := ab.PlanAuthoringTypeVocabulary(documents, ab.AuthoringVocabularyPlanOptions{PropertyDictionary: dictionaryData, Installed: !authoring}) + if err != nil { + return nil, err + } + dictionary := &ab.PropertyDictionary{} + if len(dictionaryData) > 0 { + dictionary, err = ab.UnmarshalPropertyDictionary(dictionaryData, ab.Options{Keys: plan}) + if err != nil { + return nil, err + } + } + resolver, err := newBundleResolverMode(plan, dictionary, typeIDs, !authoring) + if err != nil { + return nil, err + } + for _, key := range sortedBundleNames(resolver.nativeTypeKeys) { + if native := resolver.nativeTypeKeys[key]; native != key { + warn(fmt.Sprintf("type %s: internal key %q uses v1's separator; mapped to native key %q", typeIDs[key], key, native)) + } + } + + for _, def := range dictionary.Properties { + names := map[string]bool{} + for _, option := range def.Options { + if names[option.Name] { + warn(fmt.Sprintf("property %q has multiple options named %q; cross-space references fall back to the first matching name", def.Key, option.Name)) + } + names[option.Name] = true + } + if def.FormatUnknown { + warn(fmt.Sprintf("property %q has no definition; values are preserved without inventing a property", def.Key)) + } + if def.Uninstalled { + warn(fmt.Sprintf("property %q was uninstalled; restoring its definition live so imported values remain editable", def.Key)) + } + } + var conversionErr error + opts := ab.Options{Keys: resolver, ResolveFormat: resolver.resolveFormat, ResolveOptions: resolver, ResolveProperties: resolver, SpaceId: spaceID} + source := ab.IndexFileName + opts.OnWarning = func(issue ab.Issue) { + warn(fmt.Sprintf("%s: %s", source, issue)) + switch issue.Code { + case ab.IssueCodeFoldedParticipantsWithoutSpace, ab.IssueCodeFoldedTypesWithoutResolver: + conversionErr = fmt.Errorf("%s: %s", source, issue) + } + } + idx, err := ab.UnmarshalIndex(indexData, opts) + if err != nil { + return nil, err + } + entries := map[string][]byte{} + files := map[string]string{} + ids := map[string]bool{} + paths := map[string]string{} + addSnapshot := func(kind model.SmartBlockType, snapshot *model.SmartBlockSnapshotBase) error { + id := snapshot.GetDetails().GetFields()["id"].GetStringValue() + if id == "" || strings.ContainsAny(id, "/\\") || strings.Trim(id, ".") == "" { + return fmt.Errorf("unsafe output object id %q", id) + } + if ids[id] { + return fmt.Errorf("duplicate output object id %q", id) + } + ids[id] = true + data, err := encodeV1Snapshot(kind, snapshot, encoding) + if err != nil { + return err + } + directory := "objects" + switch kind { + case model.SmartBlockType_STType: + directory = "types" + case model.SmartBlockType_STRelation: + directory = "relations" + case model.SmartBlockType_STRelationOption: + directory = "relationsOptions" + case model.SmartBlockType_Template: + directory = "templates" + } + name := directory + "/" + id + "." + encoding + folded := strings.ToLower(name) + if previous, exists := paths[folded]; exists { + return fmt.Errorf("output paths %q and %q collide on case-insensitive filesystems", previous, name) + } + paths[folded] = name + entries[name] = data + return nil + } + // what the index declares the source space never held (§2c). A + // declaration for a type the bundle DOES carry is a contradiction the + // validator refuses; ignore it here too, so an over-declaring index can + // never retype live objects to Page behind the validator's back. + missingTypes := map[string]bool{} + resolver.missingTypes = map[string]bool{} + if idx.Unresolved != nil { + for _, id := range idx.Unresolved.Types { + key := strings.TrimPrefix(id, ab.TypeRefPrefix) + if _, carried := typeIDs[key]; carried { + continue + } + missingTypes[id] = true + resolver.missingTypes[key] = true + } + } + names := sortedBundleNames(documents) + for _, name := range names { + source = name + kind, snapshot, err := ab.Unmarshal(documents[name], opts) + if err != nil { + return nil, fmt.Errorf("%s: %w", name, err) + } + if kind == model.SmartBlockType_FileObject || kind == model.SmartBlockType_File { + id := snapshot.GetDetails().GetFields()["id"].GetStringValue() + blob := "" + if idx.Manifest != nil { + blob = idx.Manifest.Files[id] + } + if blob != "" { + target := "files/" + id + "/" + filepath.Base(blob) + if !fs.ValidPath(target) || strings.ContainsAny(id, "/\\") { + return nil, fmt.Errorf("unsafe file id %q", id) + } + files[target] = blob + snapshot.Details.Fields["source"] = stringValue(target) + } else { + delete(snapshot.Details.Fields, "source") + if snapshot.FileInfo == nil || snapshot.FileInfo.FileId == "" { + warn(fmt.Sprintf("%s: file bytes and recoverable remote metadata are absent", name)) + } else { + warn(fmt.Sprintf("%s: file bytes are not bundled; remote metadata retained (source network %q)", name, idx.NetworkId)) + } + } + } + normalizeMissingTypes(name, kind, snapshot, missingTypes, warn) + if err := completeBundleSnapshot(kind, snapshot, resolver); err != nil { + return nil, fmt.Errorf("%s: %w", name, err) + } + if err := addSnapshot(kind, snapshot); err != nil { + return nil, err + } + } + source = dictionaryPath + properties, err := resolver.propertySnapshots(opts) + if err != nil { + return nil, err + } + for _, id := range sortedBundleNames(properties) { + if err := addSnapshot(model.SmartBlockType_STRelation, properties[id]); err != nil { + return nil, err + } + } + optionSnapshots := resolver.optionSnapshots() + for _, id := range sortedBundleNames(optionSnapshots) { + if err := addSnapshot(model.SmartBlockType_STRelationOption, optionSnapshots[id]); err != nil { + return nil, err + } + } + widgets, err := ab.WidgetsSnapshot(idx) + if err != nil { + return nil, err + } + if widgets != nil { + if err := addSnapshot(model.SmartBlockType_Widget, widgets); err != nil { + return nil, err + } + } + if !authoring { + details := map[string]*types.Value{"name": stringValue(idx.Name), "description": stringValue(idx.Description), "homepage": stringValue(ab.WireHomepage(idx.SpaceHomepage()))} + if idx.Icon != nil { + // Use the object codec for all icon variants and palette values too. + var rawIndex map[string]json.RawMessage + if err := json.Unmarshal(indexData, &rawIndex); err != nil { + return nil, err + } + iconData, err := json.Marshal(map[string]any{"formatVersion": "2.0", "id": "space-icon", "icon": rawIndex["icon"]}) + if err != nil { + return nil, err + } + _, iconSnapshot, err := ab.Unmarshal(iconData, opts) + if err != nil { + return nil, fmt.Errorf("space icon: %w", err) + } + for _, key := range []string{"iconEmoji", "iconImage", "iconName", "iconOption"} { + if value, ok := iconSnapshot.Details.Fields[key]; ok { + details[key] = value + } + } + } + + if err := addSnapshot(model.SmartBlockType_Workspace, metadataSnapshot("_anyblock_space", "", "ot-space", details)); err != nil { + return nil, err + } + } + profile, err := encodeBundleProfile(idx) + if err != nil { + return nil, err + } + entries["profile"] = profile + if resolver.err != nil { + return nil, resolver.err + } + if conversionErr != nil { + return nil, conversionErr + } + if authoring && (idx.Icon != nil || idx.Description != "") { + warn("index.json: v1 profile has no space emoji or description fields; object icons and content are preserved") + } + return &Result{Entries: entries, Documents: len(documents), Files: files, SourceNetworkID: idx.NetworkId, + Unresolved: unresolvedTargets(idx.Unresolved)}, nil +} + +// The codec preserves document semantics; these fields are required by the +// native archive installer. In particular, templates are discovered through +// targetObjectType, not just the codec's ObjectTypes cache. +func completeBundleSnapshot(kind model.SmartBlockType, snapshot *model.SmartBlockSnapshotBase, resolver *bundleResolver) error { + for _, objectType := range snapshot.ObjectTypes { + key, err := vocabulary.TypeKeyFromUrl(objectType) + if err != nil { + return err + } + if _, ok := resolver.TypeIdByKey(string(key)); !ok { + if resolver.err != nil { + return resolver.err + } + // a declared missing type reached an identity slot that + // normalizeMissingTypes did not rewrite: loud, never a nil + // error that would skip the rest of this function + return fmt.Errorf("type %q is declared missing and was not normalized", key) + } + } + details := snapshot.Details.Fields + if kind == model.SmartBlockType_STType { + if len(snapshot.ObjectTypes) == 0 { + snapshot.ObjectTypes = []string{"ot-objectType"} + } + // The native type/template readers use scalar strings for these slots. + if values := details["defaultTemplateId"].GetListValue().GetValues(); len(values) == 1 { + details["defaultTemplateId"] = values[0] + } + } + if kind == model.SmartBlockType_Template { + if values := details["targetObjectType"].GetListValue().GetValues(); len(values) == 1 { + details["targetObjectType"] = values[0] + } + if details["targetObjectType"].GetStringValue() == "" { + if len(snapshot.ObjectTypes) < 2 { + return fmt.Errorf("template has no target type") + } + key, err := vocabulary.TypeKeyFromUrl(snapshot.ObjectTypes[1]) + if err != nil { + return err + } + id, ok := resolver.TypeIdByKey(string(key)) + if !ok { + return resolver.err + } + details["targetObjectType"] = stringValue(id) + } + } + + // ObjectTypes holds type keys in the legacy ot- spelling, not references + // to the type document IDs. Rewrite the definition and every membership + // together, after resolving ID-valued template/query/property targets. + if kind == model.SmartBlockType_STType { + snapshot.Key = resolver.nativeTypeKey(snapshot.Key) + // Carry the native identity for import paths that inspect details before + // rebuilding them; paths that derive it from Snapshot.Key agree. + details["uniqueKey"] = stringValue("ot-" + snapshot.Key) + } + for i, objectType := range snapshot.ObjectTypes { + key, err := vocabulary.TypeKeyFromUrl(objectType) + if err != nil { + return err + } + snapshot.ObjectTypes[i] = "ot-" + resolver.nativeTypeKey(string(key)) + } + return nil +} + +func sortedBundleNames[V any](entries map[string]V) []string { + names := make([]string, 0, len(entries)) + for name := range entries { + names = append(names, name) + } + sort.Strings(names) + return names +} diff --git a/bundle/convert/encode.go b/bundle/convert/encode.go new file mode 100644 index 0000000..bd3e3e9 --- /dev/null +++ b/bundle/convert/encode.go @@ -0,0 +1,34 @@ +package convert + +import ( + "fmt" + + envelopepb "github.com/anyproto/any-block/codec/anyblockjson/envelope" + "github.com/anyproto/any-block/format/v1/model" + "github.com/gogo/protobuf/jsonpb" + "github.com/gogo/protobuf/proto" +) + +func encodeV1Snapshot(sbType model.SmartBlockType, snapshot *model.SmartBlockSnapshotBase, encoding string) ([]byte, error) { + envelope := &envelopepb.SnapshotWithType{ + SbType: sbType, + Snapshot: &envelopepb.ChangeSnapshot{Data: snapshot}, + } + var output []byte + var err error + switch encoding { + case "pb": + output, err = proto.Marshal(envelope) + case "json": + model.RegisterJSONEnums() + var text string + text, err = (&jsonpb.Marshaler{Indent: " "}).MarshalToString(envelope) + output = []byte(text) + default: + return nil, fmt.Errorf("unknown encoding %q: use pb or json", encoding) + } + if err != nil { + return nil, fmt.Errorf("encode v1: %w", err) + } + return output, nil +} diff --git a/bundle/convert/full_test.go b/bundle/convert/full_test.go new file mode 100644 index 0000000..1c2dc00 --- /dev/null +++ b/bundle/convert/full_test.go @@ -0,0 +1,139 @@ +package convert + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io/fs" + "os" + "strings" + "testing" + "testing/fstest" + + pb "github.com/anyproto/any-block/codec/anyblockjson/envelope" + "github.com/anyproto/any-block/format/v1/model" + "github.com/anyproto/any-block/internal/testfixtures" + "github.com/gogo/protobuf/proto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func decodeEntries(t *testing.T, result *Result) map[string]*pb.SnapshotWithType { + t.Helper() + out := map[string]*pb.SnapshotWithType{} + for name, data := range result.Entries { + if !strings.HasSuffix(name, ".pb") { + continue + } + sn := &pb.SnapshotWithType{} + require.NoError(t, proto.Unmarshal(data, sn)) + out[sn.Snapshot.Data.Details.Fields["id"].GetStringValue()] = sn + } + return out +} +func TestFullExport(t *testing.T) { + fixture := os.DirFS("../../format/v2/examples/exported_space") + var warnings []string + result, err := Bundle(fixture, Options{SpaceID: "root.suffix", OnWarning: func(s string) { warnings = append(warnings, s) }}) + require.NoError(t, err) + entries := decodeEntries(t, result) + require.Equal(t, "files/ridge.png", result.Files["files/bafyreiridgephoto/ridge.png"]) + require.Equal(t, "files/bafyreiridgephoto/ridge.png", entries["bafyreiridgephoto"].Snapshot.Data.Details.Fields["source"].GetStringValue()) + require.Contains(t, entries, "rel-0f1e2d3c4b5a69788796a5b4") + require.NotContains(t, entries, "rel-deadbeefdeadbeefdeadbeef") + option := entries["opt-0f1e2d3c4b5a69788796a5b6"].Snapshot.Data + require.Equal(t, "0f1e2d3c4b5a69788796a5b6", option.Key) + require.Equal(t, "grey", option.Details.Fields["relationOptionColor"].GetStringValue()) + require.NotEmpty(t, warnings) + require.Equal(t, model.SmartBlockType_Workspace, entries["_anyblock_space"].SbType) + _, err = Authoring(fixture, Options{}) + require.Error(t, err) +} +func TestFullManifestUsesDictionaryAndBlobPaths(t *testing.T) { + fixture := fstest.MapFS{} + root := os.DirFS("../../format/v2/examples/exported_space") + require.NoError(t, fs.WalkDir(root, ".", func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + data, err := fs.ReadFile(root, name) + if err != nil { + return err + } + fixture[name] = &fstest.MapFile{Data: data} + return nil + })) + var idx map[string]any + require.NoError(t, json.Unmarshal(fixture["index.json"].Data, &idx)) + manifest := idx["manifest"].(map[string]any) + manifest["properties"] = "metadata/dictionary.data" + manifest["files"].(map[string]any)["bafyreiridgephoto"] = "files/payload.json" + fixture["metadata/dictionary.data"] = fixture["properties.json"] + delete(fixture, "properties.json") + fixture["files/payload.json"] = fixture["files/ridge.png"] + delete(fixture, "files/ridge.png") + data, err := json.Marshal(idx) + require.NoError(t, err) + fixture["index.json"] = &fstest.MapFile{Data: data} + result, err := Bundle(fixture, Options{SpaceID: "root.suffix"}) + require.NoError(t, err) + require.Equal(t, "files/payload.json", result.Files["files/bafyreiridgephoto/payload.json"]) +} +func TestFullRemoteFileAndParticipant(t *testing.T) { + remote := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"version":1,"cid":%q,"encryption_keys":{"/0/":%q}}`, testfixtures.ContentID, testfixtures.FileVariantKey))) + identity := testfixtures.AccountIdentity + fixture := fstest.MapFS{ + "index.json": {Data: []byte(`{"formatVersion":"2.0","name":"Remote"}`)}, + "file.json": {Data: []byte(fmt.Sprintf(`{"formatVersion":"2.0","kind":"file_object","id":"remote-file","type":"File","file_remote":%q}`, remote))}, + "person.json": {Data: []byte(fmt.Sprintf(`{"formatVersion":"2.0","kind":"participant","id":"participant-%s","properties":{"Name":"Member"}}`, identity))}, + } + result, err := Bundle(fixture, Options{SpaceID: testfixtures.SpaceID}) + require.NoError(t, err) + entries := decodeEntries(t, result) + file := entries["remote-file"].Snapshot.Data + require.Equal(t, testfixtures.ContentID, file.FileInfo.FileId) + require.Equal(t, testfixtures.FileVariantKey, file.FileInfo.EncryptionKeys[0].Key) + require.NotContains(t, file.Details.Fields, "source") + require.Empty(t, result.Files) + require.Contains(t, entries, testfixtures.ParticipantID(testfixtures.SpaceID, identity)) +} +func TestFullStoredOptionsAndTemplateTarget(t *testing.T) { + fixture := fstest.MapFS{ + "index.json": {Data: []byte(`{"formatVersion":"2.0"}`)}, + "properties.json": {Data: []byte(fmt.Sprintf(`{"formatVersion":"2.0","properties":[{"name":"Status","property":"status","internal_key":"customStatus","format":"select","options":[{"name":"Same","internal_key":"first","api_key":"firstOption"},{"name":"Same","internal_key":"second"}]},{"name":"Owner","internal_key":"customOwner","format":"objects","object_types":[%q,"type-participant"]}]}`, testfixtures.ContentID))}, + "template.json": {Data: []byte(`{"formatVersion":"2.0","kind":"template","id":"tpl","type":"Template","properties":{"Template's Type":["type-page"]}}`)}, + } + result, err := Bundle(fixture, Options{}) + require.NoError(t, err) + entries := decodeEntries(t, result) + require.Contains(t, entries, "opt-first") + require.Contains(t, entries, "opt-second") + require.Equal(t, "firstOption", entries["opt-first"].Snapshot.Data.Details.Fields["apiObjectKey"].GetStringValue()) + require.Equal(t, "ot-page", entries["tpl"].Snapshot.Data.Details.Fields["targetObjectType"].GetStringValue()) + targets := entries["rel-customOwner"].Snapshot.Data.Details.Fields["relationFormatObjectTypes"].GetListValue().Values + assert.Equal(t, testfixtures.ContentID, targets[0].GetStringValue()) +} + +func TestFullSpaceIconByContentCidReachesTheSpace(t *testing.T) { + fixture := fstest.MapFS{"index.json": {Data: []byte(`{"formatVersion":"2.0","icon":{"format":"cid","cid":"` + testfixtures.ContentID + `"}}`)}} + result, err := Bundle(fixture, Options{}) + require.NoError(t, err, "a content cid names no object and is not a dangling target") + details := decodeEntries(t, result)["_anyblock_space"].Snapshot.Data.Details.Fields + require.Equal(t, testfixtures.ContentID, details["iconImage"].GetListValue().GetValues()[0].GetStringValue()) +} + +func TestFullSpaceIconVariants(t *testing.T) { + for _, icon := range []string{`{"format":"icon","name":"star","color":"red"}`, `{"format":"color","color":"blue"}`} { + fixture := fstest.MapFS{"index.json": {Data: []byte(`{"formatVersion":"2.0","icon":` + icon + `}`)}} + result, err := Bundle(fixture, Options{}) + require.NoError(t, err) + details := decodeEntries(t, result)["_anyblock_space"].Snapshot.Data.Details.Fields + require.NotZero(t, details["iconOption"].GetNumberValue()) + if strings.Contains(icon, "star") { + require.Equal(t, "star", details["iconName"].GetStringValue()) + } + } +} diff --git a/bundle/convert/profile.go b/bundle/convert/profile.go new file mode 100644 index 0000000..b090788 --- /dev/null +++ b/bundle/convert/profile.go @@ -0,0 +1,54 @@ +package convert + +import ( + ab "github.com/anyproto/any-block/codec/anyblockjson" + "github.com/anyproto/any-block/format/v1/model" + "github.com/gogo/protobuf/proto" +) + +// These are the fields used from Heart's pb.Profile and pb.WidgetBlock +// (pb/protos/snapshot.proto). Keeping the wire-compatible subset here avoids +// pulling the application and its services into this standalone CLI. The root +// profile is always binary protobuf, even when snapshots use JSON encoding. +// Profile is the wire-compatible subset of native archive metadata. +type Profile struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3"` + Homepage string `protobuf:"bytes,5,opt,name=spaceDashboardId,proto3"` + Widgets []*ProfileWidget `protobuf:"bytes,9,rep,name=widgets,proto3"` +} + +func (p *Profile) Reset() { *p = Profile{} } +func (p *Profile) String() string { return proto.CompactTextString(p) } +func (*Profile) ProtoMessage() {} + +// ProfileWidget identifies one sidebar target in the native archive. +type ProfileWidget struct { + Layout int32 `protobuf:"varint,1,opt,name=layout,proto3"` + Target string `protobuf:"bytes,2,opt,name=targetObjectId,proto3"` + Limit int32 `protobuf:"varint,3,opt,name=objectLimit,proto3"` +} + +func (p *ProfileWidget) Reset() { *p = ProfileWidget{} } +func (p *ProfileWidget) String() string { return proto.CompactTextString(p) } +func (*ProfileWidget) ProtoMessage() {} + +func encodeBundleProfile(idx *ab.Index) ([]byte, error) { + profile := &Profile{Name: idx.Name, Homepage: ab.WireHomepage(idx.SpaceHomepage())} + // Read enum values from the same builder that emits the sidebar snapshot, + // so adding a supported layout cannot drift between the two representations. + widgets, err := ab.WidgetsSnapshot(idx) + if err != nil { + return nil, err + } + if widgets != nil { + for _, block := range widgets.Blocks { + widget, ok := block.Content.(*model.BlockContentOfWidget) + if !ok { + continue + } + w := idx.Widgets[len(profile.Widgets)] + profile.Widgets = append(profile.Widgets, &ProfileWidget{Layout: int32(widget.Widget.Layout), Target: ab.WireWidgetTarget(w.Target), Limit: w.Limit}) + } + } + return proto.Marshal(profile) +} diff --git a/bundle/convert/profile_test.go b/bundle/convert/profile_test.go new file mode 100644 index 0000000..870da61 --- /dev/null +++ b/bundle/convert/profile_test.go @@ -0,0 +1,19 @@ +package convert + +import ( + "encoding/hex" + ab "github.com/anyproto/any-block/codec/anyblockjson" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func TestBundleProfileWireCompatibility(t *testing.T) { + // Independent wire fixture: Profile.name=1, dashboard=5, widgets=9; + // WidgetBlock.layout=1 (Link=0 omitted), target=2, limit=3. + wire, err := hex.DecodeString("0a01582a04686f6d654a081204686f6d651803") + require.NoError(t, err) + data, err := encodeBundleProfile(&ab.Index{Name: "X", Entrypoint: "home", Widgets: []ab.Widget{{Target: "home", Limit: 3}}}) + require.NoError(t, err) + assert.Equal(t, wire, data) +} diff --git a/bundle/convert/resolver.go b/bundle/convert/resolver.go new file mode 100644 index 0000000..bef75bf --- /dev/null +++ b/bundle/convert/resolver.go @@ -0,0 +1,426 @@ +package convert + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strings" + + ab "github.com/anyproto/any-block/codec/anyblockjson" + "github.com/anyproto/any-block/codec/anyblockjson/domain" + "github.com/anyproto/any-block/codec/anyblockjson/vocabulary" + "github.com/anyproto/any-block/format/v1/model" + "github.com/gogo/protobuf/types" + "github.com/ipfs/go-cid" +) + +// Remap the entire planned property namespace, including candidate sets and +// type scopes, so ambiguous names keep the codec's normal resolution rules. +type bundleResolver struct { + *ab.AuthoringTypeVocabulary + keys map[string]string + definitions map[string]ab.PropertyDefinition + typeIDs map[string]string + nativeTypeKeys map[string]string + options map[string]*bundleOption + installed bool + // missingTypes are the STORED KEYS the index declares the source space + // never held (SPEC §2c, unresolved.types). A lookup for one is a soft + // miss: it returns not-found without arming err, so each slot degrades + // the way it already degrades an absent reference instead of failing + // the whole conversion. + missingTypes map[string]bool + err error +} + +type bundleOption struct { + key, id string + definition ab.OptionDefinition + order int +} + +func newBundleResolver(plan *ab.AuthoringTypeVocabulary, dictionary *ab.PropertyDictionary, typeIDs map[string]string) (*bundleResolver, error) { + return newBundleResolverMode(plan, dictionary, typeIDs, false) +} + +func newBundleResolverMode(plan *ab.AuthoringTypeVocabulary, dictionary *ab.PropertyDictionary, typeIDs map[string]string, installed bool) (*bundleResolver, error) { + r := &bundleResolver{installed: installed, AuthoringTypeVocabulary: plan, keys: map[string]string{}, definitions: map[string]ab.PropertyDefinition{}, typeIDs: typeIDs, options: map[string]*bundleOption{}} + var err error + nativeTypes := map[string]string{} + for key, id := range typeIDs { + if !installed || !vocabulary.HasObjectTypeByKey(domain.TypeKey(key)) { + nativeTypes[key] = id + } + } + r.nativeTypeKeys, err = planNativeTypeKeys(nativeTypes) + if err != nil { + return nil, err + } + for _, def := range dictionary.Properties { + oldKey := string(def.Key) + key := oldKey + if !def.KeyIsInternal && !vocabulary.HasRelation(def.Key) { + var random [12]byte + if _, err := rand.Read(random[:]); err != nil { + return nil, fmt.Errorf("mint property %q: %w", def.Name, err) + } + key = hex.EncodeToString(random[:]) + } + if _, exists := r.definitions[key]; exists { + return nil, fmt.Errorf("duplicate output property key %q", key) + } + r.keys[oldKey] = key + def.Key = domain.RelationKey(key) + r.definitions[key] = def + for _, option := range def.Options { + r.addOption(key, option) + } + } + return r, nil +} + +// Heart's unique-key wire syntax is "ot-". Its UnmarshalUniqueKey +// rejects more than one hyphen, although v2 legitimately allows hyphens in +// authored internal keys. Keep compatible keys; give incompatible ones stable +// native identities. Document IDs remain unchanged and still bind all links. +func planNativeTypeKeys(typeIDs map[string]string) (map[string]string, error) { + result := make(map[string]string, len(typeIDs)) + owners := make(map[string]string, len(typeIDs)) + for key := range typeIDs { + if !strings.Contains(key, "-") { + owners[key] = key + } + } + for _, key := range sortedBundleNames(typeIDs) { + native := key + if strings.Contains(key, "-") { + hash := sha256.Sum256([]byte("anyblock:v1:type:" + key)) + native = hex.EncodeToString(hash[:12]) + } + if owner, exists := owners[native]; exists && owner != key { + return nil, fmt.Errorf("type keys %q and %q map to the same native key %q", owner, key, native) + } + if vocabulary.HasObjectTypeByKey(domain.TypeKey(native)) { + return nil, fmt.Errorf("native type key %q collides with a built-in type", native) + } + owners[native] = key + result[key] = native + } + return result, nil +} + +func (r *bundleResolver) nativeTypeKey(key string) string { + if native, exists := r.nativeTypeKeys[key]; exists { + return native + } + return key +} + +func (r *bundleResolver) mappedKey(key string) string { + if mapped, ok := r.keys[key]; ok { + return mapped + } + return key +} + +func (r *bundleResolver) PropertyKey(term string) (string, bool) { + key, ok := r.AuthoringTypeVocabulary.PropertyKey(term) + return r.mappedKey(key), ok +} + +func (r *bundleResolver) PropertySlug(key string) string { + if def, ok := r.definitions[key]; ok { + return ab.PropertyLabel(key, def.Name) + } + return r.AuthoringTypeVocabulary.PropertySlug(key) +} + +func (r *bundleResolver) PropertyKeyCandidates(term string) []string { + if key, ok := r.keys[term]; ok { + return []string{key} + } + candidates := r.AuthoringTypeVocabulary.PropertyKeyCandidates(term) + for i, key := range candidates { + candidates[i] = r.mappedKey(key) + } + sort.Strings(candidates) + return candidates +} + +func (r *bundleResolver) TypePropertyKeys(key string) []string { + keys := r.AuthoringTypeVocabulary.TypePropertyKeys(key) + for i, key := range keys { + keys[i] = r.mappedKey(key) + } + return keys +} + +func (r *bundleResolver) PropertyTermFacts(term string) ab.KeyTermFacts { + facts := r.AuthoringTypeVocabulary.PropertyTermFacts(term) + if _, minted := r.definitions[term]; minted { + facts.LiveStoredKey = true + } else if r.mappedKey(term) != term { + facts.LiveStoredKey = false + } + return facts +} + +func (r *bundleResolver) resolveFormat(key domain.RelationKey) (model.RelationFormat, bool) { + def, ok := r.definitions[r.mappedKey(string(key))] + return def.Format, ok && !def.FormatUnknown +} + +func (r *bundleResolver) PropertyId(def ab.PropertyDefinition) (string, bool) { + key := r.mappedKey(string(def.Key)) + if _, ok := r.definitions[key]; ok || vocabulary.HasRelation(domain.RelationKey(key)) { + return "rel-" + key, true + } + r.err = fmt.Errorf("property %q has no definition in properties.json", key) + return "", false +} + +func (r *bundleResolver) PropertyById(id string) (ab.PropertyDefinition, bool) { + for key, def := range r.definitions { + if id == "rel-"+key { + return def, true + } + } + return ab.PropertyDefinition{}, false +} + +func (r *bundleResolver) TypeIdByKey(key string) (string, bool) { + if id, ok := r.typeIDs[key]; ok { + return id, true + } + if _, err := vocabulary.GetType(domain.TypeKey(key)); err == nil { + return domain.TypeKey(key).URL(), true + } + // Old exports can retain a target type's object ID after its definition + // disappeared. Preserve that reference for the native missing-object policy. + if r.installed { + if _, err := cid.Decode(key); err == nil { + return key, true + } + } + if r.missingTypes[key] { + // declared missing: the caller degrades, the conversion goes on + return "", false + } + r.err = fmt.Errorf("type %q has no declaration in the bundle", key) + return "", false +} + +func (r *bundleResolver) TypeKeyById(id string) (string, bool) { + for key, target := range r.typeIDs { + if id == target { + return key, true + } + } + key, err := vocabulary.TypeKeyFromUrl(id) + if err != nil { + return "", false + } + _, err = vocabulary.GetType(key) + return string(key), err == nil +} + +func (r *bundleResolver) addOption(key string, def ab.OptionDefinition) *bundleOption { + identity := key + "\x00" + def.Name + if def.InternalKey != "" { + identity = key + "\x00key:" + def.InternalKey + } + if option, ok := r.options[identity]; ok { + if option.definition.Name != def.Name { + r.err = fmt.Errorf("option key %q has conflicting names in property %q", def.InternalKey, key) + } + return option + } + // Options are local to their property. Repeated values (including defaults + // and filter values) resolve to the same object; equal names on different + // properties must never share an option. + hash := sha256.Sum256([]byte(identity)) + optionKey := def.InternalKey + if optionKey == "" { + optionKey = hex.EncodeToString(hash[:12]) + def.InternalKey = optionKey + } + option := &bundleOption{key: key, id: "opt-" + archiveSafeId(optionKey), definition: def, order: 1} + for _, existing := range r.options { + if existing.id == option.id && existing.key != key { + r.err = fmt.Errorf("option key %q belongs to multiple properties", option.definition.InternalKey) + } + if existing.key == key { + option.order++ + } + } + r.options[identity] = option + return option +} + +// archiveSafeId makes a stored key usable as a native archive entry id. An +// option's key is minted from its name, so "C/C++" puts a slash in it, and +// the archive names an entry by the object's id, where a slash is a path. +// Only the archive id is escaped: the option's real key travels as the +// snapshot's Key, which is what the destination derives its object id +// from, and every value naming the option resolves to this same id. +// The escape character is escaped FIRST, so the mapping is injective: two +// keys that differ only by an escape sequence ("a/b" and "a%2Fb") must not +// collapse onto one archive entry, where one value is silently dropped and +// which one survives depends on map iteration order. +func archiveSafeId(key string) string { + return strings.NewReplacer("%", "%25", "/", "%2F", "\\", "%5C").Replace(key) +} + +func (r *bundleResolver) OptionId(key domain.RelationKey, name string) (string, bool) { + if name == "" { + return "", false + } + mapped := r.mappedKey(string(key)) + var first *bundleOption + for _, option := range r.options { + if option.key == mapped && option.definition.Name == name && (first == nil || option.order < first.order) { + first = option + } + } + if first != nil { + return first.id, true + } + return r.addOption(mapped, ab.OptionDefinition{Name: name}).id, true +} + +func (r *bundleResolver) OptionName(key domain.RelationKey, id string) (string, bool) { + for _, option := range r.options { + if option.key == r.mappedKey(string(key)) && option.id == id { + return option.definition.Name, true + } + } + return "", false +} + +// missingObjectSentinel is the store's own marker for a reference an +// importer could not resolve (SPEC §9). It can sit in a relation's stored +// target list and reach a dictionary entry written by an older export. +const missingObjectSentinel = "_missing_object" + +func (r *bundleResolver) propertySnapshots(opts ab.Options) (map[string]*model.SmartBlockSnapshotBase, error) { + out := map[string]*model.SmartBlockSnapshotBase{} + keys := make([]string, 0, len(r.definitions)) + for key := range r.definitions { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + def := r.definitions[key] + if def.FormatUnknown { + continue + } + details := map[string]*types.Value{} + if installed, ok := ab.InstalledRelationDetails(key, opts); ok { + details = installed.Fields + } + details["isHidden"] = boolValue(def.Hidden) + details["relationReadonlyValue"] = boolValue(def.Readonly) + if def.ApiKey != "" { + details["apiObjectKey"] = stringValue(def.ApiKey) + } + if def.DefaultValueSet || def.DefaultValue != nil { + value, err := ab.UnmarshalPropertyValueChecked(key, def.DefaultValue, opts) + if err != nil { + return nil, fmt.Errorf("property %q default: %w", key, err) + } + if value != nil { + details["relationDefaultValue"] = value + } + } + details["name"] = stringValue(def.Name) + details["relationKey"] = stringValue(key) + details["relationFormat"] = numberValue(float64(def.Format)) + details["layout"] = numberValue(float64(model.ObjectType_relation)) + if def.Description != "" { + details["description"] = stringValue(def.Description) + } + if def.IncludeTimeSet { + details["relationFormatIncludeTime"] = &types.Value{Kind: &types.Value_NullValue{}} + if def.IncludeTime != nil { + details["relationFormatIncludeTime"] = boolValue(*def.IncludeTime) + } + } + if ab.MultiValuedFormat(def.Format) { + details["relationMaxCount"] = numberValue(float64(def.MaxCount)) + } else { + details["relationMaxCount"] = numberValue(1) + } + if len(def.ObjectTypes) > 0 { + var targets []*types.Value + for _, target := range def.ObjectTypes { + if target == missingObjectSentinel { + // the app's own sentinel, written by an old importer where a + // type it could not resolve belonged and copied by older + // exports: it names no type, so the entry keeps its other + // targets and goes on without it + if opts.OnWarning != nil { + opts.OnWarning(ab.Issue{Path: "/properties/" + key + "/object_types", + Message: fmt.Sprintf("property %q targets the missing-object sentinel %q, which names no type; dropped", key, target)}) + } + continue + } + id, ok := r.TypeIdByKey(target) + if !ok { + if r.missingTypes[target] { + // a target type the source space never held: the + // list expresses the absence by being shorter (§9) + if opts.OnWarning != nil { + opts.OnWarning(ab.Issue{Path: "/properties/" + key + "/object_types", + Message: fmt.Sprintf("property %q targets type %q, which no document carries and the source space never held; dropped", key, target)}) + } + continue + } + return nil, r.err + } + targets = append(targets, stringValue(id)) + } + if len(targets) > 0 { + details["relationFormatObjectTypes"] = listValue(targets) + } + } + id := "rel-" + key + out[id] = metadataSnapshot(id, key, "ot-relation", details) + } + return out, nil +} + +func (r *bundleResolver) optionSnapshots() map[string]*model.SmartBlockSnapshotBase { + out := map[string]*model.SmartBlockSnapshotBase{} + for _, option := range r.options { + details := map[string]*types.Value{ + "name": stringValue(option.definition.Name), "relationKey": stringValue(option.key), + "layout": numberValue(float64(model.ObjectType_relationOption)), "orderId": stringValue(fmt.Sprintf("%08d", option.order)), + } + if option.definition.ApiKey != "" { + details["apiObjectKey"] = stringValue(option.definition.ApiKey) + } + if option.definition.Color != "" { + details["relationOptionColor"] = stringValue(option.definition.Color) + } + out[option.id] = metadataSnapshot(option.id, option.definition.InternalKey, "ot-relationOption", details) + } + return out +} + +func metadataSnapshot(id, key, objectType string, details map[string]*types.Value) *model.SmartBlockSnapshotBase { + details["id"] = stringValue(id) + return &model.SmartBlockSnapshotBase{Key: key, ObjectTypes: []string{objectType}, Details: &types.Struct{Fields: details}, Blocks: []*model.Block{{Id: id, Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}} +} + +func stringValue(v string) *types.Value { + return &types.Value{Kind: &types.Value_StringValue{StringValue: v}} +} +func numberValue(v float64) *types.Value { + return &types.Value{Kind: &types.Value_NumberValue{NumberValue: v}} +} +func boolValue(v bool) *types.Value { return &types.Value{Kind: &types.Value_BoolValue{BoolValue: v}} } +func listValue(v []*types.Value) *types.Value { + return &types.Value{Kind: &types.Value_ListValue{ListValue: &types.ListValue{Values: v}}} +} diff --git a/bundle/convert/resolver_test.go b/bundle/convert/resolver_test.go new file mode 100644 index 0000000..715dd5e --- /dev/null +++ b/bundle/convert/resolver_test.go @@ -0,0 +1,15 @@ +package convert + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func TestNativeTypeKeyPlanningPreservesCompatibleKeysAndRejectsCollisions(t *testing.T) { + planned, err := planNativeTypeKeys(map[string]string{"trip": "type-trip", "bookclub-person": "type-bookclub-person"}) + require.NoError(t, err) + assert.Equal(t, "trip", planned["trip"]) + _, err = planNativeTypeKeys(map[string]string{"bookclub-person": "type-bookclub-person", planned["bookclub-person"]: "type-other"}) + require.ErrorContains(t, err, "same native key") +} diff --git a/bundle/convert/unresolved_test.go b/bundle/convert/unresolved_test.go new file mode 100644 index 0000000..ce9ada0 --- /dev/null +++ b/bundle/convert/unresolved_test.go @@ -0,0 +1,285 @@ +package convert + +// unresolved_test.go pins what the converter does with a bundle whose index +// DECLARES targets it does not carry, and why (§2c): the conversion runs, +// the three classes reach the caller on the result so an importer can keep +// a deleted id and tombstone it, and every declared target is forwarded as +// a warning line the caller can show. An undeclared dangling target is +// still the exporter bug it always was. + +import ( + "encoding/json" + "io/fs" + "os" + "strings" + "testing" + "testing/fstest" + + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-block/format/v1/model" +) + +func convertCid(seed string) string { + sum, err := mh.Sum([]byte(seed), mh.SHA2_256, -1) + if err != nil { + panic(err) + } + return cid.NewCidV1(cid.DagCBOR, sum).String() +} + +func exportedSpaceFixture(t *testing.T) fstest.MapFS { + t.Helper() + fixture := fstest.MapFS{} + root := os.DirFS("../../format/v2/examples/exported_space") + require.NoError(t, fs.WalkDir(root, ".", func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + data, err := fs.ReadFile(root, name) + if err != nil { + return err + } + fixture[name] = &fstest.MapFile{Data: data} + return nil + })) + return fixture +} + +func withIndex(t *testing.T, fixture fstest.MapFS, edit func(idx map[string]any)) { + t.Helper() + var idx map[string]any + require.NoError(t, json.Unmarshal(fixture["index.json"].Data, &idx)) + edit(idx) + data, err := json.Marshal(idx) + require.NoError(t, err) + fixture["index.json"] = &fstest.MapFile{Data: data} +} + +func TestBundleCarriesDeclaredUnresolvedTargets(t *testing.T) { + deleted, omitted, absent := convertCid("deleted"), convertCid("omitted"), convertCid("absent") + fixture := exportedSpaceFixture(t) + withIndex(t, fixture, func(idx map[string]any) { + idx["homepage"] = absent + idx["widgets"] = append(idx["widgets"].([]any), map[string]any{"target": deleted}, map[string]any{"target": omitted}) + idx["unresolved"] = map[string]any{ + "targets": []string{absent, deleted, omitted}, + "deleted": []string{deleted}, + "omitted": []string{omitted}, + } + }) + + var warnings []string + result, err := Bundle(fixture, Options{SpaceID: "root.suffix", OnWarning: func(s string) { warnings = append(warnings, s) }}) + require.NoError(t, err, "a declared loss is admitted on the full surface") + assert.Equal(t, []string{deleted}, result.Unresolved.Deleted) + assert.Equal(t, []string{omitted}, result.Unresolved.Omitted) + assert.Equal(t, []string{absent}, result.Unresolved.Absent) + + joined := strings.Join(warnings, "\n") + assert.Contains(t, joined, deleted) + assert.Contains(t, joined, omitted) + assert.Contains(t, joined, "not synced") + + entries := decodeEntries(t, result) + space := entries["_anyblock_space"].Snapshot.Data.Details.Fields + assert.Equal(t, absent, space["homepage"].GetStringValue(), "the homepage passes through verbatim; the importer decides") + var widgetTargets []string + for _, entry := range entries { + if entry.SbType != model.SmartBlockType_Widget { + continue + } + for _, block := range entry.Snapshot.Data.Blocks { + if link := block.GetLink(); link != nil { + widgetTargets = append(widgetTargets, link.TargetBlockId) + } + } + } + assert.Contains(t, widgetTargets, deleted) + assert.Contains(t, widgetTargets, omitted) +} + +func TestBundleStillRefusesAnUndeclaredDanglingTarget(t *testing.T) { + absent := convertCid("absent") + fixture := exportedSpaceFixture(t) + withIndex(t, fixture, func(idx map[string]any) { idx["homepage"] = absent }) + + _, err := Bundle(fixture, Options{SpaceID: "root.suffix"}) + require.ErrorContains(t, err, `homepage references object "`+absent+`"`) +} + +// An object whose type no document carries and the space never held — an +// old import's leftover — is imported as a Page, and a template for such a +// type as a template for Page, each with a warning naming the document and +// the type. Restoring the dangling key verbatim would reproduce an object +// that cannot be searched by type and reads as broken; a Page is what the +// user can work with. +func TestBundleNormalizesAMissingTypeToPage(t *testing.T) { + fixture := exportedSpaceFixture(t) + fixture["objects/orphan.anyblock.json"] = &fstest.MapFile{Data: []byte( + `{"formatVersion":"2.0","id":"bafyreiorphan","type":"69aab06861fab2bc0d9afc59","type_internal_key":"69aab06861fab2bc0d9afc59","properties":{"Name":"Orphan"}}`)} + fixture["templates/orphantpl.anyblock.json"] = &fstest.MapFile{Data: []byte( + `{"formatVersion":"2.0","kind":"template","id":"bafyreiorphantpl","type":"Template","type_internal_key":"template","template_for":"type-6832fe62421aa906e701c31b","properties":{"Name":""}}`)} + withIndex(t, fixture, func(idx map[string]any) { + idx["unresolved"] = map[string]any{"properties": idx["unresolved"].(map[string]any)["properties"], + "types": []string{"type-69aab06861fab2bc0d9afc59", "type-6832fe62421aa906e701c31b"}} + }) + + var warnings []string + result, err := Bundle(fixture, Options{SpaceID: "root.suffix", OnWarning: func(s string) { warnings = append(warnings, s) }}) + require.NoError(t, err, "a declared missing type is admitted") + assert.Equal(t, []string{"type-6832fe62421aa906e701c31b", "type-69aab06861fab2bc0d9afc59"}, result.Unresolved.Types) + + entries := decodeEntries(t, result) + assert.Equal(t, []string{"ot-page"}, entries["bafyreiorphan"].Snapshot.Data.ObjectTypes, "imported as a Page") + assert.Equal(t, []string{"ot-template", "ot-page"}, entries["bafyreiorphantpl"].Snapshot.Data.ObjectTypes, "a template for Page") + joined := strings.Join(warnings, "\n") + assert.Contains(t, joined, "orphan.anyblock.json") + assert.Contains(t, joined, "69aab06861fab2bc0d9afc59") + assert.Contains(t, joined, "Page") +} + +// A dictionary entry may carry the app's own `_missing_object` sentinel in +// its target types: an old importer wrote it where a type it could not +// resolve belonged, and older exports copy it. It names no type, so the +// entry's other targets are kept, the sentinel is dropped with a warning, +// and the conversion goes on — three real spaces failed on exactly this. +func TestBundleDropsTheMissingObjectSentinelFromDictionaryTargets(t *testing.T) { + fixture := exportedSpaceFixture(t) + var dict map[string]any + require.NoError(t, json.Unmarshal(fixture["properties.json"].Data, &dict)) + for _, entry := range dict["properties"].([]any) { + e := entry.(map[string]any) + if e["internal_key"] == "creator" { + e["object_types"] = []any{"type-participant", "_missing_object"} + } + } + data, err := json.Marshal(dict) + require.NoError(t, err) + fixture["properties.json"] = &fstest.MapFile{Data: data} + + var warnings []string + result, err := Bundle(fixture, Options{SpaceID: "root.suffix", OnWarning: func(s string) { warnings = append(warnings, s) }}) + require.NoError(t, err, "a stored sentinel is not a type the bundle owes") + targets := decodeEntries(t, result)["rel-creator"].Snapshot.Data.Details.Fields["relationFormatObjectTypes"].GetListValue().GetValues() + require.Len(t, targets, 1, "the real target survives, the sentinel does not") + assert.NotContains(t, targets[0].GetStringValue(), "_missing_object") + assert.Contains(t, strings.Join(warnings, "\n"), "creator") + assert.Contains(t, strings.Join(warnings, "\n"), "_missing_object") +} + +// A relation option's stored key is minted from its name, so a name like +// "C/C++" puts a slash in the key. The native archive names an entry by the +// object's id, and a slash there is a path — one real space refused to +// convert on exactly this. The archive id is escaped; the option keeps its +// real key, which is what the destination derives its object id from, and +// every value that names the option resolves to the same escaped id. +func TestBundleEscapesASlashInAnOptionKeyForTheArchive(t *testing.T) { + fixture := fstest.MapFS{ + "index.json": {Data: []byte(`{"formatVersion":"2.0"}`)}, + "properties.json": {Data: []byte(`{"formatVersion":"2.0","properties":[` + + `{"name":"Language","property":"language","internal_key":"customLanguage","format":"select",` + + `"options":[{"name":"C/C++","internal_key":"69bbfc78877a91b1d12d1a7c_C/C++"}]}]}`)}, + "page.json": {Data: []byte(`{"formatVersion":"2.0","id":"page","type":"Page","properties":{"language":"C/C++"}}`)}, + } + result, err := Bundle(fixture, Options{}) + require.NoError(t, err) + + entries := decodeEntries(t, result) + var optionId string + for id, entry := range entries { + if entry.Snapshot.Data.Details.Fields["name"].GetStringValue() == "C/C++" { + optionId = id + assert.NotContains(t, id, "/", "the archive id is path-safe") + assert.Equal(t, "69bbfc78877a91b1d12d1a7c_C/C++", entry.Snapshot.Data.Key, "the option keeps its real key") + } + } + require.NotEmpty(t, optionId, "the option travels") + // the custom property lands under the native key the converter mints + // for it, so the value is found by content rather than by key + found := false + for _, value := range entries["page"].Snapshot.Data.Details.Fields { + for _, item := range value.GetListValue().GetValues() { + if item.GetStringValue() == optionId { + found = true + } + } + } + assert.True(t, found, "the value names the option by the same escaped id") +} + +// A declared missing type (SPEC §2c, unresolved.types) is named from more +// slots than an object's own type. `query_source.types` and a property's +// `object_types` reach it too, and those used to hard-fail the whole +// conversion with "has no declaration in the bundle" — a bundle the +// validator had just admitted with a warning. The identity slots become +// Page; a slot that merely REFERENCES the type degrades the way §9 already +// degrades an absent reference, and the conversion goes on. +func TestBundleConvertsADeclaredMissingTypeInEveryReferenceSlot(t *testing.T) { + const ghost = "type-69aab06861fab2bc0d9afc59" + fixture := exportedSpaceFixture(t) + fixture["objects/set.anyblock.json"] = &fstest.MapFile{Data: []byte( + `{"formatVersion":"2.0","id":"bafyreiset","type":"Page","properties":{"Name":"Ghost set"},` + + `"query_source":{"types":["` + ghost + `"]}}`)} + var dict map[string]any + require.NoError(t, json.Unmarshal(fixture["properties.json"].Data, &dict)) + for _, entry := range dict["properties"].([]any) { + if e := entry.(map[string]any); e["internal_key"] == "creator" { + e["object_types"] = []any{"type-participant", ghost} + } + } + data, err := json.Marshal(dict) + require.NoError(t, err) + fixture["properties.json"] = &fstest.MapFile{Data: data} + withIndex(t, fixture, func(idx map[string]any) { + idx["unresolved"] = map[string]any{"types": []string{ghost}} + }) + + var warnings []string + result, err := Bundle(fixture, Options{SpaceID: "root.suffix", OnWarning: func(s string) { warnings = append(warnings, s) }}) + require.NoError(t, err, "the validator admits this bundle, so the converter must too") + + entries := decodeEntries(t, result) + require.Contains(t, entries, "bafyreiset", "the set still travels") + targets := entries["rel-creator"].Snapshot.Data.Details.Fields["relationFormatObjectTypes"].GetListValue().GetValues() + require.Len(t, targets, 1, "the live target survives; the missing one is dropped") + assert.NotContains(t, targets[0].GetStringValue(), "69aab06861fab2bc0d9afc59") + assert.Contains(t, strings.Join(warnings, "\n"), "69aab06861fab2bc0d9afc59") +} + +// Two option keys that differ only by an escape sequence must stay two +// options. The archive id escapes `/`, so `a/b` and `a%2Fb` collapsed onto +// one entry, one of the two values was dropped, and which one survived +// depended on map iteration order — the composer's byte-determinism +// invariant broken by an id function that was not injective. +func TestBundleKeepsEscapedAndUnescapedOptionKeysDistinct(t *testing.T) { + fixture := fstest.MapFS{ + "index.json": {Data: []byte(`{"formatVersion":"2.0"}`)}, + "properties.json": {Data: []byte(`{"formatVersion":"2.0","properties":[` + + `{"name":"Language","property":"language","internal_key":"customLanguage","format":"select",` + + `"options":[{"name":"Slash","internal_key":"a/b"},{"name":"Escaped","internal_key":"a%2Fb"}]}]}`)}, + } + var first map[string]string + for run := 0; run < 8; run++ { + result, err := Bundle(fixture, Options{}) + require.NoError(t, err) + names := map[string]string{} + for id, entry := range decodeEntries(t, result) { + if name := entry.Snapshot.Data.Details.Fields["name"].GetStringValue(); name == "Slash" || name == "Escaped" { + names[name] = id + } + } + require.Len(t, names, 2, "both options travel; neither id may swallow the other") + assert.NotEqual(t, names["Slash"], names["Escaped"], "distinct keys, distinct archive ids") + if first == nil { + first = names + } + assert.Equal(t, first, names, "run %d: the same input composes to the same ids", run) + } +} diff --git a/bundle/inspectunresolved_test.go b/bundle/inspectunresolved_test.go new file mode 100644 index 0000000..e688f97 --- /dev/null +++ b/bundle/inspectunresolved_test.go @@ -0,0 +1,171 @@ +package bundle + +// inspectunresolved_test.go pins what a DECLARED dangling index target buys +// on each validation surface (§2c, §12). The declaration is the exemption, +// and only on the full surface: an export that says what it could not carry, +// and why, is admitted with the loss stated at the severity the class earns; +// an author's dangling reference is an authoring error whatever the index +// says, and an UNDECLARED dangling target is an exporter bug on both. + +import ( + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-block/codec/anyblockjson" + "github.com/anyproto/any-block/internal/testfixtures" +) + +func declaringBundle(unresolved string) fstest.MapFS { + index := `{"formatVersion":"2.0","entrypoint":"page","homepage":"gone","widgets":[{"target":"page"}]` + if unresolved != "" { + index += `,"unresolved":` + unresolved + } + index += `}` + return fstest.MapFS{ + "index.json": &fstest.MapFile{Data: []byte(index)}, + "objects/page.json": &fstest.MapFile{Data: []byte(`{"formatVersion":"2.0","id":"page"}`)}, + "properties.json": &fstest.MapFile{Data: []byte(`{"formatVersion":"2.0"}`)}, + } +} + +func TestInspect_ADeclaredDeletedTargetIsAdmittedAsInfo(t *testing.T) { + fsys := declaringBundle(`{"targets":["gone"],"deleted":["gone"]}`) + + require.NoError(t, Validate(fsys), "a tombstone the space still names is ordinary state") + report, err := Inspect(fsys) + require.NoError(t, err) + require.Len(t, report.Issues, 1) + assert.Equal(t, SeverityInfo, report.Issues[0].Severity) + assert.Equal(t, anyblockjson.IssueCodeDeletedTarget, report.Issues[0].Code) + assert.Equal(t, "homepage", report.Issues[0].Path) + assert.Contains(t, report.Issues[0].Message, "gone") + assert.Empty(t, report.Errors()) + + err = ValidateAuthoring(fsys) + require.ErrorContains(t, err, `homepage references object "gone"`, "an author cannot declare a loss away") +} + +func TestInspect_ADeclaredOmittedTargetIsAWarning(t *testing.T) { + fsys := declaringBundle(`{"targets":["gone"],"omitted":["gone"]}`) + + require.NoError(t, Validate(fsys)) + report, err := Inspect(fsys) + require.NoError(t, err) + require.Len(t, report.Issues, 1) + assert.Equal(t, SeverityWarning, report.Issues[0].Severity) + assert.Equal(t, anyblockjson.IssueCodeOmittedTarget, report.Issues[0].Code) +} + +func TestInspect_ADeclaredAbsentTargetIsAWarning(t *testing.T) { + fsys := declaringBundle(`{"targets":["gone"]}`) + + require.NoError(t, Validate(fsys)) + report, err := Inspect(fsys) + require.NoError(t, err) + require.Len(t, report.Issues, 1) + assert.Equal(t, SeverityWarning, report.Issues[0].Severity) + assert.Equal(t, anyblockjson.IssueCodeUnresolvedTarget, report.Issues[0].Code) + assert.Contains(t, report.Issues[0].Message, "not synced") +} + +func TestInspect_AnUndeclaredTargetStaysAnError(t *testing.T) { + fsys := declaringBundle("") + + err := Validate(fsys) + require.ErrorContains(t, err, `homepage references object "gone"`) + report, err := Inspect(fsys) + require.NoError(t, err) + require.Len(t, report.Errors(), 1) + assert.Equal(t, SeverityError, report.Errors()[0].Severity) + assert.Equal(t, "homepage", report.Errors()[0].Path) +} + +// Validate's verdict and Inspect's errors are one list: whatever Validate +// says, Inspect's error-severity issues say, line for line. +func TestInspect_ErrorsAreWhatValidateRefusesOn(t *testing.T) { + fsys := declaringBundle("") + delete(fsys, "objects/page.json") + + err := Validate(fsys) + require.Error(t, err) + report, inspectErr := Inspect(fsys) + require.NoError(t, inspectErr) + require.Len(t, report.Errors(), 3, "entrypoint, homepage and the widget target") + for _, issue := range report.Errors() { + assert.Contains(t, err.Error(), issue.Message) + } + assert.EqualError(t, report.Err(), err.Error()) +} + +// A space icon by content cid names no object, in either spelling: the +// `cid` variant, and the legacy overloaded `file` an older exporter wrote. +func TestInspect_AContentCidIconIsNotATarget(t *testing.T) { + for name, icon := range map[string]string{ + "cid variant": `{"format":"cid","cid":"` + testfixtures.ContentID + `"}`, + "legacy file": `{"format":"file","file":"` + testfixtures.ContentID + `"}`, + } { + t.Run(name, func(t *testing.T) { + fsys := fstest.MapFS{ + "index.json": &fstest.MapFile{Data: []byte(`{"formatVersion":"2.0","entrypoint":"page","icon":` + icon + `}`)}, + "objects/page.json": &fstest.MapFile{Data: []byte(`{"formatVersion":"2.0","id":"page"}`)}, + "properties.json": &fstest.MapFile{Data: []byte(`{"formatVersion":"2.0"}`)}, + } + report, err := Inspect(fsys) + require.NoError(t, err) + assert.Empty(t, report.Issues) + }) + } +} + +// A type a document names and no document carries is admitted when the +// index declares it (unresolved.types): a warning on the full surface, an +// error on the authoring surface and whenever undeclared — the same rule a +// dangling index target follows (§2c). +func TestInspect_ADeclaredMissingTypeIsAWarning(t *testing.T) { + bundleWith := func(unresolved string) fstest.MapFS { + index := `{"formatVersion":"2.0","entrypoint":"page"` + if unresolved != "" { + index += `,"unresolved":` + unresolved + } + index += `}` + return fstest.MapFS{ + "index.json": &fstest.MapFile{Data: []byte(index)}, + "objects/page.json": &fstest.MapFile{Data: []byte(`{"formatVersion":"2.0","id":"page","type":"gone","type_internal_key":"gone"}`)}, + "properties.json": &fstest.MapFile{Data: []byte(`{"formatVersion":"2.0"}`)}, + } + } + declared := bundleWith(`{"types":["type-gone"]}`) + require.NoError(t, Validate(declared)) + report, err := Inspect(declared) + require.NoError(t, err) + require.Len(t, report.Issues, 1) + assert.Equal(t, SeverityWarning, report.Issues[0].Severity) + assert.Equal(t, anyblockjson.IssueCodeUnresolvedType, report.Issues[0].Code) + assert.Contains(t, report.Issues[0].Message, "type-gone") + require.ErrorContains(t, ValidateAuthoring(declared), `references type "type-gone"`) + + require.ErrorContains(t, Validate(bundleWith("")), `references type "type-gone"`) +} + +// An index may not declare a type the bundle CARRIES. The declaration means +// "the source space never held this", and the converter acts on it by +// importing every object of that type as a Page — so an over-declaring +// index silently retypes live objects and orphans their type document. +// Refused at the door, like every other self-contradicting report (§2c). +func TestInspect_ADeclaredTypeTheBundleCarriesIsRefused(t *testing.T) { + fsys := fstest.MapFS{ + "index.json": &fstest.MapFile{Data: []byte( + `{"formatVersion":"2.0","entrypoint":"page","unresolved":{"types":["type-wine"]}}`)}, + "objects/page.json": &fstest.MapFile{Data: []byte(`{"formatVersion":"2.0","id":"page","type":"Wine","type_internal_key":"wine"}`)}, + "types/wine.json": &fstest.MapFile{Data: []byte( + `{"formatVersion":"2.0","kind":"object_type","id":"type-wine","internal_key":"wine","type":"Type","properties":{"Name":"Wine"}}`)}, + "properties.json": &fstest.MapFile{Data: []byte(`{"formatVersion":"2.0"}`)}, + } + + err := Validate(fsys) + require.ErrorContains(t, err, "type-wine") + require.ErrorContains(t, err, "carries") +} diff --git a/bundle/report.go b/bundle/report.go new file mode 100644 index 0000000..fc40b0f --- /dev/null +++ b/bundle/report.go @@ -0,0 +1,95 @@ +package bundle + +import ( + "fmt" + "sort" + "strings" + + "github.com/anyproto/any-block/codec/anyblockjson" +) + +// Severity grades a bundle-level issue (§12). Only SeverityError makes +// Validate refuse; the other two are what Inspect exists to carry. +type Severity uint8 + +const ( + // SeverityError: the bundle is not valid AnyBlock. Validate refuses it. + SeverityError Severity = iota + // SeverityWarning: the bundle is valid and states a loss — a target the + // space had no row for, or one it holds that this export did not write. + SeverityWarning + // SeverityInfo: the bundle is valid and states something that is not a + // loss — a target the space deleted, by design. + SeverityInfo +) + +func (s Severity) String() string { + switch s { + case SeverityError: + return "error" + case SeverityWarning: + return "warning" + case SeverityInfo: + return "info" + } + return fmt.Sprintf("severity(%d)", uint8(s)) +} + +// ReportIssue is one bundle-level verdict: a severity, a stable code where one +// exists (callers branch on Code, never on Message), the index field or +// bundle path it is about, and the wording Validate has always used. +type ReportIssue struct { + Severity Severity + Code anyblockjson.IssueCode + Path string + Message string +} + +// Report is everything one walk of a bundle had to say, errors first, then +// warnings, then info, each group sorted so two walks of one bundle read +// alike. +type Report struct { + Issues []ReportIssue +} + +// Errors returns the error-severity issues — exactly what Validate refuses +// on. +func (r *Report) Errors() []ReportIssue { + var out []ReportIssue + for _, issue := range r.Issues { + if issue.Severity == SeverityError { + out = append(out, issue) + } + } + return out +} + +// Err is Validate's verdict for this report: nil when no issue is an error, +// else the same joined, sorted message Validate has always returned. +func (r *Report) Err() error { + errs := r.Errors() + if len(errs) == 0 { + return nil + } + messages := make([]string, 0, len(errs)) + for _, issue := range errs { + messages = append(messages, issue.Message) + } + sort.Strings(messages) + return fmt.Errorf("bundle validation failed:\n- %s", strings.Join(messages, "\n- ")) +} + +func (r *Report) add(issue ReportIssue) { r.Issues = append(r.Issues, issue) } + +func (r *Report) sorted() { + sort.SliceStable(r.Issues, func(i, j int) bool { + a, b := r.Issues[i], r.Issues[j] + if a.Severity != b.Severity { + return a.Severity < b.Severity + } + if a.Path != b.Path { + return a.Path < b.Path + } + return a.Message < b.Message + }) +} diff --git a/bundle/validate.go b/bundle/validate.go index 68e39bc..8dee913 100644 --- a/bundle/validate.go +++ b/bundle/validate.go @@ -180,7 +180,22 @@ func bundlePathAliasKey(name string) string { // pass the resulting root.FS(), rather than use os.DirFS, when validating // untrusted bundle contents. func Validate(fsys fs.FS) error { - return validate(fsys, fullFormatSurface) + report, err := inspect(fsys, fullFormatSurface) + if err != nil { + return err + } + return report.Err() +} + +// Inspect is Validate with the whole verdict kept: every error Validate +// would refuse on, plus the warnings and info a valid bundle states about +// itself — chiefly the index targets it declares it cannot carry, graded by +// the class the writer gave them (§2c: a tombstone is info, an omitted or +// absent object a warning). The error return is reserved for a walk that +// could not run at all — no index, an unreadable one — and is the same +// error Validate returns for it. +func Inspect(fsys fs.FS) (*Report, error) { + return inspect(fsys, fullFormatSurface) } // ValidateAuthoring is Validate for a bundle an author WROTE (§2g): every @@ -213,7 +228,19 @@ func Validate(fsys fs.FS) error { // calling one function or the other, exactly as `NoDerivedTypeIds` is a fact // about the writer (§9). func ValidateAuthoring(fsys fs.FS) error { - return validate(fsys, authoringSurface) + report, err := inspect(fsys, authoringSurface) + if err != nil { + return err + } + return report.Err() +} + +// InspectAuthoring is Inspect on the authoring surface: the same report, +// with every dangling index target an error whatever the index declares — +// an author's dangling reference is an authoring error, and the authoring +// index schema has no `unresolved` member to declare one in. +func InspectAuthoring(fsys fs.FS) (*Report, error) { + return inspect(fsys, authoringSurface) } // bundleSurface says which of the two questions of §2g a walk is asking. It @@ -225,35 +252,36 @@ const ( authoringSurface ) -func validate(fsys fs.FS, surface bundleSurface) error { +func inspect(fsys fs.FS, surface bundleSurface) (*Report, error) { authoring := surface == authoringSurface + report := &Report{} authoritativePaths := newAuthoritativeBundlePaths() indexStatus, inspectErr := inspectExactBundleFile(fsys, anyblockjson.IndexFileName, authoritativePaths) switch indexStatus { case exactBundleFileAlias: - return fmt.Errorf("%s does not use exact directory-entry spelling", anyblockjson.IndexFileName) + return nil, fmt.Errorf("%s does not use exact directory-entry spelling", anyblockjson.IndexFileName) case exactBundleFileMissing: - return ErrIndexNotFound + return nil, ErrIndexNotFound case exactBundleFileNotRegular: - return fmt.Errorf("%s is not a regular file", anyblockjson.IndexFileName) + return nil, fmt.Errorf("%s is not a regular file", anyblockjson.IndexFileName) case exactBundleFileInspectionError: - return fmt.Errorf("cannot inspect %s: %w", anyblockjson.IndexFileName, inspectErr) + return nil, fmt.Errorf("cannot inspect %s: %w", anyblockjson.IndexFileName, inspectErr) case exactBundleFileReadable: // The authoritative index is admitted before its first content read. default: - return fmt.Errorf("cannot inspect %s: unknown admission status", anyblockjson.IndexFileName) + return nil, fmt.Errorf("cannot inspect %s: unknown admission status", anyblockjson.IndexFileName) } indexData, err := fs.ReadFile(fsys, anyblockjson.IndexFileName) if errors.Is(err, fs.ErrNotExist) { - return ErrIndexNotFound + return nil, ErrIndexNotFound } if err != nil { - return fmt.Errorf("read %s: %w", anyblockjson.IndexFileName, err) + return nil, fmt.Errorf("read %s: %w", anyblockjson.IndexFileName, err) } idx, err := anyblockjson.UnmarshalIndex(indexData, anyblockjson.Options{}) if err != nil { - return fmt.Errorf("validate %s: %w", anyblockjson.IndexFileName, err) + return nil, fmt.Errorf("validate %s: %w", anyblockjson.IndexFileName, err) } var issues []string @@ -458,7 +486,7 @@ func validate(fsys fs.FS, surface bundleSurface) error { return nil }) if err != nil { - return fmt.Errorf("walk bundle: %w", err) + return nil, fmt.Errorf("walk bundle: %w", err) } // Reported after the walk, over sorted keys, so the diagnostic is the @@ -520,19 +548,71 @@ func validate(fsys fs.FS, surface bundleSurface) error { } } + // What the index DECLARES it cannot carry, and why (§2c). On the full + // surface the declaration is the exemption: the loss is stated at the + // severity its class earns instead of refused. On the authoring surface + // it buys nothing — an author's dangling reference is an authoring + // error, and the authoring index schema has no member to declare one in. + declaredTargets := map[string]struct{}{} + declaredDeleted := map[string]struct{}{} + declaredOmitted := map[string]struct{}{} + if idx.Unresolved != nil { + for _, id := range idx.Unresolved.Targets { + declaredTargets[id] = struct{}{} + } + for _, id := range idx.Unresolved.Deleted { + declaredDeleted[id] = struct{}{} + } + for _, id := range idx.Unresolved.Omitted { + declaredOmitted[id] = struct{}{} + } + } requireObject := func(field, id string) { if id == "" || anyblockjson.IsReservedWidgetTarget(id) || anyblockjson.IsReservedHomepage(id) { return } - if _, exists := documentPaths[id]; !exists { - issues = append(issues, fmt.Sprintf("%s references object %q, but the bundle contains no document with that id", field, id)) + if _, exists := documentPaths[id]; exists { + return + } + if _, declared := declaredTargets[id]; declared && !authoring { + switch { + case has(declaredDeleted, id): + report.add(ReportIssue{Severity: SeverityInfo, Code: anyblockjson.IssueCodeDeletedTarget, Path: field, + Message: fmt.Sprintf("%s references object %q, which the space deleted; the index declares it (unresolved.deleted), and a restore keeps the id with a tombstone", field, id)}) + case has(declaredOmitted, id): + report.add(ReportIssue{Severity: SeverityWarning, Code: anyblockjson.IssueCodeOmittedTarget, Path: field, + Message: fmt.Sprintf("%s references object %q, which the space holds and this export did not write; the index declares it (unresolved.omitted)", field, id)}) + default: + report.add(ReportIssue{Severity: SeverityWarning, Code: anyblockjson.IssueCodeUnresolvedTarget, Path: field, + Message: fmt.Sprintf("%s references object %q, which the bundle does not carry and the space had no row for at export time — not synced, or never in this space; the index declares it (unresolved.targets)", field, id)}) + } + return } + report.add(ReportIssue{Severity: SeverityError, Path: field, + Message: fmt.Sprintf("%s references object %q, but the bundle contains no document with that id", field, id)}) } // The type namespace's cross-document check. A derived type id is an // address and the bundle is the only place one can be checked: a single // document cannot know whether `type-habit` is here. Reported once per // distinct (slot, id, file) so a type named from forty objects does not // produce forty lines. + declaredTypes := map[string]struct{}{} + if idx.Unresolved != nil { + for _, id := range idx.Unresolved.Types { + declaredTypes[id] = struct{}{} + } + } + // a declaration means the source space never held the type, and a + // reader acts on it by importing every object of that type as a Page. + // Declaring one the bundle CARRIES would retype live objects and orphan + // their type document, so the contradiction is refused at the door. + for id := range declaredTypes { + if path, carried := documentPaths[id]; carried { + issues = append(issues, fmt.Sprintf( + "unresolved.types names %q, but the bundle carries that type at %s; "+ + "the list states what the source space never held", id, path)) + } + } reportedTypeUse := map[derivedTypeUse]struct{}{} for _, use := range typeUses { if _, exists := documentPaths[use.ref]; exists { @@ -542,6 +622,16 @@ func validate(fsys fs.FS, surface bundleSurface) error { continue } reportedTypeUse[use] = struct{}{} + // a type the index DECLARES it cannot carry (§2c) is admitted on + // the full surface as the loss it states: the source space never + // held the type, and a reader imports the object as a Page. The + // authoring surface refuses it like every dangling reference. + if _, declared := declaredTypes[use.ref]; declared && !authoring { + report.add(ReportIssue{Severity: SeverityWarning, Code: anyblockjson.IssueCodeUnresolvedType, Path: use.source, + Message: fmt.Sprintf("%s: %s references type %q, which no document carries and the source space never held; "+ + "the index declares it (unresolved.types), and a reader imports the object as a Page", use.source, use.slot, use.ref)}) + continue + } issues = append(issues, fmt.Sprintf( "%s: %s references type %q, but the bundle contains no document with that id — "+ "a type document's id IS its derived id (SPEC §9), and since the manifest lost its "+ @@ -598,11 +688,16 @@ func validate(fsys fs.FS, surface bundleSurface) error { } } } - if len(issues) == 0 { - return nil + for _, message := range issues { + report.add(ReportIssue{Severity: SeverityError, Message: message}) } - sort.Strings(issues) - return fmt.Errorf("bundle validation failed:\n- %s", strings.Join(issues, "\n- ")) + report.sorted() + return report, nil +} + +func has(set map[string]struct{}, id string) bool { + _, ok := set[id] + return ok } func appendManifestRoleIssues(bindings map[string]map[string][]string, issues *[]string) { diff --git a/cmd/anyblock/README.md b/cmd/anyblock/README.md index 58fdd5e..49bd7ff 100644 --- a/cmd/anyblock/README.md +++ b/cmd/anyblock/README.md @@ -10,7 +10,8 @@ Every path above exists in this repository, so the three commands run as written. `validate` finds documents by their `.json` extension and reports a path that yields none, so a mistyped path fails rather than passing silently. -The conversion commands operate on one snapshot/document. `validate` runs the +With a file input, the conversion commands operate on one snapshot/document. +`to-v1` also accepts an authoring bundle directory (see below). `validate` runs the one-document codec over every `.json` document either way, so the derived-id reservation (`type-` and `participant-` ids belong to the matching documents, SPEC §9) is checked on a lone document as well as inside a @@ -23,30 +24,91 @@ a bundled key is exempt, since every reader carries the shipped table), and file bindings. A directory without `index.json` is treated as a collection of independent documents. +A valid bundle's own account of what it could not carry (`unresolved`, SPEC +§2c) is printed to stderr as `info: …` (a target the source space deleted) or +`warning: …` (one it holds and did not export, or one it had no row for) +lines, one per declared target, and the bundle still reports `ok bundle`. +`validate -strict` fails on warnings for CI over authored bundles; info never +fails. An undeclared dangling target is an error either way. A type the +documents name that no type document carries prints as a warning when the +index declares it under `unresolved.types` (a reader imports the object as a +Page) and as an error otherwise. + `to-v2 -include-file-remote` preserves a file object's remote CID, encryption keys, and optional indexed variant metadata in the base64 `file_remote` field. It defaults to false. `to-v1` restores a supported payload automatically and warns when it ignores a malformed or future version. -These commands convert one document; a bundle can carry its source -`network_id` in `index.json` for import to assess remote recovery (SPEC §2h). +Single-document conversion handles the snapshot only; a bundle can carry its +source `network_id` in `index.json` for import to assess remote recovery (SPEC §2h). The identifier's value is not validated during export or bundle validation. Neither conversion command downloads file bytes. -## What a round trip does not carry +## Convert an authoring bundle to v1 + +```sh +go run ./cmd/anyblock to-v1 -in ./format/v2/examples/habit_tracker -out /tmp/habit-tracker-v1 +go run ./cmd/anyblock to-v1 -in ./format/v2/examples/habit_tracker -out /tmp/habit-tracker-v1.zip -zip +``` + +A directory input selects bundle conversion. It must contain `index.json`, +object JSON documents, and `properties.json` when custom properties are used. +All three document kinds must satisfy the **v2 authoring subset**. Full space +exports, file objects, participant documents, and stored identity legends are +outside this mode and are rejected. Input ZIP archives are not supported; +unpack them first. The source bundle is never modified. + +The command validates the complete bundle, plans its type vocabulary, mints +one stored key for each custom property, and converts documents using shared +format, property, type, and option resolvers. Dates become native timestamps; +select values and filters refer to option objects. Declared options retain +colors and order, including unused choices. Additional choices found in +values are created after the declared vocabulary. Equal option names on +different properties remain distinct. + +V2 permits hyphens in custom type keys; Heart's native `ot-` identity +parser does not. Bundle conversion maps those keys to stable 24-hex native +keys and updates every object's type membership. It reports each mapping on +stderr. Type document IDs and links retain their source values. Compatible +type keys are preserved, and every custom type receives a matching native +`uniqueKey` in its details. Import paths that read this field can match by +identity; paths that rebuild it from the snapshot key derive the same value. + +Output contains `objects/`, `types/`, `templates/`, `relations/`, and +`relationsOptions/` snapshots as needed, plus the binary root `profile`. +Templates, default templates, query sources, object relationships, and sidebar +widgets are wired to the bundle's objects. The sidebar is also represented by +a native widget snapshot. Object blocks, icons, covers, and views pass through +the same codec used for single-document conversion. The v1 profile has no +space emoji or description fields; their omission is reported as a warning. + +`-encoding pb` (the default) writes `.pb` snapshots; `-encoding json` writes +protobuf-envelope `.json` snapshots. The `profile` stays binary in both cases. +`-zip` packages the same layout directly at the archive root. The output +extension does not enable ZIP output; use the flag. Custom property and block +IDs are minted on each conversion, so repeated conversions are not byte-identical. + +Bundle output must be a **new path outside the input directory**. Existing +files, directories, and symlinks are refused. Validation and conversion finish +before output writing begins; a failed write removes the new partial output. +`-space-id` remains optional and follows the participant-reference rules below. +For bundles it supplies no live-space lookup: declared types use their bundle +IDs, while built-in types use their bundled addresses. + +## What a single-document round trip does not carry `to-v1` mints a fresh id for every container the format does not name — table rows, columns and their cells. Converting the same document twice therefore produces different bytes, by design: `Options.GenerateId` defaults to a random 24-hex id. The v2 side is byte-stable; only this direction varies. -The CLI converts with the bundled vocabulary and no option resolver, so an -`option_ids` legend is dropped on the way to v1 and cannot be rebuilt on the +Single-document conversion uses the bundled vocabulary and no option resolver, +so an `option_ids` legend is dropped on the way to v1 and cannot be rebuilt on the way back. Property values survive; the legend binding a value's spelling to a stored option id does not. A caller that needs the legend preserved should use the Go API and supply `Options.ResolveOptions`. -It wires no `TypeResolver` either, so it cannot turn a `type-` -reference (SPEC §9) into a space's type object id. With `-space-id` — which +Single-document conversion wires no `TypeResolver` either, so it cannot turn +a `type-` reference (SPEC §9) into a space's type object id. With `-space-id` — which says the document is being read into that space — `to-v1` refuses rather than writing the folded string where an address belongs, the same pre-write refusal folded participant references get. Without `-space-id` the conversion @@ -109,11 +171,19 @@ non-empty `-space-id` is rejected by either direction before the input is read and before an output file or its parent directories are created. These pre-write failures leave an already-existing output file unchanged. -On success, missing output parent directories are created and the output file -is created or replaced. The final filesystem write is not atomic: an error +For single-document conversion, on success, missing output parent directories +are created and the output file is created or replaced. The final filesystem write is not atomic: an error while creating the parent directory or writing the file is outside the pre-write preservation guarantee. -Bundle composition and application-specific property/option resolution live -in the root `bundle` package rather than being hidden inside single-document -conversion. +The root `bundle` package provides bundle validation and v2 export composition. +The `bundle/convert` package supplies the authoring-to-v1 adapter, offline +resolvers, and native archive metadata described above. The CLI calls the same +package as Heart's importer; single-document conversion remains unchanged. + +Full space exports can also be converted with `to-v1 -full -in SPACE_EXPORT +-out NATIVE_OUTPUT [-zip] -space-id DESTINATION_SPACE`. This uses +`bundle/convert.Bundle`, including stored definitions, participants, manifest +attachments, remote file metadata, and space settings. Without `-full`, bundle +conversion retains the strict authoring subset. Attachments are streamed to the +output; bytes omitted from the export still need source-network access. diff --git a/cmd/anyblock/bundle.go b/cmd/anyblock/bundle.go new file mode 100644 index 0000000..ed559d2 --- /dev/null +++ b/cmd/anyblock/bundle.go @@ -0,0 +1,158 @@ +package main + +import ( + "archive/zip" + "bytes" + "errors" + "fmt" + bundleconvert "github.com/anyproto/any-block/bundle/convert" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" +) + +// Bundle conversion is an authoring operation. Exported space backups have +// additional identity, file and provenance requirements and are deliberately +// refused by ValidateAuthoring rather than partially reconstructed here. +func runToV1Bundle(input, output, encoding, spaceID string, zipped bool) error { + if encoding != "pb" && encoding != "json" { + return fmt.Errorf("unknown encoding %q: use pb or json", encoding) + } + if err := checkBundleOutput(input, output); err != nil { + return err + } + root, err := os.OpenRoot(input) + if err != nil { + return fmt.Errorf("open bundle: %w", err) + } + defer root.Close() + entries, count, err := convertAuthoringBundle(root.FS(), encoding, spaceID) + if err != nil { + return fmt.Errorf("convert authoring bundle: %w", err) + } + if err := writeV1Bundle(output, entries, zipped); err != nil { + return err + } + fmt.Printf("converted %d authored objects into %d v1 snapshots plus profile: %s\n", count, len(entries)-1, output) + return nil +} + +func convertAuthoringBundle(fsys fs.FS, encoding, spaceID string) (map[string][]byte, int, error) { + result, err := bundleconvert.Authoring(fsys, bundleconvert.Options{ + Encoding: encoding, SpaceID: spaceID, + OnWarning: func(message string) { fmt.Fprintf(cliWarningOutput, "warning: %s\n", message) }, + }) + if err != nil { + return nil, 0, err + } + return result.Entries, result.Documents, nil +} + +func sortedBundleNames[V any](entries map[string]V) []string { + names := make([]string, 0, len(entries)) + for name := range entries { + names = append(names, name) + } + sort.Strings(names) + return names +} + +func checkBundleOutput(input, output string) error { + if _, err := os.Lstat(output); err == nil { + return fmt.Errorf("bundle output already exists: %s", output) + } else if !errors.Is(err, fs.ErrNotExist) { + return err + } + input, err := filepath.EvalSymlinks(input) + if err != nil { + return err + } + input, err = filepath.Abs(input) + if err != nil { + return err + } + output, err = filepath.Abs(output) + if err != nil { + return err + } + // Resolve the nearest existing parent, including output paths whose parent + // directories do not exist yet, to catch symlink aliases back into input. + parent := output + var tail []string + for { + resolved, err := filepath.EvalSymlinks(parent) + if err == nil { + output = resolved + for i := len(tail) - 1; i >= 0; i-- { + output = filepath.Join(output, tail[i]) + } + break + } + if !errors.Is(err, fs.ErrNotExist) { + return err + } + tail = append(tail, filepath.Base(parent)) + parent = filepath.Dir(parent) + } + relative, err := filepath.Rel(input, output) + if err != nil { + return err + } + if relative == "." || (relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))) { + return fmt.Errorf("bundle output must be outside the input directory") + } + return nil +} + +func writeV1Bundle(output string, entries map[string][]byte, zipped bool) (err error) { + names := sortedBundleNames(entries) + if zipped { + var buffer bytes.Buffer + archive := zip.NewWriter(&buffer) + for _, name := range names { + writer, err := archive.Create(name) + if err != nil { + return err + } + if _, err := writer.Write(entries[name]); err != nil { + return err + } + } + if err := archive.Close(); err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(output), 0o755); err != nil { + return err + } + file, err := os.OpenFile(output, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return err + } + _, writeErr := file.Write(buffer.Bytes()) + closeErr := file.Close() + if err := errors.Join(writeErr, closeErr); err != nil { + _ = os.Remove(output) + return err + } + return nil + } + if err := os.MkdirAll(filepath.Dir(output), 0o755); err != nil { + return err + } + if err := os.Mkdir(output, 0o755); err != nil { + return err + } + defer func() { + if err != nil { + _ = os.RemoveAll(output) + } + }() + for _, name := range names { + if err := writeOutput(filepath.Join(output, filepath.FromSlash(name)), entries[name]); err != nil { + return err + } + } + return nil +} diff --git a/cmd/anyblock/bundle_full.go b/cmd/anyblock/bundle_full.go new file mode 100644 index 0000000..8a5140a --- /dev/null +++ b/cmd/anyblock/bundle_full.go @@ -0,0 +1,87 @@ +package main + +import ( + "archive/zip" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + bundleconvert "github.com/anyproto/any-block/bundle/convert" +) + +func runToV1FullBundle(input, output, encoding, spaceID string, zipped bool) (err error) { + if err = checkBundleOutput(input, output); err != nil { + return err + } + root, err := os.OpenRoot(input) + if err != nil { + return err + } + defer root.Close() + result, err := bundleconvert.Bundle(root.FS(), bundleconvert.Options{Encoding: encoding, SpaceID: spaceID, OnWarning: func(message string) { fmt.Fprintf(cliWarningOutput, "warning: %s\n", message) }}) + if err != nil { + return fmt.Errorf("convert full bundle: %w", err) + } + if err = os.MkdirAll(filepath.Dir(output), 0755); err != nil { + return err + } + var archive *zip.Writer + var target *os.File + if zipped { + target, err = os.OpenFile(output, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) + if err != nil { + return err + } + archive = zip.NewWriter(target) + } else if err = os.Mkdir(output, 0755); err != nil { + return err + } + defer func() { + if archive != nil { + err = errors.Join(err, archive.Close(), target.Close()) + } + if err != nil { + _ = os.RemoveAll(output) + } + }() + write := func(name string, copyData func(io.Writer) error) error { + if archive != nil { + writer, e := archive.Create(name) + if e != nil { + return e + } + return copyData(writer) + } + name = filepath.Join(output, filepath.FromSlash(name)) + if e := os.MkdirAll(filepath.Dir(name), 0755); e != nil { + return e + } + file, e := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) + if e != nil { + return e + } + return errors.Join(copyData(file), file.Close()) + } + for _, name := range sortedBundleNames(result.Entries) { + if err = write(name, func(w io.Writer) error { _, e := w.Write(result.Entries[name]); return e }); err != nil { + return err + } + } + for _, name := range sortedBundleNames(result.Files) { + if err = write(name, func(w io.Writer) error { + f, e := root.Open(result.Files[name]) + if e != nil { + return e + } + defer f.Close() + _, e = io.Copy(w, f) + return e + }); err != nil { + return err + } + } + fmt.Printf("converted %d documents and %d attachments: %s\n", result.Documents, len(result.Files), output) + return nil +} diff --git a/cmd/anyblock/bundle_full_test.go b/cmd/anyblock/bundle_full_test.go new file mode 100644 index 0000000..f188496 --- /dev/null +++ b/cmd/anyblock/bundle_full_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "archive/zip" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestToV1FullBundleStreamsAttachments(t *testing.T) { + input := "../../format/v2/examples/exported_space" + expected, err := os.ReadFile(filepath.Join(input, "files/ridge.png")) + require.NoError(t, err) + for _, zipped := range []bool{false, true} { + output := filepath.Join(t.TempDir(), "native") + args := []string{"-full", "-in", input, "-out", output, "-space-id", "root.suffix"} + if zipped { + args = append(args, "-zip") + } + require.NoError(t, runToV1(args)) + name := "files/bafyreiridgephoto/ridge.png" + if zipped { + archive, err := zip.OpenReader(output) + require.NoError(t, err) + reader, err := archive.Open(name) + require.NoError(t, err) + actual, err := io.ReadAll(reader) + require.NoError(t, err) + require.Equal(t, expected, actual) + require.NoError(t, reader.Close()) + require.NoError(t, archive.Close()) + } else { + actual, err := os.ReadFile(filepath.Join(output, name)) + require.NoError(t, err) + require.Equal(t, expected, actual) + } + require.ErrorContains(t, runToV1(args), "already exists") + } +} diff --git a/cmd/anyblock/bundle_test.go b/cmd/anyblock/bundle_test.go new file mode 100644 index 0000000..2b39609 --- /dev/null +++ b/cmd/anyblock/bundle_test.go @@ -0,0 +1,352 @@ +package main + +import ( + "archive/zip" + "bytes" + bundleconvert "github.com/anyproto/any-block/bundle/convert" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "testing/fstest" + "time" + + env "github.com/anyproto/any-block/codec/anyblockjson/envelope" + "github.com/anyproto/any-block/format/v1/model" + "github.com/gogo/protobuf/jsonpb" + "github.com/gogo/protobuf/proto" + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type bundleProfile = bundleconvert.Profile + +func authoringBundleFixture() fstest.MapFS { + documents := map[string]string{ + "index.json": `{"formatVersion":"2.0","name":"Weekends","entrypoint":"home","widgets":[{"target":"home"},{"target":"type-trip","layout":"view","limit":6},{"target":"_recent_open","layout":"compact_list","limit":4}]}`, + "properties.json": `{"formatVersion":"2.0","properties":[ + {"name":"Budget","format":"number"}, + {"name":"State","format":"select","options":[{"name":"Booked","color":"blue"},{"name":"Dream","color":"yellow"}]}, + {"name":"Cost state","format":"select","options":["Booked"]}, + {"name":"When","format":"date","include_time":false}, + {"name":"Expenses","format":"objects","object_types":["Expense"]}, + {"name":"Trip","format":"objects","object_types":["Trip"]}, + {"name":"Tags","format":"multi_select","options":[{"name":"Art","color":"purple"},"Water"]} + ]}`, + "types/trip.json": `{"formatVersion":"2.0","kind":"object_type","id":"type-trip","internal_key":"trip","properties":{"Name":"Trip"},"type_settings":{"default_template":"template-trip","property_definitions":[{"property":"Name"},{"property":"Budget","section":"featured"},{"property":"State"},{"property":"When"},{"property":"Expenses"},{"property":"Tags"}]},"blocks":[{"type":"paragraph","text":"Plan a weekend."}]}`, + "types/expense.json": `{"formatVersion":"2.0","kind":"object_type","id":"type-expense","internal_key":"expense","properties":{"Name":"Expense"},"type_settings":{"property_definitions":[{"property":"Cost state"},{"property":"Trip"}]}}`, + "templates/trip.json": `{"formatVersion":"2.0","kind":"template","type":"template","template_for":"Trip","id":"template-trip","properties":{"Name":"New weekend","State":"Booked","When":"2026-09-18T00:00:00Z"},"blocks":[{"type":"paragraph","text":"Pick a destination."}]}`, + "objects/home.json": `{"formatVersion":"2.0","id":"home","type":"Query","properties":{"Name":"Weekends"},"query_source":{"types":["Trip"],"properties":["Budget"]},"blocks":[{"type":"dataview","properties":[{"property":"State","format":"select"},{"property":"Budget","format":"number"}],"views":[{"name":"Booked weekends","type":"table","columns":[{"property":"State"},{"property":"Budget"}],"filters":[{"property":"State","condition":"in","value":["Booked"]}]}]}]}`, + "objects/trip.json": `{"formatVersion":"2.0","id":"weekend","type":"Trip","properties":{"Name":"Prague","Budget":750.5,"State":"Booked","When":"2026-09-18T00:00:00Z","Expenses":["hotel"],"Tags":["Art","Music"]},"icon":{"format":"emoji","emoji":"🚆"},"cover":{"format":"gradient","gradient":"sky"},"blocks":[{"type":"heading_2","text":"Saturday"},{"type":"paragraph","text":"Walk across Charles Bridge."},{"type":"link","object_id":"hotel"}]}`, + "objects/expense.json": `{"formatVersion":"2.0","id":"hotel","type":"Expense","properties":{"Name":"Two hotel nights","Cost state":"Booked","Trip":["weekend"]},"blocks":[{"type":"paragraph","text":"A room for two near the old town."}]}`, + } + out := fstest.MapFS{} + for name, data := range documents { + out[name] = &fstest.MapFile{Data: []byte(data), Mode: 0o644} + } + return out +} + +func writeAuthoringFixture(t *testing.T, fixture fstest.MapFS) string { + t.Helper() + dir := t.TempDir() + for name, file := range fixture { + require.NoError(t, writeOutput(filepath.Join(dir, filepath.FromSlash(name)), file.Data)) + } + return dir +} + +func decodeBundleSnapshots(t *testing.T, entries map[string][]byte, encoding string) map[string]*env.SnapshotWithType { + t.Helper() + out := map[string]*env.SnapshotWithType{} + for name, data := range entries { + if name == "profile" { + continue + } + wrapper := &env.SnapshotWithType{} + if encoding == "pb" { + require.NoError(t, proto.Unmarshal(data, wrapper)) + } else { + require.NoError(t, jsonpb.Unmarshal(bytes.NewReader(data), wrapper)) + } + require.NotNil(t, wrapper.Snapshot) + require.NotNil(t, wrapper.Snapshot.Data) + id := wrapper.Snapshot.Data.Details.Fields["id"].GetStringValue() + require.NotEmpty(t, id) + require.NotContains(t, out, id) + out[id] = wrapper + } + return out +} + +func TestToV1AuthoringBundle(t *testing.T) { + for _, tc := range []struct { + encoding string + zip bool + }{{"pb", true}, {"json", false}} { + t.Run(tc.encoding, func(t *testing.T) { + warnings := captureCLIWarnings(t) + input := writeAuthoringFixture(t, authoringBundleFixture()) + output := filepath.Join(t.TempDir(), "converted") + args := []string{"to-v1", "-in", input, "-out", output, "-encoding", tc.encoding, "-space-id", testSpaceID} + if tc.zip { + args = append(args, "-zip") + } + require.NoError(t, run(args)) + assert.NotContains(t, warnings.String(), "folded") + entries := map[string][]byte{} + if tc.zip { + archive, err := zip.OpenReader(output) + require.NoError(t, err) + defer archive.Close() + for _, file := range archive.File { + reader, err := file.Open() + require.NoError(t, err) + data, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + entries[file.Name] = data + } + } else { + require.NoError(t, filepath.WalkDir(output, func(name string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + rel, err := filepath.Rel(output, name) + if err != nil { + return err + } + data, err := os.ReadFile(name) + entries[filepath.ToSlash(rel)] = data + return err + })) + } + snapshots := decodeBundleSnapshots(t, entries, tc.encoding) + byName := map[string]*model.SmartBlockSnapshotBase{} + for _, wrapper := range snapshots { + if wrapper.SbType == model.SmartBlockType_STRelation { + s := wrapper.Snapshot.Data + byName[s.Details.Fields["name"].GetStringValue()] = s + } + } + require.Len(t, byName, 7) + key := func(name string) string { require.Contains(t, byName, name); return byName[name].Key } + for _, s := range byName { + assert.Regexp(t, "^[0-9a-f]{24}$", s.Key) + assert.Equal(t, s.Key, s.Details.Fields["relationKey"].GetStringValue()) + } + value := func(id, property string) *types.Value { + require.Contains(t, snapshots, id) + return snapshots[id].Snapshot.Data.Details.Fields[key(property)] + } + assert.Equal(t, 750.5, value("weekend", "Budget").GetNumberValue()) + date, err := time.Parse(time.RFC3339, "2026-09-18T00:00:00Z") + require.NoError(t, err) + assert.Equal(t, float64(date.Unix()), value("weekend", "When").GetNumberValue()) + assert.Equal(t, float64(date.Unix()), value("template-trip", "When").GetNumberValue()) + assert.False(t, byName["When"].Details.Fields["relationFormatIncludeTime"].GetBoolValue()) + assert.Equal(t, 0.0, byName["Expenses"].Details.Fields["relationMaxCount"].GetNumberValue()) + assert.Equal(t, "type-expense", byName["Expenses"].Details.Fields["relationFormatObjectTypes"].GetListValue().Values[0].GetStringValue()) + assert.Equal(t, "hotel", value("weekend", "Expenses").GetListValue().Values[0].GetStringValue()) + assert.Equal(t, "weekend", value("hotel", "Trip").GetListValue().Values[0].GetStringValue()) + booked := value("weekend", "State").GetListValue().Values[0].GetStringValue() + costBooked := value("hotel", "Cost state").GetListValue().Values[0].GetStringValue() + assert.NotEqual(t, booked, costBooked) + assert.Equal(t, "Booked", snapshots[booked].Snapshot.Data.Details.Fields["name"].GetStringValue()) + assert.Equal(t, "blue", snapshots[booked].Snapshot.Data.Details.Fields["relationOptionColor"].GetStringValue()) + assert.Equal(t, booked, value("template-trip", "State").GetListValue().Values[0].GetStringValue()) + optionCount := 0 + for _, wrapper := range snapshots { + if wrapper.SbType != model.SmartBlockType_STRelationOption { + continue + } + optionCount++ + d := wrapper.Snapshot.Data.Details.Fields + if d["name"].GetStringValue() == "Dream" { + assert.Equal(t, "yellow", d["relationOptionColor"].GetStringValue()) + assert.Greater(t, d["orderId"].GetStringValue(), snapshots[booked].Snapshot.Data.Details.Fields["orderId"].GetStringValue()) + } + } + assert.Equal(t, 6, optionCount, "unused declared and newly used options are both emitted") + assert.Equal(t, 20, len(snapshots)) + tripType := snapshots["type-trip"].Snapshot.Data + assert.Equal(t, []string{"ot-objectType"}, tripType.ObjectTypes) + assert.Equal(t, "ot-trip", tripType.Details.Fields["uniqueKey"].GetStringValue()) + assert.Equal(t, "template-trip", tripType.Details.Fields["defaultTemplateId"].GetStringValue()) + assert.Equal(t, "rel-"+key("Budget"), tripType.Details.Fields["recommendedFeaturedRelations"].GetListValue().Values[0].GetStringValue()) + assert.Equal(t, "rel-name", tripType.Details.Fields["recommendedRelations"].GetListValue().Values[0].GetStringValue()) + assert.Equal(t, "type-trip", snapshots["template-trip"].Snapshot.Data.Details.Fields["targetObjectType"].GetStringValue()) + weekend := snapshots["weekend"].Snapshot.Data + assert.Equal(t, []string{"ot-trip"}, weekend.ObjectTypes) + assert.Equal(t, "🚆", weekend.Details.Fields["iconEmoji"].GetStringValue()) + assert.Equal(t, "sky", weekend.Details.Fields["coverId"].GetStringValue()) + texts := []string{} + for _, b := range weekend.Blocks { + if b.GetText() != nil { + texts = append(texts, b.GetText().Text) + } + if b.GetLink() != nil { + assert.Equal(t, "hotel", b.GetLink().TargetBlockId) + } + } + assert.Equal(t, []string{"Saturday", "Walk across Charles Bridge."}, texts) + home := snapshots["home"].Snapshot.Data + setOf := home.Details.Fields["setOf"].GetListValue().Values + assert.Equal(t, "type-trip", setOf[0].GetStringValue()) + assert.Equal(t, "rel-"+key("Budget"), setOf[1].GetStringValue()) + for _, b := range home.Blocks { + if b.GetDataview() == nil { + continue + } + view := b.GetDataview().Views[0] + assert.Equal(t, key("State"), view.Relations[0].Key) + assert.Equal(t, key("State"), view.Filters[0].RelationKey) + assert.Equal(t, booked, view.Filters[0].Value.GetListValue().Values[0].GetStringValue()) + } + profile := &bundleProfile{} + require.NoError(t, proto.Unmarshal(entries["profile"], profile)) + assert.Equal(t, "Weekends", profile.Name) + assert.Equal(t, "home", profile.Homepage) + require.Len(t, profile.Widgets, 3) + assert.Equal(t, "recentOpen", profile.Widgets[2].Target) + assert.Equal(t, int32(model.BlockContentWidget_CompactList), profile.Widgets[2].Layout) + assert.Contains(t, snapshots, "widgets") + }) + } +} + +func TestBundleConversionRejectsBeforeWriting(t *testing.T) { + for _, test := range []struct{ name, path, data string }{ + {"missing target", "index.json", `{"formatVersion":"2.0","name":"Bad","entrypoint":"missing"}`}, + {"exported document", "objects/export.json", `{"formatVersion":"2.0","kind":"participant","id":"participant-person","properties":{"Name":"Person"}}`}, + {"unknown type", "objects/unknown.json", `{"formatVersion":"2.0","id":"unknown","type":"Typo","properties":{"Name":"Unknown"}}`}, + {"exported dictionary", "properties.json", `{"formatVersion":"2.0","properties":[{"internal_key":"stored-key","name":"Budget","format":"number"}]}`}, + } { + t.Run(test.name, func(t *testing.T) { + captureCLIWarnings(t) + fixture := authoringBundleFixture() + fixture[test.path] = &fstest.MapFile{Data: []byte(test.data)} + input := writeAuthoringFixture(t, fixture) + parent := filepath.Join(t.TempDir(), "not-created") + err := runToV1([]string{"-in", input, "-out", filepath.Join(parent, "bundle"), "-zip"}) + require.Error(t, err) + _, err = os.Stat(parent) + assert.ErrorIs(t, err, fs.ErrNotExist) + }) + } +} + +func TestBundleOutputProtection(t *testing.T) { + input := writeAuthoringFixture(t, authoringBundleFixture()) + output := filepath.Join(t.TempDir(), "existing") + require.NoError(t, os.WriteFile(output, []byte("keep me"), 0o644)) + for _, extra := range [][]string{nil, {"-zip"}} { + args := append([]string{"-in", input, "-out", output}, extra...) + require.ErrorContains(t, runToV1(args), "already exists") + } + data, err := os.ReadFile(output) + require.NoError(t, err) + assert.Equal(t, "keep me", string(data)) + require.ErrorContains(t, runToV1([]string{"-in", input, "-out", filepath.Join(input, "new", "output")}), "outside") + alias := filepath.Join(t.TempDir(), "alias") + require.NoError(t, os.Symlink(input, alias)) + require.ErrorContains(t, runToV1([]string{"-in", input, "-out", filepath.Join(alias, "new", "output")}), "outside") + require.ErrorContains(t, runToV1([]string{"-in", input, "-out", filepath.Join(t.TempDir(), "bad"), "-encoding", "auto"}), "unknown encoding") + require.ErrorContains(t, runToV1([]string{"-in", filepath.Join(input, "objects", "trip.json"), "-out", filepath.Join(t.TempDir(), "bad"), "-zip"}), "directory") +} + +func TestBundleDoesNotFollowSymlinkDocuments(t *testing.T) { + input := writeAuthoringFixture(t, authoringBundleFixture()) + require.NoError(t, os.Symlink(filepath.Join(input, "objects", "trip.json"), filepath.Join(input, "linked.json"))) + require.Error(t, runToV1([]string{"-in", input, "-out", filepath.Join(t.TempDir(), "result")})) +} + +func TestBundleConvertsExistingAuthoringExample(t *testing.T) { + captureCLIWarnings(t) + entries, count, err := convertAuthoringBundle(os.DirFS(filepath.Join("..", "..", "format", "v2", "examples", "habit_tracker")), "pb", "") + require.NoError(t, err) + assert.Positive(t, count) + snapshots := decodeBundleSnapshots(t, entries, "pb") + assert.Contains(t, snapshots, "type-habit") + assert.Contains(t, snapshots, "page-start") +} + +func TestBundleRejectsOutputCaseCollision(t *testing.T) { + fixture := authoringBundleFixture() + fixture["objects/other.json"] = &fstest.MapFile{Data: []byte(`{"formatVersion":"2.0","id":"Weekend","type":"Trip","properties":{"Name":"Another weekend"}}`)} + captureCLIWarnings(t) + _, _, err := convertAuthoringBundle(fixture, "pb", "") + require.ErrorContains(t, err, "case-insensitive") +} + +func TestBundleWithoutDictionaryOrWidgets(t *testing.T) { + fixture := fstest.MapFS{ + "index.json": {Data: []byte(`{"formatVersion":"2.0","name":"Notes","entrypoint":"home"}`)}, + "home.json": {Data: []byte(`{"formatVersion":"2.0","id":"home","type":"Page","properties":{"Name":"Notes"},"blocks":[{"type":"paragraph","text":"Start here."}]}`)}, + } + entries, count, err := convertAuthoringBundle(fixture, "pb", "") + require.NoError(t, err) + assert.Equal(t, 1, count) + assert.Len(t, entries, 2) + profile := &bundleProfile{} + require.NoError(t, proto.Unmarshal(entries["profile"], profile)) + assert.Equal(t, "home", profile.Homepage) + assert.Empty(t, profile.Widgets) +} + +func TestBundleHyphenatedTypeKeysUseNativeIdentities(t *testing.T) { + fixture := authoringBundleFixture() + // Both custom types now use legal v2 keys that Heart cannot parse inside + // its native ot- identity. Built-in Task and template keys stay intact. + for _, file := range fixture { + data := strings.NewReplacer( + `"type-trip"`, `"type-bookclub-meeting"`, + `"internal_key":"trip"`, `"internal_key":"bookclub-meeting"`, + `"type-expense"`, `"type-bookclub-person"`, + `"internal_key":"expense"`, `"internal_key":"bookclub-person"`, + ).Replace(string(file.Data)) + file.Data = []byte(data) + } + fixture["objects/task.json"] = &fstest.MapFile{Data: []byte(`{"formatVersion":"2.0","id":"task","type":"Task","properties":{"Name":"Bring a book"}}`)} + warnings := captureCLIWarnings(t) + entries, _, err := convertAuthoringBundle(fixture, "pb", "") + require.NoError(t, err) + snapshots := decodeBundleSnapshots(t, entries, "pb") + meeting := snapshots["type-bookclub-meeting"].Snapshot.Data + person := snapshots["type-bookclub-person"].Snapshot.Data + for _, typeSnapshot := range []*model.SmartBlockSnapshotBase{meeting, person} { + assert.Regexp(t, `^[0-9a-f]{24}$`, typeSnapshot.Key) + uniqueKey := typeSnapshot.Details.Fields["uniqueKey"].GetStringValue() + assert.Equal(t, "ot-"+typeSnapshot.Key, uniqueKey) + assert.Len(t, strings.Split(uniqueKey, "-"), 2, "Heart's UnmarshalUniqueKey accepts exactly one separator") + } + assert.NotEqual(t, meeting.Key, person.Key) + assert.Equal(t, []string{"ot-" + meeting.Key}, snapshots["weekend"].Snapshot.Data.ObjectTypes) + assert.Equal(t, []string{"ot-" + person.Key}, snapshots["hotel"].Snapshot.Data.ObjectTypes) + assert.Equal(t, []string{"ot-task"}, snapshots["task"].Snapshot.Data.ObjectTypes) + template := snapshots["template-trip"].Snapshot.Data + assert.Equal(t, []string{"ot-template", "ot-" + meeting.Key}, template.ObjectTypes) + assert.Equal(t, "type-bookclub-meeting", template.Details.Fields["targetObjectType"].GetStringValue()) + assert.Equal(t, "type-bookclub-meeting", snapshots["home"].Snapshot.Data.Details.Fields["setOf"].GetListValue().Values[0].GetStringValue()) + for _, wrapper := range snapshots { + if wrapper.SbType != model.SmartBlockType_STRelation { + continue + } + details := wrapper.Snapshot.Data.Details.Fields + if details["name"].GetStringValue() == "Expenses" { + assert.Equal(t, "type-bookclub-person", details["relationFormatObjectTypes"].GetListValue().Values[0].GetStringValue()) + } + } + assert.Contains(t, warnings.String(), `internal key "bookclub-meeting"`) + assert.Contains(t, warnings.String(), `internal key "bookclub-person"`) + again, _, err := convertAuthoringBundle(fixture, "pb", "") + require.NoError(t, err) + repeated := decodeBundleSnapshots(t, again, "pb") + assert.Equal(t, meeting.Key, repeated["type-bookclub-meeting"].Snapshot.Data.Key, "repeated conversion keeps the native type identity") +} diff --git a/cmd/anyblock/main.go b/cmd/anyblock/main.go index 0f549cd..685c8ad 100644 --- a/cmd/anyblock/main.go +++ b/cmd/anyblock/main.go @@ -59,13 +59,22 @@ func run(args []string) error { } } -func runValidate(paths []string) error { - if len(paths) == 1 && (paths[0] == "-h" || paths[0] == "--help") { - fmt.Fprintln(os.Stdout, "usage: anyblock validate ...") +func runValidate(args []string) error { + if len(args) == 1 && (args[0] == "-h" || args[0] == "--help") { + fmt.Fprintln(os.Stdout, "usage: anyblock validate [-strict] ...") return nil } + flags := flag.NewFlagSet("validate", flag.ContinueOnError) + strict := flags.Bool("strict", false, "fail on warnings too (a declared omitted or absent target); info never fails") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + return err + } + paths := flags.Args() if len(paths) == 0 { - return fmt.Errorf("usage: anyblock validate ...") + return fmt.Errorf("usage: anyblock validate [-strict] ...") } failures := 0 for _, root := range paths { @@ -79,9 +88,29 @@ func runValidate(paths []string) error { return err } if info.IsDir() { - err = validateBundleDirectory(root) + var report *anyblockbundle.Report + report, err = validateBundleDirectory(root) switch { case err == nil: + // what a valid bundle states about itself (§2c): printed at + // the severity its class earns, and failing only under -strict + // and only for warnings — info is a tombstone the space still + // names, which is by design + warned := false + for _, issue := range report.Issues { + if issue.Severity == anyblockbundle.SeverityError { + continue + } + if issue.Severity == anyblockbundle.SeverityWarning { + warned = true + } + fmt.Fprintf(cliWarningOutput, "%s: %s\n", issue.Severity, issue.Message) + } + if *strict && warned { + failures++ + fmt.Fprintf(os.Stderr, "invalid bundle %s: warnings are errors under -strict\n", root) + continue + } fmt.Printf("ok bundle %s\n", root) continue case !errors.Is(err, anyblockbundle.ErrIndexNotFound): @@ -121,17 +150,28 @@ func runValidate(paths []string) error { return nil } -func validateBundleDirectory(name string) (err error) { +// validateBundleDirectory inspects a bundle rooted at name. A nil error +// with a report means the bundle is valid and the report carries what it +// states about itself; an error is either a refusal (report.Err) or a walk +// that could not run. +func validateBundleDirectory(name string) (report *anyblockbundle.Report, err error) { root, err := os.OpenRoot(name) if err != nil { - return fmt.Errorf("open bundle root %s: %w", name, err) + return nil, fmt.Errorf("open bundle root %s: %w", name, err) } defer func() { if closeErr := root.Close(); err == nil && closeErr != nil { err = fmt.Errorf("close bundle root %s: %w", name, closeErr) } }() - return anyblockbundle.Validate(root.FS()) + report, err = anyblockbundle.Inspect(root.FS()) + if err != nil { + return nil, err + } + if err := report.Err(); err != nil { + return nil, err + } + return report, nil } func validateFile(path string) error { @@ -153,9 +193,11 @@ func validateFile(path string) error { func runToV1(args []string) error { flags := flag.NewFlagSet("to-v1", flag.ContinueOnError) flags.SetOutput(os.Stdout) - in := flags.String("in", "", "AnyBlock v2 object document") - out := flags.String("out", "", "AnyBlock v1 snapshot envelope") + in := flags.String("in", "", "AnyBlock v2 object document or authoring bundle directory") + out := flags.String("out", "", "AnyBlock v1 snapshot, bundle directory, or ZIP with -zip") encoding := flags.String("encoding", "pb", "output encoding: pb or json") + full := flags.Bool("full", false, "convert full space exports, including attachments and stored definitions") + zipOutput := flags.Bool("zip", false, "write a bundle as a ZIP archive (directory input only)") spaceID := flags.String("space-id", "", "space receiving the v1 snapshot (required to rebuild folded participant references)") if err := flags.Parse(args); err != nil { if errors.Is(err, flag.ErrHelp) { @@ -169,6 +211,22 @@ func runToV1(args []string) error { if err := validateOptionalSpaceID(*spaceID); err != nil { return err } + info, err := os.Stat(*in) + if err != nil { + return fmt.Errorf("read input: %w", err) + } + if info.IsDir() { + if *full { + return runToV1FullBundle(*in, *out, *encoding, *spaceID, *zipOutput) + } + return runToV1Bundle(*in, *out, *encoding, *spaceID, *zipOutput) + } + if *full { + return fmt.Errorf("-full requires a bundle directory as -in") + } + if *zipOutput { + return fmt.Errorf("-zip requires a bundle directory as -in") + } data, err := os.ReadFile(*in) if err != nil { return fmt.Errorf("read input: %w", err) @@ -188,12 +246,21 @@ func runToV1(args []string) error { if err := outcome.preWriteError(); err != nil { return err } + output, err := encodeV1Snapshot(sbType, snapshot, *encoding) + if err != nil { + return err + } + return writeOutput(*out, output) +} + +func encodeV1Snapshot(sbType model.SmartBlockType, snapshot *model.SmartBlockSnapshotBase, encoding string) ([]byte, error) { envelope := &envelopepb.SnapshotWithType{ SbType: sbType, Snapshot: &envelopepb.ChangeSnapshot{Data: snapshot}, } var output []byte - switch *encoding { + var err error + switch encoding { case "pb": output, err = proto.Marshal(envelope) case "json": @@ -202,12 +269,12 @@ func runToV1(args []string) error { text, err = (&jsonpb.Marshaler{Indent: " "}).MarshalToString(envelope) output = []byte(text) default: - return fmt.Errorf("unknown encoding %q: use pb or json", *encoding) + return nil, fmt.Errorf("unknown encoding %q: use pb or json", encoding) } if err != nil { - return fmt.Errorf("encode v1: %w", err) + return nil, fmt.Errorf("encode v1: %w", err) } - return writeOutput(*out, output) + return output, nil } func runToV2(args []string) error { diff --git a/cmd/anyblock/validatereport_test.go b/cmd/anyblock/validatereport_test.go new file mode 100644 index 0000000..823f44e --- /dev/null +++ b/cmd/anyblock/validatereport_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeDeclaringBundle(t *testing.T, unresolved string) string { + t.Helper() + dir := t.TempDir() + index := `{"formatVersion":"2.0","entrypoint":"page","homepage":"gone","widgets":[{"target":"page"}],"unresolved":` + unresolved + `}` + require.NoError(t, os.WriteFile(filepath.Join(dir, "index.json"), []byte(index), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "objects"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "objects", "page.json"), []byte(`{"formatVersion":"2.0","id":"page"}`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "properties.json"), []byte(`{"formatVersion":"2.0"}`), 0o644)) + return dir +} + +// A bundle that declares what it cannot carry validates; what it declared +// is printed at the severity its class earns, so the person running the +// command sees the loss without the command failing on it. +func TestValidatePrintsDeclaredTargetsWithoutFailing(t *testing.T) { + dir := writeDeclaringBundle(t, `{"targets":["gone"],"deleted":["gone"]}`) + warnings := captureCLIWarnings(t) + + require.NoError(t, runValidate([]string{dir})) + + assert.Contains(t, warnings.String(), "info: homepage") + assert.Contains(t, warnings.String(), `"gone"`) +} + +// -strict is for CI over authored bundles: a warning fails the command, +// info does not. +func TestValidateStrictFailsOnWarnings(t *testing.T) { + warningDir := writeDeclaringBundle(t, `{"targets":["gone"],"omitted":["gone"]}`) + infoDir := writeDeclaringBundle(t, `{"targets":["gone"],"deleted":["gone"]}`) + captureCLIWarnings(t) + + require.NoError(t, runValidate([]string{warningDir}), "without -strict a warning does not fail") + require.Error(t, runValidate([]string{"-strict", warningDir})) + require.NoError(t, runValidate([]string{"-strict", infoDir}), "info never fails") +} diff --git a/cmd/anyblock/validatesymlinks_test.go b/cmd/anyblock/validatesymlinks_test.go index 4db4825..f7c787c 100644 --- a/cmd/anyblock/validatesymlinks_test.go +++ b/cmd/anyblock/validatesymlinks_test.go @@ -22,7 +22,7 @@ func TestCLIContainsAuthoritativeSymlinkTargets(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(escaping, "index.json"), dictionaryManifestIndex(), 0o644)) require.NoError(t, os.Symlink(outside, filepath.Join(escaping, "dictionary.data"))) - secureErr := validateBundleDirectory(escaping) + _, secureErr := validateBundleDirectory(escaping) require.ErrorContains(t, secureErr, `manifest.properties cannot inspect target "dictionary.data"`) require.ErrorContains(t, secureErr, "path escapes from parent") diff --git a/codec/anyblockjson/authoringtypes.go b/codec/anyblockjson/authoringtypes.go index 8ebd912..b2e6f82 100644 --- a/codec/anyblockjson/authoringtypes.go +++ b/codec/anyblockjson/authoringtypes.go @@ -30,6 +30,14 @@ type AuthoringTypeVocabulary struct { propertyStored map[string]struct{} propertyNames []string typeProperties map[string][]string + // propertySpellings is the dictionary's own statement of how a key is + // SPELLED in this bundle's documents (§2f): each entry's `property` + // member, mapped to the stored key it names. A spelling stated by one + // entry outranks a display NAME another entry happens to carry, which + // is what tells the entry that spells "Tag" apart from the entry that + // is merely named Tag. A spelling two entries state is nobody's and is + // left to the claim count. + propertySpellings map[string]string } // AuthoringVocabularyPlanOptions supplies the bundle-level declarations that @@ -75,6 +83,11 @@ func PlanAuthoringTypeVocabulary( ) (*AuthoringTypeVocabulary, error) { type declaration struct { source, name, key, alias string + // uninstalled: the user removed the type (§2a). It owns its stored + // key — every `type-` reference resolves — but claims no + // spelling, so a live type that took the freed name is not shadowed + // by the corpse. Same rule the store resolver applies on export. + uninstalled bool } paths := make([]string, 0, len(documents)) for source := range documents { @@ -87,6 +100,7 @@ func PlanAuthoringTypeVocabulary( var doc struct { Kind string `json:"kind"` InternalKey string `json:"internal_key"` + Uninstalled bool `json:"uninstalled"` PropertyKeys map[string]string `json:"property_internal_keys"` Properties map[string]json.RawMessage `json:"properties"` } @@ -133,10 +147,11 @@ func PlanAuthoringTypeVocabulary( }}} } declarations = append(declarations, declaration{ - source: source, - name: displayName, - key: doc.InternalKey, - alias: vocabulary.MintApiSlugFromName(displayName), + source: source, + name: displayName, + key: doc.InternalKey, + alias: vocabulary.MintApiSlugFromName(displayName), + uninstalled: doc.Uninstalled, }) } @@ -155,13 +170,14 @@ func PlanAuthoringTypeVocabulary( }) v := &AuthoringTypeVocabulary{ - byKey: make(map[string]string, len(declarations)), - claims: make(map[string][]string, len(declarations)*2), - stored: make(map[string]struct{}, len(declarations)), - propertyByKey: map[string]string{}, - propertyClaims: map[string][]string{}, - propertyStored: map[string]struct{}{}, - typeProperties: map[string][]string{}, + byKey: make(map[string]string, len(declarations)), + claims: make(map[string][]string, len(declarations)*2), + stored: make(map[string]struct{}, len(declarations)), + propertyByKey: map[string]string{}, + propertyClaims: map[string][]string{}, + propertyStored: map[string]struct{}{}, + typeProperties: map[string][]string{}, + propertySpellings: map[string]string{}, } keyOwner := make(map[string]declaration, len(declarations)) var issues []Issue @@ -189,6 +205,9 @@ func PlanAuthoringTypeVocabulary( if _, admitted := v.stored[decl.key]; !admitted { continue } + if decl.uninstalled { + continue + } terms := []struct { term, role string }{{decl.name, "display name"}, {decl.alias, "legacy derived alias"}} @@ -238,6 +257,9 @@ func PlanAuthoringTypeVocabulary( return nil, &ValidationError{Issues: issues} } for _, decl := range declarations { + if decl.uninstalled { + continue + } v.names = append(v.names, decl.name) } sort.Strings(v.names) @@ -314,6 +336,36 @@ func (v *AuthoringTypeVocabulary) planAuthoringProperties( for _, def := range dictionary.Properties { register(string(def.Key), def.Name, true) } + // the entries' own spellings, read off the bytes: the decoded + // definition inverts `property` to a stored key and keeps no record + // of how the entry spelled it + var raw struct { + Properties []struct { + Property string `json:"property"` + InternalKey string `json:"internal_key"` + } `json:"properties"` + } + if json.Unmarshal(dictionaryData, &raw) == nil && len(raw.Properties) == len(dictionary.Properties) { + contested := map[string]bool{} + for i, entry := range raw.Properties { + spelling := nfcTerm(entry.Property) + key := entry.InternalKey + if key == "" { + key = string(dictionary.Properties[i].Key) + } + if spelling == "" || key == "" || spelling == key { + continue + } + if owner, seen := v.propertySpellings[spelling]; seen && owner != key { + contested[spelling] = true + continue + } + v.propertySpellings[spelling] = key + } + for spelling := range contested { + delete(v.propertySpellings, spelling) + } + } } typeDocs := make(map[string]authoringTypePropertyDocument, len(paths)) @@ -363,7 +415,14 @@ func (v *AuthoringTypeVocabulary) planAuthoringProperties( key, ambiguous := v.resolvePlannedProperty(term) if len(ambiguous) > 1 { // The scope cannot identify itself through an already ambiguous - // name. Authors can make this exact with internal_key or a legend. + // name. Authors can make this exact with internal_key or a + // legend — and an export already did: it writes the stored key + // beside every spelling, so a stated internal_key decides here + // (six real declarations spelled `Tag` beside `internal_key: + // "tag"` in a space that also minted a custom Tag). + if property.InternalKey != "" { + register(property.InternalKey, property.Name, false) + } continue } register(key, property.Name, false) @@ -409,6 +468,12 @@ func (v *AuthoringTypeVocabulary) planAuthoringProperties( if key == "" { var ambiguous []string key, ambiguous = v.resolvePlannedProperty(term) + if len(ambiguous) > 1 && property.InternalKey != "" { + // the spelling is contested and the entry states which + // claimant it meant: the stored key is the tie-break, + // exactly the remedy the refusal below names + key, ambiguous = property.InternalKey, nil + } if len(ambiguous) > 1 { return &ValidationError{Issues: []Issue{{ Path: fmt.Sprintf("/type_settings/property_definitions/%d/property", i), @@ -508,6 +573,11 @@ func (v *AuthoringTypeVocabulary) PropertyKeyCandidates(spelling string) []strin spelling = nfcTerm(spelling) var out []string if v != nil { + // the dictionary's own spelling of a key is the one claimant that + // counts: the bundle said, in so many words, what this term means + if key, ok := v.propertySpellings[spelling]; ok { + return []string{key} + } out = append(out, v.propertyClaims[spelling]...) } if key, ok := BundledPropertyKeyByName(spelling); ok { diff --git a/codec/anyblockjson/authoringtypes_test.go b/codec/anyblockjson/authoringtypes_test.go index 0a6ac8b..b97fb22 100644 --- a/codec/anyblockjson/authoringtypes_test.go +++ b/codec/anyblockjson/authoringtypes_test.go @@ -361,3 +361,52 @@ func assertValidationIssueAt(t *testing.T, err error, path string) { require.NotEmpty(t, validationErr.Issues) assert.Equal(t, path, validationErr.Issues[0].Path) } + +// An exported declaration states BOTH the spelling and the stored key. The +// spelling still wins when it resolves — that is the member the document's +// own legend speaks for — but when the spelling is CONTESTED the stated +// `internal_key` is the tie-break, which is the remedy the refusal itself +// names. Six real declarations spelled `Tag` with `internal_key: "tag"` in +// a space that also minted a custom property named Tag, and the pair was +// refused for an ambiguity it was carrying the answer to. +func TestAuthoringVocabularyInternalKeyOutranksAnAmbiguousSpelling(t *testing.T) { + dictionary := []byte(`{"formatVersion":"2.0","properties":[` + + `{"property":"68cdaa41e9223c9dc7ce5f30","internal_key":"68cdaa41e9223c9dc7ce5f30","name":"Tag","format":"select"}]}`) + typeDoc := []byte(`{"formatVersion":"2.0","kind":"object_type","id":"type-6a7b","internal_key":"6a7b","properties":{"Name":"Recipe"},` + + `"type_settings":{"layout":"basic","property_definitions":[{"property":"Tag","internal_key":"tag","name":"Tag","format":"select"}]}}`) + + vocab, err := PlanAuthoringTypeVocabulary( + map[string][]byte{"types/recipe.json": typeDoc}, + AuthoringVocabularyPlanOptions{PropertyDictionary: dictionary, Installed: true}) + + require.NoError(t, err, "the declaration carries its stored key; nothing is ambiguous") + assert.Equal(t, []string{"tag"}, vocab.TypePropertyKeys("6a7b")) +} + +// A dictionary entry's `property` member is the bundle's own statement of +// how that key is SPELLED in its documents (§2f), and it outranks a display +// name another entry happens to carry. The measured space renamed the +// bundled Tag to "Regs" and minted a custom property named Tag: the +// exporter spelled the bundled key `Tag` (the table's spelling) and the +// custom one by its stored key, then 542 object documents spelling `Tag` +// were refused on read as ambiguous between the entry that SPELLS it and +// the entry that is merely NAMED it. +func TestAuthoringVocabulary_ADictionarySpellingOutranksANameClaim(t *testing.T) { + dictionary := []byte(`{"formatVersion":"2.0","properties":[` + + `{"property":"Description","internal_key":"description","name":"Summary","format":"text","bundled_diverged":true},` + + `{"property":"68cdaa41e9223c9dc7ce5f30","internal_key":"68cdaa41e9223c9dc7ce5f30","name":"Description","format":"text"}]}`) + vocab, err := PlanAuthoringTypeVocabulary(map[string][]byte{}, + AuthoringVocabularyPlanOptions{PropertyDictionary: dictionary, Installed: true}) + require.NoError(t, err) + + assert.Equal(t, []string{"description"}, vocab.PropertyKeyCandidates("Description"), + "the entry that spells the term claims it; the entry merely named after it does not") + key, ok := vocab.PropertyKey("Description") + assert.True(t, ok) + assert.Equal(t, "description", key) + + object := []byte(`{"formatVersion":"2.0","id":"o1","type":"Page","properties":{"Description":"the bundled one"}}`) + _, snapshot, err := Unmarshal(object, Options{Keys: vocab}) + require.NoError(t, err, "no ambiguity: the dictionary said what the spelling means") + assert.Equal(t, "the bundled one", snapshot.Details.Fields["description"].GetStringValue()) +} diff --git a/codec/anyblockjson/dictionary.go b/codec/anyblockjson/dictionary.go index 5b91d61..bdbebb2 100644 --- a/codec/anyblockjson/dictionary.go +++ b/codec/anyblockjson/dictionary.go @@ -583,6 +583,13 @@ func dictionaryEntryOmapWithOptions(def PropertyDefinition, opts Options) (*omap } targets := make([]string, 0, len(def.ObjectTypes)) for _, key := range def.ObjectTypes { + // a stored `_missing_object` sentinel in the relation's target list + // drops here exactly as it drops from a document's `object_types` + // (§9): it names no type, and the predicate is the one the document + // slot and the comparator already share + if DroppedMissingObjectRef(opts, key) { + continue + } // a type is named by its derived id wherever a key admits one (§9); // a vocabulary's spelling is the fallback for a key the gate refuses spelling := dictionaryTypeSpelling(key) diff --git a/codec/anyblockjson/export.go b/codec/anyblockjson/export.go index 851914e..f1d27e0 100644 --- a/codec/anyblockjson/export.go +++ b/codec/anyblockjson/export.go @@ -229,7 +229,10 @@ const ( // a template. validate.go's matching refusal died at the freeze — the // version gate answers for every pre-freeze document now (§15 #9). typeKeyTemplate = "template" - storeKeyItems = "objects" + // typeKeyObjectType is the bundled key of the Type type — what every + // type document's own type is, by definition (§2a). + typeKeyObjectType = "objectType" + storeKeyItems = "objects" // codeLangField is the internal fields key holding a code block's // language (§5.1) codeLangField = "lang" @@ -875,7 +878,8 @@ func (e *exporter) recordPropertyKey(term, key string) { return } if bundledBinds(term, key, (BundledKeyVocabulary{}).PropertyKey) && - termInverts(term, key, e.opts.keys().PropertyKey) { + termInverts(term, key, e.opts.keys().PropertyKey) && + !e.contestedPropertySpelling(term, key) { return } if reason, refused := legendEntryRefusal(term, key, true); refused { @@ -933,6 +937,37 @@ func legendEntryRefusal(term, key string, deny bool) (string, bool) { return "", false } +// contestedPropertySpelling reports that the writer's own space has more +// than one live property answering to the spelling — the bundled key the +// table binds it to AND a custom property that carries the same display +// name. The two questions recordPropertyKey asks both say "no entry owed": +// the table binds the spelling, and the writer's vocabulary resolves it +// bundled-first. A reader planning from the bundle's dictionary sees both +// claimants and refuses the document. Measured: a space that renamed the +// bundled Tag to "Regs" and minted a custom property named Tag shipped 542 +// object documents spelling `Tag` with no entry, every one refused on read. +// Only a scoped vocabulary can answer; a package-only writer has no space +// to ask, and its documents were never contested. +// +// The question is whether ANY claimant other than the key being written +// answers to the spelling — not whether the set has two members. A store +// vocabulary lists the space's own claimants and may or may not add the +// bundled binding; the space that measured this had renamed the bundled +// Tag to "Regs", so its set for "Tag" held the custom property alone, and +// a count of two would have said nothing was owed. +func (e *exporter) contestedPropertySpelling(term, key string) bool { + scoped, ok := e.opts.keys().(ScopedKeyVocabulary) + if !ok { + return false + } + for _, candidate := range scoped.PropertyKeyCandidates(term) { + if candidate != key { + return true + } + } + return false +} + // termInverts reports whether `term`, written for the stored key `key` with // NO legend entry, reads back as `key` through one reader's table. It asks // the table the same way the importer does — Options.propertyKey/typeKey @@ -1584,6 +1619,16 @@ func (e *exporter) buildDoc(sbType model.SmartBlockType) (*omap, error) { doc.set("formatVersion", FormatVersion) typeKeys := e.modelledTypeKeys(true) + // a type document IS a Type, so its own type is the bundled objectType + // whatever the store holds — derivable from the kind, like the layout + // keys §2a drops. Three real type objects from an old markdown import + // carried `ot-type`, a key no space ever minted, and copying it named a + // type document no bundle could carry. + if e.isTypeDoc() && len(typeKeys) > 0 && typeKeys[0] != typeKeyObjectType { + e.warn("/"+memberTypeInternalKey, "type document stores its own type as %q; a type document is a Type, so %q is written instead", + typeKeys[0], typeKeyObjectType) + typeKeys = append([]string{typeKeyObjectType}, typeKeys[1:]...) + } typeTerm := "" if len(typeKeys) > 0 { typeTerm = e.typeSlug(typeKeys[0]) @@ -1668,6 +1713,14 @@ func (e *exporter) buildDoc(sbType model.SmartBlockType) (*omap, error) { doc.setNonEmpty("template_for", e.typeKeyRef(typeKeys[1])) } doc.setNonEmpty(memberInternalKey, e.snapshot.Key) + // a type the user REMOVED from the space travels as an ordinary type + // document carrying `uninstalled: true` (§2a) — the mirror of the member + // a removed property's declaration and dictionary entry carry — and + // restores hidden, as the user left it. The stored flag is lifted here + // and refused in `properties` (droppedPropertyKey), never spelled as one. + if e.isTypeDoc() && e.detail(detailKeyIsUninstalled).GetBoolValue() { + doc.set(memberUninstalled, true) + } // a relation document states its own definition next (§2d): `format`, // `include_time`, `object_types` — before `icon`, because what a property // IS outranks what it looks like, and before the legends, so the type @@ -1954,6 +2007,10 @@ func (e *exporter) droppedPropertyKey(k string, warn func(path, format string, a if _, dropped := typeProvenanceKeys[k]; dropped { return true } + // lifted onto the envelope as `uninstalled` (§2a), never a property + if k == detailKeyIsUninstalled { + return true + } } // a PARTICIPANT document does not carry createdDate (§3): the object // is derived from the ACL and has no creation change, so the stored diff --git a/codec/anyblockjson/flat_invariants_test.go b/codec/anyblockjson/flat_invariants_test.go index 1a8cf33..2d5d78e 100644 --- a/codec/anyblockjson/flat_invariants_test.go +++ b/codec/anyblockjson/flat_invariants_test.go @@ -576,6 +576,11 @@ func invertedTypes(objectTypes []string, sbType model.SmartBlockType) []string { if len(keys) == 0 { return nil } + // a type document IS a Type: export writes objectType as its own type + // whatever the store held (§2a), so that is what comes back + if isTypeSmartBlock(sbType) { + keys[0] = "objectType" + } out := []string{"ot-" + keys[0]} if sbType == model.SmartBlockType_Template && len(keys) > 1 { out = append(out, "ot-"+keys[1]) diff --git a/codec/anyblockjson/iconcid_test.go b/codec/anyblockjson/iconcid_test.go new file mode 100644 index 0000000..76ef29a --- /dev/null +++ b/codec/anyblockjson/iconcid_test.go @@ -0,0 +1,63 @@ +package anyblockjson + +// iconcid_test.go pins the icon's fifth variant (§2b): an image addressed by +// its CONTENT cid, with no file object behind it — a participant's avatar, a +// 1-to-1 space's icon — fetched through the gateway or the identity repo. +// `file` used to carry both meanings and let a reader tell them apart by +// decoding the CID's codec; a slot with two meanings is a defect, so the +// content address gets its own member and the codec decides which one an +// export writes: dag-cbor is an object id, dag-pb or raw is content. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-block/internal/testfixtures" +) + +func TestExport_AContentCidIconIsItsOwnVariant(t *testing.T) { + t.Run("a dag-pb cid is content, not an object", func(t *testing.T) { + icon, _, _, _ := exportedIconCover(t, map[string]*types.Value{ + "iconImage": strList(testfixtures.ContentID), "iconOption": num(5)}) + assert.Equal(t, `{"format":"cid","cid":"`+testfixtures.ContentID+`","color":"pink"}`, compactJSON(t, icon)) + }) + t.Run("a dag-cbor cid is still a file object reference", func(t *testing.T) { + icon, _, _, _ := exportedIconCover(t, map[string]*types.Value{"iconImage": strList(testfixtures.ObjectID)}) + assert.Equal(t, `{"format":"file","file":"`+testfixtures.ObjectID+`"}`, compactJSON(t, icon)) + }) + t.Run("it imports back onto the same slot and is a fixpoint", func(t *testing.T) { + doc := `{"formatVersion":"2.0","id":"o1","icon":{"format":"cid","cid":"` + testfixtures.ContentID + `"}}` + require.NoError(t, Validate([]byte(doc), Options{})) + _, snap, err := Unmarshal([]byte(doc), Options{GenerateId: seqIds("g")}) + require.NoError(t, err) + assert.Equal(t, strList(testfixtures.ContentID), snap.Details.Fields[detailKeyIconImage]) + icon, _, _, _ := exportedIconCover(t, map[string]*types.Value{"iconImage": strList(testfixtures.ContentID)}) + assert.Equal(t, `{"format":"cid","cid":"`+testfixtures.ContentID+`"}`, compactJSON(t, icon)) + }) + t.Run("an author cannot mint a content address", func(t *testing.T) { + doc := `{"formatVersion":"2.0","id":"o1","type":"Page","icon":{"format":"cid","cid":"` + testfixtures.ContentID + `"}}` + require.Error(t, ValidateAuthoring([]byte(doc))) + }) +} + +func TestIndex_AContentCidIconNamesNoObject(t *testing.T) { + t.Run("the variant round-trips through the index", func(t *testing.T) { + data, err := MarshalIndex(&Index{Name: "Corpus", Icon: &Icon{Format: "cid", Cid: testfixtures.ContentID}}, Options{}) + require.NoError(t, err) + assert.Contains(t, string(data), `"format": "cid"`) + back, err := UnmarshalIndex(data, Options{}) + require.NoError(t, err) + require.NotNil(t, back.Icon) + assert.Equal(t, testfixtures.ContentID, back.Icon.Cid) + assert.Empty(t, back.IconImageId(), "a content cid names no object the bundle could carry") + }) + t.Run("the legacy overloaded spelling is input compatibility, and names no object either", func(t *testing.T) { + legacy := &Index{Name: "Corpus", Icon: &Icon{Format: "file", File: testfixtures.ContentID}} + assert.Empty(t, legacy.IconImageId()) + object := &Index{Name: "Corpus", Icon: &Icon{Format: "file", File: testfixtures.ObjectID}} + assert.Equal(t, testfixtures.ObjectID, object.IconImageId()) + }) +} diff --git a/codec/anyblockjson/iconcover.go b/codec/anyblockjson/iconcover.go index e6ac471..19e09d9 100644 --- a/codec/anyblockjson/iconcover.go +++ b/codec/anyblockjson/iconcover.go @@ -357,6 +357,13 @@ func iconOf(detail func(string) *types.Value, warn func(path, format string, arg // emit what its own Validate rejects (§11, I1) warn("/icon", "icon image %q is not an object id and is dropped — "+ "this format holds a reference to an image object, never a URL or a path", images[0]) + } else if isContentCid(images[0]) { + // a content address, not an object: no file object stands behind + // it and none can be asked for, so it gets the variant that says + // so rather than a `file` a reader would try to resolve (§2b) + ic.Format = "cid" + ic.Cid = images[0] + return ic } else { ic.Format = "file" ic.File = images[0] @@ -394,6 +401,8 @@ func iconOmap(ic *Icon) *omap { format = "emoji" case ic.File != "": format = "file" + case ic.Cid != "": + format = "cid" case ic.Color != nil: format = "color" default: @@ -409,6 +418,8 @@ func iconOmap(ic *Icon) *omap { m.set("emoji", ic.Emoji) case "file": m.set("file", ic.File) + case "cid": + m.set("cid", ic.Cid) } if ic.Color != nil { m.set("color", ic.Color) @@ -518,6 +529,11 @@ func (e *exporter) calloutIcon(t *model.BlockContentText) *omap { "a reference to an image object, never a URL or a path", t.IconImage) return nil } + if isContentCid(t.IconImage) { + m.set("format", "cid") + m.set("cid", t.IconImage) + return m + } m.set("format", "file") m.set("file", e.opts.foldRef(t.IconImage)) return m @@ -533,6 +549,8 @@ func calloutIconFrom(ic *Icon, t *model.BlockContentText) { t.IconEmoji = ic.Emoji case "file": t.IconImage = ic.File + case "cid": + t.IconImage = ic.Cid } } @@ -585,8 +603,13 @@ type Icon struct { Format string `json:"format"` Emoji string `json:"emoji"` File string `json:"file"` - Name string `json:"name"` - Color any `json:"color"` + // Cid is the `cid` variant's content address (§2b): an image no file + // object holds — a participant avatar, a 1-to-1 space icon — fetched by + // content id through the gateway or the identity repo. Never an object + // reference, so nothing folds, declares or resolves it. + Cid string `json:"cid"` + Name string `json:"name"` + Color any `json:"color"` } // Cover is the other typed field (§2b), Icon's counterpart: the banner an @@ -630,13 +653,19 @@ func (imp *importer) applyIcon(details *types.Struct) { switch ic.Format { case "emoji": setStr(detailKeyIconEmoji, ic.Emoji) - case "file": + case "file", "cid": // `iconImage` is a `file` relation, so its stored shape is a list — // the same shape the ordinary property path writes (wrapToList), and - // the shape all 12 011 populated cases in the corpus hold + // the shape all 12 011 populated cases in the corpus hold. A content + // cid lands on the same slot: the store never told the two apart, + // only the wire does (§2b), and a content address is never folded. + image := ic.Cid + if ic.Format == "file" { + image = imp.unfoldRef(ic.File) + } details.Fields[detailKeyIconImage] = &types.Value{Kind: &types.Value_ListValue{ ListValue: &types.ListValue{Values: []*types.Value{ - {Kind: &types.Value_StringValue{StringValue: imp.unfoldRef(ic.File)}}, + {Kind: &types.Value_StringValue{StringValue: image}}, }}, }} case "icon": diff --git a/codec/anyblockjson/iconcover_test.go b/codec/anyblockjson/iconcover_test.go index 6507aab..d8f76d0 100644 --- a/codec/anyblockjson/iconcover_test.go +++ b/codec/anyblockjson/iconcover_test.go @@ -86,9 +86,9 @@ func TestExport_EveryEmittedIconShape(t *testing.T) { "iconName": str("folder"), "iconOption": num(10), "iconEmoji": str("🌎")}, icon: `{"format":"icon","name":"folder","color":"lime","emoji":"🌎"}`, }, - "an avatar image and its colour — 55 documents": { - details: map[string]*types.Value{"iconImage": strList("bafybeic7zrh5fa"), "iconOption": num(5)}, - icon: `{"format":"file","file":"bafybeic7zrh5fa","color":"pink"}`, + "an avatar image and its colour — 55 documents; a content cid, not an object": { + details: map[string]*types.Value{"iconImage": strList(testfixtures.ContentID), "iconOption": num(5)}, + icon: `{"format":"cid","cid":"` + testfixtures.ContentID + `","color":"pink"}`, }, "a colour with no source — 29 documents": { details: map[string]*types.Value{"iconOption": num(9)}, @@ -519,11 +519,11 @@ func TestValidate_AWrongIconGetsExactlyOneIssue(t *testing.T) { {`{"formatVersion":"2.0","icon":{"format":"emoji","emoji":"📕","name":"rocket"}}`, "/icon/name", `property "name" is not allowed`}, {`{"formatVersion":"2.0","icon":{"format":"image","url":"https://x/y.png"}}`, - "/icon/format", "value must be one of 'emoji', 'file', 'icon', 'color'"}, + "/icon/format", "value must be one of 'emoji', 'file', 'cid', 'icon', 'color'"}, {`{"formatVersion":"2.0","icon":{"format":"url","url":"https://x/y.png"}}`, - "/icon/format", "value must be one of 'emoji', 'file', 'icon', 'color'"}, + "/icon/format", "value must be one of 'emoji', 'file', 'cid', 'icon', 'color'"}, {`{"formatVersion":"2.0","icon":{"emoji":"🚀"}}`, - "/icon", "an icon is one of 'emoji', 'file', 'icon', 'color'"}, + "/icon", "an icon is one of 'emoji', 'file', 'cid', 'icon', 'color'"}, {`{"formatVersion":"2.0","icon":{"format":"emoji"}}`, "/icon", "missing property 'emoji'"}, {`{"formatVersion":"2.0","icon":{"format":"icon","name":"rocket","color":"turquoise"}}`, @@ -559,9 +559,9 @@ func TestValidate_AWrongIconGetsExactlyOneIssue(t *testing.T) { // How this can fail: hardcode the variant names in iconFormatIssues and this // test still passes on the happy path but the appended variant never appears. func TestValidate_TheFormatUnionIsReadFromTheSchema(t *testing.T) { - assert.Equal(t, []string{"emoji", "file", "icon", "color"}, schemaFormatEnum("icon")) + assert.Equal(t, []string{"emoji", "file", "cid", "icon", "color"}, schemaFormatEnum("icon")) assert.Equal(t, []string{"image", "color", "gradient"}, schemaFormatEnum("cover")) - assert.Equal(t, []string{"emoji", "file"}, schemaFormatEnum("plainIcon"), + assert.Equal(t, []string{"emoji", "file", "cid"}, schemaFormatEnum("plainIcon"), "the narrowed definition answers with the narrowed set, not the one it refs") } diff --git a/codec/anyblockjson/iconunion_test.go b/codec/anyblockjson/iconunion_test.go index cb7a0c8..64906e4 100644 --- a/codec/anyblockjson/iconunion_test.go +++ b/codec/anyblockjson/iconunion_test.go @@ -15,8 +15,8 @@ import ( // BOTH enums failed both and the reader got two contradictory verdicts, the // WIDER one first: // -// /blocks/0/icon/format: value must be one of 'emoji', 'file', 'icon', 'color' -// /blocks/0/icon/format: value must be one of 'emoji', 'file' +// /blocks/0/icon/format: value must be one of 'emoji', 'file', 'cid', 'icon', 'color' +// /blocks/0/icon/format: value must be one of 'emoji', 'file', 'cid' // // A greedy repairer reads line 1, writes `format: "icon"`, and gets a fresh // error — two round trips where one would do. That is the disease this whole @@ -56,12 +56,12 @@ func TestValidate_ARestrictedIconSlotNamesOneUnion(t *testing.T) { }) } - // the control: the OBJECT slot still names all four, so the fix cannot pass + // the control: the OBJECT slot still names all five, so the fix cannot pass // by narrowing every slot to the callout's set t.Run("the object slot still names all four", func(t *testing.T) { issues := firstIssues(t, `{"formatVersion": "2.0", "type": "page", "icon": {"format": "url", "url": "http://x"}}`) require.Len(t, issues, 1) - assert.Contains(t, issues[0].Message, "'emoji', 'file', 'icon', 'color'") + assert.Contains(t, issues[0].Message, "'emoji', 'file', 'cid', 'icon', 'color'") }) // and both slots still accept what they should diff --git a/codec/anyblockjson/import.go b/codec/anyblockjson/import.go index 4a4bb2b..e7f79c3 100644 --- a/codec/anyblockjson/import.go +++ b/codec/anyblockjson/import.go @@ -72,7 +72,10 @@ type jsonDoc struct { // TypeSettings is a kind:object_type document's definition group (§2a): // the five lifted settings plus property_definitions. TypeSettings *jsonTypeSettings `json:"type_settings"` - FileRemote *string `json:"file_remote"` + // Uninstalled says the user REMOVED this type from its space (§2a): it + // restores hidden, as the mirror member on a property declaration does. + Uninstalled bool `json:"uninstalled"` + FileRemote *string `json:"file_remote"` // PropertyKeys is the §3 spelling→stored-key legend: what this document says // its own key spellings mean, consulted before any vocabulary so a reader // without the space still lands on the right relation. Its values are @@ -1011,6 +1014,11 @@ func (imp *importer) build() (model.SmartBlockType, *model.SmartBlockSnapshotBas if err := imp.applyTypeSettings(details, sbType); err != nil { return 0, nil, err } + if imp.doc.Uninstalled && isTypeSmartBlock(sbType) { + // the stored flag the app sets on removal; the runtime mirrors it + // into isDeleted on load, which is what hides the type (§2a) + details.Fields[detailKeyIsUninstalled] = &types.Value{Kind: &types.Value_BoolValue{BoolValue: true}} + } if err := imp.applyQuerySource(details, sbType); err != nil { return 0, nil, err } diff --git a/codec/anyblockjson/index.go b/codec/anyblockjson/index.go index 711daa5..ae87700 100644 --- a/codec/anyblockjson/index.go +++ b/codec/anyblockjson/index.go @@ -452,6 +452,60 @@ type Unresolved struct { // usually names a widget the user deleted, which is what the ledger is // FOR, so a missing document is its normal state rather than a loss. Targets []string `json:"targets"` + // Deleted is the subset of Targets the space DELETED: tombstones in the + // source store. A reference to a deleted object is ordinary Anytype + // state — the app does not rewrite every object that mentions a deleted + // one — so a reader treats these as known, by design, and a restore + // keeps the id and writes a tombstone. Sorted, folded like Targets. + Deleted []string `json:"deleted"` + // Omitted is the subset of Targets the space still HOLDS and this + // export did not write: an archived object under an export without + // archived objects, or one outside a partial export's scope. A loss in + // this bundle, not in the space. Sorted, folded like Targets. + // + // A target in neither subset is ABSENT — the space had no row for it, + // typically because it had not synced — and that is the one class a + // reader warns about. + Omitted []string `json:"omitted"` + // Types are the TYPES this bundle's documents name by derived id — + // `type-`, in a `type_internal_key`, a `template_for`, + // an `object_types` entry — that no type document carries and the + // space has no row for: an object or template an old import created + // with a type it never made. Spelled as the derived id, which is the + // address the document uses and the validator checks, and never + // folded. A reader imports such an object as a Page and warns (§2c). + // Sorted. + Types []string `json:"types"` +} + +// check refuses a report that contradicts itself: a subset entry that +// `targets` does not declare, or an id that is both a tombstone and an +// object the space still holds. Both doors run it, the way both refuse the +// empty object. +func (u *Unresolved) check() error { + if u == nil { + return nil + } + declared := map[string]struct{}{} + for _, id := range u.Targets { + declared[id] = struct{}{} + } + deleted := map[string]struct{}{} + for _, id := range u.Deleted { + if _, ok := declared[id]; !ok { + return fmt.Errorf("unresolved.deleted names %q, which unresolved.targets does not declare", id) + } + deleted[id] = struct{}{} + } + for _, id := range u.Omitted { + if _, ok := declared[id]; !ok { + return fmt.Errorf("unresolved.omitted names %q, which unresolved.targets does not declare", id) + } + if _, ok := deleted[id]; ok { + return fmt.Errorf("unresolved.deleted and unresolved.omitted both name %q; an id is a tombstone or an object the space holds, not both", id) + } + } + return nil } // empty reports whether the report says nothing — the shape setNonEmpty @@ -460,7 +514,7 @@ type Unresolved struct { // empty object rather than let it read as a promise that everything // resolves. func (u *Unresolved) empty() bool { - return u == nil || (len(u.Properties) == 0 && len(u.Targets) == 0) + return u == nil || (len(u.Properties) == 0 && len(u.Targets) == 0 && len(u.Deleted) == 0 && len(u.Omitted) == 0 && len(u.Types) == 0) } // EntryPoint returns the entry point the bundle *declares*: the entrypoint @@ -691,6 +745,11 @@ func UnmarshalIndex(data []byte, opts Options) (*Index, error) { idx.AutoWidgetTargets = mapStrings(idx.AutoWidgetTargets, opts.unfoldRef) if idx.Unresolved != nil { idx.Unresolved.Targets = mapStrings(idx.Unresolved.Targets, opts.unfoldRef) + idx.Unresolved.Deleted = mapStrings(idx.Unresolved.Deleted, opts.unfoldRef) + idx.Unresolved.Omitted = mapStrings(idx.Unresolved.Omitted, opts.unfoldRef) + if err := idx.Unresolved.check(); err != nil { + return nil, &ValidationError{Issues: []Issue{{Path: "/unresolved", Message: err.Error()}}} + } } if idx.Icon != nil && idx.Icon.File != "" { idx.Icon.File = opts.unfoldRef(idx.Icon.File) @@ -842,11 +901,14 @@ func indexIconOmap(ic *Icon) *omap { return iconOmap(ic) } -// IconImageId is the object id of an index's image icon, or "" when the index -// has no icon or an emoji one. The bundle wiring resolves that id to an image -// NAME, which is what the installer actually takes. +// IconImageId is the OBJECT id of an index's image icon, or "" when the +// index has no icon, an emoji one, or an image addressed by content cid +// (§2b): a `cid` icon names no object the bundle could carry, and neither +// does the legacy overloaded spelling — a `file` holding a content cid, +// read for compatibility and never written. The bundle wiring resolves the +// object id to an image NAME, which is what the installer actually takes. func (i *Index) IconImageId() string { - if i == nil || i.Icon == nil || i.Icon.Format != "file" { + if i == nil || i.Icon == nil || i.Icon.Format != "file" || isContentCid(i.Icon.File) { return "" } return i.Icon.File @@ -956,9 +1018,15 @@ func MarshalIndex(idx *Index, opts Options) ([]byte, error) { // reference — a report that spelled a type one way while the widget // targeting it spelled it another would name two different things. if !idx.Unresolved.empty() { + if err := idx.Unresolved.check(); err != nil { + return nil, err + } u := &omap{} u.setNonEmpty("properties", stringsToAny(sortedCopy(idx.Unresolved.Properties))) u.setNonEmpty("targets", stringsToAny(sortedCopy(mapStrings(idx.Unresolved.Targets, opts.foldRef)))) + u.setNonEmpty("deleted", stringsToAny(sortedCopy(mapStrings(idx.Unresolved.Deleted, opts.foldRef)))) + u.setNonEmpty("omitted", stringsToAny(sortedCopy(mapStrings(idx.Unresolved.Omitted, opts.foldRef)))) + u.setNonEmpty("types", stringsToAny(sortedCopy(idx.Unresolved.Types))) doc.set("unresolved", u) } return marshalCanonical(doc) diff --git a/codec/anyblockjson/indexunresolved_test.go b/codec/anyblockjson/indexunresolved_test.go index ad84ba6..b4df58c 100644 --- a/codec/anyblockjson/indexunresolved_test.go +++ b/codec/anyblockjson/indexunresolved_test.go @@ -107,7 +107,7 @@ func TestIndexUnresolved_TheSchemaPublishesTheShape(t *testing.T) { for m := range def.Properties { stated[m] = true } - assert.Equal(t, map[string]bool{"properties": true, "targets": true}, stated) + assert.Equal(t, map[string]bool{"properties": true, "targets": true, "deleted": true, "omitted": true, "types": true}, stated) } // ReferencedObjectIds is the one list of slots that name an object, shared @@ -157,3 +157,117 @@ func TestIndexObjectReferencesKeepEachLocation(t *testing.T) { }, idx.ObjectReferences(Options{})) assert.Equal(t, []string{"same-target"}, idx.ReferencedObjectIds(Options{})) } + +// The two subsets say WHY a target is dangling, which the store knew and the +// bundle would otherwise lose: `deleted` names the tombstones (the space +// deleted them, by design), `omitted` names the objects the space holds +// that this export did not write (archived under an export without archived +// objects, or outside a partial export's scope). What is in neither is +// absent — no row in the space — and that is the one class that is a loss. +func TestIndexUnresolved_DeletedAndOmittedAreSubsetsOfTargets(t *testing.T) { + idx := &Index{ + Name: "Corpus", + Homepage: "bafyreigone", + Unresolved: &Unresolved{ + Targets: []string{"bafyreigone", "bafyreiarchived", "bafyreiabsent"}, + Deleted: []string{"bafyreigone"}, + Omitted: []string{"bafyreiarchived"}, + }, + } + + data, err := MarshalIndex(idx, Options{}) + require.NoError(t, err) + assert.Contains(t, string(data), `"deleted"`) + assert.Contains(t, string(data), `"omitted"`) + + back, err := UnmarshalIndex(data, Options{}) + require.NoError(t, err) + require.NotNil(t, back.Unresolved) + assert.Equal(t, []string{"bafyreiabsent", "bafyreiarchived", "bafyreigone"}, back.Unresolved.Targets, "sorted") + assert.Equal(t, []string{"bafyreigone"}, back.Unresolved.Deleted) + assert.Equal(t, []string{"bafyreiarchived"}, back.Unresolved.Omitted) + + again, err := MarshalIndex(back, Options{}) + require.NoError(t, err) + assert.Equal(t, string(data), string(again), "the report round-trips byte for byte") +} + +// A subset entry that is not in `targets` is a contradiction, not a report: +// the writer says an id is deleted while also saying the index does not +// name it. Both doors refuse it, the way both refuse the empty object. +func TestIndexUnresolved_ASubsetEntryMustBeADeclaredTarget(t *testing.T) { + stray := []byte(`{"formatVersion":"2.0","name":"Corpus","unresolved":{"targets":["bafyreigone"],"deleted":["bafyreiother"]}}`) + _, err := UnmarshalIndex(stray, Options{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "bafyreiother") + + _, err = MarshalIndex(&Index{ + Name: "Corpus", + Unresolved: &Unresolved{Targets: []string{"bafyreigone"}, Omitted: []string{"bafyreiother"}}, + }, Options{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "bafyreiother") +} + +// An id cannot be both a tombstone and an object the space still holds. +func TestIndexUnresolved_DeletedAndOmittedDoNotOverlap(t *testing.T) { + both := []byte(`{"formatVersion":"2.0","name":"Corpus","unresolved":{"targets":["bafyreigone"],"deleted":["bafyreigone"],"omitted":["bafyreigone"]}}`) + _, err := UnmarshalIndex(both, Options{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "bafyreigone") + + _, err = MarshalIndex(&Index{ + Name: "Corpus", + Unresolved: &Unresolved{ + Targets: []string{"bafyreigone"}, + Deleted: []string{"bafyreigone"}, + Omitted: []string{"bafyreigone"}, + }, + }, Options{}) + require.Error(t, err) +} + +// The subsets are references too, folded and unfolded with `targets` — a +// subset spelled one way and the target it classifies spelled another would +// never match. +func TestIndexUnresolved_SubsetsAreSpelledLikeTargets(t *testing.T) { + opts := Options{ResolveProperties: newTypeIdVocabulary()} + data, err := MarshalIndex(&Index{ + Name: "Corpus", + Unresolved: &Unresolved{Targets: []string{"typeid-wine"}, Omitted: []string{"typeid-wine"}}, + }, opts) + require.NoError(t, err) + assert.Equal(t, 2, strings.Count(string(data), `"type-wine"`), "folded in both lists") + assert.NotContains(t, string(data), "typeid-wine") + + back, err := UnmarshalIndex(data, opts) + require.NoError(t, err) + assert.Equal(t, []string{"typeid-wine"}, back.Unresolved.Targets) + assert.Equal(t, []string{"typeid-wine"}, back.Unresolved.Omitted) +} + +// The third loss a bundle can state (§2c): a TYPE its documents name by +// derived id that no type document carries and the space has no row for — +// an object or template an old import created with a type it never made. +// Spelled as the derived id, because that is the address a document uses +// and the one the validator checks; never folded, since it is already the +// bundle-internal spelling. A reader imports such an object as a Page and +// warns. +func TestIndexUnresolved_TypesTheDocumentsNameAndNothingCarries(t *testing.T) { + idx := &Index{ + Name: "Corpus", + Unresolved: &Unresolved{Types: []string{"type-69aab09861fab2bc0d9afdb6", "type-69aab06861fab2bc0d9afc59"}}, + } + data, err := MarshalIndex(idx, Options{}) + require.NoError(t, err) + assert.Contains(t, string(data), `"types"`) + + back, err := UnmarshalIndex(data, Options{}) + require.NoError(t, err) + require.NotNil(t, back.Unresolved) + assert.Equal(t, []string{"type-69aab06861fab2bc0d9afc59", "type-69aab09861fab2bc0d9afdb6"}, back.Unresolved.Types, "sorted") + + bare := []byte(`{"formatVersion":"2.0","name":"Corpus","unresolved":{"types":["69aab06861fab2bc0d9afc59"]}}`) + _, err = UnmarshalIndex(bare, Options{}) + require.Error(t, err, "a type is named by its derived id, never a bare key") +} diff --git a/codec/anyblockjson/legendadmission_test.go b/codec/anyblockjson/legendadmission_test.go index 69bd606..ef02b72 100644 --- a/codec/anyblockjson/legendadmission_test.go +++ b/codec/anyblockjson/legendadmission_test.go @@ -177,7 +177,9 @@ func (denyBindingVocabulary) PropertyKey(slug string) (string, bool) { // The type namespace has the same two shapes at the envelope `type`, which // carries no length or charset rule of its own (§3) and so hands the stored -// key to the ledger untouched. +// key to the ledger untouched. The vehicle is a PAGE: a type document's own +// type is normalized to objectType whatever the store holds (§2a), so only +// an ordinary document can carry a hostile key in that slot. func TestExport_TypeLegendRefusesAnEntryItCannotHold(t *testing.T) { for _, tc := range []struct { name, key, wantWarn string @@ -211,7 +213,7 @@ func TestExport_TypeLegendRefusesAnEntryItCannotHold(t *testing.T) { } // when - data, err := Marshal(model.SmartBlockType_STType, snap, opts) + data, err := Marshal(model.SmartBlockType_Page, snap, opts) // then require.NoError(t, err) diff --git a/codec/anyblockjson/missingref_test.go b/codec/anyblockjson/missingref_test.go index fb1349b..19feb9c 100644 --- a/codec/anyblockjson/missingref_test.go +++ b/codec/anyblockjson/missingref_test.go @@ -1,13 +1,17 @@ package anyblockjson // missingref_test.go — the missing-reference rule (§9): a reference to an -// object that does not exist in the SPACE is not written as if it did — a -// SINGULAR slot (block object_id, mention target) rewrites to the -// `_missing_object` sentinel, a LIST slot (objects/files property values, -// `object_types`) drops the entry. And the distinction that makes or breaks -// the rule: "missing from this EXPORT" is not "missing from the space" — -// only the store's own testimony, through the ObjectExistenceResolver -// capability, may move anything. No capability, no change, sentinel included. +// object the SPACE holds no row for is KEPT VERBATIM and warned, in every +// slot — a singular slot (block object_id, mention target) and a list slot +// (objects/files property values, `object_types`) alike. The `_missing_object` +// sentinel is the importer's answer, never the exporter's: a stored sentinel +// is kept as-is in a singular slot and dropped silently from a list, and +// export never manufactures one — a backup taken during a sync gap keeps the +// id for the day the object arrives. And the distinction that makes or +// breaks the warning: "missing from this EXPORT" is not "missing from the +// space" — only the store's own testimony, through the +// ObjectExistenceResolver capability, may say anything. No capability, no +// warning and no change, sentinel included. import ( "strings" @@ -138,9 +142,9 @@ func TestMissingReference_SingularBlockSlots(t *testing.T) { // then require.NoError(t, err) require.NoError(t, Validate(data, Options{}), "Marshal never emits what Validate rejects (I1)") - assert.Contains(t, string(data), `"object_id": "_missing_object"`) - assert.NotContains(t, string(data), deadCid, "the dead id must not be written as if it existed") - require.Len(t, warnings, 1, "a rewrite destroys the stored id; the warning is its last appearance") + assert.Contains(t, string(data), `"object_id": "`+deadCid+`"`, "the id is kept: the space may simply not have synced it yet") + assert.NotContains(t, string(data), missingObjectId, "the sentinel is the importer's answer, never the exporter's") + require.Len(t, warnings, 1, "the loss is stated, not enacted") assert.Contains(t, warnings[0].Message, deadCid) assert.Equal(t, IssueCodeUnresolvedTarget, warnings[0].Code) assert.Equal(t, "/blocks/b1/object_id", warnings[0].Path) @@ -181,7 +185,7 @@ func TestMissingReference_SingularBlockSlots(t *testing.T) { assert.Empty(t, warnings) }) - t.Run("no capability wired: nothing is rewritten", func(t *testing.T) { + t.Run("no capability wired: nothing is warned or rewritten", func(t *testing.T) { // given — a package-only export has no store to ask, and the absence // of an answer is not evidence of absence; a name-only resolver // (the pre-capability shape) must not arm the rule either @@ -218,9 +222,9 @@ func TestMissingReference_SingularBlockSlots(t *testing.T) { }) } -// A mention is a singular slot inside inline markup: the target rewrites to -// the sentinel, the mention's own text stays, and the snapshot's marks — -// caller-owned state — are never mutated. +// A mention is a singular slot inside inline markup: an absent target is +// kept and warned, the mention's own text stays, and the snapshot's marks — +// caller-owned state — are never mutated by the fold that still runs. func TestMissingReference_MentionTargets(t *testing.T) { mention := func(param string) *model.Block { return textBlock("b1", model.BlockContentText_Paragraph, "Ping Alice", @@ -231,7 +235,7 @@ func TestMissingReference_MentionTargets(t *testing.T) { }) } - t.Run("a missing mention target rewrites to the sentinel", func(t *testing.T) { + t.Run("a missing mention target is kept and warned", func(t *testing.T) { // given var warnings []Issue opts := missingRefOptions(&warnings) @@ -243,10 +247,11 @@ func TestMissingReference_MentionTargets(t *testing.T) { // then require.NoError(t, err) require.NoError(t, Validate(data, Options{}), "Marshal never emits what Validate rejects (I1)") - assert.Contains(t, string(data), `Alice`) + assert.Contains(t, string(data), `Alice`) + assert.NotContains(t, string(data), missingObjectId) require.Len(t, warnings, 1) assert.Contains(t, warnings[0].Message, deadCid) - // the snapshot is caller-owned: the rewrite must be copy-on-write + assert.Equal(t, IssueCodeUnresolvedTarget, warnings[0].Code) assert.Equal(t, "/blocks/b1/text/marks/0/param", warnings[0].Path) assert.Equal(t, deadCid, snap.Blocks[1].GetText().Marks.Marks[0].Param, "exportMarks mutated the caller's snapshot") @@ -311,17 +316,16 @@ func TestMissingReference_MentionTargets(t *testing.T) { // then require.NoError(t, err) - assert.Contains(t, string(data), `Alice`) + assert.Contains(t, string(data), `Alice`) require.Len(t, warnings, 1) assert.Equal(t, "/blocks/r1-c1/text/marks/0/param", warnings[0].Path) }) } -// An objects/files property value is a LIST slot: a missing entry drops — -// the sentinel silently, a real id with a warning that is the id's last -// appearance — and the emptied list stays `[]`, because the key's presence -// is meaningful (§3) and dropping dangling entries must not erase the fact -// that the property was set. +// An objects/files property value is a LIST slot: an absent real id is +// KEPT and warned, a stored sentinel drops silently (it carries nothing — +// which object it was is already gone), and a list emptied by that drop +// stays `[]`, because the key's presence is meaningful (§3). func TestMissingReference_PropertyValueLists(t *testing.T) { withRelated := func(v *types.Value) *model.SmartBlockSnapshotBase { snap := blockSnapshot() @@ -329,7 +333,7 @@ func TestMissingReference_PropertyValueLists(t *testing.T) { return snap } - t.Run("missing entries drop, live entries close ranks", func(t *testing.T) { + t.Run("an absent entry is kept and warned; the sentinel drops silently", func(t *testing.T) { // given var warnings []Issue opts := missingRefOptions(&warnings) @@ -341,7 +345,7 @@ func TestMissingReference_PropertyValueLists(t *testing.T) { // then require.NoError(t, err) require.NoError(t, Validate(data, Options{}), "Marshal never emits what Validate rejects (I1)") - assert.Contains(t, compactDoc(data), `"related":["`+liveCid+`"]`) + assert.Contains(t, compactDoc(data), `"related":["`+liveCid+`","`+deadCid+`"]`) require.Len(t, warnings, 1, "the real id warns; the sentinel — which carries nothing — drops silently") assert.Equal(t, IssueCodeUnresolvedTarget, warnings[0].Code) assert.Equal(t, "/properties/related/1", warnings[0].Path) @@ -398,9 +402,9 @@ func TestMissingReference_PropertyValueLists(t *testing.T) { // A property document's `object_types` is the same list slot in the type // namespace (§2d): a resolvable type id becomes its derived id, a bare key -// spells its derived id too — vocabulary, not a reference — and only what -// the store disowns drops. An id nothing could translate stays an id: `type-` -// is never put in front of a CID. +// spells its derived id too — vocabulary, not a reference — an absent id is +// kept and warned, and only the stored sentinel drops. An id nothing could +// translate stays an id: `type-` is never put in front of a CID. func TestMissingReference_PropertySettingsObjectTypes(t *testing.T) { relSnap := func(targets *types.Value) *model.SmartBlockSnapshotBase { return relationSnapshot(map[string]*types.Value{ @@ -414,7 +418,7 @@ func TestMissingReference_PropertySettingsObjectTypes(t *testing.T) { return o } - t.Run("dead id and sentinel drop; live id and bare key survive", func(t *testing.T) { + t.Run("the sentinel drops; dead id, live id and bare key survive", func(t *testing.T) { // given — the corpus shape: 56 properties carry an object id naming // nothing, type ids from the account where a shipped use case was // AUTHORED (an object id differs in every space; a type key does not) @@ -428,9 +432,10 @@ func TestMissingReference_PropertySettingsObjectTypes(t *testing.T) { // then require.NoError(t, err) require.NoError(t, Validate(data, Options{}), "Marshal never emits what Validate rejects (I1)") - assert.Contains(t, compactDoc(data), `"object_types":["type-page","type-wine"]`) + assert.Contains(t, compactDoc(data), `"object_types":["type-page","`+deadCid+`","type-wine"]`) require.Len(t, warnings, 1) assert.Contains(t, warnings[0].Message, deadCid) + assert.Equal(t, IssueCodeUnresolvedTarget, warnings[0].Code) assert.Equal(t, "/properties/relationFormatObjectTypes/1", warnings[0].Path) }) @@ -465,9 +470,9 @@ func TestMissingReference_PropertySettingsObjectTypes(t *testing.T) { }) } -// The round trip is a fixpoint after one generation: the first export -// rewrites and drops, import stores what was written, and every export -// after that is byte-identical — §11 guarantee 3 under the new rule. +// The round trip is a fixpoint from the first generation: the absent id is +// kept, import stores it, and every export after that is byte-identical — +// §11 guarantee 3 under the rule, with nothing lost on the way. func TestMissingReference_RoundTripStable(t *testing.T) { // given — a dead singular target AND a dead list entry in one snapshot snap := blockSnapshot( @@ -486,7 +491,9 @@ func TestMissingReference_RoundTripStable(t *testing.T) { // then assert.Equal(t, string(first), string(second), - "Export(Import(Export(S))) must equal Export(S): the rewrite converges in one generation") + "Export(Import(Export(S))) must equal Export(S)") + assert.Contains(t, string(first), deadCid, "the absent id survives the round trip") + assert.NotContains(t, string(first), missingObjectId) } // Over the hostile corpus a resolver that declares EVERYTHING missing @@ -510,7 +517,9 @@ func TestMissingReference_HostileIdsAreUntouchable(t *testing.T) { } // The predicate snapshotdiff consults is the export's own; pin its verdicts -// at the seam so the comparator and the codec cannot drift apart. +// at the seam so the comparator and the codec cannot drift apart. Only the +// stored sentinel ever drops, and only when the capability is wired: a real +// id — absent or not — is kept. func TestDroppedMissingObjectRef(t *testing.T) { armed := Options{ResolveObjectNames: missingRefStore()} for name, tc := range map[string]struct { @@ -518,7 +527,7 @@ func TestDroppedMissingObjectRef(t *testing.T) { entry string want bool }{ - "dead cid, capability wired": {armed, deadCid, true}, + "dead cid, capability wired": {armed, deadCid, false}, "sentinel, capability wired": {armed, missingObjectId, true}, "live cid": {armed, liveCid, false}, "untitled but existing": {armed, untitledCid, false}, diff --git a/codec/anyblockjson/refs.go b/codec/anyblockjson/refs.go index 1f0e915..bbe061b 100644 --- a/codec/anyblockjson/refs.go +++ b/codec/anyblockjson/refs.go @@ -135,6 +135,76 @@ func DroppedDeletedIconRef(opts Options, id string) bool { return known && deleted } +// UnresolvedClass says WHY an id the index names is not in the bundle +// (§2c, Index.Unresolved). The store already draws this line — a deleted +// object keeps a tombstone row so a link to it can be told apart from a +// link to an object that has not loaded — and the composer, the validator +// and the comparator must all read it the same way, so the predicate lives +// here and is exported. +type UnresolvedClass uint8 + +const ( + // UnresolvedAbsent: the space holds no row for the id — it has not + // synced, or it was never here — or nothing could ask. This is the one + // class that is a loss, and the fail-safe direction: a store that did + // not answer, a capability that is not wired, and an id the store was + // never the authority for all land here. + UnresolvedAbsent UnresolvedClass = iota + // UnresolvedDeleted: a tombstone. The space deleted the object, by + // design, and a reference to it is ordinary state. + UnresolvedDeleted + // UnresolvedOmitted: the space still holds the object and this export + // did not write it — archived under an export without archived objects, + // or outside a partial export's scope. A loss in this bundle, not in + // the space. + UnresolvedOmitted +) + +// ClassifyUnresolvedTarget asks the wired store which UnresolvedClass an +// unresolved index target belongs to. Only CID-shaped ids are asked +// (isObjectIdShaped); the existence capability must be wired and must +// answer; and a row whose deletion cannot be asked is reported as omitted, +// because "the space holds it" is the half that was answered. +func ClassifyUnresolvedTarget(opts Options, id string) UnresolvedClass { + if !isObjectIdShaped(id) { + return UnresolvedAbsent + } + existence, ok := opts.ResolveObjectNames.(ObjectExistenceResolver) + if !ok { + return UnresolvedAbsent + } + exists, known := existence.ObjectExists(id) + if !known || !exists { + return UnresolvedAbsent + } + if deletion, ok := opts.ResolveObjectNames.(ObjectDeletionResolver); ok { + if deleted, known := deletion.ObjectDeleted(id); known && deleted { + return UnresolvedDeleted + } + } + return UnresolvedOmitted +} + +// isContentCid reports whether s is a CONTENT address rather than an object +// id: a CID whose codec is dag-pb or raw — the shape file bytes get, and +// the shape a participant avatar or a 1-to-1 space icon carries when no file +// object stands behind the image (§2b). An object id is dag-cbor. The two +// never share a codec, so this is a classifier, not a heuristic. +func isContentCid(s string) bool { + if len(s) < 46 { + return false + } + c, err := cid.Decode(s) + if err != nil { + return false + } + switch c.Prefix().Codec { + case cid.DagProtobuf, cid.Raw: + return true + } + return false +} + // isObjectIdShaped reports whether s parses as a content id (CID) — the // shape of every object and file id a space actually mints. It is the gate // that keeps the existence question OFF everything that is not a space @@ -175,23 +245,26 @@ func missingFromSpace(opts Options, id string) bool { // DroppedMissingObjectRef reports whether export drops entry from a // LIST-valued reference slot — an objects/files property value (§3), a -// property document's `object_types` (§2d): the stored `_missing_object` -// sentinel, or an object id the wired store says the space does not hold. -// A list expresses absence by being shorter; singular slots rewrite to the -// sentinel instead (§9) and are not this predicate's business. +// property document's `object_types` (§2d). Exactly one thing drops: the +// stored `_missing_object` sentinel, and only when the existence capability +// is wired. A real id is kept whatever the store says of it (§9): an object +// the space holds no row for may simply not have synced yet, and the +// sentinel is the importer's answer for a reference it cannot resolve, +// never the exporter's — a backup that replaced the id would have lost, for +// good, the one thing a later sync could still make whole. // // Exported because snapshotdiff — the comparator behind the corpus sweep — // must apply the SAME predicate to both sides, or every dropped-by-design -// entry reports as data loss (the drift class that once produced 1,344 +// sentinel reports as data loss (the drift class that once produced 1,344 // false failures in one sweep, §11). With no capability wired it drops // nothing, sentinel included: a package-only export passes every entry // through verbatim. func DroppedMissingObjectRef(opts Options, entry string) bool { - if entry == missingObjectId { - _, ok := opts.ResolveObjectNames.(ObjectExistenceResolver) - return ok + if entry != missingObjectId { + return false } - return missingFromSpace(opts, entry) + _, ok := opts.ResolveObjectNames.(ObjectExistenceResolver) + return ok } // isAccountIdentity reports whether s is a member's account identity — the @@ -464,59 +537,57 @@ func (e *exporter) objectRef(id string) string { // singularObjectRef renders a SINGULAR reference slot — a block's // `object_id` (link, bookmark, file kinds, dataview) — under the -// missing-reference rule (§9): a target the space does not hold is written -// as the `_missing_object` sentinel, because omission cannot express "no -// target" here — only deleting the block could, and that would lose the -// fact that a link existed. A target the store DOES hold, the store cannot -// speak for (missingFromSpace's gates), or that already IS the sentinel -// passes to the ordinary objectRef untouched. +// missing-reference rule (§9): a target the space holds no row for is +// written VERBATIM and warned. The id is kept because absence from the +// store is not absence from the world — the object may not have synced +// yet — and the importer already writes the `_missing_object` sentinel for +// any target it cannot resolve, so rewriting here preserved nothing and +// destroyed the one thing a later sync could still make whole. A target the +// store DOES hold, the store cannot speak for (missingFromSpace's gates), or +// that already IS the sentinel passes to the ordinary objectRef silently. // -// The rewrite warns, naming the id: unlike the sentinel — which says -// nothing beyond "gone" — the id is real information, and the warning is -// its last appearance anywhere. After one round trip the slot is a -// fixpoint: the sentinel is kept as-is, so re-exports are byte-stable. +// The warning names the id and the slot, so an export report can say what +// the space could not serve. The slot is a fixpoint from the first +// generation: the id is kept, import stores it, and re-exports are +// byte-stable. func (e *exporter) singularObjectRef(path, slot, id string) string { if missingFromSpace(e.opts, id) { - e.warnReference(path, "%s %q names no object in this space and is written as %q — "+ - "the slot cannot say \"no target\" without deleting the block, and the sentinel "+ - "keeps the fact that a reference existed", slot, id, missingObjectId) - return e.objectRef(missingObjectId) + e.warnReference(path, "%s %q names no object in this space — not synced, or never here — "+ + "and is written as it stands; the importer resolves it, or writes the sentinel", slot, id) } return e.objectRef(id) } // droppedMissingListEntry is the LIST half of the missing-reference rule -// (§9): an objects/files property value entry, or an `object_types` entry, -// that the space does not hold is dropped — a list expresses absence by -// being shorter. The predicate is the exported DroppedMissingObjectRef, so +// (§9): an objects/files property value entry, or an `object_types` entry. +// A stored `_missing_object` sentinel drops silently — it carries nothing, +// which object it was is already gone, and the corpus holds ~990 of them in +// property values alone, which would triple a warning channel that was just +// cut down to what is worth reading (§12). A real id the space holds no +// row for is KEPT and warned, for the reason singularObjectRef gives: it +// may not have synced yet, and the importer is the one that writes the +// sentinel. The drop predicate is the exported DroppedMissingObjectRef, so // the comparator applies exactly what export applied. -// -// Only a REAL id warns. A stored `_missing_object` sentinel drops silently: -// it carries nothing — which object it was is already gone — and the corpus -// holds ~990 of them in property values alone, which would triple a warning -// channel that was just cut down to what is worth reading (§12). func (e *exporter) droppedMissingListEntry(path, id string) bool { - if !DroppedMissingObjectRef(e.opts, id) { + if missingFromSpace(e.opts, id) { + e.warnReference(path, "%q names no object in this space — not synced, or never here — "+ + "and is kept as it stands; the importer resolves it, or drops it", id) return false } - if id != missingObjectId { - e.warnReference(path, "%q names no object in this space and is dropped — "+ - "a list expresses absence by being shorter", id) - } - return true + return DroppedMissingObjectRef(e.opts, id) } // exportMarks applies the reference rules to inline markup (§8, §9). A -// `` whose target the space does not hold is -// rewritten to the `_missing_object` sentinel — a mention is a singular -// slot; dropping the mark would lose the fact that a mention existed while -// its text stayed. Then the derived-id fold (foldRef) runs on every mention, -// object-mark and object-link target, exactly as it runs on every other -// reference slot: a text mention of a member spells `participant-` -// like the `Assignee` value beside it, so a reader joins both to the same -// document. Copy-on-write: the snapshot's own marks are caller-owned state -// and are never mutated, and the common case — nothing to rewrite — returns -// the input slice untouched. +// `` whose target the space holds no row for is +// kept verbatim and warned, like every other singular slot +// (singularObjectRef): the mention's text and its address both stay. Then +// the derived-id fold (foldRef) runs on every mention, object-mark and +// object-link target, exactly as it runs on every other reference slot: a +// text mention of a member spells `participant-` like the +// `Assignee` value beside it, so a reader joins both to the same document. +// Copy-on-write: the snapshot's own marks are caller-owned state and are +// never mutated, and the common case — nothing to fold — returns the input +// slice untouched. func (e *exporter) exportMarks(path string, marks []*model.BlockContentTextMark) []*model.BlockContentTextMark { out := marks copied := false @@ -536,10 +607,8 @@ func (e *exporter) exportMarks(path string, marks []*model.BlockContentTextMark) switch m.Type { case model.BlockContentTextMark_Mention: if missingFromSpace(e.opts, m.Param) { - e.warnReference(fmt.Sprintf("%s/text/marks/%d/param", path, i), "mention target %q names no object in this space and is written as %q — "+ - "the mention's own text stays; only its address is gone", m.Param, missingObjectId) - replace(i, m, missingObjectId) - continue + e.warnReference(fmt.Sprintf("%s/text/marks/%d/param", path, i), "mention target %q names no object in this space — "+ + "not synced, or never here — and is written as it stands; the importer resolves it, or writes the sentinel", m.Param) } if folded := e.opts.foldRef(m.Param); folded != m.Param { replace(i, m, folded) diff --git a/codec/anyblockjson/shadowedbundledspelling_test.go b/codec/anyblockjson/shadowedbundledspelling_test.go new file mode 100644 index 0000000..f5da1a5 --- /dev/null +++ b/codec/anyblockjson/shadowedbundledspelling_test.go @@ -0,0 +1,86 @@ +package anyblockjson + +// shadowedbundledspelling_test.go pins the legend entry a bundled spelling +// owes when a LIVE custom property carries the same display name (§3). The +// writer's own vocabulary resolves the spelling bundled-first, so the two +// existing questions — "does the bundled table bind it" and "does the +// writer's vocabulary invert it" — both said yes and no entry was written. +// A reader planning from the bundle's dictionary sees two live claimants +// and refuses the document. Measured: one space renamed the bundled Tag to +// "Regs" and minted a custom property named Tag; 542 object documents +// spelled `Tag` with no legend line, every one refused on read. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-block/format/v1/model" +) + +const customDescriptionKey = "68cdaa41e9223c9dc7ce5f30" + +// shadowingSpaceVocabulary is the writer's side of that space: the bundled +// spelling resolves bundled-first, and the claimant set names both. +type shadowingSpaceVocabulary struct { + BundledKeyVocabulary + // claimantsOnly is the store vocabulary's shape: the space's own + // claimants without the bundled binding added — one entry, not two. + claimantsOnly bool +} + +func (shadowingSpaceVocabulary) PropertySlug(key string) string { + if key == customDescriptionKey { + return "Description" + } + return BundledKeyVocabulary{}.PropertySlug(key) +} +func (v shadowingSpaceVocabulary) PropertyKeyCandidates(spelling string) []string { + if spelling == "Description" { + if v.claimantsOnly { + return []string{customDescriptionKey} + } + return []string{customDescriptionKey, "description"} + } + if key, ok := BundledPropertyKeyByName(spelling); ok { + return []string{key} + } + return nil +} +func (shadowingSpaceVocabulary) TypeKeyCandidates(spelling string) []string { + if key, ok := BundledTypeKeyByName(spelling); ok { + return []string{key} + } + return nil +} +func (shadowingSpaceVocabulary) TypePropertyKeys(string) []string { return nil } +func (shadowingSpaceVocabulary) PropertyTermFacts(string) KeyTermFacts { return KeyTermFacts{} } +func (shadowingSpaceVocabulary) TypeTermFacts(string) KeyTermFacts { return KeyTermFacts{} } + +func TestExport_ABundledSpellingShadowedByACustomNameOwesALegendEntry(t *testing.T) { + snap := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "o1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{ + "id": str("o1"), + "description": str("the bundled one"), + }), + } + + for name, vocab := range map[string]shadowingSpaceVocabulary{ + "claimants plus the bundled binding": {}, + "the space's claimants alone": {claimantsOnly: true}, + } { + t.Run(name, func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_Page, snap, Options{Keys: vocab}) + require.NoError(t, err) + + doc := decodeEnvelope(t, data) + assert.Equal(t, map[string]string{"Description": "description"}, doc.PropertyKeys, + "the spelling is contested in this space, so the document says which key it means") + assert.Contains(t, doc.Properties, "Description") + }) + } +} diff --git a/codec/anyblockjson/snapshotdiff/missingref_test.go b/codec/anyblockjson/snapshotdiff/missingref_test.go index 11265ec..c66de20 100644 --- a/codec/anyblockjson/snapshotdiff/missingref_test.go +++ b/codec/anyblockjson/snapshotdiff/missingref_test.go @@ -1,12 +1,15 @@ package snapshotdiff // missingref_test.go — the missing-reference rule (§9) reaches this -// comparator in the SAME change that taught export: an objects/files value -// entry or an `object_types` entry naming an object the space does not hold -// is dropped by design, and the comparison applies the format's own -// predicate (DroppedMissingObjectRef) to both sides. Without that, every -// document the rule touches would report its dropped entries as data loss — -// the drift class that once produced 1,344 false failures in one sweep. +// comparator through the format's own predicate (DroppedMissingObjectRef), +// applied to both sides. Export keeps an ABSENT real id verbatim now — the +// sentinel is the importer's answer, never the exporter's — so the only +// entry export drops is a stored `_missing_object` sentinel in a list slot, +// and that is the only entry the comparator may excuse. A real id that +// vanishes is loss, whether the store holds it or not. Without the +// predicate, every document carrying a stored sentinel would report its +// drop as data loss — the drift class that once produced 1,344 false +// failures in one sweep. import ( "testing" @@ -75,11 +78,13 @@ func pageSnap(related *types.Value) *model.SmartBlockSnapshotBase { // The real shape of the sweep: an actual round trip through the codec, with // the SAME options handed to export, import and Compare — exactly how -// Heart's extraction-time roundtrip harness wires it. The dropped entries must not report. +// Heart's extraction-time roundtrip harness wires it. The absent id travels +// and compares equal; the stored sentinel drops and must not report. // -// How this can fail: teach export the drop without this file's normalization -// and the objects-format row reports `detail "related" changed` on every -// object carrying a dangling reference — ~990 documents in the last corpus. +// How this can fail: drop the absent id on export and the round-tripped +// side is shorter than the original on every object carrying a dangling +// reference; excuse it in the comparator and a real loss hides behind the +// excuse. ~990 corpus documents carry a stored sentinel in property values. func TestCompare_MissingReferenceDropIsNotLoss(t *testing.T) { t.Run("objects-format value through a real round trip", func(t *testing.T) { // given @@ -87,6 +92,7 @@ func TestCompare_MissingReferenceDropIsNotLoss(t *testing.T) { orig := pageSnap(list(liveCid, deadCid, missingObjectSentinel)) data, err := anyblockjson.Marshal(model.SmartBlockType_Page, orig, opts) require.NoError(t, err) + require.Contains(t, string(data), deadCid, "the absent id is kept on export") sbType, got, err := anyblockjson.Unmarshal(data, opts) require.NoError(t, err) @@ -94,7 +100,35 @@ func TestCompare_MissingReferenceDropIsNotLoss(t *testing.T) { diffs := Compare(orig, got, sbType, opts) // then - assert.Empty(t, diffs, "a dropped-by-design entry is a normalization, not loss") + assert.Empty(t, diffs, "the sentinel drop is a normalization, not loss; the kept id compares equal") + }) + + t.Run("an absent id kept on both sides is not loss", func(t *testing.T) { + // given — the wired store says deadCid has no row; both sides carry it + opts := missingRefOpts() + orig := pageSnap(list(liveCid, deadCid)) + got := pageSnap(list(liveCid, deadCid)) + + // when + diffs := Compare(orig, got, model.SmartBlockType_Page, opts) + + // then + assert.Empty(t, diffs) + }) + + t.Run("an absent id that vanishes IS loss", func(t *testing.T) { + // given — export no longer drops a real id, so a comparator that + // excused its absence would hide a real loss + opts := missingRefOpts() + orig := pageSnap(list(liveCid, deadCid)) + got := pageSnap(list(liveCid)) + + // when + diffs := Compare(orig, got, model.SmartBlockType_Page, opts) + + // then + require.Len(t, diffs, 1) + assert.Contains(t, diffs[0], `detail "related" changed`) }) t.Run("object_types on a property document through a real round trip", func(t *testing.T) { @@ -104,10 +138,11 @@ func TestCompare_MissingReferenceDropIsNotLoss(t *testing.T) { opts.ResolveProperties = relTypeResolver{} orig := relationSnap(map[string]*types.Value{ "relationFormat": number(float64(model.RelationFormat_object)), - "relationFormatObjectTypes": list("typeid-page", deadCid, "wine"), + "relationFormatObjectTypes": list("typeid-page", deadCid, "wine", missingObjectSentinel), }) data, err := anyblockjson.Marshal(model.SmartBlockType_STRelation, orig, opts) require.NoError(t, err) + require.Contains(t, string(data), deadCid, "the absent id is kept on export") sbType, got, err := anyblockjson.Unmarshal(data, opts) require.NoError(t, err) @@ -119,10 +154,10 @@ func TestCompare_MissingReferenceDropIsNotLoss(t *testing.T) { }) } -// The suppression is scoped exactly to what export drops: a LIVE entry that -// vanishes still reports, and with no existence capability in the options -// nothing is suppressed — export dropped nothing, so a shorter list really -// is loss. +// The suppression is scoped exactly to what export drops — the stored +// sentinel, with the capability wired: a LIVE entry that vanishes still +// reports, and with no existence capability in the options nothing is +// suppressed — export dropped nothing, so a shorter list really is loss. func TestCompare_MissingReferenceScopeStaysTight(t *testing.T) { t.Run("a live entry that vanishes still reports", func(t *testing.T) { // given diff --git a/codec/anyblockjson/snapshotdiff/snapshotdiff.go b/codec/anyblockjson/snapshotdiff/snapshotdiff.go index a0e6fd7..597c48c 100644 --- a/codec/anyblockjson/snapshotdiff/snapshotdiff.go +++ b/codec/anyblockjson/snapshotdiff/snapshotdiff.go @@ -241,6 +241,12 @@ func Compare(orig, got *model.SmartBlockSnapshotBase, sbType model.SmartBlockTyp if gotFields[k] == nil && omittable && anyblockjson.OmittedUninstallStamp(k, orig.Details.Fields[k]) { continue } + // a type document's `isUninstalled` stored FALSE (§2a): the + // envelope member is written true only, so absent is the same + // statement. Same ownership of the predicate. + if gotFields[k] == nil && anyblockjson.DroppedTypeUninstallFlag(sbType, k, orig.Details.Fields[k]) { + continue + } // an omitted widget document's residual keys (§2c): the two // object timestamps, and a name that was EMPTY — a non-empty // name keeps the whole document, so within the omitted scope it @@ -359,6 +365,12 @@ var typeKeyIdPrefix = domain.TypeKey("").URL() func compareObjectTypes(orig, got *model.SmartBlockSnapshotBase, sbType model.SmartBlockType) []string { origKeys := typeKeysOf(orig) gotKeys := typeKeysOf(got) + // a type document's own type is written as objectType whatever the + // store held (§2a) — the same normalization export applies, so a + // stored `ot-type` coming back as objectType is not loss + if anyblockjson.IsTypeSmartBlock(sbType) && len(origKeys) > 0 { + origKeys[0] = anyblockjson.TypeKeyObjectType + } modelled := modelledTypeSlots(origKeys, sbType) var out []string diff --git a/codec/anyblockjson/snapshotdiff/typedoc_test.go b/codec/anyblockjson/snapshotdiff/typedoc_test.go new file mode 100644 index 0000000..c5bb385 --- /dev/null +++ b/codec/anyblockjson/snapshotdiff/typedoc_test.go @@ -0,0 +1,53 @@ +package snapshotdiff + +// typedoc_test.go pins two normalizations a TYPE document takes on export +// (SPEC §2a) so the comparator reads them as what they are, not as loss: +// its own type is written as objectType whatever the store held, and a +// stored `isUninstalled: false` comes back absent — written `true` only. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + + "github.com/anyproto/any-block/codec/anyblockjson" + "github.com/anyproto/any-block/format/v1/model" +) + +func typeDocSnap(objectTypes []string, extra map[string]*types.Value) *model.SmartBlockSnapshotBase { + fields := map[string]*types.Value{ + "id": {Kind: &types.Value_StringValue{StringValue: "typeid-wine"}}, + "name": {Kind: &types.Value_StringValue{StringValue: "Wine"}}, + "uniqueKey": {Kind: &types.Value_StringValue{StringValue: "ot-wine"}}, + } + for k, v := range extra { + fields[k] = v + } + return &model.SmartBlockSnapshotBase{Key: "wine", ObjectTypes: objectTypes, Details: &types.Struct{Fields: fields}} +} + +func TestCompare_ATypeDocumentsOwnTypeNormalizesToObjectType(t *testing.T) { + orig := typeDocSnap([]string{"ot-type"}, nil) + got := typeDocSnap([]string{"ot-objectType"}, nil) + assert.Empty(t, Compare(orig, got, model.SmartBlockType_STType, anyblockjson.Options{}), + "the stored junk was normalized, by design; nothing was lost") + + page := &model.SmartBlockSnapshotBase{ObjectTypes: []string{"ot-type"}, Details: &types.Struct{Fields: map[string]*types.Value{ + "id": {Kind: &types.Value_StringValue{StringValue: "p1"}}}}} + pageBack := &model.SmartBlockSnapshotBase{ObjectTypes: []string{"ot-objectType"}, Details: page.Details} + assert.NotEmpty(t, Compare(page, pageBack, model.SmartBlockType_Page, anyblockjson.Options{}), + "on an ordinary document the same change is a real change") +} + +func TestCompare_AStoredFalseUninstallFlagOnATypeComesBackAbsent(t *testing.T) { + stored := map[string]*types.Value{"isUninstalled": {Kind: &types.Value_BoolValue{BoolValue: false}}} + orig := typeDocSnap([]string{"ot-objectType"}, stored) + got := typeDocSnap([]string{"ot-objectType"}, nil) + assert.Empty(t, Compare(orig, got, model.SmartBlockType_STType, anyblockjson.Options{}), + "written true only: absent is the same statement as false") + + trueOrig := typeDocSnap([]string{"ot-objectType"}, map[string]*types.Value{"isUninstalled": {Kind: &types.Value_BoolValue{BoolValue: true}}}) + assert.NotEmpty(t, Compare(trueOrig, got, model.SmartBlockType_STType, anyblockjson.Options{}), + "a TRUE flag that vanishes is loss: the type came back installed") +} diff --git a/codec/anyblockjson/typekeys_test.go b/codec/anyblockjson/typekeys_test.go index 392b336..84fb515 100644 --- a/codec/anyblockjson/typekeys_test.go +++ b/codec/anyblockjson/typekeys_test.go @@ -370,13 +370,30 @@ func TestExport_ASharedTypeSpellingIsHarmless(t *testing.T) { ObjectTypes: []string{"wiki person"}, }} + // the envelope slot, on an ordinary document: a type document's own type + // is normalized to objectType whatever the store holds (§2a), so only a + // page can carry the shared spelling there + page := &model.SmartBlockSnapshotBase{ + Blocks: []*model.Block{{Id: "p1", + Content: &model.BlockContentOfSmartblock{Smartblock: &model.BlockContentSmartblock{}}}}, + Details: fields(map[string]*types.Value{"id": str("p1")}), + ObjectTypes: []string{"ot-" + customTypeKey}, + } + pageData, err := Marshal(model.SmartBlockType_Page, page, Options{Keys: vocab}) + require.NoError(t, err) + pageDoc := decodeEnvelope(t, pageData) + assert.Equal(t, "wiki person", pageDoc.Type, "the vocabulary's spelling, shared or not") + assert.Equal(t, customTypeKey, pageDoc.TypeInternalKey, "and the key says which type it is") + _, pageBack, err := Unmarshal(pageData, Options{GenerateId: seqIds("h")}) + require.NoError(t, err) + assert.Equal(t, []string{"ot-" + customTypeKey}, pageBack.ObjectTypes) + + // the property target slot, on the type document that declares it data, err := Marshal(model.SmartBlockType_STType, snap, Options{Keys: vocab, ResolveProperties: resolver}) require.NoError(t, err) doc := decodeEnvelope(t, data) - assert.Equal(t, "wiki person", doc.Type, "the vocabulary's spelling, shared or not") - assert.Equal(t, customTypeKey, doc.TypeInternalKey, "and the key says which type it is") require.Len(t, doc.TypeProps(), 1) assert.Equal(t, []string{"wiki person"}, doc.TypeProps()[0].ObjectTypes, "a key the fold gate refuses is written verbatim, its own address") @@ -390,7 +407,7 @@ func TestExport_ASharedTypeSpellingIsHarmless(t *testing.T) { r := &recordingPropertyResolver{} _, snap3, err := Unmarshal(data, Options{GenerateId: seqIds("h"), ResolveProperties: r}) require.NoError(t, err) - assert.Equal(t, []string{"ot-" + customTypeKey}, snap3.ObjectTypes) + assert.Equal(t, []string{"ot-objectType"}, snap3.ObjectTypes, "a type document is a Type") require.Len(t, r.defs, 1) assert.Equal(t, []string{"wiki person"}, r.defs[0].ObjectTypes, "verbatim-first: a stored key is its own address in a package-only reader") diff --git a/codec/anyblockjson/typeoftype_test.go b/codec/anyblockjson/typeoftype_test.go new file mode 100644 index 0000000..3fbfaee --- /dev/null +++ b/codec/anyblockjson/typeoftype_test.go @@ -0,0 +1,51 @@ +package anyblockjson + +// typeoftype_test.go pins one derivable fact (§2a): a type document IS a +// Type, so its own type is the bundled `objectType` whatever the store +// holds. Three real type objects, all from an old markdown import, carried +// `ot-type` — a key no space in the account ever minted — and the export +// copied it as `type_internal_key: "type"`, which then named a type +// document the bundle could not carry. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-block/format/v1/model" +) + +func TestExport_ATypeDocumentsOwnTypeIsAlwaysObjectType(t *testing.T) { + snapshot := func(objectTypes ...string) *model.SmartBlockSnapshotBase { + return &model.SmartBlockSnapshotBase{ + Key: "wine", + ObjectTypes: objectTypes, + Details: &types.Struct{Fields: map[string]*types.Value{ + "id": {Kind: &types.Value_StringValue{StringValue: "typeid-wine"}}, + "name": {Kind: &types.Value_StringValue{StringValue: "Wine"}}, + "uniqueKey": {Kind: &types.Value_StringValue{StringValue: "ot-wine"}}, + }}, + } + } + t.Run("a bogus stored type key is normalized and warned", func(t *testing.T) { + var warnings []Issue + opts := Options{ResolveProperties: newTypeIdVocabulary(), OnWarning: func(i Issue) { warnings = append(warnings, i) }} + data, err := Marshal(model.SmartBlockType_STType, snapshot("ot-type"), opts) + require.NoError(t, err) + assert.Contains(t, compactDoc(data), `"type_internal_key":"objectType"`) + assert.NotContains(t, compactDoc(data), `"type":"type"`) + require.Len(t, warnings, 1) + assert.Equal(t, "/type_internal_key", warnings[0].Path) + assert.Contains(t, warnings[0].Message, `"type"`) + }) + t.Run("the correct key is written silently", func(t *testing.T) { + var warnings []Issue + opts := Options{ResolveProperties: newTypeIdVocabulary(), OnWarning: func(i Issue) { warnings = append(warnings, i) }} + data, err := Marshal(model.SmartBlockType_STType, snapshot("ot-objectType"), opts) + require.NoError(t, err) + assert.Contains(t, compactDoc(data), `"type_internal_key":"objectType"`) + assert.Empty(t, warnings) + }) +} diff --git a/codec/anyblockjson/typesettings.go b/codec/anyblockjson/typesettings.go index 9929c73..eb351bd 100644 --- a/codec/anyblockjson/typesettings.go +++ b/codec/anyblockjson/typesettings.go @@ -229,6 +229,28 @@ func (e *exporter) isTypeDoc() bool { return isTypeSmartBlock(e.sbType) } +// TypeKeyObjectType is the bundled key of the Type type: what every type +// document's own type is, by definition (§2a). Export writes it whatever +// the store holds, and the comparator reads the same normalization. +const TypeKeyObjectType = typeKeyObjectType + +// IsTypeSmartBlock reports a type document's smartblock kinds — the scope +// of the §2a normalizations, exported so the comparator applies exactly the +// predicate export applies. +func IsTypeSmartBlock(sbType model.SmartBlockType) bool { + return isTypeSmartBlock(sbType) +} + +// DroppedTypeUninstallFlag reports a type document's `isUninstalled` stored +// FALSE: the envelope member is written `true` only, absent being the same +// statement, so the key comes back absent and nothing was lost (§2a). A +// TRUE flag is not this predicate's business — it travels as `uninstalled` +// and comes back as stored state. Exported so snapshotdiff applies the +// same rule rather than reporting the omission as loss. +func DroppedTypeUninstallFlag(sbType model.SmartBlockType, key string, v *types.Value) bool { + return isTypeSmartBlock(sbType) && key == detailKeyIsUninstalled && !v.GetBoolValue() +} + // buildTypeSettings renders the §2a group, or nil off a type document. The // five settings members follow the §4 omit-empty canon (see // DroppedEmptyTypeSetting for why the §2d mirror does not apply); diff --git a/codec/anyblockjson/uninstalledtype_test.go b/codec/anyblockjson/uninstalledtype_test.go new file mode 100644 index 0000000..724b86c --- /dev/null +++ b/codec/anyblockjson/uninstalledtype_test.go @@ -0,0 +1,91 @@ +package anyblockjson + +// uninstalledtype_test.go pins how a type the user REMOVED from a space +// travels (§2a). Removing a type sets `isUninstalled`, the runtime mirrors +// that into `isDeleted`, and until now the export never wrote the type at +// all — while every object of it kept the key. 64 of the 72 type identities +// the corpus sweep could not resolve were exactly this: full rows, names +// and layouts intact, hidden. The type now travels as an ordinary type +// document carrying `uninstalled: true`, the mirror of the member a removed +// property already has, and restores hidden. + +import ( + "testing" + + "github.com/gogo/protobuf/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-block/format/v1/model" +) + +func uninstalledTypeSnapshot(uninstalled bool) *model.SmartBlockSnapshotBase { + fields := map[string]*types.Value{ + "id": {Kind: &types.Value_StringValue{StringValue: "typeid-wine"}}, + "name": {Kind: &types.Value_StringValue{StringValue: "Wine"}}, + "uniqueKey": {Kind: &types.Value_StringValue{StringValue: "ot-wine"}}, + } + if uninstalled { + fields["isUninstalled"] = &types.Value{Kind: &types.Value_BoolValue{BoolValue: true}} + fields["isDeleted"] = &types.Value{Kind: &types.Value_BoolValue{BoolValue: true}} + } + return &model.SmartBlockSnapshotBase{Key: "wine", ObjectTypes: []string{"ot-objectType"}, Details: &types.Struct{Fields: fields}} +} + +func TestExport_AnUninstalledTypeIsStatedOnTheEnvelope(t *testing.T) { + opts := Options{ResolveProperties: newTypeIdVocabulary()} + + t.Run("the flag is lifted, not spelled as a property", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_STType, uninstalledTypeSnapshot(true), opts) + require.NoError(t, err) + doc := compactDoc(data) + assert.Contains(t, doc, `"uninstalled":true`) + assert.NotContains(t, doc, "Is uninstalled") + assert.NotContains(t, doc, "Is deleted") + require.NoError(t, Validate(data, Options{}), "Marshal never emits what Validate rejects (I1)") + }) + + t.Run("an installed type carries no member", func(t *testing.T) { + data, err := Marshal(model.SmartBlockType_STType, uninstalledTypeSnapshot(false), opts) + require.NoError(t, err) + assert.NotContains(t, compactDoc(data), "uninstalled") + }) + + t.Run("import writes the flag back and the round trip is a fixpoint", func(t *testing.T) { + first, err := Marshal(model.SmartBlockType_STType, uninstalledTypeSnapshot(true), opts) + require.NoError(t, err) + sbType, snap, err := Unmarshal(first, opts) + require.NoError(t, err) + assert.True(t, snap.Details.Fields["isUninstalled"].GetBoolValue(), "restored hidden, as the user left it") + second, err := Marshal(sbType, snap, opts) + require.NoError(t, err) + assert.Equal(t, string(first), string(second)) + }) + + t.Run("the member belongs to type documents only", func(t *testing.T) { + page := `{"formatVersion":"2.0","id":"p1","type":"Page","uninstalled":true}` + require.Error(t, Validate([]byte(page), Options{})) + authored := `{"formatVersion":"2.0","kind":"object_type","id":"type-wine","internal_key":"wine","type":"Type","properties":{"Name":"Wine"},"uninstalled":true}` + require.Error(t, ValidateAuthoring([]byte(authored)), "an author declares a type to use it; there is nothing to uninstall") + }) +} + +// An uninstalled type owns its stored key — every `type-` reference +// still resolves — but not its display name: a live type that took the +// freed spelling must not be shadowed by the corpse. Same rule the store +// resolver applies on the export side. +func TestAuthoringVocabulary_AnUninstalledTypeOwnsItsKeyNotItsName(t *testing.T) { + live := []byte(`{"formatVersion":"2.0","kind":"object_type","id":"type-idea2","internal_key":"idea2","type":"Type","properties":{"Name":"Idea"}}`) + corpse := []byte(`{"formatVersion":"2.0","kind":"object_type","id":"type-idea","internal_key":"idea","type":"Type","properties":{"Name":"Idea"},"uninstalled":true}`) + + vocab, err := PlanAuthoringTypeVocabulary(map[string][]byte{"types/a.json": live, "types/b.json": corpse}, + AuthoringVocabularyPlanOptions{Installed: true}) + require.NoError(t, err) + + key, ok := vocab.TypeKey("Idea") + assert.True(t, ok, "one live claimant, not two") + assert.Equal(t, "idea2", key) + assert.Equal(t, []string{"idea2"}, vocab.TypeKeyCandidates("Idea")) + key, _ = vocab.TypeKey("idea") + assert.Equal(t, "idea", key, "the corpse's stored key is still an address") +} diff --git a/codec/anyblockjson/unresolvedclass_test.go b/codec/anyblockjson/unresolvedclass_test.go new file mode 100644 index 0000000..693f28e --- /dev/null +++ b/codec/anyblockjson/unresolvedclass_test.go @@ -0,0 +1,45 @@ +package anyblockjson + +// unresolvedclass_test.go pins the one predicate that says WHY an index +// target dangles (§2c). The store already draws the line the bundle needs: +// a deleted object keeps a tombstone row ({id, isDeleted}) precisely so a +// link to it can be told apart from a link to an object that has not +// loaded, and the composer, the validator and the comparator must all read +// that line the same way. + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestClassifyUnresolvedTarget(t *testing.T) { + live, tomb, absent := testCid("live"), testCid("tomb"), testCid("absent") + store := deletingStore{ + testObjectStore: testObjectStore{live: "Live", tomb: ""}, + tombstones: map[string]bool{tomb: true}, + } + opts := Options{ResolveObjectNames: store} + + t.Run("a tombstone is deleted", func(t *testing.T) { + assert.Equal(t, UnresolvedDeleted, ClassifyUnresolvedTarget(opts, tomb)) + }) + t.Run("a live row the export did not write is omitted", func(t *testing.T) { + assert.Equal(t, UnresolvedOmitted, ClassifyUnresolvedTarget(opts, live)) + }) + t.Run("no row is absent", func(t *testing.T) { + assert.Equal(t, UnresolvedAbsent, ClassifyUnresolvedTarget(opts, absent)) + }) + t.Run("a store that cannot answer is absent: the fail-safe direction is loss", func(t *testing.T) { + assert.Equal(t, UnresolvedAbsent, ClassifyUnresolvedTarget(Options{ResolveObjectNames: unansweringStore{}}, tomb)) + }) + t.Run("no capability wired is absent", func(t *testing.T) { + assert.Equal(t, UnresolvedAbsent, ClassifyUnresolvedTarget(Options{}, tomb)) + }) + t.Run("an id the store was never the authority for is never asked", func(t *testing.T) { + assert.Equal(t, UnresolvedAbsent, ClassifyUnresolvedTarget(opts, "type-wine")) + }) + t.Run("a row whose deletion cannot be asked is omitted, not absent", func(t *testing.T) { + assert.Equal(t, UnresolvedOmitted, ClassifyUnresolvedTarget(Options{ResolveObjectNames: testObjectStore{live: "Live"}}, live)) + }) +} diff --git a/codec/anyblockjson/validate.go b/codec/anyblockjson/validate.go index bffb1ad..17dbc39 100644 --- a/codec/anyblockjson/validate.go +++ b/codec/anyblockjson/validate.go @@ -80,8 +80,23 @@ const ( type IssueCode string const ( - // IssueCodeUnresolvedTarget identifies a reference to a missing object. + // IssueCodeUnresolvedTarget identifies a reference to an object the + // space had no row for — ABSENT: not synced, or never in this space. + // The one unresolved class that is a loss (§2c, §9). IssueCodeUnresolvedTarget IssueCode = "unresolved_target" + // IssueCodeDeletedTarget identifies a reference to an object the space + // DELETED — a tombstone. Ordinary state, by design; reported so a + // report is complete, never as a loss (§2c). + IssueCodeDeletedTarget IssueCode = "deleted_target" + // IssueCodeOmittedTarget identifies a reference to an object the space + // still holds that the export did not write — archived under an export + // without archived objects, or outside a partial export's scope (§2c). + IssueCodeOmittedTarget IssueCode = "omitted_target" + // IssueCodeUnresolvedType identifies a document naming a type, by + // derived id, that no type document carries and the space has no row + // for — an old import's leftover. A reader imports the object as a + // Page and warns (§2c). + IssueCodeUnresolvedType IssueCode = "unresolved_type" // IssueCodeTypeIdentityMismatch identifies an export rejected because a // type document and its references would use different identities. IssueCodeTypeIdentityMismatch IssueCode = "type_identity_mismatch" diff --git a/docs/superpowers/specs/2026-09-11-uninstalled-types-design.md b/docs/superpowers/specs/2026-09-11-uninstalled-types-design.md new file mode 100644 index 0000000..07cb238 --- /dev/null +++ b/docs/superpowers/specs/2026-09-11-uninstalled-types-design.md @@ -0,0 +1,199 @@ +# Uninstalled types, and two related repairs + +Date: 2026-09-11. Status: approved in discussion; this note records it. +Follows `2026-09-11-unresolved-references-design.md`. + +## 1. Evidence + +After the unresolved-references change, 9 of 79 corpus bundles still fail +validation on 143 references to 72 type identities no type document +carries. Read against the retained source stores: + +| Class | References | Identities | Store evidence | +| --- | ---: | ---: | --- | +| uninstalled types | 135 | 64 | a full row with `isUninstalled` and `isDeleted`, name and layout intact | +| type documents whose own type is `type` | 3 | 1 | legacy markdown imports; no store in the account holds a type keyed `type` | +| types an old import never created | 5 | 5 | two markdown pages, two template targets, one profile object; the `type` id has no row in its space | +| ambiguous `Tag` declaration | 1 | | six declarations spell `Tag` with `internal_key: "tag"` beside a custom property named `Tag` | + +The first class is a plain export omission. Removing a type from a space +sets `isUninstalled`; `injectDerivedDetails` mirrors that into +`isDeleted`; the exporter enumerates through `List`, which injects +`isDeleted != true`, and its emit skip refuses anything flagged deleted. +The space still holds the definition and the objects still carry the key. + +## 2. Decisions + +1. **An uninstalled type is exported, and restored uninstalled.** It + travels as an ordinary `object_type` document with `uninstalled: true` + on the envelope, the mirror of the member a removed property already + has on its declaration and dictionary entry (§15 #22). Import writes + `isUninstalled` back, the runtime mirrors it into `isDeleted`, and the + type lands hidden, resolvable for its objects, reinstallable from the + library. Restoring it live would change a choice the user made, so that + is not done. When the destination already holds a live type with that + key, the destination wins and stays live. +2. **The planner lets an uninstalled type own its key, not its name.** + `type-` references resolve; a live type with the same display name + is not shadowed by the corpse. Same rule the store resolver applies. +3. **A type document's type is `objectType` by definition.** It is + derivable from the kind, like the layout keys §2a already drops, so + export writes `objectType` and warns when the store holds something + else. +4. **`internal_key` wins over the spelling** in a property declaration. + Export writes an agreeing pair, an author never writes `internal_key`, + and the authoring subset refuses it, so the precedence only ever + decides in favor of what the exporter knew. +5. **The store resolver's vocabulary listing names uninstalled types.** + The type listing passes an explicit `isDeleted` none-filter, as it + already does for `isArchived`, so a double-flag row keeps its + id-to-key naming while staying out of the name namespace. +6. **A bundled spelling a live custom property also carries owes a legend + entry.** Found by the fresh export: with the declarations resolving, 542 + object documents in the Tag space spelled `Tag` with no legend line, + because the writer's vocabulary resolves the spelling bundled-first and + the bundled table binds it, so neither existing question said an entry + was owed. When a scoped vocabulary reports more than one claimant for + the spelling, the export writes the entry. +7. **The comparator reads the two type-document normalizations as + normalizations.** Its own type coming back as `objectType`, and a + stored `isUninstalled: false` coming back absent, are not loss; the + codec exports both predicates and `snapshotdiff` consults them. +8. **A dictionary entry's spelling outranks a display-name claim.** Repair + 6 did not fire on real data: the store resolver withholds a label from + the custom property precisely because it shadows the bundled name, so + its claimant set for "Tag" held the bundled key alone, while the + reader's planner registered the custom entry's `name: "Tag"` as a + claim. The dictionary states outright, through the entry's `property` + member, that the bundled key is spelled "Tag" in this bundle; the + planner now records every entry's spelling and lets it decide the + candidate set before any name claim. This is what makes the Tag space + importable as already exported. Repair 6 stays as defense for a + vocabulary whose claimant set does include the shadowing name. + +9. **A type the space never held imports as a Page** (decided later the + same day, replacing the "keep the key verbatim" recommendation). The + composer declares such derived ids under `unresolved.types`, the + validator admits a declared one as a warning (`unresolved_type`) and + refuses an undeclared one, the export report warns per reference, and + the converter rewrites the object's type — a template's target — to + Page with a warning naming the document and the type. Verbatim, the + object is unsearchable by type and gets a guessed layout; a Page is + what the user can work with. +10. **A stored `_missing_object` in a relation's target list drops from the + dictionary entry.** Found by the first import sweep over fresh + exports: three spaces failed in the converter on `type + "_missing_object" has no declaration`, because an old importer had + written the sentinel into `relationFormatObjectTypes` and the + dictionary path copied it while the document path already dropped + it. The dictionary writer now applies the same predicate, and the + converter drops a sentinel target from an older export with a warning. +11. **An option key with a slash gets an escaped archive id.** The + converter names a native archive entry by the object id, and an + option's key is minted from its name, so "C/C++" refused a whole space + as an unsafe id. The archive id escapes the slash; the option's real + key travels as the snapshot key, and every value naming the option + resolves to the same escaped id. +12. **An absent bookmark object restores as a URL-only bookmark.** The pb + importer logged an error and kept the stale target when a bookmark + block's object was not in the import. The target is now cleared with + a warning, so the bookmark syncer fetches the object again from the + URL — the most a restore can recover, and the harness no longer reads + the log line as a failed import. + +## 5. Defects found by a five-lens review of this work, and fixed + +A five-lens Opus review over the finished change set found these. Each is +now fixed with a test that fails without the fix. The nine-space sweep had +passed green with all four present: the corpus does not exercise them. + +13. **A declared missing type validated and then refused to convert.** The + census declares a type reached through four document slots, but the + repair rewrote only an object's own type and a template's target. A type + named by `query_source.types`, by a dictionary entry's `object_types`, + or by a type's `property_definitions` passed validation with the "imports + as a Page" warning and then hard-failed conversion. The resolver now + treats a declared missing key as a SOFT miss: it returns not-found + without arming the sticky error, so each slot degrades the way §9 already + degrades an absent reference. Identity slots still become Page; a slot + that merely references the type drops the entry with a warning. +14. **An over-declaring index silently retyped live objects.** Nothing + checked a declared type against the documents the bundle carries, so an + index naming a carried type retyped every object of it to Page and + orphaned the type document. The validator now refuses that contradiction, + and the converter ignores a declaration for a type it can see. +15. **The composer never censused the property dictionary.** A relation + document is never written, so a property's target types reach a bundle + only through its dictionary entry — a slot the validator checks and the + composer did not, so the composer could ship a bundle its own validator + refuses. The dictionary's targets now join the type census. +16. **The option archive id was not injective.** `/` was escaped without + escaping the escape character first, so `a/b` and `a%2Fb` collapsed onto + one entry: one option was dropped, and which one survived depended on map + iteration order, breaking byte-determinism. `%` is escaped first. + +## 6. Deferred, with the design settled (2026-09-12) + +An uninstalled BUNDLED type still restores installed. The importer drops the +creation payload for a bundled key and installs the type before the +create-or-reset branch, and that install clears both the uninstalled and the +deleted flag, so decision 2's guard always sees a freshly reinstalled row and +strips the incoming flag. Measured: 0 of the 68 removed type keys in the +corpus are bundled, so this is unobserved there. It is reachable for any user +who removes a shipped type such as Task and then restores. + +The user framed the governing principle: **a backup states what the space was +at the moment it was taken, so restoring it should reproduce that moment.** +Under that rule the matrix is + +| backup | destination | today | principle | +|---|---|---|---| +| live | uninstalled | live | live | +| live | absent | live | live | +| uninstalled | absent | **live** | uninstalled | +| uninstalled | live | **live** | uninstalled | + +The first two rows already hold: a bundled type is reinstalled by the install +path, and a custom one by the state reset, which removes both flags because +the backup carries neither. Rows three and four both fail, and row four is +decision 2's guard, which deliberately lets the destination win. + +The agreed resolution, NOT implemented: scope the guard by the importer's +existing new-space flag. On a restore into a new space the bundle is the +authority in both directions, which makes row three and row four correct. On +an import into an existing space the guard stays, so a bundle can never hide +a type the destination has live — the case that matters when a use case or a +shared experience is installed into a space someone is working in. That is +one condition, not the pre-import liveness capture first proposed. + +Deferred by the user on 2026-09-12: leave the behavior as it is for now. + +## 3. Touch points + +any-block: `codec/anyblockjson` export (envelope lift, type normalization, +`authoredIdentity`), import (write-back), `authoringtypes.go` (claims), +`format/v2/schema/object.schema.json` (gated `uninstalled`), SPEC §2a, §2b +declaration table, §15. Heart: `core/block/export/collection.go` +(enumeration), `core/block/export/anyblock/anyblock.go` (emit gate), +`pkg/lib/anyblockjson/storeresolver/keyvocab.go` (listing filter), +`core/block/import/common/objectcreator` (live type wins). + +## 4. Tests + +Codec: lift and write-back round trip; no member without the flag; the +authoring subset refuses it; the planner keeps the key and yields the +name; type normalization with its warning; `internal_key` tie-break on +the `Tag` shape; a contested bundled spelling owes a legend entry; the +comparator's two normalizations. Heart: an uninstalled type row is exported with the +member; single-document export of one succeeds; the vocabulary names a +double-flag row; an incoming uninstalled type does not uninstall a live +destination type. Corpus: the existing bundles predate the exporter fix, so the check is a +fresh export of the nine failing spaces from the retained account with +the rebuilt exporter, validated with the format's own tool. Result: five +validate outright — the Tag space among them, settled by repair 8 on its +first fresh export — and four fail only on the five never-created types. +With every repair in: all nine fresh bundles validate (the five never-created +types as `unresolved_type` warnings), and the headless import sweep imports +all nine into a fresh account — 5 Page normalizations, 7 dictionary sentinel +drops, 5 bookmarks rebuilt from their URLs, no importer errors. On the old +corpus that is 79 of 79 active spaces, from 17 at the start of the day. diff --git a/docs/superpowers/specs/2026-09-11-unresolved-references-design.md b/docs/superpowers/specs/2026-09-11-unresolved-references-design.md new file mode 100644 index 0000000..a7221cc --- /dev/null +++ b/docs/superpowers/specs/2026-09-11-unresolved-references-design.md @@ -0,0 +1,333 @@ +# Unresolved references: deleted, omitted, absent + +Date: 2026-09-11. Status: approved design, awaiting review of this text. + +Repositories: `any-block` (format, composer, validator, converter) and +`anytype-heart` (exporter wiring, export report, importer). One any-sync +dependency is named in §6. + +## 1. Problem + +Real space exports are rejected by `bundle.Validate` before a single object +is restored. On the 79 non-empty bundles of the September account sweep, 62 +failed validation. The dominant cause is `index.json` naming objects the +bundle does not carry: + +| Slot | Bundles | References | +| --- | ---: | ---: | +| homepage | 56 | 56 | +| icon.file | 29 | 29 | +| widget target | 12 | 66 | + +The exporter is behaving as designed. Three rules combine: + +1. The exporter never writes an object flagged deleted (`writeDoc` in + Heart's export service). +2. The format treats a tombstone as an existing row, so document-level + references to deleted objects stay verbatim (SPEC §9). +3. The composer lifts homepage, widget targets and the icon into the index + verbatim, lists what it did not write in `unresolved.targets`, and the + validator refuses every listed id anyway (SPEC §2c, "Stating a target + does not make it legal"). + +The corpus confirms the mechanism: of the missing index targets, 41 of 56 +homepages, 20 of 29 icons and 12 of 62 widget targets are still referenced +verbatim by other documents in the same bundle, which means the store held a +row for them at export time. They are tombstones. + +The user's framing, adopted here: Anytype deliberately does not rewrite +every object that mentions a deleted one. A reference to a deleted object is +normal state and must import without complaint. A reference to an object +the space does not hold at all, typically because it has not synced yet, is +a real loss and must reach the user as a warning at export and at import. + +## 2. Goals and non-goals + +Goals: + +- Every whole-space export whose only defect is dangling index targets + validates and restores. +- The bundle states why each dangling target is dangling, so a reader can + distinguish a known deletion from a loss without the source store. +- Authored bundles keep strict validation: an author's dangling reference is + an authoring error. +- Restore preserves the "deleted" meaning of a reference instead of + degrading it to "not found". +- Export stops manufacturing the `_missing_object` sentinel for references + whose target may merely be unsynced. + +Non-goals: + +- Missing type references (`type-` naming no type document) and the + ambiguous `Tag` declaration. Nine bundles fail on those and they need the + key-aware audit described in the import handoff. §8 lists one likely + contributor. +- Changing what the exporter collects. Deleted objects stay unexported. +- Any change to reserved listings or the auto-widget ledger. + +## 3. Classification at export + +The composer already holds the store resolver set. At `Finish`, for each id +`Index.ReferencedObjectIds` names that no written document carries, it asks +the two existing capabilities, `ObjectExistenceResolver` and +`ObjectDeletionResolver`, and classifies: + +| exists | deleted | class | meaning | +| --- | --- | --- | --- | +| true | true | deleted | tombstone; the space deleted it, by design | +| true | false | omitted | the space holds it and this export did not write it: archived under an export without archived objects, or outside a partial export's scope | +| false | any | absent | the space holds no row; not yet synced or never here | +| unknown | | absent | the store failed to answer; the fail-safe direction is "loss" | + +Only CID-shaped ids are asked. A `type-` widget target, a participant +reference or any other derived id is declared but never classified, and +therefore counts as absent. Reserved listings and the auto-widget ledger are +never declared, as today. The codec exports the classification helper, so +the composer, the comparator and the validator share one predicate and one +id-shape gate. + +One slot is not declared: a space icon whose image object is a tombstone. +The document-level rule already drops a deleted icon because an icon is +optional (`DroppedDeletedIconRef`, SPEC §2b and §9). The composer applies +the same predicate to `icon.file` at `Finish`, drops it with the same +warning, and the index falls through to whatever icon channel is left. An +omitted or absent icon is kept and declared like any other target. + +Both resolver methods ride `GetDetails`, a raw document lookup with no +default filter injection. Tombstones and archived rows are visible on that +path. This was verified against `spaceindex.getDetails` and +`storeresolver.objectRow`. + +## 4. Index format + +`unresolved` gains two members beside the existing `properties` and +`targets`: + +```json +{ "unresolved": { + "targets": ["bafy…A", "bafy…B", "bafy…C"], + "deleted": ["bafy…A"], + "omitted": ["bafy…B"] } } +``` + +- `targets` keeps its current meaning: every id the index names that the + bundle does not carry. It is the admission list. +- `deleted` and `omitted` are subsets of `targets`, disjoint from each + other. An id in neither is absent. +- All three are sorted string lists, folded like every other reference, + and written only when non-empty. The empty `unresolved` object stays + refused. +- The full index schema admits the two members. The authoring index schema + is unchanged: it is closed and has no `unresolved` member, so an authored + bundle cannot declare a loss. + +Why subsets rather than a reason per entry: it keeps the current `targets` +semantics and the current exemption logic intact, and the format already +speaks in sorted lists. + +## 5. Validation + +### 5.1 Structured result + +`bundle.Validate` returns one flat error today. The design adds +`bundle.Inspect(fsys) (*Report, error)` and `bundle.InspectAuthoring`, where +`Report` carries issues with a severity (`error`, `warning`, `info`), a +stable code and a path. `Validate` and `ValidateAuthoring` keep their +signatures and fail exactly when the report holds an error-severity issue, +so existing callers see no change except the admissions in §5.2. + +### 5.2 Rules + +| Case | Full surface | Authoring surface | +| --- | --- | --- | +| index target declared in `targets` and in `deleted` | info, code `deleted_target` | error | +| index target declared in `targets` and in `omitted` | warning, code `omitted_target` | error | +| index target declared in `targets`, in neither subset | warning, code `unresolved_target` | error | +| index target not declared | error | error | +| `deleted` or `omitted` names an id not in `targets` | error | error | +| `deleted` and `omitted` overlap | error | error | +| `manifest.files` names a missing document | error | error | + +The declaration is the only thing that makes a dangling target admissible, +and only on the full surface. An undeclared dangling target stays an error +on both surfaces because it means the exporter shipped a reference without +noticing. + +The reserved-target and auto-widget exemptions are unchanged. + +### 5.3 CLI + +`anyblock validate` prints warnings and info after errors, grouped by +severity, and exits non-zero only on errors. A `-strict` flag makes +warnings fail as well, for CI use on authored bundles. + +## 6. Restore + +### 6.1 Converter + +`bundle/convert.Bundle` runs `Inspect` on the full surface, fails on errors, +forwards warnings and info through `OnWarning`, and returns the three sets +on the result, unfolded to store ids: + +```go +type Result struct { + … + Unresolved struct{ Deleted, Omitted, Absent []string } +} +``` + +Homepage and widgets are converted verbatim as today. + +### 6.2 Importer + +Heart's importer rewrites every reference whose id is not in the import's +old-to-new map to `_missing_object`, in `common.UpdateLinksToObjects` and +`UpdateObjectIDsInDetails`, and `WidgetObject.Init` then strips widget links +to the sentinel. That collapses deleted and absent alike. The change: + +- **Declared deleted.** The importer seeds the old-to-new map with each + declared-deleted id mapped to itself, so every rewrite site keeps the id: + links, bookmarks, file and dataview targets, mention and object marks, + icon images, relation values and widget links. It then ensures a + tombstone for the id in the destination: + - id has a live row in the destination (restore into the same space): + keep the id, write nothing; the reference re-links. + - id has a tombstone already: nothing to do. + - id has no row: write a tombstone. +- **Declared omitted or absent, and undeclared.** Today's behavior: the + sentinel is written, the widget is stripped on next load. Each declared + id produces an import report entry naming the slot and the class, at + warning severity. +- **Import report for deleted ids.** One entry per declared-deleted id at + info severity, so the report stays complete without drowning the + warnings that matter. + +Tombstone route. The durable route is the synced deletion log, so that +every device sees the tombstone and a full reindex rebuilds it: +`reindexDeletedObjects` rebuilds tombstones only from +`AllDeletedTreeIds`. any-sync's `settingsObject.DeleteObject` refuses an id +with no local head entry, so this needs a small any-sync addition: record a +deletion for an id that has no tree in this space. Until that exists, the +importer writes a store-only tombstone through `spaceindex.DeleteObject`. +Its known limits: other devices do not see it, and a full reindex drops it, +after which the reference degrades to absent, which is exactly today's +state. The any-sync change is tracked as a follow-up in §8 and is not a +blocker for this work. + +### 6.3 Round trip + +- Deleted: exported as declared deleted, restored with a tombstone, + re-exported as declared deleted. Stable in one generation. +- Absent: exported verbatim and declared, restored as the sentinel, + re-exported as a stored sentinel. Converges in one generation, as the + current §9 rule already promises for rewrites. +- Omitted: same as absent on restore. On a later export of the source it + either becomes an ordinary document or stays omitted. +- Index slots: the homepage relation is longtext and is never remapped, so + an absent homepage stays the raw id in the restored space and is declared + absent again on every later export, with the same warning each time, + until the user picks a new homepage. That is stable, and it is the + correct signal. + +## 7. Export-side document references + +Approved change to SPEC §9 "References the space cannot serve": + +- **Absent targets are kept verbatim.** Singular slots no longer rewrite + to `_missing_object`; list slots no longer drop the entry. Each keeps its + id and produces a warning naming the slot, the id and the class. The + importer already manufactures the sentinel for anything it cannot + resolve, so nothing that would be lost on restore is preserved by the + rewrite, and a backup taken during a sync gap keeps the id for the day + the object arrives. +- **Stored sentinels are unchanged.** A `_missing_object` the space + already holds is written as is in singular slots and dropped from lists, + as today. +- **The deleted-icon drop is unchanged.** It answers a deletion, which is + decidable, not a sync state. +- `snapshotdiff` and `DroppedMissingObjectRef` follow: the comparator no + longer treats an absent real id as a normalization, because nothing is + dropped. + +The export report gains two codes beside `unresolved_target`: +`omitted_target` (warning) and `deleted_target` (info). Absent index targets +and absent document references share `unresolved_target`. Source-path +attribution stays as it is. + +## 8. Follow-ups and future improvements + +Recorded here so they are not lost. None is in scope for this change. + +1. **Sync-layer refinement of "absent".** The space sync status keeps the + head-sync missing id list per space (`spaceSyncStatus.UpdateMissingIds`). + A new optional resolver capability can sharpen an absent id into + "pending sync", when peers hold it, versus "unknown to the network". The + index would gain a `pending` subset and the warning would say so. +2. **Warn when exporting offline or long unsynced.** Signals exist today: + the space sync status reports `Offline`, a non-zero syncing objects + counter and a non-empty missing id list, and every object carries + `syncStatus` and `syncDate` details. The export should add a report + warning, and the clients a confirmation, when the space is offline, has + objects pending, or has not completed a sync for a long time, since a + backup taken in that state is the one most likely to declare absent + targets. +3. **Vocabulary query hides uninstalled types and relations.** The + resolver's type and relation listing in `keyvocab.go` goes through + `Query`, which injects `isDeleted != true`, and a UI-removed type or + relation carries both `isUninstalled` and `isDeleted` in production + (`injectDerivedDetails`). Those rows fall out of the id-to-key naming, + and `TypeKeyById` has no point-lookup fallback. Fix: pass an explicit + `isDeleted` filter with the none condition, as the listing already does + for `isArchived`, and add a test asserting `TypeKeyById` on a + double-flag row. Likely a contributor to the missing type references + class. +4. **any-sync: record a deletion for an id with no local tree.** Needed for + the durable tombstone route in §6.2. +5. **Missing type references and the ambiguous `Tag` declaration.** The + key-aware audit from the import handoff. +6. **Fidelity verification** of restored spaces beyond absence of import + errors, as the handoff already suggests. + +## 9. Testing + +any-block: + +- Composer with a fake resolver: each of deleted, omitted, absent and + unknown lands in the right list; non-CID targets are declared and not + classified; reserved and ledger entries are never declared; folding is + applied to the subsets; a deleted `icon.file` is dropped with a warning + and not declared, while an omitted or absent one is declared. +- Index codec: round trip of the two new members, schema admission on the + full schema, refusal on the authoring schema, empty-object refusal kept. +- Validator matrix from §5.2 on both surfaces, including the subset + consistency errors and the `manifest.files` error. +- Converter: `Result.Unresolved` populated and unfolded; a bundle with only + declared targets converts. +- Export codec: absent singular and list references kept verbatim with a + warning; stored sentinel behavior unchanged; deleted-icon drop unchanged; + `snapshotdiff` agrees. +- The worked authoring example still validates warning-free. + +Heart: + +- Exporter: report codes per class, source-path attribution kept. +- Importer: declared-deleted id kept in every rewrite site and a tombstone + written; live row in destination re-links without a tombstone; absent id + becomes the sentinel with a report entry; widget with a deleted target is + kept, widget with an absent target is stripped; homepage verbatim. +- Sweep: rerun the 79 active spaces with deleted skips enabled. Expected: + the homepage, icon and widget classes no longer fail validation; the + remaining failures are the type reference and `Tag` classes. + +## 10. Spec text to change + +- SPEC §2c "What the bundle names and cannot answer for": the two new + members, the subset rules, and the reversal of "stating a target does not + make it legal" for the full surface. +- SPEC §9 "References the space cannot serve": absent targets kept + verbatim; the sentinel is the importer's, not the exporter's. +- SPEC §12: the structured report and the severity rules. +- SPEC §15: a decided entry recording this design and the deleted versus + absent principle. +- `bundle/README.md`, `bundle/convert/README.md`, `cmd/anyblock/README.md` + for the new surfaces and flags. diff --git a/format/v2/README.md b/format/v2/README.md index baeca52..ea5db74 100644 --- a/format/v2/README.md +++ b/format/v2/README.md @@ -12,6 +12,7 @@ the format Anytype exports to, imports from, and serves over API v2. | [`examples/reader`](examples/reader) | those nine steps, runnable, standard library only | | [`examples/exported_space`](examples/exported_space) | a tiny export-shaped bundle to run them on | | [`examples/habit_tracker`](examples/habit_tracker) | an authoring bundle — what a person writes by hand | +| [`SKILL.md`](SKILL.md) | agent skill for creating a bundle around a user's use case | | [`SPEC.md`](SPEC.md) | normative and complete | | [`PRINCIPLES.md`](PRINCIPLES.md) | the ten rules the format answers to | diff --git a/format/v2/SKILL.md b/format/v2/SKILL.md new file mode 100644 index 0000000..615c737 --- /dev/null +++ b/format/v2/SKILL.md @@ -0,0 +1,107 @@ +--- +name: anyblock-v2-bundle +description: Create AnyBlock v2 bundles for Anytype from a user's use case, with suitable object types, typed properties, meaningful blocks, and useful views. Use for authoring a new bundle or extending an authored bundle. +--- + +# Create an AnyBlock v2 bundle + +Build a bundle that helps the user manage information and discover useful insights. A valid JSON bundle is only the starting point: its types, properties, content, and views should support the user's actual workflow. + +## Authoritative local references + +Repository root: `/Users/roman/anytype/any-block`. + +Read the authoring schemas before generating documents. These are the schemas for creating a bundle from scratch: + +| Surface | Absolute schema path | +| --- | --- | +| Objects, types, and templates | `/Users/roman/anytype/any-block/format/v2/schema/authoring/object.schema.json` | +| Bundle index and navigation | `/Users/roman/anytype/any-block/format/v2/schema/authoring/index.schema.json` | +| Shared property definitions | `/Users/roman/anytype/any-block/format/v2/schema/authoring/properties.schema.json` | + +Use these local files to inspect the grammar. In generated JSON, set `formatVersion` to `"2.0"` and `$schema` to the corresponding published URL, for example `https://schemas.anytype.io/anyblock/2.0/authoring/object.schema.json` (replace `object` with `index` or `properties` for those files). + +**Do not read the full `SPEC.md`: it is huge.** Author from this skill, the authoring schemas, and the examples below. Only consult `/Users/roman/anytype/any-block/format/v2/SPEC.md` when a validation error remains unclear after checking the schema and examples. Search for the failing field or error and read only the relevant section: §2a for types, §2c for bundles, §2f–2g for properties and authoring, §6.2 for dataviews, or §9 for references. Use `/Users/roman/anytype/any-block/format/v2/INLINE_MARKUP.md` when writing rich inline text. + +## Start from the authoring examples + +Read the relevant files in this complete, maintained authoring bundle: + +| Example | Absolute path | +| --- | --- | +| Index, entrypoint, and widgets | `/Users/roman/anytype/any-block/format/v2/examples/habit_tracker/index.json` | +| Shared properties and select options | `/Users/roman/anytype/any-block/format/v2/examples/habit_tracker/properties.json` | +| Custom type, property membership, and views | `/Users/roman/anytype/any-block/format/v2/examples/habit_tracker/types/habit.json` | +| Typed object with property values and nested blocks | `/Users/roman/anytype/any-block/format/v2/examples/habit_tracker/objects/morning-run.json` | +| Typed object with a checklist | `/Users/roman/anytype/any-block/format/v2/examples/habit_tracker/objects/weekly-review.json` | +| Welcome content and object links | `/Users/roman/anytype/any-block/format/v2/examples/habit_tracker/objects/start.json` | + +Adapt the structure to the user's domain. The example's `Page` is a welcome document; it is not the default type for the user's records. Export examples carry bookkeeping that fresh authoring should omit. + +## Model the user's use case + +Understand what the user wants to capture, update, connect, and learn from the information. Infer this from their request and supplied material; ask a focused question only when a missing answer would materially change the model. + +Choose a small set of meaningful types around the entities the user manages, such as projects, contacts, recipes, habits, or research findings. Reuse a suitable built-in type when its meaning and properties fit; otherwise define a custom type. Avoid `"type": "Page"` whenever a more useful domain type fits. Use Page for genuinely general content, such as a welcome guide, when appropriate. Do not confuse the semantic `type` with `kind`: ordinary domain objects normally omit `kind`, whose default is `page` regardless of their type. + +Check built-in definitions in `/Users/roman/anytype/any-block/codec/anyblockjson/vocabulary/types.json` and `/Users/roman/anytype/any-block/codec/anyblockjson/vocabulary/relations.json` (the stored property vocabulary) before reusing names or defining custom ones. Do not copy their export or internal metadata into authored documents, or declare a custom type whose identity or name collides with a built-in. + +Choose properties that make records easy to maintain, filter, sort, and compare. Use object references for relationships between entities and consistent select options for categories or stages. Think about the questions the user wants answered—what needs attention, what is overdue, which items belong together, or how activity changes over time—and include the data needed to answer them. If historical insights matter, model dated observations or events; a single current value cannot supply a history. + +## Declare types and populate their properties + +- Give each custom type a document under `types/` with `kind: "object_type"`, a unique `internal_key`, matching `id: "type-"`, and `properties.Name` containing its display name. Put its layout, plural name, property membership, and default view in `type_settings`. +- Define shared custom properties once in root `properties.json`, with their formats and any options or target types. Reference them by display name from each type's `type_settings.property_definitions`. Inline property definitions are also supported, but definitions of the same property must agree across types. Use `section: "featured"` for the few values the user should see first. +- Give every ordinary object its chosen `type` by display name, a unique bundle-local `id`, and `properties.Name`. **Populate the applicable properties declared by that type**, using values from the user's material or clearly identified example data. A typed object should not be just a title with all its useful information buried in prose. Keep domain properties aligned with the type's declared property set; add a useful missing property to the type instead of silently introducing an ad hoc field on one object. +- Do not invent facts to fill every property. Omit unknown values, preserve meaningful `false`, `0`, empty arrays, or `null` where valid, and make any sample data distinguishable from real user data. +- Write property names consistently across definitions, object values, dataview columns, filters, and sorts. Text is a string, numbers are JSON numbers, checkboxes are booleans, and dates are RFC 3339 UTC strings. `select` and `multi_select` values are arrays of option names, even for one selection. `objects` and `files` values are arrays of object ids. +- For a relationship property, use `format: "objects"` and constrain `object_types` to suitable type display names or derived type ids. Its values reference the related object documents' ids, not their names or filenames. + +## Give objects meaningful blocks + +**Every authored content object should have a non-empty `blocks` array as well as its properties.** Use blocks for the information someone needs when opening it: context, notes, instructions, decisions, evidence, checklists, or links to related objects. Avoid filler or simply repeating all property values as paragraphs. Type documents should have useful blocks too, usually a dataview over their objects; templates should provide reusable content structure when the workflow calls for one. + +Write blocks as a flat array in reading order. Omit `indent` at level zero; increase it by at most one level for a child. Never use nested `children` arrays. Use supported block types such as `paragraph`, `heading_2`, `bulleted_list_item`, `checkbox`, `callout`, `link`, and `dataview`. Put inline formatting in `text`, following the local inline-markup guide. The title belongs in `properties.Name`; do not invent a title block. + +Omit block, view, sort, and filter ids in the authoring profile. Keep the document-level ids that cross-file references need. Do not add export-only legends, `type_internal_key`, attribution, system timestamps, or stored app state. + +## Make navigation and insights useful + +Add views that answer the user's questions using the declared properties: a filtered list of items needing attention, a board grouped by a meaningful select property, or a table sorted by a relevant date or number. Choose useful columns and clear view names. Do not claim computed metrics or automatic updates unless the bundle actually implements them; explain any manually maintained metric in the content. + +A type's dataview belongs in the type document's `blocks` and ranges over that type's objects. Dataviews contain view definitions, not embedded record rows. Omit `object_id` for the host's own dataview. For a separate Query, declare `query_source.types`; for a Collection, declare `collection_items`. An inline dataview targeting a different Query or Collection uses that object's id. Use the authoring schema and the habit type example for sources, filters, and grouping. + +Choose a useful entrypoint and link to the main types or views. Declare it in root `index.json` and explicitly list the intended sidebar widgets; setting an entrypoint does not insert a widget. A type widget with `layout: "view"` shows that type's default view. Keep navigation focused on the user's workflow. + +## Assemble, validate, and deliver + +Use one JSON document per object, with a simple directory layout matching the example: + +```text +bundle/ + index.json + properties.json + types/ + .json + objects/ + .json +``` + +Include `properties.json` when declaring shared custom properties. Keep each object's identity in its `id`; references use ids, not file paths. Avoid reserved ids and keep `type-` ids exclusively for the matching type documents. The authoring index uses directory discovery and does not include a `manifest`. + +**An existing validator CLI is available at `/Users/roman/anytype/any-block/cmd/anyblock`.** Use its `validate` subcommand for individual JSON files or bundle directories. It reports diagnostics and exits nonzero on failure. Its usage is documented in `/Users/roman/anytype/any-block/cmd/anyblock/README.md`. The `go run` command below works without installing an `anyblock` binary globally or adding one to `PATH`. + +**After generating or modifying a bundle, run the validator on the actual output before delivery.** Validate each file against its corresponding local authoring schema, then execute the existing CLI from the repository root, replacing `/absolute/path/to/bundle` with the generated bundle's path: + +```sh +cd /Users/roman/anytype/any-block +go run ./cmd/anyblock validate /absolute/path/to/bundle +``` + +The CLI checks the full format and bundle semantics; it does not enforce every authoring restriction and currently has no strict authoring flag. For strict authoring validation, use `bundle.ValidateAuthoring(os.DirFS(bundlePath))` from `github.com/anyproto/any-block/bundle`. Also validate the index with `anyblockjson.ValidateAuthoringIndex` and the dictionary, when present, with `anyblockjson.ValidateAuthoringPropertyDictionary` from `github.com/anyproto/any-block/codec/anyblockjson`; the bundle authoring validator does not apply those two subset schemas itself. + +Read the validator's diagnostics, fix failures, and rerun validation until it passes. If an error cannot be understood from the schema and examples, consult only the relevant spec section as described above. Do not merely recommend the command to the user or substitute validation of the example bundle for validation of the generated output. If validation cannot run, state that limitation explicitly and do not claim the bundle passed. + +Review the bundle's coherence beyond schema validity: references resolve, objects carry their types' relevant properties, objects have useful blocks, and views have the data they need to answer the intended questions. Do not present schema validation as proof that a view was exercised in the app. + +Deliver the bundle at the user's requested location, with a brief explanation of its types, how to use its views, any sample data or manual updates, and the validation actually performed. If an archive is requested, package the validated bundle with `index.json` at the archive root. diff --git a/format/v2/SPEC.md b/format/v2/SPEC.md index a0589d4..50ef074 100644 --- a/format/v2/SPEC.md +++ b/format/v2/SPEC.md @@ -315,13 +315,14 @@ Fields, in **canonical order** (§4): | `type` | string | no | The object's type **document spelling**, canonically its NFC display name (`Page`, `Task`, `Property`) — the key vocabulary of §3, not the stored `ot-`-prefixed key and not a derived API slug. Legacy derived slugs such as `object_type` remain input-only compatibility spellings; canonical re-export uses the display name. Maps to `object_types[0]` in the snapshot. Absent when the snapshot has no object types (legacy/system objects). **The stored key stands beside it in `type_internal_key`, on every typed document**, so a reader resolves the key and shows the spelling; only a document that states no key — an authored one — has its `type` inverted through the §3 chain in the type namespace (the vocabulary in force: the bundled table offline, the space's stored names and compatibility spellings inside a node), and the resulting stored key is handed to the wiring, which resolves it — matching an existing type or creating one (the Markdown importer's behavior). A term the chain does not know passes through verbatim — an exact stored key is always its own address (§3). No spelling is reserved: `template` is an ordinary type term that a key or vocabulary may bind wherever it likes, because `kind` — a field no chain touches — is the sole template authority. With no `kind`, even a literal `"type": "template"` is an ordinary page type (§10). | | `template_for` | string | no | Only for templates: the target type (`object_types[1]`), written as the type's **derived id** `type-` (§9) — a reference by key, the same spelling every id-valued slot folds a type to, so a template names its type the way a filter or a `Set of` value does and a reader never resolves a spelling here. That describes the DEFAULT shape; a SINGLE DOCUMENT exported under the `NoDerivedTypeIds` mode spells the vocabulary here instead — the same word the envelope `type` writes — and a reader resolves it through the §3 chain. No bundle carries that shape: the mode is scoped to one document and the `bundle` package refuses it (§9). On input a display name (`"Task"`, `"Habit"` for a type this bundle declares) or the legacy `ot-` is accepted through the §3 chain, for authoring (§2g); canonical export writes the derived id. Admitted on `kind: "template"` and nothing else — present without it, or without a `type` beside it to be `object_types[0]`, is a validation error. Note what this is NOT keyed off: the template's own type. A template whose `object_types` do not begin with the template key is a shape the model permits. The target does not depend on what `object_types[0]` holds. | | `internal_key` | string | no | Identity key of *system* objects (types, properties). This is the STORED identity key (a `uniqueKey`'s internal part), written verbatim: unlike every key slot in §3 it is **not** translated, so for an object whose stored key is a minted BSON it does not match the slug the public API serves as that object's `key`. The name says what the value is — an id the app MINTS (a bson for a custom definition, the camelCase bundled key for a bundled one), never something an author derives — where the word `key` used to name this stored id AND a property definition's spelling one level down, one word for two concepts (§15 #14). Because it is verbatim, its charset is whatever the store already holds: a relation option's key is built from the option's *name*, so `completion_status_Not Started`, `…_C/C++` and `…_тогглы` are all real stored keys. The rule is therefore a deny rule — non-empty — not an allowlist. An allowlist was tried and falsified: it failed 59 objects of a 36 808-object account, every one a relation option. Length and charset are not bounded either: the app mints an option's key from the option's name, and the name is whatever an import carried, so any bound here makes an object the store already holds unexportable. `Marshal` never emits what `Validate` rejects (§11) is the stronger promise. Never emitted for ordinary documents. | +| `uninstalled` | bool | no | Only for type documents (`kind: "object_type"`): the user **REMOVED** this type from the space (stored `isUninstalled`), and the bundle carries it for backup fidelity — its objects still carry the key, and the definition is still the space's (§2a). The mirror of the member a removed property's declaration and dictionary entry carry (§2f). Written `true` only; absent is the same statement as `false`. A reader restores it hidden, as the user left it, never as an installed type, and a live type owning the same stored key in the destination stays live. Output-only: the authoring subset refuses it. Present on any other kind → validation error. | | `property_settings` | object | on `kind: "property"` | Only for property documents, where it is **required**: the definition of the property this document IS — one `propertyDefinition` (§2d, §2e). Carries `format` (required, a §3 format NAME — never a raw enum number; stands for the stored `relationFormat` key, which `properties` refuses), `include_time` and `object_types`, each present exactly when its stored key is, value included. Illegal on every other kind. | | `icon` | object | no | The object's icon — ONE object whose `format` selects the variant (§2b). Stands for the stored `iconEmoji` / `iconImage` / `iconName` / `iconOption` keys, which `properties` refuses. | | `cover` | object | no | The object's cover — same shape, three variants (§2b). Stands for the stored `coverId` / `coverType` / `coverScale` / `coverX` / `coverY` keys, which `properties` refuses. | | `properties` | object | no | The object's properties, §3. | | `type_settings` | object | no | Only for type documents (`kind: "object_type"`, `"bundled_object_type"`): everything that defines the TYPE, in one gated subtree — `layout`, `api_key`, `plural_name`, `default_template`, `default_view`, and `property_definitions` (§2a). Present on any other kind → validation error. The root spelling `type_properties` is refused with the repair named. | | `file_remote` | string | no | Only for `kind: "file_object"` or legacy `"file"`: standard base64 encoding of independently versioned JSON containing the remote CID, encryption keys, and optional indexed variant metadata (§2h). | -| `property_internal_keys` | object | no | Legend: the stored property key each spelling in this document names (§3). Written for every spelling the **bundled table does not bind to the key being written** — a spelling the table cannot invert (a space's own key) *and* the **identity entry**, which is the ordinary case: a custom key written verbatim names itself, because nothing else in the document says the term is a stored key rather than somebody's display-name spelling. A reader consults it **before** its own vocabulary and takes the value as **authoritative**: it is not liveness-checked, deliberately (§3). Absent only from a document whose every spelling is bundled. | +| `property_internal_keys` | object | no | Legend: the stored property key each spelling in this document names (§3). Written for every spelling the **bundled table does not bind to the key being written** — a spelling the table cannot invert (a space's own key) *and* the **identity entry**, which is the ordinary case: a custom key written verbatim names itself, because nothing else in the document says the term is a stored key rather than somebody's display-name spelling — and for a bundled spelling that a **live custom property also carries as its name**: the writer's own vocabulary resolves such a spelling bundled-first, so the table and the writer both say "no entry owed" while a reader planning from the bundle's dictionary sees two live claimants; a scoped vocabulary reporting more than one claimant is what makes the entry owed (measured: one space renamed the bundled Tag to "Regs" and minted a custom Tag; 542 documents spelled `Tag` with no entry). A reader consults it **before** its own vocabulary and takes the value as **authoritative**: it is not liveness-checked, deliberately (§3). Absent only from a document whose every spelling is bundled. | | `type_internal_key` | string | no | The STORED type key the `type` spelling names — the bundled key (`page`, `task`) or the minted key of a space's own type — written on **every** document that states a `type`, bundled or not (§15 #28). A scalar, because an object has exactly one type: a map overstated the shape. Import takes it as **authoritative** and never resolves the spelling beside it; the spelling is the caption a reader shows. Canonical export writes it after `type`; a key the writable-key rule cannot hold (over-long, control characters) is not written, with a warning, and `type` then carries the key verbatim. Present without `type` is a validation error. The former `type_internal_keys` map is retired: a template's target and every `object_types` entry are the type's derived id `type-` (§9) and need no legend, so the map had exactly one entry left to hold. (In a SINGLE DOCUMENT exported under the `NoDerivedTypeIds` mode those two slots spell the vocabulary rather than the derived id; the type namespace carries no legend either way, and a bundle refuses that mode — §9.) A document carrying the map is refused with the repair named (§10). | | `option_ids` | object | no | Legend: the id of the option each select/multi_select **name** in this document stands for — nested, `{property spelling: {option name: option id}}` (§3, §9a). Written **unconditionally** wherever export spells an option by name; dropped by `OmitIds` (§9). Read as a **hint**, not an address: an id is honored only where the target space still serves it as a live option of that relation, and otherwise the name resolves exactly as it did before the legend existed. | | `blocks` | array | no | The document's blocks as a **flat pre-order array**; nesting via `indent` (§4). | @@ -449,6 +450,32 @@ query-source lift (§6.2) and is the only kind test that lift makes: on a type document there is nothing left to lift, so a type document carries no `query_source` either, and one that states it is refused. +**A type the user removed still travels, and comes back removed.** Removing +a type from a space sets `isUninstalled`, and the app mirrors that into +`isDeleted` on load, which is what hides it — the definition stays, and +every object of the type keeps the key. Measured on the 79-bundle corpus +before this rule: 64 of the 72 type identities that objects named and no +document carried were exactly this, full rows with names and layouts +intact, dropped because the export enumerated through a query that refuses +`isDeleted` rows. The type now travels as an ordinary type document +carrying `uninstalled: true` on the envelope, the mirror of the member a +removed property's declaration and dictionary entry carry (§2f); the stored +flag is lifted there and never spelled as a property. Import writes the +flag back, so the type restores hidden and reinstallable from the library, +never as an installed type — restoring it live would change a choice the +user made — and a live type owning the same stored key in the destination +stays live. In the vocabulary an uninstalled type owns its stored key, so +`type-` references resolve, but claims no spelling, so a live type +that took the freed name is not shadowed by the corpse; the store resolver +applies the same rule on the export side. + +**A type document's own type is `objectType`, by definition.** It is +derivable from the kind, like the layout keys above, so export writes it +whatever the store holds and warns when the two differ. Three real type +objects, all from an old markdown import, stored `ot-type` — a key no space +in the account ever minted — and copying it named a type document no +bundle could carry. + Seven candidates FAILED the admission test, and six of them stay in `properties`: `is_hidden` (cannot be proven install-only), `layout_width`/`layout_align` (the type object's own @@ -490,7 +517,7 @@ style. | Field | Type | Req | Notes | |---|---|---|---| | `property` | string | no* | The property's document-facing SPELLING — a key slot like any other, inverted through `property_internal_keys` (§3). Deliberately not called a key: the word used to name this spelling AND the envelope's stored id at once (§15 #14). | -| `internal_key` | string | no* | The property's STORED internal key, verbatim — never run through the §3 ladder, because a stored id is its own address and the bundled fold would rebind a slug-shaped one (`due_date` onto `dueDate`). Export writes it beside `property` for fidelity; an author never needs it, and cannot produce a correct one for a custom property (the app mints those — a bson id). *An entry must state an identity: `property`, or `internal_key`, or a `name` the spelling derives from; when both `property` and `internal_key` are present the spelling wins, and export writes an agreeing pair. A custom property whose entry states no `internal_key` gets a FRESH minted internal key from the import wiring's create path, the way the app mints one when a user creates a property — the spelling must not silently become the stored key. | +| `internal_key` | string | no* | The property's STORED internal key, verbatim — never run through the §3 ladder, because a stored id is its own address and the bundled fold would rebind a slug-shaped one (`due_date` onto `dueDate`). Export writes it beside `property` for fidelity; an author never needs it, and cannot produce a correct one for a custom property (the app mints those — a bson id). *An entry must state an identity: `property`, or `internal_key`, or a `name` the spelling derives from; when both `property` and `internal_key` are present the spelling wins, and export writes an agreeing pair — except where the spelling is CONTESTED (several live claimants in the bundle's vocabulary), where the stated `internal_key` is the tie-break: it is the remedy the refusal names, and an export states it beside every spelling. A custom property whose entry states no `internal_key` gets a FRESH minted internal key from the import wiring's create path, the way the app mints one when a user creates a property — the spelling must not silently become the stored key. | | `name` | string | no | Display name. Import uses it only when the property must be **created**; an existing property keeps its own name. Every bundled key already exists, so a name given for one is inert — `{"property": "Description", "name": "Summary"}` renders as *Description*. Validation warns. If the label is the point, mint a custom key instead of reusing a bundled one. | | `format` | string | no | Property format (§3 names). Same import rule as `name`; a conflict with an existing property's format is an error at the wiring level (the package cannot see the space). | | `options` | (string \| object)[] | no | A select/multi_select property's **vocabulary, in display order**. Each entry is a bare option name, or `{"name": …, "color": …}` when the option's color is part of the design — the color belongs to the option rather than to a parallel array, so inserting or reordering an option cannot shift it. `color` is one of `grey`, `yellow`, `orange`, `red`, `pink`, `purple`, `blue`, `ice`, `teal`, `lime` (`util/constant`); anything else is a validation error rather than a silently ignored value. The bare string is **canonical** only when the option carries NONE of `color`, `internal_key` and `api_key`; any one of the three present makes the object form canonical, and export writes every member it holds. The criterion is all three because the object form is the only one that can STATE what it holds: a bare name for an option carrying a stored key or an api key would erase the option's stored identity and the spelling its API callers address it by, and neither is derivable from the name (below). A scalar for the simple case and an object for the enriched one is the SHAPE cells follow in §6.1; the criterion is this one, and it is the same one §2f's dictionary entry uses — one serializer writes both homes. Leaving a color out does not mean *no* color: the wiring assigns one, cycling the palette in declaration order and skipping whatever the vocabulary claims explicitly, so a vocabulary that names no colors still gets distinct ones. (The app assigns one at random on every other creation path; cycling keeps a converted bundle identical run to run.) Options are otherwise discovered only from values that happen to be used, so a vocabulary entry no record carries would never exist — its kanban column simply absent — and a discovered option carries no `orderId`. Declaring them lets the wiring create each one up front with an order id. The app's own vocabulary listing puts every option carrying an `orderId` first, those ascending, then the ones carrying none, `createdDate` descending — the picker's subscription sorts `orderId` ascending with no empty-placement, which lists the order-less ones first, and the picker then re-sorts the received rows so that an option with an order id precedes one without. Since a new option is minted with the smallest order id of its siblings, ascending order ids and descending creation dates agree on newest-first. Two options tying on both — a `createdDate` is a whole-second stamp — are then ordered by the option's own id, ascending: a third key the app never needs and a writer does, without which two options minted in the same second swap places between runs. A bundle writes the array in the RENDERED order, with that tie-break (§2f). (Sorting objects by their tag COLUMN is a different feature with a different rule, `[orderId, name]` concatenated per record — `pkg/lib/database.BuildOrderMap`; it says nothing about how a vocabulary lists.) Names discovered from usage rather than declared are ordered after the declared ones. The object form takes two more members, `internal_key` and `api_key` — the option's STORED key, which the app mints and an author never writes, and its public API key (stored `apiObjectKey`), the spelling callers address it by. Export states each where the store holds one. The stored key is what lets a bundle STATE a vocabulary rather than describe it (§2f, where the dictionary entry states a vocabulary in these same members — the dictionary's entry and a type's are the two homes of this shape that admit them, since no option document carries either). The api key is not a slug of the name: it does not follow a rename and nothing rewrites it, and it travels because no restore mints one (§15 #21). Only meaningful on `select`/`multi_select`; duplicate names are a validation error in a TYPE's definition, across both forms — authoring resolves an option by its name and so cannot state one twice. The property dictionary is the exception: its entries carry explicit `internal_key`s, which tell same-named twins apart, and real spaces hold them (§2f). | @@ -571,6 +598,7 @@ object**, whose `format` member says which kind it is: ```json { "icon": { "format": "emoji", "emoji": "📕" } } { "icon": { "format": "file", "file": "bafyreicfdcmfn…" } } +{ "icon": { "format": "cid", "cid": "bafybeic42xaqr…" } } { "icon": { "format": "icon", "name": "hammer", "color": "orange" } } { "icon": { "format": "color", "color": "teal" } } @@ -579,15 +607,34 @@ object**, whose `format` member says which kind it is: { "cover": { "format": "gradient", "gradient": "pinkOrange" } } ``` -### `icon` — four variants +### `icon` — five variants | `format` | required | optional | stands for | |---|---|---|---| | `emoji` | `emoji` (non-empty string) | `color` | `iconEmoji` | -| `file` | `file` (object reference) | `color` | `iconImage[0]` | +| `file` | `file` (object reference) | `color` | `iconImage[0]` when the stored value is an object id | +| `cid` | `cid` (a raw content cid) | `color` | `iconImage[0]` when the stored value is a content cid — output-only | | `icon` | `name` (a built-in icon name) | `color`, `emoji` (output-only) | `iconName` | | `color` | `color` | — | `iconOption` alone | +- **`cid` is an image with no object behind it.** A participant's avatar + and a 1-to-1 space's icon are stored as the image's raw content cid — the + client fetches the bytes by content id through the gateway or the + identity repo, and API v2 serves the same value — and no file object in + the space, let alone the bundle, stands behind it. `file` used to carry + both meanings and left a reader to decode the codec to tell them apart; + one member with two meanings is a defect, so the content address has its + own variant, and the CODEC decides which one an export writes: an object + id is dag-cbor (`bafyrei…`), a content cid is dag-pb or raw (`bafybei…`), + and the two never share a codec, so this is a classifier rather than a + heuristic. Nothing folds, declares or resolves a `cid`: it names no + object the bundle could carry, so `bundle.Validate` requires no document + for it and the composer never lists it under `unresolved` (§2c). Import + writes it back to the same stored slot. It is output-only in the + authoring subset (§2g): an author cannot mint a content address, and + supplies an image as a file object. A `file` holding a content cid — the + spelling exporters wrote before this variant — is read as the variant for + compatibility, and is never written. - **`color` is on every variant, because it is orthogonal to the source.** 87 production objects attach one to something other than a named icon — 53 workspaces and 2 profiles give an avatar image its background color, 3 an @@ -1165,11 +1212,13 @@ the one file that describes the bundle as a whole: ```json { "unresolved": { "properties": ["66602dc5e5672d06c0e19245"], - "targets": ["bafyrei…"] } } + "targets": ["bafyrei…A", "bafyrei…B", "bafyrei…C"], + "deleted": ["bafyrei…A"], + "omitted": ["bafyrei…B"] } } ``` -Two sorted lists, each present only when it has something to report, and -each earning its place differently. +Two sorted lists and two subsets of the second, each present only when it +has something to report, and each earning its place differently. - **`properties`** — the stored property keys the documents reference and nothing could define, written verbatim (a key nothing defines has no @@ -1200,11 +1249,71 @@ each earning its place differently. audited space: its homepage and 16 of its 23 widget targets, 17 in all, which is exactly what `bundle.Validate` reports for those slots. -**Stating a target does not make it legal.** `bundle.Validate` still refuses -a bundle whose index points at a document it does not carry. What the -statement buys is the distinction the refusal cannot make: an export that -KNEW what it could not carry, against one that shipped a dangling reference -without noticing. +- **`deleted`** and **`omitted`** — two subsets of `targets`, disjoint, + saying WHY a target dangles. The store already draws the line: a deleted + object keeps a tombstone row (`{id, isDeleted}`) precisely so that a link + to it can be told apart from a link to an object that has not loaded, and + the deletion is a space-level, synced fact. The composer asks the two + capabilities the exporter already wires (`ObjectExistenceResolver`, + `ObjectDeletionResolver`, §9) through one exported predicate, + `ClassifyUnresolvedTarget`, and every reader applies the same one: + + | the store says | class | meaning | + |---|---|---| + | a tombstone | `deleted` | the space deleted it, by design. Anytype does not rewrite every object that mentions a deleted one, so a reference to a tombstone is ordinary state | + | a row, not a tombstone | `omitted` | the space holds it and this export did not write it: archived under an export without archived objects, or outside a partial export's scope. A loss in this bundle, not in the space | + | no row, or no answer | neither: **absent** | the space had no row for it — not yet synced, or never here. The one class that is a loss, and the fail-safe direction: a store that did not answer, a capability that is not wired, and an id the store was never the authority for (a `type-` widget target, a participant) all land here | + + Only CID-shaped ids are asked. One slot is not declared at all: a space + icon whose image object is a tombstone follows the document-level icon + rule (§2b, §9) — an icon is optional, so it is dropped with a warning and + the index falls through to whatever channel is left. +- **`types`** — the TYPES the bundle's documents name by derived id (a + `type_internal_key`, a `template_for`, an `object_types` entry) that no + type document carries. A type is never deleted in Anytype and every + uninstalled one travels (§2a), so what remains here is a type the source + space never held: an object or template an old import created with a + type it never made. Measured on the corpus: five, from three importers. + Spelled as the derived id — the address the document uses and the + validator checks — and never folded; a bundled key never appears, since + every reader carries the shipped table. Declaring it is what makes the + bundle admissible on the full surface, as a warning; an undeclared one is + still refused as an exporter bug, and the authoring surface refuses both. + A reader imports an object of such a type as a **Page**, and a template + for one as a template for Page, each with a warning naming the document + and the type: restoring the dangling key verbatim would reproduce an + object no type search finds and the app can only guess a layout for, + which reads as broken, and a Page is what the user can work with. That + substitution is for the slots that say what an object **is**. A slot that + merely REFERENCES the type — a `query_source.types` entry, a property's + `object_types` — drops the entry with a warning instead, the way §9 + already has a list express an absent reference by being shorter: nothing + there is broken, only a stale constraint, and retyping it to Page would + widen a query or a target set to every page in the space. + + **Declaring a type the bundle CARRIES is refused.** The list states what + the source space never held, and a reader acts on it by retyping objects; + an index naming a carried type would silently retype live objects and + orphan their type document, so the contradiction is refused at the door + like every other self-contradicting report. + +**Stating a target is what makes it admissible, and only on the full +surface.** `bundle.Validate` admits a declared target, and `bundle.Inspect` +reports it at the severity its class earns (§12): a `deleted` entry is +info, an `omitted` entry a warning, and a target in neither subset a warning +under the `unresolved_target` code. An UNDECLARED dangling target is still +refused on both surfaces, because it means the exporter shipped a reference +without noticing — the distinction the old refusal could not make is now +the whole rule. `bundle.ValidateAuthoring` refuses every dangling target +whatever the index says: an author's dangling reference is an authoring +error, and the authoring index schema has no `unresolved` member to declare +one in (§2g). A subset entry that `targets` does not declare, or an id in +both subsets, is refused at both doors, like the empty object. On restore +the importer keeps a `deleted` id in every reference slot and writes a +tombstone for it in the destination, so the reference restores as what it +was; an `omitted` or absent id takes the sentinel the importer has always +written for a target it cannot resolve (§9), and each declared id is +reported by class. **An absent `unresolved` is not a completeness claim**, and the empty object is refused rather than allowed to become one. What a writer checks here is @@ -1454,16 +1563,17 @@ can answer — the §11 normalization, a respelling and not a rebinding, since the id is the store's own spelling for the same type. Only a key nothing can answer for stays a key, and that is where a key is its own address (§3): a key is vocabulary, and a vocabulary miss is never evidence of nonexistence, -so the term survives rather than dropping. What no longer -passes is an entry the space's own store disowns (§9): the -`_missing_object` sentinel, and an object id the wired existence capability -says names no row — 56 production properties carry one, type ids from the -account where a shipped use case was AUTHORED, and an object id differs in -every space while a type key does not. Both drop, the real id with a warning -naming it; `object_types` is a list, and a list expresses absence by being -shorter. Note the store answers for CORPSES too: an uninstalled type still -has a row and still inverts through `TypeKeyById` (its id names something), -so only an id with no row at all drops. Without any resolver every stored ID +so the term survives rather than dropping. An object id the wired +existence capability says names no row — 56 production properties carry +one, type ids from the account where a shipped use case was AUTHORED, and +an object id differs in every space while a type key does not — is KEPT +verbatim with a warning naming it (§9): the store can only say the row is +not here, not that the object is nowhere, and the sentinel is the +importer's answer. What no longer passes is only a stored `_missing_object` +sentinel: `object_types` is a list, a list expresses absence by being +shorter, and the sentinel carries nothing. Note the store answers for +CORPSES too: an uninstalled type still has a row and still inverts through +`TypeKeyById` (its id names something). Without any resolver every stored ID passes through verbatim, and the offline round trip is byte-exact for every entry alike: a bare key still crosses as `type-` and comes back that key, and an id the store merely could not be asked about is still the stored @@ -3054,7 +3164,7 @@ Values are encoded by the property's format: | `checkbox` | boolean | | `date` | RFC 3339 date-time string, UTC (`"2026-07-06T15:04:05Z"`); import converts back to unix seconds. Import also accepts date-only strings (UTC midnight), non-UTC offsets (converted to UTC), and fractional seconds (truncated to whole seconds). Export always writes the full UTC form — **except** for a stored value outside the years RFC 3339 can express (0000–9999), which export writes as the **raw number**, with a warning. There is no string form for such a value, and writing one anyway (`"57482-01-22T22:43:20Z"`, from milliseconds stored where seconds belong) would not parse back, so the value would return as a *string* on a date property and stay one. A reader must therefore accept a number here; the number is a stored value it cannot interpret as a date, not a second date encoding. | | `select`, `multi_select` | array of option **names** (strings) — see below | -| `objects`, `files` | array of object ids (strings). A resolver-wired export drops an entry the SPACE does not hold — the stored `_missing_object` sentinel included — and the emptied list stays `[]`, because the key's presence is meaningful; a package-only export drops nothing (§9) | +| `objects`, `files` | array of object ids (strings). A resolver-wired export keeps an entry the SPACE holds no row for, with a warning, and drops only a stored `_missing_object` sentinel; the emptied list stays `[]`, because the key's presence is meaningful; a package-only export drops nothing (§9) | | `properties` | array of **stored property keys** (strings), verbatim. This is the one property-naming slot in the format that does NOT spell a property the document-facing way: every other one — a property block's `property`, a link block's `properties`, a dataview column's, a type's `property_definitions` entry — carries the SPELLING and resolves it through the chain above, and this carries the key the chain resolves to. Export writes the stored key raw here while spelling the same key by name in a property block beside it, legend line and all; import mirrors that, so with `property_internal_keys: {"priority": "67abc"}` in the document a property block's `"property": "priority"` names `67abc` and this slot's `["priority"]` stores the literal string `priority`. A reader therefore looks each element up in `properties.json` DIRECTLY (§2f) and never through the legend — the value already is the key a legend maps a spelling to — and says so when nothing defines it, the way it says so for a reference no document carries. Not to be confused with the `properties` MEMBER of a link block (§5), which holds spellings and resolves them. No export in the 79-bundle corpus declares this format on anything — 0 occurrences of `"format": "properties"` across all 24,889 documents and all 5,385 dictionary entries — which is why the cardinality rule below sat in this document while the importer wrapped only the other four | | unresolvable format | value passes through verbatim in both directions | @@ -5199,7 +5309,7 @@ a document is found by its id and by nothing else (§2c). | `bafyrei…` — a bare object id (a CID, lowercase base32; older spaces also hold 24-hex bson ids) | every reference slot: object/file property values, `collection_items`, block `object_id`s, filter values, sort `custom_order`, `object_orders`, icon/cover `file`, index `entrypoint`/`homepage`/widget `target` | the document whose envelope `id` is that string | the object exists in its space and did not travel, or the space deleted it — **the bundle cannot tell you which**, and neither can a reader. Measured: 1,265 of 10,053 reference occurrences in a deliberately narrow census (property values, `collection_items`, block targets, icon/cover) name no document here, over 723 distinct ids. For the ids `index.json` itself names, the export says so: `unresolved.targets` (§2c) | | `type-` — a type, by its stored key (§9 *Derived ids*) | a type document's own `id`; `template_for`; every `object_types`; `query_source.types` (§6.2); the `Template's Type` and `Default type id` values; a view's `default_type_id`; a filter `value`; a link or dataview block's `object_id`; a widget `target` | the document whose `id` is that string. The key is the text after the prefix, so the reference says WHICH type without any lookup at all | a **bundled** key (`type-page`) needs no document — every reader has it in the shipped table, and `bundle.Validate` exempts it. A minted key (`type-68c2…`) that finds no document is a real dangling reference. Measured over the audited space (3,286 documents), which is the population every figure in this row counts: 354 occurrences across nine slots; 92 typed documents (29 distinct keys) name a `type-` no document here carries | | `participant-` — a space member, by account identity (§9 *The participant fold*) | a participant document's own `id`, the two attribution properties, and any slot whose VALUE passes the identity's checksum — the classifier is the value's shape, never the property's name | the participant document with that id. An importer rebuilds the store's composite `_participant__` against its own `Options.SpaceId` | a reader that sets no `SpaceId` stores the folded id, which addresses nobody; it is told so once per document (§13). Measured: 6,569 occurrences | -| `_missing_object` — the space's own sentinel for a reference it could not serve | singular slots only: a block `object_id`, a `` target. A list slot drops the entry instead of writing the sentinel | it does not resolve — **it is the answer.** The link or mention existed and its target does not | already nothing: which object it was is gone. Measured: 12 | +| `_missing_object` — the space's own sentinel for a reference its IMPORTER could not serve | singular slots only: a block `object_id`, a `` target. A list slot drops a stored sentinel instead of writing one. Export never writes it: an absent id is kept verbatim and warned (§9) | it does not resolve — **it is the answer.** The link or mention existed and its target does not | already nothing: which object it was is gone. Measured: 12 | | `_anytype_profile` — the platform's own profile object | `Created by`, on all 1,880 of this space's participant documents and nowhere else | a platform address, not a bundle id | no bundle carries a document for it and none is missing | | `_participant__` — an unfolded participant composite | a value naming a member of a DIFFERENT space, which passes through whole in both directions rather than being re-homed | read the identity out of it; it is not this space's member | it names another space's member, so this bundle owes no document. Measured: no reference slot in the 79-bundle corpus holds one — the eight textual occurrences sit inside prose in a `text` member, which nothing resolves | | `_date_` — a virtual date object | mention targets, and a link block's `object_id`. Measured corpus-wide: 104 (103 mentions, 1 block target) | the date is IN the id; the platform mints the object on demand | nothing is missing: no space stores a row for it, and no export can | @@ -5786,45 +5896,58 @@ no fold at all. ### References the space cannot serve -A reference to an object that does not exist in the SPACE is not written as -if it did. The space stores already state this for the references -their importers resolved — `_missing_object` -(`pkg/lib/localstore/addr.MissingObject`) stands 1,089 times across a -28,617-document corpus — and export now applies the same honesty to ids -that dangle without the sentinel, and a consistent policy to the sentinels -it re-exports. - -**The split is by what the slot can express.** +A reference to an object the SPACE holds no row for is written exactly as +the space holds it, and the loss is STATED rather than enacted. The space +stores already carry a sentinel for the references their importers could +not resolve — `_missing_object` (`pkg/lib/localstore/addr.MissingObject`) +stands 1,089 times across a 28,617-document corpus — and that sentinel is +the importer's answer, never the exporter's: export keeps an absent id +verbatim and warns, and only a sentinel the space already holds moves. + +**Why the id is kept.** "Missing from this export", "missing from this +device" and "missing from the world" are three different facts, and the +store can only ever testify to the second. An object A synced ahead of the +object B it mentions leaves B with no row on this device until B arrives; +a backup taken in that window that rewrote B's id would have destroyed, +for good, the one thing a later sync could still make whole. The importer +writes the sentinel for every target it cannot resolve anyway +(`common.UpdateLinksToObjects`), so a rewrite at export preserved nothing +on the restore side — it only removed the choice. The exporter's job is to +carry what the space holds and say what the space could not serve; the +sentinel is a verdict, and the verdict belongs to the side that resolves. + +**What moves, by what the slot can express.** - A **singular slot** — a block's `object_id` (link, bookmark, file, image, video, audio, pdf, dataview) and a `` target (§8) - — REWRITES the id to `_missing_object`. Omission cannot express "no - target" there: only deleting the block (or the mark) could, and that - would lose the fact that a link or mention existed — the mention's text - stays, only its address is gone. A stored sentinel is kept as-is. + — keeps an absent id verbatim, with a warning naming the slot and the + id. A stored sentinel is kept as-is, silently: it already says "gone". - A **list slot** — an objects/files property value (§3), a property - document's `object_types` (§2d) — DROPS the entry: a list expresses - absence by being shorter. A stored sentinel drops too. The emptied list - stays `[]`, never omitted: the key's presence is meaningful (§3), and for - `object_types` an empty list is a cleared target set (§2d). -- Everything else is deliberately out of scope: collection `collection_items`, filter + document's `object_types` (§2d) — keeps an absent id verbatim, with the + same warning. A stored sentinel DROPS, silently: a list expresses absence + by being shorter, the sentinel carries nothing — which object it was is + already gone — and the corpus holds ~990 of them in property values + alone, which would triple a warning channel that was just cut down to + what is worth reading (§12). The emptied list stays `[]`, never omitted: + the key's presence is meaningful (§3), and for `object_types` an empty + list is a cleared target set (§2d). +- Everything else is out of scope: collection `collection_items`, filter values, custom orders, `object_orders`, a type's `default_template_id`, - and object-link marks keep their ids verbatim. Each of those can be - extended later on this section's precedent; none was in the evidence. + and object-link marks keep their ids verbatim and warn about nothing. **"Missing from this export" and "missing from the space" are different -facts, and only the second may cause a rewrite.** An export of a single -object references its neighbours in the space; those objects exist and were -simply not exported, and rewriting them would corrupt a perfectly good -export. The exporter never sees the export set — it works one document at a -time — so the only question it can ask is of the STORE, which is the right -question: does this space hold a row for this id? The answer comes through -the `ObjectExistenceResolver` capability (§13), asked affirmatively — -`known && !exists` — so a store failure moves nothing. And it is a NEW -capability because the resolver already standing in the object namespace -cannot answer it: `ObjectNameResolver`'s ok is `name != ""`, which reads -"exists but untitled" as "no". Untitled objects are common; conflating the -two questions rewrites live references. +facts, and only the second may warn.** An export of a single object +references its neighbours in the space; those objects exist and were +simply not exported, and warning about them would drown the channel. The +exporter never sees the export set — it works one document at a time — so +the only question it can ask is of the STORE: does this space hold a row +for this id? The answer comes through the `ObjectExistenceResolver` +capability (§13), asked affirmatively — `known && !exists` — so a store +failure says nothing. And it is a separate capability because the resolver +already standing in the object namespace cannot answer it: +`ObjectNameResolver`'s ok is `name != ""`, which reads "exists but +untitled" as "no". Untitled objects are common; conflating the two +questions would warn about live references. **The question only reaches ids the space index is the authority for**: CID-shaped ids (`isObjectIdShaped` — `cid.Decode` behind a length gate). @@ -5832,38 +5955,40 @@ A `_date_…` id is virtual, `_ot…`/`_br…` resolve against the bundled tables, a participant composite against the fold, a bare type key against the key vocabulary, a widget link target against the editor's constants — a store that was never an id's authority cannot declare it missing, so -none of those are ever asked about, let alone rewritten. A deleted -object's tombstone is a row: its id still means something in this space, -and references to it are untouched — with ONE deliberate exception, the -icon. An icon is optional where a link or mention target is not, so an -`iconImage` whose target the space DELETED is dropped rather than kept or -rewritten: export asks the narrower question through the -`ObjectDeletionResolver` capability (§13, `DroppedDeletedIconRef` — the -predicate is exported so the comparator applies the same rule), and the -document falls through to whatever icon channel is left, exactly as an -image that is not an object id already does (§2b). Measured before the -rule: 134 corpus bookmark documents shipped an icon pointing at a favicon -whose file object was a tombstone in their own space's store. A store -failure (`known == false`) drops nothing, and no other reference slot asks -about deletion at all. - -**With no capability wired, nothing moves — the sentinel included.** A -package-only export passes every reference through verbatim, exactly as -before this rule existed: the absence of an answer is not evidence of -absence, and the offline round trip stays byte-exact. - -**Warnings follow what is lost.** A rewrite or a real-id drop destroys the -stored id — the warning is that id's last appearance anywhere — so both -warn, naming the id. A stored sentinel kept or dropped says nothing: which -object it was is already gone, and ~990 silent sentinel drops per corpus -would drown the channel §12 just reclaimed. - -**Round trip**: the change converges in one generation and is a fixpoint -after — the first export rewrites and drops, import stores what was -written, and `Export(Import(Export(S))) = Export(S)` holds (§11 guarantee -3). The comparator applies the same exported predicate -(`DroppedMissingObjectRef`) to both sides, so a dropped-by-design entry is -a normalization, not loss (§11). +none of those are ever asked about. A deleted object's tombstone is a row: +its id still means something in this space, and references to it are +untouched and unwarned — with ONE deliberate exception, the icon. An icon +is optional where a link or mention target is not, so an `iconImage` +whose target the space DELETED is dropped rather than kept: export asks +the narrower question through the `ObjectDeletionResolver` capability +(§13, `DroppedDeletedIconRef` — the predicate is exported so the +comparator applies the same rule), and the document falls through to +whatever icon channel is left, exactly as an image that is not an object +id already does (§2b). Measured before the rule: 134 corpus bookmark +documents shipped an icon pointing at a favicon whose file object was a +tombstone in their own space's store. A store failure (`known == false`) +drops nothing, and no other reference slot asks about deletion at all. +The bundle-level counterpart — an index target the space deleted, held or +never had — is §2c's `unresolved` account. + +**With no capability wired, nothing moves and nothing warns — the +sentinel included.** A package-only export passes every reference through +verbatim, exactly as before this rule existed: the absence of an answer is +not evidence of absence, and the offline round trip stays byte-exact. + +**Warnings follow what the space could not serve.** An absent real id +warns, naming the id and the slot, so an export report can say what was +not there; the id itself stays in the document. A stored sentinel, kept or +dropped, says nothing: which object it was is already gone. + +**Round trip**: the rule is a fixpoint from the first generation — the +export keeps the id, import stores what was written, and +`Export(Import(Export(S))) = Export(S)` holds (§11 guarantee 3) with +nothing lost on the way. The only movement is the sentinel drop from a +list, which converges in one generation. The comparator applies the same +exported predicate (`DroppedMissingObjectRef`) to both sides, so a +dropped sentinel is a normalization, not loss, and a real id that vanishes +is loss whatever the store says of it (§11). ### 9a. The legends, and compact ids @@ -6402,15 +6527,15 @@ reads every dropped icon as data loss, the drift class that once produced The missing-reference rule (§9) adds one normalization, armed only when the wiring supplies the `ObjectExistenceResolver` capability (§13) — under bare options it adds nothing and every reference passes verbatim: -**a reference to an object the space's store holds no row for is rewritten -to `_missing_object` in a singular slot (block `object_id`s, mention -targets) and dropped from a list slot (objects/files values, -`object_types`), and a stored sentinel follows the same split — kept in a -singular slot, dropped from a list.** The movement converges in one -generation: the first export writes the sentinel or the shorter list, -import stores exactly that, and every later export is byte-identical — so -guarantee 3 below holds, with the rewritten id's warning as its last -appearance anywhere. The predicate is exported +**a stored `_missing_object` sentinel is kept in a singular slot (block +`object_id`s, mention targets) and dropped from a list slot (objects/files +values, `object_types`). A real id the store holds no row for is not +moved at all — it is kept verbatim and warned, because the sentinel is the +importer's answer and a backup must carry what a later sync could still +make whole.** The one movement converges in one generation: the first +export writes the shorter list, import stores exactly that, and every +later export is byte-identical — so guarantee 3 below holds, and the kept +id's warning is a statement, not a last appearance. The predicate is exported (`DroppedMissingObjectRef`) and the comparator applies it to BOTH sides of the objects/files and `relationFormatObjectTypes` comparisons, the same-commit discipline every owned predicate above follows: a @@ -6780,6 +6905,18 @@ fail neither test belong in authoring guidance and in review. **inline-markup parsing** (§8) — grammar errors report the block's JSON path and the offending snippet. The indent bound [0, 32] lives in the schema. +- **A bundle has two verdicts, and `bundle.Inspect` keeps both.** + `bundle.Validate` answers yes or no; `bundle.Inspect` returns the whole + walk as a `Report` whose issues carry a severity (`error`, `warning`, + `info`), a stable code where one exists (callers branch on the code, + never on the wording) and the index field or bundle path they are about. + `Validate` is `Inspect` refusing exactly on the error-severity issues, + with the wording it always had; `InspectAuthoring` and `ValidateAuthoring` + are the same pair on the authoring surface. What earns each grade is + §2c's classification of a declared dangling target; everything Validate + used to refuse is still an error. The CLI prints warnings and info after + the `ok bundle` line and fails on warnings only under `-strict`; info + never fails, because a tombstone the space still names is by design. - **Validate and strict, default-vocabulary Unmarshal agree, in both directions.** With `Options{}` (or an equivalent resolver-free bundled key vocabulary), whatever `Validate` accepts `Unmarshal` decodes, and whatever @@ -8318,6 +8455,66 @@ being true. once the property leaves documents, sparing a bundled-relation `revision` bump and a space-by-space reviser pass. +- **#33 A type the space never held imports as a Page** — settled in §2c. + The five type references left on the corpus after #32 were objects an + old import created with a type it never made: two markdown pages, two + pb templates' targets, one pb profile object, each key minted at the + moment of its own import. The composer declares such derived ids under + `unresolved.types`, the full surface admits a declared one as a warning + and refuses an undeclared one, and the converter imports the object as a + Page — a template as one for Page — with a warning per document. Kept + verbatim, the object would have been unsearchable by type and rendered + with a guessed layout, exactly as it is in the source; a Page is usable. + Found on the same sweep and settled beside it: a stored `_missing_object` + in a relation's target list, written by an old importer, drops from the + dictionary entry's `object_types` by the predicate the document slot + already applies (§9) — the dictionary path had copied it, and three + spaces failed on read for a target that names no type. + +- **#32 Uninstalled types travel, and two repairs beside them** — settled + in §2a and §2b. A type the user removed is exported as a type document + carrying `uninstalled: true` and restored hidden; it owns its key in the + vocabulary and claims no spelling. A type document's own type is written + as `objectType`, derivable from the kind, with a warning when the store + disagrees. In a property declaration the stated `internal_key` is the + tie-break when the spelling is contested, the remedy the refusal itself + named; and a dictionary entry's own `property` spelling outranks a + display name another entry carries, because the entry that SPELLS a + term is the bundle's statement of what the term means, while the entry + merely NAMED it spells itself some other way — 542 documents in one + space were refused for that ambiguity before the rule. Measured: 64 of 72 unresolved type identities on the corpus were + uninstalled types; 3 were the `ot-type` legacy; six declarations spelled + `Tag` beside `internal_key: "tag"` in a space with a custom Tag. + +- **#31 The `cid` icon variant** — settled in §2b. A participant avatar + and a 1-to-1 space icon are raw content cids with no file object behind + them; `file` carried them beside real object ids and left the codec to + tell the two apart, and at the bundle level that overload made every + 1-to-1 space export declare its own icon unresolved. A content address + now has its own variant, written by codec, never folded, declared or + resolved, read back onto `iconImage`, refused by the authoring subset, + with the old overloaded spelling kept as input compatibility. The + measured case: the sweep's 29 `icon.file` failures, of which the audited + example is a dag-pb cid referenced only by a participant document. + +- **#30 Deleted, omitted, absent** — settled in §2c and §9. A dangling + reference has three causes and the store already tells them apart: a + tombstone (the user deleted the object; Anytype does not rewrite every + object that mentions it, so the reference is ordinary state), a row the + export did not write (archived, or out of a partial export's scope), and + no row at all (not yet synced, or never in this space — the one real + loss). The index declares the first two as subsets of `unresolved.targets`, + the validator admits a declared target on the full surface at the + severity its class earns and refuses every dangling target on the + authoring surface, and export stops manufacturing `_missing_object` for an + absent id: kept verbatim and warned, because the sentinel is the + importer's answer and a backup taken during a sync gap must carry what a + later sync could still make whole. Measured on the 79-bundle corpus: 56 + homepages, 29 icons and 62 widget targets rejected for this alone, 41, 20 + and 12 of them provably tombstones. Deferred with it: a `pending` class + from the head-sync missing-id list, and a warning at export time when the + space is offline or has not synced for a long time. + - **#20 Files by reference** — settled in §2h. Optional `file_remote` on each file document carries a base64-encoded JSON payload with independent version 1, the root CID, encryption keys, and optional indexed variants. diff --git a/format/v2/schema/index.schema.json b/format/v2/schema/index.schema.json index fdd2134..dfedef2 100644 --- a/format/v2/schema/index.schema.json +++ b/format/v2/schema/index.schema.json @@ -31,7 +31,7 @@ }, "icon": { "$ref": "https://schemas.anytype.io/anyblock/2.0/object.schema.json#/$defs/icon", - "description": "The space's icon — exactly the shape `icon` has on any object (§2b), so an author never has to remember a second convention: an emoji, the **object id** of an image in the bundle, a built-in icon name, or a bare colour. Two flat keys used to stand here with no rule for which one wins, and they disagreed with the object surface about whether an image is a scalar or a list. This field was then narrowed to emoji-or-image, which silently dropped the icon of a space whose icon is a bare colour — 20 of 77 real spaces, the letter avatars the client draws when no image was ever set. An image needs the image object AND its file in the archive, which is why a generated bundle uses an emoji. (The installer resolves the space icon by image *name*, not id — the wiring does that lookup, so this field stays consistent with every other id in the format; an emoji or colour icon reaches no installer field at all, and is carried so a round trip does not lose it.)" + "description": "The space's icon — exactly the shape `icon` has on any object (§2b), so an author never has to remember a second convention: an emoji, the **object id** of an image in the bundle, the raw content cid of an image no file object holds (a 1-to-1 space's icon, output-only), a built-in icon name, or a bare colour. Two flat keys used to stand here with no rule for which one wins, and they disagreed with the object surface about whether an image is a scalar or a list. This field was then narrowed to emoji-or-image, which silently dropped the icon of a space whose icon is a bare colour — 20 of 77 real spaces, the letter avatars the client draws when no image was ever set. An image needs the image object AND its file in the archive, which is why a generated bundle uses an emoji. (The installer resolves the space icon by image *name*, not id — the wiring does that lookup, so this field stays consistent with every other id in the format; an emoji or colour icon reaches no installer field at all, and is carried so a round trip does not lose it.)" }, "entrypoint": { "type": "string", @@ -129,6 +129,36 @@ "pattern": "^[^_]" }, "description": "The ids this index names that no document in the bundle carries — an `entrypoint`, a `homepage`, a widget `target`, an image icon. Spelled like every other reference in this file, the derived-id fold included (§9), so the report and the slot it reports on name the same thing. A reserved listing (`_set`, `_widgets`) never appears: those name built-in screens, resolve everywhere, and are not the bundle's to carry. Neither does the auto-widget ledger's content — an entry there usually names a widget the user deleted, which is what the ledger is FOR, so a missing document is its normal state rather than a loss." + }, + "deleted": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^_]" + }, + "description": "The subset of `targets` the space DELETED: each names a tombstone in the source store. A reference to a deleted object is ordinary Anytype state — the app does not rewrite every object that mentions a deleted one — so a reader treats these as known, by design, and a restore keeps the id and writes a tombstone rather than a dangling reference. Every entry must also appear in `targets`, and none may appear in `omitted`." + }, + "omitted": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[^_]" + }, + "description": "The subset of `targets` the space still HOLDS and this export did not write: an archived object under an export without archived objects, or an object outside a partial export's scope. Not a loss in the space, only in this bundle. Every entry must also appear in `targets`, and none may appear in `deleted`. A target in neither subset is ABSENT — the space had no row for it, typically because it had not synced — and that is the one class a reader should warn about." + }, + "types": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 6, + "pattern": "^type-" + }, + "description": "The types this bundle's documents name by derived id — `type-` in a `type_internal_key`, a `template_for`, an `object_types` entry — that no type document carries and the source space has no row for: an object or template an old import created with a type it never made. Spelled as the derived id, the address a document uses and the validator checks; never folded. A reader imports such an object as a Page, and a template as one for Page, and warns; a slot that merely REFERENCES the type (a `query_source.types` entry, a property's `object_types`) drops the entry instead of retyping anything. Declaring the loss is what makes the bundle admissible on the full surface; an undeclared one is still refused as an exporter bug, and so is a declaration naming a type the bundle actually carries." } } }, diff --git a/format/v2/schema/object.schema.json b/format/v2/schema/object.schema.json index 030b07a..9792976 100644 --- a/format/v2/schema/object.schema.json +++ b/format/v2/schema/object.schema.json @@ -98,6 +98,11 @@ "properties": { "$ref": "#/$defs/propertyMap" }, + "uninstalled": { + "type": "boolean", + "const": true, + "description": "Only on type documents (kind \"object_type\"): the user REMOVED this type from the space (stored `isUninstalled`), and the bundle carries it for backup fidelity — its objects still carry the key, and the definition is still the space's. The mirror of the member a removed property's declaration and dictionary entry carry (§2f). Written `true` only; absent is the same statement as `false`. A reader restores it hidden, as the user left it, never as an installed type; a type that owns the same stored key and is live in the destination stays live. Output-only: the authoring subset refuses it, because an author declares a type to use it." + }, "type_settings": { "description": "Only on type documents (kind \"object_type\" or \"bundled_object_type\"): everything that defines the TYPE, in one gated subtree (§2a). `layout` is the recommended layout of objects of this type (a layout name; a stored number outside the vocabulary passes through raw). `api_key` is the type's public API key — NOT a slug: of 1,326 corpus type documents with one, it differs from the document's own spelling in 247 (the `property` type's api key is `relation`, the word the public API kept when the format renamed the concept) — calling it a slug would imply it is the term used elsewhere in the document, which for those 247 it is not. `plural_name` is the plural display name. `default_template` is the object id of the template new objects of this type start from. `default_view` is the default view type (a §6.2 view-type name; a stored number outside the vocabulary passes through raw). `property_definitions` is the type's property list — one propertyDefinition + `section` per entry (§2e); it lived at the document root as `type_properties` until v0.32, and the word is `property_definitions` rather than `properties` because the document already uses that word for property VALUES at the root — one word per concept.", "type": "object", @@ -273,7 +278,8 @@ "then": {}, "else": { "properties": { - "type_settings": false + "type_settings": false, + "uninstalled": false } } }, @@ -423,7 +429,7 @@ "minLength": 1, "maxLength": 255, "pattern": "^[^/]+$", - "description": "The id of an object in this space, a bundle-local slug, or a raw content CID — the WHOLE string, and nothing else. A reference carries no caption, display hint or name: there is no separator inside it and no half of it to discard, so `bafyrei\u2026#alice` is one id, one that no space mints, and it names nothing. (Earlier drafts of this format spelled a reference `#` and asked readers to split at the first `#`. That form is gone; `#` is now an ordinary character, kept legal here only because a reference is written through verbatim from whatever a space stores.) To show a human-readable name for a reference, look the id up: index the bundle's documents by their envelope `id` and read the target's `Name`. Never a URL and never a filesystem path — this format does no I/O, so it can only name an object or content the reader can resolve. To supply an image by URL, use a layer that can fetch it: it uploads the image, mints the file object and writes that object's id here. An object id uses the dag-cbor CID codec (normally beginning `bafyrei`); a raw or dag-pb content CID normally begins `bafybei` and resolves against content storage rather than the bundle's object documents." + "description": "The id of an object in this space, a bundle-local slug, or a raw content CID — the WHOLE string, and nothing else. A reference carries no caption, display hint or name: there is no separator inside it and no half of it to discard, so `bafyrei\u2026#alice` is one id, one that no space mints, and it names nothing. (Earlier drafts of this format spelled a reference `#` and asked readers to split at the first `#`. That form is gone; `#` is now an ordinary character, kept legal here only because a reference is written through verbatim from whatever a space stores.) To show a human-readable name for a reference, look the id up: index the bundle's documents by their envelope `id` and read the target's `Name`. Never a URL and never a filesystem path — this format does no I/O, so it can only name an object or content the reader can resolve. To supply an image by URL, use a layer that can fetch it: it uploads the image, mints the file object and writes that object's id here. An object id uses the dag-cbor CID codec (normally beginning `bafyrei`). A raw or dag-pb content CID (normally beginning `bafybei`) is content, not an object: it resolves against content storage, an icon carries one in its `cid` variant (§2b), and no reference slot should hold one." }, "iconColor": { "description": "One of the ten palette colours — the same palette select options use (§2a) — or, for a stored value the palette has no name for, its raw number. That escape is bounded at 9007199254740991 (2^53-1), the same bound `size` carries: it is the largest integer this format moves through a v1 float value and writes back out denoting the same number, and the reference exporter drops a stored value above it with a warning rather than emitting one this schema refuses.", @@ -457,7 +463,7 @@ ], "properties": { "format": { - "description": "Which kind of icon this is: an emoji, an image object in this space, a built-in named icon, or a bare colour." + "description": "Which kind of icon this is: an emoji, an image object in this space, an image by raw content cid (no object behind it), a built-in named icon, or a bare colour." } }, "allOf": [ @@ -511,7 +517,39 @@ "format": {}, "file": { "$ref": "#/$defs/objectRef", - "description": "The image. Normally the id of a file object in this bundle; for a participant avatar or an invite icon it is the raw content cid of the image instead — see $defs/objectRef for how the two are told apart." + "description": "The image: the id of a file object in this bundle. A raw content cid — a participant avatar, an invite or 1-to-1 space icon — is not a file object and travels in the `cid` variant instead; a `file` holding one is read as that variant for compatibility with documents written before it existed, and is never written." + }, + "color": { + "$ref": "#/$defs/iconColor" + } + } + } + }, + { + "if": { + "properties": { + "format": { + "const": "cid" + } + }, + "required": [ + "format" + ] + }, + "then": { + "required": [ + "format", + "cid" + ], + "additionalProperties": false, + "properties": { + "format": {}, + "cid": { + "type": "string", + "minLength": 46, + "maxLength": 255, + "pattern": "^[^/]+$", + "description": "The image's raw content cid (dag-pb or raw codec, normally beginning `bafybei`): an image no file object in this bundle holds — a participant's avatar, a 1-to-1 space's icon — which a client fetches by content id through the gateway or the identity repo. Never an object reference: nothing in the bundle resolves it, nothing declares it unresolved, and a reader that cannot reach content storage shows no icon. Output-only: an author cannot mint a content address, and supplies an image as a file object instead." }, "color": { "$ref": "#/$defs/iconColor" @@ -592,10 +630,11 @@ "enum": [ "emoji", "file", + "cid", "icon", "color" ], - "description": "Which kind of icon this is: an emoji, an image object in this space, a built-in named icon, or a bare colour." + "description": "Which kind of icon this is: an emoji, an image object in this space, an image by raw content cid (no object behind it), a built-in named icon, or a bare colour." } } }, @@ -610,7 +649,8 @@ "format": { "enum": [ "emoji", - "file" + "file", + "cid" ] } }