GO-7383 Import real space exports: full converter, unresolved references, uninstalled types

bundle/README.md

@@ -108,6 +108,29 @@ Repeated targets at different locations remain separate diagnostics.
 target inventories. Authored index fields without a source snapshot have a
 path and target but no source object.
 
+`Stats.UnresolvedDeleted` and `Stats.UnresolvedOmitted` are the subsets the
+wired store could classify (SPEC §2c, `anyblockjson.ClassifyUnresolvedTarget`):
+tombstones, and rows the space still holds that this export did not write.
+What is in neither is absent — the one class that is a loss. `index.json`
+states the same subsets as `unresolved.deleted` and `unresolved.omitted`,
+and a space icon whose image object is a tombstone is dropped with a warning
+rather than declared, following the document-level icon rule.
+
+`Stats.UnresolvedTypes` and `Stats.UnresolvedTypeReferences` name the types
+the written documents reference by derived id that no written type document
+carries (SPEC §2c): a type the source space never held, left by an old
+import. `index.json` states the same list as `unresolved.types`. A bundled
+key never appears; an uninstalled type travels as a type document carrying
+`uninstalled: true`, so it is not one of these either.
+
+`Inspect` is `Validate` with the whole verdict kept: a `Report` of issues
+graded `error`, `warning` or `info`, each with a stable code and the index
+field it is about. On the full surface a declared target is admitted — a
+deleted one as info, an omitted or absent one as a warning — while an
+undeclared dangling target stays an error; `InspectAuthoring` and
+`ValidateAuthoring` refuse every dangling target. `Validate` is `Inspect`
+refusing exactly on the error-severity issues, with the same wording.
+
 Codec `unresolved_target` warnings put the complete source location in `Path`,
 for example `/blocks/<stored-block-id>/object_id`,
 `/blocks/<stored-block-id>/text/marks/0/param`, or `/properties/<stored-key>/1`.
@@ -119,12 +142,12 @@ or `<object-id>/properties/<stored-key>/1`; the message names the missing target
 Index references with a known source use the same object prefix and `SourcePath`
 in the report; otherwise they use `index.json#` followed by the index pointer.
 
-## Known integration issue: icons in 1-to-1 spaces
+## Icons by content cid (1-to-1 spaces, participants)
 
-In 1-to-1 spaces, `spaceIcon` can reference a raw file CID with no corresponding
-file object in the space's object store. Export preserves that CID in
-`index.json` at `/icon/file`, but the bundle's target check expects an exported
-object and reports `unresolved_target`. This is a known mismatch between the
-stored icon reference and the bundle's object-reference model; it does not by
-itself mean an object was deleted or a file export failed. Resolving these raw
-icon CIDs is not yet implemented.
+In 1-to-1 spaces `spaceIcon`, and on participant objects `iconImage`, hold a
+raw content CID with no file object behind it. Export writes that as the
+icon's `cid` variant (SPEC §2b), decided by the CID's codec, and neither the
+composer nor `Validate` treats it as an object the bundle should carry. A
+bundle written before the variant existed carries the same CID under
+`icon.file`; that spelling is read as the `cid` variant for compatibility and
+names no object either. Import writes the CID back to the stored slot.

bundle/compose.go

@@ -137,8 +137,27 @@ type Stats struct {
 	// its normal state. The slots are the ones bundle.Validate refuses on,
 	// so an export states exactly what a later validation would find.
 	UnresolvedTargets []string
+	// UnresolvedDeleted and UnresolvedOmitted are the subsets of
+	// UnresolvedTargets the wired store could classify (§2c,
+	// anyblockjson.ClassifyUnresolvedTarget): tombstones, and rows the space
+	// still holds that this export did not write. What is in neither is
+	// absent — the one class that is a loss. Sorted. index.json states the
+	// same subsets (Index.Unresolved.Deleted / Omitted).
+	UnresolvedDeleted []string
+	UnresolvedOmitted []string
 	// UnresolvedReferences retains each missing index target with source location.
 	UnresolvedReferences []anyblockjson.ObjectReference
+	// UnresolvedTypes are the derived type ids the written documents name
+	// — in a type_internal_key, a template_for, an object_types entry — that
+	// no written type document carries (§2c): an object or template an old
+	// import created with a type it never made. Bundled keys are never
+	// here; every reader carries the shipped table. Sorted. index.json
+	// states the same list (Index.Unresolved.Types), and a reader imports
+	// such an object as a Page and warns.
+	UnresolvedTypes []string
+	// UnresolvedTypeReferences retains each such reference with the slot
+	// (Path) and the written document id (ObjectID) it came from.
+	UnresolvedTypeReferences []anyblockjson.ObjectReference
 	// RefusedOptions names the vocabularies the dictionary cannot state and
 	// why — one `key: reason` line each, sorted. The writer refuses a
 	// vocabulary on a property whose format does not admit one (§2a), and
@@ -244,7 +263,11 @@ type Composer struct {
 	// function Marshal used to write them, rather than a second opinion that
 	// could disagree about a type's derived id. They answer the one question
 	// no document can: whether an id this index names is carried here.
-	documentIds           map[string]bool
+	documentIds map[string]bool
+	// typeRefs are the derived type ids the written documents name, by
+	// slot and document — the validator's own census (derivedTypeUses),
+	// taken here so the index can state what no document carries (§2c).
+	typeRefs              map[derivedTypeUse]struct{}
 	indexReferenceSources map[anyblockjson.ObjectReference]struct{}
 	widgetViewLabels      map[string]map[string]string
 
@@ -291,6 +314,7 @@ func NewComposer(opts anyblockjson.Options, spaceName string) (*Composer, error)
 		used:         map[string]bool{},
 		declared:     map[string]map[declaredProperty]bool{},
 		documentIds:  map[string]bool{},
+		typeRefs:     map[derivedTypeUse]struct{}{},
 		spaceSettings: spaceSettingsCandidates{
 			names:        map[string]struct{}{},
 			descriptions: map[string]struct{}{},
@@ -498,11 +522,11 @@ func (c *Composer) observeRelation(sbType model.SmartBlockType, base *model.Smar
 // Close, so whatever the composition needs to know about a document it has
 // to take from the bytes it is about to write (design §1.1).
 func (c *Composer) ObserveWritten(sbType model.SmartBlockType, base *model.SmartBlockSnapshotBase, doc []byte) error {
+	var envelope bundleDocumentEnvelope
+	if err := json.Unmarshal(doc, &envelope); err != nil {
+		return fmt.Errorf("read document envelope: %w", err)
+	}
 	if sbType == model.SmartBlockType_FileObject || sbType == model.SmartBlockType_File {
-		var envelope bundleDocumentEnvelope
-		if err := json.Unmarshal(doc, &envelope); err != nil {
-			return fmt.Errorf("read file envelope: %w", err)
-		}
 		if c.opts.IncludeFileRemote && envelope.FileRemote == nil {
 			return fmt.Errorf("observe written file: IncludeFileRemote requires file_remote; use the same Options for Marshal and NewComposer")
 		}
@@ -546,6 +570,12 @@ func (c *Composer) ObserveWritten(sbType model.SmartBlockType, base *model.Smart
 	if id := anyblockjson.FoldDocumentId(c.opts, sbType,
 		base.GetDetails().GetFields()["id"].GetStringValue(), base.GetKey()); id != "" {
 		c.documentIds[id] = true
+		// the derived type ids this document names, by slot — the same
+		// census the validator takes, so what the index declares and what
+		// the validator would refuse cannot disagree (§2c)
+		for _, use := range derivedTypeUses(id, envelope) {
+			c.typeRefs[use] = struct{}{}
+		}
 	}
 	for key := range used {
 		c.used[key] = true
@@ -1062,6 +1092,18 @@ func (c *Composer) Finish() (index, properties []byte, stats Stats, err error) {
 	idx.Name = spaceSettings.Name
 	idx.Description = spaceSettings.Description
 	idx.Icon = spaceSettings.Icon
+	// the space icon follows the document-level icon rule (§2b, §9): an
+	// icon is optional, so one whose image object the space DELETED is
+	// dropped and warned rather than carried as a declared target, and the
+	// index falls through to whatever icon channel is left
+	if iconID := idx.IconImageId(); iconID != "" && anyblockjson.DroppedDeletedIconRef(c.opts, iconID) {
+		idx.Icon = nil
+		if c.opts.OnWarning != nil {
+			c.opts.OnWarning(anyblockjson.Issue{Path: "/icon/file", Message: fmt.Sprintf(
+				"space icon %q names an image object the space deleted and is dropped — an icon is optional, "+
+					"and the index falls through to whatever icon channel is left", iconID)})
+		}
+	}
 	idx.Homepage = spaceSettings.Homepage
 	// the caller's name is the fallback for a space whose document has none
 	if idx.Name == "" {
@@ -1101,10 +1143,26 @@ func (c *Composer) Finish() (index, properties []byte, stats Stats, err error) {
 	// question "is this export incomplete" is answered where the bundle is
 	// described, rather than by a reader discovering silence.
 	unresolvedTargets := c.unresolvedIndexTargets(&idx)
-	if len(orphans) > 0 || len(unresolvedTargets) > 0 {
+	unresolvedDeleted, unresolvedOmitted := c.classifyUnresolvedTargets(unresolvedTargets)
+	// a relation document is never written (§2f), so a property's target
+	// types reach the bundle only through its dictionary entry — a slot the
+	// validator censuses, and one the document walk above cannot see
+	for _, def := range dict.Properties {
+		for _, target := range def.ObjectTypes {
+			spelling := anyblockjson.TypeKeySpelling(target)
+			if anyblockjson.IsDerivedTypeId(spelling) {
+				c.typeRefs[derivedTypeUse{ref: spelling, slot: "object_types", source: anyblockjson.PropertiesFileName}] = struct{}{}
+			}
+		}
+	}
+	unresolvedTypes, unresolvedTypeRefs := c.unresolvedTypeReferences()
+	if len(orphans) > 0 || len(unresolvedTargets) > 0 || len(unresolvedTypes) > 0 {
 		idx.Unresolved = &anyblockjson.Unresolved{
 			Properties: orphans,
 			Targets:    unresolvedTargets,
+			Deleted:    unresolvedDeleted,
+			Omitted:    unresolvedOmitted,
+			Types:      unresolvedTypes,
 		}
 	}
 	idxData, err := anyblockjson.MarshalIndex(&idx, c.opts)
@@ -1130,6 +1188,10 @@ func (c *Composer) Finish() (index, properties []byte, stats Stats, err error) {
 	}
 	stats.RefusedOptions = refusedOptions
 	stats.UnresolvedTargets = unresolvedTargets
+	stats.UnresolvedDeleted = unresolvedDeleted
+	stats.UnresolvedOmitted = unresolvedOmitted
+	stats.UnresolvedTypes = unresolvedTypes
+	stats.UnresolvedTypeReferences = unresolvedTypeRefs
 	stats.UnresolvedReferences = c.unresolvedIndexReferences(&idx)
 	return idxData, dictData, stats, nil
 }
@@ -1160,6 +1222,56 @@ func (c *Composer) unresolvedIndexTargets(idx *anyblockjson.Index) []string {
 	return out
 }
 
+// unresolvedTypeReferences names the derived type ids the written documents
+// reference that no written type document carries (§2c), as a sorted set
+// and as the references themselves with the slot and the document each
+// came from. Bundled keys never appear: derivedTypeUses skips them, since
+// every reader carries the shipped table. Called with the mutex held.
+func (c *Composer) unresolvedTypeReferences() ([]string, []anyblockjson.ObjectReference) {
+	seen := map[string]bool{}
+	var ids []string
+	var refs []anyblockjson.ObjectReference
+	for use := range c.typeRefs {
+		if c.documentIds[use.ref] {
+			continue
+		}
+		if !seen[use.ref] {
+			seen[use.ref] = true
+			ids = append(ids, use.ref)
+		}
+		refs = append(refs, anyblockjson.ObjectReference{TargetObjectID: use.ref, Path: "/" + use.slot, ObjectID: use.source})
+	}
+	sort.Strings(ids)
+	sort.Slice(refs, func(i, j int) bool {
+		a, b := refs[i], refs[j]
+		if a.TargetObjectID != b.TargetObjectID {
+			return a.TargetObjectID < b.TargetObjectID
+		}
+		if a.ObjectID != b.ObjectID {
+			return a.ObjectID < b.ObjectID
+		}
+		return a.Path < b.Path
+	})
+	return ids, refs
+}
+
+// classifyUnresolvedTargets asks the wired store WHY each unresolved target
+// dangles (§2c, anyblockjson.ClassifyUnresolvedTarget) and splits the
+// tombstones from the rows the space still holds. Both come back sorted,
+// which they are already, since the input is. With no store wired nothing
+// is classified and every target reads as absent, the fail-safe direction.
+func (c *Composer) classifyUnresolvedTargets(targets []string) (deleted, omitted []string) {
+	for _, id := range targets {
+		switch anyblockjson.ClassifyUnresolvedTarget(c.opts, id) {
+		case anyblockjson.UnresolvedDeleted:
+			deleted = append(deleted, id)
+		case anyblockjson.UnresolvedOmitted:
+			omitted = append(omitted, id)
+		}
+	}
+	return deleted, omitted
+}
+
 // hasSemanticState distinguishes a genuinely empty composition from one in
 // which every source document was deliberately omitted because index.json or
 // properties.json carries it instead. spaceName is only a fallback label, so