Retain transition coverage in exploration (#131)

Author: SandroMaglione

.changeset/exploration-transition-coverage.md

@@ -0,0 +1,5 @@
+---
+"@typeonce/effect-machine": minor
+---
+
+Add exact `transitionCoverage` to `MachineTest.Exploration`. Coverage includes startup and every concretely planned event, including state-limit candidates, while unplanned depth- and transition-limit frontiers remain misses.

README.md

@@ -549,7 +549,8 @@ The testing entrypoint provides complementary layers:
 - `MachineTest.run` and `verify` inspect pure planner traces;
 - `coverage` reports exact transition-definition and conditional-branch hits;
 - invariants and generated scenarios check application laws;
-- `explore` performs bounded breadth-first state-space exploration;
+- `explore` performs bounded breadth-first state-space exploration and retains
+  exact transition-definition and branch coverage for every plan it computes;
 - `probe` causally acknowledges live runtime commands;
 - runtime command models cover timers, invokes, bursts, and scheduling.
 

src/internal/testing/machine/exploration.ts

@@ -30,6 +30,7 @@ import type {
 import type { EnsureExecutable } from "../../machine/readiness.js"
 import { assertInvariants, type InvariantError } from "./invariant.js"
 import { appendTrace, run } from "./trace.js"
+import { makeTransitionCoverageCollector } from "./transitionCoverage.js"
 
 type AnyMachine = Machine.Machine.Any
 
@@ -119,6 +120,10 @@ export const explore = <M extends AnyMachine, Key extends ExplorationKey>(
 
     const initialContext = stateContext(machine, initialTrace)
     const initialKey = validateKey(options.stateKey(initialContext))
+    const transitionCoverage = makeTransitionCoverageCollector<M>(machine)
+    for (const microstep of initialTrace.initial.plan.microsteps) {
+      transitionCoverage.observeMicrostep(microstep)
+    }
     const nodes: Array<ExplorationNode<M, Key>> = [{ ...initialContext, key: initialKey }]
     const nodeDraftsByKey = new Map<Key, number>([[initialKey, 0]])
     const edges: Array<ExplorationEdgeDraft<M>> = []
@@ -165,6 +170,8 @@ export const explore = <M extends AnyMachine, Key extends ExplorationKey>(
         const scenario = scenarioWithEvent(source.trace, event)
         const targetTrace = yield* appendTrace(machine, source.trace, event, scenario)
         plannedTransitions += 1
+        const step = targetTrace.steps[targetTrace.steps.length - 1]!
+        for (const microstep of step.plan.microsteps) transitionCoverage.observeMicrostep(microstep)
         if (invariants.length > 0) {
           yield* assertInvariants(machine, targetTrace, invariants)
         }
@@ -178,7 +185,7 @@ export const explore = <M extends AnyMachine, Key extends ExplorationKey>(
             target: existing,
             edge: {
               event,
-              step: targetTrace.steps[targetTrace.steps.length - 1]!,
+              step,
               discovered: false
             }
           })
@@ -207,7 +214,7 @@ export const explore = <M extends AnyMachine, Key extends ExplorationKey>(
           target: targetIndex,
           edge: {
             event,
-            step: targetTrace.steps[targetTrace.steps.length - 1]!,
+            step,
             discovered: true
           }
         })
@@ -243,6 +250,7 @@ export const explore = <M extends AnyMachine, Key extends ExplorationKey>(
         retainedEdges: edges.length,
         maxDepth: nodes.reduce((maximum, node) => Math.max(maximum, node.depth), 0)
       },
+      transitionCoverage: transitionCoverage.summary(),
       completeness
     }
   })

src/internal/testing/machine/transitionCoverage.ts

@@ -0,0 +1,116 @@
+/**
+ * Exact transition-definition and branch coverage collected from retained
+ * planner microsteps.
+ *
+ * @since 0.14.0
+ */
+
+import * as Machine from "../../../Machine.js"
+import type {
+  CoverageSummary,
+  Microstep,
+  TransitionBranchCoverageItem,
+  TransitionCoverage,
+  TransitionDefinitionCoverageItem
+} from "../../../testing/MachineTest.js"
+
+type AnyMachine = Machine.Machine.Any
+
+type StateNodePath<M extends AnyMachine> = Machine.Machine.StateNodeIdentifier<Machine.Machine.States<M>>
+
+type EventTag<M extends AnyMachine> = Machine.Machine.TagOf<Machine.Machine.Events<M>[number]>
+
+type MachineTransitionCoverage<M extends AnyMachine> = TransitionCoverage<
+  StateNodePath<M>,
+  EventTag<M>,
+  StateNodePath<M>
+>
+
+const coverageSummary = <Item>(declared: ReadonlyArray<Item>, hit: ReadonlySet<number>): CoverageSummary<Item> => {
+  const hits: Array<Item> = []
+  const misses: Array<Item> = []
+  declared.forEach((item, index) => (hit.has(index) ? hits : misses).push(item))
+  return {
+    total: declared.length,
+    hit: hits.length,
+    missing: misses.length,
+    hits,
+    misses
+  }
+}
+
+export const sameTransitionTrigger = (
+  left: Machine.Machine.TransitionTrigger,
+  right: Machine.Machine.TransitionTrigger
+): boolean => {
+  if (left.type !== right.type) return false
+  if (left.type === "event") return right.type === "event" && left.event === right.event
+  if (left.type === "invoke") {
+    return right.type === "invoke" && left.id === right.id && left.outcome === right.outcome
+  }
+  return true
+}
+
+export interface TransitionCoverageCollector<M extends AnyMachine> {
+  readonly observeMicrostep: (microstep: Microstep<M, unknown>) => void
+  readonly summary: () => MachineTransitionCoverage<M>
+}
+
+export const makeTransitionCoverageCollector = <M extends AnyMachine>(
+  machine: M
+): TransitionCoverageCollector<M> => {
+  const definitions = Machine.transitionDefinitions(machine).map(
+    (definition, index): TransitionDefinitionCoverageItem<StateNodePath<M>, EventTag<M>, StateNodePath<M>> => ({
+      id: `transition:${index}`,
+      index,
+      source: definition.source,
+      trigger: definition.trigger,
+      reenter: definition.reenter,
+      branches: definition.branches
+    })
+  )
+  const definitionHits = new Set<number>()
+  const branchOffsets: Array<number> = []
+  const branches: Array<TransitionBranchCoverageItem<StateNodePath<M>, EventTag<M>, StateNodePath<M>>> = []
+  for (let definitionIndex = 0; definitionIndex < definitions.length; definitionIndex++) {
+    branchOffsets.push(branches.length)
+    const definition = definitions[definitionIndex]!
+    definition.branches.forEach((branch, branchIndex) => {
+      branches.push({
+        id: `transition:${definitionIndex}:branch:${branchIndex}`,
+        definitionIndex,
+        branchIndex,
+        source: definition.source,
+        trigger: definition.trigger,
+        reenter: definition.reenter,
+        branch
+      })
+    })
+  }
+  const branchHits = new Set<number>()
+
+  return {
+    observeMicrostep(microstep) {
+      for (const retained of microstep.transitions) {
+        const definitionIndex = definitions.findIndex((definition) =>
+          definition.source === retained.source &&
+          definition.reenter === retained.reenter &&
+          sameTransitionTrigger(definition.trigger, retained.trigger)
+        )
+        if (definitionIndex === -1) continue
+        definitionHits.add(definitionIndex)
+        const definition = definitions[definitionIndex]!
+        if (
+          Number.isSafeInteger(retained.branchIndex) && retained.branchIndex >= 0 &&
+          retained.branchIndex < definition.branches.length
+        ) {
+          branchHits.add(branchOffsets[definitionIndex]! + retained.branchIndex)
+        }
+      }
+    },
+    summary: () => ({
+      definitions: coverageSummary(definitions, definitionHits),
+      branches: coverageSummary(branches, branchHits)
+    })
+  }
+}

src/internal/testing/machine/verification.ts

@@ -29,8 +29,6 @@ import type {
   StateCoverageItem,
   Trace,
   TraceStep,
-  TransitionBranchCoverageItem,
-  TransitionDefinitionCoverageItem,
   VerificationLaw,
   VerificationLawGroup,
   VerificationViolation,
@@ -41,6 +39,7 @@ import { toArbitraryWithReport } from "./arbitrary.js"
 import type { FiniteModel } from "./finiteModel.js"
 import * as ReferenceModel from "./referenceModel.js"
 import { rawConfigurationPaths, run } from "./trace.js"
+import { makeTransitionCoverageCollector, sameTransitionTrigger } from "./transitionCoverage.js"
 
 export {
   advanceCommand,
@@ -470,18 +469,6 @@ const normalizeTraces = <M extends AnyMachine>(
 ): ReadonlyArray<Trace<M>> =>
   Array.isArray(traceOrTraces) ? traceOrTraces as ReadonlyArray<Trace<M>> : [traceOrTraces as Trace<M>]
 
-const sameTransitionTrigger = (
-  left: Machine.Machine.TransitionTrigger,
-  right: Machine.Machine.TransitionTrigger
-): boolean => {
-  if (left.type !== right.type) return false
-  if (left.type === "event") return right.type === "event" && left.event === right.event
-  if (left.type === "invoke") {
-    return right.type === "invoke" && left.id === right.id && left.outcome === right.outcome
-  }
-  return true
-}
-
 const targetWithinSelection = (
   target: string | undefined,
   branch: Machine.Machine.TransitionBranch,
@@ -576,48 +563,7 @@ export const coverage = <M extends AnyMachine>(
   const entryHits = new Set<number>()
   const exitHits = new Set<number>()
 
-  const definitions = Machine.transitionDefinitions(machine).map(
-    (
-      definition,
-      index
-    ): TransitionDefinitionCoverageItem<
-      StateNodePath<M>,
-      Machine.Machine.TagOf<Machine.Machine.Events<M>[number]>,
-      StateNodePath<M>
-    > => ({
-      id: `transition:${index}`,
-      index,
-      source: definition.source,
-      trigger: definition.trigger,
-      reenter: definition.reenter,
-      branches: definition.branches
-    })
-  )
-  const transitionHits = new Set<number>()
-  const branchOffsets: Array<number> = []
-  const branches: Array<
-    TransitionBranchCoverageItem<
-      StateNodePath<M>,
-      Machine.Machine.TagOf<Machine.Machine.Events<M>[number]>,
-      StateNodePath<M>
-    >
-  > = []
-  for (let definitionIndex = 0; definitionIndex < definitions.length; definitionIndex++) {
-    branchOffsets.push(branches.length)
-    const definition = definitions[definitionIndex]!
-    definition.branches.forEach((branch, branchIndex) => {
-      branches.push({
-        id: `transition:${definitionIndex}:branch:${branchIndex}`,
-        definitionIndex,
-        branchIndex,
-        source: definition.source,
-        trigger: definition.trigger,
-        reenter: definition.reenter,
-        branch
-      })
-    })
-  }
-  const branchHits = new Set<number>()
+  const transitionCoverage = makeTransitionCoverageCollector(machine)
 
   const declaredEvents = publicEventTags(machine)
   const declaredEventTags = declaredEvents.tags
@@ -675,6 +621,7 @@ export const coverage = <M extends AnyMachine>(
   }
 
   const observeMicrostep = (microstep: Microstep<M, any>): void => {
+    transitionCoverage.observeMicrostep(microstep)
     microsteps += 1
     if (microstep.changed) changedMicrosteps += 1
     raisedEvents += microstep.raisedEvents.length
@@ -692,20 +639,6 @@ export const coverage = <M extends AnyMachine>(
         historyTargets += 1
         if (retained.resolvedTarget !== undefined) resolvedHistoryTargets += 1
       }
-      const definitionIndex = definitions.findIndex((definition) =>
-        definition.source === retained.source &&
-        definition.reenter === retained.reenter &&
-        sameTransitionTrigger(definition.trigger, retained.trigger)
-      )
-      if (definitionIndex === -1) continue
-      transitionHits.add(definitionIndex)
-      const definition = definitions[definitionIndex]!
-      if (
-        Number.isSafeInteger(retained.branchIndex) && retained.branchIndex >= 0 &&
-        retained.branchIndex < definition.branches.length
-      ) {
-        branchHits.add(branchOffsets[definitionIndex]! + retained.branchIndex)
-      }
     }
   }
 
@@ -745,10 +678,7 @@ export const coverage = <M extends AnyMachine>(
       entry: coverageSummary(activeNodes, entryHits),
       exit: coverageSummary(activeNodes, exitHits)
     },
-    transitions: {
-      definitions: coverageSummary(definitions, transitionHits),
-      branches: coverageSummary(branches, branchHits)
-    },
+    transitions: transitionCoverage.summary(),
     events: declaredEvents.diagnostics.length === 0
       ? {
         available: true,

src/testing/MachineTest.ts

@@ -1375,6 +1375,17 @@ export interface Exploration<M extends AnyMachine, Key extends ExplorationKey =
   readonly start: Graph.NodeIndex
   readonly limits: ResolvedExplorationLimits
   readonly stats: ExplorationStats
+  /**
+   * Exact declared transition branches witnessed by concretely planned work.
+   * This is observed evidence, not a claim that every reachable branch was
+   * explored. Startup and state-limit plans count; unplanned depth- and
+   * transition-limit frontiers do not.
+   */
+  readonly transitionCoverage: TransitionCoverage<
+    StateNodePath<M>,
+    Machine.Machine.TagOf<Machine.Machine.Events<M>[number]>,
+    StateNodePath<M>
+  >
   readonly completeness: ExplorationCompleteness<M, Key>
 }
 
@@ -1409,7 +1420,10 @@ export type ExploreOptions<M extends AnyMachine, Key extends ExplorationKey = Ex
  *
  * Invariants are checked against startup and every concretely planned edge,
  * so a failure retains a shortest discovered counterexample. Staged actions
- * and runtime activities are not executed.
+ * and runtime activities are not executed. `transitionCoverage` includes
+ * startup and every event plan that was actually computed, including a plan
+ * retained at a state-limit frontier. Depth- and transition-limit frontiers
+ * have no plan and therefore contribute no transition hits.
  *
  * **Example**
  *