test: compare the ported ConstraintSetParser against the oracle

parity/src/test/kotlin/tech/annexflow/parity/constraintset/ConstraintSetDifferentialTest.kt

@@ -0,0 +1,135 @@
+// Copyright 2023, Sergei Gagarin and the project contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package tech.annexflow.parity.constraintset
+
+import kotlin.test.Test
+import kotlin.test.fail
+
+/**
+ * Parses/measures a range of generated documents with both implementations and requires identical
+ * outcomes.
+ *
+ * The oracle is upstream, so a disagreement is a defect in the port until proven otherwise. When
+ * this fails, read the report and fix the port — do not relax the harness to accommodate it.
+ *
+ * Both [ConstraintSetSubject.parse] and [ConstraintSetSubject.measure] entry points are exercised,
+ * following `solver.SolverDifferentialTest`'s structure — and each is compared only against its own
+ * counterpart on the other side, never `parse` against `measure`. The two are different contracts
+ * (`measure` resolves a fixed root via a real `Measurer`; `parse` does not — see
+ * `ConstraintSetSubject.measure`'s kdoc) and a cross comparison would fail for reasons that say
+ * nothing about the port. Before this addition, `measure` — the only entry point that resolves bias
+ * or a chain's style/weight (see `AxisLivenessTest`'s `hBias`/`chainStyle`/`hWeight` cases) — was
+ * never run against the 2000-seed corpus at all: a liveness suite certifying those axes live through
+ * `measure` said nothing about whether the port agrees with the oracle on them, because this test,
+ * the one thing that actually compares the two implementations at scale, never called it.
+ */
+class ConstraintSetDifferentialTest {
+    private val seeds = 1L..2000L
+    private val maxExamplesPerEntry = 5
+
+    /**
+     * Both floors are measured against the current corpus (2000 seeds, unmutated port), not guessed
+     * — see the task report for the instrumented run each number came from. `parse` and `measure`
+     * land on the *same* populated count and geometry-row total: row count only depends on how many
+     * widgets/guidelines/barriers a document materialises, which is identical either way — the two
+     * entry points disagree (or, per this test, don't) on the *numbers inside* each row (a
+     * `MATCH_CONSTRAINT` widget spread to width 0 under `parse` but to a real span under `measure`;
+     * see `ConstraintSetSubject.measure`'s kdoc), not on how many rows exist. That distinction is
+     * exactly why a separate divergence check per entry point matters even though the floors turned
+     * out identical: two entry points can agree on shape while disagreeing on content, and it's the
+     * content this test exists to compare.
+     */
+    private enum class Entry(val label: String, val minimumPopulated: Int, val minimumGeometryRows: Int) {
+        // Measured: 1876 populated, 10576 geometry rows. 5000 stays under half of that, per the
+        // rationale `minimumGeometryRows` originally carried (headroom for ordinary generator
+        // changes, unreachable by a corpus of substance-free documents).
+        PARSE("parse", 1800, 5000),
+
+        // Measured: 1876 populated, 10576 geometry rows — identical to PARSE (see the class kdoc for
+        // why). Same floors as PARSE follow directly from that.
+        MEASURE("measure", 1800, 5000),
+    }
+
+    private fun run(entry: Entry, subject: ConstraintSetSubject, spec: ConstraintSetSpec): ConstraintSetOutcome =
+        when (entry) {
+            Entry.PARSE -> subject.parse(spec)
+            Entry.MEASURE -> subject.measure(spec)
+        }
+
+    @Test
+    fun thePortAgreesWithTheOracle() {
+        // Keyed per entry point rather than one shared list: PARSE runs before MEASURE for every
+        // seed, so a global cap would let early parse divergences crowd out every measure
+        // divergence from the reported examples while the counts stayed correct and the diagnostic
+        // went blind.
+        val divergences = mutableMapOf(Entry.PARSE to mutableListOf<String>(), Entry.MEASURE to mutableListOf<String>())
+        var totalDivergences = 0
+        val populated = mutableMapOf(Entry.PARSE to 0, Entry.MEASURE to 0)
+        val geometryRows = mutableMapOf(Entry.PARSE to 0, Entry.MEASURE to 0)
+
+        for (seed in seeds) {
+            val spec = Scenarios.generate(seed)
+            for (entry in Entry.entries) {
+                val oracle = run(entry, OracleConstraintSet, spec)
+                val port = run(entry, PortConstraintSet, spec)
+                if (oracle is ConstraintSetOutcome.Populated) {
+                    populated[entry] = populated.getValue(entry) + 1
+                    geometryRows[entry] = geometryRows.getValue(entry) + oracle.geometry.count { it == '\n' }
+                }
+                if (oracle != port) {
+                    totalDivergences++
+                    val examples = divergences.getValue(entry)
+                    if (examples.size < maxExamplesPerEntry) examples += report(entry, spec, oracle, port)
+                }
+            }
+        }
+
+        // Two ways a decayed generator passes the equality check while testing nothing:
+        // wholesale rejection, where both sides throw on every document and there is nothing left
+        // to compare; and wholesale vacuity, where both sides return `Populated` for every
+        // document but the documents carry no widgets — e.g. the emitter regressing to `{}` for
+        // every spec. Both sides would then agree trivially on empty geometry for all 2000 seeds,
+        // the populated count would clear the floor below, and the test would pass having compared
+        // nothing at all. `minimumPopulated` catches the first; `minimumGeometryRows`, which a
+        // corpus of empty documents cannot satisfy, catches the second — checked per entry point,
+        // because one of them failing to run at all (e.g. `measure` throwing on every document) is
+        // exactly the failure worth catching, and a shared total could hide it behind the other
+        // entry's healthy numbers.
+        for (entry in Entry.entries) {
+            if (populated.getValue(entry) < entry.minimumPopulated) {
+                fail(
+                    "only ${populated.getValue(entry)} of ${seeds.count()} documents populated through " +
+                        "${entry.label}; the generator is emitting junk",
+                )
+            }
+            if (geometryRows.getValue(entry) < entry.minimumGeometryRows) {
+                fail(
+                    "only ${geometryRows.getValue(entry)} geometry rows across ${populated.getValue(entry)} " +
+                        "populated documents through ${entry.label}; the generator is emitting substance-free " +
+                        "documents",
+                )
+            }
+        }
+
+        if (totalDivergences > 0) {
+            val examples = Entry.entries.flatMap { divergences.getValue(it) }
+            fail(
+                "$totalDivergences of ${seeds.count() * Entry.entries.size} comparisons diverged " +
+                    "(showing up to $maxExamplesPerEntry per entry point):\n\n${examples.joinToString("\n\n")}",
+            )
+        }
+    }
+
+    private fun report(
+        entry: Entry,
+        spec: ConstraintSetSpec,
+        oracle: ConstraintSetOutcome,
+        port: ConstraintSetOutcome,
+    ): String = buildString {
+        appendLine("seed ${spec.seed} via ${entry.label}")
+        appendLine(emit(spec))
+        appendLine("oracle: $oracle")
+        appendLine("port:   $port")
+    }
+}

parity/src/test/kotlin/tech/annexflow/parity/constraintset/ConstraintSetOutcome.kt

@@ -0,0 +1,105 @@
+// Copyright 2023, Sergei Gagarin and the project contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package tech.annexflow.parity.constraintset
+
+/**
+ * One widget's laid-out box, named without reference to either implementation's classes.
+ *
+ * [visibility], [alpha], the rotations, the scales, the translations and the pivots never affect
+ * `left`/`top`/`width`/`height` — they land on `WidgetFrame`, a sibling of the box computed by the
+ * solver, not an input to it (see `WidgetFrame.kt`). Folded into the same row rather than a new
+ * section: it keeps exactly one line per widget, which is what `ConstraintSetDifferentialTest`'s
+ * `minimumGeometryRows` floor (a count of newlines in `geometry`) assumes stays true across edits.
+ */
+data class GeometryRow(
+    val id: String,
+    val left: Int,
+    val top: Int,
+    val width: Int,
+    val height: Int,
+    val visibility: Int,
+    val alpha: Float,
+    val rotationX: Float,
+    val rotationY: Float,
+    val rotationZ: Float,
+    val scaleX: Float,
+    val scaleY: Float,
+    val translationX: Float,
+    val translationY: Float,
+    val translationZ: Float,
+    val pivotX: Float,
+    val pivotY: Float,
+)
+
+/** One custom attribute, already stringified by whichever subject read it. */
+data class CustomRow(val widgetId: String, val name: String, val value: String)
+
+/** One design element produced by `parseDesignElementsJSON`. */
+data class ElementRow(val id: String, val type: String, val params: Map<String, String>)
+
+fun renderGeometry(rows: List<GeometryRow>): String =
+    rows.joinToString(separator = "") { r ->
+        "${r.id} l=${r.left} t=${r.top} w=${r.width} h=${r.height} vis=${r.visibility} alpha=${r.alpha} " +
+            "rX=${r.rotationX} rY=${r.rotationY} rZ=${r.rotationZ} sX=${r.scaleX} sY=${r.scaleY} " +
+            "tX=${r.translationX} tY=${r.translationY} tZ=${r.translationZ} pvX=${r.pivotX} pvY=${r.pivotY}\n"
+    }
+
+// Sorted, unlike geometry: custom attributes come out of a HashMap, and the two implementations
+// have no reason to iterate one in the same order. Geometry keeps the caller's order because the
+// subjects walk the container's children, which is a list on both sides.
+fun renderCustom(rows: List<CustomRow>): String =
+    rows.sortedWith(compareBy({ it.widgetId }, { it.name }))
+        .joinToString(separator = "") { "${it.widgetId}.${it.name}=${it.value}\n" }
+
+fun renderElements(rows: List<ElementRow>): String =
+    rows.sortedBy { it.id }.joinToString(separator = "") { row ->
+        val params = row.params.entries.sortedBy { it.key }.joinToString(" ") { "${it.key}=${it.value}" }
+        "${row.id} type=${row.type}${if (params.isEmpty()) "" else " $params"}\n"
+    }
+
+/**
+ * Everything observable about parsing one document, normalised so the two implementations become
+ * comparable despite living in different packages. The failure side is deliberately the same shape
+ * as `LayoutOutcome` ([Leaked], [Crashed]); the success side is not — this type splits into
+ * [Populated] and [Elements] for its two entry points, where `LayoutOutcome` has only one
+ * (`LaidOut`).
+ */
+sealed interface ConstraintSetOutcome {
+    /** The layout entry point: a document parsed, applied to a container and laid out. */
+    data class Populated(val geometry: String, val custom: String) : ConstraintSetOutcome
+
+    /**
+     * The `parseDesignElementsJSON` entry point, which yields a list and no geometry at all. A
+     * separate case rather than a third field on [Populated]: the two entry points never run
+     * together, so a shared shape would leave half of it empty on every scenario.
+     */
+    data class Elements(val rendered: String) : ConstraintSetOutcome
+
+    /**
+     * An exception the parser is documented to raise — `CLParsingException` above all.
+     *
+     * Compared by portable category, not exception class: the port is multiplatform and cannot
+     * raise JVM-specific classes on Native or JS, so class equality would demand something no
+     * correct port could deliver.
+     */
+    data class Leaked(val category: String) : ConstraintSetOutcome
+
+    /**
+     * Anything else escaping. Distinct from [Leaked] on purpose: a parsing exception on malformed
+     * input is the parser working, while an `IndexOutOfBoundsException` is a defect on whichever
+     * side raised it. Collapsing the two would let a port that crashes match an oracle that rejects.
+     */
+    data class Crashed(val error: String) : ConstraintSetOutcome
+
+    companion object {
+        fun categorise(throwable: Throwable): String =
+            when (throwable) {
+                is IndexOutOfBoundsException -> "IndexOutOfBounds"
+                is NullPointerException -> "NullPointer"
+                is ArithmeticException -> "Arithmetic"
+                is NumberFormatException -> "NumberFormat"
+                else -> throwable::class.simpleName ?: "Unknown"
+            }
+    }
+}

parity/src/test/kotlin/tech/annexflow/parity/constraintset/ConstraintSetOutcomeTest.kt

@@ -0,0 +1,56 @@
+// Copyright 2023, Sergei Gagarin and the project contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package tech.annexflow.parity.constraintset
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class ConstraintSetOutcomeTest {
+    private fun row(id: String, left: Int, top: Int, width: Int, height: Int) = GeometryRow(
+        id, left, top, width, height,
+        visibility = 0, alpha = Float.NaN,
+        rotationX = Float.NaN, rotationY = Float.NaN, rotationZ = Float.NaN,
+        scaleX = Float.NaN, scaleY = Float.NaN,
+        translationX = Float.NaN, translationY = Float.NaN, translationZ = Float.NaN,
+        pivotX = Float.NaN, pivotY = Float.NaN,
+    )
+
+    @Test
+    fun geometryRendersOneRowPerWidgetInGivenOrder() {
+        val rendered = renderGeometry(listOf(row("b", 10, 20, 30, 40), row("a", 0, 0, 5, 5)))
+        assertEquals(
+            "b l=10 t=20 w=30 h=40 vis=0 alpha=NaN rX=NaN rY=NaN rZ=NaN sX=NaN sY=NaN tX=NaN tY=NaN tZ=NaN " +
+                "pvX=NaN pvY=NaN\n" +
+                "a l=0 t=0 w=5 h=5 vis=0 alpha=NaN rX=NaN rY=NaN rZ=NaN sX=NaN sY=NaN tX=NaN tY=NaN tZ=NaN " +
+                "pvX=NaN pvY=NaN\n",
+            rendered,
+        )
+    }
+
+    @Test
+    fun customRowsAreSortedSoIterationOrderCannotLeakIn() {
+        val rendered = renderCustom(
+            listOf(
+                CustomRow("w", "zeta", "1.0"),
+                CustomRow("w", "alpha", "2.0"),
+                CustomRow("a", "beta", "3.0"),
+            ),
+        )
+        assertEquals("a.beta=3.0\nw.alpha=2.0\nw.zeta=1.0\n", rendered)
+    }
+
+    @Test
+    fun elementParamsAreSorted() {
+        val rendered = renderElements(
+            listOf(ElementRow("e", "button", mapOf("b" to "2", "a" to "1"))),
+        )
+        assertEquals("e type=button a=1 b=2\n", rendered)
+    }
+
+    @Test
+    fun categoriseMapsToPortableNames() {
+        assertEquals("IndexOutOfBounds", ConstraintSetOutcome.categorise(IndexOutOfBoundsException()))
+        assertEquals("NullPointer", ConstraintSetOutcome.categorise(NullPointerException()))
+    }
+}