jsonwrite writes valid, separate arrays (4.2.37)

.github/workflows/build-linux.yml

@@ -1505,6 +1505,38 @@ jobs:
           fi
           echo "[json] jsonwrite serializes and jsonparse round-trips"
 
+      - name: JSON arrays (Non-20.04)
+        if: matrix.id != 'docker-ubuntu-20.04'
+        shell: bash
+        run: |
+          set -eo pipefail
+          # jsonwrite groups counted concepts (item1, item2) into arrays.
+          # Until 4.2.37 an array followed by a plain sibling was not JSON,
+          # two arrays side by side merged, and "a1b2" counted as 12. Each
+          # case must match its expected file and survive a jsonparse round
+          # trip unchanged.
+          FIX=.github/workflows/tests/json-arrays
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log"
+          mkdir -p "$FIX/logs"
+          ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          if [ -f "$FIX/logs/make_ana.log" ]; then cat "$FIX/logs/make_ana.log"; fi
+          OUT="$FIX/input/text.txt_log/out.txt"
+          if [ ! -f "$OUT" ]; then
+            echo "FAIL(json arrays): no out.txt -- the analyzer did not build"; exit 1
+          fi
+          got=$(tr -d '\r' < "$OUT")
+          want=$(printf '%s\n' 'A:same' 'B:same' 'C:same' 'E:same' 'F:same')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(json arrays): round trip, out.txt was:"; echo "$got"; exit 1
+          fi
+          for c in A B C E F; do
+            # Strip CR on both sides: the Windows checkout gives expected/ CRLF.
+            if ! diff <(tr -d '\r' < "$FIX/input/text.txt_log/$c.json") <(tr -d '\r' < "$FIX/expected/$c.json"); then
+              echo "FAIL(json arrays): case $c differs from expected/$c.json"; exit 1
+            fi
+          done
+          echo "[json arrays] counted concepts become valid, separate arrays"
+
       - name: callanalyzer runs another analyzer, loaded once
         if: matrix.id != 'docker-ubuntu-20.04'
         shell: bash

.github/workflows/build-windows.yml

@@ -1389,6 +1389,37 @@ jobs:
           fi
           echo "[json] jsonwrite serializes and jsonparse round-trips"
 
+      - name: JSON arrays
+        shell: bash
+        run: |
+          set -eo pipefail
+          # jsonwrite groups counted concepts (item1, item2) into arrays.
+          # Until 4.2.37 an array followed by a plain sibling was not JSON,
+          # two arrays side by side merged, and "a1b2" counted as 12. Each
+          # case must match its expected file and survive a jsonparse round
+          # trip unchanged.
+          FIX=.github/workflows/tests/json-arrays
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log"
+          mkdir -p "$FIX/logs"
+          ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          if [ -f "$FIX/logs/make_ana.log" ]; then cat "$FIX/logs/make_ana.log"; fi
+          OUT="$FIX/input/text.txt_log/out.txt"
+          if [ ! -f "$OUT" ]; then
+            echo "FAIL(json arrays): no out.txt -- the analyzer did not build"; exit 1
+          fi
+          got=$(tr -d '\r' < "$OUT")
+          want=$(printf '%s\n' 'A:same' 'B:same' 'C:same' 'E:same' 'F:same')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(json arrays): round trip, out.txt was:"; echo "$got"; exit 1
+          fi
+          for c in A B C E F; do
+            # Strip CR on both sides: the Windows checkout gives expected/ CRLF.
+            if ! diff <(tr -d '\r' < "$FIX/input/text.txt_log/$c.json") <(tr -d '\r' < "$FIX/expected/$c.json"); then
+              echo "FAIL(json arrays): case $c differs from expected/$c.json"; exit 1
+            fi
+          done
+          echo "[json arrays] counted concepts become valid, separate arrays"
+
       - name: callanalyzer runs another analyzer, loaded once
         shell: bash
         run: |

.github/workflows/tests/json-arrays/expected/A.json

@@ -0,0 +1,17 @@
+{
+  "caseA": {
+    "item": [
+      {
+        "id": "1",
+        "v": "item1"
+      },
+      {
+        "id": "2",
+        "v": "item2"
+      }
+    ],
+    "name": {
+      "v": "name"
+    }
+  }
+}

.github/workflows/tests/json-arrays/expected/B.json

@@ -0,0 +1,24 @@
+{
+  "caseB": {
+    "item": [
+      {
+        "id": "1",
+        "v": "item1"
+      },
+      {
+        "id": "2",
+        "v": "item2"
+      }
+    ],
+    "thing": [
+      {
+        "id": "1",
+        "v": "thing1"
+      },
+      {
+        "id": "2",
+        "v": "thing2"
+      }
+    ]
+  }
+}

.github/workflows/tests/json-arrays/expected/C.json

@@ -0,0 +1,14 @@
+{
+  "caseC": {
+    "a1b": [
+      {
+        "id": "1",
+        "v": "a1b1"
+      },
+      {
+        "id": "2",
+        "v": "a1b2"
+      }
+    ]
+  }
+}

.github/workflows/tests/json-arrays/expected/E.json

@@ -0,0 +1,18 @@
+{
+  "caseE": {
+    "row": [
+      {
+        "id": "1"
+      },
+      {
+        "id": "2",
+        "cell": {
+          "v": "cell"
+        }
+      }
+    ],
+    "tail": {
+      "v": "tail"
+    }
+  }
+}

.github/workflows/tests/json-arrays/expected/F.json

@@ -0,0 +1,36 @@
+{
+  "caseF": {
+    "person": [
+      {
+        "id": "1",
+        "v": "person1",
+        "phone": [
+          {
+            "id": "1",
+            "v": "phone1"
+          },
+          {
+            "id": "2",
+            "v": "phone2"
+          }
+        ],
+        "email": {
+          "v": "email"
+        }
+      },
+      {
+        "id": "2",
+        "v": "person2",
+        "phone": [
+          {
+            "id": "1",
+            "v": "phone1"
+          }
+        ]
+      }
+    ],
+    "count": {
+      "v": "count"
+    }
+  }
+}

.github/workflows/tests/json-arrays/input/text.txt

@@ -0,0 +1 @@
+The fixture builds its own knowledge base, not from this text.

.github/workflows/tests/json-arrays/spec/analyzer.seq

@@ -0,0 +1,2 @@
+tokenize	nil	# tokenize only -- fixture builds its own KB
+nlp	arrays	# jsonwrite array grouping, round-tripped through jsonparse

.github/workflows/tests/json-arrays/spec/arrays.nlp

@@ -0,0 +1,74 @@
+###############################################
+# FILE:     arrays.nlp
+# SUBJ:     Regression for how jsonwrite groups counted concepts into arrays.
+# NOTE:     Until 4.2.37 jsonwrite (and the JsonKB it was copied from) wrote
+#           the comma after each child, so an array followed by a plain
+#           sibling came out with a stray brace and no comma -- not JSON.
+#           Two arrays side by side merged into one, and a digit inside a
+#           name ("a1b2") was taken as part of the count.
+#
+#           Each case is written to <case>.json and compared in CI with
+#           expected/<case>.json, then read back with jsonparse and written
+#           again; the second document must equal the first.
+#
+#           Expected out.txt:
+#           A:same
+#           B:same
+#           C:same
+#           E:same
+#           F:same
+###############################################
+
+@DECL
+Mk(L("p"),L("name")) {
+	L("c") = makeconcept(L("p"),L("name"));
+	addstrval(L("c"),"v",L("name"));
+	return L("c");
+}
+Check(L("label"),L("con")) {
+	jsonwrite(L("label") + ".json",L("con"));
+	L("first") = readfile(G("$apppath") + "/input/text.txt_log/" + L("label") + ".json");
+	L("back") = makeconcept(findroot(),L("label") + "back");
+	L("wrapped") = jsonparse(L("first"),L("back"));
+	jsonwrite(L("label") + "-again.json",down(L("back")));
+	L("second") = readfile(G("$apppath") + "/input/text.txt_log/" + L("label") + "-again.json");
+	if (L("wrapped") && L("first") == L("second"))
+		"out.txt" << L("label") << ":same\n";
+	else
+		"out.txt" << L("label") << ":DIFFERENT\n";
+}
+@@DECL
+
+@CODE
+# A: an array, then a plain sibling
+L("r") = makeconcept(findroot(),"caseA");
+Mk(L("r"),"item1"); Mk(L("r"),"item2"); Mk(L("r"),"name");
+Check("A",L("r"));
+
+# B: two arrays side by side
+L("r") = makeconcept(findroot(),"caseB");
+Mk(L("r"),"item1"); Mk(L("r"),"item2"); Mk(L("r"),"thing1"); Mk(L("r"),"thing2");
+Check("B",L("r"));
+
+# C: a digit inside the base name
+L("r") = makeconcept(findroot(),"caseC");
+Mk(L("r"),"a1b1"); Mk(L("r"),"a1b2");
+Check("C",L("r"));
+
+# E: array elements with no attributes, one with a child, then a plain sibling
+L("r") = makeconcept(findroot(),"caseE");
+makeconcept(L("r"),"row1");
+L("x") = makeconcept(L("r"),"row2");
+Mk(L("x"),"cell");
+Mk(L("r"),"tail");
+Check("E",L("r"));
+
+# F: an array inside an element of an array, then a plain sibling
+L("r") = makeconcept(findroot(),"caseF");
+L("p") = Mk(L("r"),"person1");
+Mk(L("p"),"phone1"); Mk(L("p"),"phone2"); Mk(L("p"),"email");
+L("p") = Mk(L("r"),"person2");
+Mk(L("p"),"phone1");
+Mk(L("r"),"count");
+Check("F",L("r"));
+@@CODE