Compiled analyzers can call interpreter-only builtins (4.2.25)

.github/workflows/build-windows.yml

@@ -1243,6 +1243,36 @@ jobs:
 
           Compress-Archive -Path (Join-Path $outRoot '*') -DestinationPath (Join-Path $env:GITHUB_WORKSPACE 'nlpengine-compile-libs.zip') -Force
 
+      - name: Compiled analyzer calls interpreter-only builtins
+        shell: bash
+        run: |
+          set -eo pipefail
+          # A builtin added to the interpreter without a compiled Arun:: twin made
+          # every compiled analyzer that called it fail to build: 23 of them, among
+          # them jsonwrite, which the KB template calls. The generator now routes
+          # them through Arun::fnbridge. Build this fixture against the compile
+          # libraries just packaged, run it, and compare with the interpreted run.
+          FIX=.github/workflows/tests/compiled-bridge
+          LIBS="$(cygpath -m "$GITHUB_WORKSPACE")/compile-libs"
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" "$FIX/run" "$FIX/build"
+          mkdir -p "$FIX/logs"
+          ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          tr -d '\r' < "$FIX/input/text.txt_log/output.txt" > bridge-interpreted.txt
+          n=$(wc -l < bridge-interpreted.txt)
+          if [ "$n" -ne 23 ]; then
+            echo "FAIL(compiled-bridge): interpreted run printed $n lines, wanted 23:"; cat bridge-interpreted.txt; exit 1
+          fi
+          rm -rf "$FIX/input/text.txt_log" "$FIX/output"
+          ./bin/Release/nlp.exe -COMPILEANA -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          cmake -S "$FIX" -B "$FIX/build" -A x64 -DENGINE_LIBS="$LIBS"
+          cmake --build "$FIX/build" --config Release
+          rm -rf "$FIX/input/text.txt_log" "$FIX/output"
+          ./bin/Release/nlp.exe -COMPILED -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          if ! diff bridge-interpreted.txt <(tr -d '\r' < "$FIX/input/text.txt_log/output.txt"); then
+            echo "FAIL(compiled-bridge): compiled output differs from interpreted"; exit 1
+          fi
+          echo "[compiled-bridge] interpreter-only builtins work in a compiled analyzer"
+
       - name: Upload nlpengine-compile-libs.zip
         uses: actions/upload-artifact@v4
         with:

.github/workflows/tests/compiled-bridge/.gitignore

@@ -0,0 +1,10 @@
+run/
+kb/
+build/
+logs/
+output/
+input/*_log/
+*.dll
+*.exp
+*.lib
+*.pdb

.github/workflows/tests/compiled-bridge/CMakeLists.txt

@@ -0,0 +1,50 @@
+# Builds this fixture as a compiled analyzer (analyzer.dll) against the
+# packaged compile libraries, as the VS Code extension and the compile service
+# do (their generated CMakeLists.txt is the model). Used by the Windows CI step
+# "Compiled analyzer calls interpreter-only builtins".
+#
+#   nlp -COMPILEANA -ANA <this folder> ...     (writes run/*.cpp)
+#   cmake -S <this folder> -B <this folder>/build -A x64 -DENGINE_LIBS=<compile-libs>
+#   cmake --build <this folder>/build --config Release
+cmake_minimum_required(VERSION 3.16)
+project(nlp_generated_library LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+if(NOT ENGINE_LIBS)
+    message(FATAL_ERROR "Pass -DENGINE_LIBS=<unpacked nlpengine-compile-libs folder>.")
+endif()
+
+set(ANALYZER_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${ANALYZER_DIR}")
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${ANALYZER_DIR}")
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${ANALYZER_DIR}")
+foreach(OUTPUTCONFIG ${CMAKE_CONFIGURATION_TYPES})
+    string(TOUPPER ${OUTPUTCONFIG} OUTPUTCONFIG_UPPER)
+    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${OUTPUTCONFIG_UPPER} "${ANALYZER_DIR}")
+    set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${OUTPUTCONFIG_UPPER} "${ANALYZER_DIR}")
+    set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${OUTPUTCONFIG_UPPER} "${ANALYZER_DIR}")
+endforeach()
+
+file(GLOB GENERATED_CPP "${ANALYZER_DIR}/run/*.cpp" "${ANALYZER_DIR}/kb/*.cpp")
+if(NOT GENERATED_CPP)
+    message(FATAL_ERROR "No generated C++ in run/ or kb/. Run nlp -COMPILEANA first.")
+endif()
+
+add_library(nlp_generated SHARED ${GENERATED_CPP})
+set_target_properties(nlp_generated PROPERTIES OUTPUT_NAME "analyzer" PREFIX "")
+
+target_include_directories(nlp_generated PRIVATE
+    "${ANALYZER_DIR}"
+    "${ANALYZER_DIR}/run"
+    "${ANALYZER_DIR}/kb"
+    "${ENGINE_LIBS}/include/Api"
+    "${ENGINE_LIBS}/include/cs"
+    "${ENGINE_LIBS}/include"
+)
+
+file(GLOB ENGINE_LIBS_ALL "${ENGINE_LIBS}/lib/*.lib")
+target_link_libraries(nlp_generated PRIVATE ${ENGINE_LIBS_ALL})
+target_compile_definitions(nlp_generated PRIVATE _CRT_SECURE_NO_WARNINGS)
+target_compile_options(nlp_generated PRIVATE /wd4005 /FI"StdAfx.h" /MP)

.github/workflows/tests/compiled-bridge/StdAfx.h

@@ -0,0 +1,13 @@
+// Auto-generated stub. Engine-generated .cpp files include "StdAfx.h" by convention.
+#pragma once
+#ifdef _WIN32
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#include <tchar.h>
+#endif
+#include "my_tchar.h"

.github/workflows/tests/compiled-bridge/input/text.txt

@@ -0,0 +1 @@
+hello world

.github/workflows/tests/compiled-bridge/spec/abs.nlp

@@ -0,0 +1,10 @@
+###############################################
+# FILE:     abs.nlp
+# SUBJ:     abs() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("r") = abs(-5);
+"output.txt" << "abs: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/analyzer.seq

@@ -0,0 +1,24 @@
+tokenize	nil	# 
+nlp	abs	# 
+nlp	arrayreverse	# 
+nlp	arrayslice	# 
+nlp	arraysort	# 
+nlp	arrayunique	# 
+nlp	push	# 
+nlp	strindexof	# 
+nlp	strlastindexof	# 
+nlp	striscaps	# 
+nlp	strjoin	# 
+nlp	rematch	# 
+nlp	refind	# 
+nlp	resubst	# 
+nlp	fileexists	# 
+nlp	direxists	# 
+nlp	filesize	# 
+nlp	readfile	# 
+nlp	readlines	# 
+nlp	dirlist	# 
+nlp	deletefile	# 
+nlp	randomint	# 
+nlp	jsonwrite	# 
+nlp	jsonparse	# 

.github/workflows/tests/compiled-bridge/spec/arrayreverse.nlp

@@ -0,0 +1,12 @@
+###############################################
+# FILE:     arrayreverse.nlp
+# SUBJ:     arrayreverse() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("a")[0] = 1; L("a")[1] = 2; L("a")[2] = 3;
+L("b") = arrayreverse(L("a"));
+L("r") = str(L("b")[0]) + str(L("b")[1]) + str(L("b")[2]);
+"output.txt" << "arrayreverse: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/arrayslice.nlp

@@ -0,0 +1,12 @@
+###############################################
+# FILE:     arrayslice.nlp
+# SUBJ:     arrayslice() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("a")[0] = "p"; L("a")[1] = "q"; L("a")[2] = "r";
+L("b") = arrayslice(L("a"), 1);
+L("r") = L("b")[0] + L("b")[1];
+"output.txt" << "arrayslice: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/arraysort.nlp

@@ -0,0 +1,12 @@
+###############################################
+# FILE:     arraysort.nlp
+# SUBJ:     arraysort() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("a")[0] = 30; L("a")[1] = 10; L("a")[2] = 20;
+L("b") = arraysort(L("a"), 1);
+L("r") = str(L("b")[0]) + "," + str(L("b")[1]) + "," + str(L("b")[2]);
+"output.txt" << "arraysort: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/arrayunique.nlp

@@ -0,0 +1,12 @@
+###############################################
+# FILE:     arrayunique.nlp
+# SUBJ:     arrayunique() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("a")[0] = "x"; L("a")[1] = "x"; L("a")[2] = "y";
+L("b") = arrayunique(L("a"));
+L("r") = str(arraylength(L("b")));
+"output.txt" << "arrayunique: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/deletefile.nlp

@@ -0,0 +1,10 @@
+###############################################
+# FILE:     deletefile.nlp
+# SUBJ:     deletefile() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("r") = str(deletefile("no-such-file.txt"));
+"output.txt" << "deletefile: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/direxists.nlp

@@ -0,0 +1,10 @@
+###############################################
+# FILE:     direxists.nlp
+# SUBJ:     direxists() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("r") = str(direxists("no-such-dir"));
+"output.txt" << "direxists: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/dirlist.nlp

@@ -0,0 +1,10 @@
+###############################################
+# FILE:     dirlist.nlp
+# SUBJ:     dirlist() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("r") = str(arraylength(dirlist("no-such-dir")));
+"output.txt" << "dirlist: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/fileexists.nlp

@@ -0,0 +1,10 @@
+###############################################
+# FILE:     fileexists.nlp
+# SUBJ:     fileexists() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("r") = str(fileexists("no-such-file.txt"));
+"output.txt" << "fileexists: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/filesize.nlp

@@ -0,0 +1,10 @@
+###############################################
+# FILE:     filesize.nlp
+# SUBJ:     filesize() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("r") = filesize("no-such-file.txt");
+"output.txt" << "filesize: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/jsonparse.nlp

@@ -0,0 +1,12 @@
+###############################################
+# FILE:     jsonparse.nlp
+# SUBJ:     jsonparse() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("c") = makeconcept(findroot(), "jp");
+L("ok") = jsonparse("{\"a\": 1}", L("c"));
+L("r") = str(L("ok")) + " " + str(numval(L("c"), "a"));
+"output.txt" << "jsonparse: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/jsonwrite.nlp

@@ -0,0 +1,14 @@
+###############################################
+# FILE:     jsonwrite.nlp
+# SUBJ:     jsonwrite() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("c") = makeconcept(findroot(), "jw");
+addstrval(L("c"), "k", "v");
+L("f") = G("$apppath") + "/output/bridge.json";
+jsonwrite(L("f"), L("c"));
+L("r") = str(fileexists(L("f"))) + " " + str(strlength(readfile(L("f"))) > 10);
+"output.txt" << "jsonwrite: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/push.nlp

@@ -0,0 +1,12 @@
+###############################################
+# FILE:     push.nlp
+# SUBJ:     push() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("a")[0] = 2; L("a")[1] = 3;
+L("b") = push(1, L("a"));
+L("r") = str(L("b")[0]) + str(L("b")[1]) + str(L("b")[2]);
+"output.txt" << "push: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/randomint.nlp

@@ -0,0 +1,11 @@
+###############################################
+# FILE:     randomint.nlp
+# SUBJ:     randomint() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("n") = randomint(5, 5);
+L("r") = L("n");
+"output.txt" << "randomint: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/readfile.nlp

@@ -0,0 +1,10 @@
+###############################################
+# FILE:     readfile.nlp
+# SUBJ:     readfile() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("r") = "[" + readfile("no-such-file.txt") + "]";
+"output.txt" << "readfile: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/readlines.nlp

@@ -0,0 +1,10 @@
+###############################################
+# FILE:     readlines.nlp
+# SUBJ:     readlines() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("r") = str(arraylength(readlines("no-such-file.txt")));
+"output.txt" << "readlines: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/refind.nlp

@@ -0,0 +1,10 @@
+###############################################
+# FILE:     refind.nlp
+# SUBJ:     refind() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("r") = refind("abcabc", "c");
+"output.txt" << "refind: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/rematch.nlp

@@ -0,0 +1,10 @@
+###############################################
+# FILE:     rematch.nlp
+# SUBJ:     rematch() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("r") = str(rematch("abc", "a.c")) + str(rematch("abc", "x"));
+"output.txt" << "rematch: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/resubst.nlp

@@ -0,0 +1,10 @@
+###############################################
+# FILE:     resubst.nlp
+# SUBJ:     resubst() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("r") = resubst("abcabc", "b", "X");
+"output.txt" << "resubst: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/strindexof.nlp

@@ -0,0 +1,10 @@
+###############################################
+# FILE:     strindexof.nlp
+# SUBJ:     strindexof() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("r") = strindexof("hello", "l");
+"output.txt" << "strindexof: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/striscaps.nlp

@@ -0,0 +1,10 @@
+###############################################
+# FILE:     striscaps.nlp
+# SUBJ:     striscaps() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("r") = str(striscaps("Hello")) + str(striscaps("hello"));
+"output.txt" << "striscaps: " << L("r") << "\n";
+@@CODE

.github/workflows/tests/compiled-bridge/spec/strjoin.nlp

@@ -0,0 +1,11 @@
+###############################################
+# FILE:     strjoin.nlp
+# SUBJ:     strjoin() in interpreted and compiled analyzers. A compiled
+#           analyzer calls it through Arun::fnbridge.
+###############################################
+
+@CODE
+L("a")[0] = "x"; L("a")[1] = "y"; L("a")[2] = "z";
+L("r") = strjoin(L("a"), "-");
+"output.txt" << "strjoin: " << L("r") << "\n";
+@@CODE