File and directory helpers on std::filesystem; mkdir() without the shell (#323) (4.2.31)

.github/workflows/build-linux.yml

@@ -1154,6 +1154,40 @@ jobs:
           fi
           echo "[dict-shape] dictionary join, split, _name lines and bare entries"
 
+      - name: mkdir() and kbdumptree() without the shell (#323) (Non-20.04)
+        if: matrix.id != 'docker-ubuntu-20.04'
+        shell: bash
+        run: |
+          set -eo pipefail
+          # mkdir() was a shell "mkdir PATH": a path with spaces (or, on
+          # Windows, forward slashes) made nothing, a path with & ran a
+          # command, and it always returned 1. kbdumptree() ran "cat" on
+          # Linux and macOS. Both use std::filesystem / C++ now (#323).
+          FIX=.github/workflows/tests/fs-ops
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" "$FIX/made" pwned_323.txt
+          mkdir -p "$FIX/logs"
+          ./bin/nlp -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          OUT="$FIX/input/text.txt_log/output.txt"
+          if [ ! -f "$OUT" ]; then
+            echo "FAIL(fs-ops): no output.txt -- the analyzer did not build"; exit 1
+          fi
+          got=$(tr -d '\r' < "$OUT")
+          want=$(printf '%s\n' 'plain 1' 'spaces 1' 'nested 1' 'again 1' 'dump 1')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(fs-ops): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1
+          fi
+          for d in plain "with spaces" a/b/c; do
+            [ -d "$FIX/made/$d" ] || { echo "FAIL(fs-ops): mkdir() did not make $d"; exit 1; }
+          done
+          if [ -e pwned_323.txt ]; then
+            echo "FAIL(fs-ops): mkdir() ran a shell command from its path"; exit 1
+          fi
+          DUMP="$FIX/made/dump.kb"
+          grep -q 'add hier "concept" "fsTest" "child"' "$DUMP" || { echo "FAIL(fs-ops): dump.kb lacks the child"; cat "$DUMP"; exit 1; }
+          grep -q '^"blue"' "$DUMP" || { echo "FAIL(fs-ops): dump.kb lacks the attribute"; cat "$DUMP"; exit 1; }
+          [ "$(tr -d '\r' < "$DUMP" | grep -v '^$' | tail -1)" = quit ] || { echo "FAIL(fs-ops): dump.kb does not end in quit"; cat "$DUMP"; exit 1; }
+          echo "[fs-ops] mkdir() and kbdumptree() work without the shell"
+
       - name: factorial() up to 20! (Non-20.04)
         if: matrix.id != 'docker-ubuntu-20.04'
         shell: bash

.github/workflows/build-windows.yml

@@ -1048,6 +1048,39 @@ jobs:
           fi
           echo "[dict-shape] dictionary join, split, _name lines and bare entries"
 
+      - name: mkdir() and kbdumptree() without the shell (#323)
+        shell: bash
+        run: |
+          set -eo pipefail
+          # mkdir() was a shell "mkdir PATH": a path with spaces (or, on
+          # Windows, forward slashes) made nothing, a path with & ran a
+          # command, and it always returned 1. kbdumptree() ran "cat" on
+          # Linux and macOS. Both use std::filesystem / C++ now (#323).
+          FIX=.github/workflows/tests/fs-ops
+          rm -rf "$FIX/logs" "$FIX/input/text.txt_log" "$FIX/output" "$FIX/made" pwned_323.txt
+          mkdir -p "$FIX/logs"
+          ./bin/Release/nlp.exe -ANA "$FIX" -WORK ./ "$FIX/input/text.txt"
+          OUT="$FIX/input/text.txt_log/output.txt"
+          if [ ! -f "$OUT" ]; then
+            echo "FAIL(fs-ops): no output.txt -- the analyzer did not build"; exit 1
+          fi
+          got=$(tr -d '\r' < "$OUT")
+          want=$(printf '%s\n' 'plain 1' 'spaces 1' 'nested 1' 'again 1' 'dump 1')
+          if [ "$got" != "$want" ]; then
+            echo "FAIL(fs-ops): output.txt was:"; echo "$got"; echo "wanted:"; echo "$want"; exit 1
+          fi
+          for d in plain "with spaces" a/b/c; do
+            [ -d "$FIX/made/$d" ] || { echo "FAIL(fs-ops): mkdir() did not make $d"; exit 1; }
+          done
+          if [ -e pwned_323.txt ]; then
+            echo "FAIL(fs-ops): mkdir() ran a shell command from its path"; exit 1
+          fi
+          DUMP="$FIX/made/dump.kb"
+          grep -q 'add hier "concept" "fsTest" "child"' "$DUMP" || { echo "FAIL(fs-ops): dump.kb lacks the child"; cat "$DUMP"; exit 1; }
+          grep -q '^"blue"' "$DUMP" || { echo "FAIL(fs-ops): dump.kb lacks the attribute"; cat "$DUMP"; exit 1; }
+          [ "$(tr -d '\r' < "$DUMP" | grep -v '^$' | tail -1)" = quit ] || { echo "FAIL(fs-ops): dump.kb does not end in quit"; cat "$DUMP"; exit 1; }
+          echo "[fs-ops] mkdir() and kbdumptree() work without the shell"
+
       - name: factorial() up to 20!
         shell: bash
         run: |

.github/workflows/tests/fs-ops/input/text.txt

@@ -0,0 +1 @@
+x

.github/workflows/tests/fs-ops/spec/analyzer.seq

@@ -0,0 +1,2 @@
+tokenize	nil	# 
+nlp	ops	# 

.github/workflows/tests/fs-ops/spec/ops.nlp

@@ -0,0 +1,22 @@
+###############################################
+# FILE:     ops.nlp
+# SUBJ:     mkdir() and kbdumptree() (#323).
+# NOTE:     mkdir() was a shell "mkdir PATH": paths with spaces or
+#           forward slashes (on Windows) made nothing, a path with & ran
+#           a command, and it always returned 1. kbdumptree() ran "cat"
+#           on Linux and macOS. Expected output is in the CI step.
+###############################################
+
+@CODE
+L("base") = G("$apppath") + "/made";
+"output.txt" << "plain " << mkdir(L("base") + "/plain") << "\n";
+"output.txt" << "spaces " << mkdir(L("base") + "/with spaces") << "\n";
+"output.txt" << "nested " << mkdir(L("base") + "/a/b/c") << "\n";
+"output.txt" << "again " << mkdir(L("base") + "/plain") << "\n";
+# Must not run "echo": the CI step checks no pwned_323.txt appears.
+mkdir(L("base") + "/x & echo pwned > pwned_323.txt");
+
+L("con") = makeconcept(findroot(), "fsTest");
+addstrval(makeconcept(L("con"), "child"), "color", "blue");
+"output.txt" << "dump " << kbdumptree(L("con"), L("base") + "/dump.kb") << "\n";
+@@CODE

.github/workflows/tests/fs-ops/tmp/attr.kb

@@ -0,0 +1,9 @@
+ind attr
+"concept" "fsTest" "child"
+0
+"concept" "sys" "dict" "a" "color"
+pst
+"blue"
+end ind
+
+

.github/workflows/tests/fs-ops/tmp/hier.kb

@@ -0,0 +1,2 @@
+add hier "concept" "fsTest"
+add hier "concept" "fsTest" "child"

.github/workflows/tests/fs-ops/tmp/phr.kb

@@ -0,0 +1,3 @@
+
+quit
+

cs/libconsh/cg.cpp

@@ -3066,9 +3066,12 @@ delete f_attr;												// 08/07/03 AM.
 // Terminate the last command file.
 f_phr  << std::endl << _T("quit") << std::endl << std::endl;
 
-// TODO: CATENATE THE COMMAND FILES TO THE OUTPUT FILE.
-#ifndef LINUX
-// Do the catenate in C++.
+// Catenate the command files to the output file, in C++ on every
+// platform: Linux and macOS ran "cat", through the shell (#323). Flush
+// the files still open first, or their last lines may be missing.
+f_hier.close();
+f_word.close();
+f_phr.close();
 _TCHAR buf[MAXMSG];
 std::_t_ofstream *f_dump = new std::_t_ofstream(TCHAR2A(file));	// Final output file.
 std::_t_ifstream *f_in = new std::_t_ifstream(TCHAR2A(o_hier));
@@ -3096,14 +3099,6 @@ delete f_in;
 
 delete f_dump;
 
-#else
-_TCHAR cmd[MAXPATH*9];
-_TCHAR *cp = _T("cat");
-_stprintf(cmd, _T("%s %s %s %s %s > %s"),
-	cp, o_hier, o_word, o_attr, o_phr, file);
-int systemRet = _tsystem(cmd);
-#endif
-
 *cgerr << _T("[Done with modular kb dump.]") << std::endl;
 return true;
 }

cs/libprim/dir.cpp

@@ -10,6 +10,7 @@ All rights reserved.
 *
 *******************************************************************************/
 #include "StdAfx.h"
+#include <filesystem>		// #323.
 #ifdef LINUX
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -34,14 +35,8 @@ make_dir(_TCHAR *dir)
 {
 if (!dir || !*dir)
 	return false;
-#ifdef LINUX
-if (mkdir(dir, 0755) == 0)
-	return true;
-#else
-if (_tmkdir(dir) == 0)
-	return true;
-#endif
-return false;
+std::error_code ec;
+return std::filesystem::create_directory(std::filesystem::path(dir), ec);	// #323.
 }
 
 
@@ -57,12 +52,9 @@ rm_dir(_TCHAR *dir)
 {
 if (!dir || !*dir)
 	return false;
-#ifdef LINUX
-if (rmdir(dir) == 0)
-	return true;
-#else
-if (_trmdir(dir) == 0)
-	return true;
-#endif
-return false;
+std::error_code ec;
+std::filesystem::path p(dir);
+if (!std::filesystem::is_directory(p, ec))
+	return false;
+return std::filesystem::remove(p, ec);	// Only if empty.	// #323.
 }

lite/fn.cpp

@@ -12,6 +12,7 @@ All rights reserved.
 *******************************************************************************/
 
 #include "StdAfx.h"
+#include <filesystem>		// #323.
 #include <limits.h>	//MAXINT,MAXLONG,MAX_INT,MAX_LONG			// 01/20/00 AM.
 #include <locale.h>	// For char handling.							// 01/06/03 AM.
 #include <math.h>		// For math, log10.								// 04/29/04 AM.
@@ -11284,10 +11285,14 @@ ok = (_tsystem(cmd) ? false : true);
 #endif
 #endif
 
-_TCHAR cmd[2048];	// 09/08/28 AM.
-_stprintf(cmd,_T("mkdir %s"), str);	// 09/08/28 AM.
-run_silent(cmd);			// system() w/o DOC CMD PROMPT POPUP.	// 09/28/08 AM.
-ok = true;	// 09/28/08 AM.
+// Was a shell "mkdir STR": a path with spaces failed, one with & or ;
+// ran a command, and it always reported success (#323).
+{
+std::error_code ec;
+std::filesystem::path dp(str);
+std::filesystem::create_directories(dp, ec);
+ok = std::filesystem::is_directory(dp, ec);
+}
 
 if (!ok)
 	{

lite/fnrun.cpp

@@ -14,6 +14,7 @@ All rights reserved.
 
 
 #include "StdAfx.h"
+#include <filesystem>		// #323.
 #ifdef LINUX
 #ifdef _ODBC
 #include <sql.h>
@@ -17641,10 +17642,14 @@ ok = (_tsystem(cmd) ? false : true);
 #endif
 #endif
 
-_TCHAR cmd[2048];	// 09/28/08 AM.
-_stprintf(cmd,_T("mkdir %s"), str);	// 09/28/08 AN.
-run_silent(cmd);			// system() w/o DOC CMD PROMPT POPUP.	// 09/28/08 AM.
-ok = true;	// 09/28/08 AM.
+// Was a shell "mkdir STR": a path with spaces failed, one with & or ;
+// ran a command, and it always reported success (#323).
+{
+std::error_code ec;
+std::filesystem::path dp(str);
+std::filesystem::create_directories(dp, ec);
+ok = std::filesystem::is_directory(dp, ec);
+}
 
 if (!ok)
 	{

lite/io.cpp

@@ -11,6 +11,7 @@ All rights reserved.
 *******************************************************************************/
 
 #include "StdAfx.h"
+#include <filesystem>		// #323.
 #include "machine.h"		// 03/09/00 AM.
 #include "u_out.h"		// 01/19/06 AM.
 #include "prim/libprim.h"	// 01/18/06 AM.
@@ -91,9 +92,12 @@ for (;;)																			// 12/18/01 AM.
 bool file_exists(const _TCHAR *iname)
 {
 
-//ifstream inFile(iname, std::ios::in | std::ios::nocreate);
-std::_t_ifstream inFile(CTCHAR2CA(iname), std::ios::in);
-return (inFile ? true : false);
+// A file, not a directory: opening a directory for reading succeeds on
+// Linux and macOS, so a directory used to "exist" there (#323).
+if (!iname || !*iname)
+	return false;
+std::error_code ec;
+return std::filesystem::is_regular_file(std::filesystem::path(iname), ec);
 }
 
 

lite/mach.cpp

@@ -11,6 +11,7 @@ All rights reserved.
 *******************************************************************************/
 
 #include "StdAfx.h"
+#include <filesystem>		// #323.
 #include <time.h>
 #ifndef LINUX
 #include <direct.h>
@@ -265,25 +266,11 @@ void _tmain( void ){
 
 LITE_API bool dir_exists(_TCHAR *name)
 {
-_TCHAR cwd[MAXSTR];
-
-/* Get the current working directory: */
-#ifndef LINUX
-if(_tgetcwd( cwd, MAXSTR) == NULL)
-	{
-	std::_t_strstream gerrStr;
-	gerrStr << _T("[dir_exists: Couldn't get working dir.]") << std::ends;
-	errOut(&gerrStr,false);
-	return false;
-	}
-
-if(_tchdir(name))
+// Was a chdir() there and back, and on Linux and macOS always true (#323).
+if (!name || !*name)
 	return false;
-
-// Was able to change to it. Now change back.
-_tchdir(cwd);
-#endif
-return true;
+std::error_code ec;
+return std::filesystem::is_directory(std::filesystem::path(name), ec);
 }
 
 
@@ -297,11 +284,11 @@ return true;
 
 LITE_API bool make_dir(_TCHAR *name)
 {
-#ifndef LINUX
-if(_tmkdir(name) == 0)
-	return true;
-#endif
-return false;
+// On Linux and macOS this used to do nothing and return false (#323).
+if (!name || !*name)
+	return false;
+std::error_code ec;
+return std::filesystem::create_directory(std::filesystem::path(name), ec);
 }
 
 
@@ -315,11 +302,14 @@ return false;
 
 LITE_API bool rm_dir(_TCHAR *name)
 {
-#ifndef LINUX
-if(_trmdir(name) == 0)
-	return true;
-#endif
-return false;
+// On Linux and macOS this used to do nothing and return false (#323).
+if (!name || !*name)
+	return false;
+std::error_code ec;
+std::filesystem::path p(name);
+if (!std::filesystem::is_directory(p, ec))
+	return false;
+return std::filesystem::remove(p, ec);	// Only if empty.
 }
 
 
@@ -335,9 +325,8 @@ LITE_API bool remove_path(_TCHAR *name)
 {
 if (!safe_dir(name))
 	return false;
-if(_tremove(name) == 0)
-	return true;
-return false;
+std::error_code ec;
+return std::filesystem::remove(std::filesystem::path(name), ec);
 }
 
 /********************************************