Development Environment improvements

Taskfile.yml

@@ -47,19 +47,30 @@ tasks:
         msg: "Couldn't locate decompiler executable in '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler'"
     cmds:
       - '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/{{.DECOMP_CONFIG}}" "./iso_data" "./decompiler_out" --version "{{.DECOMP_CONFIG_VERSION}}" --config-override ''{"decompile_code": false, "levels_extract": true, "allowed_objects": []}'''
+  compile-game-if-needed:
+    internal: true
+    cmds:
+      - |
+        if ! test -f out/{{.GAME}}/iso/KERNEL.CGO; then
+          echo "Game is not compiled yet ('out/{{.GAME}}/iso/KERNEL.CGO' is missing) - running (mi) in 1 seconds..."
+          {{.PYTHON}} -c "import time; time.sleep(1)"
+          {{.GOALC_BIN_RELEASE_DIR}}/goalc{{.EXE_FILE_EXTENSION}} --user-auto --game {{.GAME}} --cmd "(mi)"
+        fi
   boot-game:
-    desc: "Boots the game, it will fail if it's not already compiled!"
+    desc: "Boots the game, compiling it first if it's not compiled yet"
     preconditions:
       - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
         msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
     cmds:
+      - task: compile-game-if-needed
       - "{{.GK_BIN_RELEASE_DIR}}/gk -v --game {{.GAME}} -- -boot -fakeiso -debug"
   boot-game-retail:
-    desc: "Boots the game without debug mode, it will fail if it's not already compiled!"
+    desc: "Boots the game without debug mode, compiling it first if it's not compiled yet"
     preconditions:
       - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
         msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
     cmds:
+      - task: compile-game-if-needed
       - "{{.GK_BIN_RELEASE_DIR}}/gk -v --game {{.GAME}} -- -boot -fakeiso"
   run-game:
     desc: "Start the game's runtime, to start the game itself the REPL is required"

common/util/FileUtil.cpp

@@ -278,7 +278,24 @@ fs::path get_iso_dir_for_game(GameVersion game_version) {
     expected_subdir = "jak3";
   }
   const auto temp_dir = get_jak_project_dir() / "iso_data" / expected_subdir;
-  if (fs::exists(temp_dir)) {
+  if (fs::exists(temp_dir / "DGO")) {
+    g_iso_data_directory = temp_dir;
+    return g_iso_data_directory;
+  }
+  // no DGO folder means no game data was ever extracted to the default iso_data folder -
+  // fall back to the JAK_ISO_DATA_DIR environment variable if it is set
+  const auto env_iso_data = get_env("JAK_ISO_DATA_DIR");
+  if (!env_iso_data.empty()) {
+    fs::path env_path = env_iso_data;
+    // the env var may point at the iso_data root (containing jak1/ etc.) or directly at the
+    // extracted game data
+    if (fs::exists(env_path / expected_subdir)) {
+      env_path /= expected_subdir;
+    }
+    lg::info("'{}' has no extracted game data, using JAK_ISO_DATA_DIR instead: {}",
+             temp_dir.string(), env_path.string());
+    g_iso_data_directory = env_path;
+  } else if (fs::exists(temp_dir)) {
     g_iso_data_directory = temp_dir;
   }
   return g_iso_data_directory;

decompiler/extractor/main.cpp

@@ -375,6 +375,20 @@ int main(int argc, char** argv) {
       iso_data_path = input_file_path;
     } else {
       iso_data_path = file_util::get_jak_project_dir() / "iso_data" / data_subfolder;
+      // no DGO folder means no game data was ever extracted here
+      if (!fs::exists(iso_data_path / "DGO")) {
+        const auto env_iso_data = get_env("JAK_ISO_DATA_DIR");
+        // see if we can use JAK_ISO_DATA_DIR
+        if (!env_iso_data.empty()) {
+          fs::path env_path = env_iso_data;
+          if (fs::exists(env_path / data_subfolder)) {
+            env_path /= data_subfolder;
+          }
+          lg::info("'{}' has no extracted game data, using JAK_ISO_DATA_DIR instead: {}",
+                   iso_data_path.string(), env_path.string());
+          iso_data_path = env_path;
+        }
+      }
     }
   }
 

decompiler/level_extractor/extract_level.cpp

@@ -448,7 +448,8 @@ void extract_all_levels(const ObjectFileDB& db,
                       game_version_names[config.game_version] / "entities";
   file_util::create_dir_if_needed(entities_dir);
 
-  int num_workers = dgo_names.size();
+  // cap the number of workers - one thread per level holds the whole uncompressed level in memory 
+  int num_workers = std::min((int)dgo_names.size(), (int)std::thread::hardware_concurrency());
   if (tex_db.replace_texture_dir) {
     num_workers = 1;
   }

decompiler/main.cpp

@@ -83,6 +83,19 @@ int main(int argc, char** argv) {
   }
 
   in_folder = in_folder / config.game_name;
+  // no DGO folder means no game data was ever extracted here - fall back to JAK_ISO_DATA_DIR if it is set
+  if (!fs::exists(in_folder / "DGO")) {
+    const auto env_iso_data = get_env("JAK_ISO_DATA_DIR");
+    if (!env_iso_data.empty()) {
+      fs::path env_path = env_iso_data;
+      if (fs::exists(env_path / config.game_name)) {
+        env_path /= config.game_name;
+      }
+      lg::info("'{}' has no extracted game data, using JAK_ISO_DATA_DIR instead: {}",
+               in_folder.string(), env_path.string());
+      in_folder = env_path;
+    }
+  }
   // Verify the in_folder is correct
   if (!exists(in_folder)) {
     lg::error("Aborting - 'in_folder' does not exist '{}'", in_folder.string());

goalc/compiler/compilation/CompilerControl.cpp

@@ -3,15 +3,19 @@
  * Compiler implementation for forms which actually control the compiler.
  */
 
+#include <chrono>
 #include <regex>
 #include <stack>
+#include <thread>
 
 #include "common/repl/repl_wrapper.h"
 #include "common/util/DgoWriter.h"
 #include "common/util/FileUtil.h"
 #include "common/util/Timer.h"
 #include "common/util/string_util.h"
 
+#include "decompiler/extractor/extractor_util.h"
+
 #include "goalc/compiler/Compiler.h"
 #include "goalc/compiler/IR.h"
 #include "goalc/compiler/docs/DocTypes.h"
@@ -834,7 +838,65 @@ Val* Compiler::compile_make(const goos::Object& form, const goos::Object& rest,
     report = get_true_or_false(form, args.get_named("report"));
   }
 
-  m_make.make(args.unnamed.at(0).as_string()->data, force, verbose, report);
+  const auto& target = args.unnamed.at(0).as_string()->data;
+
+  // if we're building the iso group but the game was never extracted, automatically run the
+  // extraction (the equivalent of `task extract`) once before building
+  if (target == "GROUP:iso") {
+    const auto game_name = version_to_game_name(m_version);
+    const auto decomp_out = file_util::get_jak_project_dir() / "decompiler_out" / game_name;
+    if (!fs::exists(decomp_out / "textures" / "tpage-dir.txt")) {
+      const auto iso_dir = file_util::get_iso_dir_for_game(m_version);
+      if (iso_dir.empty() || !fs::exists(iso_dir / "DGO")) {
+        lg::warn(
+            "The game has not been extracted yet and no ISO data was found - the build will "
+            "likely fail. Place the game data in iso_data/{} or set JAK_ISO_DATA_DIR.",
+            game_name);
+      } else {
+        lg::warn("The game has not been extracted yet ('{}' is missing).", decomp_out.string());
+        lg::warn("Automatically extracting from '{}' .", iso_dir.string());
+        // run the extraction in a separate decompiler process (the same thing `task extract`
+        // does) - it is memory heavy and can't be allowed to take down the compiler process
+        try {
+          const auto version_info = get_version_info_or_default(iso_dir);
+#ifdef _WIN32
+          const std::string decomp_exe_name = "decompiler.exe";
+#else
+          const std::string decomp_exe_name = "decompiler";
+#endif
+          const auto decompiler_path =
+              fs::path(file_util::get_current_executable_path()).parent_path() / decomp_exe_name;
+          if (!fs::exists(decompiler_path)) {
+            lg::error("Could not find the decompiler at '{}', attempting the build anyway...",
+                      decompiler_path.string());
+          } else {
+            const auto config_path = file_util::get_jak_project_dir() / "decompiler" / "config" /
+                                     game_name / fmt::format("{}_config.jsonc", game_name);
+            auto cmd = fmt::format(
+                "\"{}\" \"{}\" \"{}\" \"{}\" --version \"{}\" --config-override "
+                "\"{{\\\"decompile_code\\\": false, \\\"levels_extract\\\": true, "
+                "\\\"allowed_objects\\\": []}}\"",
+                decompiler_path.string(), config_path.string(),
+                (file_util::get_jak_project_dir() / "iso_data").string(),
+                (file_util::get_jak_project_dir() / "decompiler_out").string(),
+                version_info.decomp_config_version);
+#ifdef _WIN32
+            // cmd.exe strips the outer quotes of the whole command string, wrap it in an extra
+            // pair so the quoted executable path survives
+            cmd = fmt::format("\"{}\"", cmd);
+#endif
+            if (system(cmd.c_str()) != 0) {
+              lg::error("Extraction failed, attempting the build anyway...");
+            }
+          }
+        } catch (const std::exception& e) {
+          lg::error("Extraction failed ({}), attempting the build anyway...", e.what());
+        }
+      }
+    }
+  }
+
+  m_make.make(target, force, verbose, report);
   return get_none();
 }
 

goalc/main.cpp

@@ -103,7 +103,8 @@ int main(int argc, char** argv) {
   // if a command is provided on the command line, no REPL just run the compiler on it
   try {
     if (!cmd.empty()) {
-      compiler = std::make_unique<Compiler>(game_version, emitter::InstructionSet::X86);
+      compiler = std::make_unique<Compiler>(game_version, emitter::InstructionSet::X86,
+                                            std::make_optional(repl_config), username);
       compiler->run_front_end_on_string(cmd);
       return 0;
     }

goalc/make/MakeSystem.cpp

@@ -5,6 +5,8 @@
 #include "common/util/FileUtil.h"
 #include "common/util/Timer.h"
 #include "common/util/string_util.h"
+#include "common/util/unicode_util.h"
+#include "common/versions/versions.h"
 
 #include "goalc/make/CompilerReport.h"
 #include "goalc/make/Tools.h"
@@ -96,8 +98,30 @@ MakeSystem::MakeSystem(const std::optional<REPL::Config> repl_config, const std:
                  file_util::get_iso_dir_for_game(m_repl_config->game_version).string());
     set_constant("*use-iso-data-path*", true);
   } else {
-    set_constant("*iso-data*", file_util::get_file_path({"iso_data"}));
-    set_constant("*use-iso-data-path*", false);
+    bool use_env_iso_data = false;
+    if (m_repl_config) {
+      // no DGO folder means no game data was never extracted
+      // fall back to the JAK_ISO_DATA_DIR environment variable if it is set
+      const auto game_name = version_to_game_name(m_repl_config->game_version);
+      if (!fs::exists(file_util::get_jak_project_dir() / "iso_data" / game_name / "DGO")) {
+        const auto env_iso_data = get_env("JAK_ISO_DATA_DIR");
+        if (!env_iso_data.empty()) {
+          fs::path env_path = env_iso_data;
+          if (fs::exists(env_path / game_name)) {
+            env_path /= game_name;
+          }
+          lg::info("iso_data/{} has no extracted game data, using JAK_ISO_DATA_DIR instead: {}",
+                   game_name, env_path.string());
+          set_constant("*iso-data*", env_path.string());
+          set_constant("*use-iso-data-path*", true);
+          use_env_iso_data = true;
+        }
+      }
+    }
+    if (!use_env_iso_data) {
+      set_constant("*iso-data*", file_util::get_file_path({"iso_data"}));
+      set_constant("*use-iso-data-path*", false);
+    }
   }
 
   add_tool<DgoTool>();