From 8df61e6c4bf375c49473f5a1cc451c3171ef4d3d Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Sun, 8 Nov 2026 21:47:42 +0100 Subject: [PATCH 1/4] Build: prepare the shaders at the end of the build, not at the first start make_game.py now runs hookshot --prepare-shaders as its last step, so the minutes of shader preparation happen under the builder's progress bar instead of behind the game's black window at its first start (issue 711). The command has been in the game since PR 320, but nothing ran it. Its @@stage/@@progress lines go on to the builder; its own @@done and @@error go on as plain text, since both builders read those as the build's end or failure. A PC without usable Vulkan (exit 3) or any other failure leaves the work to the first start, as before. HOOKSHOT_NO_PREPARE=1 skips the step. The Windows builder gets a "Prepare the shaders" row. Co-Authored-By: Claude Opus 5.5 --- builder/HookshotBuilder/MainWindow.xaml.cs | 4 +- nx2/make_game.py | 24 +++++++++ tests/test_prepare_at_build.py | 58 ++++++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/test_prepare_at_build.py diff --git a/builder/HookshotBuilder/MainWindow.xaml.cs b/builder/HookshotBuilder/MainWindow.xaml.cs index 5dfc532d..76a5498f 100644 --- a/builder/HookshotBuilder/MainWindow.xaml.cs +++ b/builder/HookshotBuilder/MainWindow.xaml.cs @@ -719,10 +719,12 @@ static readonly (string Id, string Build, string Update)[] Stages = ("lift", "Translate the game to PC code", "Translate the game to PC code"), ("compile", "Compile", "Compile changed files"), ("finish", "Put the game together", "Swap in the new game"), + ("shaders", "Prepare the shaders", "Prepare the shaders"), }; static readonly Dictionary Weight = new() { - ["download"] = (0, 0.04), ["read"] = (0.04, 0.04), ["lift"] = (0.08, 0.14), ["compile"] = (0.22, 0.74), ["finish"] = (0.96, 0.04), + ["download"] = (0, 0.04), ["read"] = (0.04, 0.04), ["lift"] = (0.08, 0.14), ["compile"] = (0.22, 0.68), ["finish"] = (0.90, 0.02), + ["shaders"] = (0.92, 0.08), // (hookshot --prepare-shaders: minutes on some GPUs, the first start's wait moved here) }; void MakeRows(bool update) diff --git a/nx2/make_game.py b/nx2/make_game.py index ec733289..d468fe35 100644 --- a/nx2/make_game.py +++ b/nx2/make_game.py @@ -755,6 +755,28 @@ def ship_state_list(out): print("the shipped pipeline state list: %d pipelines" % ((os.path.getsize(src) - 12) // 528), flush=True) +def prepare_shaders(cmd): + """The shaders prepared now, under the builder's progress bar, not at the game's first start behind a black window (minutes on + some GPUs, issue 711): hookshot --prepare-shaders (host/hos_main.c, gpu/vk/vk_main.cpp), the same work as that first start. + Its @@stage and @@progress lines go on to the builder; its own @@done and @@error would end or fail the build there, so they + go on as plain text. A PC whose Vulkan can't (exit 3), or any other failure, leaves the work to the first start, as before""" + try: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", + bufsize=1) + except OSError as e: + print("the shaders were not prepared now (%s): the game prepares them at its first start" % e, flush=True) + return 1 + no_throttle(getattr(p, "_handle", None)) + for line in p.stdout: + line = line.rstrip() + print(line if line.startswith(("@@stage ", "@@progress ")) or not line.startswith("@@") else line[2:], flush=True) + p.stdout.close() + code = p.wait() + if code: + print("the shaders were not prepared now (exit %d): the game prepares them at its first start" % code, flush=True) + return code + + def folder_language(out): """the language a game folder was built in: the code line of its language.txt""" try: @@ -1236,6 +1258,8 @@ def compiled(line): for sub in ("sa", "objcache", "probe"): shutil.rmtree(os.path.join(work, sub), ignore_errors=True) print("the build files in %s were deleted (HOOKSHOT_KEEP_BUILD_FILES=1 keeps them for quicker updates)" % work, flush=True) + if not os.environ.get("HOOKSHOT_NO_PREPARE"): # (=1: left to the first start, as before) + prepare_shaders([os.path.join(game_dir, BESIDE[0]), "--prepare-shaders"]) total_seconds = time.time() - STARTED previous = record_times(work, run_files, compile_seconds, total_seconds, modules=plan[1], kind="update" if a.update else "build") diff --git a/tests/test_prepare_at_build.py b/tests/test_prepare_at_build.py new file mode 100644 index 00000000..fe4eb232 --- /dev/null +++ b/tests/test_prepare_at_build.py @@ -0,0 +1,58 @@ +"""make_game.py's last step, hookshot --prepare-shaders, relayed to the builder: its stage and progress lines go on, its own +@@done and @@error don't (the builders read those as the build's end or failure), and a failure leaves the work to the first start. + +python -I -B tests/test_prepare_at_build.py +""" +import contextlib +import importlib.util +import io +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "nx2" / "make_game.py" +spec = importlib.util.spec_from_file_location("make_game_prepare", SOURCE) +make_game = importlib.util.module_from_spec(spec) +spec.loader.exec_module(make_game) + + +def relayed(child): + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = make_game.prepare_shaders([sys.executable, "-c", child]) + return code, out.getvalue().splitlines() + + +class PrepareAtBuildTests(unittest.TestCase): + def test_progress_goes_on_and_done_does_not(self): + code, lines = relayed("print('hookshot v'); print('@@stage shaders Preparing shaders'); print('@@progress 0.500');" + "print('@@done shaders')") + self.assertEqual(code, 0) + self.assertEqual(lines, ["hookshot v", "@@stage shaders Preparing shaders", "@@progress 0.500", "done shaders"]) + + def test_no_vulkan_is_not_a_failed_build(self): + code, lines = relayed("import sys; print('@@error no Vulkan device'); sys.exit(3)") + self.assertEqual(code, 3) + self.assertFalse(any(l.startswith("@@") for l in lines)) + self.assertIn("error no Vulkan device", lines) + self.assertTrue(lines[-1].startswith("the shaders were not prepared now (exit 3)")) + + def test_missing_game_is_not_a_failed_build(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = make_game.prepare_shaders([str(ROOT / "no-such-folder" / "hookshot.exe"), "--prepare-shaders"]) + self.assertEqual(code, 1) + self.assertNotIn("@@", out.getvalue()) + + def test_the_build_runs_it_after_the_game_folder_is_complete(self): + text = SOURCE.read_text(encoding="utf-8") + main = text[text.index("\ndef main("):] + at = main.index("prepare_shaders([os.path.join(game_dir, BESIDE[0]), \"--prepare-shaders\"])") + for before in ("os.replace(dst + \".new\", dst)", "shader_census_cmd(", "seed_pipelines(out)"): + self.assertLess(main.index(before), at, before) + self.assertLess(at, main.index("say(\"done\", out)")) + + +if __name__ == "__main__": + unittest.main() From 126449c0371a7d99d2075a55eb9b7e333ad65390 Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Mon, 9 Nov 2026 03:22:54 +0100 Subject: [PATCH 2/4] Builders: a "Prepare shaders now" tick, on by default Unticked, the builders pass make_game.py --no-prepare-shaders and the game prepares its shaders at its first start, as before. The choice is kept (builder-settings.json on Windows, builder.ini on Linux) and used for updates too. Co-Authored-By: Claude Opus 5.5 --- builder/HookshotBuilder/MainWindow.xaml | 3 +++ builder/HookshotBuilder/MainWindow.xaml.cs | 10 ++++++++-- builder/linux/hookshot_builder.py | 12 +++++++++--- nx2/make_game.py | 4 +++- tests/test_prepare_at_build.py | 10 ++++++++++ 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/builder/HookshotBuilder/MainWindow.xaml b/builder/HookshotBuilder/MainWindow.xaml index 3d281e96..f1ea91fb 100644 --- a/builder/HookshotBuilder/MainWindow.xaml +++ b/builder/HookshotBuilder/MainWindow.xaml @@ -171,6 +171,9 @@ + + diff --git a/builder/HookshotBuilder/MainWindow.xaml.cs b/builder/HookshotBuilder/MainWindow.xaml.cs index 76a5498f..708086b1 100644 --- a/builder/HookshotBuilder/MainWindow.xaml.cs +++ b/builder/HookshotBuilder/MainWindow.xaml.cs @@ -260,6 +260,7 @@ sealed class BuilderSettings public string? Romfs { get; set; } public string? Exefs { get; set; } public bool Dlss { get; set; } // (NVIDIA DLSS ticked: NVIDIA's license was accepted) + public bool PrepareLater { get; set; } // ("Prepare shaders now" unticked) } static string SettingsPath => System.IO.Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "HookshotBuilder", "builder-settings.json"); @@ -269,6 +270,7 @@ void ApplySettings() BuilderSettings? s = null; try { s = System.Text.Json.JsonSerializer.Deserialize(File.ReadAllText(SettingsPath)); } catch (Exception) { } if (s == null) return; + PrepareTick.IsChecked = !s.PrepareLater; if (!string.IsNullOrEmpty(s.Out)) OutPath.Text = s.Out; DlssTick.IsChecked = s.Dlss; if (s.Romfs != null && s.Exefs != null && Directory.Exists(s.Romfs) && File.Exists(System.IO.Path.Combine(s.Exefs, "main"))) @@ -293,7 +295,8 @@ void SaveSettings() try { var s = new BuilderSettings { Out = OutPath.Text, Nup = romfs == null ? chosenNup : null, Romfs = romfs, Exefs = exefs, - Dlss = DlssTick.IsChecked == true }; + Dlss = DlssTick.IsChecked == true, + PrepareLater = PrepareTick.IsChecked != true }; Directory.CreateDirectory(System.IO.Path.GetDirectoryName(SettingsPath)!); File.WriteAllText(SettingsPath, System.Text.Json.JsonSerializer.Serialize(s)); } @@ -692,6 +695,7 @@ async void StartBuild(object sender, RoutedEventArgs e) else if (chosenNup != null) args.AddRange(new[] { "--nup", chosenNup }); else return; args.AddRange(new[] { "--out", OutPath.Text, "--language", lang.Code }); + if (PrepareTick.IsChecked != true) args.Add("--no-prepare-shaders"); Run(tools, args, dlss: DlssTick.IsChecked == true); } @@ -1175,7 +1179,9 @@ internal void UpdateBuild(Tools tools, string folder) // (a folder built with NVIDIA DLSS, its license accepted then, is built with it again) var dlss = File.Exists(System.IO.Path.Combine(folder, "ngx", "nvngx_dlss.dll")) && File.Exists(System.IO.Path.Combine(Storage.NgxSdk, "LICENSE.txt")); - Run(tools, new List { "--update", folder }, keepRows: true, dlss: dlss); + var args = new List { "--update", folder }; + if (PrepareTick.IsChecked != true) args.Add("--no-prepare-shaders"); // (as the last build chose: ApplySettings) + Run(tools, args, keepRows: true, dlss: dlss); } // ================= the window ================= diff --git a/builder/linux/hookshot_builder.py b/builder/linux/hookshot_builder.py index a4d235a7..2622ada6 100644 --- a/builder/linux/hookshot_builder.py +++ b/builder/linux/hookshot_builder.py @@ -178,6 +178,7 @@ def __init__(self, launch=None, update=None, fetched=False, tag=None): names = [n for _, n in make_game.LANGUAGES] code = self.cfg.get("language", "en-US") self.language = tk.StringVar(value=dict(make_game.LANGUAGES).get(code, names[0])) + self.prepare = tk.BooleanVar(value=self.cfg.get("prepare", "1") != "0") # (make_game.py's last step: off, at the first start) self.check_text = tk.StringVar() self.space_text = tk.StringVar() self.stage_text = tk.StringVar() @@ -245,6 +246,9 @@ def show_form(self): ttk.Combobox(f, textvariable=self.language, values=[n for _, n in make_game.LANGUAGES], state="readonly").grid( row=row, column=1, sticky="w", **self.pad) row += 1 + ttk.Checkbutton(f, text="Prepare shaders now (unticked: at the game's first start, behind a black window)", + variable=self.prepare).grid(row=row, column=0, columnspan=3, sticky="w", **self.pad) + row += 1 ttk.Label(f, text="Takes about 45 minutes and needs 60 GB.").grid(row=row, column=0, columnspan=3, sticky="w", **self.pad) row += 1 ttk.Label(f, textvariable=self.space_text).grid(row=row, column=0, columnspan=3, sticky="w", **self.pad) @@ -414,7 +418,8 @@ def start_make_update(self, folder): self.status.set("This builder is the latest version.") self.lookup_latest() return - cmd = [sys.executable, "-I", os.path.join(ROOT, "nx2", "make_game.py"), "--update", folder] + cmd = [sys.executable, "-I", os.path.join(ROOT, "nx2", "make_game.py"), "--update", folder] + \ + ([] if self.prepare.get() else ["--no-prepare-shaders"]) # (as the last build chose) self.heading.configure(text="Updating your game") self.stage_text.set("Compiling what changed") self.started = time.time() @@ -458,12 +463,13 @@ def build(self): except ValueError as error: return messagebox.showerror("Hookshot", str(error)) code = next((c for c, n in make_game.LANGUAGES if n == self.language.get()), "en-US") - self.cfg.update(kind=self.kind.get(), nup=self.nup.get(), romfs=self.romfs.get(), exefs=self.exefs.get(), out=out, language=code) + self.cfg.update(kind=self.kind.get(), nup=self.nup.get(), romfs=self.romfs.get(), exefs=self.exefs.get(), out=out, language=code, + prepare="1" if self.prepare.get() else "0") save_config(self.cfg) # (issue 531) Without HOOKSHOT_JOBS or RX_JOBS, make_game chooses as on Windows, from the memory free now; this window chose # fewer, 80% of the total memory at 3.5 GB each, and the CPU sat at 29% cmd = [sys.executable, "-I", os.path.join(ROOT, "nx2", "make_game.py")] + src + ["--out", out, "--language", code] + \ - ([] if jobs is None else ["--jobs", str(jobs)]) + ([] if jobs is None else ["--jobs", str(jobs)]) + ([] if self.prepare.get() else ["--no-prepare-shaders"]) self.show_progress() self.started = time.time() self.proc = subprocess.Popen(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, diff --git a/nx2/make_game.py b/nx2/make_game.py index d468fe35..8754be56 100644 --- a/nx2/make_game.py +++ b/nx2/make_game.py @@ -958,6 +958,8 @@ def parse_args(): p.add_argument("--language", default="en-US", help="the game's language: a code from LANGUAGES, written to language.txt") p.add_argument("--check", action="store_true") p.add_argument("--update", help="a game folder built before: built again from the game files it keeps (the game's update offer)") + p.add_argument("--no-prepare-shaders", action="store_true", + help="leave the shaders' preparation to the game's first start (the builder's \"Prepare shaders now\" unticked)") p.add_argument("--log", metavar="FILE", help="also save build output to a UTF-8 file (replaces the file if it exists)") # (NVIDIA DLSS, off unless asked: NVIDIA's SDK into HOOKSHOT_NGX_SDK first, under NVIDIA's license. scripts/ngx_sdk.py) p.add_argument("--fetch-ngx-sdk", action="store_true", help="download NVIDIA's DLSS SDK into HOOKSHOT_NGX_SDK before the build") @@ -1258,7 +1260,7 @@ def compiled(line): for sub in ("sa", "objcache", "probe"): shutil.rmtree(os.path.join(work, sub), ignore_errors=True) print("the build files in %s were deleted (HOOKSHOT_KEEP_BUILD_FILES=1 keeps them for quicker updates)" % work, flush=True) - if not os.environ.get("HOOKSHOT_NO_PREPARE"): # (=1: left to the first start, as before) + if not a.no_prepare_shaders and not os.environ.get("HOOKSHOT_NO_PREPARE"): # (either: left to the first start, as before) prepare_shaders([os.path.join(game_dir, BESIDE[0]), "--prepare-shaders"]) total_seconds = time.time() - STARTED previous = record_times(work, run_files, compile_seconds, total_seconds, modules=plan[1], diff --git a/tests/test_prepare_at_build.py b/tests/test_prepare_at_build.py index fe4eb232..69d6797b 100644 --- a/tests/test_prepare_at_build.py +++ b/tests/test_prepare_at_build.py @@ -53,6 +53,16 @@ def test_the_build_runs_it_after_the_game_folder_is_complete(self): self.assertLess(main.index(before), at, before) self.assertLess(at, main.index("say(\"done\", out)")) + def test_the_builders_can_leave_it_to_the_first_start(self): + text = SOURCE.read_text(encoding="utf-8") + self.assertIn('"--no-prepare-shaders"', text) + main = text[text.index("\ndef main("):] + guard = main[:main.index("prepare_shaders([os.path.join(game_dir")].rsplit("\n", 2)[-2] + self.assertIn("a.no_prepare_shaders", guard) + self.assertIn('HOOKSHOT_NO_PREPARE', guard) + for builder in ("builder/HookshotBuilder/MainWindow.xaml.cs", "builder/linux/hookshot_builder.py"): + self.assertEqual((ROOT / builder).read_text(encoding="utf-8").count("--no-prepare-shaders"), 2, builder) # (build and update) + if __name__ == "__main__": unittest.main() From c6deacd22c73aa8f73b97c8f16db9649c3ebc7de Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Mon, 9 Nov 2026 07:53:37 +0100 Subject: [PATCH 3/4] tests: the Linux builder's "Prepare shaders now" tick in the build-jobs harness The harness's window now has the tick (ticked, the default), the saved settings carry prepare=1, and a new case checks that unticked it passes --no-prepare-shaders and saves prepare=0. Co-Authored-By: Claude Opus 5.5 --- tests/test_linux_builder_jobs.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_linux_builder_jobs.py b/tests/test_linux_builder_jobs.py index 024499a7..c898f243 100644 --- a/tests/test_linux_builder_jobs.py +++ b/tests/test_linux_builder_jobs.py @@ -42,7 +42,7 @@ def __init__(self, folder, environ, kind="nup"): exefs.mkdir(exist_ok=True) self.gui = SimpleNamespace( out=value(str(folder / "out")), kind=value(kind), nup=value(str(nup)), - romfs=value(str(romfs)), exefs=value(str(exefs)), language=value("English (US)"), + romfs=value(str(romfs)), exefs=value(str(exefs)), language=value("English (US)"), prepare=value(True), cfg={"previous": "kept"}, show_progress=lambda: self.events.append(("progress",)), read=lambda proc: None) def start_process(command, **kwargs): @@ -90,7 +90,8 @@ def assert_started(self, harness, jobs=None): self.assertEqual(options, dict(cwd=str(ROOT), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, encoding="utf-8", errors="replace", start_new_session=True)) self.assertEqual(harness.events[0][1], dict(previous="kept", kind=gui.kind.get(), nup=gui.nup.get(), - romfs=gui.romfs.get(), exefs=gui.exefs.get(), out=gui.out.get(), language="en-US")) + romfs=gui.romfs.get(), exefs=gui.exefs.get(), out=gui.out.get(), language="en-US", + prepare="1")) self.assertEqual(harness.events[3][1], dict(target=gui.read, args=(gui.proc,), daemon=True)) def test_positive_hookshot_jobs_is_passed_on(self): @@ -99,6 +100,14 @@ def test_positive_hookshot_jobs_is_passed_on(self): self.assert_started(harness, 4) self.assertEqual(harness.environ, {"HOOKSHOT_JOBS": "4", "RX_JOBS": "2"}) + def test_unticked_prepare_leaves_the_shaders_to_the_first_start(self): + harness = BuildHarness(self.folder, {}) + harness.gui.prepare = value(False) + harness.run() + command, _ = harness.processes[0] + self.assertEqual(command[-1], "--no-prepare-shaders") + self.assertEqual(harness.events[0][1]["prepare"], "0") + def test_zero_falls_back_to_rx_jobs_for_extracted_folders(self): harness = BuildHarness(self.folder, {"HOOKSHOT_JOBS": "0", "RX_JOBS": "2"}, kind="folders") harness.run() From d6594bf9359ecd3542ce7900eed5582f679e76c2 Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Mon, 9 Nov 2026 11:22:25 +0100 Subject: [PATCH 4/4] Build: the shaders' step after review: saved choice on updates, honest row, watchdog, the game's own environment - Windows builder: the update flows (the launcher's Update, the --update start after a self-update) read the saved PrepareLater setting (BuilderSettings.PrepareNow) instead of the tick, which ApplySettings never set there. - The "Prepare the shaders" row only when they are prepared; it ends skipped, with the reason under it, when no @@stage shaders came or make_game printed its "the shaders were not prepared now ()" line (a plain line: older builders only show it). The Linux builder notes it on its ready line. - make_game: run() gains show (a line filter), cwd, base (environment) and silence (a reader thread; a step quiet that long is killed). prepare_shaders uses it, in the game folder and game_env() (the Play button's environment, now shared with the Linux builder), and stops a run silent for 15 minutes: left to the first start, never a failed build. - HOOKSHOT_NO_PREPARE skips unless "0", as the game reads its switches; RX_TARGET=windows on Linux (CROSS) skips. - The build times are recorded and said before the preparation (build_times.json stays comparable). - Linux builder: a new @@stage clears the compile's clock and countdown. - Tests: prepare_shaders against stub games (relay, exit 3, crash, silent hang, missing exe, environment and folder), run()'s silence and filter, the skip decision, the Linux builder's stage handling; the Windows builder's settings and row decisions in the builder_workers harness. Co-Authored-By: Claude Opus 5.5 --- builder/HookshotBuilder/BuilderSettings.cs | 22 ++ builder/HookshotBuilder/MainWindow.xaml.cs | 85 +++++--- builder/HookshotBuilder/ShaderStep.cs | 30 +++ builder/linux/hookshot_builder.py | 15 +- nx2/make_game.py | 148 +++++++++++--- .../BuilderWorkersTests.csproj | 2 + tests/builder_workers/Program.cs | 30 +++ tests/test_prepare_at_build.py | 193 +++++++++++++++--- 8 files changed, 434 insertions(+), 91 deletions(-) create mode 100644 builder/HookshotBuilder/BuilderSettings.cs create mode 100644 builder/HookshotBuilder/ShaderStep.cs diff --git a/builder/HookshotBuilder/BuilderSettings.cs b/builder/HookshotBuilder/BuilderSettings.cs new file mode 100644 index 00000000..0b7da667 --- /dev/null +++ b/builder/HookshotBuilder/BuilderSettings.cs @@ -0,0 +1,22 @@ +namespace HookshotBuilder; + +/// What the player chose last time (the install folder, the game files, the shaders' tick): builder-settings.json beside the versions +sealed class BuilderSettings +{ + public string? Out { get; set; } + public string? Nup { get; set; } + public string? Romfs { get; set; } + public string? Exefs { get; set; } + public bool Dlss { get; set; } // (NVIDIA DLSS ticked: NVIDIA's license was accepted) + public bool PrepareLater { get; set; } // ("Prepare shaders now" unticked) + + public static BuilderSettings? Load(string path) + { + try { return System.Text.Json.JsonSerializer.Deserialize(System.IO.File.ReadAllText(path)); } + catch (Exception) { return null; } + } + + /// (the updates, from the launcher's Update and from the --update start after a self-update, begin before ApplySettings sets + /// the tick from these settings: they read the saved choice itself. None saved: prepared, as the tick's default) + public static bool PrepareNow(string path) => Load(path)?.PrepareLater != true; +} diff --git a/builder/HookshotBuilder/MainWindow.xaml.cs b/builder/HookshotBuilder/MainWindow.xaml.cs index 708086b1..7afcd26f 100644 --- a/builder/HookshotBuilder/MainWindow.xaml.cs +++ b/builder/HookshotBuilder/MainWindow.xaml.cs @@ -252,23 +252,13 @@ void LaunchPlayClick(object sender, RoutedEventArgs e) void LaunchFolderClick(object sender, RoutedEventArgs e) => Process.Start(new ProcessStartInfo(launchFolder) { UseShellExecute = true }); - // ---- what the player chose last time (the install folder, the game files): builder-settings.json beside the versions ---- - sealed class BuilderSettings - { - public string? Out { get; set; } - public string? Nup { get; set; } - public string? Romfs { get; set; } - public string? Exefs { get; set; } - public bool Dlss { get; set; } // (NVIDIA DLSS ticked: NVIDIA's license was accepted) - public bool PrepareLater { get; set; } // ("Prepare shaders now" unticked) - } + // ---- what the player chose last time (BuilderSettings.cs): builder-settings.json beside the versions ---- static string SettingsPath => System.IO.Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "HookshotBuilder", "builder-settings.json"); void ApplySettings() { - BuilderSettings? s = null; - try { s = System.Text.Json.JsonSerializer.Deserialize(File.ReadAllText(SettingsPath)); } catch (Exception) { } + var s = BuilderSettings.Load(SettingsPath); if (s == null) return; PrepareTick.IsChecked = !s.PrepareLater; if (!string.IsNullOrEmpty(s.Out)) OutPath.Text = s.Out; @@ -309,7 +299,7 @@ void Preview(string what) switch (what) { case "build": - MakeRows(update: false); + MakeRows(update: false, shaders: true); SetStep(1); ShowPage(BuildPage); rows["read"].Begin(); rows["read"].Done(); rows["read"].Small("Version checked"); @@ -695,8 +685,8 @@ async void StartBuild(object sender, RoutedEventArgs e) else if (chosenNup != null) args.AddRange(new[] { "--nup", chosenNup }); else return; args.AddRange(new[] { "--out", OutPath.Text, "--language", lang.Code }); - if (PrepareTick.IsChecked != true) args.Add("--no-prepare-shaders"); - Run(tools, args, dlss: DlssTick.IsChecked == true); + prepareNow = PrepareTick.IsChecked == true; + Run(tools, ShaderStep.Args(args, prepareNow), dlss: DlssTick.IsChecked == true); } const string DlssLicense = @@ -723,19 +713,30 @@ static readonly (string Id, string Build, string Update)[] Stages = ("lift", "Translate the game to PC code", "Translate the game to PC code"), ("compile", "Compile", "Compile changed files"), ("finish", "Put the game together", "Swap in the new game"), - ("shaders", "Prepare the shaders", "Prepare the shaders"), + (ShaderStep.Id, "Prepare the shaders", "Prepare the shaders"), }; - static readonly Dictionary Weight = new() + static readonly Dictionary WithShaders = new() { ["download"] = (0, 0.04), ["read"] = (0.04, 0.04), ["lift"] = (0.08, 0.14), ["compile"] = (0.22, 0.68), ["finish"] = (0.90, 0.02), - ["shaders"] = (0.92, 0.08), // (hookshot --prepare-shaders: minutes on some GPUs, the first start's wait moved here) + [ShaderStep.Id] = (0.92, 0.08), // (hookshot --prepare-shaders: minutes on some GPUs, the first start's wait moved here) }; + static readonly Dictionary WithoutShaders = new() + { + ["download"] = (0, 0.04), ["read"] = (0.04, 0.04), ["lift"] = (0.08, 0.14), ["compile"] = (0.22, 0.74), ["finish"] = (0.96, 0.04), + }; + Dictionary Weight = WithoutShaders; + // (this build prepares the shaders: the tick for a build, the saved choice for an update. The game's @@stage shaders came, and + // make_game's line when it left them to the first start: the row's end says which) + bool prepareNow, shadersBegan; + string? shadersLater; + TimeSpan? builtIn; // (at @@times: the build without the shaders' minutes, as build_times.json keeps it) - void MakeRows(bool update) + void MakeRows(bool update, bool shaders) { StageList.Children.Clear(); rows.Clear(); - var list = Stages.Where(s => update || s.Id != "download").ToList(); + Weight = shaders ? WithShaders : WithoutShaders; + var list = Stages.Where(s => (update || s.Id != "download") && (shaders || s.Id != ShaderStep.Id)).ToList(); for (var i = 0; i < list.Count; i++) { var row = new StepRow(i + 1, update ? list[i].Update : list[i].Build, last: i == list.Count - 1); @@ -758,7 +759,7 @@ void Run(Tools tools, List args, bool keepRows = false, bool dlss = fals if (!keepRows) { updating = false; - MakeRows(update: false); + MakeRows(update: false, shaders: prepareNow); SetStep(1); BuildTitle.Text = "Building your game"; UpdateBar.Visibility = Visibility.Collapsed; @@ -775,6 +776,9 @@ void Run(Tools tools, List args, bool keepRows = false, bool dlss = fals compileEta = -1; thisCompile = 0; previousTotal = -1; + shadersBegan = false; + shadersLater = null; + builtIn = null; ShowPage(BuildPage); CancelButton.Visibility = Visibility.Visible; SetProgress(updating ? Weight["read"].From : 0); @@ -801,6 +805,7 @@ void Line(string line) current = id; if (rows.TryGetValue(id, out var row)) row.Begin(); if (id == "compile") compileClock.Restart(); + if (id == ShaderStep.Id) shadersBegan = true; if (Weight.TryGetValue(id, out var w)) SetProgress(w.From); } else if (line.StartsWith("@@progress ")) @@ -819,6 +824,7 @@ void Line(string line) return; } else if (line.StartsWith("@@done ")) finished = line[7..]; + else if (ShaderStep.Reason(line) is { } later) shadersLater = later; else if (FilesLine.Match(line) is { Success: true } m) filesOf = int.Parse(m.Groups[1].Value); else if (FileLine.IsMatch(line)) filesDone++; if (current == "compile" && filesChanged < 0 && filesOf > 0 && rows.TryGetValue("compile", out var c)) @@ -849,6 +855,7 @@ void Number(string line) compileEta = eta; break; case "@@times" when p.Length == 4: + builtIn = elapsed.Elapsed; long.TryParse(p[2], out thisCompile); long.TryParse(p[3], out previousTotal); break; @@ -922,7 +929,12 @@ void Ended(string? failure) runner = null; if (finished.Length > 0) { - foreach (var r in rows.Values) r.Done(); + foreach (var (id, r) in rows) + { + if (id != ShaderStep.Id) r.Done(); + else if (ShaderStep.Skipped(shadersBegan, shadersLater) is { } why) { r.Skipped(); r.Small(why); } + else r.Done(); + } SetProgress(1); ShowReady(); return; @@ -963,7 +975,7 @@ void ShowReady() SetStep(2); ReadyTitle.Text = updating ? "Your game is updated" : "Your game is ready"; var compiledNote = thisCompile > 0 ? $" (compiled in {Span(TimeSpan.FromSeconds(thisCompile))})" : ""; - ReadyNote.Text = $"{(updating ? "Updated" : "Built")} in {Span(elapsed.Elapsed)}{compiledNote}{Against(elapsed.Elapsed)} · {finished}"; + ReadyNote.Text = $"{(updating ? "Updated" : "Built")} in {Span(elapsed.Elapsed)}{compiledNote}{Against(builtIn ?? elapsed.Elapsed)} · {finished}"; if (!updating && Environment.GetEnvironmentVariable("HOOKSHOT_UI_PREVIEW") == null) Finder.Shortcuts(finished, DesktopTick.IsChecked == true, StartTick.IsChecked == true); else if (updating && Environment.GetEnvironmentVariable("HOOKSHOT_UI_PREVIEW") == null) @@ -1138,7 +1150,8 @@ public void BeginUpdate(string folder, string? tag) StepPill.Visibility = Visibility.Collapsed; VersionChip.Visibility = Visibility.Visible; VersionChipText.Text = from != null && from != to ? $"{from} → {to}" : to; - MakeRows(update: true); + prepareNow = BuilderSettings.PrepareNow(SettingsPath); + MakeRows(update: true, shaders: prepareNow); BuildTitle.Text = "Updating your game"; BuildNote.Text = "Getting the update…"; RingBox.Visibility = Visibility.Collapsed; @@ -1179,9 +1192,9 @@ internal void UpdateBuild(Tools tools, string folder) // (a folder built with NVIDIA DLSS, its license accepted then, is built with it again) var dlss = File.Exists(System.IO.Path.Combine(folder, "ngx", "nvngx_dlss.dll")) && File.Exists(System.IO.Path.Combine(Storage.NgxSdk, "LICENSE.txt")); - var args = new List { "--update", folder }; - if (PrepareTick.IsChecked != true) args.Add("--no-prepare-shaders"); // (as the last build chose: ApplySettings) - Run(tools, args, keepRows: true, dlss: dlss); + // (prepareNow: the saved choice, read in BeginUpdate. The launcher's Update and the --update start after a self-update never + // ran ApplySettings, so the tick there is only its default) + Run(tools, ShaderStep.Args(new List { "--update", folder }, prepareNow), keepRows: true, dlss: dlss); } // ================= the window ================= @@ -1296,6 +1309,24 @@ public void Done() time.Foreground = B("InkFaint"); } + /// (not done: left to later, the shaders' step when they were not prepared now) + public void Skipped() + { + if (running) Tick(); + running = false; + watch.Stop(); + disc.Fill = Brushes.Transparent; + disc.Stroke = B("GlassLineStrong"); + disc.StrokeThickness = 1.5; + halo.Fill = Brushes.Transparent; + number.Text = "–"; + number.Foreground = B("InkFaint"); + tick.Visibility = Visibility.Collapsed; + name.Foreground = B("InkFaint"); + name.FontWeight = FontWeights.Medium; + time.Foreground = B("InkFaint"); + } + public void Stopped() { running = false; diff --git a/builder/HookshotBuilder/ShaderStep.cs b/builder/HookshotBuilder/ShaderStep.cs new file mode 100644 index 00000000..e906f006 --- /dev/null +++ b/builder/HookshotBuilder/ShaderStep.cs @@ -0,0 +1,30 @@ +using System.Text.RegularExpressions; + +namespace HookshotBuilder; + +/// The build's last step, the shaders' preparation (nx2/make_game.py prepare_shaders): the argument that leaves it to the game's +/// first start, and what its row says once the build is done +static class ShaderStep +{ + public const string Id = "shaders"; + public const string Later = "--no-prepare-shaders"; + // (make_game.py NOT_PREPARED: a plain line, so a builder that does not know it only shows it) + static readonly Regex NotPrepared = new(@"^the shaders were not prepared now \((.*)\): "); + + /// make_game's arguments with the player's choice (unticked: the game prepares its shaders at its first start) + public static List Args(List args, bool prepare) + { + if (!prepare) args.Add(Later); + return args; + } + + /// Why make_game left the shaders to the first start, when this line says so; else null + public static string? Reason(string line) => NotPrepared.Match(line) is { Success: true } m ? m.Groups[1].Value : null; + + /// The line under the row of a finished build when the shaders were not prepared, or null when they were (a check mark). + /// began: the game's @@stage shaders came; reason: make_game's not-prepared line + public static string? Skipped(bool began, string? reason) => + reason != null ? $"Left to the game's first start: {reason}" + : began ? null + : "Left to the game's first start"; +} diff --git a/builder/linux/hookshot_builder.py b/builder/linux/hookshot_builder.py index 2622ada6..f70cb5ac 100644 --- a/builder/linux/hookshot_builder.py +++ b/builder/linux/hookshot_builder.py @@ -30,7 +30,7 @@ CONFIG = os.path.join(os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser("~/.config"), "hookshot", "builder.ini") STAGES = {"read": "Reading your game files", "lift": "Translating the game to PC code", "compile": "Compiling", - "finish": "Putting the game folder together"} + "finish": "Putting the game folder together", "shaders": "Preparing shaders"} def left_text(seconds): @@ -70,8 +70,7 @@ def play(folder): exe = linux_update.game_exe(folder) if not exe: return - env = {k: v for k, v in os.environ.items() if not k.startswith(("HOOKSHOT_", "PYTHON", "APPDIR", "APPIMAGE", "ARGV0", "OWD"))} - env.pop("LD_LIBRARY_PATH", None) + env = make_game.game_env() # (as make_game's shader preparation and hookshot-launch: the game's own SDL, the system's libraries) env["HOOKSHOT_NO_UPDATE_CHECK"] = "1" subprocess.Popen([exe], cwd=folder, env=env, start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) @@ -488,6 +487,7 @@ def show_progress(self, title="Building"): self.eta = None self.compile_started = None self.times = None + self.shaders_later = False f = self.page = ttk.Frame(self) f.pack(fill="both", expand=True, pady=self.px(8)) f.columnconfigure(0, weight=1) @@ -553,8 +553,9 @@ def on_line(self, line): stage = line.split(" ", 2)[1] self.stage_text.set(STAGES.get(stage, line.split(" ", 2)[-1])) self.bar.configure(mode="determinate", value=0) - if stage == "compile": - self.compile_started = time.time() + # (the compile's clock and countdown are the compile's: a later step, the minutes-long shaders one, without them) + self.compile_started = time.time() if stage == "compile" else None + self.eta = None elif line.startswith("@@progress "): try: self.bar.configure(value=float(line.split()[1])) @@ -597,6 +598,8 @@ def on_line(self, line): self.built = line[7:] elif line.startswith("@@error "): self.error = line[8:] + elif line.startswith(make_game.NOT_PREPARED): + self.shaders_later = True if hasattr(self, "log"): self.log.insert("end", line + "\n") self.log.see("end") @@ -614,6 +617,8 @@ def on_exit(self, code): text += ", compiled in %d:%02d" % (int(compiled) // 60, int(compiled) % 60) if previous > 0: text += " · last time %d:%02d" % (int(previous) // 3600, int(previous) // 60 % 60) + if self.shaders_later: + text += " · the shaders are prepared at its first start" self.stage_text.set(text) self.bar.configure(value=1.0) try: diff --git a/nx2/make_game.py b/nx2/make_game.py index 8754be56..aff5ee06 100644 --- a/nx2/make_game.py +++ b/nx2/make_game.py @@ -17,6 +17,9 @@ @@check (--check) the answer @@done finished @@error stopped + +and one plain line the builders also read (older ones only show it): "the shaders were not prepared now (): ..." when the +build's last step, the shaders' preparation, was skipped or failed and is left to the game's first start (NOT_PREPARED) """ import argparse import contextlib @@ -24,10 +27,12 @@ import hashlib import json import os +import queue import re import shutil import stat import subprocess +import threading import time import sys import tempfile @@ -214,19 +219,65 @@ def clock(seconds): return "%d:%02d:%02d" % (s // 3600, s // 60 % 60, s % 60) -def run(cmd, env=None, on_line=None, check=True, output=None): - e = dict(os.environ, PYTHONIOENCODING="utf-8") +class Silent(Exception): + """a step printed nothing for longer than run()'s silence""" + + +def output_lines(p, silence): + """A step's lines as they come; with silence (seconds), read on a thread so that a step which goes quiet that long (a hung + driver, a crash dialog left open) raises Silent instead of waiting for ever""" + if silence is None: + yield from p.stdout + return + lines = queue.Queue() + + def read(): + try: + for line in p.stdout: + lines.put(line) + except (OSError, ValueError): + pass + finally: + with contextlib.suppress(OSError, ValueError): + p.stdout.close() # (here, not by run(): a stopped step's pipe may still be read by this thread) + lines.put(None) + + threading.Thread(target=read, daemon=True).start() + while True: + try: + line = lines.get(timeout=silence) + except queue.Empty: + raise Silent() from None + if line is None: + return + yield line + + +def run(cmd, env=None, on_line=None, check=True, output=None, show=None, cwd=ROOT, base=None, silence=None): + """A step, its output printed as it comes. show: what of each line is printed (its None: nothing; on_line and a failure's + reason still get the line itself). base: the environment it starts from (os.environ). silence: seconds without a line after + which the step is stopped; it returns None then (with check, the build fails)""" + e = dict(os.environ if base is None else base, PYTHONIOENCODING="utf-8") e.update(env or {}) - p = subprocess.Popen(cmd, cwd=ROOT, env=e, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", + p = subprocess.Popen(cmd, cwd=cwd, env=e, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", bufsize=1) no_throttle(getattr(p, "_handle", None)) lines = [] - for line in p.stdout: - line = line.rstrip() - print(line, file=output, flush=True) - lines.append(line) - if on_line: - on_line(line) + try: + for line in output_lines(p, silence): + line = line.rstrip() + shown = show(line) if show else line + if shown is not None: + print(shown, file=output, flush=True) + lines.append(line) + if on_line: + on_line(line) + except Silent: + p.kill() + p.wait() + if check: + fail("%s printed nothing for %s and was stopped" % (os.path.basename(cmd[0]), span(silence))) + return None p.stdout.close() code = p.wait() if code and check: @@ -755,25 +806,66 @@ def ship_state_list(out): print("the shipped pipeline state list: %d pipelines" % ((os.path.getsize(src) - 12) // 528), flush=True) -def prepare_shaders(cmd): +# (the builders read this line's start: the step was skipped or failed, and the shaders' row says so instead of a check mark) +NOT_PREPARED = "the shaders were not prepared now" +# (a run that prints nothing this long is stopped. Not its whole length: that is 25 minutes on some AMD GPUs, and the driver's own +# step near the end can sit at 99% for minutes) +PREPARE_SILENCE = 15 * 60 +GAME_ENV_DROP = ("HOOKSHOT_", "PYTHON", "APPDIR", "APPIMAGE", "ARGV0", "OWD") + + +def span(seconds): + return "%d minutes" % (seconds // 60) if seconds >= 120 else "%g seconds" % seconds + + +def game_env(environ=None): + """The environment the game runs in when the tools start it: without what the AppImage runtime and the tools put there (their + libraries first on the search path, Python's settings, the HOOKSHOT_ build switches), so it loads its own SDL and the system's + Vulkan, as when the player starts it. The Linux builder's Play button uses it too; builder/linux/hookshot-launch drops the same""" + env = {k: v for k, v in (os.environ if environ is None else environ).items() if not k.startswith(GAME_ENV_DROP)} + env.pop("LD_LIBRARY_PATH", None) + return env + + +def not_prepared(why): + print("%s (%s): the game prepares them at its first start" % (NOT_PREPARED, why), flush=True) + + +def prepare_skipped(a, environ=None): + """Why the build leaves the shaders to the first start, or None: it prepares them""" + environ = os.environ if environ is None else environ + if a.no_prepare_shaders: + return '"Prepare shaders now" is unticked' + if environ.get("HOOKSHOT_NO_PREPARE", "0") != "0": # (set and not 0, as the game reads HOOKSHOT_NO_UPDATE_CHECK) + return "HOOKSHOT_NO_PREPARE is set" + if CROSS: # (hookshot.exe can't run here, and under Wine it would prepare for this PC's GPU, not the one that plays it) + return "a Windows game built on Linux prepares them on the PC it runs on" + return None + + +def show_prepared(line): + """The game's @@stage and @@progress lines go on to the builder; its own @@done and @@error would end or fail the build there, + so they go on as plain text""" + return line if line.startswith(("@@stage ", "@@progress ")) or not line.startswith("@@") else line[2:] + + +def prepare_shaders(cmd, folder=None, silence=PREPARE_SILENCE): """The shaders prepared now, under the builder's progress bar, not at the game's first start behind a black window (minutes on - some GPUs, issue 711): hookshot --prepare-shaders (host/hos_main.c, gpu/vk/vk_main.cpp), the same work as that first start. - Its @@stage and @@progress lines go on to the builder; its own @@done and @@error would end or fail the build there, so they - go on as plain text. A PC whose Vulkan can't (exit 3), or any other failure, leaves the work to the first start, as before""" + some GPUs, issue 711): hookshot --prepare-shaders (host/hos_main.c, gpu/vk/vk_main.cpp), the same work as that first start, in + the game's folder and environment, as the builders' Play starts it. A PC whose Vulkan can't (exit 3), a run silent for `silence` seconds (then stopped), or + any other failure leaves the work to the first start, as before: never a failed build. Returns its exit code (None: stopped)""" try: - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", - bufsize=1) + code = run(cmd, check=False, show=show_prepared, cwd=folder or os.path.dirname(cmd[0]) or None, + base=game_env(), silence=silence) except OSError as e: - print("the shaders were not prepared now (%s): the game prepares them at its first start" % e, flush=True) + not_prepared(str(e)) return 1 - no_throttle(getattr(p, "_handle", None)) - for line in p.stdout: - line = line.rstrip() - print(line if line.startswith(("@@stage ", "@@progress ")) or not line.startswith("@@") else line[2:], flush=True) - p.stdout.close() - code = p.wait() - if code: - print("the shaders were not prepared now (exit %d): the game prepares them at its first start" % code, flush=True) + if code is None: + not_prepared("the game printed nothing for %s and was stopped" % span(silence)) + elif code == 3: + not_prepared("no usable Vulkan here") + elif code: + not_prepared("the game stopped with exit status %d" % code) return code @@ -1260,14 +1352,18 @@ def compiled(line): for sub in ("sa", "objcache", "probe"): shutil.rmtree(os.path.join(work, sub), ignore_errors=True) print("the build files in %s were deleted (HOOKSHOT_KEEP_BUILD_FILES=1 keeps them for quicker updates)" % work, flush=True) - if not a.no_prepare_shaders and not os.environ.get("HOOKSHOT_NO_PREPARE"): # (either: left to the first start, as before) - prepare_shaders([os.path.join(game_dir, BESIDE[0]), "--prepare-shaders"]) total_seconds = time.time() - STARTED previous = record_times(work, run_files, compile_seconds, total_seconds, modules=plan[1], kind="update" if a.update else "build") print("the build took %s, compiled in %s%s" % (clock(total_seconds), clock(compile_seconds), "" if previous is None else " (the last one %s)" % clock(previous)), flush=True) say("times", "%.0f" % total_seconds, "%.0f" % compile_seconds, "-1" if previous is None else "%.0f" % previous) + # (after the times: they stay comparable from build to build, without a GPU's minutes that can be switched off) + skipped = prepare_skipped(a) + if skipped: + not_prepared(skipped) + else: + prepare_shaders([os.path.join(game_dir, BESIDE[0]), "--prepare-shaders"], out) say("done", out) diff --git a/tests/builder_workers/BuilderWorkersTests.csproj b/tests/builder_workers/BuilderWorkersTests.csproj index 9d591d12..f89d1579 100644 --- a/tests/builder_workers/BuilderWorkersTests.csproj +++ b/tests/builder_workers/BuilderWorkersTests.csproj @@ -9,5 +9,7 @@ + + diff --git a/tests/builder_workers/Program.cs b/tests/builder_workers/Program.cs index 6842544f..9d1c4087 100644 --- a/tests/builder_workers/Program.cs +++ b/tests/builder_workers/Program.cs @@ -50,6 +50,35 @@ static async Task EmptyParent(string python, Tools tools) Require(value.GetProperty("jobs").GetString() == "", "Automatic changed an inherited empty entry."); } + /// The shaders' step: the update's argument from the saved choice (the launcher's Update and the --update start read it, not + /// the tick), and what the row says when make_game left the shaders to the first start + static void ShaderStepChecks(string root) + { + var settings = Path.Combine(root, "builder-settings.json"); + Require(BuilderSettings.PrepareNow(settings), "No saved settings must prepare the shaders, as the tick's default."); + File.WriteAllText(settings, "not json"); + Require(BuilderSettings.PrepareNow(settings), "Unreadable settings must prepare the shaders."); + File.WriteAllText(settings, """{"Out":"authored output","PrepareLater":true}"""); + Require(!BuilderSettings.PrepareNow(settings), "A saved unticked choice must leave the shaders to the first start."); + Require(ShaderStep.Args(new List { "--update", "authored game" }, BuilderSettings.PrepareNow(settings)) + .SequenceEqual(new[] { "--update", "authored game", "--no-prepare-shaders" }), "The update ignored the saved choice."); + File.WriteAllText(settings, """{"Out":"authored output","PrepareLater":false}"""); + Require(ShaderStep.Args(new List { "--update", "authored game" }, BuilderSettings.PrepareNow(settings)) + .SequenceEqual(new[] { "--update", "authored game" }), "A saved ticked choice must prepare the shaders."); + + Require(ShaderStep.Reason("the shaders were not prepared now (no usable Vulkan here): the game prepares them at its first start") + == "no usable Vulkan here", "make_game's not-prepared line was not read."); + Require(ShaderStep.Reason("the shaders were not prepared now (\"Prepare shaders now\" is unticked (here)): the game prepares them at its first start") + == "\"Prepare shaders now\" is unticked (here)", "A reason with brackets was cut."); + Require(ShaderStep.Reason("@@stage shaders Preparing shaders") is null && ShaderStep.Reason("done shaders") is null, + "Other lines are not a skipped step."); + Require(ShaderStep.Skipped(began: true, reason: null) is null, "Prepared shaders must get their check mark."); + Require(ShaderStep.Skipped(began: false, reason: null) is { } none && none.Contains("first start"), + "No @@stage shaders (an older make_game, or none ran) must not be a check mark."); + Require(ShaderStep.Skipped(began: true, reason: "the game printed nothing for 15 minutes and was stopped") is { } stopped + && stopped.Contains("15 minutes"), "A stopped preparation must say why under its row."); + } + static async Task Main(string[] args) { if (args.Length is < 1 or > 2) throw new Exception("Pass the Python executable for the authored child."); @@ -74,6 +103,7 @@ static async Task Main(string[] args) return; } + ShaderStepChecks(root); Require(BuildWorkers.Choices(4).SequenceEqual(new object[] { BuildWorkers.Automatic, 1, 2, 3, 4 }), "The menu must offer Automatic first and one count per logical processor."); Require(BuildWorkers.Choices(0).SequenceEqual(new object[] { BuildWorkers.Automatic, 1 }), "CPU fallback must still offer one worker."); diff --git a/tests/test_prepare_at_build.py b/tests/test_prepare_at_build.py index 69d6797b..5d52366a 100644 --- a/tests/test_prepare_at_build.py +++ b/tests/test_prepare_at_build.py @@ -1,67 +1,194 @@ -"""make_game.py's last step, hookshot --prepare-shaders, relayed to the builder: its stage and progress lines go on, its own -@@done and @@error don't (the builders read those as the build's end or failure), and a failure leaves the work to the first start. +"""make_game.py's last step, hookshot --prepare-shaders, run against stub games: its stage and progress lines go on to the builder, +its own @@done and @@error don't (the builders read those as the build's end or failure), a failure or a silent hang leaves the work +to the first start (never a failed build), and the game runs in its own environment, as the builders' Play starts it. python -I -B tests/test_prepare_at_build.py """ +import ast import contextlib import importlib.util import io +import json +import os +import re import sys +import tempfile +import time import unittest from pathlib import Path +from types import SimpleNamespace +from unittest import mock ROOT = Path(__file__).resolve().parents[1] SOURCE = ROOT / "nx2" / "make_game.py" spec = importlib.util.spec_from_file_location("make_game_prepare", SOURCE) make_game = importlib.util.module_from_spec(spec) spec.loader.exec_module(make_game) +BUILDER = ROOT / "builder" / "linux" / "hookshot_builder.py" -def relayed(child): +def stub(code): + """A stand-in for hookshot --prepare-shaders: authored Python, no game, no GPU""" + return [sys.executable, "-I", "-c", code] + + +def prepared(cmd, **kwargs): out = io.StringIO() with contextlib.redirect_stdout(out): - code = make_game.prepare_shaders([sys.executable, "-c", child]) + code = make_game.prepare_shaders(cmd, **kwargs) return code, out.getvalue().splitlines() -class PrepareAtBuildTests(unittest.TestCase): - def test_progress_goes_on_and_done_does_not(self): - code, lines = relayed("print('hookshot v'); print('@@stage shaders Preparing shaders'); print('@@progress 0.500');" - "print('@@done shaders')") +def not_prepared(lines): + return [l for l in lines if l.startswith(make_game.NOT_PREPARED)] + + +class PrepareShadersTests(unittest.TestCase): + def test_progress_goes_on_and_the_games_done_does_not(self): + code, lines = prepared(stub("print('hookshot v'); print('@@stage shaders Preparing shaders'); print('@@progress 0.500');" + "print('@@done shaders')")) self.assertEqual(code, 0) self.assertEqual(lines, ["hookshot v", "@@stage shaders Preparing shaders", "@@progress 0.500", "done shaders"]) - def test_no_vulkan_is_not_a_failed_build(self): - code, lines = relayed("import sys; print('@@error no Vulkan device'); sys.exit(3)") + def test_no_vulkan_leaves_them_to_the_first_start(self): + code, lines = prepared(stub("import sys; print('@@error no usable Vulkan here'); sys.exit(3)")) self.assertEqual(code, 3) self.assertFalse(any(l.startswith("@@") for l in lines)) - self.assertIn("error no Vulkan device", lines) - self.assertTrue(lines[-1].startswith("the shaders were not prepared now (exit 3)")) + self.assertIn("error no usable Vulkan here", lines) + self.assertEqual(len(not_prepared(lines)), 1) + self.assertIn("(no usable Vulkan here)", lines[-1]) + + def test_a_crash_leaves_them_to_the_first_start(self): + code, lines = prepared(stub("import sys; print('@@stage shaders Preparing shaders'); sys.exit(5)")) + self.assertEqual(code, 5) + self.assertIn("exit status 5", not_prepared(lines)[0]) + + def test_a_silent_hang_is_stopped_and_not_a_failed_build(self): + # (prints its stage, then nothing: as a driver deadlock or a crash dialog left open. silence is 15 minutes in a build) + started = time.monotonic() + code, lines = prepared(stub("import time; print('@@stage shaders Preparing shaders', flush=True); time.sleep(120)"), + silence=1) + self.assertLess(time.monotonic() - started, 30) + self.assertIsNone(code) + self.assertEqual(lines[0], "@@stage shaders Preparing shaders") + self.assertIn("printed nothing for 1 seconds and was stopped", not_prepared(lines)[0]) + + def test_steady_output_is_not_silence(self): + code, lines = prepared(stub("import time\nfor i in range(6):\n print('@@progress %.3f' % (i / 6), flush=True); " + "time.sleep(0.4)"), silence=1) + self.assertEqual(code, 0) + self.assertEqual(not_prepared(lines), []) def test_missing_game_is_not_a_failed_build(self): + code, lines = prepared([str(ROOT / "no-such-folder" / "hookshot.exe"), "--prepare-shaders"]) + self.assertEqual(code, 1) + self.assertEqual(len(not_prepared(lines)), 1) + self.assertFalse(any(l.startswith("@@") for l in lines)) + + def test_the_game_runs_in_its_folder_without_the_tools_environment(self): + dropped = dict(HOOKSHOT_TOOLS="/builder/tools", HOOKSHOT_NO_PREPARE="0", PYTHONPATH="/builder/python", + PYTHONHOME="/builder", APPDIR="/appdir", APPIMAGE="/tmp/Hookshot.AppImage", ARGV0="Hookshot", OWD="/old", + LD_LIBRARY_PATH="/builder/lib") + with tempfile.TemporaryDirectory(prefix="hookshot-prepare-") as folder, \ + mock.patch.dict(os.environ, dict(dropped, HOOKSHOT_TEST_KEPT_NOT="x", KEPT_FOR_THE_GAME="yes")): + code, lines = prepared(stub("import json, os; print(json.dumps({'cwd': os.getcwd(), 'env': dict(os.environ)}))"), + folder=folder) + seen = json.loads(lines[0]) + self.assertEqual(code, 0) + self.assertEqual(os.path.normcase(os.path.realpath(seen["cwd"])), os.path.normcase(os.path.realpath(folder))) + for name in list(dropped) + ["HOOKSHOT_TEST_KEPT_NOT"]: + self.assertNotIn(name, seen["env"], name) + self.assertEqual(seen["env"]["KEPT_FOR_THE_GAME"], "yes") + + def test_game_env_is_the_play_buttons(self): + env = make_game.game_env({"HOOKSHOT_X": "1", "PYTHONPATH": "p", "APPIMAGE": "a", "APPDIR": "d", "ARGV0": "0", "OWD": "o", + "LD_LIBRARY_PATH": "l", "PATH": "/usr/bin", "HOME": "/home/p"}) + self.assertEqual(env, {"PATH": "/usr/bin", "HOME": "/home/p"}) + # (the shell launcher can't import it: the same names there) + launch = (ROOT / "builder" / "linux" / "hookshot-launch").read_text(encoding="utf-8") + names = re.search(r"\$1 ~ /\^\(([A-Z_|0-9]+)\)/ \|\| \$1 == \"LD_LIBRARY_PATH\"", launch).group(1).split("|") + self.assertEqual(tuple(names), make_game.GAME_ENV_DROP) + + +class RunSilenceTests(unittest.TestCase): + def test_a_silent_step_fails_the_build_when_checked(self): out = io.StringIO() + with contextlib.redirect_stdout(out), self.assertRaises(SystemExit): + make_game.run(stub("import time; time.sleep(120)"), silence=1) + self.assertIn("@@error", out.getvalue()) + self.assertIn("printed nothing for 1 seconds", out.getvalue()) + + def test_show_filters_what_is_printed_but_not_what_is_seen(self): + out, seen = io.StringIO(), [] with contextlib.redirect_stdout(out): - code = make_game.prepare_shaders([str(ROOT / "no-such-folder" / "hookshot.exe"), "--prepare-shaders"]) - self.assertEqual(code, 1) - self.assertNotIn("@@", out.getvalue()) - - def test_the_build_runs_it_after_the_game_folder_is_complete(self): - text = SOURCE.read_text(encoding="utf-8") - main = text[text.index("\ndef main("):] - at = main.index("prepare_shaders([os.path.join(game_dir, BESIDE[0]), \"--prepare-shaders\"])") - for before in ("os.replace(dst + \".new\", dst)", "shader_census_cmd(", "seed_pipelines(out)"): - self.assertLess(main.index(before), at, before) - self.assertLess(at, main.index("say(\"done\", out)")) - - def test_the_builders_can_leave_it_to_the_first_start(self): - text = SOURCE.read_text(encoding="utf-8") - self.assertIn('"--no-prepare-shaders"', text) - main = text[text.index("\ndef main("):] - guard = main[:main.index("prepare_shaders([os.path.join(game_dir")].rsplit("\n", 2)[-2] - self.assertIn("a.no_prepare_shaders", guard) - self.assertIn('HOOKSHOT_NO_PREPARE', guard) - for builder in ("builder/HookshotBuilder/MainWindow.xaml.cs", "builder/linux/hookshot_builder.py"): - self.assertEqual((ROOT / builder).read_text(encoding="utf-8").count("--no-prepare-shaders"), 2, builder) # (build and update) + code = make_game.run(stub("print('keep'); print('drop')"), on_line=seen.append, + show=lambda line: None if line == "drop" else line.upper()) + self.assertEqual(code, 0) + self.assertEqual(out.getvalue().splitlines(), ["KEEP"]) + self.assertEqual(seen, ["keep", "drop"]) + + +class SkipTests(unittest.TestCase): + def skipped(self, environ, flag=False, cross=False): + with mock.patch.object(make_game, "CROSS", cross): + return make_game.prepare_skipped(SimpleNamespace(no_prepare_shaders=flag), environ) + + def test_prepared_by_default(self): + self.assertIsNone(self.skipped({})) + + def test_the_unticked_box(self): + self.assertIn("unticked", self.skipped({}, flag=True)) + + def test_hookshot_no_prepare_as_the_game_reads_its_switches(self): + self.assertIsNone(self.skipped({"HOOKSHOT_NO_PREPARE": "0"})) + self.assertIsNotNone(self.skipped({"HOOKSHOT_NO_PREPARE": "1"})) + self.assertIsNotNone(self.skipped({"HOOKSHOT_NO_PREPARE": ""})) # (set, as the game's getenv && strcmp(.., "0")) + + def test_a_windows_game_built_on_linux(self): + self.assertIn("Windows game built on Linux", self.skipped({}, cross=True)) + + def test_the_build_times_leave_the_preparation_out(self): + # (one check of main's order: the times are recorded and said before the preparation, which comes before @@done) + main = SOURCE.read_text(encoding="utf-8").split("\ndef main(", 1)[1] + self.assertLess(main.index("record_times("), main.index("prepare_skipped(a)")) + self.assertLess(main.index('say("times"'), main.index("prepare_skipped(a)")) + self.assertLess(main.index("prepare_skipped(a)"), main.index('say("done", out)')) + + +def builder_method(name): + """One method of the Linux builder's window, compiled alone (no Tk)""" + tree = ast.parse(BUILDER.read_text(encoding="utf-8")) + cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "Builder") + node = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == name) + stages = next(n for n in tree.body if isinstance(n, ast.Assign) and getattr(n.targets[0], "id", "") == "STAGES") + namespace = {"make_game": make_game, "time": time} + exec(compile(ast.Module(body=[stages, node], type_ignores=[]), str(BUILDER), "exec"), namespace) + return namespace[name] + + +class LinuxBuilderTests(unittest.TestCase): + def window(self): + text = [] + return SimpleNamespace(stage_text=SimpleNamespace(set=text.append), bar=SimpleNamespace(configure=lambda **k: None), + compile_started=None, eta=None, shaders_later=False), text + + def test_a_later_stage_clears_the_compiles_clock_and_countdown(self): + on_line = builder_method("on_line") + w, text = self.window() + on_line(w, "@@stage compile Compiling") + on_line(w, "@@eta 300") + self.assertIsNotNone(w.compile_started) + self.assertEqual(w.eta, 300) + on_line(w, "@@stage shaders Preparing shaders") + self.assertIsNone(w.compile_started) + self.assertIsNone(w.eta) + self.assertEqual(text[-1], "Preparing shaders") + + def test_the_not_prepared_line_is_noted(self): + on_line = builder_method("on_line") + w, _ = self.window() + on_line(w, "the shaders were not prepared now (no usable Vulkan here): the game prepares them at its first start") + self.assertTrue(w.shaders_later) if __name__ == "__main__":