From 0979c07c45919529ed9058b7c0dc44f63e879f69 Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Sun, 8 Nov 2026 21:47:42 +0100 Subject: [PATCH 1/2] Build: prepare the shaders at the end of the build, not at the first start make_game.py now runs hookshot --prepare-shaders as its last step, so the minutes of shader preparation happen under the builder's progress bar instead of behind the game's black window at its first start (issue 711). The command has been in the game since PR 320, but nothing ran it. Its @@stage/@@progress lines go on to the builder; its own @@done and @@error go on as plain text, since both builders read those as the build's end or failure. A PC without usable Vulkan (exit 3) or any other failure leaves the work to the first start, as before. HOOKSHOT_NO_PREPARE=1 skips the step. The Windows builder gets a "Prepare the shaders" row. Co-Authored-By: Claude Opus 5.5 --- builder/HookshotBuilder/MainWindow.xaml.cs | 4 +- nx2/make_game.py | 24 +++++++++ tests/test_prepare_at_build.py | 58 ++++++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/test_prepare_at_build.py diff --git a/builder/HookshotBuilder/MainWindow.xaml.cs b/builder/HookshotBuilder/MainWindow.xaml.cs index d68b5141..bc9a458a 100644 --- a/builder/HookshotBuilder/MainWindow.xaml.cs +++ b/builder/HookshotBuilder/MainWindow.xaml.cs @@ -685,10 +685,12 @@ static readonly (string Id, string Build, string Update)[] Stages = ("lift", "Translate the game to PC code", "Translate the game to PC code"), ("compile", "Compile", "Compile changed files"), ("finish", "Put the game together", "Swap in the new game"), + ("shaders", "Prepare the shaders", "Prepare the shaders"), }; static readonly Dictionary Weight = new() { - ["download"] = (0, 0.04), ["read"] = (0.04, 0.04), ["lift"] = (0.08, 0.14), ["compile"] = (0.22, 0.74), ["finish"] = (0.96, 0.04), + ["download"] = (0, 0.04), ["read"] = (0.04, 0.04), ["lift"] = (0.08, 0.14), ["compile"] = (0.22, 0.68), ["finish"] = (0.90, 0.02), + ["shaders"] = (0.92, 0.08), // (hookshot --prepare-shaders: minutes on some GPUs, the first start's wait moved here) }; void MakeRows(bool update) diff --git a/nx2/make_game.py b/nx2/make_game.py index e05bf0a3..3f11bc6d 100644 --- a/nx2/make_game.py +++ b/nx2/make_game.py @@ -727,6 +727,28 @@ def seed_pipelines(out): print("the shipped pipeline list: %d KB, built before the first title" % (os.path.getsize(seed) >> 10), flush=True) +def prepare_shaders(cmd): + """The shaders prepared now, under the builder's progress bar, not at the game's first start behind a black window (minutes on + some GPUs, issue 711): hookshot --prepare-shaders (host/hos_main.c, gpu/vk/vk_main.cpp), the same work as that first start. + Its @@stage and @@progress lines go on to the builder; its own @@done and @@error would end or fail the build there, so they + go on as plain text. A PC whose Vulkan can't (exit 3), or any other failure, leaves the work to the first start, as before""" + try: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", + bufsize=1) + except OSError as e: + print("the shaders were not prepared now (%s): the game prepares them at its first start" % e, flush=True) + return 1 + no_throttle(getattr(p, "_handle", None)) + for line in p.stdout: + line = line.rstrip() + print(line if line.startswith(("@@stage ", "@@progress ")) or not line.startswith("@@") else line[2:], flush=True) + p.stdout.close() + code = p.wait() + if code: + print("the shaders were not prepared now (exit %d): the game prepares them at its first start" % code, flush=True) + return code + + def folder_language(out): """the language a game folder was built in: the code line of its language.txt""" try: @@ -1111,6 +1133,8 @@ def compiled(line): # changed; the builder says how much stays and lets it go. Kept in a development checkout) if os.path.normcase(work) != os.path.normcase(os.path.abspath(os.path.join(ROOT, "work"))): shutil.rmtree(exefs, ignore_errors=True) + if not os.environ.get("HOOKSHOT_NO_PREPARE"): # (=1: left to the first start, as before) + prepare_shaders([os.path.join(game_dir, BESIDE[0]), "--prepare-shaders"]) total_seconds = time.time() - STARTED previous = record_times(work, run_files, compile_seconds, total_seconds, modules=plan[1], kind="update" if a.update else "build") diff --git a/tests/test_prepare_at_build.py b/tests/test_prepare_at_build.py new file mode 100644 index 00000000..fe4eb232 --- /dev/null +++ b/tests/test_prepare_at_build.py @@ -0,0 +1,58 @@ +"""make_game.py's last step, hookshot --prepare-shaders, relayed to the builder: its stage and progress lines go on, its own +@@done and @@error don't (the builders read those as the build's end or failure), and a failure leaves the work to the first start. + +python -I -B tests/test_prepare_at_build.py +""" +import contextlib +import importlib.util +import io +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "nx2" / "make_game.py" +spec = importlib.util.spec_from_file_location("make_game_prepare", SOURCE) +make_game = importlib.util.module_from_spec(spec) +spec.loader.exec_module(make_game) + + +def relayed(child): + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = make_game.prepare_shaders([sys.executable, "-c", child]) + return code, out.getvalue().splitlines() + + +class PrepareAtBuildTests(unittest.TestCase): + def test_progress_goes_on_and_done_does_not(self): + code, lines = relayed("print('hookshot v'); print('@@stage shaders Preparing shaders'); print('@@progress 0.500');" + "print('@@done shaders')") + self.assertEqual(code, 0) + self.assertEqual(lines, ["hookshot v", "@@stage shaders Preparing shaders", "@@progress 0.500", "done shaders"]) + + def test_no_vulkan_is_not_a_failed_build(self): + code, lines = relayed("import sys; print('@@error no Vulkan device'); sys.exit(3)") + self.assertEqual(code, 3) + self.assertFalse(any(l.startswith("@@") for l in lines)) + self.assertIn("error no Vulkan device", lines) + self.assertTrue(lines[-1].startswith("the shaders were not prepared now (exit 3)")) + + def test_missing_game_is_not_a_failed_build(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = make_game.prepare_shaders([str(ROOT / "no-such-folder" / "hookshot.exe"), "--prepare-shaders"]) + self.assertEqual(code, 1) + self.assertNotIn("@@", out.getvalue()) + + def test_the_build_runs_it_after_the_game_folder_is_complete(self): + text = SOURCE.read_text(encoding="utf-8") + main = text[text.index("\ndef main("):] + at = main.index("prepare_shaders([os.path.join(game_dir, BESIDE[0]), \"--prepare-shaders\"])") + for before in ("os.replace(dst + \".new\", dst)", "shader_census_cmd(", "seed_pipelines(out)"): + self.assertLess(main.index(before), at, before) + self.assertLess(at, main.index("say(\"done\", out)")) + + +if __name__ == "__main__": + unittest.main() From 122a485d019630fe080843b96eb2462293e41880 Mon Sep 17 00:00:00 2001 From: klimovich008 Date: Mon, 9 Nov 2026 03:22:54 +0100 Subject: [PATCH 2/2] Builders: a "Prepare shaders now" tick, on by default Unticked, the builders pass make_game.py --no-prepare-shaders and the game prepares its shaders at its first start, as before. The choice is kept (builder-settings.json on Windows, builder.ini on Linux) and used for updates too. Co-Authored-By: Claude Opus 5.5 --- builder/HookshotBuilder/MainWindow.xaml | 3 +++ builder/HookshotBuilder/MainWindow.xaml.cs | 10 ++++++++-- builder/linux/hookshot_builder.py | 12 +++++++++--- nx2/make_game.py | 4 +++- tests/test_prepare_at_build.py | 10 ++++++++++ 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/builder/HookshotBuilder/MainWindow.xaml b/builder/HookshotBuilder/MainWindow.xaml index e358b23d..9abde96b 100644 --- a/builder/HookshotBuilder/MainWindow.xaml +++ b/builder/HookshotBuilder/MainWindow.xaml @@ -170,6 +170,9 @@ + + diff --git a/builder/HookshotBuilder/MainWindow.xaml.cs b/builder/HookshotBuilder/MainWindow.xaml.cs index bc9a458a..9e77d777 100644 --- a/builder/HookshotBuilder/MainWindow.xaml.cs +++ b/builder/HookshotBuilder/MainWindow.xaml.cs @@ -255,6 +255,7 @@ sealed class BuilderSettings public string? Nup { get; set; } public string? Romfs { get; set; } public string? Exefs { get; set; } + public bool PrepareLater { get; set; } // ("Prepare shaders now" unticked) } static string SettingsPath => System.IO.Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "HookshotBuilder", "builder-settings.json"); @@ -264,6 +265,7 @@ void ApplySettings() BuilderSettings? s = null; try { s = System.Text.Json.JsonSerializer.Deserialize(File.ReadAllText(SettingsPath)); } catch (Exception) { } if (s == null) return; + PrepareTick.IsChecked = !s.PrepareLater; if (!string.IsNullOrEmpty(s.Out)) OutPath.Text = s.Out; if (s.Romfs != null && s.Exefs != null && Directory.Exists(s.Romfs) && File.Exists(System.IO.Path.Combine(s.Exefs, "main"))) { @@ -287,7 +289,8 @@ void SaveSettings() { try { - var s = new BuilderSettings { Out = OutPath.Text, Nup = romfs == null ? chosenNup : null, Romfs = romfs, Exefs = exefs }; + var s = new BuilderSettings { Out = OutPath.Text, Nup = romfs == null ? chosenNup : null, Romfs = romfs, Exefs = exefs, + PrepareLater = PrepareTick.IsChecked != true }; Directory.CreateDirectory(System.IO.Path.GetDirectoryName(SettingsPath)!); File.WriteAllText(SettingsPath, System.Text.Json.JsonSerializer.Serialize(s)); } @@ -675,6 +678,7 @@ async void StartBuild(object sender, RoutedEventArgs e) else if (chosenNup != null) args.AddRange(new[] { "--nup", chosenNup }); else return; args.AddRange(new[] { "--out", OutPath.Text, "--language", lang.Code }); + if (PrepareTick.IsChecked != true) args.Add("--no-prepare-shaders"); Run(tools, args); } @@ -1121,7 +1125,9 @@ internal void UpdateBuild(Tools tools, string folder) { download = null; if (rows.TryGetValue("download", out var d)) { d.Done(); d.Small("Build tools checked"); } - Run(tools, new List { "--update", folder }, keepRows: true); + var args = new List { "--update", folder }; + if (PrepareTick.IsChecked != true) args.Add("--no-prepare-shaders"); // (as the last build chose: ApplySettings) + Run(tools, args, keepRows: true); } // ================= the window ================= diff --git a/builder/linux/hookshot_builder.py b/builder/linux/hookshot_builder.py index b24044e7..fe998a0a 100644 --- a/builder/linux/hookshot_builder.py +++ b/builder/linux/hookshot_builder.py @@ -172,6 +172,7 @@ def __init__(self, launch=None, update=None, fetched=False, tag=None): names = [n for _, n in make_game.LANGUAGES] code = self.cfg.get("language", "en-US") self.language = tk.StringVar(value=dict(make_game.LANGUAGES).get(code, names[0])) + self.prepare = tk.BooleanVar(value=self.cfg.get("prepare", "1") != "0") # (make_game.py's last step: off, at the first start) self.check_text = tk.StringVar() self.space_text = tk.StringVar() self.stage_text = tk.StringVar() @@ -239,6 +240,9 @@ def show_form(self): ttk.Combobox(f, textvariable=self.language, values=[n for _, n in make_game.LANGUAGES], state="readonly").grid( row=row, column=1, sticky="w", **self.pad) row += 1 + ttk.Checkbutton(f, text="Prepare shaders now (unticked: at the game's first start, behind a black window)", + variable=self.prepare).grid(row=row, column=0, columnspan=3, sticky="w", **self.pad) + row += 1 ttk.Label(f, text="Takes about 45 minutes and needs 60 GB.").grid(row=row, column=0, columnspan=3, sticky="w", **self.pad) row += 1 ttk.Label(f, textvariable=self.space_text).grid(row=row, column=0, columnspan=3, sticky="w", **self.pad) @@ -406,7 +410,8 @@ def start_make_update(self, folder): self.status.set("This builder is the latest version.") self.lookup_latest() return - cmd = [sys.executable, "-I", os.path.join(ROOT, "nx2", "make_game.py"), "--update", folder] + cmd = [sys.executable, "-I", os.path.join(ROOT, "nx2", "make_game.py"), "--update", folder] + \ + ([] if self.prepare.get() else ["--no-prepare-shaders"]) # (as the last build chose) self.heading.configure(text="Updating your game") self.stage_text.set("Compiling what changed") self.started = time.time() @@ -450,12 +455,13 @@ def build(self): except ValueError as error: return messagebox.showerror("Hookshot", str(error)) code = next((c for c, n in make_game.LANGUAGES if n == self.language.get()), "en-US") - self.cfg.update(kind=self.kind.get(), nup=self.nup.get(), romfs=self.romfs.get(), exefs=self.exefs.get(), out=out, language=code) + self.cfg.update(kind=self.kind.get(), nup=self.nup.get(), romfs=self.romfs.get(), exefs=self.exefs.get(), out=out, language=code, + prepare="1" if self.prepare.get() else "0") save_config(self.cfg) # (issue 531) Without HOOKSHOT_JOBS or RX_JOBS, make_game chooses as on Windows, from the memory free now; this window chose # fewer, 80% of the total memory at 3.5 GB each, and the CPU sat at 29% cmd = [sys.executable, "-I", os.path.join(ROOT, "nx2", "make_game.py")] + src + ["--out", out, "--language", code] + \ - ([] if jobs is None else ["--jobs", str(jobs)]) + ([] if jobs is None else ["--jobs", str(jobs)]) + ([] if self.prepare.get() else ["--no-prepare-shaders"]) self.show_progress() self.started = time.time() self.proc = subprocess.Popen(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, diff --git a/nx2/make_game.py b/nx2/make_game.py index 3f11bc6d..429b7d3f 100644 --- a/nx2/make_game.py +++ b/nx2/make_game.py @@ -877,6 +877,8 @@ def parse_args(): p.add_argument("--language", default="en-US", help="the game's language: a code from LANGUAGES, written to language.txt") p.add_argument("--check", action="store_true") p.add_argument("--update", help="a game folder built before: built again from the game files it keeps (the game's update offer)") + p.add_argument("--no-prepare-shaders", action="store_true", + help="leave the shaders' preparation to the game's first start (the builder's \"Prepare shaders now\" unticked)") p.add_argument("--log", metavar="FILE", help="also save build output to a UTF-8 file (replaces the file if it exists)") a = p.parse_args() a.default_work = p.get_default("work") # (main: the builder's own work folder, stop_stray_compilers) @@ -1133,7 +1135,7 @@ def compiled(line): # changed; the builder says how much stays and lets it go. Kept in a development checkout) if os.path.normcase(work) != os.path.normcase(os.path.abspath(os.path.join(ROOT, "work"))): shutil.rmtree(exefs, ignore_errors=True) - if not os.environ.get("HOOKSHOT_NO_PREPARE"): # (=1: left to the first start, as before) + if not a.no_prepare_shaders and not os.environ.get("HOOKSHOT_NO_PREPARE"): # (either: left to the first start, as before) prepare_shaders([os.path.join(game_dir, BESIDE[0]), "--prepare-shaders"]) total_seconds = time.time() - STARTED previous = record_times(work, run_files, compile_seconds, total_seconds, modules=plan[1], diff --git a/tests/test_prepare_at_build.py b/tests/test_prepare_at_build.py index fe4eb232..69d6797b 100644 --- a/tests/test_prepare_at_build.py +++ b/tests/test_prepare_at_build.py @@ -53,6 +53,16 @@ def test_the_build_runs_it_after_the_game_folder_is_complete(self): self.assertLess(main.index(before), at, before) self.assertLess(at, main.index("say(\"done\", out)")) + def test_the_builders_can_leave_it_to_the_first_start(self): + text = SOURCE.read_text(encoding="utf-8") + self.assertIn('"--no-prepare-shaders"', text) + main = text[text.index("\ndef main("):] + guard = main[:main.index("prepare_shaders([os.path.join(game_dir")].rsplit("\n", 2)[-2] + self.assertIn("a.no_prepare_shaders", guard) + self.assertIn('HOOKSHOT_NO_PREPARE', guard) + for builder in ("builder/HookshotBuilder/MainWindow.xaml.cs", "builder/linux/hookshot_builder.py"): + self.assertEqual((ROOT / builder).read_text(encoding="utf-8").count("--no-prepare-shaders"), 2, builder) # (build and update) + if __name__ == "__main__": unittest.main()