Build: prepare the shaders at the end of the build, not at the first start

builder/HookshotBuilder/MainWindow.xaml

@@ -170,6 +170,9 @@
                   </Style>
                 </ComboBox.FocusVisualStyle>
               </ComboBox>
+              <!-- (make_game.py's last step, the game's prepare-shaders run: unticked, the game prepares them at its first start) -->
+              <CheckBox x:Name="PrepareTick" Style="{StaticResource Tick}" Content="Prepare shaders now" IsChecked="True" Margin="0,12,0,0"
+                        ToolTip="Prepares the game's shaders at the end of the build, a few minutes (longer on some GPUs). Unticked, the game prepares them at its first start, behind a black window."/>
             </StackPanel>
           </Grid>
 

builder/HookshotBuilder/MainWindow.xaml.cs

@@ -255,6 +255,7 @@ sealed class BuilderSettings
         public string? Nup { get; set; }
         public string? Romfs { get; set; }
         public string? Exefs { get; set; }
+        public bool PrepareLater { get; set; }   // ("Prepare shaders now" unticked)
     }
     static string SettingsPath => System.IO.Path.Combine(
         Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "HookshotBuilder", "builder-settings.json");
@@ -264,6 +265,7 @@ void ApplySettings()
         BuilderSettings? s = null;
         try { s = System.Text.Json.JsonSerializer.Deserialize<BuilderSettings>(File.ReadAllText(SettingsPath)); } catch (Exception) { }
         if (s == null) return;
+        PrepareTick.IsChecked = !s.PrepareLater;
         if (!string.IsNullOrEmpty(s.Out)) OutPath.Text = s.Out;
         if (s.Romfs != null && s.Exefs != null && Directory.Exists(s.Romfs) && File.Exists(System.IO.Path.Combine(s.Exefs, "main")))
         {
@@ -287,7 +289,8 @@ void SaveSettings()
     {
         try
         {
-            var s = new BuilderSettings { Out = OutPath.Text, Nup = romfs == null ? chosenNup : null, Romfs = romfs, Exefs = exefs };
+            var s = new BuilderSettings { Out = OutPath.Text, Nup = romfs == null ? chosenNup : null, Romfs = romfs, Exefs = exefs,
+                                         PrepareLater = PrepareTick.IsChecked != true };
             Directory.CreateDirectory(System.IO.Path.GetDirectoryName(SettingsPath)!);
             File.WriteAllText(SettingsPath, System.Text.Json.JsonSerializer.Serialize(s));
         }
@@ -675,6 +678,7 @@ async void StartBuild(object sender, RoutedEventArgs e)
         else if (chosenNup != null) args.AddRange(new[] { "--nup", chosenNup });
         else return;
         args.AddRange(new[] { "--out", OutPath.Text, "--language", lang.Code });
+        if (PrepareTick.IsChecked != true) args.Add("--no-prepare-shaders");
         Run(tools, args);
     }
 
@@ -685,10 +689,12 @@ static readonly (string Id, string Build, string Update)[] Stages =
         ("lift", "Translate the game to PC code", "Translate the game to PC code"),
         ("compile", "Compile", "Compile changed files"),
         ("finish", "Put the game together", "Swap in the new game"),
+        ("shaders", "Prepare the shaders", "Prepare the shaders"),
     };
     static readonly Dictionary<string, (double From, double Size)> Weight = new()
     {
-        ["download"] = (0, 0.04), ["read"] = (0.04, 0.04), ["lift"] = (0.08, 0.14), ["compile"] = (0.22, 0.74), ["finish"] = (0.96, 0.04),
+        ["download"] = (0, 0.04), ["read"] = (0.04, 0.04), ["lift"] = (0.08, 0.14), ["compile"] = (0.22, 0.68), ["finish"] = (0.90, 0.02),
+        ["shaders"] = (0.92, 0.08),   // (hookshot --prepare-shaders: minutes on some GPUs, the first start's wait moved here)
     };
 
     void MakeRows(bool update)
@@ -1119,7 +1125,9 @@ internal void UpdateBuild(Tools tools, string folder)
     {
         download = null;
         if (rows.TryGetValue("download", out var d)) { d.Done(); d.Small("Build tools checked"); }
-        Run(tools, new List<string> { "--update", folder }, keepRows: true);
+        var args = new List<string> { "--update", folder };
+        if (PrepareTick.IsChecked != true) args.Add("--no-prepare-shaders");   // (as the last build chose: ApplySettings)
+        Run(tools, args, keepRows: true);
     }
 
     // ================= the window =================

builder/linux/hookshot_builder.py

@@ -172,6 +172,7 @@ def __init__(self, launch=None, update=None, fetched=False, tag=None):
         names = [n for _, n in make_game.LANGUAGES]
         code = self.cfg.get("language", "en-US")
         self.language = tk.StringVar(value=dict(make_game.LANGUAGES).get(code, names[0]))
+        self.prepare = tk.BooleanVar(value=self.cfg.get("prepare", "1") != "0")   # (make_game.py's last step: off, at the first start)
         self.check_text = tk.StringVar()
         self.space_text = tk.StringVar()
         self.stage_text = tk.StringVar()
@@ -239,6 +240,9 @@ def show_form(self):
         ttk.Combobox(f, textvariable=self.language, values=[n for _, n in make_game.LANGUAGES], state="readonly").grid(
             row=row, column=1, sticky="w", **self.pad)
         row += 1
+        ttk.Checkbutton(f, text="Prepare shaders now (unticked: at the game's first start, behind a black window)",
+                        variable=self.prepare).grid(row=row, column=0, columnspan=3, sticky="w", **self.pad)
+        row += 1
         ttk.Label(f, text="Takes about 45 minutes and needs 60 GB.").grid(row=row, column=0, columnspan=3, sticky="w", **self.pad)
         row += 1
         ttk.Label(f, textvariable=self.space_text).grid(row=row, column=0, columnspan=3, sticky="w", **self.pad)
@@ -406,7 +410,8 @@ def start_make_update(self, folder):
             self.status.set("This builder is the latest version.")
             self.lookup_latest()
             return
-        cmd = [sys.executable, "-I", os.path.join(ROOT, "nx2", "make_game.py"), "--update", folder]
+        cmd = [sys.executable, "-I", os.path.join(ROOT, "nx2", "make_game.py"), "--update", folder] + \
+            ([] if self.prepare.get() else ["--no-prepare-shaders"])   # (as the last build chose)
         self.heading.configure(text="Updating your game")
         self.stage_text.set("Compiling what changed")
         self.started = time.time()
@@ -450,12 +455,13 @@ def build(self):
         except ValueError as error:
             return messagebox.showerror("Hookshot", str(error))
         code = next((c for c, n in make_game.LANGUAGES if n == self.language.get()), "en-US")
-        self.cfg.update(kind=self.kind.get(), nup=self.nup.get(), romfs=self.romfs.get(), exefs=self.exefs.get(), out=out, language=code)
+        self.cfg.update(kind=self.kind.get(), nup=self.nup.get(), romfs=self.romfs.get(), exefs=self.exefs.get(), out=out, language=code,
+                        prepare="1" if self.prepare.get() else "0")
         save_config(self.cfg)
         # (issue 531) Without HOOKSHOT_JOBS or RX_JOBS, make_game chooses as on Windows, from the memory free now; this window chose
         # fewer, 80% of the total memory at 3.5 GB each, and the CPU sat at 29%
         cmd = [sys.executable, "-I", os.path.join(ROOT, "nx2", "make_game.py")] + src + ["--out", out, "--language", code] + \
-            ([] if jobs is None else ["--jobs", str(jobs)])
+            ([] if jobs is None else ["--jobs", str(jobs)]) + ([] if self.prepare.get() else ["--no-prepare-shaders"])
         self.show_progress()
         self.started = time.time()
         self.proc = subprocess.Popen(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1,

nx2/make_game.py

@@ -727,6 +727,28 @@ def seed_pipelines(out):
     print("the shipped pipeline list: %d KB, built before the first title" % (os.path.getsize(seed) >> 10), flush=True)
 
 
+def prepare_shaders(cmd):
+    """The shaders prepared now, under the builder's progress bar, not at the game's first start behind a black window (minutes on
+    some GPUs, issue 711): hookshot --prepare-shaders (host/hos_main.c, gpu/vk/vk_main.cpp), the same work as that first start.
+    Its @@stage and @@progress lines go on to the builder; its own @@done and @@error would end or fail the build there, so they
+    go on as plain text. A PC whose Vulkan can't (exit 3), or any other failure, leaves the work to the first start, as before"""
+    try:
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace",
+                             bufsize=1)
+    except OSError as e:
+        print("the shaders were not prepared now (%s): the game prepares them at its first start" % e, flush=True)
+        return 1
+    no_throttle(getattr(p, "_handle", None))
+    for line in p.stdout:
+        line = line.rstrip()
+        print(line if line.startswith(("@@stage ", "@@progress ")) or not line.startswith("@@") else line[2:], flush=True)
+    p.stdout.close()
+    code = p.wait()
+    if code:
+        print("the shaders were not prepared now (exit %d): the game prepares them at its first start" % code, flush=True)
+    return code
+
+
 def folder_language(out):
     """the language a game folder was built in: the code line of its language.txt"""
     try:
@@ -855,6 +877,8 @@ def parse_args():
     p.add_argument("--language", default="en-US", help="the game's language: a code from LANGUAGES, written to language.txt")
     p.add_argument("--check", action="store_true")
     p.add_argument("--update", help="a game folder built before: built again from the game files it keeps (the game's update offer)")
+    p.add_argument("--no-prepare-shaders", action="store_true",
+                   help="leave the shaders' preparation to the game's first start (the builder's \"Prepare shaders now\" unticked)")
     p.add_argument("--log", metavar="FILE", help="also save build output to a UTF-8 file (replaces the file if it exists)")
     a = p.parse_args()
     a.default_work = p.get_default("work")   # (main: the builder's own work folder, stop_stray_compilers)
@@ -1111,6 +1135,8 @@ def compiled(line):
     # changed; the builder says how much stays and lets it go. Kept in a development checkout)
     if os.path.normcase(work) != os.path.normcase(os.path.abspath(os.path.join(ROOT, "work"))):
         shutil.rmtree(exefs, ignore_errors=True)
+    if not a.no_prepare_shaders and not os.environ.get("HOOKSHOT_NO_PREPARE"):   # (either: left to the first start, as before)
+        prepare_shaders([os.path.join(game_dir, BESIDE[0]), "--prepare-shaders"])
     total_seconds = time.time() - STARTED
     previous = record_times(work, run_files, compile_seconds, total_seconds, modules=plan[1],
                             kind="update" if a.update else "build")

tests/test_prepare_at_build.py

@@ -0,0 +1,68 @@
+"""make_game.py's last step, hookshot --prepare-shaders, relayed to the builder: its stage and progress lines go on, its own
+@@done and @@error don't (the builders read those as the build's end or failure), and a failure leaves the work to the first start.
+
+python -I -B tests/test_prepare_at_build.py
+"""
+import contextlib
+import importlib.util
+import io
+import sys
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+SOURCE = ROOT / "nx2" / "make_game.py"
+spec = importlib.util.spec_from_file_location("make_game_prepare", SOURCE)
+make_game = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(make_game)
+
+
+def relayed(child):
+    out = io.StringIO()
+    with contextlib.redirect_stdout(out):
+        code = make_game.prepare_shaders([sys.executable, "-c", child])
+    return code, out.getvalue().splitlines()
+
+
+class PrepareAtBuildTests(unittest.TestCase):
+    def test_progress_goes_on_and_done_does_not(self):
+        code, lines = relayed("print('hookshot v'); print('@@stage shaders Preparing shaders'); print('@@progress 0.500');"
+                              "print('@@done shaders')")
+        self.assertEqual(code, 0)
+        self.assertEqual(lines, ["hookshot v", "@@stage shaders Preparing shaders", "@@progress 0.500", "done shaders"])
+
+    def test_no_vulkan_is_not_a_failed_build(self):
+        code, lines = relayed("import sys; print('@@error no Vulkan device'); sys.exit(3)")
+        self.assertEqual(code, 3)
+        self.assertFalse(any(l.startswith("@@") for l in lines))
+        self.assertIn("error no Vulkan device", lines)
+        self.assertTrue(lines[-1].startswith("the shaders were not prepared now (exit 3)"))
+
+    def test_missing_game_is_not_a_failed_build(self):
+        out = io.StringIO()
+        with contextlib.redirect_stdout(out):
+            code = make_game.prepare_shaders([str(ROOT / "no-such-folder" / "hookshot.exe"), "--prepare-shaders"])
+        self.assertEqual(code, 1)
+        self.assertNotIn("@@", out.getvalue())
+
+    def test_the_build_runs_it_after_the_game_folder_is_complete(self):
+        text = SOURCE.read_text(encoding="utf-8")
+        main = text[text.index("\ndef main("):]
+        at = main.index("prepare_shaders([os.path.join(game_dir, BESIDE[0]), \"--prepare-shaders\"])")
+        for before in ("os.replace(dst + \".new\", dst)", "shader_census_cmd(", "seed_pipelines(out)"):
+            self.assertLess(main.index(before), at, before)
+        self.assertLess(at, main.index("say(\"done\", out)"))
+
+    def test_the_builders_can_leave_it_to_the_first_start(self):
+        text = SOURCE.read_text(encoding="utf-8")
+        self.assertIn('"--no-prepare-shaders"', text)
+        main = text[text.index("\ndef main("):]
+        guard = main[:main.index("prepare_shaders([os.path.join(game_dir")].rsplit("\n", 2)[-2]
+        self.assertIn("a.no_prepare_shaders", guard)
+        self.assertIn('HOOKSHOT_NO_PREPARE', guard)
+        for builder in ("builder/HookshotBuilder/MainWindow.xaml.cs", "builder/linux/hookshot_builder.py"):
+            self.assertEqual((ROOT / builder).read_text(encoding="utf-8").count("--no-prepare-shaders"), 2, builder)   # (build and update)
+
+
+if __name__ == "__main__":
+    unittest.main()