diff --git a/builder/HookshotBuilder/BuilderSettings.cs b/builder/HookshotBuilder/BuilderSettings.cs new file mode 100644 index 00000000..0b7da667 --- /dev/null +++ b/builder/HookshotBuilder/BuilderSettings.cs @@ -0,0 +1,22 @@ +namespace HookshotBuilder; + +/// What the player chose last time (the install folder, the game files, the shaders' tick): builder-settings.json beside the versions +sealed class BuilderSettings +{ + public string? Out { get; set; } + public string? Nup { get; set; } + public string? Romfs { get; set; } + public string? Exefs { get; set; } + public bool Dlss { get; set; } // (NVIDIA DLSS ticked: NVIDIA's license was accepted) + public bool PrepareLater { get; set; } // ("Prepare shaders now" unticked) + + public static BuilderSettings? Load(string path) + { + try { return System.Text.Json.JsonSerializer.Deserialize(System.IO.File.ReadAllText(path)); } + catch (Exception) { return null; } + } + + /// (the updates, from the launcher's Update and from the --update start after a self-update, begin before ApplySettings sets + /// the tick from these settings: they read the saved choice itself. None saved: prepared, as the tick's default) + public static bool PrepareNow(string path) => Load(path)?.PrepareLater != true; +} diff --git a/builder/HookshotBuilder/MainWindow.xaml b/builder/HookshotBuilder/MainWindow.xaml index 3d281e96..f1ea91fb 100644 --- a/builder/HookshotBuilder/MainWindow.xaml +++ b/builder/HookshotBuilder/MainWindow.xaml @@ -171,6 +171,9 @@ + + diff --git a/builder/HookshotBuilder/MainWindow.xaml.cs b/builder/HookshotBuilder/MainWindow.xaml.cs index 5dfc532d..7afcd26f 100644 --- a/builder/HookshotBuilder/MainWindow.xaml.cs +++ b/builder/HookshotBuilder/MainWindow.xaml.cs @@ -252,23 +252,15 @@ void LaunchPlayClick(object sender, RoutedEventArgs e) void LaunchFolderClick(object sender, RoutedEventArgs e) => Process.Start(new ProcessStartInfo(launchFolder) { UseShellExecute = true }); - // ---- what the player chose last time (the install folder, the game files): builder-settings.json beside the versions ---- - sealed class BuilderSettings - { - public string? Out { get; set; } - public string? Nup { get; set; } - public string? Romfs { get; set; } - public string? Exefs { get; set; } - public bool Dlss { get; set; } // (NVIDIA DLSS ticked: NVIDIA's license was accepted) - } + // ---- what the player chose last time (BuilderSettings.cs): builder-settings.json beside the versions ---- static string SettingsPath => System.IO.Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "HookshotBuilder", "builder-settings.json"); void ApplySettings() { - BuilderSettings? s = null; - try { s = System.Text.Json.JsonSerializer.Deserialize(File.ReadAllText(SettingsPath)); } catch (Exception) { } + var s = BuilderSettings.Load(SettingsPath); if (s == null) return; + PrepareTick.IsChecked = !s.PrepareLater; if (!string.IsNullOrEmpty(s.Out)) OutPath.Text = s.Out; DlssTick.IsChecked = s.Dlss; if (s.Romfs != null && s.Exefs != null && Directory.Exists(s.Romfs) && File.Exists(System.IO.Path.Combine(s.Exefs, "main"))) @@ -293,7 +285,8 @@ void SaveSettings() try { var s = new BuilderSettings { Out = OutPath.Text, Nup = romfs == null ? chosenNup : null, Romfs = romfs, Exefs = exefs, - Dlss = DlssTick.IsChecked == true }; + Dlss = DlssTick.IsChecked == true, + PrepareLater = PrepareTick.IsChecked != true }; Directory.CreateDirectory(System.IO.Path.GetDirectoryName(SettingsPath)!); File.WriteAllText(SettingsPath, System.Text.Json.JsonSerializer.Serialize(s)); } @@ -306,7 +299,7 @@ void Preview(string what) switch (what) { case "build": - MakeRows(update: false); + MakeRows(update: false, shaders: true); SetStep(1); ShowPage(BuildPage); rows["read"].Begin(); rows["read"].Done(); rows["read"].Small("Version checked"); @@ -692,7 +685,8 @@ async void StartBuild(object sender, RoutedEventArgs e) else if (chosenNup != null) args.AddRange(new[] { "--nup", chosenNup }); else return; args.AddRange(new[] { "--out", OutPath.Text, "--language", lang.Code }); - Run(tools, args, dlss: DlssTick.IsChecked == true); + prepareNow = PrepareTick.IsChecked == true; + Run(tools, ShaderStep.Args(args, prepareNow), dlss: DlssTick.IsChecked == true); } const string DlssLicense = @@ -719,17 +713,30 @@ static readonly (string Id, string Build, string Update)[] Stages = ("lift", "Translate the game to PC code", "Translate the game to PC code"), ("compile", "Compile", "Compile changed files"), ("finish", "Put the game together", "Swap in the new game"), + (ShaderStep.Id, "Prepare the shaders", "Prepare the shaders"), }; - static readonly Dictionary Weight = new() + static readonly Dictionary WithShaders = new() + { + ["download"] = (0, 0.04), ["read"] = (0.04, 0.04), ["lift"] = (0.08, 0.14), ["compile"] = (0.22, 0.68), ["finish"] = (0.90, 0.02), + [ShaderStep.Id] = (0.92, 0.08), // (hookshot --prepare-shaders: minutes on some GPUs, the first start's wait moved here) + }; + static readonly Dictionary WithoutShaders = new() { ["download"] = (0, 0.04), ["read"] = (0.04, 0.04), ["lift"] = (0.08, 0.14), ["compile"] = (0.22, 0.74), ["finish"] = (0.96, 0.04), }; + Dictionary Weight = WithoutShaders; + // (this build prepares the shaders: the tick for a build, the saved choice for an update. The game's @@stage shaders came, and + // make_game's line when it left them to the first start: the row's end says which) + bool prepareNow, shadersBegan; + string? shadersLater; + TimeSpan? builtIn; // (at @@times: the build without the shaders' minutes, as build_times.json keeps it) - void MakeRows(bool update) + void MakeRows(bool update, bool shaders) { StageList.Children.Clear(); rows.Clear(); - var list = Stages.Where(s => update || s.Id != "download").ToList(); + Weight = shaders ? WithShaders : WithoutShaders; + var list = Stages.Where(s => (update || s.Id != "download") && (shaders || s.Id != ShaderStep.Id)).ToList(); for (var i = 0; i < list.Count; i++) { var row = new StepRow(i + 1, update ? list[i].Update : list[i].Build, last: i == list.Count - 1); @@ -752,7 +759,7 @@ void Run(Tools tools, List args, bool keepRows = false, bool dlss = fals if (!keepRows) { updating = false; - MakeRows(update: false); + MakeRows(update: false, shaders: prepareNow); SetStep(1); BuildTitle.Text = "Building your game"; UpdateBar.Visibility = Visibility.Collapsed; @@ -769,6 +776,9 @@ void Run(Tools tools, List args, bool keepRows = false, bool dlss = fals compileEta = -1; thisCompile = 0; previousTotal = -1; + shadersBegan = false; + shadersLater = null; + builtIn = null; ShowPage(BuildPage); CancelButton.Visibility = Visibility.Visible; SetProgress(updating ? Weight["read"].From : 0); @@ -795,6 +805,7 @@ void Line(string line) current = id; if (rows.TryGetValue(id, out var row)) row.Begin(); if (id == "compile") compileClock.Restart(); + if (id == ShaderStep.Id) shadersBegan = true; if (Weight.TryGetValue(id, out var w)) SetProgress(w.From); } else if (line.StartsWith("@@progress ")) @@ -813,6 +824,7 @@ void Line(string line) return; } else if (line.StartsWith("@@done ")) finished = line[7..]; + else if (ShaderStep.Reason(line) is { } later) shadersLater = later; else if (FilesLine.Match(line) is { Success: true } m) filesOf = int.Parse(m.Groups[1].Value); else if (FileLine.IsMatch(line)) filesDone++; if (current == "compile" && filesChanged < 0 && filesOf > 0 && rows.TryGetValue("compile", out var c)) @@ -843,6 +855,7 @@ void Number(string line) compileEta = eta; break; case "@@times" when p.Length == 4: + builtIn = elapsed.Elapsed; long.TryParse(p[2], out thisCompile); long.TryParse(p[3], out previousTotal); break; @@ -916,7 +929,12 @@ void Ended(string? failure) runner = null; if (finished.Length > 0) { - foreach (var r in rows.Values) r.Done(); + foreach (var (id, r) in rows) + { + if (id != ShaderStep.Id) r.Done(); + else if (ShaderStep.Skipped(shadersBegan, shadersLater) is { } why) { r.Skipped(); r.Small(why); } + else r.Done(); + } SetProgress(1); ShowReady(); return; @@ -957,7 +975,7 @@ void ShowReady() SetStep(2); ReadyTitle.Text = updating ? "Your game is updated" : "Your game is ready"; var compiledNote = thisCompile > 0 ? $" (compiled in {Span(TimeSpan.FromSeconds(thisCompile))})" : ""; - ReadyNote.Text = $"{(updating ? "Updated" : "Built")} in {Span(elapsed.Elapsed)}{compiledNote}{Against(elapsed.Elapsed)} · {finished}"; + ReadyNote.Text = $"{(updating ? "Updated" : "Built")} in {Span(elapsed.Elapsed)}{compiledNote}{Against(builtIn ?? elapsed.Elapsed)} · {finished}"; if (!updating && Environment.GetEnvironmentVariable("HOOKSHOT_UI_PREVIEW") == null) Finder.Shortcuts(finished, DesktopTick.IsChecked == true, StartTick.IsChecked == true); else if (updating && Environment.GetEnvironmentVariable("HOOKSHOT_UI_PREVIEW") == null) @@ -1132,7 +1150,8 @@ public void BeginUpdate(string folder, string? tag) StepPill.Visibility = Visibility.Collapsed; VersionChip.Visibility = Visibility.Visible; VersionChipText.Text = from != null && from != to ? $"{from} → {to}" : to; - MakeRows(update: true); + prepareNow = BuilderSettings.PrepareNow(SettingsPath); + MakeRows(update: true, shaders: prepareNow); BuildTitle.Text = "Updating your game"; BuildNote.Text = "Getting the update…"; RingBox.Visibility = Visibility.Collapsed; @@ -1173,7 +1192,9 @@ internal void UpdateBuild(Tools tools, string folder) // (a folder built with NVIDIA DLSS, its license accepted then, is built with it again) var dlss = File.Exists(System.IO.Path.Combine(folder, "ngx", "nvngx_dlss.dll")) && File.Exists(System.IO.Path.Combine(Storage.NgxSdk, "LICENSE.txt")); - Run(tools, new List { "--update", folder }, keepRows: true, dlss: dlss); + // (prepareNow: the saved choice, read in BeginUpdate. The launcher's Update and the --update start after a self-update never + // ran ApplySettings, so the tick there is only its default) + Run(tools, ShaderStep.Args(new List { "--update", folder }, prepareNow), keepRows: true, dlss: dlss); } // ================= the window ================= @@ -1288,6 +1309,24 @@ public void Done() time.Foreground = B("InkFaint"); } + /// (not done: left to later, the shaders' step when they were not prepared now) + public void Skipped() + { + if (running) Tick(); + running = false; + watch.Stop(); + disc.Fill = Brushes.Transparent; + disc.Stroke = B("GlassLineStrong"); + disc.StrokeThickness = 1.5; + halo.Fill = Brushes.Transparent; + number.Text = "–"; + number.Foreground = B("InkFaint"); + tick.Visibility = Visibility.Collapsed; + name.Foreground = B("InkFaint"); + name.FontWeight = FontWeights.Medium; + time.Foreground = B("InkFaint"); + } + public void Stopped() { running = false; diff --git a/builder/HookshotBuilder/ShaderStep.cs b/builder/HookshotBuilder/ShaderStep.cs new file mode 100644 index 00000000..e906f006 --- /dev/null +++ b/builder/HookshotBuilder/ShaderStep.cs @@ -0,0 +1,30 @@ +using System.Text.RegularExpressions; + +namespace HookshotBuilder; + +/// The build's last step, the shaders' preparation (nx2/make_game.py prepare_shaders): the argument that leaves it to the game's +/// first start, and what its row says once the build is done +static class ShaderStep +{ + public const string Id = "shaders"; + public const string Later = "--no-prepare-shaders"; + // (make_game.py NOT_PREPARED: a plain line, so a builder that does not know it only shows it) + static readonly Regex NotPrepared = new(@"^the shaders were not prepared now \((.*)\): "); + + /// make_game's arguments with the player's choice (unticked: the game prepares its shaders at its first start) + public static List Args(List args, bool prepare) + { + if (!prepare) args.Add(Later); + return args; + } + + /// Why make_game left the shaders to the first start, when this line says so; else null + public static string? Reason(string line) => NotPrepared.Match(line) is { Success: true } m ? m.Groups[1].Value : null; + + /// The line under the row of a finished build when the shaders were not prepared, or null when they were (a check mark). + /// began: the game's @@stage shaders came; reason: make_game's not-prepared line + public static string? Skipped(bool began, string? reason) => + reason != null ? $"Left to the game's first start: {reason}" + : began ? null + : "Left to the game's first start"; +} diff --git a/builder/linux/hookshot_builder.py b/builder/linux/hookshot_builder.py index a4d235a7..f70cb5ac 100644 --- a/builder/linux/hookshot_builder.py +++ b/builder/linux/hookshot_builder.py @@ -30,7 +30,7 @@ CONFIG = os.path.join(os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser("~/.config"), "hookshot", "builder.ini") STAGES = {"read": "Reading your game files", "lift": "Translating the game to PC code", "compile": "Compiling", - "finish": "Putting the game folder together"} + "finish": "Putting the game folder together", "shaders": "Preparing shaders"} def left_text(seconds): @@ -70,8 +70,7 @@ def play(folder): exe = linux_update.game_exe(folder) if not exe: return - env = {k: v for k, v in os.environ.items() if not k.startswith(("HOOKSHOT_", "PYTHON", "APPDIR", "APPIMAGE", "ARGV0", "OWD"))} - env.pop("LD_LIBRARY_PATH", None) + env = make_game.game_env() # (as make_game's shader preparation and hookshot-launch: the game's own SDL, the system's libraries) env["HOOKSHOT_NO_UPDATE_CHECK"] = "1" subprocess.Popen([exe], cwd=folder, env=env, start_new_session=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) @@ -178,6 +177,7 @@ def __init__(self, launch=None, update=None, fetched=False, tag=None): names = [n for _, n in make_game.LANGUAGES] code = self.cfg.get("language", "en-US") self.language = tk.StringVar(value=dict(make_game.LANGUAGES).get(code, names[0])) + self.prepare = tk.BooleanVar(value=self.cfg.get("prepare", "1") != "0") # (make_game.py's last step: off, at the first start) self.check_text = tk.StringVar() self.space_text = tk.StringVar() self.stage_text = tk.StringVar() @@ -245,6 +245,9 @@ def show_form(self): ttk.Combobox(f, textvariable=self.language, values=[n for _, n in make_game.LANGUAGES], state="readonly").grid( row=row, column=1, sticky="w", **self.pad) row += 1 + ttk.Checkbutton(f, text="Prepare shaders now (unticked: at the game's first start, behind a black window)", + variable=self.prepare).grid(row=row, column=0, columnspan=3, sticky="w", **self.pad) + row += 1 ttk.Label(f, text="Takes about 45 minutes and needs 60 GB.").grid(row=row, column=0, columnspan=3, sticky="w", **self.pad) row += 1 ttk.Label(f, textvariable=self.space_text).grid(row=row, column=0, columnspan=3, sticky="w", **self.pad) @@ -414,7 +417,8 @@ def start_make_update(self, folder): self.status.set("This builder is the latest version.") self.lookup_latest() return - cmd = [sys.executable, "-I", os.path.join(ROOT, "nx2", "make_game.py"), "--update", folder] + cmd = [sys.executable, "-I", os.path.join(ROOT, "nx2", "make_game.py"), "--update", folder] + \ + ([] if self.prepare.get() else ["--no-prepare-shaders"]) # (as the last build chose) self.heading.configure(text="Updating your game") self.stage_text.set("Compiling what changed") self.started = time.time() @@ -458,12 +462,13 @@ def build(self): except ValueError as error: return messagebox.showerror("Hookshot", str(error)) code = next((c for c, n in make_game.LANGUAGES if n == self.language.get()), "en-US") - self.cfg.update(kind=self.kind.get(), nup=self.nup.get(), romfs=self.romfs.get(), exefs=self.exefs.get(), out=out, language=code) + self.cfg.update(kind=self.kind.get(), nup=self.nup.get(), romfs=self.romfs.get(), exefs=self.exefs.get(), out=out, language=code, + prepare="1" if self.prepare.get() else "0") save_config(self.cfg) # (issue 531) Without HOOKSHOT_JOBS or RX_JOBS, make_game chooses as on Windows, from the memory free now; this window chose # fewer, 80% of the total memory at 3.5 GB each, and the CPU sat at 29% cmd = [sys.executable, "-I", os.path.join(ROOT, "nx2", "make_game.py")] + src + ["--out", out, "--language", code] + \ - ([] if jobs is None else ["--jobs", str(jobs)]) + ([] if jobs is None else ["--jobs", str(jobs)]) + ([] if self.prepare.get() else ["--no-prepare-shaders"]) self.show_progress() self.started = time.time() self.proc = subprocess.Popen(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, @@ -482,6 +487,7 @@ def show_progress(self, title="Building"): self.eta = None self.compile_started = None self.times = None + self.shaders_later = False f = self.page = ttk.Frame(self) f.pack(fill="both", expand=True, pady=self.px(8)) f.columnconfigure(0, weight=1) @@ -547,8 +553,9 @@ def on_line(self, line): stage = line.split(" ", 2)[1] self.stage_text.set(STAGES.get(stage, line.split(" ", 2)[-1])) self.bar.configure(mode="determinate", value=0) - if stage == "compile": - self.compile_started = time.time() + # (the compile's clock and countdown are the compile's: a later step, the minutes-long shaders one, without them) + self.compile_started = time.time() if stage == "compile" else None + self.eta = None elif line.startswith("@@progress "): try: self.bar.configure(value=float(line.split()[1])) @@ -591,6 +598,8 @@ def on_line(self, line): self.built = line[7:] elif line.startswith("@@error "): self.error = line[8:] + elif line.startswith(make_game.NOT_PREPARED): + self.shaders_later = True if hasattr(self, "log"): self.log.insert("end", line + "\n") self.log.see("end") @@ -608,6 +617,8 @@ def on_exit(self, code): text += ", compiled in %d:%02d" % (int(compiled) // 60, int(compiled) % 60) if previous > 0: text += " · last time %d:%02d" % (int(previous) // 3600, int(previous) // 60 % 60) + if self.shaders_later: + text += " · the shaders are prepared at its first start" self.stage_text.set(text) self.bar.configure(value=1.0) try: diff --git a/nx2/make_game.py b/nx2/make_game.py index ec733289..aff5ee06 100644 --- a/nx2/make_game.py +++ b/nx2/make_game.py @@ -17,6 +17,9 @@ @@check (--check) the answer @@done finished @@error stopped + +and one plain line the builders also read (older ones only show it): "the shaders were not prepared now (): ..." when the +build's last step, the shaders' preparation, was skipped or failed and is left to the game's first start (NOT_PREPARED) """ import argparse import contextlib @@ -24,10 +27,12 @@ import hashlib import json import os +import queue import re import shutil import stat import subprocess +import threading import time import sys import tempfile @@ -214,19 +219,65 @@ def clock(seconds): return "%d:%02d:%02d" % (s // 3600, s // 60 % 60, s % 60) -def run(cmd, env=None, on_line=None, check=True, output=None): - e = dict(os.environ, PYTHONIOENCODING="utf-8") +class Silent(Exception): + """a step printed nothing for longer than run()'s silence""" + + +def output_lines(p, silence): + """A step's lines as they come; with silence (seconds), read on a thread so that a step which goes quiet that long (a hung + driver, a crash dialog left open) raises Silent instead of waiting for ever""" + if silence is None: + yield from p.stdout + return + lines = queue.Queue() + + def read(): + try: + for line in p.stdout: + lines.put(line) + except (OSError, ValueError): + pass + finally: + with contextlib.suppress(OSError, ValueError): + p.stdout.close() # (here, not by run(): a stopped step's pipe may still be read by this thread) + lines.put(None) + + threading.Thread(target=read, daemon=True).start() + while True: + try: + line = lines.get(timeout=silence) + except queue.Empty: + raise Silent() from None + if line is None: + return + yield line + + +def run(cmd, env=None, on_line=None, check=True, output=None, show=None, cwd=ROOT, base=None, silence=None): + """A step, its output printed as it comes. show: what of each line is printed (its None: nothing; on_line and a failure's + reason still get the line itself). base: the environment it starts from (os.environ). silence: seconds without a line after + which the step is stopped; it returns None then (with check, the build fails)""" + e = dict(os.environ if base is None else base, PYTHONIOENCODING="utf-8") e.update(env or {}) - p = subprocess.Popen(cmd, cwd=ROOT, env=e, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", + p = subprocess.Popen(cmd, cwd=cwd, env=e, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", bufsize=1) no_throttle(getattr(p, "_handle", None)) lines = [] - for line in p.stdout: - line = line.rstrip() - print(line, file=output, flush=True) - lines.append(line) - if on_line: - on_line(line) + try: + for line in output_lines(p, silence): + line = line.rstrip() + shown = show(line) if show else line + if shown is not None: + print(shown, file=output, flush=True) + lines.append(line) + if on_line: + on_line(line) + except Silent: + p.kill() + p.wait() + if check: + fail("%s printed nothing for %s and was stopped" % (os.path.basename(cmd[0]), span(silence))) + return None p.stdout.close() code = p.wait() if code and check: @@ -755,6 +806,69 @@ def ship_state_list(out): print("the shipped pipeline state list: %d pipelines" % ((os.path.getsize(src) - 12) // 528), flush=True) +# (the builders read this line's start: the step was skipped or failed, and the shaders' row says so instead of a check mark) +NOT_PREPARED = "the shaders were not prepared now" +# (a run that prints nothing this long is stopped. Not its whole length: that is 25 minutes on some AMD GPUs, and the driver's own +# step near the end can sit at 99% for minutes) +PREPARE_SILENCE = 15 * 60 +GAME_ENV_DROP = ("HOOKSHOT_", "PYTHON", "APPDIR", "APPIMAGE", "ARGV0", "OWD") + + +def span(seconds): + return "%d minutes" % (seconds // 60) if seconds >= 120 else "%g seconds" % seconds + + +def game_env(environ=None): + """The environment the game runs in when the tools start it: without what the AppImage runtime and the tools put there (their + libraries first on the search path, Python's settings, the HOOKSHOT_ build switches), so it loads its own SDL and the system's + Vulkan, as when the player starts it. The Linux builder's Play button uses it too; builder/linux/hookshot-launch drops the same""" + env = {k: v for k, v in (os.environ if environ is None else environ).items() if not k.startswith(GAME_ENV_DROP)} + env.pop("LD_LIBRARY_PATH", None) + return env + + +def not_prepared(why): + print("%s (%s): the game prepares them at its first start" % (NOT_PREPARED, why), flush=True) + + +def prepare_skipped(a, environ=None): + """Why the build leaves the shaders to the first start, or None: it prepares them""" + environ = os.environ if environ is None else environ + if a.no_prepare_shaders: + return '"Prepare shaders now" is unticked' + if environ.get("HOOKSHOT_NO_PREPARE", "0") != "0": # (set and not 0, as the game reads HOOKSHOT_NO_UPDATE_CHECK) + return "HOOKSHOT_NO_PREPARE is set" + if CROSS: # (hookshot.exe can't run here, and under Wine it would prepare for this PC's GPU, not the one that plays it) + return "a Windows game built on Linux prepares them on the PC it runs on" + return None + + +def show_prepared(line): + """The game's @@stage and @@progress lines go on to the builder; its own @@done and @@error would end or fail the build there, + so they go on as plain text""" + return line if line.startswith(("@@stage ", "@@progress ")) or not line.startswith("@@") else line[2:] + + +def prepare_shaders(cmd, folder=None, silence=PREPARE_SILENCE): + """The shaders prepared now, under the builder's progress bar, not at the game's first start behind a black window (minutes on + some GPUs, issue 711): hookshot --prepare-shaders (host/hos_main.c, gpu/vk/vk_main.cpp), the same work as that first start, in + the game's folder and environment, as the builders' Play starts it. A PC whose Vulkan can't (exit 3), a run silent for `silence` seconds (then stopped), or + any other failure leaves the work to the first start, as before: never a failed build. Returns its exit code (None: stopped)""" + try: + code = run(cmd, check=False, show=show_prepared, cwd=folder or os.path.dirname(cmd[0]) or None, + base=game_env(), silence=silence) + except OSError as e: + not_prepared(str(e)) + return 1 + if code is None: + not_prepared("the game printed nothing for %s and was stopped" % span(silence)) + elif code == 3: + not_prepared("no usable Vulkan here") + elif code: + not_prepared("the game stopped with exit status %d" % code) + return code + + def folder_language(out): """the language a game folder was built in: the code line of its language.txt""" try: @@ -936,6 +1050,8 @@ def parse_args(): p.add_argument("--language", default="en-US", help="the game's language: a code from LANGUAGES, written to language.txt") p.add_argument("--check", action="store_true") p.add_argument("--update", help="a game folder built before: built again from the game files it keeps (the game's update offer)") + p.add_argument("--no-prepare-shaders", action="store_true", + help="leave the shaders' preparation to the game's first start (the builder's \"Prepare shaders now\" unticked)") p.add_argument("--log", metavar="FILE", help="also save build output to a UTF-8 file (replaces the file if it exists)") # (NVIDIA DLSS, off unless asked: NVIDIA's SDK into HOOKSHOT_NGX_SDK first, under NVIDIA's license. scripts/ngx_sdk.py) p.add_argument("--fetch-ngx-sdk", action="store_true", help="download NVIDIA's DLSS SDK into HOOKSHOT_NGX_SDK before the build") @@ -1242,6 +1358,12 @@ def compiled(line): print("the build took %s, compiled in %s%s" % (clock(total_seconds), clock(compile_seconds), "" if previous is None else " (the last one %s)" % clock(previous)), flush=True) say("times", "%.0f" % total_seconds, "%.0f" % compile_seconds, "-1" if previous is None else "%.0f" % previous) + # (after the times: they stay comparable from build to build, without a GPU's minutes that can be switched off) + skipped = prepare_skipped(a) + if skipped: + not_prepared(skipped) + else: + prepare_shaders([os.path.join(game_dir, BESIDE[0]), "--prepare-shaders"], out) say("done", out) diff --git a/tests/builder_workers/BuilderWorkersTests.csproj b/tests/builder_workers/BuilderWorkersTests.csproj index 9d591d12..f89d1579 100644 --- a/tests/builder_workers/BuilderWorkersTests.csproj +++ b/tests/builder_workers/BuilderWorkersTests.csproj @@ -9,5 +9,7 @@ + + diff --git a/tests/builder_workers/Program.cs b/tests/builder_workers/Program.cs index 6842544f..9d1c4087 100644 --- a/tests/builder_workers/Program.cs +++ b/tests/builder_workers/Program.cs @@ -50,6 +50,35 @@ static async Task EmptyParent(string python, Tools tools) Require(value.GetProperty("jobs").GetString() == "", "Automatic changed an inherited empty entry."); } + /// The shaders' step: the update's argument from the saved choice (the launcher's Update and the --update start read it, not + /// the tick), and what the row says when make_game left the shaders to the first start + static void ShaderStepChecks(string root) + { + var settings = Path.Combine(root, "builder-settings.json"); + Require(BuilderSettings.PrepareNow(settings), "No saved settings must prepare the shaders, as the tick's default."); + File.WriteAllText(settings, "not json"); + Require(BuilderSettings.PrepareNow(settings), "Unreadable settings must prepare the shaders."); + File.WriteAllText(settings, """{"Out":"authored output","PrepareLater":true}"""); + Require(!BuilderSettings.PrepareNow(settings), "A saved unticked choice must leave the shaders to the first start."); + Require(ShaderStep.Args(new List { "--update", "authored game" }, BuilderSettings.PrepareNow(settings)) + .SequenceEqual(new[] { "--update", "authored game", "--no-prepare-shaders" }), "The update ignored the saved choice."); + File.WriteAllText(settings, """{"Out":"authored output","PrepareLater":false}"""); + Require(ShaderStep.Args(new List { "--update", "authored game" }, BuilderSettings.PrepareNow(settings)) + .SequenceEqual(new[] { "--update", "authored game" }), "A saved ticked choice must prepare the shaders."); + + Require(ShaderStep.Reason("the shaders were not prepared now (no usable Vulkan here): the game prepares them at its first start") + == "no usable Vulkan here", "make_game's not-prepared line was not read."); + Require(ShaderStep.Reason("the shaders were not prepared now (\"Prepare shaders now\" is unticked (here)): the game prepares them at its first start") + == "\"Prepare shaders now\" is unticked (here)", "A reason with brackets was cut."); + Require(ShaderStep.Reason("@@stage shaders Preparing shaders") is null && ShaderStep.Reason("done shaders") is null, + "Other lines are not a skipped step."); + Require(ShaderStep.Skipped(began: true, reason: null) is null, "Prepared shaders must get their check mark."); + Require(ShaderStep.Skipped(began: false, reason: null) is { } none && none.Contains("first start"), + "No @@stage shaders (an older make_game, or none ran) must not be a check mark."); + Require(ShaderStep.Skipped(began: true, reason: "the game printed nothing for 15 minutes and was stopped") is { } stopped + && stopped.Contains("15 minutes"), "A stopped preparation must say why under its row."); + } + static async Task Main(string[] args) { if (args.Length is < 1 or > 2) throw new Exception("Pass the Python executable for the authored child."); @@ -74,6 +103,7 @@ static async Task Main(string[] args) return; } + ShaderStepChecks(root); Require(BuildWorkers.Choices(4).SequenceEqual(new object[] { BuildWorkers.Automatic, 1, 2, 3, 4 }), "The menu must offer Automatic first and one count per logical processor."); Require(BuildWorkers.Choices(0).SequenceEqual(new object[] { BuildWorkers.Automatic, 1 }), "CPU fallback must still offer one worker."); diff --git a/tests/test_linux_builder_jobs.py b/tests/test_linux_builder_jobs.py index 024499a7..c898f243 100644 --- a/tests/test_linux_builder_jobs.py +++ b/tests/test_linux_builder_jobs.py @@ -42,7 +42,7 @@ def __init__(self, folder, environ, kind="nup"): exefs.mkdir(exist_ok=True) self.gui = SimpleNamespace( out=value(str(folder / "out")), kind=value(kind), nup=value(str(nup)), - romfs=value(str(romfs)), exefs=value(str(exefs)), language=value("English (US)"), + romfs=value(str(romfs)), exefs=value(str(exefs)), language=value("English (US)"), prepare=value(True), cfg={"previous": "kept"}, show_progress=lambda: self.events.append(("progress",)), read=lambda proc: None) def start_process(command, **kwargs): @@ -90,7 +90,8 @@ def assert_started(self, harness, jobs=None): self.assertEqual(options, dict(cwd=str(ROOT), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, encoding="utf-8", errors="replace", start_new_session=True)) self.assertEqual(harness.events[0][1], dict(previous="kept", kind=gui.kind.get(), nup=gui.nup.get(), - romfs=gui.romfs.get(), exefs=gui.exefs.get(), out=gui.out.get(), language="en-US")) + romfs=gui.romfs.get(), exefs=gui.exefs.get(), out=gui.out.get(), language="en-US", + prepare="1")) self.assertEqual(harness.events[3][1], dict(target=gui.read, args=(gui.proc,), daemon=True)) def test_positive_hookshot_jobs_is_passed_on(self): @@ -99,6 +100,14 @@ def test_positive_hookshot_jobs_is_passed_on(self): self.assert_started(harness, 4) self.assertEqual(harness.environ, {"HOOKSHOT_JOBS": "4", "RX_JOBS": "2"}) + def test_unticked_prepare_leaves_the_shaders_to_the_first_start(self): + harness = BuildHarness(self.folder, {}) + harness.gui.prepare = value(False) + harness.run() + command, _ = harness.processes[0] + self.assertEqual(command[-1], "--no-prepare-shaders") + self.assertEqual(harness.events[0][1]["prepare"], "0") + def test_zero_falls_back_to_rx_jobs_for_extracted_folders(self): harness = BuildHarness(self.folder, {"HOOKSHOT_JOBS": "0", "RX_JOBS": "2"}, kind="folders") harness.run() diff --git a/tests/test_prepare_at_build.py b/tests/test_prepare_at_build.py new file mode 100644 index 00000000..5d52366a --- /dev/null +++ b/tests/test_prepare_at_build.py @@ -0,0 +1,195 @@ +"""make_game.py's last step, hookshot --prepare-shaders, run against stub games: its stage and progress lines go on to the builder, +its own @@done and @@error don't (the builders read those as the build's end or failure), a failure or a silent hang leaves the work +to the first start (never a failed build), and the game runs in its own environment, as the builders' Play starts it. + +python -I -B tests/test_prepare_at_build.py +""" +import ast +import contextlib +import importlib.util +import io +import json +import os +import re +import sys +import tempfile +import time +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "nx2" / "make_game.py" +spec = importlib.util.spec_from_file_location("make_game_prepare", SOURCE) +make_game = importlib.util.module_from_spec(spec) +spec.loader.exec_module(make_game) +BUILDER = ROOT / "builder" / "linux" / "hookshot_builder.py" + + +def stub(code): + """A stand-in for hookshot --prepare-shaders: authored Python, no game, no GPU""" + return [sys.executable, "-I", "-c", code] + + +def prepared(cmd, **kwargs): + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = make_game.prepare_shaders(cmd, **kwargs) + return code, out.getvalue().splitlines() + + +def not_prepared(lines): + return [l for l in lines if l.startswith(make_game.NOT_PREPARED)] + + +class PrepareShadersTests(unittest.TestCase): + def test_progress_goes_on_and_the_games_done_does_not(self): + code, lines = prepared(stub("print('hookshot v'); print('@@stage shaders Preparing shaders'); print('@@progress 0.500');" + "print('@@done shaders')")) + self.assertEqual(code, 0) + self.assertEqual(lines, ["hookshot v", "@@stage shaders Preparing shaders", "@@progress 0.500", "done shaders"]) + + def test_no_vulkan_leaves_them_to_the_first_start(self): + code, lines = prepared(stub("import sys; print('@@error no usable Vulkan here'); sys.exit(3)")) + self.assertEqual(code, 3) + self.assertFalse(any(l.startswith("@@") for l in lines)) + self.assertIn("error no usable Vulkan here", lines) + self.assertEqual(len(not_prepared(lines)), 1) + self.assertIn("(no usable Vulkan here)", lines[-1]) + + def test_a_crash_leaves_them_to_the_first_start(self): + code, lines = prepared(stub("import sys; print('@@stage shaders Preparing shaders'); sys.exit(5)")) + self.assertEqual(code, 5) + self.assertIn("exit status 5", not_prepared(lines)[0]) + + def test_a_silent_hang_is_stopped_and_not_a_failed_build(self): + # (prints its stage, then nothing: as a driver deadlock or a crash dialog left open. silence is 15 minutes in a build) + started = time.monotonic() + code, lines = prepared(stub("import time; print('@@stage shaders Preparing shaders', flush=True); time.sleep(120)"), + silence=1) + self.assertLess(time.monotonic() - started, 30) + self.assertIsNone(code) + self.assertEqual(lines[0], "@@stage shaders Preparing shaders") + self.assertIn("printed nothing for 1 seconds and was stopped", not_prepared(lines)[0]) + + def test_steady_output_is_not_silence(self): + code, lines = prepared(stub("import time\nfor i in range(6):\n print('@@progress %.3f' % (i / 6), flush=True); " + "time.sleep(0.4)"), silence=1) + self.assertEqual(code, 0) + self.assertEqual(not_prepared(lines), []) + + def test_missing_game_is_not_a_failed_build(self): + code, lines = prepared([str(ROOT / "no-such-folder" / "hookshot.exe"), "--prepare-shaders"]) + self.assertEqual(code, 1) + self.assertEqual(len(not_prepared(lines)), 1) + self.assertFalse(any(l.startswith("@@") for l in lines)) + + def test_the_game_runs_in_its_folder_without_the_tools_environment(self): + dropped = dict(HOOKSHOT_TOOLS="/builder/tools", HOOKSHOT_NO_PREPARE="0", PYTHONPATH="/builder/python", + PYTHONHOME="/builder", APPDIR="/appdir", APPIMAGE="/tmp/Hookshot.AppImage", ARGV0="Hookshot", OWD="/old", + LD_LIBRARY_PATH="/builder/lib") + with tempfile.TemporaryDirectory(prefix="hookshot-prepare-") as folder, \ + mock.patch.dict(os.environ, dict(dropped, HOOKSHOT_TEST_KEPT_NOT="x", KEPT_FOR_THE_GAME="yes")): + code, lines = prepared(stub("import json, os; print(json.dumps({'cwd': os.getcwd(), 'env': dict(os.environ)}))"), + folder=folder) + seen = json.loads(lines[0]) + self.assertEqual(code, 0) + self.assertEqual(os.path.normcase(os.path.realpath(seen["cwd"])), os.path.normcase(os.path.realpath(folder))) + for name in list(dropped) + ["HOOKSHOT_TEST_KEPT_NOT"]: + self.assertNotIn(name, seen["env"], name) + self.assertEqual(seen["env"]["KEPT_FOR_THE_GAME"], "yes") + + def test_game_env_is_the_play_buttons(self): + env = make_game.game_env({"HOOKSHOT_X": "1", "PYTHONPATH": "p", "APPIMAGE": "a", "APPDIR": "d", "ARGV0": "0", "OWD": "o", + "LD_LIBRARY_PATH": "l", "PATH": "/usr/bin", "HOME": "/home/p"}) + self.assertEqual(env, {"PATH": "/usr/bin", "HOME": "/home/p"}) + # (the shell launcher can't import it: the same names there) + launch = (ROOT / "builder" / "linux" / "hookshot-launch").read_text(encoding="utf-8") + names = re.search(r"\$1 ~ /\^\(([A-Z_|0-9]+)\)/ \|\| \$1 == \"LD_LIBRARY_PATH\"", launch).group(1).split("|") + self.assertEqual(tuple(names), make_game.GAME_ENV_DROP) + + +class RunSilenceTests(unittest.TestCase): + def test_a_silent_step_fails_the_build_when_checked(self): + out = io.StringIO() + with contextlib.redirect_stdout(out), self.assertRaises(SystemExit): + make_game.run(stub("import time; time.sleep(120)"), silence=1) + self.assertIn("@@error", out.getvalue()) + self.assertIn("printed nothing for 1 seconds", out.getvalue()) + + def test_show_filters_what_is_printed_but_not_what_is_seen(self): + out, seen = io.StringIO(), [] + with contextlib.redirect_stdout(out): + code = make_game.run(stub("print('keep'); print('drop')"), on_line=seen.append, + show=lambda line: None if line == "drop" else line.upper()) + self.assertEqual(code, 0) + self.assertEqual(out.getvalue().splitlines(), ["KEEP"]) + self.assertEqual(seen, ["keep", "drop"]) + + +class SkipTests(unittest.TestCase): + def skipped(self, environ, flag=False, cross=False): + with mock.patch.object(make_game, "CROSS", cross): + return make_game.prepare_skipped(SimpleNamespace(no_prepare_shaders=flag), environ) + + def test_prepared_by_default(self): + self.assertIsNone(self.skipped({})) + + def test_the_unticked_box(self): + self.assertIn("unticked", self.skipped({}, flag=True)) + + def test_hookshot_no_prepare_as_the_game_reads_its_switches(self): + self.assertIsNone(self.skipped({"HOOKSHOT_NO_PREPARE": "0"})) + self.assertIsNotNone(self.skipped({"HOOKSHOT_NO_PREPARE": "1"})) + self.assertIsNotNone(self.skipped({"HOOKSHOT_NO_PREPARE": ""})) # (set, as the game's getenv && strcmp(.., "0")) + + def test_a_windows_game_built_on_linux(self): + self.assertIn("Windows game built on Linux", self.skipped({}, cross=True)) + + def test_the_build_times_leave_the_preparation_out(self): + # (one check of main's order: the times are recorded and said before the preparation, which comes before @@done) + main = SOURCE.read_text(encoding="utf-8").split("\ndef main(", 1)[1] + self.assertLess(main.index("record_times("), main.index("prepare_skipped(a)")) + self.assertLess(main.index('say("times"'), main.index("prepare_skipped(a)")) + self.assertLess(main.index("prepare_skipped(a)"), main.index('say("done", out)')) + + +def builder_method(name): + """One method of the Linux builder's window, compiled alone (no Tk)""" + tree = ast.parse(BUILDER.read_text(encoding="utf-8")) + cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "Builder") + node = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == name) + stages = next(n for n in tree.body if isinstance(n, ast.Assign) and getattr(n.targets[0], "id", "") == "STAGES") + namespace = {"make_game": make_game, "time": time} + exec(compile(ast.Module(body=[stages, node], type_ignores=[]), str(BUILDER), "exec"), namespace) + return namespace[name] + + +class LinuxBuilderTests(unittest.TestCase): + def window(self): + text = [] + return SimpleNamespace(stage_text=SimpleNamespace(set=text.append), bar=SimpleNamespace(configure=lambda **k: None), + compile_started=None, eta=None, shaders_later=False), text + + def test_a_later_stage_clears_the_compiles_clock_and_countdown(self): + on_line = builder_method("on_line") + w, text = self.window() + on_line(w, "@@stage compile Compiling") + on_line(w, "@@eta 300") + self.assertIsNotNone(w.compile_started) + self.assertEqual(w.eta, 300) + on_line(w, "@@stage shaders Preparing shaders") + self.assertIsNone(w.compile_started) + self.assertIsNone(w.eta) + self.assertEqual(text[-1], "Preparing shaders") + + def test_the_not_prepared_line_is_noted(self): + on_line = builder_method("on_line") + w, _ = self.window() + on_line(w, "the shaders were not prepared now (no usable Vulkan here): the game prepares them at its first start") + self.assertTrue(w.shaders_later) + + +if __name__ == "__main__": + unittest.main()