From 39531be50caaaf5453ace7a255d9a6ca149856c7 Mon Sep 17 00:00:00 2001 From: Jim Hazen Date: Sat, 25 Jul 2026 15:45:41 -0700 Subject: [PATCH 1/8] Resolve zig automatically instead of hard-requiring an exact system match Burrito.wrap/1 used to exit(1) up front if `zig` wasn't on PATH at all, or if it didn't match the pinned version exactly. That's a hard blocker for anyone without zig installed, or whose system zig has moved on (e.g. a rolling-release distro) since Burrito last bumped its pin. Add Burrito.Util.ZigResolver, used by a new ResolveZig fetch step, that resolves a working zig in this order: 1. BURRITO_ZIG_PATH env var, if set (explicit override) 2. a previously downloaded, managed copy at the pinned version 3. system zig on PATH, if its version matches exactly (today's behavior, unchanged, for anyone already set up correctly) 4. otherwise, download the pinned version for the host OS/CPU from ziglang.org, verify its sha256 against a hardcoded checksum, cache it, and use that Only the exact pinned version is ever used to build -- Burrito's own build.zig sources have only ever been compatible with one zig version at a time (see the 0.15 -> 0.16 migration in #221/#225), so this isn't a relaxation of which zig gets used, just of where it has to come from. The resolved path threads through Context (a new zig_bin field) to both System.cmd("zig", ...) call sites in Steps.Build.PackAndBuild, which previously always shelled out to bare "zig" on PATH regardless of what pre_check had verified. --- lib/builder/builder.ex | 5 +- lib/builder/context.ex | 1 + lib/burrito.ex | 24 +-- lib/steps/build/pack_and_build.ex | 8 +- lib/steps/fetch/resolve_zig.ex | 20 +++ lib/util/zig_resolver.ex | 239 ++++++++++++++++++++++++++++++ 6 files changed, 273 insertions(+), 24 deletions(-) create mode 100644 lib/steps/fetch/resolve_zig.ex create mode 100644 lib/util/zig_resolver.ex diff --git a/lib/builder/builder.ex b/lib/builder/builder.ex index 3606116..c4f1611 100644 --- a/lib/builder/builder.ex +++ b/lib/builder/builder.ex @@ -42,7 +42,7 @@ defmodule Burrito.Builder do """ @phases [ - fetch: [Fetch.Init, Fetch.FetchMusl, Fetch.ResolveERTS], + fetch: [Fetch.Init, Fetch.ResolveZig, Fetch.FetchMusl, Fetch.ResolveERTS], patch: [Patch.CopyERTS, Patch.RecompileNIFs], build: [Build.PackAndBuild, Build.CopyRelease] ] @@ -103,7 +103,8 @@ defmodule Burrito.Builder do work_dir: "", self_dir: self_path, extra_build_env: [], - halted: false + halted: false, + zig_bin: "zig" } Log.info(:build, "Burrito is building target: #{target.alias}") diff --git a/lib/builder/context.ex b/lib/builder/context.ex index 3fb689f..a595a18 100644 --- a/lib/builder/context.ex +++ b/lib/builder/context.ex @@ -12,5 +12,6 @@ defmodule Burrito.Builder.Context do field(:self_dir, String.t()) field(:extra_build_env, list({String.t(), String.t()})) field(:halted, boolean()) + field(:zig_bin, String.t()) end end diff --git a/lib/burrito.ex b/lib/burrito.ex index df0f80a..26db8ae 100644 --- a/lib/burrito.ex +++ b/lib/burrito.ex @@ -27,10 +27,14 @@ defmodule Burrito do end defp pre_check() do - if Enum.any?(~w(zig xz), &(System.find_executable(&1) == nil)) do + # NOTE: `zig` is intentionally not checked here. `Burrito.Steps.Fetch.ResolveZig` + # resolves (and, if needed, downloads) a compatible zig per-target during the + # build's fetch phase instead of requiring one on PATH up front -- see its + # moduledoc, and `Burrito.Util.ZigResolver`, for the resolution order. + if System.find_executable("xz") == nil do Log.error( :build, - "You MUST have `zig` and `xz` installed to use Burrito, we couldn't find all of them in your PATH!" + "You MUST have `xz` installed to use Burrito, we couldn't find it in your PATH!" ) exit(1) @@ -42,21 +46,5 @@ defmodule Burrito do "We couldn't find 7z/7zz in your PATH, 7z/7zz is required to build Windows releases. They will fail if you don't fix this!" ) end - - check_zig_version() - end - - defp check_zig_version() do - {res, _} = System.cmd("zig", ["version"]) - version = String.trim(res) |> Version.parse!() - - if version != @zig_version_expected do - Log.error( - :build, - "Your Zig version does not match the one Burrito requires! We need `#{Version.to_string(@zig_version_expected)}`, you have: `#{Version.to_string(version)}`" - ) - - exit(1) - end end end diff --git a/lib/steps/build/pack_and_build.ex b/lib/steps/build/pack_and_build.ex index ea32bad..5846974 100644 --- a/lib/steps/build/pack_and_build.ex +++ b/lib/steps/build/pack_and_build.ex @@ -16,7 +16,7 @@ defmodule Burrito.Steps.Build.PackAndBuild do zig_build_args = ["-Dtarget=#{build_triplet}"] - create_metadata_file(context.self_dir, zig_build_args, context.mix_release) + create_metadata_file(context.self_dir, zig_build_args, context.mix_release, context.zig_bin) # TODO: Why do we need to do this??? # This is to bypass a VERY strange bug inside Linux containers... @@ -36,7 +36,7 @@ defmodule Burrito.Steps.Build.PackAndBuild do Log.info(:step, "Zig build env: #{inspect(build_env)}") build_result = - System.cmd("zig", ["build"] ++ zig_build_args, + System.cmd(context.zig_bin, ["build"] ++ zig_build_args, cd: context.self_dir, env: build_env, into: IO.stream() @@ -66,10 +66,10 @@ defmodule Burrito.Steps.Build.PackAndBuild do Path.join(File.cwd!(), [plugin_path]) end - defp create_metadata_file(self_path, args, release) do + defp create_metadata_file(self_path, args, release, zig_bin) do Log.info(:step, "Generating wrapper metadata file...") - {zig_version_string, 0} = System.cmd("zig", ["version"], cd: self_path) + {zig_version_string, 0} = System.cmd(zig_bin, ["version"], cd: self_path) metadata_map = %{ app_name: Atom.to_string(release.name), diff --git a/lib/steps/fetch/resolve_zig.ex b/lib/steps/fetch/resolve_zig.ex new file mode 100644 index 0000000..d6ae248 --- /dev/null +++ b/lib/steps/fetch/resolve_zig.ex @@ -0,0 +1,20 @@ +defmodule Burrito.Steps.Fetch.ResolveZig do + alias Burrito.Builder.Context + alias Burrito.Builder.Log + alias Burrito.Builder.Step + alias Burrito.Util.ZigResolver + + @behaviour Step + + @impl Step + def execute(%Context{} = context) do + case ZigResolver.resolve() do + {:ok, zig_bin} -> + %Context{context | zig_bin: zig_bin} + + {:error, reason} -> + Log.error(:step, reason) + %Context{context | halted: true} + end + end +end diff --git a/lib/util/zig_resolver.ex b/lib/util/zig_resolver.ex new file mode 100644 index 0000000..1d292a1 --- /dev/null +++ b/lib/util/zig_resolver.ex @@ -0,0 +1,239 @@ +defmodule Burrito.Util.ZigResolver do + @moduledoc """ + Resolves a path to a `zig` executable matching Burrito's pinned version + (see `Burrito.get_versions/0`), in this order: + + 1. `BURRITO_ZIG_PATH` environment variable, if set — an explicit, always-trusted + override. + 2. A previously downloaded, managed copy at the pinned version, if already cached. + 3. The system `zig` on `$PATH`, if its version matches the pinned version exactly — + this preserves today's behavior unchanged for anyone who already has the right + version installed. + 4. Otherwise, download the pinned version for the current host OS/CPU, verify its + checksum, cache it, and use that. + + Only the exact pinned version is ever used to build — Burrito's own `build.zig` + sources have historically only been compatible with one zig version at a time (see + the 0.15 -> 0.16 migration), so accepting "any installed zig" isn't a safe + relaxation of the old check. This resolver removes the need for the *system* to + have zig installed at all, and stops an incompatible system zig from blocking a + build, without weakening which zig version actually gets used. + """ + + alias Burrito.Builder.Log + alias Burrito.Util + + # sha256 checksums for the pinned zig release, one per (os, cpu) Burrito supports + # building on. Sourced from https://ziglang.org/download/index.json for the + # version in Burrito.get_versions().zig -- update alongside that version. + @zig_checksums %{ + {:linux, :x86_64} => "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00", + {:linux, :aarch64} => "ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17", + {:darwin, :x86_64} => "0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7", + {:darwin, :aarch64} => "b23d70deaa879b5c2d486ed3316f7eaa53e84acf6fc9cc747de152450d401489", + {:windows, :x86_64} => "68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e" + } + + @spec resolve() :: {:ok, String.t()} | {:error, String.t()} + def resolve do + expected = Burrito.get_versions().zig + + with :miss <- from_override(), + :miss <- from_managed_cache(expected), + :miss <- from_system_path(expected) do + download_and_cache(expected) + end + end + + @spec zig_version_at(String.t()) :: {:ok, Version.t()} | :error + def zig_version_at(path) do + case System.cmd(path, ["version"]) do + {out, 0} -> {:ok, out |> String.trim() |> Version.parse!()} + _ -> :error + end + rescue + _ -> :error + end + + defp from_override do + case System.get_env("BURRITO_ZIG_PATH") do + nil -> + :miss + + path -> + if File.exists?(path) do + Log.info(:step, "Using BURRITO_ZIG_PATH override: #{path}") + {:ok, path} + else + {:error, "BURRITO_ZIG_PATH is set to `#{path}`, but no file exists there!"} + end + end + end + + defp from_managed_cache(expected) do + path = managed_zig_path(expected) + + if File.exists?(path) do + Log.info(:step, "Using cached managed Zig #{expected}: #{path}") + {:ok, path} + else + :miss + end + end + + defp from_system_path(expected) do + case System.find_executable("zig") do + nil -> + :miss + + path -> + case zig_version_at(path) do + {:ok, ^expected} -> {:ok, path} + _ -> :miss + end + end + end + + defp download_and_cache(expected) do + os = Util.get_current_os() + cpu = Util.get_current_cpu() + + case Map.fetch(@zig_checksums, {os, cpu}) do + :error -> + {:error, + "No managed Zig download is known for #{os}/#{cpu}. Install Zig #{expected} " <> + "yourself and set BURRITO_ZIG_PATH to point at it."} + + {:ok, expected_sha256} -> + Log.info( + :step, + "No compatible Zig found on PATH; fetching managed Zig #{expected} for #{os}/#{cpu}..." + ) + + fetch_and_install(os, cpu, expected, expected_sha256) + end + end + + defp fetch_and_install(os, cpu, version, expected_sha256) do + {:ok, _} = Application.ensure_all_started(:req) + + archive_ext = if os == :windows, do: "zip", else: "tar.xz" + file_name = "zig-#{cpu}-#{zig_os_name(os)}-#{version}.#{archive_ext}" + url = "https://ziglang.org/download/#{version}/#{file_name}" + + Log.info(:step, "Downloading: #{url}") + + resp = + case Util.get_proxy() do + proxy = %{scheme: scheme, host: host, port: port} when scheme in ["http", "https"] -> + Log.info(:step, "Using PROXY: #{proxy}") + proxy = {String.to_atom(scheme), host, port, []} + Req.get!(url, raw: true, connect_options: [proxy: proxy]) + + _ -> + Req.get!(url, raw: true) + end + + cond do + resp.status != 200 -> + {:error, "Failed to download Zig from #{url} (got HTTP #{resp.status})"} + + sha256(resp.body) != expected_sha256 -> + {:error, + "Checksum mismatch for #{file_name}! Expected #{expected_sha256}, got " <> + "#{sha256(resp.body)}. Refusing to use this download."} + + true -> + install(resp.body, archive_ext, os, version) + end + end + + defp install(archive_bytes, archive_ext, os, version) do + extract_dir = + Path.join(System.tmp_dir!(), "burrito-zig-extract-#{:erlang.unique_integer([:positive])}") + + File.mkdir_p!(extract_dir) + + with :ok <- extract(archive_bytes, archive_ext, extract_dir), + {:ok, unpacked_dir} <- sole_entry(extract_dir) do + install_dir = managed_install_dir(version) + File.mkdir_p!(Path.dirname(install_dir)) + File.rm_rf!(install_dir) + File.rename!(unpacked_dir, install_dir) + File.rm_rf!(extract_dir) + + zig_path = managed_zig_path(version) + + if os != :windows do + File.chmod!(zig_path, 0o755) + end + + Log.success(:step, "Installed managed Zig #{version}: #{zig_path}") + {:ok, zig_path} + else + {:error, reason} -> + File.rm_rf!(extract_dir) + {:error, reason} + end + end + + defp extract(bytes, "tar.xz", dest_dir) do + tmp_tar = Path.join(dest_dir, "archive.tar.xz") + File.write!(tmp_tar, bytes) + + case System.cmd("tar", ["-xJf", tmp_tar, "-C", dest_dir]) do + {_, 0} -> + File.rm(tmp_tar) + :ok + + {out, _} -> + {:error, "Failed to extract Zig tarball: #{out}"} + end + end + + defp extract(bytes, "zip", dest_dir) do + tmp_zip = Path.join(dest_dir, "archive.zip") + File.write!(tmp_zip, bytes) + + case :zip.extract(String.to_charlist(tmp_zip), cwd: String.to_charlist(dest_dir)) do + {:ok, _} -> + File.rm(tmp_zip) + :ok + + {:error, reason} -> + {:error, "Failed to extract Zig archive: #{inspect(reason)}"} + end + end + + # The Zig archive always contains exactly one top-level `zig--/` + # directory (the compiler binary plus its supporting `lib/` sources) -- find it + # rather than hardcoding its name, since the compression step wrote other files + # (the archive itself) into the same scratch directory. + defp sole_entry(dir) do + case File.ls!(dir) |> Enum.reject(&(&1 in ["archive.tar.xz", "archive.zip"])) do + [only] -> {:ok, Path.join(dir, only)} + other -> {:error, "Expected exactly one entry in extracted Zig archive, got: #{inspect(other)}"} + end + end + + defp managed_install_dir(version) do + os = Util.get_current_os() + cpu = Util.get_current_cpu() + Path.join([managed_root(), "#{version}-#{os}-#{cpu}"]) + end + + defp managed_zig_path(version) do + exe = if Util.get_current_os() == :windows, do: "zig.exe", else: "zig" + Path.join(managed_install_dir(version), exe) + end + + defp managed_root do + :filename.basedir(:user_cache, "burrito_file_cache") |> to_string() |> Path.join("zig") + end + + defp zig_os_name(:darwin), do: "macos" + defp zig_os_name(:windows), do: "windows" + defp zig_os_name(:linux), do: "linux" + + defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) +end From f144c099618c56e9b304df339a719e041ae3bce7 Mon Sep 17 00:00:00 2001 From: Jim Hazen Date: Sat, 25 Jul 2026 15:51:12 -0700 Subject: [PATCH 2/8] Extract the downloaded Zig archive on the same filesystem as the cache dir File.rename!/2 (like POSIX rename(2)) cannot cross a device boundary. System.tmp_dir!() is frequently a separate filesystem (tmpfs) from the managed cache directory under $XDG_CACHE_HOME -- caught live by a sprite test where /tmp and ~/.cache were on different mounts. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LRBbtQUQJTce1uASHx4en5 --- lib/util/zig_resolver.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/util/zig_resolver.ex b/lib/util/zig_resolver.ex index 1d292a1..5346a18 100644 --- a/lib/util/zig_resolver.ex +++ b/lib/util/zig_resolver.ex @@ -149,8 +149,13 @@ defmodule Burrito.Util.ZigResolver do end defp install(archive_bytes, archive_ext, os, version) do + # Extract as a sibling of the final install location, not under System.tmp_dir!() -- + # that's frequently a separate filesystem (e.g. tmpfs) from the managed cache dir, + # and File.rename!/2 (like POSIX rename(2)) cannot cross a device boundary. + File.mkdir_p!(managed_root()) + extract_dir = - Path.join(System.tmp_dir!(), "burrito-zig-extract-#{:erlang.unique_integer([:positive])}") + Path.join(managed_root(), "extract-#{:erlang.unique_integer([:positive])}") File.mkdir_p!(extract_dir) From 67c7861c08a9ae635f7066c76701f94079e862fb Mon Sep 17 00:00:00 2001 From: Jim Hazen Date: Sat, 25 Jul 2026 15:53:00 -0700 Subject: [PATCH 3/8] Thread the resolved zig path through NIF cross-compilation too Steps.Patch.RecompileNIFs shells out to `make` with CC/CXX/AR/RANLIB set to bare `zig cc`/`zig c++`/`zig ar`/`zig ranlib` strings, executed by make's own subshell -- a third call site relying on `zig` being on PATH, missed by the first pass since it's reached only when a dependency has a NIF (elixir_make in its compilers list). Caught live: the nozig sprite test got past ResolveZig and the wrapper build, then failed recompiling exqlite's NIF with "zig: not found". Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LRBbtQUQJTce1uASHx4en5 --- lib/steps/patch/recompile_nifs.ex | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/steps/patch/recompile_nifs.ex b/lib/steps/patch/recompile_nifs.ex index 30c3f62..9e7816f 100644 --- a/lib/steps/patch/recompile_nifs.ex +++ b/lib/steps/patch/recompile_nifs.ex @@ -29,7 +29,8 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do cflags, cxxflags, nif_env, - nif_make_args + nif_make_args, + context.zig_bin ) end) end @@ -59,7 +60,7 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do end) end - defp maybe_recompile_nif({_, _, false}, _, _, _, _, _, _, _), do: :no_nif + defp maybe_recompile_nif({_, _, false}, _, _, _, _, _, _, _, _), do: :no_nif defp maybe_recompile_nif( {dep, path, true}, @@ -69,7 +70,8 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do extra_cflags, extra_cxxflags, extra_env, - extra_make_args + extra_make_args, + zig_bin ) do dep = Atom.to_string(dep) @@ -94,12 +96,12 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do env: [ {"MIX_APP_PATH", output_priv_dir}, - {"RANLIB", "zig ranlib"}, - {"AR", "zig ar"}, + {"RANLIB", "#{zig_bin} ranlib"}, + {"AR", "#{zig_bin} ar"}, {"CC", - "zig cc -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cflags}"}, + "#{zig_bin} cc -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cflags}"}, {"CXX", - "zig c++ -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cxxflags}"} + "#{zig_bin} c++ -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cxxflags}"} ] ++ erts_env ++ extra_env, into: IO.stream() ) From 590e66dc4db1f4e38f5b9f3aa83a7fe72826b1c0 Mon Sep 17 00:00:00 2001 From: Jim Hazen Date: Sat, 25 Jul 2026 16:49:17 -0700 Subject: [PATCH 4/8] Address review findings: dedupe download logic, fix override validation, concurrency-safe managed install, quote zig_bin in shell strings Four issues from a self-review of the zig-auto-resolve branch, all introduced or exacerbated by this branch (a fifth finding -- Req.get! raising instead of returning {:error, _} -- was left alone since it's an existing FetchMusl convention this branch only followed, not one it introduced or worsened): - Extracted the proxy-aware Req.get! logic (previously duplicated near-verbatim from Fetch.FetchMusl.do_download/2) into a shared Burrito.Util.Downloader.get!/1, used by both FetchMusl and ZigResolver. Also routed ZigResolver's downloaded archive bytes through the existing Burrito.Util.FileCache, same as FetchMusl already does for the musl runtime, instead of a second bespoke "does this already exist" mechanism. - BURRITO_ZIG_PATH now runs through the same zig_version_at/1 check every other resolution path uses, instead of a bare File.exists? (which also passed for directories). A wrong-version or non-zig override now fails clearly at resolution time instead of confusingly deep inside a build step. - The managed-zig install path had a real concurrency gap: the extraction scratch dir was named with :erlang.unique_integer/1 alone, which is unique per-BEAM-instance, not across OS processes, so two concurrent `mix release` invocations could collide; and the final install swap (rm_rf + rename) had no protection against a concurrent installer's directory being deleted out from under it. Now: the scratch dir also includes System.pid(), and the install swap is first-writer-wins -- if a concurrent build already finished installing this exact version, adopt it and discard our own extraction instead of overwriting. - CC/CXX/AR/RANLIB are built as shell command strings that make hands to /bin/sh -c. Before this branch the interpolated value was always the literal word "zig", which can't contain a space; now it's a resolved filesystem path (the managed cache dir under the user's cache home, or a BURRITO_ZIG_PATH override), which can. Added a small POSIX single-quote helper and applied it to zig_bin in all four env vars. Also added a narrow extraction-time check: the one path our own code touches (the sole top-level archive entry we go on to File.rename!) must resolve inside the scratch directory. This is a real but partial mitigation -- it can't retroactively catch a malicious *nested* member written outside the scratch dir during extraction itself, since a post-hoc scan can only see what landed inside the directory it's scanning. The primary defense stays the sha256 checksum verified strictly before extraction ever runs. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LRBbtQUQJTce1uASHx4en5 --- lib/steps/fetch/fetch_musl.ex | 13 +-- lib/steps/patch/recompile_nifs.ex | 17 +++- lib/util/downloader.ex | 25 +++++ lib/util/zig_resolver.ex | 162 +++++++++++++++++++++--------- 4 files changed, 153 insertions(+), 64 deletions(-) create mode 100644 lib/util/downloader.ex diff --git a/lib/steps/fetch/fetch_musl.ex b/lib/steps/fetch/fetch_musl.ex index 5f6fc60..afe0892 100644 --- a/lib/steps/fetch/fetch_musl.ex +++ b/lib/steps/fetch/fetch_musl.ex @@ -4,6 +4,7 @@ defmodule Burrito.Steps.Fetch.FetchMusl do alias Burrito.Builder.Step alias Burrito.Builder.Target + alias Burrito.Util.Downloader alias Burrito.Util.FileCache # Linked against musl libc v1.2.5 @@ -51,19 +52,9 @@ defmodule Burrito.Steps.Fetch.FetchMusl do def execute(context), do: context defp do_download(url, cache_key) do - {:ok, _} = Application.ensure_all_started(:req) Log.info(:step, "Downloading file: #{url}") - resp = - case Burrito.Util.get_proxy() do - proxy = %{scheme: scheme, host: host, port: port} when scheme in ["http", "https"] -> - Log.info(:step, "Using PROXY: #{proxy}") - proxy = {String.to_atom(scheme), host, port, []} - Req.get!(url, raw: true, connect_options: [proxy: proxy]) - - _ -> - Req.get!(url, raw: true) - end + resp = Downloader.get!(url) if resp.status != 200 do raise "Failed to fetch musl runtime: #{url}! (Got #{resp.status}) -- please file an issue! Thanks!" diff --git a/lib/steps/patch/recompile_nifs.ex b/lib/steps/patch/recompile_nifs.ex index 9e7816f..4c87d6f 100644 --- a/lib/steps/patch/recompile_nifs.ex +++ b/lib/steps/patch/recompile_nifs.ex @@ -96,12 +96,12 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do env: [ {"MIX_APP_PATH", output_priv_dir}, - {"RANLIB", "#{zig_bin} ranlib"}, - {"AR", "#{zig_bin} ar"}, + {"RANLIB", "#{shell_quote(zig_bin)} ranlib"}, + {"AR", "#{shell_quote(zig_bin)} ar"}, {"CC", - "#{zig_bin} cc -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cflags}"}, + "#{shell_quote(zig_bin)} cc -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cflags}"}, {"CXX", - "#{zig_bin} c++ -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cxxflags}"} + "#{shell_quote(zig_bin)} c++ -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cxxflags}"} ] ++ erts_env ++ extra_env, into: IO.stream() ) @@ -163,4 +163,13 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do {"ERTS_INCLUDE_DIR", erts_include} ] end + + # CC/CXX/AR/RANLIB above are shell command *strings* that `make` hands to + # /bin/sh -c -- zig_bin is a resolved filesystem path (the managed Zig cache + # dir, or a user-supplied BURRITO_ZIG_PATH), not a fixed literal like the + # bare "zig" this used to be, so it can contain spaces or other characters + # that would otherwise split into unintended argv words. + defp shell_quote(path) do + "'" <> String.replace(path, "'", "'\\''") <> "'" + end end diff --git a/lib/util/downloader.ex b/lib/util/downloader.ex new file mode 100644 index 0000000..a6cf9dc --- /dev/null +++ b/lib/util/downloader.ex @@ -0,0 +1,25 @@ +defmodule Burrito.Util.Downloader do + @moduledoc """ + Shared HTTP GET used by Burrito's own download steps (musl runtime, managed Zig, + ...) -- proxy-aware, raw bytes. Raises on transport failure, same as the plain + `Req.get!/2` it wraps; callers are responsible for checking `resp.status`. + """ + + alias Burrito.Builder.Log + alias Burrito.Util + + @spec get!(String.t()) :: Req.Response.t() + def get!(url) do + {:ok, _} = Application.ensure_all_started(:req) + + case Util.get_proxy() do + proxy = %{scheme: scheme, host: host, port: port} when scheme in ["http", "https"] -> + Log.info(:step, "Using PROXY: #{proxy}") + proxy = {String.to_atom(scheme), host, port, []} + Req.get!(url, raw: true, connect_options: [proxy: proxy]) + + _ -> + Req.get!(url, raw: true) + end + end +end diff --git a/lib/util/zig_resolver.ex b/lib/util/zig_resolver.ex index 5346a18..089e07f 100644 --- a/lib/util/zig_resolver.ex +++ b/lib/util/zig_resolver.ex @@ -3,8 +3,9 @@ defmodule Burrito.Util.ZigResolver do Resolves a path to a `zig` executable matching Burrito's pinned version (see `Burrito.get_versions/0`), in this order: - 1. `BURRITO_ZIG_PATH` environment variable, if set — an explicit, always-trusted - override. + 1. `BURRITO_ZIG_PATH` environment variable, if set — an explicit override. Still + validated (must exist and report the pinned version) so a typo or wrong-version + override fails clearly here instead of confusingly deep in a build step. 2. A previously downloaded, managed copy at the pinned version, if already cached. 3. The system `zig` on `$PATH`, if its version matches the pinned version exactly — this preserves today's behavior unchanged for anyone who already has the right @@ -22,6 +23,8 @@ defmodule Burrito.Util.ZigResolver do alias Burrito.Builder.Log alias Burrito.Util + alias Burrito.Util.Downloader + alias Burrito.Util.FileCache # sha256 checksums for the pinned zig release, one per (os, cpu) Burrito supports # building on. Sourced from https://ziglang.org/download/index.json for the @@ -38,7 +41,7 @@ defmodule Burrito.Util.ZigResolver do def resolve do expected = Burrito.get_versions().zig - with :miss <- from_override(), + with :miss <- from_override(expected), :miss <- from_managed_cache(expected), :miss <- from_system_path(expected) do download_and_cache(expected) @@ -55,17 +58,32 @@ defmodule Burrito.Util.ZigResolver do _ -> :error end - defp from_override do + defp from_override(expected) do case System.get_env("BURRITO_ZIG_PATH") do nil -> :miss path -> - if File.exists?(path) do - Log.info(:step, "Using BURRITO_ZIG_PATH override: #{path}") - {:ok, path} - else - {:error, "BURRITO_ZIG_PATH is set to `#{path}`, but no file exists there!"} + cond do + not File.regular?(path) -> + {:error, "BURRITO_ZIG_PATH is set to `#{path}`, but no file exists there!"} + + true -> + case zig_version_at(path) do + {:ok, ^expected} -> + Log.info(:step, "Using BURRITO_ZIG_PATH override: #{path}") + {:ok, path} + + {:ok, other} -> + {:error, + "BURRITO_ZIG_PATH points at Zig #{other}, but Burrito requires exactly " <> + "#{expected}!"} + + :error -> + {:error, + "BURRITO_ZIG_PATH is set to `#{path}`, but it doesn't look like a working " <> + "`zig` binary!"} + end end end end @@ -115,66 +133,62 @@ defmodule Burrito.Util.ZigResolver do end defp fetch_and_install(os, cpu, version, expected_sha256) do - {:ok, _} = Application.ensure_all_started(:req) - archive_ext = if os == :windows, do: "zip", else: "tar.xz" file_name = "zig-#{cpu}-#{zig_os_name(os)}-#{version}.#{archive_ext}" url = "https://ziglang.org/download/#{version}/#{file_name}" + cache_key = :crypto.hash(:sha, url) |> Base.encode16() - Log.info(:step, "Downloading: #{url}") - - resp = - case Util.get_proxy() do - proxy = %{scheme: scheme, host: host, port: port} when scheme in ["http", "https"] -> - Log.info(:step, "Using PROXY: #{proxy}") - proxy = {String.to_atom(scheme), host, port, []} - Req.get!(url, raw: true, connect_options: [proxy: proxy]) + archive_bytes = + case FileCache.fetch(cache_key) do + {:hit, data} -> + Log.info(:step, "Found matching cached Zig download, using that") + data _ -> - Req.get!(url, raw: true) + download(url, cache_key) end - cond do - resp.status != 200 -> - {:error, "Failed to download Zig from #{url} (got HTTP #{resp.status})"} + if sha256(archive_bytes) != expected_sha256 do + {:error, + "Checksum mismatch for #{file_name}! Expected #{expected_sha256}, got " <> + "#{sha256(archive_bytes)}. Refusing to use this download."} + else + install(archive_bytes, archive_ext, os, version) + end + end - sha256(resp.body) != expected_sha256 -> - {:error, - "Checksum mismatch for #{file_name}! Expected #{expected_sha256}, got " <> - "#{sha256(resp.body)}. Refusing to use this download."} + defp download(url, cache_key) do + Log.info(:step, "Downloading: #{url}") + resp = Downloader.get!(url) - true -> - install(resp.body, archive_ext, os, version) + if resp.status != 200 do + raise "Failed to download Zig from #{url} (got HTTP #{resp.status})" end + + FileCache.put_if_not_exist(cache_key, resp.body) + resp.body end defp install(archive_bytes, archive_ext, os, version) do - # Extract as a sibling of the final install location, not under System.tmp_dir!() -- - # that's frequently a separate filesystem (e.g. tmpfs) from the managed cache dir, - # and File.rename!/2 (like POSIX rename(2)) cannot cross a device boundary. File.mkdir_p!(managed_root()) + # PID + a per-VM unique integer: collision-resistant across concurrent OS + # processes too, not just within one BEAM instance (a bare + # :erlang.unique_integer/1 is only unique per-VM, so two `mix release` + # processes started at the same moment could otherwise pick the same name). extract_dir = - Path.join(managed_root(), "extract-#{:erlang.unique_integer([:positive])}") + Path.join( + managed_root(), + "extract-#{System.pid()}-#{:erlang.unique_integer([:positive])}" + ) File.mkdir_p!(extract_dir) with :ok <- extract(archive_bytes, archive_ext, extract_dir), {:ok, unpacked_dir} <- sole_entry(extract_dir) do - install_dir = managed_install_dir(version) - File.mkdir_p!(Path.dirname(install_dir)) - File.rm_rf!(install_dir) - File.rename!(unpacked_dir, install_dir) + result = place_install(unpacked_dir, version, os) File.rm_rf!(extract_dir) - - zig_path = managed_zig_path(version) - - if os != :windows do - File.chmod!(zig_path, 0o755) - end - - Log.success(:step, "Installed managed Zig #{version}: #{zig_path}") - {:ok, zig_path} + result else {:error, reason} -> File.rm_rf!(extract_dir) @@ -182,6 +196,40 @@ defmodule Burrito.Util.ZigResolver do end end + # First writer wins: if a concurrent build already finished installing this + # exact version while we were downloading/extracting, don't delete or + # overwrite a directory another process may already be executing `zig` out + # of -- just adopt the existing install and discard our own extraction. + defp place_install(unpacked_dir, version, os) do + install_dir = managed_install_dir(version) + zig_path = managed_zig_path(version) + + if File.exists?(zig_path) do + Log.info( + :step, + "Managed Zig #{version} was already installed by a concurrent build, using that" + ) + else + File.mkdir_p!(Path.dirname(install_dir)) + + case File.rename(unpacked_dir, install_dir) do + :ok -> + if os != :windows, do: File.chmod!(zig_path, 0o755) + + {:error, _reason} -> + # Lost a race with a concurrent installer between the check above and + # this rename. If the winner's install landed, use it; otherwise this + # really is a failure. + unless File.exists?(zig_path) do + raise "Failed to install Zig #{version} to #{install_dir}" + end + end + end + + Log.success(:step, "Installed managed Zig #{version}: #{zig_path}") + {:ok, zig_path} + end + defp extract(bytes, "tar.xz", dest_dir) do tmp_tar = Path.join(dest_dir, "archive.tar.xz") File.write!(tmp_tar, bytes) @@ -213,11 +261,27 @@ defmodule Burrito.Util.ZigResolver do # The Zig archive always contains exactly one top-level `zig--/` # directory (the compiler binary plus its supporting `lib/` sources) -- find it # rather than hardcoding its name, since the compression step wrote other files - # (the archive itself) into the same scratch directory. + # (the archive itself) into the same scratch directory. Also confirm that entry + # actually resolves inside dest_dir: the checksum gate in fetch_and_install/4 + # already guards against a tampered archive reaching extraction at all, but this + # is a cheap second check specifically on the one path our own code goes on to + # `File.rename!/2` -- it doesn't (and, short of a dedicated safe-tar-extraction + # library, can't after the fact) catch a malicious *nested* member written + # outside dest_dir during extraction itself. defp sole_entry(dir) do case File.ls!(dir) |> Enum.reject(&(&1 in ["archive.tar.xz", "archive.zip"])) do - [only] -> {:ok, Path.join(dir, only)} - other -> {:error, "Expected exactly one entry in extracted Zig archive, got: #{inspect(other)}"} + [only] -> + entry_path = Path.join(dir, only) + expanded_dir = Path.expand(dir) + + if String.starts_with?(Path.expand(entry_path), expanded_dir <> "/") do + {:ok, entry_path} + else + {:error, "Extracted Zig archive's top-level entry escaped the extraction directory"} + end + + other -> + {:error, "Expected exactly one entry in extracted Zig archive, got: #{inspect(other)}"} end end From b3fc08be4ed9952a061525ec211169f3d79f67e6 Mon Sep 17 00:00:00 2001 From: Jim Hazen Date: Sat, 25 Jul 2026 16:50:37 -0700 Subject: [PATCH 5/8] Dedupe the third copy of the proxy-aware download block too --- lib/util/default_erts_resolver.ex | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/lib/util/default_erts_resolver.ex b/lib/util/default_erts_resolver.ex index 8185c99..e17b5eb 100644 --- a/lib/util/default_erts_resolver.ex +++ b/lib/util/default_erts_resolver.ex @@ -3,6 +3,7 @@ defmodule Burrito.Util.DefaultERTSResolver do alias Burrito.Builder.Log alias Burrito.Util + alias Burrito.Util.Downloader alias Burrito.Util.FileCache alias Burrito.Util.ERTSResolver alias Burrito.Util.ERTSUniversalMachineFetcher @@ -96,19 +97,9 @@ defmodule Burrito.Util.DefaultERTSResolver do end defp do_download(url, cache_key) do - {:ok, _} = Application.ensure_all_started(:req) Log.info(:step, "Downloading file: #{url}") - resp = - case Burrito.Util.get_proxy() do - proxy = %{scheme: scheme, host: host, port: port} when scheme in ["http", "https"] -> - Log.info(:step, "Using PROXY: #{proxy}") - proxy = {String.to_atom(scheme), host, port, []} - Req.get!(url, raw: true, connect_options: [proxy: proxy]) - - _ -> - Req.get!(url, raw: true) - end + resp = Downloader.get!(url) if resp.status != 200 do raise "Failed to fetch #{url}! (Got #{resp.status}) Perhaps we haven't built a pre-compiled Erlang for this release yet? If this was a 404, please file an issue! Thanks!" From 01cc2629bec4729689dbaa520d9e474f9aa66e88 Mon Sep 17 00:00:00 2001 From: Jim Hazen Date: Sat, 25 Jul 2026 16:52:14 -0700 Subject: [PATCH 6/8] Trim inline comments to match this codebase's terser house style --- lib/steps/patch/recompile_nifs.ex | 7 ++----- lib/util/zig_resolver.ex | 25 ++++--------------------- 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/lib/steps/patch/recompile_nifs.ex b/lib/steps/patch/recompile_nifs.ex index 4c87d6f..18691ff 100644 --- a/lib/steps/patch/recompile_nifs.ex +++ b/lib/steps/patch/recompile_nifs.ex @@ -164,11 +164,8 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do ] end - # CC/CXX/AR/RANLIB above are shell command *strings* that `make` hands to - # /bin/sh -c -- zig_bin is a resolved filesystem path (the managed Zig cache - # dir, or a user-supplied BURRITO_ZIG_PATH), not a fixed literal like the - # bare "zig" this used to be, so it can contain spaces or other characters - # that would otherwise split into unintended argv words. + # CC/CXX/AR/RANLIB are shell strings `make` passes to /bin/sh -c; zig_bin can + # now be a real path (not the fixed literal "zig"), so it needs quoting. defp shell_quote(path) do "'" <> String.replace(path, "'", "'\\''") <> "'" end diff --git a/lib/util/zig_resolver.ex b/lib/util/zig_resolver.ex index 089e07f..90c7c35 100644 --- a/lib/util/zig_resolver.ex +++ b/lib/util/zig_resolver.ex @@ -172,10 +172,7 @@ defmodule Burrito.Util.ZigResolver do defp install(archive_bytes, archive_ext, os, version) do File.mkdir_p!(managed_root()) - # PID + a per-VM unique integer: collision-resistant across concurrent OS - # processes too, not just within one BEAM instance (a bare - # :erlang.unique_integer/1 is only unique per-VM, so two `mix release` - # processes started at the same moment could otherwise pick the same name). + # unique_integer/1 alone is only unique per-VM, not across concurrent OS processes extract_dir = Path.join( managed_root(), @@ -196,10 +193,7 @@ defmodule Burrito.Util.ZigResolver do end end - # First writer wins: if a concurrent build already finished installing this - # exact version while we were downloading/extracting, don't delete or - # overwrite a directory another process may already be executing `zig` out - # of -- just adopt the existing install and discard our own extraction. + # first writer wins -- adopt a concurrent build's install rather than clobber it defp place_install(unpacked_dir, version, os) do install_dir = managed_install_dir(version) zig_path = managed_zig_path(version) @@ -217,9 +211,7 @@ defmodule Burrito.Util.ZigResolver do if os != :windows, do: File.chmod!(zig_path, 0o755) {:error, _reason} -> - # Lost a race with a concurrent installer between the check above and - # this rename. If the winner's install landed, use it; otherwise this - # really is a failure. + # lost the race between the check above and this rename unless File.exists?(zig_path) do raise "Failed to install Zig #{version} to #{install_dir}" end @@ -258,16 +250,7 @@ defmodule Burrito.Util.ZigResolver do end end - # The Zig archive always contains exactly one top-level `zig--/` - # directory (the compiler binary plus its supporting `lib/` sources) -- find it - # rather than hardcoding its name, since the compression step wrote other files - # (the archive itself) into the same scratch directory. Also confirm that entry - # actually resolves inside dest_dir: the checksum gate in fetch_and_install/4 - # already guards against a tampered archive reaching extraction at all, but this - # is a cheap second check specifically on the one path our own code goes on to - # `File.rename!/2` -- it doesn't (and, short of a dedicated safe-tar-extraction - # library, can't after the fact) catch a malicious *nested* member written - # outside dest_dir during extraction itself. + # find the archive's sole top-level dir, and confirm it didn't escape dest_dir defp sole_entry(dir) do case File.ls!(dir) |> Enum.reject(&(&1 in ["archive.tar.xz", "archive.zip"])) do [only] -> From 590c3224353311326f4db732afb1a3e1be56a6fd Mon Sep 17 00:00:00 2001 From: Jim Hazen Date: Mon, 27 Jul 2026 13:02:58 -0700 Subject: [PATCH 7/8] Match this codebase's do_download naming and error-message conventions A deeper style-fit pass compared zig_resolver.ex against the two direct precedents for this exact failure mode (default_erts_resolver.ex, fetch_musl.ex): both name the helper do_download, and both end the download-failure message with "please file an issue! Thanks!". Matched both here so the new resolver reads as native to this codebase rather than as an outside contribution with its own conventions. --- lib/util/zig_resolver.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/util/zig_resolver.ex b/lib/util/zig_resolver.ex index 90c7c35..1dbf626 100644 --- a/lib/util/zig_resolver.ex +++ b/lib/util/zig_resolver.ex @@ -145,7 +145,7 @@ defmodule Burrito.Util.ZigResolver do data _ -> - download(url, cache_key) + do_download(url, cache_key) end if sha256(archive_bytes) != expected_sha256 do @@ -157,12 +157,12 @@ defmodule Burrito.Util.ZigResolver do end end - defp download(url, cache_key) do + defp do_download(url, cache_key) do Log.info(:step, "Downloading: #{url}") resp = Downloader.get!(url) if resp.status != 200 do - raise "Failed to download Zig from #{url} (got HTTP #{resp.status})" + raise "Failed to download Zig from #{url}! (Got #{resp.status}) -- please file an issue! Thanks!" end FileCache.put_if_not_exist(cache_key, resp.body) From 1860f0c8aa9d9d99fa1032bfe6b12bf715c86397 Mon Sep 17 00:00:00 2001 From: Jim Hazen Date: Mon, 27 Jul 2026 13:54:43 -0700 Subject: [PATCH 8/8] Add CI coverage for no-zig and wrong-zig resolver fallback tiers build_examples already covers the already-correctly-configured case across hosts, but nothing exercised the two scenarios ZigResolver exists for: no system zig at all, and a system zig that doesn't match the pinned version. Added build_no_system_zig and build_wrong_system_zig, mirroring build_examples's own job shape (this file already repeats near-identical job blocks per scenario for run_examples_windows/ linux/macos, rather than a shared/parameterized workflow). Verified locally against both real scenarios on isolated Fly Sprites before adding: one with no zig on $PATH, one with zig 0.15.2 present (the prior pin). Both build and run cleanly end-to-end (all 5 targets, exit 0) once p7zip was installed on the sprite -- confirmed separately against a real prior CI run that ubuntu-latest already ships p7zip by default, so this isn't a gap the real runner will hit. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01JB25iAFxMVFobn96APu8dW --- .github/workflows/burrito-xcomp-check.yaml | 75 ++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/.github/workflows/burrito-xcomp-check.yaml b/.github/workflows/burrito-xcomp-check.yaml index c89eed8..79ab932 100644 --- a/.github/workflows/burrito-xcomp-check.yaml +++ b/.github/workflows/burrito-xcomp-check.yaml @@ -88,6 +88,81 @@ jobs: path: ./examples/**/burrito_out/* retention-days: 1 + #### Zig auto-resolution checks #### + # Prove the two failure modes ZigResolver exists for: no system zig at all, and a + # system zig that doesn't match the pinned version. Single-host (ubuntu-latest) is + # enough to exercise the resolver's tiers; build_examples already covers the + # already-correctly-configured case across hosts. + build_no_system_zig: + name: build_no_system_zig + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: erlef/setup-beam@v1 + with: + otp-version: "27.3" + elixir-version: "1.18.3" + + - run: sudo apt-get -y install xz-utils + + - uses: actions/cache/restore@v3 + id: cache-restore-no-zig + with: + path: /home/runner/.cache/burrito_file_cache/ + key: burrito-download-cache_no-system-zig + + - name: cli_example + working-directory: "./examples/cli_example" + run: "mix deps.get && mix release" + + - uses: actions/cache/save@v3 + id: cache-save-no-zig + with: + path: /home/runner/.cache/burrito_file_cache/ + key: burrito-download-cache_no-system-zig + + - run: chmod +x examples/cli_example/burrito_out/example_cli_app_linux + - run: examples/cli_example/burrito_out/example_cli_app_linux + + build_wrong_system_zig: + name: build_wrong_system_zig + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: erlef/setup-beam@v1 + with: + otp-version: "27.3" + elixir-version: "1.18.3" + + # Deliberately the prior pin, not the current one, so the resolver's exact-match + # check on $PATH must reject it and fall through to downloading the pinned version. + - uses: goto-bus-stop/setup-zig@v2 + with: + version: "0.15.2" + + - run: sudo apt-get -y install xz-utils + + - uses: actions/cache/restore@v3 + id: cache-restore-wrong-zig + with: + path: /home/runner/.cache/burrito_file_cache/ + key: burrito-download-cache_wrong-system-zig + + - name: cli_example + working-directory: "./examples/cli_example" + run: "mix deps.get && mix release" + + - uses: actions/cache/save@v3 + id: cache-save-wrong-zig + with: + path: /home/runner/.cache/burrito_file_cache/ + key: burrito-download-cache_wrong-system-zig + + - run: chmod +x examples/cli_example/burrito_out/example_cli_app_linux + - run: examples/cli_example/burrito_out/example_cli_app_linux + #### Run example binaries #### # Windows binaries: built on Linux/macOS hosts, run on windows-latest run_examples_windows: