diff --git a/.github/workflows/burrito-xcomp-check.yaml b/.github/workflows/burrito-xcomp-check.yaml index c89eed8..79ab932 100644 --- a/.github/workflows/burrito-xcomp-check.yaml +++ b/.github/workflows/burrito-xcomp-check.yaml @@ -88,6 +88,81 @@ jobs: path: ./examples/**/burrito_out/* retention-days: 1 + #### Zig auto-resolution checks #### + # Prove the two failure modes ZigResolver exists for: no system zig at all, and a + # system zig that doesn't match the pinned version. Single-host (ubuntu-latest) is + # enough to exercise the resolver's tiers; build_examples already covers the + # already-correctly-configured case across hosts. + build_no_system_zig: + name: build_no_system_zig + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: erlef/setup-beam@v1 + with: + otp-version: "27.3" + elixir-version: "1.18.3" + + - run: sudo apt-get -y install xz-utils + + - uses: actions/cache/restore@v3 + id: cache-restore-no-zig + with: + path: /home/runner/.cache/burrito_file_cache/ + key: burrito-download-cache_no-system-zig + + - name: cli_example + working-directory: "./examples/cli_example" + run: "mix deps.get && mix release" + + - uses: actions/cache/save@v3 + id: cache-save-no-zig + with: + path: /home/runner/.cache/burrito_file_cache/ + key: burrito-download-cache_no-system-zig + + - run: chmod +x examples/cli_example/burrito_out/example_cli_app_linux + - run: examples/cli_example/burrito_out/example_cli_app_linux + + build_wrong_system_zig: + name: build_wrong_system_zig + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: erlef/setup-beam@v1 + with: + otp-version: "27.3" + elixir-version: "1.18.3" + + # Deliberately the prior pin, not the current one, so the resolver's exact-match + # check on $PATH must reject it and fall through to downloading the pinned version. + - uses: goto-bus-stop/setup-zig@v2 + with: + version: "0.15.2" + + - run: sudo apt-get -y install xz-utils + + - uses: actions/cache/restore@v3 + id: cache-restore-wrong-zig + with: + path: /home/runner/.cache/burrito_file_cache/ + key: burrito-download-cache_wrong-system-zig + + - name: cli_example + working-directory: "./examples/cli_example" + run: "mix deps.get && mix release" + + - uses: actions/cache/save@v3 + id: cache-save-wrong-zig + with: + path: /home/runner/.cache/burrito_file_cache/ + key: burrito-download-cache_wrong-system-zig + + - run: chmod +x examples/cli_example/burrito_out/example_cli_app_linux + - run: examples/cli_example/burrito_out/example_cli_app_linux + #### Run example binaries #### # Windows binaries: built on Linux/macOS hosts, run on windows-latest run_examples_windows: diff --git a/lib/builder/builder.ex b/lib/builder/builder.ex index 3606116..c4f1611 100644 --- a/lib/builder/builder.ex +++ b/lib/builder/builder.ex @@ -42,7 +42,7 @@ defmodule Burrito.Builder do """ @phases [ - fetch: [Fetch.Init, Fetch.FetchMusl, Fetch.ResolveERTS], + fetch: [Fetch.Init, Fetch.ResolveZig, Fetch.FetchMusl, Fetch.ResolveERTS], patch: [Patch.CopyERTS, Patch.RecompileNIFs], build: [Build.PackAndBuild, Build.CopyRelease] ] @@ -103,7 +103,8 @@ defmodule Burrito.Builder do work_dir: "", self_dir: self_path, extra_build_env: [], - halted: false + halted: false, + zig_bin: "zig" } Log.info(:build, "Burrito is building target: #{target.alias}") diff --git a/lib/builder/context.ex b/lib/builder/context.ex index 3fb689f..a595a18 100644 --- a/lib/builder/context.ex +++ b/lib/builder/context.ex @@ -12,5 +12,6 @@ defmodule Burrito.Builder.Context do field(:self_dir, String.t()) field(:extra_build_env, list({String.t(), String.t()})) field(:halted, boolean()) + field(:zig_bin, String.t()) end end diff --git a/lib/burrito.ex b/lib/burrito.ex index df0f80a..26db8ae 100644 --- a/lib/burrito.ex +++ b/lib/burrito.ex @@ -27,10 +27,14 @@ defmodule Burrito do end defp pre_check() do - if Enum.any?(~w(zig xz), &(System.find_executable(&1) == nil)) do + # NOTE: `zig` is intentionally not checked here. `Burrito.Steps.Fetch.ResolveZig` + # resolves (and, if needed, downloads) a compatible zig per-target during the + # build's fetch phase instead of requiring one on PATH up front -- see its + # moduledoc, and `Burrito.Util.ZigResolver`, for the resolution order. + if System.find_executable("xz") == nil do Log.error( :build, - "You MUST have `zig` and `xz` installed to use Burrito, we couldn't find all of them in your PATH!" + "You MUST have `xz` installed to use Burrito, we couldn't find it in your PATH!" ) exit(1) @@ -42,21 +46,5 @@ defmodule Burrito do "We couldn't find 7z/7zz in your PATH, 7z/7zz is required to build Windows releases. They will fail if you don't fix this!" ) end - - check_zig_version() - end - - defp check_zig_version() do - {res, _} = System.cmd("zig", ["version"]) - version = String.trim(res) |> Version.parse!() - - if version != @zig_version_expected do - Log.error( - :build, - "Your Zig version does not match the one Burrito requires! We need `#{Version.to_string(@zig_version_expected)}`, you have: `#{Version.to_string(version)}`" - ) - - exit(1) - end end end diff --git a/lib/steps/build/pack_and_build.ex b/lib/steps/build/pack_and_build.ex index ea32bad..5846974 100644 --- a/lib/steps/build/pack_and_build.ex +++ b/lib/steps/build/pack_and_build.ex @@ -16,7 +16,7 @@ defmodule Burrito.Steps.Build.PackAndBuild do zig_build_args = ["-Dtarget=#{build_triplet}"] - create_metadata_file(context.self_dir, zig_build_args, context.mix_release) + create_metadata_file(context.self_dir, zig_build_args, context.mix_release, context.zig_bin) # TODO: Why do we need to do this??? # This is to bypass a VERY strange bug inside Linux containers... @@ -36,7 +36,7 @@ defmodule Burrito.Steps.Build.PackAndBuild do Log.info(:step, "Zig build env: #{inspect(build_env)}") build_result = - System.cmd("zig", ["build"] ++ zig_build_args, + System.cmd(context.zig_bin, ["build"] ++ zig_build_args, cd: context.self_dir, env: build_env, into: IO.stream() @@ -66,10 +66,10 @@ defmodule Burrito.Steps.Build.PackAndBuild do Path.join(File.cwd!(), [plugin_path]) end - defp create_metadata_file(self_path, args, release) do + defp create_metadata_file(self_path, args, release, zig_bin) do Log.info(:step, "Generating wrapper metadata file...") - {zig_version_string, 0} = System.cmd("zig", ["version"], cd: self_path) + {zig_version_string, 0} = System.cmd(zig_bin, ["version"], cd: self_path) metadata_map = %{ app_name: Atom.to_string(release.name), diff --git a/lib/steps/fetch/fetch_musl.ex b/lib/steps/fetch/fetch_musl.ex index 5f6fc60..afe0892 100644 --- a/lib/steps/fetch/fetch_musl.ex +++ b/lib/steps/fetch/fetch_musl.ex @@ -4,6 +4,7 @@ defmodule Burrito.Steps.Fetch.FetchMusl do alias Burrito.Builder.Step alias Burrito.Builder.Target + alias Burrito.Util.Downloader alias Burrito.Util.FileCache # Linked against musl libc v1.2.5 @@ -51,19 +52,9 @@ defmodule Burrito.Steps.Fetch.FetchMusl do def execute(context), do: context defp do_download(url, cache_key) do - {:ok, _} = Application.ensure_all_started(:req) Log.info(:step, "Downloading file: #{url}") - resp = - case Burrito.Util.get_proxy() do - proxy = %{scheme: scheme, host: host, port: port} when scheme in ["http", "https"] -> - Log.info(:step, "Using PROXY: #{proxy}") - proxy = {String.to_atom(scheme), host, port, []} - Req.get!(url, raw: true, connect_options: [proxy: proxy]) - - _ -> - Req.get!(url, raw: true) - end + resp = Downloader.get!(url) if resp.status != 200 do raise "Failed to fetch musl runtime: #{url}! (Got #{resp.status}) -- please file an issue! Thanks!" diff --git a/lib/steps/fetch/resolve_zig.ex b/lib/steps/fetch/resolve_zig.ex new file mode 100644 index 0000000..d6ae248 --- /dev/null +++ b/lib/steps/fetch/resolve_zig.ex @@ -0,0 +1,20 @@ +defmodule Burrito.Steps.Fetch.ResolveZig do + alias Burrito.Builder.Context + alias Burrito.Builder.Log + alias Burrito.Builder.Step + alias Burrito.Util.ZigResolver + + @behaviour Step + + @impl Step + def execute(%Context{} = context) do + case ZigResolver.resolve() do + {:ok, zig_bin} -> + %Context{context | zig_bin: zig_bin} + + {:error, reason} -> + Log.error(:step, reason) + %Context{context | halted: true} + end + end +end diff --git a/lib/steps/patch/recompile_nifs.ex b/lib/steps/patch/recompile_nifs.ex index 30c3f62..18691ff 100644 --- a/lib/steps/patch/recompile_nifs.ex +++ b/lib/steps/patch/recompile_nifs.ex @@ -29,7 +29,8 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do cflags, cxxflags, nif_env, - nif_make_args + nif_make_args, + context.zig_bin ) end) end @@ -59,7 +60,7 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do end) end - defp maybe_recompile_nif({_, _, false}, _, _, _, _, _, _, _), do: :no_nif + defp maybe_recompile_nif({_, _, false}, _, _, _, _, _, _, _, _), do: :no_nif defp maybe_recompile_nif( {dep, path, true}, @@ -69,7 +70,8 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do extra_cflags, extra_cxxflags, extra_env, - extra_make_args + extra_make_args, + zig_bin ) do dep = Atom.to_string(dep) @@ -94,12 +96,12 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do env: [ {"MIX_APP_PATH", output_priv_dir}, - {"RANLIB", "zig ranlib"}, - {"AR", "zig ar"}, + {"RANLIB", "#{shell_quote(zig_bin)} ranlib"}, + {"AR", "#{shell_quote(zig_bin)} ar"}, {"CC", - "zig cc -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cflags}"}, + "#{shell_quote(zig_bin)} cc -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cflags}"}, {"CXX", - "zig c++ -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cxxflags}"} + "#{shell_quote(zig_bin)} c++ -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cxxflags}"} ] ++ erts_env ++ extra_env, into: IO.stream() ) @@ -161,4 +163,10 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do {"ERTS_INCLUDE_DIR", erts_include} ] end + + # CC/CXX/AR/RANLIB are shell strings `make` passes to /bin/sh -c; zig_bin can + # now be a real path (not the fixed literal "zig"), so it needs quoting. + defp shell_quote(path) do + "'" <> String.replace(path, "'", "'\\''") <> "'" + end end diff --git a/lib/util/default_erts_resolver.ex b/lib/util/default_erts_resolver.ex index 8185c99..e17b5eb 100644 --- a/lib/util/default_erts_resolver.ex +++ b/lib/util/default_erts_resolver.ex @@ -3,6 +3,7 @@ defmodule Burrito.Util.DefaultERTSResolver do alias Burrito.Builder.Log alias Burrito.Util + alias Burrito.Util.Downloader alias Burrito.Util.FileCache alias Burrito.Util.ERTSResolver alias Burrito.Util.ERTSUniversalMachineFetcher @@ -96,19 +97,9 @@ defmodule Burrito.Util.DefaultERTSResolver do end defp do_download(url, cache_key) do - {:ok, _} = Application.ensure_all_started(:req) Log.info(:step, "Downloading file: #{url}") - resp = - case Burrito.Util.get_proxy() do - proxy = %{scheme: scheme, host: host, port: port} when scheme in ["http", "https"] -> - Log.info(:step, "Using PROXY: #{proxy}") - proxy = {String.to_atom(scheme), host, port, []} - Req.get!(url, raw: true, connect_options: [proxy: proxy]) - - _ -> - Req.get!(url, raw: true) - end + resp = Downloader.get!(url) if resp.status != 200 do raise "Failed to fetch #{url}! (Got #{resp.status}) Perhaps we haven't built a pre-compiled Erlang for this release yet? If this was a 404, please file an issue! Thanks!" diff --git a/lib/util/downloader.ex b/lib/util/downloader.ex new file mode 100644 index 0000000..a6cf9dc --- /dev/null +++ b/lib/util/downloader.ex @@ -0,0 +1,25 @@ +defmodule Burrito.Util.Downloader do + @moduledoc """ + Shared HTTP GET used by Burrito's own download steps (musl runtime, managed Zig, + ...) -- proxy-aware, raw bytes. Raises on transport failure, same as the plain + `Req.get!/2` it wraps; callers are responsible for checking `resp.status`. + """ + + alias Burrito.Builder.Log + alias Burrito.Util + + @spec get!(String.t()) :: Req.Response.t() + def get!(url) do + {:ok, _} = Application.ensure_all_started(:req) + + case Util.get_proxy() do + proxy = %{scheme: scheme, host: host, port: port} when scheme in ["http", "https"] -> + Log.info(:step, "Using PROXY: #{proxy}") + proxy = {String.to_atom(scheme), host, port, []} + Req.get!(url, raw: true, connect_options: [proxy: proxy]) + + _ -> + Req.get!(url, raw: true) + end + end +end diff --git a/lib/util/zig_resolver.ex b/lib/util/zig_resolver.ex new file mode 100644 index 0000000..1dbf626 --- /dev/null +++ b/lib/util/zig_resolver.ex @@ -0,0 +1,291 @@ +defmodule Burrito.Util.ZigResolver do + @moduledoc """ + Resolves a path to a `zig` executable matching Burrito's pinned version + (see `Burrito.get_versions/0`), in this order: + + 1. `BURRITO_ZIG_PATH` environment variable, if set — an explicit override. Still + validated (must exist and report the pinned version) so a typo or wrong-version + override fails clearly here instead of confusingly deep in a build step. + 2. A previously downloaded, managed copy at the pinned version, if already cached. + 3. The system `zig` on `$PATH`, if its version matches the pinned version exactly — + this preserves today's behavior unchanged for anyone who already has the right + version installed. + 4. Otherwise, download the pinned version for the current host OS/CPU, verify its + checksum, cache it, and use that. + + Only the exact pinned version is ever used to build — Burrito's own `build.zig` + sources have historically only been compatible with one zig version at a time (see + the 0.15 -> 0.16 migration), so accepting "any installed zig" isn't a safe + relaxation of the old check. This resolver removes the need for the *system* to + have zig installed at all, and stops an incompatible system zig from blocking a + build, without weakening which zig version actually gets used. + """ + + alias Burrito.Builder.Log + alias Burrito.Util + alias Burrito.Util.Downloader + alias Burrito.Util.FileCache + + # sha256 checksums for the pinned zig release, one per (os, cpu) Burrito supports + # building on. Sourced from https://ziglang.org/download/index.json for the + # version in Burrito.get_versions().zig -- update alongside that version. + @zig_checksums %{ + {:linux, :x86_64} => "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00", + {:linux, :aarch64} => "ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17", + {:darwin, :x86_64} => "0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7", + {:darwin, :aarch64} => "b23d70deaa879b5c2d486ed3316f7eaa53e84acf6fc9cc747de152450d401489", + {:windows, :x86_64} => "68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e" + } + + @spec resolve() :: {:ok, String.t()} | {:error, String.t()} + def resolve do + expected = Burrito.get_versions().zig + + with :miss <- from_override(expected), + :miss <- from_managed_cache(expected), + :miss <- from_system_path(expected) do + download_and_cache(expected) + end + end + + @spec zig_version_at(String.t()) :: {:ok, Version.t()} | :error + def zig_version_at(path) do + case System.cmd(path, ["version"]) do + {out, 0} -> {:ok, out |> String.trim() |> Version.parse!()} + _ -> :error + end + rescue + _ -> :error + end + + defp from_override(expected) do + case System.get_env("BURRITO_ZIG_PATH") do + nil -> + :miss + + path -> + cond do + not File.regular?(path) -> + {:error, "BURRITO_ZIG_PATH is set to `#{path}`, but no file exists there!"} + + true -> + case zig_version_at(path) do + {:ok, ^expected} -> + Log.info(:step, "Using BURRITO_ZIG_PATH override: #{path}") + {:ok, path} + + {:ok, other} -> + {:error, + "BURRITO_ZIG_PATH points at Zig #{other}, but Burrito requires exactly " <> + "#{expected}!"} + + :error -> + {:error, + "BURRITO_ZIG_PATH is set to `#{path}`, but it doesn't look like a working " <> + "`zig` binary!"} + end + end + end + end + + defp from_managed_cache(expected) do + path = managed_zig_path(expected) + + if File.exists?(path) do + Log.info(:step, "Using cached managed Zig #{expected}: #{path}") + {:ok, path} + else + :miss + end + end + + defp from_system_path(expected) do + case System.find_executable("zig") do + nil -> + :miss + + path -> + case zig_version_at(path) do + {:ok, ^expected} -> {:ok, path} + _ -> :miss + end + end + end + + defp download_and_cache(expected) do + os = Util.get_current_os() + cpu = Util.get_current_cpu() + + case Map.fetch(@zig_checksums, {os, cpu}) do + :error -> + {:error, + "No managed Zig download is known for #{os}/#{cpu}. Install Zig #{expected} " <> + "yourself and set BURRITO_ZIG_PATH to point at it."} + + {:ok, expected_sha256} -> + Log.info( + :step, + "No compatible Zig found on PATH; fetching managed Zig #{expected} for #{os}/#{cpu}..." + ) + + fetch_and_install(os, cpu, expected, expected_sha256) + end + end + + defp fetch_and_install(os, cpu, version, expected_sha256) do + archive_ext = if os == :windows, do: "zip", else: "tar.xz" + file_name = "zig-#{cpu}-#{zig_os_name(os)}-#{version}.#{archive_ext}" + url = "https://ziglang.org/download/#{version}/#{file_name}" + cache_key = :crypto.hash(:sha, url) |> Base.encode16() + + archive_bytes = + case FileCache.fetch(cache_key) do + {:hit, data} -> + Log.info(:step, "Found matching cached Zig download, using that") + data + + _ -> + do_download(url, cache_key) + end + + if sha256(archive_bytes) != expected_sha256 do + {:error, + "Checksum mismatch for #{file_name}! Expected #{expected_sha256}, got " <> + "#{sha256(archive_bytes)}. Refusing to use this download."} + else + install(archive_bytes, archive_ext, os, version) + end + end + + defp do_download(url, cache_key) do + Log.info(:step, "Downloading: #{url}") + resp = Downloader.get!(url) + + if resp.status != 200 do + raise "Failed to download Zig from #{url}! (Got #{resp.status}) -- please file an issue! Thanks!" + end + + FileCache.put_if_not_exist(cache_key, resp.body) + resp.body + end + + defp install(archive_bytes, archive_ext, os, version) do + File.mkdir_p!(managed_root()) + + # unique_integer/1 alone is only unique per-VM, not across concurrent OS processes + extract_dir = + Path.join( + managed_root(), + "extract-#{System.pid()}-#{:erlang.unique_integer([:positive])}" + ) + + File.mkdir_p!(extract_dir) + + with :ok <- extract(archive_bytes, archive_ext, extract_dir), + {:ok, unpacked_dir} <- sole_entry(extract_dir) do + result = place_install(unpacked_dir, version, os) + File.rm_rf!(extract_dir) + result + else + {:error, reason} -> + File.rm_rf!(extract_dir) + {:error, reason} + end + end + + # first writer wins -- adopt a concurrent build's install rather than clobber it + defp place_install(unpacked_dir, version, os) do + install_dir = managed_install_dir(version) + zig_path = managed_zig_path(version) + + if File.exists?(zig_path) do + Log.info( + :step, + "Managed Zig #{version} was already installed by a concurrent build, using that" + ) + else + File.mkdir_p!(Path.dirname(install_dir)) + + case File.rename(unpacked_dir, install_dir) do + :ok -> + if os != :windows, do: File.chmod!(zig_path, 0o755) + + {:error, _reason} -> + # lost the race between the check above and this rename + unless File.exists?(zig_path) do + raise "Failed to install Zig #{version} to #{install_dir}" + end + end + end + + Log.success(:step, "Installed managed Zig #{version}: #{zig_path}") + {:ok, zig_path} + end + + defp extract(bytes, "tar.xz", dest_dir) do + tmp_tar = Path.join(dest_dir, "archive.tar.xz") + File.write!(tmp_tar, bytes) + + case System.cmd("tar", ["-xJf", tmp_tar, "-C", dest_dir]) do + {_, 0} -> + File.rm(tmp_tar) + :ok + + {out, _} -> + {:error, "Failed to extract Zig tarball: #{out}"} + end + end + + defp extract(bytes, "zip", dest_dir) do + tmp_zip = Path.join(dest_dir, "archive.zip") + File.write!(tmp_zip, bytes) + + case :zip.extract(String.to_charlist(tmp_zip), cwd: String.to_charlist(dest_dir)) do + {:ok, _} -> + File.rm(tmp_zip) + :ok + + {:error, reason} -> + {:error, "Failed to extract Zig archive: #{inspect(reason)}"} + end + end + + # find the archive's sole top-level dir, and confirm it didn't escape dest_dir + defp sole_entry(dir) do + case File.ls!(dir) |> Enum.reject(&(&1 in ["archive.tar.xz", "archive.zip"])) do + [only] -> + entry_path = Path.join(dir, only) + expanded_dir = Path.expand(dir) + + if String.starts_with?(Path.expand(entry_path), expanded_dir <> "/") do + {:ok, entry_path} + else + {:error, "Extracted Zig archive's top-level entry escaped the extraction directory"} + end + + other -> + {:error, "Expected exactly one entry in extracted Zig archive, got: #{inspect(other)}"} + end + end + + defp managed_install_dir(version) do + os = Util.get_current_os() + cpu = Util.get_current_cpu() + Path.join([managed_root(), "#{version}-#{os}-#{cpu}"]) + end + + defp managed_zig_path(version) do + exe = if Util.get_current_os() == :windows, do: "zig.exe", else: "zig" + Path.join(managed_install_dir(version), exe) + end + + defp managed_root do + :filename.basedir(:user_cache, "burrito_file_cache") |> to_string() |> Path.join("zig") + end + + defp zig_os_name(:darwin), do: "macos" + defp zig_os_name(:windows), do: "windows" + defp zig_os_name(:linux), do: "linux" + + defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) +end