feat: automatically resolve zig instead of requiring an exact system version

.github/workflows/burrito-xcomp-check.yaml

@@ -88,6 +88,81 @@ jobs:
           path: ./examples/**/burrito_out/*
           retention-days: 1
 
+  #### Zig auto-resolution checks ####
+  # Prove the two failure modes ZigResolver exists for: no system zig at all, and a
+  # system zig that doesn't match the pinned version. Single-host (ubuntu-latest) is
+  # enough to exercise the resolver's tiers; build_examples already covers the
+  # already-correctly-configured case across hosts.
+  build_no_system_zig:
+    name: build_no_system_zig
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: erlef/setup-beam@v1
+        with:
+          otp-version: "27.3"
+          elixir-version: "1.18.3"
+
+      - run: sudo apt-get -y install xz-utils
+
+      - uses: actions/cache/restore@v3
+        id: cache-restore-no-zig
+        with:
+          path: /home/runner/.cache/burrito_file_cache/
+          key: burrito-download-cache_no-system-zig
+
+      - name: cli_example
+        working-directory: "./examples/cli_example"
+        run: "mix deps.get && mix release"
+
+      - uses: actions/cache/save@v3
+        id: cache-save-no-zig
+        with:
+          path: /home/runner/.cache/burrito_file_cache/
+          key: burrito-download-cache_no-system-zig
+
+      - run: chmod +x examples/cli_example/burrito_out/example_cli_app_linux
+      - run: examples/cli_example/burrito_out/example_cli_app_linux
+
+  build_wrong_system_zig:
+    name: build_wrong_system_zig
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: erlef/setup-beam@v1
+        with:
+          otp-version: "27.3"
+          elixir-version: "1.18.3"
+
+      # Deliberately the prior pin, not the current one, so the resolver's exact-match
+      # check on $PATH must reject it and fall through to downloading the pinned version.
+      - uses: goto-bus-stop/setup-zig@v2
+        with:
+          version: "0.15.2"
+
+      - run: sudo apt-get -y install xz-utils
+
+      - uses: actions/cache/restore@v3
+        id: cache-restore-wrong-zig
+        with:
+          path: /home/runner/.cache/burrito_file_cache/
+          key: burrito-download-cache_wrong-system-zig
+
+      - name: cli_example
+        working-directory: "./examples/cli_example"
+        run: "mix deps.get && mix release"
+
+      - uses: actions/cache/save@v3
+        id: cache-save-wrong-zig
+        with:
+          path: /home/runner/.cache/burrito_file_cache/
+          key: burrito-download-cache_wrong-system-zig
+
+      - run: chmod +x examples/cli_example/burrito_out/example_cli_app_linux
+      - run: examples/cli_example/burrito_out/example_cli_app_linux
+
   #### Run example binaries ####
   # Windows binaries: built on Linux/macOS hosts, run on windows-latest
   run_examples_windows:

lib/builder/builder.ex

@@ -42,7 +42,7 @@ defmodule Burrito.Builder do
   """
 
   @phases [
-    fetch: [Fetch.Init, Fetch.FetchMusl, Fetch.ResolveERTS],
+    fetch: [Fetch.Init, Fetch.ResolveZig, Fetch.FetchMusl, Fetch.ResolveERTS],
     patch: [Patch.CopyERTS, Patch.RecompileNIFs],
     build: [Build.PackAndBuild, Build.CopyRelease]
   ]
@@ -103,7 +103,8 @@ defmodule Burrito.Builder do
         work_dir: "",
         self_dir: self_path,
         extra_build_env: [],
-        halted: false
+        halted: false,
+        zig_bin: "zig"
       }
 
       Log.info(:build, "Burrito is building target: #{target.alias}")

lib/builder/context.ex

@@ -12,5 +12,6 @@ defmodule Burrito.Builder.Context do
     field(:self_dir, String.t())
     field(:extra_build_env, list({String.t(), String.t()}))
     field(:halted, boolean())
+    field(:zig_bin, String.t())
   end
 end

lib/burrito.ex

@@ -27,10 +27,14 @@ defmodule Burrito do
   end
 
   defp pre_check() do
-    if Enum.any?(~w(zig xz), &(System.find_executable(&1) == nil)) do
+    # NOTE: `zig` is intentionally not checked here. `Burrito.Steps.Fetch.ResolveZig`
+    # resolves (and, if needed, downloads) a compatible zig per-target during the
+    # build's fetch phase instead of requiring one on PATH up front -- see its
+    # moduledoc, and `Burrito.Util.ZigResolver`, for the resolution order.
+    if System.find_executable("xz") == nil do
       Log.error(
         :build,
-        "You MUST have `zig` and `xz` installed to use Burrito, we couldn't find all of them in your PATH!"
+        "You MUST have `xz` installed to use Burrito, we couldn't find it in your PATH!"
       )
 
       exit(1)
@@ -42,21 +46,5 @@ defmodule Burrito do
         "We couldn't find 7z/7zz in your PATH, 7z/7zz is required to build Windows releases. They will fail if you don't fix this!"
       )
     end
-
-    check_zig_version()
-  end
-
-  defp check_zig_version() do
-    {res, _} = System.cmd("zig", ["version"])
-    version = String.trim(res) |> Version.parse!()
-
-    if version != @zig_version_expected do
-      Log.error(
-        :build,
-        "Your Zig version does not match the one Burrito requires! We need `#{Version.to_string(@zig_version_expected)}`, you have: `#{Version.to_string(version)}`"
-      )
-
-      exit(1)
-    end
   end
 end

lib/steps/build/pack_and_build.ex

@@ -16,7 +16,7 @@ defmodule Burrito.Steps.Build.PackAndBuild do
 
     zig_build_args = ["-Dtarget=#{build_triplet}"]
 
-    create_metadata_file(context.self_dir, zig_build_args, context.mix_release)
+    create_metadata_file(context.self_dir, zig_build_args, context.mix_release, context.zig_bin)
 
     # TODO: Why do we need to do this???
     # This is to bypass a VERY strange bug inside Linux containers...
@@ -36,7 +36,7 @@ defmodule Burrito.Steps.Build.PackAndBuild do
     Log.info(:step, "Zig build env: #{inspect(build_env)}")
 
     build_result =
-      System.cmd("zig", ["build"] ++ zig_build_args,
+      System.cmd(context.zig_bin, ["build"] ++ zig_build_args,
         cd: context.self_dir,
         env: build_env,
         into: IO.stream()
@@ -66,10 +66,10 @@ defmodule Burrito.Steps.Build.PackAndBuild do
     Path.join(File.cwd!(), [plugin_path])
   end
 
-  defp create_metadata_file(self_path, args, release) do
+  defp create_metadata_file(self_path, args, release, zig_bin) do
     Log.info(:step, "Generating wrapper metadata file...")
 
-    {zig_version_string, 0} = System.cmd("zig", ["version"], cd: self_path)
+    {zig_version_string, 0} = System.cmd(zig_bin, ["version"], cd: self_path)
 
     metadata_map = %{
       app_name: Atom.to_string(release.name),

lib/steps/fetch/fetch_musl.ex

@@ -4,6 +4,7 @@ defmodule Burrito.Steps.Fetch.FetchMusl do
   alias Burrito.Builder.Step
   alias Burrito.Builder.Target
 
+  alias Burrito.Util.Downloader
   alias Burrito.Util.FileCache
 
   # Linked against musl libc v1.2.5
@@ -51,19 +52,9 @@ defmodule Burrito.Steps.Fetch.FetchMusl do
   def execute(context), do: context
 
   defp do_download(url, cache_key) do
-    {:ok, _} = Application.ensure_all_started(:req)
     Log.info(:step, "Downloading file: #{url}")
 
-    resp =
-      case Burrito.Util.get_proxy() do
-        proxy = %{scheme: scheme, host: host, port: port} when scheme in ["http", "https"] ->
-          Log.info(:step, "Using PROXY: #{proxy}")
-          proxy = {String.to_atom(scheme), host, port, []}
-          Req.get!(url, raw: true, connect_options: [proxy: proxy])
-
-        _ ->
-          Req.get!(url, raw: true)
-      end
+    resp = Downloader.get!(url)
 
     if resp.status != 200 do
       raise "Failed to fetch musl runtime: #{url}! (Got #{resp.status}) -- please file an issue! Thanks!"

lib/steps/fetch/resolve_zig.ex

@@ -0,0 +1,20 @@
+defmodule Burrito.Steps.Fetch.ResolveZig do
+  alias Burrito.Builder.Context
+  alias Burrito.Builder.Log
+  alias Burrito.Builder.Step
+  alias Burrito.Util.ZigResolver
+
+  @behaviour Step
+
+  @impl Step
+  def execute(%Context{} = context) do
+    case ZigResolver.resolve() do
+      {:ok, zig_bin} ->
+        %Context{context | zig_bin: zig_bin}
+
+      {:error, reason} ->
+        Log.error(:step, reason)
+        %Context{context | halted: true}
+    end
+  end
+end

lib/steps/patch/recompile_nifs.ex

@@ -29,7 +29,8 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do
           cflags,
           cxxflags,
           nif_env,
-          nif_make_args
+          nif_make_args,
+          context.zig_bin
         )
       end)
     end
@@ -59,7 +60,7 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do
     end)
   end
 
-  defp maybe_recompile_nif({_, _, false}, _, _, _, _, _, _, _), do: :no_nif
+  defp maybe_recompile_nif({_, _, false}, _, _, _, _, _, _, _, _), do: :no_nif
 
   defp maybe_recompile_nif(
          {dep, path, true},
@@ -69,7 +70,8 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do
          extra_cflags,
          extra_cxxflags,
          extra_env,
-         extra_make_args
+         extra_make_args,
+         zig_bin
        ) do
     dep = Atom.to_string(dep)
 
@@ -94,12 +96,12 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do
         env:
           [
             {"MIX_APP_PATH", output_priv_dir},
-            {"RANLIB", "zig ranlib"},
-            {"AR", "zig ar"},
+            {"RANLIB", "#{shell_quote(zig_bin)} ranlib"},
+            {"AR", "#{shell_quote(zig_bin)} ar"},
             {"CC",
-             "zig cc -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cflags}"},
+             "#{shell_quote(zig_bin)} cc -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cflags}"},
             {"CXX",
-             "zig c++ -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cxxflags}"}
+             "#{shell_quote(zig_bin)} c++ -target #{cross_target} -O2 -dynamic -shared -Wl,-undefined=dynamic_lookup #{extra_cxxflags}"}
           ] ++ erts_env ++ extra_env,
         into: IO.stream()
       )
@@ -161,4 +163,10 @@ defmodule Burrito.Steps.Patch.RecompileNIFs do
       {"ERTS_INCLUDE_DIR", erts_include}
     ]
   end
+
+  # CC/CXX/AR/RANLIB are shell strings `make` passes to /bin/sh -c; zig_bin can
+  # now be a real path (not the fixed literal "zig"), so it needs quoting.
+  defp shell_quote(path) do
+    "'" <> String.replace(path, "'", "'\\''") <> "'"
+  end
 end

lib/util/default_erts_resolver.ex

@@ -3,6 +3,7 @@ defmodule Burrito.Util.DefaultERTSResolver do
   alias Burrito.Builder.Log
 
   alias Burrito.Util
+  alias Burrito.Util.Downloader
   alias Burrito.Util.FileCache
   alias Burrito.Util.ERTSResolver
   alias Burrito.Util.ERTSUniversalMachineFetcher
@@ -96,19 +97,9 @@ defmodule Burrito.Util.DefaultERTSResolver do
   end
 
   defp do_download(url, cache_key) do
-    {:ok, _} = Application.ensure_all_started(:req)
     Log.info(:step, "Downloading file: #{url}")
 
-    resp =
-      case Burrito.Util.get_proxy() do
-        proxy = %{scheme: scheme, host: host, port: port} when scheme in ["http", "https"] ->
-          Log.info(:step, "Using PROXY: #{proxy}")
-          proxy = {String.to_atom(scheme), host, port, []}
-          Req.get!(url, raw: true, connect_options: [proxy: proxy])
-
-        _ ->
-          Req.get!(url, raw: true)
-      end
+    resp = Downloader.get!(url)
 
     if resp.status != 200 do
       raise "Failed to fetch #{url}! (Got #{resp.status}) Perhaps we haven't built a pre-compiled Erlang for this release yet? If this was a 404, please file an issue! Thanks!"

lib/util/downloader.ex

@@ -0,0 +1,25 @@
+defmodule Burrito.Util.Downloader do
+  @moduledoc """
+  Shared HTTP GET used by Burrito's own download steps (musl runtime, managed Zig,
+  ...) -- proxy-aware, raw bytes. Raises on transport failure, same as the plain
+  `Req.get!/2` it wraps; callers are responsible for checking `resp.status`.
+  """
+
+  alias Burrito.Builder.Log
+  alias Burrito.Util
+
+  @spec get!(String.t()) :: Req.Response.t()
+  def get!(url) do
+    {:ok, _} = Application.ensure_all_started(:req)
+
+    case Util.get_proxy() do
+      proxy = %{scheme: scheme, host: host, port: port} when scheme in ["http", "https"] ->
+        Log.info(:step, "Using PROXY: #{proxy}")
+        proxy = {String.to_atom(scheme), host, port, []}
+        Req.get!(url, raw: true, connect_options: [proxy: proxy])
+
+      _ ->
+        Req.get!(url, raw: true)
+    end
+  end
+end