Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ jobs:
- {slug: 'package_linux.powerpc64le', build_on: ''}
- {slug: 'package_linux.x86_64', build_on: ''}

# Essentially `package_linux` plus a Rust toolchain and `libclang`
# for use by the MMTK GC build (`WITH_THIRD_PARTY_GC=mmtk`)
- {slug: 'package_linux_mmtk.x86_64', build_on: ''}
- {slug: 'package_linux_mmtk.aarch64', build_on: '-arm'}

# The `package_musl` image is `alpine`-based.
- {slug: 'package_musl.x86_64', build_on: ''}

Expand Down
58 changes: 3 additions & 55 deletions linux/package_linux.jl
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
using RootfsUtils: parse_build_args, upload_gha, test_sandbox
using RootfsUtils: debootstrap
using RootfsUtils: install_awscli
using RootfsUtils: root_chroot
using RootfsUtils: install_gcc_toolchain

args = parse_build_args(ARGS, @__FILE__)
arch = args.arch
Expand Down Expand Up @@ -39,60 +39,8 @@ packages = [
]

artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do rootfs, chroot_ENV
my_chroot(args...) = root_chroot(rootfs, "bash", "-eu", "-o", "pipefail", "-c", args...; ENV=chroot_ENV)

host_triplet = "$(arch)-linux-gnu"
gcc_triplet = host_triplet
cross_tags = "target_libc+glibc-target_os+linux-target_arch+$(arch)"
if arch == "armv7l"
host_triplet = "armv7l-linux-gnueabihf"
gcc_triplet = "arm-linux-gnueabihf"
cross_tags = "target_libc+glibc-target_os+linux-target_call_abi+eabihf-target_arch+$(arch)"
end
glibc_version_dict = Dict(
"x86_64" => v"2.17",
"i686" => v"2.17",
"aarch64" => v"2.19",
"armv7l" => v"2.19",
"powerpc64le" => v"2.19",
)

# Install GCC 9 from Elliot's repo
repo_release_url = "https://github.com/staticfloat/linux-gcc-toolchains/releases/download/GCC-v9.1.0-$(host_triplet)"
gcc_install_cmd = """
cd /usr/local
curl -fL $(repo_release_url)/GCC.v9.1.0.$(host_triplet)-$(cross_tags).tar.gz | tar zx
curl -fL $(repo_release_url)/Binutils.v2.38.0.$(host_triplet)-$(cross_tags).tar.gz | tar zx
curl -fL $(repo_release_url)/Zlib.v1.2.12.$(host_triplet).tar.gz | tar zx
cd /usr/local/$(gcc_triplet)/
curl -fL $(repo_release_url)/Glibc.v$(glibc_version_dict[arch]).$(host_triplet).tar.gz | tar zx
cd /usr/local/$(gcc_triplet)/usr
curl -fL $(repo_release_url)/LinuxKernelHeaders.v5.15.14.$(host_triplet)-host+any.tar.gz | tar zx
"""
gcc_symlink_cmd = """
# Create symlinks for `gcc` -> `$(gcc_triplet)-gcc`, etc...
for tool_path in /usr/local/bin/$(gcc_triplet)-*; do
tool="\$(basename "\${tool_path}" | sed -e 's/$(gcc_triplet)-//')"
ln -vsf "$(gcc_triplet)-\${tool}" "/usr/local/bin/\${tool}"
done
# Also create symlinks for `cc` and `c++`.
ln -vsf "/usr/local/bin/gcc" "/usr/local/bin/cc"
ln -vsf "/usr/local/bin/g++" "/usr/local/bin/c++"
"""
my_chroot(gcc_install_cmd)
my_chroot(gcc_symlink_cmd)

# Show what is installed
my_chroot("which gcc")
my_chroot("which -a gcc")
my_chroot("which g++")
my_chroot("which -a g++")
my_chroot("which ld")
my_chroot("which -a ld")

my_chroot("gcc --version")
my_chroot("g++ --version")
my_chroot("ld --version")
# Install the GCC 9 cross-toolchain as the default `gcc`/`g++`/`cc`/`c++`/`ld`.
install_gcc_toolchain(rootfs, chroot_ENV, arch)

# The build jobs upload their products to S3 from within the sandbox,
# using OIDC-issued credentials; they need a recent AWS CLI to do so.
Expand Down
64 changes: 64 additions & 0 deletions linux/package_linux_mmtk.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
using RootfsUtils: parse_build_args, upload_gha, test_sandbox
using RootfsUtils: debootstrap
using RootfsUtils: install_awscli
using RootfsUtils: install_gcc_toolchain
using RootfsUtils: install_rust

args = parse_build_args(ARGS, @__FILE__)
arch = args.arch
archive = args.archive
image = args.image

# This is the `package_linux` image plus a Rust toolchain, used for Julia's
# MMTk garbage collector builds (`WITH_THIRD_PARTY_GC=mmtk`). The MMTk binding
# is compiled from source with `cargo`, and its `build.rs` runs `bindgen`
# (which needs `libclang`) to generate the Julia FFI bindings, so we add
# `clang`/`libclang-dev` on top of the usual `package_linux` package set.
packages = [
"automake",
"bash",
"bison",
"bzip2",
"ccache",
"clang",
"cmake",
"curl",
"flex",
"gdb",
"git",
"less",
"libatomic1",
"libclang-dev",
"libtool",
"lldb",
"locales",
"localepurge",
"m4",
"make",
"patch",
"patchelf",
"perl",
"pkg-config",
"python3",
"time",
"vim",
"wget",
"zstd",
]

artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do rootfs, chroot_ENV
# Install the GCC 9 cross-toolchain as the default `gcc`/`g++`/`cc`/`c++`/`ld`.
install_gcc_toolchain(rootfs, chroot_ENV, arch)

# Install the Rust toolchain used to build the MMTk garbage collector.
# Keep the version in sync with `src/gc-mmtk/mmtk_julia/rust-toolchain`
# in JuliaLang/julia.
install_rust(rootfs, chroot_ENV, arch)

# The build jobs upload their products to S3 from within the sandbox,
# using OIDC-issued credentials; they need a recent AWS CLI to do so.
install_awscli(rootfs, chroot_ENV, arch)
end

upload_gha(tarball_path)
test_sandbox(artifact_hash)
2 changes: 2 additions & 0 deletions src/RootfsUtils.jl
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ include("build_img/args.jl")
include("build_img/awscli.jl")
include("build_img/common.jl")
include("build_img/debian.jl")
include("build_img/gcc.jl")
include("build_img/rust.jl")

include("test_img/args.jl")
include("test_img/test.jl")
Expand Down
64 changes: 64 additions & 0 deletions src/build_img/gcc.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
# Install Elliot's GCC 9 cross-toolchain (GCC, Binutils, Zlib, Glibc, Linux
# kernel headers) into a Debian-based rootfs, and symlink it as the default
# `gcc`/`g++`/`cc`/`c++`/`ld`.
#
# This is the compiler toolchain used by Julia's "from binaries" Linux package
# builds, so it is shared between the `package_linux` and `package_linux_mmtk`
# images to keep them in sync.
function install_gcc_toolchain(rootfs::String, chroot_ENV::AbstractDict, arch::String)
arch = normalize_arch(arch)
my_chroot(args...) = root_chroot(rootfs, "bash", "-eu", "-o", "pipefail", "-c", args...; ENV=chroot_ENV)

host_triplet = "$(arch)-linux-gnu"
gcc_triplet = host_triplet
cross_tags = "target_libc+glibc-target_os+linux-target_arch+$(arch)"
if arch == "armv7l"
host_triplet = "armv7l-linux-gnueabihf"
gcc_triplet = "arm-linux-gnueabihf"
cross_tags = "target_libc+glibc-target_os+linux-target_call_abi+eabihf-target_arch+$(arch)"
end
glibc_version_dict = Dict(
"x86_64" => v"2.17",
"i686" => v"2.17",
"aarch64" => v"2.19",
"armv7l" => v"2.19",
"powerpc64le" => v"2.19",
)

# Install GCC 9 from Elliot's repo
repo_release_url = "https://github.com/staticfloat/linux-gcc-toolchains/releases/download/GCC-v9.1.0-$(host_triplet)"
gcc_install_cmd = """
cd /usr/local
curl -fL $(repo_release_url)/GCC.v9.1.0.$(host_triplet)-$(cross_tags).tar.gz | tar zx
curl -fL $(repo_release_url)/Binutils.v2.38.0.$(host_triplet)-$(cross_tags).tar.gz | tar zx
curl -fL $(repo_release_url)/Zlib.v1.2.12.$(host_triplet).tar.gz | tar zx
cd /usr/local/$(gcc_triplet)/
curl -fL $(repo_release_url)/Glibc.v$(glibc_version_dict[arch]).$(host_triplet).tar.gz | tar zx
cd /usr/local/$(gcc_triplet)/usr
curl -fL $(repo_release_url)/LinuxKernelHeaders.v5.15.14.$(host_triplet)-host+any.tar.gz | tar zx
"""
gcc_symlink_cmd = """
# Create symlinks for `gcc` -> `$(gcc_triplet)-gcc`, etc...
for tool_path in /usr/local/bin/$(gcc_triplet)-*; do
tool="\$(basename "\${tool_path}" | sed -e 's/$(gcc_triplet)-//')"
ln -vsf "$(gcc_triplet)-\${tool}" "/usr/local/bin/\${tool}"
done
# Also create symlinks for `cc` and `c++`.
ln -vsf "/usr/local/bin/gcc" "/usr/local/bin/cc"
ln -vsf "/usr/local/bin/g++" "/usr/local/bin/c++"
"""
my_chroot(gcc_install_cmd)
my_chroot(gcc_symlink_cmd)

# Show what is installed
my_chroot("which gcc")
my_chroot("which -a gcc")
my_chroot("which g++")
my_chroot("which -a g++")
my_chroot("which ld")
my_chroot("which -a ld")

my_chroot("gcc --version")
my_chroot("g++ --version")
my_chroot("ld --version")
end
48 changes: 48 additions & 0 deletions src/build_img/rust.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# Install a pinned Rust toolchain (rustc + cargo + std) into a Debian-based
# rootfs.
#
# Julia's MMTk garbage collector (`WITH_THIRD_PARTY_GC=mmtk`) ships a Rust
# binding that is compiled from source with `cargo build` during the Julia
# build. The binding's `rust-toolchain` file pins an exact compiler version, so
# the version we install here must be kept in sync with it:
# JuliaLang/julia : src/gc-mmtk/mmtk_julia/rust-toolchain
#
# We install the official standalone Rust distribution (NOT `rustup`) straight
# into `/usr/local`, so that `cargo`/`rustc` land on the default PATH and work
# in the non-login, non-interactive shells that Buildkite uses -- with no
# `RUSTUP_HOME`/`CARGO_HOME` plumbing and no toolchain download required at
# Julia build time. Because there is no `rustup` proxy, the `rust-toolchain`
# pin is satisfied simply by installing that exact version here.
const DEFAULT_RUST_VERSION = "1.92.0"

function rust_target(arch::String)
rust_target_mapping = Dict(
"x86_64" => "x86_64-unknown-linux-gnu",
"i686" => "i686-unknown-linux-gnu",
"aarch64" => "aarch64-unknown-linux-gnu",
"armv7l" => "armv7-unknown-linux-gnueabihf",
"powerpc64le" => "powerpc64le-unknown-linux-gnu",
)
return rust_target_mapping[normalize_arch(arch)]
end

function install_rust(rootfs::String, chroot_ENV::AbstractDict, arch::String;
version::String = DEFAULT_RUST_VERSION)
target = rust_target(arch)
my_chroot(args...) = root_chroot(rootfs, "bash", "-eu", "-o", "pipefail", "-c", args...; ENV=chroot_ENV)
my_chroot("""
tarball="rust-$(version)-$(target).tar.gz"
curl -fsSL "https://static.rust-lang.org/dist/\${tarball}" -o "/tmp/\${tarball}"
mkdir -p /tmp/rust-install
tar -C /tmp/rust-install --strip-components=1 -xf "/tmp/\${tarball}"
/tmp/rust-install/install.sh \\
--prefix=/usr/local \\
--components=rustc,cargo,rust-std-$(target)
rm -rf /tmp/rust-install "/tmp/\${tarball}"
""")

# Print the installed versions, and fail the image build if `rustc`/`cargo`
# are broken or the wrong version landed on the PATH.
my_chroot("rustc --version")
my_chroot("cargo --version")
end
Loading