diff --git a/src/JuliaC.jl b/src/JuliaC.jl index 1520fdf..16ceb08 100644 --- a/src/JuliaC.jl +++ b/src/JuliaC.jl @@ -67,6 +67,7 @@ include("patchversion.jl") include("privatize_common.jl") include("privatize_linux.jl") include("privatize_macos.jl") +include("privatize_windows.jl") export ImageRecipe, LinkRecipe, BundleRecipe export compile_products, link_products, bundle_products diff --git a/src/bundling.jl b/src/bundling.jl index 66d74b1..e8ba8a9 100644 --- a/src/bundling.jl +++ b/src/bundling.jl @@ -111,6 +111,8 @@ function privatize_libjulia!(recipe::BundleRecipe) privatize_libjulia_macos!(recipe) elseif Sys.islinux() privatize_libjulia_linux!(recipe) + elseif Sys.iswindows() + privatize_libjulia_windows!(recipe) else @warn "Privatization not implemented for this OS" end diff --git a/src/linking.jl b/src/linking.jl index 5038909..92239da 100644 --- a/src/linking.jl +++ b/src/linking.jl @@ -41,6 +41,32 @@ function get_rpath(recipe::LinkRecipe) return string(flag1, " ", flag2) end +# Resolve the bin/ directory of the bundled mingw-w64 toolchain (winlibs MinGW-w64 +# GCC distribution). This artifact is os="windows"-gated and lazy, so this only +# resolves on Windows. It ships the full binutils suite (gcc.exe, g++.exe, objcopy.exe, +# windres.exe, ld.exe, dlltool.exe, as.exe) in one directory. +function mingw_bindir() + # The `@artifact_str` macro expands at compile time and the artifact entry is + # os="windows"-gated, so it is unresolvable off-Windows. Guard with `@static if` + # so the macro is only expanded (and the lookup only attempted) on Windows; on + # other platforms this helper is never reached by the privatization path. + @static if Sys.iswindows() + return joinpath(LazyArtifacts.artifact"mingw-w64", + "extracted_files", + (Int == Int64 ? "mingw64" : "mingw32"), + "bin") + else + error("mingw_bindir() is only available on Windows") + end +end + +# Absolute path to a named tool (e.g. "gcc.exe", "objcopy.exe") in the mingw bin dir. +function mingw_tool(name::AbstractString) + tool = joinpath(mingw_bindir(), name) + isfile(tool) || error("expected mingw tool not found: $tool") + return tool +end + function get_compiler_cmd(; cplusplus::Bool=false) cc = get(ENV, "JULIA_CC", nothing) path = nothing @@ -49,11 +75,7 @@ function get_compiler_cmd(; cplusplus::Bool=false) path = nothing else @static if Sys.iswindows() - path = joinpath(LazyArtifacts.artifact"mingw-w64", - "extracted_files", - (Int==Int64 ? "mingw64" : "mingw32"), - "bin", - cplusplus ? "g++.exe" : "gcc.exe") + path = mingw_tool(cplusplus ? "g++.exe" : "gcc.exe") compiler_cmd = `$path` else compilers_cpp = ("g++", "clang++") diff --git a/src/privatize_windows.jl b/src/privatize_windows.jl new file mode 100644 index 0000000..9e9f25f --- /dev/null +++ b/src/privatize_windows.jl @@ -0,0 +1,177 @@ +""" +Windows-specific privatization for libjulia. + +Unlike the Unix paths (salt-rename via `privatize_libjulia_common!`), Windows has no +SONAME, no symbol versioning, and no DT_NEEDED, so a different mechanism is used: + +1. Inject an SxS private-assembly `RT_MANIFEST` resource into the built product + (`.exe`/`.dll`). Its `` entries list the bundled `libjulia*.dll` siblings, + creating an activation context that makes the loader prefer the copies sitting in the + product's own directory over any same-named DLL already on `PATH`. +2. Strip the stale `../bin/` prefix from the bundled `libjulia.dll`'s embedded, + colon-separated library search path (the flat `bin/` bundle layout has no `../bin/`). + +This file is standalone: it does not use `PrivatizePlatform` / the `plat_*` hooks and never +calls `privatize_libjulia_common!`. +""" + +using ObjectFile +using StructIO +import ObjectFile: COFF, Sections, section_address, section_offset, findfirst + +# Dir holding rsrc.bin (88-byte precompiled RT_MANIFEST header); @path so it survives bundling. +const TEMPLATE_DIR = @path joinpath(@__DIR__, "template") + +# Offsets of the two patched UInt32 fields from the start of the .rsrc section (IMAGE_RESOURCE_DATA_ENTRY @ 0x48). +const MANIFEST_ADDRESS_OFFSET = UInt(0x48) # IMAGE_RESOURCE_DATA_ENTRY.OffsetToData +const MANIFEST_SIZE_OFFSET = UInt(0x4c) # IMAGE_RESOURCE_DATA_ENTRY.Size + +""" + generate_manifest_xml(identity_name, dll_names) -> Vector{UInt8} + +Build the SxS private-assembly RT_MANIFEST XML. `identity_name` is the assembly identity +label; `dll_names` is the list of DLL filenames to redirect to the product's own directory. +Returns the UTF-8 bytes. +""" +function generate_manifest_xml(identity_name::AbstractString, dll_names) + io = IOBuffer() + print(io, "\n") + print(io, " \n") + for dll in dll_names + print(io, " \n") + end + print(io, "\n") + return Vector{UInt8}(String(take!(io))) +end + +# Per-product SxS assembly identity from the product basename (avoids SxS-cache clashes), name-safe. +function manifest_identity_for(product_path::AbstractString) + stem = first(splitext(basename(product_path))) # strip .exe/.dll + safe = replace(stem, r"[^A-Za-z0-9._-]" => "_") + return string("JuliaC.PrivateRuntime.", safe) +end + +# The libjulia DLLs the manifest may redirect, in canonical order. +const LIBJULIA_DLL_CANDIDATES = + ("libjulia.dll", "libjulia-internal.dll", "libjulia-codegen.dll") + +""" + inject_private_manifest!(product_path, dll_names) + +Add the SxS `RT_MANIFEST` resource (listing `dll_names`) to the PE at `product_path`: +build a `.rsrc` payload (precompiled header ++ generated manifest ++ 4-byte pad), add it as +a `.rsrc` section with mingw `objcopy`, then patch the COFF optional header's ResourceTable +data directory and the section's internal manifest address/size fields. +""" +function inject_private_manifest!(product_path, dll_names) + header = read(joinpath(TEMPLATE_DIR, "rsrc.bin")) + manifest = generate_manifest_xml(manifest_identity_for(product_path), dll_names) + + # Build the section payload: header ++ manifest ++ pad-to-4-bytes. + sectionfile = joinpath(dirname(product_path), "rsrc.bin") + open(sectionfile, "w") do rsrc_bin + write(rsrc_bin, header) + write(rsrc_bin, manifest) + if length(manifest) % sizeof(UInt32) != 0 + padding_size = sizeof(UInt32) - length(manifest) % sizeof(UInt32) + write(rsrc_bin, zeros(UInt8, padding_size)) + end + end + + # Add the section using objcopy from the mingw artifact already used for linking. + objcopy = mingw_tool("objcopy.exe") # WINDOWS-CI-ONLY: artifact + run + run(`$objcopy --add-section .rsrc=$sectionfile --set-section-flags .rsrc=data $product_path`) + rm(sectionfile) + + # Re-open and patch the headers now that objcopy has placed the section. + open(product_path; read=true, write=true, create=false, truncate=false) do io + oh = only(ObjectFile.readmeta(io)) + rsrc_section = findfirst(Sections(oh), ".rsrc") + rsrc_section === nothing && error("objcopy did not create a .rsrc section in $product_path") + + # 1) Patch the optional header's ResourceTable data directory. + magic = oh.opt_header.standard.Magic + datadirs_offset = if magic == COFF.OPTHEADER_STANDARD_MAGIC32 + oh.header_offset + sizeof(COFF.COFFHeader) + sizeof(COFF.COFFOptionalHeaderStandard) + + sizeof(UInt32) + sizeof(COFF.COFFOptionalHeaderWindows32) + elseif magic == COFF.OPTHEADER_STANDARD_MAGIC64 + oh.header_offset + sizeof(COFF.COFFHeader) + sizeof(COFF.COFFOptionalHeaderStandard) + + sizeof(COFF.COFFOptionalHeaderWindows64) + else + error("unexpected COFF optional-header magic: 0x$(string(magic, base=16))") + end + seek(oh, datadirs_offset + PatchVersion.fieldname_offset(COFF.COFFDataDirectories, :ResourceTable)) + pack(ObjectFile.handle(oh).io, COFF.COFFImageDataDirectory( + #= VirtualAddress =# section_address(rsrc_section), + #= Size =# rsrc_section.section.VirtualSize, + )) + + # 2) Patch the .rsrc data-entry's manifest address + size (relative until VA known). + seek(oh, section_offset(rsrc_section) + MANIFEST_ADDRESS_OFFSET) + write(ObjectFile.handle(oh).io, UInt32(section_address(rsrc_section) + sizeof(header))) + seek(oh, section_offset(rsrc_section) + MANIFEST_SIZE_OFFSET) + write(ObjectFile.handle(oh).io, UInt32(length(manifest))) + end + return nothing +end + +""" + fix_libjulia_libpath!(libjulia_path) + +Strip the stale `../bin/` prefix from the bundled `libjulia.dll`'s embedded, colon-separated +library search path (rewriting `@../bin/` -> `@`), in place, NUL-terminated. The rewrite only +removes bytes, so it never grows the string and is safe to do in place. +""" +function fix_libjulia_libpath!(libjulia_path) + if !isfile(libjulia_path) + error("Unable to open libjulia.dll at $(libjulia_path)") + end + open(libjulia_path; read = true, write = true) do io + needle = "../bin/" + readuntil(io, needle) + skip(io, -length(needle)) + libpath_offset = position(io) + + libpath = split(String(readuntil(io, UInt8(0))), ":") + libpath = map(libpath) do l + if startswith(l, "../bin/") + return l[8:end] + elseif startswith(l, "@../bin/") + return "@" * l[9:end] + end + return l + end + + seek(io, libpath_offset) + write(io, join(libpath, ":")) + write(io, UInt8(0)) + end +end + +# The libjulia DLLs actually present in the bundle bin/ dir, in canonical order. +function present_libjulia_dlls(bindir) + return [dll for dll in LIBJULIA_DLL_CANDIDATES if isfile(joinpath(bindir, dll))] +end + +""" +Windows privatization entry point: inject the SxS manifest into the built product and fix +the bundled libjulia.dll's embedded libpath. Standalone; does not use the plat_* hooks. +""" +function privatize_libjulia_windows!(recipe::BundleRecipe) + try + # On Windows the bundle libdir is "bin" and the product + DLLs are co-located there. + bindir = joinpath(recipe.output_dir, recipe.libdir) + product = recipe.link_recipe.outname + libjulia = joinpath(bindir, "libjulia.dll") + + dll_names = present_libjulia_dlls(bindir) + isempty(dll_names) && error("no libjulia*.dll found in $bindir to privatize") + + inject_private_manifest!(product, dll_names) + fix_libjulia_libpath!(libjulia) + catch e + error("Failed to privatize libjulia on Windows", e) + end + return nothing +end diff --git a/src/template/rsrc.bin b/src/template/rsrc.bin new file mode 100644 index 0000000..995142e Binary files /dev/null and b/src/template/rsrc.bin differ diff --git a/test/programatic.jl b/test/programatic.jl index ad8076d..fe3e67c 100644 --- a/test/programatic.jl +++ b/test/programatic.jl @@ -261,6 +261,115 @@ @test ver.minor == 32767 end end + + @testset "Windows privatization injects manifest + fixes libpath" begin + if Sys.iswindows() + outdir = mktempdir() + libname = "libwinprivtest" + libout = joinpath(outdir, libname) + link = JuliaC.LinkRecipe(image_recipe=img_lib, outname=libout, rpath=JuliaC.RPATH_BUNDLE) + JuliaC.link_products(link) + bun = JuliaC.BundleRecipe(link_recipe=link, output_dir=outdir, privatize=true) + JuliaC.bundle_products(bun) + + product = joinpath(outdir, "bin", libname * "." * Base.BinaryPlatforms.platform_dlext()) + @test isfile(product) + + # (a) The product PE now has a .rsrc section (read via ObjectFile, already a dep). + rsrc_va = open(product) do io + oh = only(JuliaC.ObjectFile.readmeta(io)) + rsrc = JuliaC.ObjectFile.findfirst(JuliaC.ObjectFile.Sections(oh), ".rsrc") + @test rsrc !== nothing + rsrc === nothing ? UInt32(0) : UInt32(JuliaC.ObjectFile.section_address(rsrc)) + end + + # (b) The optional header's ResourceTable data directory now points at .rsrc. + # Read it with raw seek/read to mirror the ld_flags testset's style and to + # independently confirm the COFF patch landed (datadirs_offset reasoning + # verified against a real PE during planning). + rt = open(product) do io + seek(io, 0x3C); pe_off = read(io, UInt32) + opt_hdr = pe_off + 4 + 20 # PE sig (4) + COFF header (20) + seek(io, opt_hdr); magic = read(io, UInt16) + # 64-bit: standard(24) + windows64(88) = 112 to the data directories; + # 32-bit: standard(24) + BaseOfData(4) + windows32(68) = 96. + dd = opt_hdr + (magic == 0x20b ? 112 : 96) + seek(io, dd + 0x10) # ResourceTable slot + (va = read(io, UInt32), size = read(io, UInt32)) + end + @test rt.va == rsrc_va + @test rt.va != 0 + @test rt.size != 0 + + # (c) The bundled libjulia.dll no longer contains the "../bin/" prefix. + libjulia = joinpath(outdir, "bin", "libjulia.dll") + @test isfile(libjulia) + @test !occursin("../bin/", String(read(libjulia))) + end + end + + @testset "Privatized library loads its own runtime copy (Windows)" begin + # SxS-manifest guarantee: loaded into a process that already has the runtime, the product loads its own sibling DLLs, so Libdl.dllist() shows two distinct runtime copies. + if Sys.iswindows() + outdir = mktempdir() + libname = "libwinprivloadtest" + libout = joinpath(outdir, libname) + link = JuliaC.LinkRecipe(image_recipe=img_lib, outname=libout, rpath=JuliaC.RPATH_BUNDLE) + JuliaC.link_products(link) + bun = JuliaC.BundleRecipe(link_recipe=link, output_dir=outdir, privatize=true) + JuliaC.bundle_products(bun) + + bindir = joinpath(outdir, "bin") + product = joinpath(bindir, libname * "." * Base.BinaryPlatforms.platform_dlext()) + @test isfile(product) + @test isfile(joinpath(bindir, "libjulia.dll")) + + # Load the product in a fresh Julia process (which already has the runtime loaded) and inspect Libdl.dllist(), mirroring the Unix "Julia dlopen test" above. + # Flushed markers localize the hang (M2 dirs right after dlopen tells us whether + # SxS activated, BEFORE the first ccall that triggers the product's runtime init); + # the watchdog ensures CI can never again hit the 6h job limit. + product_literal = repr(product) + bindir_literal = repr(bindir) + snippet = """ + using Libdl + errln(s) = (println(stderr, s); flush(stderr)) + libdirs() = unique(map(p -> lowercase(abspath(dirname(p))), + filter(p -> occursin("libjulia", lowercase(basename(p))), Libdl.dllist()))) + errln("M1_PRE_DLOPEN dirs=" * string(libdirs())) + h = Libdl.dlopen($product_literal, Libdl.RTLD_LOCAL) + errln("M2_POST_DLOPEN dirs=" * string(libdirs())) # bundled dir present here => SxS activated + sym = Libdl.dlsym(h, :jc_add_one) + errln("M3_POST_DLSYM") + errln("M4_PRE_CCALL") + r = ccall(sym, Cint, (Cint,), 41) # first ccall triggers the product's runtime init + errln("M5_POST_CCALL r=" * string(r)) + println("RESULT=", r) + d = libdirs() + println("DISTINCT_DIRS=", length(d)) + println("HAS_BUNDLED=", lowercase(abspath($bindir_literal)) in d) + try Libdl.dlclose(h) catch end + """ + logf = joinpath(outdir, "winload.log") + proc = run(pipeline(`$(Base.julia_cmd()) --startup-file=no --history-file=no -e $snippet`; + stdout=logf, stderr=logf); wait=false) + @async begin + sleep(180) + Base.process_running(proc) && (kill(proc); @warn "Windows load test exceeded 180s; killed") + end + wait(proc) + out = isfile(logf) ? read(logf, String) : "" + @info "Windows privatized-load diagnostic:\n$out" + + # The exported symbol still resolves through the privatized runtime. + @test occursin("RESULT=42", out) + # The bundled directory is one of the distinct runtime-library locations. + @test occursin("HAS_BUNDLED=true", out) + # >= 2 distinct dirs now host the runtime libs: host/system + bundled. + m = match(r"DISTINCT_DIRS=(\d+)", out) + @test m !== nothing + @test parse(Int, m.captures[1]) >= 2 + end + end end @testset "Programmatic binary (trim)" begin