Implement Windows privatization of bundled libjulia

src/JuliaC.jl

@@ -67,6 +67,7 @@ include("patchversion.jl")
 include("privatize_common.jl")
 include("privatize_linux.jl")
 include("privatize_macos.jl")
+include("privatize_windows.jl")
 
 export ImageRecipe, LinkRecipe, BundleRecipe
 export compile_products, link_products, bundle_products

src/bundling.jl

@@ -111,6 +111,8 @@ function privatize_libjulia!(recipe::BundleRecipe)
         privatize_libjulia_macos!(recipe)
     elseif Sys.islinux()
         privatize_libjulia_linux!(recipe)
+    elseif Sys.iswindows()
+        privatize_libjulia_windows!(recipe)
     else
         @warn "Privatization not implemented for this OS"
     end

src/linking.jl

@@ -41,6 +41,32 @@ function get_rpath(recipe::LinkRecipe)
     return string(flag1, " ", flag2)
 end
 
+# Resolve the bin/ directory of the bundled mingw-w64 toolchain (winlibs MinGW-w64
+# GCC distribution). This artifact is os="windows"-gated and lazy, so this only
+# resolves on Windows. It ships the full binutils suite (gcc.exe, g++.exe, objcopy.exe,
+# windres.exe, ld.exe, dlltool.exe, as.exe) in one directory.
+function mingw_bindir()
+    # The `@artifact_str` macro expands at compile time and the artifact entry is
+    # os="windows"-gated, so it is unresolvable off-Windows. Guard with `@static if`
+    # so the macro is only expanded (and the lookup only attempted) on Windows; on
+    # other platforms this helper is never reached by the privatization path.
+    @static if Sys.iswindows()
+        return joinpath(LazyArtifacts.artifact"mingw-w64",
+                        "extracted_files",
+                        (Int == Int64 ? "mingw64" : "mingw32"),
+                        "bin")
+    else
+        error("mingw_bindir() is only available on Windows")
+    end
+end
+
+# Absolute path to a named tool (e.g. "gcc.exe", "objcopy.exe") in the mingw bin dir.
+function mingw_tool(name::AbstractString)
+    tool = joinpath(mingw_bindir(), name)
+    isfile(tool) || error("expected mingw tool not found: $tool")
+    return tool
+end
+
 function get_compiler_cmd(; cplusplus::Bool=false)
     cc = get(ENV, "JULIA_CC", nothing)
     path = nothing
@@ -49,11 +75,7 @@ function get_compiler_cmd(; cplusplus::Bool=false)
         path = nothing
     else
         @static if Sys.iswindows()
-            path = joinpath(LazyArtifacts.artifact"mingw-w64",
-                            "extracted_files",
-                            (Int==Int64 ? "mingw64" : "mingw32"),
-                            "bin",
-                            cplusplus ? "g++.exe" : "gcc.exe")
+            path = mingw_tool(cplusplus ? "g++.exe" : "gcc.exe")
             compiler_cmd = `$path`
         else
             compilers_cpp = ("g++", "clang++")

src/privatize_windows.jl

@@ -0,0 +1,177 @@
+"""
+Windows-specific privatization for libjulia.
+
+Unlike the Unix paths (salt-rename via `privatize_libjulia_common!`), Windows has no
+SONAME, no symbol versioning, and no DT_NEEDED, so a different mechanism is used:
+
+1. Inject an SxS private-assembly `RT_MANIFEST` resource into the built product
+   (`.exe`/`.dll`). Its `<file>` entries list the bundled `libjulia*.dll` siblings,
+   creating an activation context that makes the loader prefer the copies sitting in the
+   product's own directory over any same-named DLL already on `PATH`.
+2. Strip the stale `../bin/` prefix from the bundled `libjulia.dll`'s embedded,
+   colon-separated library search path (the flat `bin/` bundle layout has no `../bin/`).
+
+This file is standalone: it does not use `PrivatizePlatform` / the `plat_*` hooks and never
+calls `privatize_libjulia_common!`.
+"""
+
+using ObjectFile
+using StructIO
+import ObjectFile: COFF, Sections, section_address, section_offset, findfirst
+
+# Dir holding rsrc.bin (88-byte precompiled RT_MANIFEST header); @path so it survives bundling.
+const TEMPLATE_DIR = @path joinpath(@__DIR__, "template")
+
+# Offsets of the two patched UInt32 fields from the start of the .rsrc section (IMAGE_RESOURCE_DATA_ENTRY @ 0x48).
+const MANIFEST_ADDRESS_OFFSET = UInt(0x48)  # IMAGE_RESOURCE_DATA_ENTRY.OffsetToData
+const MANIFEST_SIZE_OFFSET    = UInt(0x4c)  # IMAGE_RESOURCE_DATA_ENTRY.Size
+
+"""
+    generate_manifest_xml(identity_name, dll_names) -> Vector{UInt8}
+
+Build the SxS private-assembly RT_MANIFEST XML. `identity_name` is the assembly identity
+label; `dll_names` is the list of DLL filenames to redirect to the product's own directory.
+Returns the UTF-8 bytes.
+"""
+function generate_manifest_xml(identity_name::AbstractString, dll_names)
+    io = IOBuffer()
+    print(io, "<assembly xmlns=\"urn:schemas-microsoft-com:asm.v1\" manifestVersion=\"1.0\">\n")
+    print(io, "    <assemblyIdentity type=\"win32\" name=\"", identity_name,
+              "\" version=\"1.0.0.0\"></assemblyIdentity>\n")
+    for dll in dll_names
+        print(io, "    <file name=\"", dll, "\"></file>\n")
+    end
+    print(io, "</assembly>\n")
+    return Vector{UInt8}(String(take!(io)))
+end
+
+# Per-product SxS assembly identity from the product basename (avoids SxS-cache clashes), name-safe.
+function manifest_identity_for(product_path::AbstractString)
+    stem = first(splitext(basename(product_path)))      # strip .exe/.dll
+    safe = replace(stem, r"[^A-Za-z0-9._-]" => "_")
+    return string("JuliaC.PrivateRuntime.", safe)
+end
+
+# The libjulia DLLs the manifest may redirect, in canonical order.
+const LIBJULIA_DLL_CANDIDATES =
+    ("libjulia.dll", "libjulia-internal.dll", "libjulia-codegen.dll")
+
+"""
+    inject_private_manifest!(product_path, dll_names)
+
+Add the SxS `RT_MANIFEST` resource (listing `dll_names`) to the PE at `product_path`:
+build a `.rsrc` payload (precompiled header ++ generated manifest ++ 4-byte pad), add it as
+a `.rsrc` section with mingw `objcopy`, then patch the COFF optional header's ResourceTable
+data directory and the section's internal manifest address/size fields.
+"""
+function inject_private_manifest!(product_path, dll_names)
+    header = read(joinpath(TEMPLATE_DIR, "rsrc.bin"))
+    manifest = generate_manifest_xml(manifest_identity_for(product_path), dll_names)
+
+    # Build the section payload: header ++ manifest ++ pad-to-4-bytes.
+    sectionfile = joinpath(dirname(product_path), "rsrc.bin")
+    open(sectionfile, "w") do rsrc_bin
+        write(rsrc_bin, header)
+        write(rsrc_bin, manifest)
+        if length(manifest) % sizeof(UInt32) != 0
+            padding_size = sizeof(UInt32) - length(manifest) % sizeof(UInt32)
+            write(rsrc_bin, zeros(UInt8, padding_size))
+        end
+    end
+
+    # Add the section using objcopy from the mingw artifact already used for linking.
+    objcopy = mingw_tool("objcopy.exe")            # WINDOWS-CI-ONLY: artifact + run
+    run(`$objcopy --add-section .rsrc=$sectionfile --set-section-flags .rsrc=data $product_path`)
+    rm(sectionfile)
+
+    # Re-open and patch the headers now that objcopy has placed the section.
+    open(product_path; read=true, write=true, create=false, truncate=false) do io
+        oh = only(ObjectFile.readmeta(io))
+        rsrc_section = findfirst(Sections(oh), ".rsrc")
+        rsrc_section === nothing && error("objcopy did not create a .rsrc section in $product_path")
+
+        # 1) Patch the optional header's ResourceTable data directory.
+        magic = oh.opt_header.standard.Magic
+        datadirs_offset = if magic == COFF.OPTHEADER_STANDARD_MAGIC32
+            oh.header_offset + sizeof(COFF.COFFHeader) + sizeof(COFF.COFFOptionalHeaderStandard) +
+                sizeof(UInt32) + sizeof(COFF.COFFOptionalHeaderWindows32)
+        elseif magic == COFF.OPTHEADER_STANDARD_MAGIC64
+            oh.header_offset + sizeof(COFF.COFFHeader) + sizeof(COFF.COFFOptionalHeaderStandard) +
+                sizeof(COFF.COFFOptionalHeaderWindows64)
+        else
+            error("unexpected COFF optional-header magic: 0x$(string(magic, base=16))")
+        end
+        seek(oh, datadirs_offset + PatchVersion.fieldname_offset(COFF.COFFDataDirectories, :ResourceTable))
+        pack(ObjectFile.handle(oh).io, COFF.COFFImageDataDirectory(
+            #= VirtualAddress =# section_address(rsrc_section),
+            #= Size          =# rsrc_section.section.VirtualSize,
+        ))
+
+        # 2) Patch the .rsrc data-entry's manifest address + size (relative until VA known).
+        seek(oh, section_offset(rsrc_section) + MANIFEST_ADDRESS_OFFSET)
+        write(ObjectFile.handle(oh).io, UInt32(section_address(rsrc_section) + sizeof(header)))
+        seek(oh, section_offset(rsrc_section) + MANIFEST_SIZE_OFFSET)
+        write(ObjectFile.handle(oh).io, UInt32(length(manifest)))
+    end
+    return nothing
+end
+
+"""
+    fix_libjulia_libpath!(libjulia_path)
+
+Strip the stale `../bin/` prefix from the bundled `libjulia.dll`'s embedded, colon-separated
+library search path (rewriting `@../bin/` -> `@`), in place, NUL-terminated. The rewrite only
+removes bytes, so it never grows the string and is safe to do in place.
+"""
+function fix_libjulia_libpath!(libjulia_path)
+    if !isfile(libjulia_path)
+        error("Unable to open libjulia.dll at $(libjulia_path)")
+    end
+    open(libjulia_path; read = true, write = true) do io
+        needle = "../bin/"
+        readuntil(io, needle)
+        skip(io, -length(needle))
+        libpath_offset = position(io)
+
+        libpath = split(String(readuntil(io, UInt8(0))), ":")
+        libpath = map(libpath) do l
+            if startswith(l, "../bin/")
+                return l[8:end]
+            elseif startswith(l, "@../bin/")
+                return "@" * l[9:end]
+            end
+            return l
+        end
+
+        seek(io, libpath_offset)
+        write(io, join(libpath, ":"))
+        write(io, UInt8(0))
+    end
+end
+
+# The libjulia DLLs actually present in the bundle bin/ dir, in canonical order.
+function present_libjulia_dlls(bindir)
+    return [dll for dll in LIBJULIA_DLL_CANDIDATES if isfile(joinpath(bindir, dll))]
+end
+
+"""
+Windows privatization entry point: inject the SxS manifest into the built product and fix
+the bundled libjulia.dll's embedded libpath. Standalone; does not use the plat_* hooks.
+"""
+function privatize_libjulia_windows!(recipe::BundleRecipe)
+    try
+        # On Windows the bundle libdir is "bin" and the product + DLLs are co-located there.
+        bindir = joinpath(recipe.output_dir, recipe.libdir)
+        product = recipe.link_recipe.outname
+        libjulia = joinpath(bindir, "libjulia.dll")
+
+        dll_names = present_libjulia_dlls(bindir)
+        isempty(dll_names) && error("no libjulia*.dll found in $bindir to privatize")
+
+        inject_private_manifest!(product, dll_names)
+        fix_libjulia_libpath!(libjulia)
+    catch e
+        error("Failed to privatize libjulia on Windows", e)
+    end
+    return nothing
+end

test/programatic.jl

@@ -261,6 +261,115 @@
             @test ver.minor == 32767
         end
     end
+
+    @testset "Windows privatization injects manifest + fixes libpath" begin
+        if Sys.iswindows()
+            outdir = mktempdir()
+            libname = "libwinprivtest"
+            libout = joinpath(outdir, libname)
+            link = JuliaC.LinkRecipe(image_recipe=img_lib, outname=libout, rpath=JuliaC.RPATH_BUNDLE)
+            JuliaC.link_products(link)
+            bun = JuliaC.BundleRecipe(link_recipe=link, output_dir=outdir, privatize=true)
+            JuliaC.bundle_products(bun)
+
+            product = joinpath(outdir, "bin", libname * "." * Base.BinaryPlatforms.platform_dlext())
+            @test isfile(product)
+
+            # (a) The product PE now has a .rsrc section (read via ObjectFile, already a dep).
+            rsrc_va = open(product) do io
+                oh = only(JuliaC.ObjectFile.readmeta(io))
+                rsrc = JuliaC.ObjectFile.findfirst(JuliaC.ObjectFile.Sections(oh), ".rsrc")
+                @test rsrc !== nothing
+                rsrc === nothing ? UInt32(0) : UInt32(JuliaC.ObjectFile.section_address(rsrc))
+            end
+
+            # (b) The optional header's ResourceTable data directory now points at .rsrc.
+            #     Read it with raw seek/read to mirror the ld_flags testset's style and to
+            #     independently confirm the COFF patch landed (datadirs_offset reasoning
+            #     verified against a real PE during planning).
+            rt = open(product) do io
+                seek(io, 0x3C); pe_off = read(io, UInt32)
+                opt_hdr = pe_off + 4 + 20                 # PE sig (4) + COFF header (20)
+                seek(io, opt_hdr); magic = read(io, UInt16)
+                # 64-bit: standard(24) + windows64(88) = 112 to the data directories;
+                # 32-bit: standard(24) + BaseOfData(4) + windows32(68) = 96.
+                dd = opt_hdr + (magic == 0x20b ? 112 : 96)
+                seek(io, dd + 0x10)                        # ResourceTable slot
+                (va = read(io, UInt32), size = read(io, UInt32))
+            end
+            @test rt.va == rsrc_va
+            @test rt.va != 0
+            @test rt.size != 0
+
+            # (c) The bundled libjulia.dll no longer contains the "../bin/" prefix.
+            libjulia = joinpath(outdir, "bin", "libjulia.dll")
+            @test isfile(libjulia)
+            @test !occursin("../bin/", String(read(libjulia)))
+        end
+    end
+
+    @testset "Privatized library loads its own runtime copy (Windows)" begin
+        # SxS-manifest guarantee: loaded into a process that already has the runtime, the product loads its own sibling DLLs, so Libdl.dllist() shows two distinct runtime copies.
+        if Sys.iswindows()
+            outdir = mktempdir()
+            libname = "libwinprivloadtest"
+            libout = joinpath(outdir, libname)
+            link = JuliaC.LinkRecipe(image_recipe=img_lib, outname=libout, rpath=JuliaC.RPATH_BUNDLE)
+            JuliaC.link_products(link)
+            bun = JuliaC.BundleRecipe(link_recipe=link, output_dir=outdir, privatize=true)
+            JuliaC.bundle_products(bun)
+
+            bindir = joinpath(outdir, "bin")
+            product = joinpath(bindir, libname * "." * Base.BinaryPlatforms.platform_dlext())
+            @test isfile(product)
+            @test isfile(joinpath(bindir, "libjulia.dll"))
+
+            # Load the product in a fresh Julia process (which already has the runtime loaded) and inspect Libdl.dllist(), mirroring the Unix "Julia dlopen test" above.
+            # Flushed markers localize the hang (M2 dirs right after dlopen tells us whether
+            # SxS activated, BEFORE the first ccall that triggers the product's runtime init);
+            # the watchdog ensures CI can never again hit the 6h job limit.
+            product_literal = repr(product)
+            bindir_literal = repr(bindir)
+            snippet = """
+                using Libdl
+                errln(s) = (println(stderr, s); flush(stderr))
+                libdirs() = unique(map(p -> lowercase(abspath(dirname(p))),
+                                       filter(p -> occursin("libjulia", lowercase(basename(p))), Libdl.dllist())))
+                errln("M1_PRE_DLOPEN dirs=" * string(libdirs()))
+                h = Libdl.dlopen($product_literal, Libdl.RTLD_LOCAL)
+                errln("M2_POST_DLOPEN dirs=" * string(libdirs()))   # bundled dir present here => SxS activated
+                sym = Libdl.dlsym(h, :jc_add_one)
+                errln("M3_POST_DLSYM")
+                errln("M4_PRE_CCALL")
+                r = ccall(sym, Cint, (Cint,), 41)                   # first ccall triggers the product's runtime init
+                errln("M5_POST_CCALL r=" * string(r))
+                println("RESULT=", r)
+                d = libdirs()
+                println("DISTINCT_DIRS=", length(d))
+                println("HAS_BUNDLED=", lowercase(abspath($bindir_literal)) in d)
+                try Libdl.dlclose(h) catch end
+                """
+            logf = joinpath(outdir, "winload.log")
+            proc = run(pipeline(`$(Base.julia_cmd()) --startup-file=no --history-file=no -e $snippet`;
+                                stdout=logf, stderr=logf); wait=false)
+            @async begin
+                sleep(180)
+                Base.process_running(proc) && (kill(proc); @warn "Windows load test exceeded 180s; killed")
+            end
+            wait(proc)
+            out = isfile(logf) ? read(logf, String) : ""
+            @info "Windows privatized-load diagnostic:\n$out"
+
+            # The exported symbol still resolves through the privatized runtime.
+            @test occursin("RESULT=42", out)
+            # The bundled directory is one of the distinct runtime-library locations.
+            @test occursin("HAS_BUNDLED=true", out)
+            # >= 2 distinct dirs now host the runtime libs: host/system + bundled.
+            m = match(r"DISTINCT_DIRS=(\d+)", out)
+            @test m !== nothing
+            @test parse(Int, m.captures[1]) >= 2
+        end
+    end
 end
 
 @testset "Programmatic binary (trim)" begin