diff --git a/.github/workflows/Typos.yml b/.github/workflows/Typos.yml index 833eb2dacbc9e..25cf9fea7cd0b 100644 --- a/.github/workflows/Typos.yml +++ b/.github/workflows/Typos.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout the JuliaLang/julia repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Check spelling with typos diff --git a/.github/workflows/Whitespace.yml b/.github/workflows/Whitespace.yml index 30fb2fd5df1fc..3d0b8fe74d334 100644 --- a/.github/workflows/Whitespace.yml +++ b/.github/workflows/Whitespace.yml @@ -15,7 +15,7 @@ jobs: timeout-minutes: 2 steps: - name: Checkout the JuliaLang/julia repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2.6.1 diff --git a/.github/workflows/cffconvert.yml b/.github/workflows/cffconvert.yml index 5dbf119426be5..45f19078a8af8 100644 --- a/.github/workflows/cffconvert.yml +++ b/.github/workflows/cffconvert.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out a copy of the repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 289247e17d0e0..afa95d7dbca36 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -22,7 +22,7 @@ function can_propagate_conditional(@nospecialize(rt), argtypes::Vector{Any}) return false end return isa(argtypes[rt.slot], Conditional) && - is_const_bool_or_bottom(rt.thentype) && is_const_bool_or_bottom(rt.thentype) + is_const_bool_or_bottom(rt.thentype) && is_const_bool_or_bottom(rt.elsetype) end function propagate_conditional(rt::InterConditional, cond::Conditional) diff --git a/Compiler/src/precompile.jl b/Compiler/src/precompile.jl index ebcfbb0edf7bf..4110447c4a9e6 100644 --- a/Compiler/src/precompile.jl +++ b/Compiler/src/precompile.jl @@ -311,8 +311,7 @@ function enqueue_specialization!(all::Bool, worklist, mi::Core.MethodInstance) while codeinst !== nothing do_compile = false if codeinst.owner !== nothing - # TODO(vchuravy) native code caching for foreign interpreters - # Skip foreign code instances + # This code instance is from a foreign interpreter, so we skip it elseif use_const_api(codeinst) # Check if invoke is jl_fptr_const_return do_compile = true elseif codeinst.invoke != C_NULL || codeinst.precompile @@ -344,7 +343,8 @@ function compile_and_emit_native(worlds::Vector{UInt}, newmodules, # Vector{Module} or Nothing mod_array, # Vector{Module} or Nothing all::Bool, - module_init_order::Vector{Any}) # Vector{Module} + module_init_order::Vector{Any}, # Vector{Module} + ext_foreign_cis::Vector{Any}) # Vector{CodeInstance} latestworld = worlds[end] # Step 1: Precompile all __init__ methods that will be required @@ -375,7 +375,13 @@ function compile_and_emit_native(worlds::Vector{UInt}, if new_ext_cis !== nothing for i in 1:length(new_ext_cis::Vector{Any}) ci = new_ext_cis[i]::CodeInstance - enqueue_specialization!(all, specialization_worklist, get_ci_mi(ci)) + if ci.owner !== nothing + # enqueue_specialization will skip over CIs from foreign interpreters + # and currently will visit at most one (do_compile) CI per method instance + push!(ext_foreign_cis, ci) + else + enqueue_specialization!(all, specialization_worklist, get_ci_mi(ci)) + end end end end diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index c97cf8b7f62a5..ed6d37229d9bd 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -147,7 +147,7 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation inferred_result = maybe_compress_codeinfo(interp, mi, inferred_result) result.is_src_volatile = false elseif ci.owner === nothing - # The global cache can only handle objects that codegen understands + # The global cache can only handle objects that codegen understands (nothing or CodeInfo) inferred_result = nothing end end diff --git a/Compiler/src/verifytrim.jl b/Compiler/src/verifytrim.jl index eef2f15d43e86..0a77a860c2c8b 100644 --- a/Compiler/src/verifytrim.jl +++ b/Compiler/src/verifytrim.jl @@ -189,7 +189,6 @@ function verify_codeinstance!(interp::NativeInterpreter, codeinst::CodeInstance, end # TODO: check for calls to Base.atexit? elseif isexpr(stmt, :call) - error = "unresolved call" farg = stmt.args[1] ftyp = widenconst(argextype(farg, codeinfo, sptypes)) if ftyp <: IntrinsicFunction @@ -250,6 +249,8 @@ function verify_codeinstance!(interp::NativeInterpreter, codeinst::CodeInstance, elseif Core.memoryrefmodify! isa ftyp error = "trim verification not yet implemented for builtin `Core.memoryrefmodify!`" else @assert false "unexpected builtin" end + else + error = "unresolved call" end extyp = argextype(SSAValue(i), codeinfo, sptypes) if extyp === Union{} diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index f90ab7dff6655..286a98c2dcfba 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -1075,3 +1075,15 @@ let io = IOBuffer() str = String(take!(io)) @test occursin("julia.write_barrier", str) end + +# sret parameters must have an alignment attribute (required by LLVM LangRef). +@testset "sret alignment attribute" begin + struct SretAlignTest + a::Float32 + b::Float32 + c::Float32 + end + @noinline f_srettest(x::Float32) = SretAlignTest(x, x+1, x+2) + ir = get_llvm(f_srettest, Tuple{Float32}, true, true, true) + @test occursin(r"sret\([^)]+\) align \d+", ir) +end diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index f7101d4319351..89b7de36fb26a 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -6531,4 +6531,18 @@ function haskey_inference_test() end @inferred haskey_inference_test() +# issue #60883: conditional propagation through wrapper functions +mutable struct A60883 + a::Int +end +inner60883(a, b) = iszero(a.a) && !b +outer60883(a, b) = inner60883(a, b) +function issue60883() + a = A60883(0) + b = iszero(a.a) + if outer60883(a, b) else end + return b # should not be narrowed to Const(false) +end +@test issue60883() === true + end # module inference diff --git a/Make.inc b/Make.inc index b4a991116bbcf..8cd068cf8d752 100644 --- a/Make.inc +++ b/Make.inc @@ -1708,7 +1708,7 @@ endif CLANGSA_FLAGS := CLANGSA_CXXFLAGS := ifeq ($(OS), Darwin) # on new XCode, the files are hidden - CLANGSA_FLAGS += -isysroot $(shell xcrun --show-sdk-path -sdk macosx) + CLANGSA_FLAGS += -isysroot $(shell xcrun --show-sdk-path --sdk macosx) endif ifeq ($(USEGCC),1) # try to help clang find the c++ files for CC by guessing the value for --prefix diff --git a/NEWS.md b/NEWS.md index 8a2d55bacf9e5..dc596460b9d39 100644 --- a/NEWS.md +++ b/NEWS.md @@ -110,6 +110,11 @@ Deprecated or removed * The method `merge(combine::Callable, d::AbstractDict...)` is now deprecated to favor `mergewith` instead ([#59775]). +Deprecated or removed +--------------------- + +* The method `merge(combine::Callable, d::AbstractDict...)` is now deprecated to favor `mergewith` instead ([#59775]). + [#47102]: https://github.com/JuliaLang/julia/issues/47102 [#48507]: https://github.com/JuliaLang/julia/issues/48507 diff --git a/base/Base.jl b/base/Base.jl index 92d3ad2c04059..0775305b171c2 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -266,7 +266,7 @@ include("threadcall.jl") # code loading include("uuid.jl") include("pkgid.jl") -include("toml_parser.jl") +include("toml/toml.jl") include("linking.jl") include("loading.jl") diff --git a/base/Makefile b/base/Makefile index 0f7767e0aef37..5842b2f53ec50 100644 --- a/base/Makefile +++ b/base/Makefile @@ -161,11 +161,10 @@ endif define symlink_system_library libname_$2 := $$(notdir $(call versioned_libname,$2,$3)) -libpath_$2 := $$(shell $$(call spawn,$$(LIBWHICH)) -p $$(libname_$2) 2>/dev/null) symlink_$2: $$(build_private_libdir)/$$(libname_$2) $$(build_private_libdir)/$$(libname_$2): - @if [ -e "$$(libpath_$2)" ]; then \ - REALPATH=$$(libpath_$2); \ + @REALPATH=`$$(call spawn,$$(LIBWHICH)) -p $$(libname_$2) 2>/dev/null`; \ + if [ -e "$$$$REALPATH" ]; then \ $$(call resolve_path,REALPATH) && \ [ -e "$$$$REALPATH" ] && \ rm -f "$$@" && \ diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index 8817d82a6add6..95d80f30ba9f2 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -369,7 +369,7 @@ function metadata(__source__, __module__, expr, ismodule) if isa(eachex, Symbol) || isexpr(eachex, :(::)) # a field declaration if last_docstr !== nothing - push!(fields, P(namify(eachex), last_docstr)) + push!(fields, P(namify(eachex)::Symbol, last_docstr)) last_docstr = nothing end elseif isexpr(eachex, :function) || isexpr(eachex, :(=)) diff --git a/base/docs/bindings.jl b/base/docs/bindings.jl index fc72375e8cebe..2b7fcae579553 100644 --- a/base/docs/bindings.jl +++ b/base/docs/bindings.jl @@ -28,7 +28,14 @@ function Base.show(io::IO, b::Binding) if b.mod === Base.active_module() print(io, b.var) else - print(io, b.mod, '.', Base.isoperator(b.var) ? ":" : "", b.var) + print(io, b.mod, '.') + if Base.isoperator(b.var) + # ensures symbols are quoted right, so e.g. :(==), :(:), :+ or :- + show(io, b.var) + else + # print ordinary identifiers without any quoting + print(io, b.var) + end end end diff --git a/base/io.jl b/base/io.jl index 0f13e59baea8d..36e0a1f9949ec 100644 --- a/base/io.jl +++ b/base/io.jl @@ -31,7 +31,9 @@ struct SystemError <: Exception end lock(::IO) = nothing +typeof(lock).name.max_methods = UInt8(1) unlock(::IO) = nothing +typeof(unlock).name.max_methods = UInt8(1) """ reseteof(io) diff --git a/base/loading.jl b/base/loading.jl index 97200262be0bf..3bed81b6eb78d 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -261,7 +261,16 @@ struct LoadingCache located::Dict{Tuple{PkgId, Union{String, Nothing}}, Union{Tuple{String, String}, Nothing}} end const LOADING_CACHE = Ref{Union{LoadingCache, Nothing}}(nothing) # n.b.: all access to and through this are protected by require_lock -LoadingCache() = LoadingCache(load_path(), Dict(), Dict(), Dict(), Set(), Dict(), Dict(), Dict()) +LoadingCache() = LoadingCache( + load_path(), + Dict{String, UUID}(), + Dict{String, Union{Bool, String}}(), + Dict{String, Union{Nothing, String}}(), + Set{String}(), + Dict{Tuple{PkgId, String}, Union{Nothing, Tuple{PkgId, String}}}(), + Dict{String, Union{Nothing, Tuple{PkgId, String}}}(), + Dict{Tuple{PkgId, Union{String, Nothing}}, Union{Tuple{String, String}, Nothing}}() +) struct TOMLCache{Dates} @@ -617,13 +626,15 @@ the form `pkgversion(@__MODULE__)` can be used. This function was introduced in Julia 1.9. """ function pkgversion(m::Module) - path = pkgdir(m) - path === nothing && return nothing @lock require_lock begin - v = get_pkgversion_from_path(path) pkgorigin = get(pkgorigins, PkgId(moduleroot(m)), nothing) - # Cache the version - if pkgorigin !== nothing && pkgorigin.version === nothing + if pkgorigin !== nothing && pkgorigin.version !== nothing + return pkgorigin.version + end + path = pkgdir(m) + path === nothing && return nothing + v = get_pkgversion_from_path(path) + if pkgorigin !== nothing pkgorigin.version = v end return v @@ -1909,9 +1920,8 @@ function compilecache_freshest_path(pkg::PkgId; try # update timestamp of precompilation file so that it is the first to be tried by code loading touch(path_to_try) - catch ex + catch # file might be read-only and then we fail to update timestamp, which is fine - ex isa IOError || rethrow() end return path_to_try @label check_next_path @@ -2133,8 +2143,8 @@ end if stalecheck try touch(path_to_try) # update timestamp of precompilation file - catch ex # file might be read-only and then we fail to update timestamp, which is fine - ex isa IOError || rethrow() + catch + # file might be read-only and then we fail to update timestamp, which is fine end end # finish loading module graph into staledeps diff --git a/base/precompilation.jl b/base/precompilation.jl index f43fee770c94a..3c6c6fa996f06 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -1258,6 +1258,11 @@ function _color_string(cstr::String, col::Union{Int64, Symbol}, hascolor) end # Can be merged with `maybe_cachefile_lock` in loading? +# Wraps the precompilation function `f` with cachefile lock handling. +# Returns the result from `f()`, which can be: +# - `nothing`: cache already existed +# - `Tuple{String, Union{Nothing, String}}`: this process just compiled +# - `Exception`: compilation failed function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint::Bool, pkg_config, pkgspidlocked, hascolor, parallel_limiter::Base.Semaphore) if !(isdefined(Base, :mkpidlock_hook) && isdefined(Base, :trymkpidlock_hook) && Base.isdefined(Base, :parse_pidfile_hook)) return f() @@ -1284,7 +1289,7 @@ function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLo Base.release(parallel_limiter) # release so other work can be done while waiting try # wait until the lock is available - @invokelatest Base.mkpidlock_hook(() -> begin + cachefile = @invokelatest Base.mkpidlock_hook(() -> begin delete!(pkgspidlocked, pkg_config) Base.acquire(f, parallel_limiter) end, diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 4553ba0546db9..e7b37e428184b 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -1363,7 +1363,7 @@ julia> fieldtypes(Foo) (Int64, String) ``` """ -fieldtypes(T::Type) = (@_foldable_meta; ntupleany(i -> fieldtype(T, i), fieldcount(T))) +fieldtypes(@nospecialize T::Type) = (@_foldable_meta; ntupleany(i -> fieldtype(T, i), fieldcount(T))) # return all instances, for types that can be enumerated diff --git a/base/shell.jl b/base/shell.jl index 09b91918f2634..78b1af5645d2e 100644 --- a/base/shell.jl +++ b/base/shell.jl @@ -171,7 +171,7 @@ function shell_split(s::AbstractString) parsed = shell_parse(s, false)[1] args = String[] for arg in parsed - push!(args, string(arg...)) + push!(args, string(arg...)::String) end args end diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index 89cba6db42c8d..6bc0c76fdbaa8 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -250,7 +250,7 @@ function annotatedstring(xs...) size = filesize(s.io) if x isa AnnotatedString for annot in x.annotations - push!(annotations, setindex(annot, annot.region .+ size, :region)) + push!(annotations, @inline(setindex(annot, annot.region .+ size, :region))) end print(s, x.string) elseif x isa SubString{<:AnnotatedString} @@ -259,7 +259,7 @@ function annotatedstring(xs...) if start <= x.offset + x.ncodeunits && stop > x.offset rstart = size + max(0, start - x.offset - 1) + 1 rstop = size + min(stop, x.offset + x.ncodeunits) - x.offset - push!(annotations, setindex(annot, rstart:rstop, :region)) + push!(annotations, @inline(setindex(annot, rstart:rstop, :region))) end end print(s, SubString(x.string.string, x.offset, x.ncodeunits, Val(:noshift))) @@ -293,12 +293,12 @@ function repeat(str::AnnotatedString, r::Integer) elseif allequal(a -> a.region, str.annotations) && first(str.annotations).region == fullregion newfullregion = firstindex(unannot):lastindex(unannot) for annot in str.annotations - push!(annotations, setindex(annot, newfullregion, :region)) + push!(annotations, @inline(setindex(annot, newfullregion, :region))) end else for offset in 0:len:(r-1)*len for annot in str.annotations - push!(annotations, setindex(annot, annot.region .+ offset, :region)) + push!(annotations, @inline(setindex(annot, annot.region .+ offset, :region))) end end end @@ -318,10 +318,10 @@ function reverse(s::AnnotatedString) lastind = lastindex(s) AnnotatedString( reverse(s.string), - [setindex(annot, + [@inline(setindex(annot, UnitRange(1 + lastind - last(annot.region), 1 + lastind - first(annot.region)), - :region) + :region)) for annot in s.annotations]) end @@ -389,16 +389,28 @@ See also: [`annotate!`](@ref). annotations(s::AnnotatedString) = s.annotations function annotations(s::SubString{<:AnnotatedString}) - RegionAnnotation[ - setindex(ann, first(ann.region)-s.offset:last(ann.region)-s.offset, :region) - for ann in annotations(s.string, s.offset+1:s.offset+s.ncodeunits)] + substr_range = s.offset+1:s.offset+s.ncodeunits + result = RegionAnnotation[] + for ann in annotations(s.string, substr_range) + # Shift the region to be relative to the substring start + shifted_region = first(ann.region)-s.offset:last(ann.region)-s.offset + # @inline setindex makes :region const knowable (#60365) + push!(result, @inline(setindex(ann, shifted_region, :region))) + end + return result end function annotations(s::AnnotatedString, pos::UnitRange{<:Integer}) # TODO optimise - RegionAnnotation[ - setindex(ann, max(first(pos), first(ann.region)):min(last(pos), last(ann.region)), :region) - for ann in s.annotations if !isempty(intersect(pos, ann.region))] + result = RegionAnnotation[] + for ann in s.annotations + if !isempty(intersect(pos, ann.region)) + clamped_region = max(first(pos), first(ann.region)):min(last(pos), last(ann.region)) + # @inline setindex makes :region const knowable (#60365) + push!(result, @inline(setindex(ann, clamped_region, :region))) + end + end + return result end annotations(s::AnnotatedString, pos::Integer) = annotations(s, pos:pos) @@ -455,7 +467,7 @@ function annotated_chartransform(f::Function, str::AnnotatedString, state=nothin start, stop = first(annot.region), last(annot.region) start_offset = last(offsets[findlast(<=(start) ∘ first, offsets)::Int]) stop_offset = last(offsets[findlast(<=(stop) ∘ first, offsets)::Int]) - push!(annots, setindex(annot, (start + start_offset):(stop + stop_offset), :region)) + push!(annots, @inline(setindex(annot, (start + start_offset):(stop + stop_offset), :region))) end AnnotatedString(takestring!(outstr), annots) end @@ -509,7 +521,7 @@ function eachregion(s::AnnotatedString, subregion::UnitRange{Int}=firstindex(s): pos = first(events).pos if pos > first(subregion) push!(regions, thisind(s, first(subregion)):prevind(s, pos)) - push!(annots, []) + push!(annots, Annotation[]) end activelist = Int[] for event in events @@ -526,7 +538,7 @@ function eachregion(s::AnnotatedString, subregion::UnitRange{Int}=firstindex(s): end if last(events).pos < nextind(s, last(subregion)) push!(regions, last(events).pos:thisind(s, last(subregion))) - push!(annots, []) + push!(annots, Annotation[]) end RegionIterator(s.string, regions, annots) end diff --git a/base/strings/annotated_io.jl b/base/strings/annotated_io.jl index 60c91be24ebfb..519c6ebb7799d 100644 --- a/base/strings/annotated_io.jl +++ b/base/strings/annotated_io.jl @@ -52,7 +52,7 @@ function write(dest::AnnotatedIOBuffer, src::AnnotatedIOBuffer) srcpos = position(src) nb = write(dest.io, src.io) isappending || _clear_annotations_in_region!(dest.annotations, destpos:destpos+nb) - srcannots = [setindex(annot, max(1 + srcpos, first(annot.region)):last(annot.region), :region) + srcannots = [@inline(setindex(annot, max(1 + srcpos, first(annot.region)):last(annot.region), :region)) for annot in src.annotations if first(annot.region) >= srcpos] _insert_annotations!(dest, srcannots, destpos - srcpos) nb @@ -78,10 +78,11 @@ function write(io::AbstractPipe, c::AnnotatedChar) end function read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{T}}) where {T <: AbstractString} - if (start = position(io)) == 0 + start = position(io) + if start == 0 AnnotatedString(read(io.io, T), copy(io.annotations)) else - annots = [setindex(annot, UnitRange{Int}(max(1, first(annot.region) - start), last(annot.region)-start), :region) + annots = [@inline(setindex(annot, UnitRange{Int}(max(1, first(annot.region) - start), last(annot.region)-start), :region)) for annot in io.annotations if last(annot.region) > start] AnnotatedString(read(io.io, T), annots) end @@ -101,7 +102,7 @@ read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar}) = read(io, AnnotatedChar{Char function truncate(io::AnnotatedIOBuffer, size::Integer) truncate(io.io, size) filter!(ann -> first(ann.region) <= size, io.annotations) - map!(ann -> setindex(ann, first(ann.region):min(size, last(ann.region)), :region), + map!(ann -> @inline(setindex(ann, first(ann.region):min(size, last(ann.region)), :region)), io.annotations, io.annotations) io end @@ -125,17 +126,17 @@ function _clear_annotations_in_region!(annotations::Vector{RegionAnnotation}, sp # Test for partial overlap if first(region) <= first(span) <= last(region) || first(region) <= last(span) <= last(region) annotations[i] = - setindex(annot, + @inline(setindex(annot, if first(region) < first(span) first(region):first(span)-1 else last(span)+1:last(region) end, - :region) + :region)) # If `span` fits exactly within `region`, then we've only copied over # the beginning overhang, but also need to conserve the end overhang. if first(region) < first(span) && last(span) < last(region) - push!(extras, (i, setindex(annot, last(span)+1:last(region), :region))) + push!(extras, (i, @inline(setindex(annot, last(span)+1:last(region), :region)))) end end end diff --git a/base/terminfo.jl b/base/terminfo.jl index be0dd53b1ac74..fec0ed800cddf 100644 --- a/base/terminfo.jl +++ b/base/terminfo.jl @@ -69,7 +69,7 @@ struct TermInfo aliases::Dict{Symbol, Symbol} end -TermInfo() = TermInfo([], Dict(), Dict(), Dict(), nothing, Dict()) +TermInfo() = TermInfo(String[], Dict{Symbol, Bool}(), Dict{Symbol, Int}(), Dict{Symbol, String}(), nothing, Dict{Symbol, Symbol}()) function read(data::IO, ::Type{TermInfoRaw}) # Parse according to `term(5)` diff --git a/base/toml_parser.jl b/base/toml/parser.jl similarity index 99% rename from base/toml_parser.jl rename to base/toml/parser.jl index 5819ab44c6f81..9c83d09d2aa54 100644 --- a/base/toml_parser.jl +++ b/base/toml/parser.jl @@ -1,12 +1,5 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -""" -`Base.TOML` is an undocumented internal part of Julia's TOML parser -implementation. Users should call the documented interface in the -TOML.jl standard library instead (by `import TOML` or `using TOML`). -""" -module TOML - using Base: IdSet # we parse DateTime into these internal structs, @@ -1233,5 +1226,3 @@ function take_chunks(l::Parser, unescape::Bool)::String empty!(l.chunks) return unescape ? unescape_string(str) : str end - -end diff --git a/stdlib/TOML/src/print.jl b/base/toml/printer.jl similarity index 75% rename from stdlib/TOML/src/print.jl rename to base/toml/printer.jl index aca013955b28f..aaef4768be347 100644 --- a/stdlib/TOML/src/print.jl +++ b/base/toml/printer.jl @@ -1,9 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -import Dates - import Base: @invokelatest -import ..isvalid_barekey_char +import ..isvalid_barekey_char # from Parser function print_toml_escaped(io::IO, s::AbstractString) for c::AbstractChar in s @@ -33,9 +31,11 @@ function print_toml_escaped(io::IO, s::AbstractString) end end -const TOMLValue = Union{AbstractVector, AbstractDict, Bool, Integer, AbstractFloat, AbstractString, - Dates.DateTime, Dates.Time, Dates.Date, Base.TOML.DateTime, Base.TOML.Time, Base.TOML.Date} +const BaseTOMLValue = Union{AbstractVector, AbstractDict, Bool, Integer, AbstractFloat, AbstractString, + Base.TOML.DateTime, Base.TOML.Time, Base.TOML.Date} +is_valid_toml_value(@nospecialize(::Any)) = false +is_valid_toml_value(@nospecialize(::BaseTOMLValue)) = true ######## # Keys # @@ -63,7 +63,7 @@ function to_toml_value(@nospecialize(f::Function), value) error("type `$(typeof(value))` is not a valid TOML type, pass a conversion function to `TOML.print`") end toml_value = f(value) - if !(toml_value isa TOMLValue) + if !is_valid_toml_value(toml_value) error("TOML syntax function for type `$(typeof(value))` did not return a valid TOML type but a `$(typeof(toml_value))`") end return toml_value @@ -88,27 +88,38 @@ function printvalue(f::Function, io::IO, value::AbstractVector, sorted::Bool) Base.print(io, "]") end -function printvalue(f::Function, io::IO, value::TOMLValue, sorted::Bool) - value isa Base.TOML.DateTime && (value = Dates.DateTime(value)) - value isa Base.TOML.Time && (value = Dates.Time(value)) - value isa Base.TOML.Date && (value = Dates.Date(value)) - value isa Dates.DateTime ? Base.print(io, Dates.format(value, Dates.dateformat"YYYY-mm-dd\THH:MM:SS.sss\Z")) : - value isa Dates.Time ? Base.print(io, Dates.format(value, Dates.dateformat"HH:MM:SS.sss")) : - value isa Dates.Date ? Base.print(io, Dates.format(value, Dates.dateformat"YYYY-mm-dd")) : - value isa Bool ? Base.print(io, value ? "true" : "false") : - value isa Integer ? print_integer(io, value) : # Julia's own printing should be compatible with TOML on integers - value isa AbstractFloat ? Base.print(io, isnan(value) ? "nan" : - isinf(value) ? string(value > 0 ? "+" : "-", "inf") : - Float64(value)) : # TOML specifies IEEE 754 binary64 for float - value isa AbstractString ? (qmark = Base.contains(value, "\n") ? "\"\"\"" : "\""; - Base.print(io, qmark); - print_toml_escaped(io, value); - Base.print(io, qmark)) : - value isa AbstractDict ? print_inline_table(f, io, value, sorted) : - error("internal error in TOML printing, unhandled value") +function printvalue(f::Function, io::IO, value::Base.TOML.DateTime, sorted::Bool) + printvalue(f, io, value.date, sorted) + Base.print(io, "T") + printvalue(f, io, value.time, sorted) + Base.print(io, "Z") +end + +function printvalue(f::Function, io::IO, value::Base.TOML.Time, sorted::Bool) + Base.print(io, string(value.hour, pad=2)) + Base.print(io, ":") + Base.print(io, string(value.minute, pad=2)) + Base.print(io, ":") + Base.print(io, string(value.second, pad=2)) + if value.ms != 0 + Base.print(io, ".") + Base.print(io, string(value.ms, pad=3)) + end +end + +function printvalue(f::Function, io::IO, value::Base.TOML.Date, sorted::Bool) + Base.print(io, string(value.year, pad=4)) + Base.print(io, "-") + Base.print(io, string(value.month, pad=2)) + Base.print(io, "-") + Base.print(io, string(value.day, pad=2)) +end + +function printvalue(f::Function, io::IO, value::Bool, sorted::Bool) + Base.print(io, value ? "true" : "false") end -function print_integer(io::IO, value::Integer) +function printvalue(f::Function, io::IO, value::Integer, sorted::Bool) value isa Signed && return Base.show(io, value) # unsigned integers are printed as hex n = 2 * ndigits(value, base=256) @@ -116,6 +127,27 @@ function print_integer(io::IO, value::Integer) return end +function printvalue(f::Function, io::IO, value::AbstractFloat, sorted::Bool) + if isnan(value) + Base.print(io, "nan") + elseif isinf(value) + Base.print(io, value > 0 ? "+inf" : "-inf") + else + Base.print(io, Float64(value)) # TOML specifies IEEE 754 binary64 for float + end +end + +function printvalue(f::Function, io::IO, value::AbstractString, sorted::Bool) + qmark = Base.contains(value, "\n") ? "\"\"\"" : "\"" + Base.print(io, qmark) + print_toml_escaped(io, value) + Base.print(io, qmark) +end + +function printvalue(f::Function, io::IO, value::AbstractDict, sorted::Bool) + print_inline_table(f, io, value, sorted) +end + function print_inline_table(f::Function, io::IO, value::AbstractDict, sorted::Bool) vkeys = collect(keys(value))::AbstractArray if sorted @@ -166,7 +198,7 @@ function print_table(f::Function, io::IO, a::AbstractDict, # First print non-tabular entries for key in akeys value = a[key] - if !isa(value, TOMLValue) + if !is_valid_toml_value(value) value = to_toml_value(f, value) end if is_tabular(value) && !(value in inline_tables) @@ -183,7 +215,7 @@ function print_table(f::Function, io::IO, a::AbstractDict, for key in akeys value = a[key] - if !isa(value, TOMLValue) + if !is_valid_toml_value(value) value = to_toml_value(f, value) end if is_table(value) && !(value in inline_tables) diff --git a/base/toml/toml.jl b/base/toml/toml.jl new file mode 100644 index 0000000000000..7426d1259869a --- /dev/null +++ b/base/toml/toml.jl @@ -0,0 +1,19 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +""" +`Base.TOML` is an undocumented internal part of Julia's TOML implementation. +Users should call the documented interface in the TOML.jl standard library +instead (by `import TOML` or `using TOML`). +""" +module TOML + +include("parser.jl") + +# We put the printing functionality in a separate module since it +# defines a function `print` and we don't want that to collide with normal +# usage of `(Base.)print` in other files +module Printer + include("printer.jl") +end + +end diff --git a/cli/Makefile b/cli/Makefile index 4602f759a4768..0ce2631a2ac64 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -88,30 +88,35 @@ DIRS = $(build_bindir) $(build_libdir) $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) ifeq ($(OS),WINNT) -$(BUILDDIR)/julia_res.o: $(JULIAHOME)/contrib/windows/julia.rc $(JULIAHOME)/VERSION +$(BUILDDIR)/julia_res.o $(BUILDDIR)/julia_res_debug.o: $(JULIAHOME)/contrib/windows/julia.rc $(JULIAHOME)/VERSION JLVER=`cat $(JULIAHOME)/VERSION` && \ JLVERi=`echo $$JLVER | perl -nle \ '/^(\d+)\.?(\d*)\.?(\d*)/ && \ print int $$1,",",int $$2,",",int $$3,",0"'` && \ $(CROSS_COMPILE)windres $< -O coff -o $@ -DJLVER=$$JLVERi -DJLVER_STR=\\\"$$JLVER\\\" EXE_OBJS += $(BUILDDIR)/julia_res.o -EXE_DOBJS += $(BUILDDIR)/julia_res.o +EXE_DOBJS += $(BUILDDIR)/julia_res_debug.o endif # Embed an Info.plist in the julia executable # Create an intermediate target Info.plist for Darwin code signing. ifeq ($(DARWIN_FRAMEWORK),1) -$(BUILDDIR)/Info.plist: $(JULIAHOME)/VERSION - /usr/libexec/PlistBuddy -x -c "Clear dict" $@ - /usr/libexec/PlistBuddy -x -c "Add :CFBundleName string julia" $@ - /usr/libexec/PlistBuddy -x -c "Add :CFBundleIdentifier string $(darwin_codesign_id_julia_ui)" $@ - /usr/libexec/PlistBuddy -x -c "Add :CFBundleInfoDictionaryVersion string 6.0" $@ - /usr/libexec/PlistBuddy -x -c "Add :CFBundleVersion string $(JULIA_COMMIT)" $@ - /usr/libexec/PlistBuddy -x -c "Add :CFBundleShortVersionString string $(JULIA_MAJOR_VERSION).$(JULIA_MINOR_VERSION).$(JULIA_PATCH_VERSION)" $@ -.INTERMEDIATE: $(BUILDDIR)/Info.plist # cleanup this file after we are done using it +.PHONY: Info.plist.phony +Info.plist.phony: + @TMPFILE=$$(mktemp -u $(abspath $(BUILDDIR)/Info.plist.XXXXXX)); \ + /usr/libexec/PlistBuddy -x -c "Add :CFBundleName string julia" $$TMPFILE; \ + /usr/libexec/PlistBuddy -x -c "Add :CFBundleIdentifier string $(darwin_codesign_id_julia_ui)" $$TMPFILE; \ + /usr/libexec/PlistBuddy -x -c "Add :CFBundleInfoDictionaryVersion string 6.0" $$TMPFILE; \ + /usr/libexec/PlistBuddy -x -c "Add :CFBundleVersion string $(JULIA_COMMIT)" $$TMPFILE; \ + /usr/libexec/PlistBuddy -x -c "Add :CFBundleShortVersionString string $(JULIA_MAJOR_VERSION).$(JULIA_MINOR_VERSION).$(JULIA_PATCH_VERSION)" $$TMPFILE; \ + if ! cmp -s $(BUILDDIR)/Info.plist $$TMPFILE; then \ + mv $$TMPFILE $(BUILDDIR)/Info.plist; \ + else \ + rm -f $$TMPFILE; \ + fi JLDFLAGS += -Wl,-sectcreate,__TEXT,__info_plist,Info.plist -$(build_bindir)/julia$(EXE): $(BUILDDIR)/Info.plist -$(build_bindir)/julia-debug$(EXE): $(BUILDDIR)/Info.plist +$(build_bindir)/julia$(EXE): Info.plist.phony +$(build_bindir)/julia-debug$(EXE): Info.plist.phony endif julia-release: $(build_bindir)/julia$(EXE) @@ -164,11 +169,14 @@ $(build_bindir)/julia-debug$(EXE): $(EXE_DOBJS) $(build_shlibdir)/libjulia-debug @$(call PRINT_LINK, $(CC) $(LOADER_CFLAGS) $(DEBUGFLAGS) $(EXE_DOBJS) -o $@ $(LOADER_LDFLAGS) $(RPATH) -ljulia-debug) $(BUILDDIR)/julia.expmap: $(SRCDIR)/julia.expmap.in $(JULIAHOME)/VERSION - sed <'$<' >'$@' -e 's/@JULIA_SHLIB_SYMBOL_VERSION@/JL_LIBJULIA_$(SOMAJOR)/' + @TMPFILE=$$(mktemp $(abspath $@.XXXXXX)); \ + sed <'$<' >$$TMPFILE -e 's/@JULIA_SHLIB_SYMBOL_VERSION@/JL_LIBJULIA_$(SOMAJOR)/'; \ + mv $$TMPFILE $@ clean: | $(CLEAN_TARGETS) rm -f $(BUILDDIR)/*.o $(BUILDDIR)/*.dbg.obj rm -f $(build_bindir)/julia* rm -f $(BUILDDIR)/julia.expmap + rm -f $(BUILDDIR)/Info.plist* .PHONY: clean release debug julia-release julia-debug diff --git a/deps/JuliaSyntax.version b/deps/JuliaSyntax.version index 94f480c65dcf7..b2169cfe86192 100644 --- a/deps/JuliaSyntax.version +++ b/deps/JuliaSyntax.version @@ -1,4 +1,4 @@ -JULIASYNTAX_BRANCH = main -JULIASYNTAX_SHA1 = 99e975a726a82994de3f8e961e6fa8d39aed0d37 +JULIASYNTAX_BRANCH = release-1.13 +JULIASYNTAX_SHA1 = 24fc593c88a6b707858f72396bb33099a1843bc5 JULIASYNTAX_GIT_URL := https://github.com/JuliaLang/JuliaSyntax.jl.git JULIASYNTAX_TAR_URL = https://api.github.com/repos/JuliaLang/JuliaSyntax.jl/tarball/$1 diff --git a/deps/checksums/JuliaSyntax-24fc593c88a6b707858f72396bb33099a1843bc5.tar.gz/md5 b/deps/checksums/JuliaSyntax-24fc593c88a6b707858f72396bb33099a1843bc5.tar.gz/md5 new file mode 100644 index 0000000000000..123db250d4993 --- /dev/null +++ b/deps/checksums/JuliaSyntax-24fc593c88a6b707858f72396bb33099a1843bc5.tar.gz/md5 @@ -0,0 +1 @@ +60071ba1110244f865929acb0a38f616 diff --git a/deps/checksums/JuliaSyntax-24fc593c88a6b707858f72396bb33099a1843bc5.tar.gz/sha512 b/deps/checksums/JuliaSyntax-24fc593c88a6b707858f72396bb33099a1843bc5.tar.gz/sha512 new file mode 100644 index 0000000000000..2994d126f988c --- /dev/null +++ b/deps/checksums/JuliaSyntax-24fc593c88a6b707858f72396bb33099a1843bc5.tar.gz/sha512 @@ -0,0 +1 @@ +76874114d9204a78b5706a750cab171abf8f8a2367554a1d04940252f3e134098469076fe8ffe157226356863644268e7b508a94c1b57dbf0162e2d57517b80a diff --git a/deps/checksums/JuliaSyntax-99e975a726a82994de3f8e961e6fa8d39aed0d37.tar.gz/md5 b/deps/checksums/JuliaSyntax-99e975a726a82994de3f8e961e6fa8d39aed0d37.tar.gz/md5 deleted file mode 100644 index 12fce1e97c1db..0000000000000 --- a/deps/checksums/JuliaSyntax-99e975a726a82994de3f8e961e6fa8d39aed0d37.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ecef4caa8b237a51f92d5622b811a0c3 diff --git a/deps/checksums/JuliaSyntax-99e975a726a82994de3f8e961e6fa8d39aed0d37.tar.gz/sha512 b/deps/checksums/JuliaSyntax-99e975a726a82994de3f8e961e6fa8d39aed0d37.tar.gz/sha512 deleted file mode 100644 index f042854e27a47..0000000000000 --- a/deps/checksums/JuliaSyntax-99e975a726a82994de3f8e961e6fa8d39aed0d37.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -56dc5158ebfaf0d5e3e5002dfeb322a137f0866add071cfa9f7a0d9ef2d40859e4c6131358c5aeaf0e9e39fe77a94ba88022028092230b059099cd87e2b795ac diff --git a/deps/checksums/Pkg-4f9884fdb867f2c928ba43dc41da5f150aaec4ab.tar.gz/md5 b/deps/checksums/Pkg-4f9884fdb867f2c928ba43dc41da5f150aaec4ab.tar.gz/md5 deleted file mode 100644 index 45f5692993470..0000000000000 --- a/deps/checksums/Pkg-4f9884fdb867f2c928ba43dc41da5f150aaec4ab.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -cb635b45a66cab302b34bf56367e69d7 diff --git a/deps/checksums/Pkg-4f9884fdb867f2c928ba43dc41da5f150aaec4ab.tar.gz/sha512 b/deps/checksums/Pkg-4f9884fdb867f2c928ba43dc41da5f150aaec4ab.tar.gz/sha512 deleted file mode 100644 index c512a594e877d..0000000000000 --- a/deps/checksums/Pkg-4f9884fdb867f2c928ba43dc41da5f150aaec4ab.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -40a0495141d6b220bbc6b4119369cdc86b22498ca6a9c83eba47aec397c4c92afa5776e9043b1545a3111ace0317ca0c2412d0ba51731a7505742f47545a5530 diff --git a/deps/checksums/Pkg-5189ac693ce94be61feb3489763d78fea967c65c.tar.gz/md5 b/deps/checksums/Pkg-5189ac693ce94be61feb3489763d78fea967c65c.tar.gz/md5 new file mode 100644 index 0000000000000..88104f5f0e226 --- /dev/null +++ b/deps/checksums/Pkg-5189ac693ce94be61feb3489763d78fea967c65c.tar.gz/md5 @@ -0,0 +1 @@ +dda65682687ad36bdbe9252cde7bd7df diff --git a/deps/checksums/Pkg-5189ac693ce94be61feb3489763d78fea967c65c.tar.gz/sha512 b/deps/checksums/Pkg-5189ac693ce94be61feb3489763d78fea967c65c.tar.gz/sha512 new file mode 100644 index 0000000000000..ded9c6e1d048b --- /dev/null +++ b/deps/checksums/Pkg-5189ac693ce94be61feb3489763d78fea967c65c.tar.gz/sha512 @@ -0,0 +1 @@ +9d99b3a3e493d28fbe5302c3977a38468599331cc04720e7b6572539b53a7200abbb9bf11ed759adaef986b4f4851cd19d7a6ab2aa56d3f0281ec19fff2986c6 diff --git a/deps/jlutilities/documenter/Manifest.toml b/deps/jlutilities/documenter/Manifest.toml index c52fbb709b087..732befa13c85c 100644 --- a/deps/jlutilities/documenter/Manifest.toml +++ b/deps/jlutilities/documenter/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.13.0-DEV" +julia_version = "1.13.0-beta2" manifest_format = "2.1" project_hash = "1e9ffa7d4739f7d125a5e2c66af8747a8effd889" @@ -53,10 +53,10 @@ version = "0.9.5" [[deps.Documenter]] deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] -git-tree-sha1 = "b37458ae37d8bdb643d763451585cd8d0e5b4a9e" +git-tree-sha1 = "56e9c37b5e7c3b4f080ab1da18d72d5c290e184a" registries = "General" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "1.16.1" +version = "1.17.0" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] @@ -90,10 +90,10 @@ version = "3.7.0+0" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "b6a684587ebe896d9f68ae777f648205940f0f70" +git-tree-sha1 = "dc34a3e3d96b4ed305b641e626dc14c12b7824b8" registries = "General" uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" -version = "2.51.3+0" +version = "2.53.0+0" [[deps.IOCapture]] deps = ["Logging", "Random"] @@ -116,10 +116,10 @@ version = "1.7.1" [[deps.JSON]] deps = ["Dates", "Logging", "Parsers", "PrecompileTools", "StructUtils", "UUIDs", "Unicode"] -git-tree-sha1 = "5b6bb73f555bc753a6153deec3717b8904f5551c" +git-tree-sha1 = "b3ad4a0255688dcb895a52fafbaae3023b588a90" registries = "General" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "1.3.0" +version = "1.4.0" [deps.JSON.extensions] JSONArrowExt = ["ArrowTypes"] @@ -146,7 +146,7 @@ version = "1.0.0" [[deps.LibCURL_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "Zstd_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.17.0+0" +version = "8.16.0+0" [[deps.LibGit2]] deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] @@ -185,14 +185,14 @@ version = "1.11.0" [[deps.MarkdownAST]] deps = ["AbstractTrees", "Markdown"] -git-tree-sha1 = "465a70f0fc7d443a00dcdc3267a497397b8a3899" +git-tree-sha1 = "93c718d892e73931841089cdc0e982d6dd9cc87b" registries = "General" uuid = "d0879d2d-cac2-40c8-9cee-1863dc0c7391" -version = "0.1.2" +version = "0.1.3" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2025.11.4" +version = "2025.12.2" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" @@ -208,7 +208,7 @@ version = "10.2.1+0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "Libdl"] uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.4+0" +version = "3.5.5+0" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] @@ -240,10 +240,10 @@ version = "1.3.3" [[deps.Preferences]] deps = ["TOML"] -git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" +git-tree-sha1 = "522f093a29b31a93e34eaea17ba055d850edea28" registries = "General" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.5.0" +version = "1.5.1" [[deps.Printf]] deps = ["Unicode"] @@ -281,10 +281,10 @@ version = "1.11.0" [[deps.StructUtils]] deps = ["Dates", "UUIDs"] -git-tree-sha1 = "79529b493a44927dd5b13dde1c7ce957c2d049e4" +git-tree-sha1 = "28145feabf717c5d65c1d5e09747ee7b1ff3ed13" registries = "General" uuid = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" -version = "2.6.0" +version = "2.6.3" [deps.StructUtils.extensions] StructUtilsMeasurementsExt = ["Measurements"] diff --git a/src/Makefile b/src/Makefile index 05230cfa19af3..4d64e6a69f966 100644 --- a/src/Makefile +++ b/src/Makefile @@ -282,19 +282,23 @@ $(BUILDDIR)/%.h.gen : $(SRCDIR)/%.d sed 's/JULIA_/JL_PROBE_/' $@ > $@.tmp mv $@.tmp $@ +# Generate `.inc` file that contains a list of `#define` macros to rename functions defined in `libjulia-internal` +# to have a `ijl_` prefix instead of `jl_`, to denote that they are coming from `libjulia-internal`. This avoids +# potential confusion with debugging tools, when inspecting a process that has both `libjulia` and `libjulia-internal` +# loaded at the same time. $(BUILDDIR)/jl_internal_funcs.inc: $(SRCDIR)/jl_exported_funcs.inc - # Generate `.inc` file that contains a list of `#define` macros to rename functions defined in `libjulia-internal` - # to have a `ijl_` prefix instead of `jl_`, to denote that they are coming from `libjulia-internal`. This avoids - # potential confusion with debugging tools, when inspecting a process that has both `libjulia` and `libjulia-internal` - # loaded at the same time. - grep 'XX(..*)' $< | sed -E 's/.*XX\((.+)\).*/#define \1 i\1/g' >$@ + @TMPFILE=$$(mktemp $(abspath $@.XXXXXX)); \ + grep 'XX(..*)' $< | sed -E 's/.*XX\((.+)\).*/#define \1 i\1/g' > $$TMPFILE; \ + mv $$TMPFILE $@ +# Generate `.inc` file that contains a list of `#define` macros to access global data through struct instances $(BUILDDIR)/jl_data_globals_defs.inc: $(SRCDIR)/jl_exported_data.inc - # Generate `.inc` file that contains a list of `#define` macros to access global data through struct instances + @TMPFILE=$$(mktemp $(abspath $@.XXXXXX)); \ { \ grep 'XX(.*)' $< | sed -E 's/.*XX\(([^,]+), .*\).*/#define jl_\1 (sysimg_global.\1)/g'; \ grep 'YY(.*)' $< | sed -E 's/.*YY\(([^,]+), .*\).*/#define jl_\1 (const_globals.jl\1)/g'; \ - } >$@ + } > $$TMPFILE; \ + mv $$TMPFILE $@ # source file rules $(BUILDDIR)/%.o: $(SRCDIR)/%.c $(HEADERS) | $(BUILDDIR) @@ -361,13 +365,17 @@ $(build_shlibdir)/libllvmcalltest.$(SHLIB_EXT): $(SRCDIR)/llvmcalltest.cpp $(LLV julia_flisp.boot.inc.phony: $(BUILDDIR)/julia_flisp.boot.inc $(BUILDDIR)/julia_flisp.boot.inc: $(BUILDDIR)/julia_flisp.boot $(FLISP_EXECUTABLE_release) - @$(call PRINT_FLISP, $(call spawn,$(FLISP_EXECUTABLE_release)) $(call cygpath_w,$(SRCDIR)/bin2hex.scm) < $< > $@) + @$(call PRINT_FLISP, TMPFILE=$$(mktemp $(abspath $@.XXXXXX)); \ + $(call spawn,$(FLISP_EXECUTABLE_release)) $(call cygpath_w,$(SRCDIR)/bin2hex.scm) < $< > $$TMPFILE; \ + mv $$TMPFILE $@) $(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm flisp/aliases.scm flisp/profile.scm \ julia-parser.scm julia-syntax.scm match.scm utils.scm ast.scm macroexpand.scm mk_julia_flisp_boot.scm) \ $(FLISP_EXECUTABLE_release) - @$(call PRINT_FLISP, $(call spawn,$(FLISP_EXECUTABLE_release)) \ - $(call cygpath_w,$(SRCDIR)/mk_julia_flisp_boot.scm) $(call cygpath_w,$(dir $<)) $(notdir $<) $(call cygpath_w,$@)) + @$(call PRINT_FLISP, TMPFILE=$$(mktemp $(abspath $@.XXXXXX)); \ + $(call spawn,$(FLISP_EXECUTABLE_release)) \ + $(call cygpath_w,$(SRCDIR)/mk_julia_flisp_boot.scm) $(call cygpath_w,$(dir $<)) $(notdir $<) $(call cygpath_w,$$TMPFILE); \ + mv $$TMPFILE $@) # additional dependency links $(BUILDDIR)/codegen-stubs.o $(BUILDDIR)/codegen-stubs.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.h llvm-julia-passes.inc) @@ -436,23 +444,28 @@ $(BUILDDIR)/flisp/libflisp-debug.a: $(addprefix $(SRCDIR)/,flisp/*.h flisp/*.c) $(MAKE) -C $(SRCDIR)/flisp debug BUILDDIR='$(abspath $(BUILDDIR)/flisp)' $(BUILDDIR)/julia_version.h: $(JULIAHOME)/VERSION - @echo "// This is an autogenerated header file" > $@.$(JULIA_BUILD_MODE).tmp - @echo "#ifndef JL_VERSION_H" >> $@.$(JULIA_BUILD_MODE).tmp - @echo "#define JL_VERSION_H" >> $@.$(JULIA_BUILD_MODE).tmp - @echo "#define JULIA_VERSION_STRING" \"$(JULIA_VERSION)\" >> $@.$(JULIA_BUILD_MODE).tmp - @echo $(JULIA_VERSION) | awk 'BEGIN {FS="[.,+-]"} \ - {print "#define JULIA_VERSION_MAJOR " $$1 "\n" \ - "#define JULIA_VERSION_MINOR " $$2 "\n" \ - "#define JULIA_VERSION_PATCH " $$3 ; \ - if (NF<4) print "#define JULIA_VERSION_IS_RELEASE 1" ; else print "#define JULIA_VERSION_IS_RELEASE 0"}' >> $@.$(JULIA_BUILD_MODE).tmp - @echo "#endif" >> $@.$(JULIA_BUILD_MODE).tmp - mv $@.$(JULIA_BUILD_MODE).tmp $@ + @TMPFILE=$$(mktemp $(abspath $@.XXXXXX)); \ + { \ + echo "// This is an autogenerated header file"; \ + echo "#ifndef JL_VERSION_H"; \ + echo "#define JL_VERSION_H"; \ + echo "#define JULIA_VERSION_STRING" \"$(JULIA_VERSION)\"; \ + echo $(JULIA_VERSION) | awk 'BEGIN {FS="[.,+-]"} \ + {print "#define JULIA_VERSION_MAJOR " $$1 "\n" \ + "#define JULIA_VERSION_MINOR " $$2 "\n" \ + "#define JULIA_VERSION_PATCH " $$3 ; \ + if (NF<4) print "#define JULIA_VERSION_IS_RELEASE 1" ; else print "#define JULIA_VERSION_IS_RELEASE 0"}'; \ + echo "#endif"; \ + } > $$TMPFILE; \ + mv $$TMPFILE $@ CXXLD = $(CXX) -shared $(BUILDDIR)/julia.expmap: $(SRCDIR)/julia.expmap.in $(JULIAHOME)/VERSION $(LLVM_CONFIG_ABSOLUTE) - sed <'$<' >'$@' -e "s/@JULIA_SHLIB_SYMBOL_VERSION@/JL_LIBJULIA_$(SOMAJOR)/" \ - -e "s/@LLVM_SHLIB_SYMBOL_VERSION@/$(LLVM_SHLIB_SYMBOL_VERSION)/" + @TMPFILE=$$(mktemp $(abspath $@.XXXXXX)); \ + sed <'$<' >$$TMPFILE -e "s/@JULIA_SHLIB_SYMBOL_VERSION@/JL_LIBJULIA_$(SOMAJOR)/" \ + -e "s/@LLVM_SHLIB_SYMBOL_VERSION@/$(LLVM_SHLIB_SYMBOL_VERSION)/"; \ + mv $$TMPFILE $@ $(build_shlibdir)/libjulia-internal.$(JL_MAJOR_MINOR_SHLIB_EXT): $(BUILDDIR)/julia.expmap $(OBJS) $(BUILDDIR)/flisp/libflisp.a $(BUILDDIR)/support/libsupport.a $(LIBUV) @$(call PRINT_LINK, $(CXXLD) $(call IMPLIB_FLAGS,$@) $(SHIPFLAGS) $(JCXXFLAGS) $(CXXLDFLAGS) $(OBJS) $(RPATH_LIB) -o $@ \ @@ -563,11 +576,11 @@ INCLUDED_CXX_FILES := \ clean: -rm -fr $(build_shlibdir)/libjulia-internal* $(build_shlibdir)/libjulia-codegen* -rm -rf $(build_shlibdir)/libccalltest* $(build_shlibdir)/libllvmcalltest* $(build_shlibdir)/libccalllazyfoo* $(build_shlibdir)/libccalllazybar* - -rm -f $(BUILDDIR)/julia_flisp.boot $(BUILDDIR)/julia_flisp.boot.inc $(BUILDDIR)/jl_internal_funcs.inc $(BUILDDIR)/jl_data_globals_defs.inc + -rm -f $(BUILDDIR)/julia_flisp.boot* $(BUILDDIR)/julia_flisp.boot.inc* $(BUILDDIR)/jl_internal_funcs.inc* $(BUILDDIR)/jl_data_globals_defs.inc* -rm -f $(BUILDDIR)/*.dbg.obj $(BUILDDIR)/*.o $(BUILDDIR)/*.dwo $(BUILDDIR)/*.$(SHLIB_EXT) $(BUILDDIR)/*.a $(BUILDDIR)/*.h.gen -rm -f $(BUILDDIR)/julia.expmap -rm -f $(BUILDDIR)/julia_version.h - -rm -f $(BUILDDIR)/compile_commands.json + -rm -f $(BUILDDIR)/compile_commands.json* clean-flisp: -$(MAKE) -C $(SRCDIR)/flisp clean BUILDDIR='$(abspath $(BUILDDIR)/flisp)' @@ -666,7 +679,7 @@ clean-analyzegc: # Compilation database generation using existing clang infrastructure .PHONY: regenerate-compile_commands regenerate-compile_commands: - TMPFILE=$$(mktemp -p $(BUILDDIR) compile_commands.json.XXXXXX); \ + @TMPFILE=$$(mktemp $(abspath $(BUILDDIR)/compile_commands.json.XXXXXX)); \ { \ CLANG_TOOLING_C_FLAGS="$$($(JULIAHOME)/contrib/escape_json.sh clang $(CLANG_TOOLING_C_FLAGS))"; \ CLANG_TOOLING_CXX_FLAGS="$$($(JULIAHOME)/contrib/escape_json.sh clang $(CLANG_TOOLING_CXX_FLAGS))"; \ diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index c11357d276482..71e331d13e233 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -681,7 +681,7 @@ static bool canPartition(const Function &F) // `external_linkage` create linkages between pkgimages. extern "C" JL_DLLEXPORT_CODEGEN void *jl_create_native_impl(LLVMOrcThreadSafeModuleRef llvmmod, int trim, int external_linkage, size_t world, - jl_array_t *mod_array, jl_array_t *worklist, int all, jl_array_t *module_init_order) + jl_array_t *mod_array, jl_array_t *worklist, int all, jl_array_t *module_init_order, jl_array_t *ext_foreign_cis) { JL_TIMING(INFERENCE, INFERENCE); auto ct = jl_current_task; @@ -697,7 +697,7 @@ void *jl_create_native_impl(LLVMOrcThreadSafeModuleRef llvmmod, int trim, int ex compiler_start_time = jl_hrtime(); jl_value_t **fargs; - JL_GC_PUSHARGS(fargs, 8); + JL_GC_PUSHARGS(fargs, 9); #ifdef _P64 jl_value_t *jl_array_ulong_type = jl_array_uint64_type; #else @@ -718,10 +718,11 @@ void *jl_create_native_impl(LLVMOrcThreadSafeModuleRef llvmmod, int trim, int ex fargs[5] = mod_array ? (jl_value_t*)mod_array : jl_nothing; // mod_array (or nothing) fargs[6] = jl_box_bool(all); fargs[7] = module_init_order ? (jl_value_t*)module_init_order : jl_nothing; // module_init_order (or nothing) + fargs[8] = ext_foreign_cis ? (jl_value_t*)ext_foreign_cis : jl_nothing; // ext_foreign_cis (or nothing) size_t last_age = ct->world_age; ct->world_age = jl_typeinf_world; - fargs[0] = jl_apply(fargs, 8); - fargs[1] = fargs[2] = fargs[3] = fargs[4] = fargs[5] = fargs[6] = fargs[7] = NULL; + fargs[0] = jl_apply(fargs, 9); + fargs[1] = fargs[2] = fargs[3] = fargs[4] = fargs[5] = fargs[6] = fargs[7] = fargs[8] = NULL; ct->world_age = last_age; jl_value_t *codeinfos = fargs[0]; JL_TYPECHK(jl_create_native, array_any, codeinfos); @@ -816,9 +817,12 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm continue; // skip any duplicates that accidentally made there way in here (or make this an error?) if (jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) params.safepoint_on_entry = false; // ensure we don't block ExpandAtomicModifyPass from inlining this code if applicable - orc::ThreadSafeModule result_m = jl_create_ts_module(name_from_method_instance(jl_get_ci_mi(codeinst)), - params.tsctx, clone.getModuleUnlocked()->getDataLayout(), - Triple(clone.getModuleUnlocked()->getTargetTriple())); + orc::ThreadSafeModule result_m = + jl_create_ts_module(name_from_method_instance(jl_get_ci_mi(codeinst)), + params.tsctx, + clone.getModuleUnlocked()->getDataLayout(), + Triple(clone.getModuleUnlocked()->getTargetTriple()), + clone.getModuleUnlocked()); jl_llvm_functions_t decls; if (!(params.params->force_emit_all) && jl_atomic_load_relaxed(&codeinst->invoke) == jl_fptr_const_return_addr) decls.functionObject = "jl_fptr_const_return"; diff --git a/src/ccall.cpp b/src/ccall.cpp index f67268b1a0007..cda99aa4043ac 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1296,6 +1296,7 @@ std::string generate_func_sig(const char *fname) if (!ctx->TargetTriple.isOSWindows()) { // llvm used to use the old mingw ABI, skipping this marking works around that difference retattrs.addStructRetAttr(lrt); + retattrs.addAlignmentAttr(Align(julia_alignment(rt))); } retattrs.addAttribute(Attribute::NoAlias); paramattrs.push_back(AttributeSet::get(LLVMCtx, retattrs)); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 275b5fe5cf06e..d99be5ec08315 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -3872,7 +3872,7 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo, bool is_promotab if (!box) { bool do_promote = vinfo.promotion_point; if (do_promote && is_promotable && vinfo.inline_roots.empty()) { - auto IP = ctx.builder.saveIP(); + IRBuilderBase::InsertPointGuard IP(ctx.builder); ctx.builder.SetInsertPoint(vinfo.promotion_point); box = emit_allocobj(ctx, (jl_datatype_t*)jt, true); Value *decayed = decay_derived(ctx, box); @@ -3884,7 +3884,6 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo, bool is_promotab originalAlloca->replaceAllUsesWith(decayed); // end illegal IR originalAlloca->eraseFromParent(); - ctx.builder.restoreIP(IP); } else { auto arg_typename = [&] JL_NOTSAFEPOINT { @@ -4268,7 +4267,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg jl_value_t *jtype = jl_svecref(sty->types, i); // n.b. ty argument must be concrete jl_cgval_t fval_info = argv[i]; - IRBuilderBase::InsertPoint savedIP; + std::optional savedIP; emit_typecheck(ctx, fval_info, jtype, "new"); fval_info = update_julia_type(ctx, fval_info, jtype); if (fval_info.typ == jl_bottom_type) @@ -4290,7 +4289,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg fval_info.inline_roots.empty() && inline_roots.empty() && // these need to be compatible, if they were to be implemented fval_info.promotion_point && fval_info.promotion_point->getParent() == ctx.builder.GetInsertBlock(); if (field_promotable) { - savedIP = ctx.builder.saveIP(); + savedIP.emplace(ctx.builder); ctx.builder.SetInsertPoint(fval_info.promotion_point); } if (!init_as_value) { @@ -4411,9 +4410,6 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg else assert(false); } - if (field_promotable) { - ctx.builder.restoreIP(savedIP); - } } if (init_as_value) { for (size_t i = nargs; i < nf; i++) { @@ -4431,7 +4427,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg } if (nargs < nf) { assert(!init_as_value); - IRBuilderBase::InsertPoint savedIP = ctx.builder.saveIP(); + IRBuilderBase::InsertPointGuard savedIP(ctx.builder); if (promotion_point) ctx.builder.SetInsertPoint(promotion_point); if (strct) { @@ -4439,7 +4435,6 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg promotion_point = ai.decorateInst(ctx.builder.CreateMemSet(strct, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0), jl_datatype_size(ty), Align(julia_alignment(ty)))); } - ctx.builder.restoreIP(savedIP); } if (type_is_ghost(lt)) return mark_julia_const(ctx, sty->instance); diff --git a/src/codegen.cpp b/src/codegen.cpp index 9dcf1264581c0..1af1569456832 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -18,6 +18,8 @@ #include #include #include +#include +#include // target machine computation #include @@ -2276,6 +2278,18 @@ static inline jl_cgval_t value_to_pointer(jl_codectx_t &ctx, const jl_cgval_t &v Align align(julia_alignment(v.typ)); Type *ty = julia_type_to_llvm(ctx, v.typ); AllocaInst *loc = emit_static_alloca(ctx, ty, align); + jl_datatype_t *dt = (jl_datatype_t *)v.typ; + size_t npointers = dt->layout->first_ptr >= 0 ? dt->layout->npointers : 0; + if (npointers > 0) { + IRBuilderBase::InsertPointGuard InsertPoint(ctx.builder); + ctx.builder.SetInsertPoint(ctx.topalloca->getParent(), ++ctx.topalloca->getIterator()); + for (size_t i = 0; i < npointers; i++) { + // make sure these are nullptr early from LLVM's perspective, in case it decides to SROA it + Value *ptr_field = emit_ptrgep(ctx, loc, jl_ptr_offset(dt, i) * sizeof(void *)); + ctx.builder.CreateAlignedStore( + Constant::getNullValue(ctx.types().T_prjlvalue), ptr_field, Align(sizeof(void *))); + } + } auto tbaa = v.V == nullptr ? ctx.tbaa().tbaa_gcframe : ctx.tbaa().tbaa_stack; auto stack_ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); recombine_value(ctx, v, loc, stack_ai, align, false); @@ -2671,17 +2685,12 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_ return jl_cgval_t(v, typ, new_tindex); } -std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &context, const DataLayout &DL, const Triple &triple) JL_NOTSAFEPOINT +std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &context, + const DataLayout &DL, const Triple &triple, + Module *source) JL_NOTSAFEPOINT { ++ModulesCreated; auto m = std::make_unique(name, context); - // According to clang darwin above 10.10 supports dwarfv4 - if (!m->getModuleFlag("Dwarf Version")) { - m->addModuleFlag(llvm::Module::Warning, "Dwarf Version", 4); - } - if (!m->getModuleFlag("Debug Info Version")) - m->addModuleFlag(llvm::Module::Warning, "Debug Info Version", - llvm::DEBUG_METADATA_VERSION); m->setDataLayout(DL); m->setTargetTriple(triple.str()); @@ -2692,9 +2701,29 @@ std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &conte m->setOverrideStackAlignment(16); } + if (source) { + // Copy module flags from source module + SmallVector Flags; + source->getModuleFlagsMetadata(Flags); + for (const auto &Flag : Flags) { + m->addModuleFlag(Flag.Behavior, Flag.Key->getString(), Flag.Val); + } + // Copy other module-level properties + m->setStackProtectorGuard(source->getStackProtectorGuard()); + m->setOverrideStackAlignment(source->getOverrideStackAlignment()); + } + else { + // No source: set default Julia flags + // According to clang darwin above 10.10 supports dwarfv4 + m->addModuleFlag(llvm::Module::Warning, "Dwarf Version", 4); + m->addModuleFlag(llvm::Module::Warning, "Debug Info Version", + llvm::DEBUG_METADATA_VERSION); + #if defined(JL_DEBUG_BUILD) - m->setStackProtectorGuard("global"); + m->setStackProtectorGuard("global"); #endif + } + return m; } @@ -3196,6 +3225,10 @@ static jl_cgval_t emit_globalref(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t * undef_var_error_ifnot(ctx, ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 0), name, (jl_value_t*)mod); return jl_cgval_t(); } + if (jl_generating_output()) { + // root is required to allow bindings to be pruned, especially by `--trim` + jl_temporary_root(ctx, constval); + } return mark_julia_const(ctx, constval); } if (rkp.kind != PARTITION_KIND_GLOBAL) { @@ -7945,6 +7978,7 @@ static jl_returninfo_t get_specsig_function(jl_codegen_params_t ¶ms, Module param.addAttribute(Attribute::NoAlias); param.addAttribute(Attribute::NoCapture); param.addAttribute(Attribute::NoUndef); + param.addAlignmentAttr(Align(props.union_align)); attrs.push_back(AttributeSet::get(M->getContext(), param)); assert(fsig.size() == 1); } @@ -8544,13 +8578,10 @@ static jl_llvm_functions_t const DataLayout &DL = jl_Module->getDataLayout(); Type *RT = Arg->getParamStructRetType(); TypeSize sz = DL.getTypeAllocSize(RT); - Align al = DL.getPrefTypeAlign(RT); - if (al > MAX_ALIGN) - al = Align(MAX_ALIGN); param.addAttribute(Attribute::NonNull); // The `dereferenceable` below does not imply `nonnull` for non addrspace(0) pointers. param.addDereferenceableAttr(sz); - param.addAlignmentAttr(al); + // Alignment is already set by get_specsig_function. } attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); // function declaration attributes } diff --git a/src/flisp/Makefile b/src/flisp/Makefile index 3c24dab77d89f..db4d729ad5ae2 100644 --- a/src/flisp/Makefile +++ b/src/flisp/Makefile @@ -142,7 +142,7 @@ INCLUDED_FLISP_FILES := flisp.c:cvalues.c flisp.c:types.c flisp.c:print.c flisp. # Compilation database generation .PHONY: regenerate-compile_commands regenerate-compile_commands: - TMPFILE=$$(mktemp -p $(BUILDDIR) compile_commands.json.XXXXXX); \ + @TMPFILE=$$(mktemp $(abspath $(BUILDDIR)/compile_commands.json.XXXXXX)); \ { \ CLANG_TOOLING_C_FLAGS="$$($(JULIAHOME)/contrib/escape_json.sh clang $(CLANG_TOOLING_C_FLAGS))"; \ echo "["; \ @@ -176,7 +176,7 @@ clean: rm -f $(BUILDDIR)/*.a rm -f $(BUILDDIR)/$(EXENAME)$(EXE) rm -f $(BUILDDIR)/$(EXENAME)-debug$(EXE) - rm -f $(BUILDDIR)/compile_commands.json + rm -f $(BUILDDIR)/compile_commands.json* rm -f $(BUILDDIR)/host/* .PHONY: flisp-deps compile-database diff --git a/src/gc-common.c b/src/gc-common.c index 811e441960eb1..4cf95ecdad2eb 100644 --- a/src/gc-common.c +++ b/src/gc-common.c @@ -625,6 +625,11 @@ int gc_slot_to_arrayidx(void *obj, void *_slot) JL_NOTSAFEPOINT start = (char*)jl_svec_data(obj); len = jl_svec_len(obj); } + else if (vt->name == jl_genericmemory_typename) { + jl_genericmemory_t *mem = (jl_genericmemory_t*)obj; + start = (char*)mem->ptr; + len = mem->length; + } if (slot < start || slot >= start + elsize * len) return -1; return (slot - start) / elsize; diff --git a/src/gc-heap-snapshot.cpp b/src/gc-heap-snapshot.cpp index 62379263b03e7..7e4917d219f83 100644 --- a/src/gc-heap-snapshot.cpp +++ b/src/gc-heap-snapshot.cpp @@ -368,6 +368,16 @@ size_t record_node_to_gc_snapshot(jl_value_t *a) JL_NOTSAFEPOINT } else { self_size = (size_t)jl_datatype_size(type); + if (type->name == jl_genericmemory_typename) { + jl_genericmemory_t *mem = (jl_genericmemory_t*)a; + int how = jl_genericmemory_how(mem); + if (how != JL_GENERICMEMORY_STRINGOWNED && how != JL_GENERICMEMORY_MALLOCD) { + // Memory's that are string-owned or point to foreign memory have + // explicit snapshot edges to pointee data. Otherwise the array + // contents are treated as part of the Memory itself. + self_size += jl_genericmemory_nbytes(mem); + } + } // print full type into ios buffer and get StringRef to it. // The ios is cleaned up below. ios_need_close = 1; @@ -559,26 +569,12 @@ void _gc_heap_snapshot_record_binding_partition_edge(jl_value_t *from, jl_value_ } -void _gc_heap_snapshot_record_hidden_edge(jl_value_t *from, void* to, size_t bytes, uint16_t alloc_type) JL_NOTSAFEPOINT +void _gc_heap_snapshot_record_foreign_memory_edge(jl_value_t *from, void* to, size_t bytes) JL_NOTSAFEPOINT { - // valid alloc_type values are 0, 1, 2 - assert(alloc_type <= 2); size_t name_or_idx = g_snapshot->names.serialize_if_necessary(g_snapshot->strings, ""); auto from_node_idx = record_node_to_gc_snapshot(from); - const char *alloc_kind = NULL; - switch (alloc_type) - { - case 0: - alloc_kind = ""; - break; - case 1: - alloc_kind = ""; - break; - case 2: - alloc_kind = ""; - break; - } + const char *alloc_kind = ""; auto to_node_idx = record_pointer_to_gc_snapshot(to, bytes, alloc_kind); _record_gc_just_edge("hidden", from_node_idx, to_node_idx, name_or_idx); diff --git a/src/gc-heap-snapshot.h b/src/gc-heap-snapshot.h index dc5b22bb72eb1..054355c8eeb62 100644 --- a/src/gc-heap-snapshot.h +++ b/src/gc-heap-snapshot.h @@ -27,7 +27,7 @@ void _gc_heap_snapshot_record_module_to_binding(jl_module_t* module, jl_value_t void _gc_heap_snapshot_record_internal_array_edge(jl_value_t *from, jl_value_t *to) JL_NOTSAFEPOINT; // Used for objects manually allocated in C (outside julia GC), to still tell the heap snapshot about the // size of the object, even though we're never going to mark that object. -void _gc_heap_snapshot_record_hidden_edge(jl_value_t *from, void* to, size_t bytes, uint16_t alloc_type) JL_NOTSAFEPOINT; +void _gc_heap_snapshot_record_foreign_memory_edge(jl_value_t *from, void* to, size_t bytes) JL_NOTSAFEPOINT; // Used for objects that are reachable from the GC roots void _gc_heap_snapshot_record_gc_roots(jl_value_t *root, char *name) JL_NOTSAFEPOINT; // Used for objects that are reachable from the finalizer list @@ -106,10 +106,10 @@ static inline void gc_heap_snapshot_record_binding_partition_edge(jl_value_t *fr } } -static inline void gc_heap_snapshot_record_hidden_edge(jl_value_t *from, void* to, size_t bytes, uint16_t alloc_type) JL_NOTSAFEPOINT +static inline void gc_heap_snapshot_record_foreign_memory_edge(jl_value_t *from, void* to, size_t bytes) JL_NOTSAFEPOINT { if (__unlikely(gc_heap_snapshot_enabled && prev_sweep_full)) { - _gc_heap_snapshot_record_hidden_edge(from, to, bytes, alloc_type); + _gc_heap_snapshot_record_foreign_memory_edge(from, to, bytes); } } diff --git a/src/gc-interface.h b/src/gc-interface.h index 4df1f154455c3..b4881a81e885b 100644 --- a/src/gc-interface.h +++ b/src/gc-interface.h @@ -101,8 +101,15 @@ JL_DLLEXPORT void jl_gc_set_max_memory(uint64_t max_mem); JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection); // Returns whether the thread with `tid` is a collector thread JL_DLLEXPORT int gc_is_collector_thread(int tid) JL_NOTSAFEPOINT; +// Enables or disables automatic full (non-generational) collections. +// When disabled (on == 0), automatic collections will only be incremental +// (young generation only). Explicit full collections via jl_gc_collect(JL_GC_FULL) +// are still honored. Returns whether automatic full collections were previously enabled. +JL_DLLEXPORT int jl_gc_enable_auto_full_collection(int on); +// Returns whether automatic full (non-generational) collections are enabled. +JL_DLLEXPORT int jl_gc_auto_full_collection_is_enabled(void); // Returns which GC implementation is being used and possibly its version according to the list of supported GCs -// NB: it should clearly identify the GC by including e.g. ‘stock’ or ‘mmtk’ as a substring. +// NB: it should clearly identify the GC by including e.g. 'stock' or 'mmtk' as a substring. JL_DLLEXPORT const char* jl_gc_active_impl(void); // Sweep Julia's stack pools and mtarray buffers. Note that this function has been added to the interface as // each GC should implement it but it will most likely not be used by other code in the runtime. diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index be773a8e625c5..bfd91749e39f8 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -336,6 +336,18 @@ JL_DLLEXPORT const char* jl_gc_active_impl(void) { return mmtk_version; } +JL_DLLEXPORT int jl_gc_enable_auto_full_collection(int on) +{ + // MMTk does not currently support collection type control + return 1; +} + +JL_DLLEXPORT int jl_gc_auto_full_collection_is_enabled(void) +{ + return 1; +} + + int64_t last_gc_total_bytes = 0; int64_t last_live_bytes = 0; // live_bytes at last collection int64_t live_bytes = 0; @@ -590,7 +602,8 @@ JL_DLLEXPORT void jl_gc_scan_julia_exc_obj(void* obj_raw, void* closure, Process static void jl_gc_free_memory(jl_genericmemory_t *m, int isaligned) JL_NOTSAFEPOINT { assert(jl_is_genericmemory(m)); - assert(jl_genericmemory_how(m) == 1 || jl_genericmemory_how(m) == 2); + assert(jl_genericmemory_how(m) == JL_GENERICMEMORY_GCMANAGED || + jl_genericmemory_how(m) == JL_GENERICMEMORY_MALLOCD); char *d = (char*)m->ptr; size_t freed_bytes = memory_block_usable_size(d, isaligned); assert(freed_bytes != 0); @@ -644,7 +657,7 @@ JL_DLLEXPORT void jl_gc_update_inlined_array(void* from, void* to) { jl_genericmemory_t *b = (jl_genericmemory_t*)jl_to; int how = jl_genericmemory_how(a); - if (how == 0 && mmtk_object_is_managed_by_mmtk(a->ptr)) { // a is inlined (a->ptr points into the mmtk object) + if (how == JL_GENERICMEMORY_INLINED && mmtk_object_is_managed_by_mmtk(a->ptr)) { // a is inlined (a->ptr points into the mmtk object) size_t offset_of_data = ((size_t)a->ptr - (size_t)a); if (offset_of_data > 0) { b->ptr = (void*)((size_t) b + offset_of_data); @@ -776,13 +789,13 @@ JL_DLLEXPORT size_t jl_gc_genericmemory_how(void *arg) JL_NOTSAFEPOINT { jl_genericmemory_t* m = (jl_genericmemory_t*)arg; if (m->ptr == (void*)((char*)m + 16)) // JL_SMALL_BYTE_ALIGNMENT (from julia_internal.h) - return 0; + return JL_GENERICMEMORY_INLINED; jl_value_t *owner = jl_genericmemory_data_owner_field(m); if (owner == (jl_value_t*)m) - return 1; + return JL_GENERICMEMORY_GCMANAGED; if (owner == NULL) - return 2; - return 3; + return JL_GENERICMEMORY_MALLOCD; + return JL_GENERICMEMORY_STRINGOWNED; } // ========================================================================= // diff --git a/src/gc-stock.c b/src/gc-stock.c index 9e0d2ab77190b..601921453060f 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -202,6 +202,7 @@ int prev_sweep_full = 1; int current_sweep_full = 0; int next_sweep_full = 0; int under_pressure = 0; +int gc_disable_auto_full_sweep = 0; // when set, automatic full collections are inhibited // Full collection heuristics static int64_t live_bytes = 0; @@ -630,7 +631,7 @@ void jl_gc_reset_alloc_count(void) JL_NOTSAFEPOINT static void jl_gc_free_memory(jl_genericmemory_t *m, int isaligned) JL_NOTSAFEPOINT { assert(jl_is_genericmemory(m)); - assert(jl_genericmemory_how(m) == 1); + assert(jl_genericmemory_how(m) == JL_GENERICMEMORY_GCMANAGED); char *d = (char*)m->ptr; size_t freed_bytes = memory_block_usable_size(d, isaligned); assert(freed_bytes != 0); @@ -2383,13 +2384,13 @@ FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_ gc_setmark_big(ptls, o, bits); } int how = jl_genericmemory_how(m); - if (how == 0 || how == 2) { - gc_heap_snapshot_record_hidden_edge(new_obj, m->ptr, jl_genericmemory_nbytes(m), how == 0 ? 2 : 0); + if (how == JL_GENERICMEMORY_MALLOCD) { + gc_heap_snapshot_record_foreign_memory_edge( + new_obj, m->ptr, jl_genericmemory_nbytes(m)); } - else if (how == 1) { + else if (how == JL_GENERICMEMORY_GCMANAGED) { if (update_meta || foreign_alloc) { size_t nb = jl_genericmemory_nbytes(m); - gc_heap_snapshot_record_hidden_edge(new_obj, m->ptr, nb, 0); if (bits == GC_OLD_MARKED) { ptls->gc_tls.gc_cache.perm_scanned_bytes += nb; } @@ -2398,7 +2399,7 @@ FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_ } } } - else if (how == 3) { + else if (how == JL_GENERICMEMORY_STRINGOWNED) { jl_value_t *owner = jl_genericmemory_data_owner_field(m); uintptr_t nptr = (1 << 2) | (bits & GC_OLD); gc_try_claim_and_push(mq, owner, &nptr); @@ -3167,6 +3168,9 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS recollect = 1; gc_count_full_sweep_reason(FULL_SWEEP_REASON_FORCED_FULL_SWEEP); } + if (gc_disable_auto_full_sweep && collection != JL_GC_FULL) { + sweep_full = 0; + } // 5. start sweeping uint64_t start_sweep_time = jl_hrtime(); JL_PROBE_GC_SWEEP_BEGIN(sweep_full); @@ -3233,7 +3237,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTS old_freed_diff = gc_mem; old_pause_time = gc_time; // thrashing estimator: if GC time more than 50% of the runtime - if (pause > mutator_time && !(thrash_counter < 4)) + if (pause > mutator_time && thrash_counter <= 4) thrash_counter += 1; else if (thrash_counter > 0) thrash_counter -= 1; @@ -4123,6 +4127,18 @@ JL_DLLEXPORT const char* jl_gc_active_impl(void) { return "Built with stock GC"; } +JL_DLLEXPORT int jl_gc_enable_auto_full_collection(int on) +{ + int prev = !gc_disable_auto_full_sweep; + gc_disable_auto_full_sweep = (on == 0); + return prev; +} + +JL_DLLEXPORT int jl_gc_auto_full_collection_is_enabled(void) +{ + return !gc_disable_auto_full_sweep; +} + #ifdef __cplusplus } #endif diff --git a/src/genericmemory.c b/src/genericmemory.c index dc1c687d1d382..e3bacc2495966 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -197,9 +197,9 @@ JL_DLLEXPORT jl_value_t *jl_genericmemory_to_string(jl_genericmemory_t *m, size_ } int how = jl_genericmemory_how(m); size_t mlength = m->length; - if (how != 0) { + if (how != JL_GENERICMEMORY_INLINED) { jl_value_t *o = jl_genericmemory_data_owner_field(m); - if (how == 3 && // implies jl_is_string(o) + if (how == JL_GENERICMEMORY_STRINGOWNED && // implies jl_is_string(o) ((mlength + sizeof(void*) + 1 <= GC_MAX_SZCLASS) == (len + sizeof(void*) + 1 <= GC_MAX_SZCLASS))) { if (jl_string_data(o)[len] != '\0') jl_string_data(o)[len] = '\0'; @@ -212,7 +212,7 @@ JL_DLLEXPORT jl_value_t *jl_genericmemory_to_string(jl_genericmemory_t *m, size_ JL_GC_POP(); return str; } - // n.b. how == 0 is always pool-allocated, so the freed bytes are computed from the pool not the object + // n.b. how == JL_GENERICMEMORY_INLINED is always pool-allocated, so the freed bytes are computed from the pool not the object return jl_pchar_to_string((const char*)m->ptr, len); } diff --git a/src/gf.c b/src/gf.c index 42c41887680f5..ae38c5c31e287 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1012,7 +1012,7 @@ int jl_foreach_reachable_mtable(int (*visit)(jl_methtable_t *mt, void *env), jl_ } else if (jl_is_mtable(v)) { jl_methtable_t *mt = (jl_methtable_t*)v; - if (mt && mt != jl_method_table) { + if (mt && mt != jl_method_table && mt->module == current_m && mt->name == name) { if (!visit(mt, env)) { result = 0; goto cleanup; diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index ae25c3cc83ca5..88fd03b90ee35 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -600,21 +600,23 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, ArrayRef argv) assert(!v.isghost); Value *vx = NULL; - if (!v.ispointer()) + if (v.inline_roots.empty() && !v.ispointer()) vx = v.V; else if (v.constant) vx = julia_const_to_llvm(ctx, v.constant); - if (v.ispointer() && vx == NULL) { + if (vx == NULL) { // try to load as original Type, to preserve llvm optimizations // but if the v.typ is not well known, use llvmt + // also handles values in split representation (inline_roots): + // the dynamic checks above ensure only primitive types reach here if (isboxed) vxt = llvmt; auto storage_type = vxt->isIntegerTy(1) ? getInt8Ty(ctx.builder.getContext()) : vxt; jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, v.tbaa); vx = ai.decorateInst(ctx.builder.CreateLoad( storage_type, - data_pointer(ctx, v))); + maybe_decay_tracked(ctx, v.V))); setName(ctx.emission_context, vx, "bitcast"); } diff --git a/src/ircode.c b/src/ircode.c index e99bd26aa304a..f1070feca9571 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -1238,7 +1238,10 @@ JL_DLLEXPORT uint8_t jl_ir_flag_has_image_globalref(jl_string_t *data) { if (jl_is_code_info(data)) return ((jl_code_info_t*)data)->has_image_globalref; - assert(jl_is_string(data)); + if (!jl_is_string(data)) { + // foreign CodeInstance with custom source/IR, doesn't track GlobalRef edges + return 0; + } jl_code_info_flags_t flags; flags.packed = jl_string_data(data)[ir_offset_flags]; return flags.bits.has_image_globalref; diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index be1b943b8511f..63fb4de3d5d50 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -2262,12 +2262,17 @@ void JuliaOJIT::enableJITDebuggingSupport() auto registerJITLoaderGDBWrapper = addAbsoluteToMap(GDBFunctions,llvm_orc_registerJITLoaderGDBWrapper); cantFail(JD.define(orc::absoluteSymbols(GDBFunctions))); (void)registerJITLoaderGDBWrapper; - if (TM->getTargetTriple().isOSBinFormatMachO()) - ObjectLayer.addPlugin(cantFail(orc::GDBJITDebugInfoRegistrationPlugin::Create(ES, JD, TM->getTargetTriple()))); + if (TM->getTargetTriple().isOSBinFormatMachO()) { + auto RegisterSym = cantFail( + safelookup(ES, {&JD}, ES.intern("_llvm_orc_registerJITLoaderGDBAllocAction"))); + ObjectLayer.addPlugin( + std::make_unique(RegisterSym.getAddress())); + } #ifndef _COMPILER_ASAN_ENABLED_ // TODO: Fix duplicated sections spam #51794 - else if (TM->getTargetTriple().isOSBinFormatELF()) + else if (TM->getTargetTriple().isOSBinFormatELF()) { //EPCDebugObjectRegistrar doesn't take a JITDylib, so we have to directly provide the call address ObjectLayer.addPlugin(std::make_unique(ES, std::make_unique(ES, registerJITLoaderGDBWrapper))); + } #endif } diff --git a/src/jitlayers.h b/src/jitlayers.h index 331d9accc8fb8..f77b21ce39d21 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -667,10 +667,16 @@ class JuliaOJIT { OptSelLayerT OptSelLayer; }; extern JuliaOJIT *jl_ExecutionEngine; -std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &ctx, const DataLayout &DL, const Triple &triple) JL_NOTSAFEPOINT; -inline orc::ThreadSafeModule jl_create_ts_module(StringRef name, orc::ThreadSafeContext ctx, const DataLayout &DL, const Triple &triple) JL_NOTSAFEPOINT { +std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &ctx, + const DataLayout &DL, const Triple &triple, + Module *source = nullptr) JL_NOTSAFEPOINT; +inline orc::ThreadSafeModule jl_create_ts_module(StringRef name, orc::ThreadSafeContext ctx, + const DataLayout &DL, const Triple &triple, + Module *source = nullptr) JL_NOTSAFEPOINT +{ auto lock = ctx.getLock(); - return orc::ThreadSafeModule(jl_create_llvm_module(name, *ctx.getContext(), DL, triple), ctx); + return orc::ThreadSafeModule( + jl_create_llvm_module(name, *ctx.getContext(), DL, triple, source), ctx); } Module &jl_codegen_params_t::shared_module() JL_NOTSAFEPOINT { diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index f3a57c6699c08..c9b1fd551e359 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1809,11 +1809,13 @@ (cons e '()) (let ((a '())) (define (arg-to-temp x) - (cond ((effect-free? x) x) - ((or (eq? (car x) '...) (eq? (car x) '&)) - `(,(car x) ,(arg-to-temp (cadr x)))) + (cond ((effect-free? x) x) + ((eq? (car x) '...) + `(... ,(arg-to-temp (cadr x)))) ((eq? (car x) 'kw) - `(,(car x) ,(cadr x) ,(arg-to-temp (caddr x)))) + `(kw ,(cadr x) ,(arg-to-temp (caddr x)))) + ((eq? (car x) 'parameters) + `(parameters ,@(map arg-to-temp (cdr x)))) (else (let ((g (make-ssavalue))) (begin (set! a (cons `(= ,g ,x) a)) diff --git a/src/julia.h b/src/julia.h index 8fca48fdbb9f2..b2fd8f66de17c 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1245,16 +1245,21 @@ STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAG 2 = malloc-allocated pointer (does not own it) 3 = has a pointer to the String object that owns the data pointer (m must be isbits) */ +#define JL_GENERICMEMORY_INLINED 0 +#define JL_GENERICMEMORY_GCMANAGED 1 +#define JL_GENERICMEMORY_MALLOCD 2 +#define JL_GENERICMEMORY_STRINGOWNED 3 + STATIC_INLINE int jl_genericmemory_how(jl_genericmemory_t *m) JL_NOTSAFEPOINT { if (m->ptr == (void*)((char*)m + 16)) // JL_SMALL_BYTE_ALIGNMENT (from julia_internal.h) - return 0; + return JL_GENERICMEMORY_INLINED; jl_value_t *owner = jl_genericmemory_data_owner_field(m); if (owner == (jl_value_t*)m) - return 1; + return JL_GENERICMEMORY_GCMANAGED; if (owner == NULL) - return 2; - return 3; + return JL_GENERICMEMORY_MALLOCD; + return JL_GENERICMEMORY_STRINGOWNED; } STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT diff --git a/src/julia_internal.h b/src/julia_internal.h index 39e49ae105f29..3cf9c77eeadeb 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -2097,7 +2097,7 @@ JL_DLLIMPORT jl_value_t *jl_dump_function_ir(jl_llvmf_dump_t *dump, char strip_i JL_DLLIMPORT jl_value_t *jl_dump_function_asm(jl_llvmf_dump_t *dump, char emit_mc, const char* asm_variant, const char *debuginfo, char binary, char raw); typedef jl_value_t *(*jl_codeinstance_lookup_t)(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t min_world, size_t max_world); -JL_DLLIMPORT void *jl_create_native(LLVMOrcThreadSafeModuleRef llvmmod, int trim, int cache, size_t world, jl_array_t *mod_array JL_MAYBE_UNROOTED, jl_array_t *worklist JL_MAYBE_UNROOTED, int all, jl_array_t *module_init_order JL_MAYBE_UNROOTED); +JL_DLLIMPORT void *jl_create_native(LLVMOrcThreadSafeModuleRef llvmmod, int trim, int cache, size_t world, jl_array_t *mod_array JL_MAYBE_UNROOTED, jl_array_t *worklist JL_MAYBE_UNROOTED, int all, jl_array_t *module_init_order JL_MAYBE_UNROOTED, jl_array_t *ext_foreign_cis JL_MAYBE_UNROOTED); JL_DLLIMPORT void *jl_emit_native(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int _external_linkage); JL_DLLIMPORT void jl_dump_native(void *native_code, const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname, diff --git a/src/module.c b/src/module.c index faa6ae41b712d..ffd2919d9a856 100644 --- a/src/module.c +++ b/src/module.c @@ -373,6 +373,8 @@ JL_DLLEXPORT void jl_update_loaded_bpart(jl_binding_t *b, jl_binding_partition_t jl_atomic_store_relaxed(&bpart->min_world, resolution.min_world); jl_atomic_store_relaxed(&bpart->max_world, resolution.max_world); bpart->restriction = resolution.binding_or_const; + if (resolution.binding_or_const) + jl_gc_wb(bpart, resolution.binding_or_const); bpart->kind = resolution.ultimate_kind; } @@ -1719,6 +1721,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b, struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, new_world, 0); new_bpart->kind = resolution.ultimate_kind | (kind & PARTITION_MASK_FLAG); new_bpart->restriction = resolution.binding_or_const; + jl_gc_wb_fresh(new_bpart, resolution.binding_or_const); assert(resolution.min_world <= new_world && resolution.max_world == ~(size_t)0); if (new_bpart->kind == old_bpart->kind && new_bpart->restriction == old_bpart->restriction) { JL_GC_POP(); diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index 31dd3e085033c..8411c9ae4bfb3 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -260,45 +260,55 @@ JL_DLLEXPORT float julia_half_to_float(uint16_t param) { (defined(__clang__) && __clang_major__ > 14)) && \ !defined(_CPU_PPC64_) && !defined(_CPU_PPC_) && \ !defined(_OS_WINDOWS_) && !defined(_CPU_RISCV64_) - #define FLOAT16_TYPE _Float16 - #define FLOAT16_TO_UINT16(x) (*(uint16_t*)&(x)) - #define FLOAT16_FROM_UINT16(x) (*(_Float16*)&(x)) + #define FLOAT16_ARG_TYPE _Float16 + #define FLOAT16_RET_TYPE _Float16 + #define FLOAT16_ARG_TO_UINT16(x) (*(uint16_t*)&(x)) + #define FLOAT16_RET_FROM_UINT16(x) (*(_Float16*)&(x)) // on older compilers, we need to emulate the platform-specific ABI -#elif defined(_CPU_X86_) || (defined(_CPU_X86_64_) && !defined(_OS_WINDOWS_)) - // on x86, we can use __m128; except on Windows where x64 calling - // conventions expect to pass __m128 by reference. - #define FLOAT16_TYPE __m128 - #define FLOAT16_TO_UINT16(x) take_from_xmm(x) - #define FLOAT16_FROM_UINT16(x) return_in_xmm(x) +#elif defined(_CPU_X86_) + // On i686, LLVM's half return convention uses XMM0, so we return __m128. + // But arguments are passed on the stack like integers, so use uint16_t. + #define FLOAT16_ARG_TYPE uint16_t + #define FLOAT16_RET_TYPE __m128 + #define FLOAT16_ARG_TO_UINT16(x) (x) + #define FLOAT16_RET_FROM_UINT16(x) return_in_xmm(x) +#elif defined(_CPU_X86_64_) && !defined(_OS_WINDOWS_) + // On x86_64 SysV, both f16 args and __m128 args go in XMM registers. + #define FLOAT16_ARG_TYPE __m128 + #define FLOAT16_RET_TYPE __m128 + #define FLOAT16_ARG_TO_UINT16(x) take_from_xmm(x) + #define FLOAT16_RET_FROM_UINT16(x) return_in_xmm(x) #elif defined(_CPU_PPC64_) || defined(_CPU_PPC_) // on PPC, pass Float16 as if it were an integer, similar to the old x86 ABI // before _Float16 - #define FLOAT16_TYPE uint16_t - #define FLOAT16_TO_UINT16(x) (x) - #define FLOAT16_FROM_UINT16(x) (x) + #define FLOAT16_ARG_TYPE uint16_t + #define FLOAT16_RET_TYPE uint16_t + #define FLOAT16_ARG_TO_UINT16(x) (x) + #define FLOAT16_RET_FROM_UINT16(x) (x) #else // otherwise, pass using floating-point calling conventions - #define FLOAT16_TYPE float - #define FLOAT16_TO_UINT16(x) ((uint16_t)*(uint32_t*)&(x)) - #define FLOAT16_FROM_UINT16(x) ({ uint32_t tmp = (uint32_t)(x); *(float*)&tmp; }) + #define FLOAT16_ARG_TYPE float + #define FLOAT16_RET_TYPE float + #define FLOAT16_ARG_TO_UINT16(x) ((uint16_t)*(uint32_t*)&(x)) + #define FLOAT16_RET_FROM_UINT16(x) ({ uint32_t tmp = (uint32_t)(x); *(float*)&tmp; }) #endif -JL_DLLEXPORT float julia__gnu_h2f_ieee(FLOAT16_TYPE param) +JL_DLLEXPORT float julia__gnu_h2f_ieee(FLOAT16_ARG_TYPE param) { - uint16_t param16 = FLOAT16_TO_UINT16(param); + uint16_t param16 = FLOAT16_ARG_TO_UINT16(param); return half_to_float(param16); } -JL_DLLEXPORT FLOAT16_TYPE julia__gnu_f2h_ieee(float param) +JL_DLLEXPORT FLOAT16_RET_TYPE julia__gnu_f2h_ieee(float param) { uint16_t res = float_to_half(param); - return FLOAT16_FROM_UINT16(res); + return FLOAT16_RET_FROM_UINT16(res); } -JL_DLLEXPORT FLOAT16_TYPE julia__truncdfhf2(double param) +JL_DLLEXPORT FLOAT16_RET_TYPE julia__truncdfhf2(double param) { uint16_t res = double_to_half(param); - return FLOAT16_FROM_UINT16(res); + return FLOAT16_RET_FROM_UINT16(res); } diff --git a/src/staticdata.c b/src/staticdata.c index f8d1c1388a303..053b0b37603b4 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -862,7 +862,7 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ else if (jl_is_genericmemory(v)) { jl_genericmemory_t *m = (jl_genericmemory_t*)v; const char *data = (const char*)m->ptr; - if (jl_genericmemory_how(m) == 3) { + if (jl_genericmemory_how(m) == JL_GENERICMEMORY_STRINGOWNED) { assert(jl_is_string(jl_genericmemory_data_owner_field(m))); } else if (layout->flags.arrayelem_isboxed) { @@ -1447,7 +1447,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED jl_genericmemory_t *m = (jl_genericmemory_t*)v; const jl_datatype_layout_t *layout = t->layout; size_t len = m->length; - // if (jl_genericmemory_how(m) == 3) { + // if (jl_genericmemory_how(m) == JL_GENERICMEMORY_STRINGOWNED) { // jl_value_t *owner = jl_genericmemory_data_owner_field(m); // write_uint(f, len); // write_pointerfield(s, owner); @@ -1507,7 +1507,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED if (len == 0) { // TODO: should we have a zero-page, instead of writing each type's fragment separately? write_padding(s->const_data, layout->size ? layout->size : isbitsunion); } - else if (jl_genericmemory_how(m) == 3) { + else if (jl_genericmemory_how(m) == JL_GENERICMEMORY_STRINGOWNED) { assert(jl_is_string(jl_genericmemory_data_owner_field(m))); write_padding(s->const_data, 1); } @@ -2953,11 +2953,35 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, size_t num_mis; jl_get_llvm_cis(native_functions, &num_mis, NULL); arraylist_grow(&MIs, num_mis); + + // Record MethodInstances for user-provided code (as reported by codegen) jl_get_llvm_cis(native_functions, &num_mis, (jl_code_instance_t**)MIs.items); for (size_t i = 0; i < num_mis; i++) { jl_code_instance_t *ci = (jl_code_instance_t*)MIs.items[i]; MIs.items[i] = (void*)jl_get_ci_mi(ci); } + + // Record MethodInstances for built-ins (used when dynamically dispatching to a + // built-in, e.g., in the Core._apply_iterate implementation) + jl_datatype_t *tt = NULL; + JL_GC_PUSH1(&tt); + for (size_t i = 0; i < jl_n_builtins; i++) { + jl_value_t *builtin = jl_builtin_instances[i]; + if (builtin == NULL) + continue; + + jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(builtin); + jl_value_t *params[2]; + params[0] = dt->name->wrapper; + params[1] = jl_tparam0(jl_anytuple_type); + tt = (jl_datatype_t*)jl_apply_tuple_type_v(params, 2); + jl_method_instance_t *mi = (jl_method_instance_t *)jl_method_lookup_by_tt( + tt, /* world */ 1, /* mt */ jl_nothing + ); + assert(!jl_is_nothing(mi)); + arraylist_push(&MIs, mi); + } + JL_GC_POP(); } } if (jl_options.trim) { @@ -3345,18 +3369,20 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli ff = f; } - jl_array_t *mod_array = NULL, *extext_methods = NULL, *new_ext_cis = NULL; + jl_array_t *mod_array = NULL, *extext_methods = NULL, *new_ext_cis = NULL, *ext_foreign_cis = NULL; int64_t checksumpos = 0; int64_t checksumpos_ff = 0; int64_t datastartpos = 0; - JL_GC_PUSH3(&mod_array, &extext_methods, &new_ext_cis); + JL_GC_PUSH4(&mod_array, &extext_methods, &new_ext_cis, &ext_foreign_cis); + + ext_foreign_cis = jl_alloc_vec_any(0); mod_array = jl_get_loaded_modules(); // __toplevel__ modules loaded in this session (from Base.loaded_modules_array) if (worklist) { if (_native_data != NULL) { if (suppress_precompile) newly_inferred = NULL; - *_native_data = jl_create_native(NULL, 0, 1, jl_atomic_load_acquire(&jl_world_counter), NULL, suppress_precompile ? (jl_array_t*)jl_an_empty_vec_any : worklist, 0, module_init_order); + *_native_data = jl_create_native(NULL, 0, 1, jl_atomic_load_acquire(&jl_world_counter), NULL, suppress_precompile ? (jl_array_t*)jl_an_empty_vec_any : worklist, 0, module_init_order, ext_foreign_cis); } jl_write_header_for_incremental(f, worklist, mod_array, udeps, srctextpos, &checksumpos); if (emit_split) { @@ -3369,7 +3395,7 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } } else if (_native_data != NULL) { - *_native_data = jl_create_native(NULL, jl_options.trim, 0, jl_atomic_load_acquire(&jl_world_counter), mod_array, NULL, jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL, module_init_order); + *_native_data = jl_create_native(NULL, jl_options.trim, 0, jl_atomic_load_acquire(&jl_world_counter), mod_array, NULL, jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL, module_init_order, ext_foreign_cis); } if (_native_data != NULL) native_functions = *_native_data; @@ -3405,6 +3431,15 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli new_ext_cis = jl_compute_new_ext_cis(); } + // Merge foreign & external CIs + if (ext_foreign_cis) { + size_t n_ext = jl_array_nrows(ext_foreign_cis); + for (size_t i = 0; i < n_ext; i++) { + jl_array_ptr_1d_push(new_ext_cis, jl_array_ptr_ref(ext_foreign_cis, i)); + } + } + ext_foreign_cis = NULL; // not needed anymore, free it + // Collect method extensions extext_methods = jl_alloc_vec_any(0); jl_collect_extext_methods(extext_methods, mod_array); diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 047db9af1cccf..285dcfc1dabe8 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -502,7 +502,7 @@ static jl_array_t *queue_external_cis(jl_array_t *list, jl_query_cache *query_ca int dispatch_status = jl_atomic_load_relaxed(&m->dispatch_status); if (!(dispatch_status & METHOD_SIG_LATEST_WHICH)) continue; // ignore replaced methods - if (ci->owner == jl_nothing && jl_atomic_load_relaxed(&ci->inferred) && jl_is_method(m) && jl_object_in_image((jl_value_t*)m->module)) { + if (jl_atomic_load_relaxed(&ci->inferred) && jl_is_method(m) && jl_object_in_image((jl_value_t*)m->module)) { int found = has_backedge_to_worklist(mi, &visited, &stack, query_cache); assert(found == 0 || found == 1 || found == 2); assert(stack.len == 0); diff --git a/src/support/Makefile b/src/support/Makefile index 69e97a1f65cc7..280b461677ba6 100644 --- a/src/support/Makefile +++ b/src/support/Makefile @@ -80,7 +80,7 @@ INCLUDED_SUPPORT_FILES := hashing.c:MurmurHash3.c # Compilation database generation .PHONY: regenerate-compile_commands regenerate-compile_commands: - TMPFILE=$$(mktemp -p $(BUILDDIR) compile_commands.json.XXXXXX); \ + @TMPFILE=$$(mktemp $(abspath $(BUILDDIR)/compile_commands.json.XXXXXX)); \ { \ CLANG_TOOLING_S_FLAGS="$$($(JULIAHOME)/contrib/escape_json.sh clang $(JCPPFLAGS) $(DEBUGFLAGS))"; \ CLANG_TOOLING_C_FLAGS="$$($(JULIAHOME)/contrib/escape_json.sh clang $(JCPPFLAGS) $(JCFLAGS) $(DEBUGFLAGS))"; \ @@ -123,7 +123,7 @@ clean: rm -f $(BUILDDIR)/core* rm -f $(BUILDDIR)/libsupport.a rm -f $(BUILDDIR)/libsupport-debug.a - rm -f $(BUILDDIR)/compile_commands.json + rm -f $(BUILDDIR)/compile_commands.json* rm -f $(BUILDDIR)/host/* .PHONY: compile-database diff --git a/src/support/dtypes.h b/src/support/dtypes.h index 2ee2d3529c10d..44db067019cd9 100644 --- a/src/support/dtypes.h +++ b/src/support/dtypes.h @@ -27,7 +27,9 @@ #define WIN32_LEAN_AND_MEAN /* Clang does not like fvisibility=hidden with windows headers. This adds the visibility attribute there. Arguably this is a clang bug. */ -#define DECLSPEC_IMPORT __declspec(dllimport) __attribute__ ((visibility("default"))) +# ifndef _COMPILER_MICROSOFT_ +# define DECLSPEC_IMPORT __declspec(dllimport) __attribute__ ((visibility("default"))) +# endif #include #if defined(_COMPILER_MICROSOFT_) && !defined(_SSIZE_T_) && !defined(_SSIZE_T_DEFINED) @@ -59,15 +61,21 @@ typedef intptr_t ssize_t; */ #ifdef _OS_WINDOWS_ +# ifndef _COMPILER_MICROSOFT_ +# define JL_VISIBILITY_DEFAULT __attribute__ ((visibility("default"))) +# else +# define JL_VISIBILITY_DEFAULT +# define JL_VISIBILITY_HIDDEN +# endif #define STDCALL __stdcall # ifdef JL_LIBRARY_EXPORTS_INTERNAL -# define JL_DLLEXPORT __declspec(dllexport) __attribute__ ((visibility("default"))) +# define JL_DLLEXPORT __declspec(dllexport) JL_VISIBILITY_DEFAULT # endif # ifdef JL_LIBRARY_EXPORTS_CODEGEN -# define JL_DLLEXPORT_CODEGEN __declspec(dllexport) __attribute__ ((visibility("default"))) +# define JL_DLLEXPORT_CODEGEN __declspec(dllexport) JL_VISIBILITY_DEFAULT # endif #define JL_HIDDEN -#define JL_DLLIMPORT __declspec(dllimport) __attribute__ ((visibility("default"))) +#define JL_DLLIMPORT __declspec(dllimport) JL_VISIBILITY_DEFAULT #else #define STDCALL #define JL_DLLIMPORT __attribute__ ((visibility("default"))) diff --git a/stdlib/Artifacts/test/runtests.jl b/stdlib/Artifacts/test/runtests.jl index cb81c16347abf..256bc7fdb6122 100644 --- a/stdlib/Artifacts/test/runtests.jl +++ b/stdlib/Artifacts/test/runtests.jl @@ -242,14 +242,29 @@ end anon = Module(:__anon__) Core.eval(anon, Meta.parse("using $(imports), Test")) # Ensure that we get the expected exception, since this test runs with --depwarn=error - Core.eval(anon, quote - try - artifact"socrates" - @assert false "this @artifact_str macro invocation should have failed!" - catch e - @test startswith("using Pkg instead of using LazyArtifacts is deprecated", e.msg) - end - end) + depwarn_flag = Base.JLOptions().depwarn + # 0: --depwarn=no + # 1: --depwarn=yes + # 2: --depwarn=error + if depwarn_flag == 0 + @warn "Skipping one test, because we are running with --depwarn=no" + @test_skip false + elseif depwarn_flag == 1 + expected_msg = "using Pkg instead of using LazyArtifacts is deprecated" + @test_logs (:warn,expected_msg) Core.eval(anon, :(artifact"socrates")) + elseif depwarn_flag == 2 + Core.eval(anon, quote + try + artifact"socrates" + # The previous line should have thrown, so we should not reach the next line: + error("this @artifact_str macro invocation should have failed!") + catch e + @test startswith("using Pkg instead of using LazyArtifacts is deprecated", e.msg) + end + end) + else + error("Unexpected value for Base.JLOptions().depwarn: $(depwarn_flag)") + end end end end diff --git a/stdlib/Dates/src/io.jl b/stdlib/Dates/src/io.jl index 76b3c8d4e0dfc..256a992f87fb9 100644 --- a/stdlib/Dates/src/io.jl +++ b/stdlib/Dates/src/io.jl @@ -747,3 +747,12 @@ for date_type in (:Date, :DateTime) # Parsable output will have type info displayed, thus it is implied @eval Base.typeinfo_implicit(::Type{$date_type}) = true end + +# minimal Base.TOML support +Base.TOML.Printer.printvalue(f::Function, io::IO, value::Date, sorted::Bool) = + Base.print(io, Dates.format(value, dateformat"YYYY-mm-dd")) +Base.TOML.Printer.printvalue(f::Function, io::IO, value::Time, sorted::Bool) = + Base.print(io, Dates.format(value, dateformat"HH:MM:SS.sss")) +Base.TOML.Printer.printvalue(f::Function, io::IO, value::DateTime, sorted::Bool) = + Base.print(io, Dates.format(value, dateformat"YYYY-mm-dd\THH:MM:SS.sss\Z")) +Base.TOML.Printer.is_valid_toml_value(@nospecialize(x::Union{Date,Time,DateTime})) = true diff --git a/stdlib/FileWatching/src/pidfile.jl b/stdlib/FileWatching/src/pidfile.jl index 3a3ac7e754817..0975fa0e6d7d1 100644 --- a/stdlib/FileWatching/src/pidfile.jl +++ b/stdlib/FileWatching/src/pidfile.jl @@ -1,3 +1,6 @@ +""" +A simple utility tool for creating advisory pidfiles (lock files). +""" module Pidfile @@ -372,3 +375,5 @@ function Base.close(lock::LockMonitor) end end # module + +public Pidfile diff --git a/stdlib/InteractiveUtils/src/InteractiveUtils.jl b/stdlib/InteractiveUtils/src/InteractiveUtils.jl index e7327ef5be331..a452d343680da 100644 --- a/stdlib/InteractiveUtils/src/InteractiveUtils.jl +++ b/stdlib/InteractiveUtils/src/InteractiveUtils.jl @@ -11,8 +11,8 @@ Base.Experimental.@optlevel 1 export apropos, edit, less, code_warntype, code_llvm, code_native, methodswith, varinfo, versioninfo, subtypes, supertypes, @which, @edit, @less, @functionloc, @code_warntype, - @code_typed, @code_lowered, @code_llvm, @code_native, @time_imports, clipboard, @trace_compile, @trace_dispatch, - @activate + @code_typed, @code_lowered, @code_llvm, @code_native, @time_imports, clipboard, + has_system_clipboard, @trace_compile, @trace_dispatch, @activate import Base.Docs.apropos @@ -253,7 +253,7 @@ function methodswith(@nospecialize(t::Type); supertypes::Bool=false) end # subtypes -function _subtypes_in!(mods::Array, x::Type) +function _subtypes_in!(mods::Array, @nospecialize(x::Type)) xt = unwrap_unionall(x) if !isabstracttype(x) || !isa(xt, DataType) # Fast path diff --git a/stdlib/InteractiveUtils/src/clipboard.jl b/stdlib/InteractiveUtils/src/clipboard.jl index 1cbdff9f45537..fb2c3260cf14b 100644 --- a/stdlib/InteractiveUtils/src/clipboard.jl +++ b/stdlib/InteractiveUtils/src/clipboard.jl @@ -2,7 +2,23 @@ # clipboard copy and paste +""" + has_system_clipboard() -> Bool + +Return `true` if the operating system has a clipboard mechanism available. +On macOS and Windows this is always `true`. On Linux and FreeBSD, it checks +for the presence of `xclip`, `xsel`, or `wl-copy` (from `wl-clipboard`). +On other systems, returns `false`. + +!!! compat "Julia 1.14" + This function requires Julia 1.14 or later. + +See also [`clipboard`](@ref). +""" +function has_system_clipboard end + if Sys.isapple() + has_system_clipboard() = true function clipboard(x) pbcopy_cmd = `pbcopy` @@ -31,6 +47,14 @@ if Sys.isapple() end elseif Sys.islinux() || Sys.KERNEL === :FreeBSD + function has_system_clipboard() + try + clipboardcmd() + return true + catch + return false + end + end _clipboardcmd = nothing const _clipboard_copy = Dict( :xsel => Sys.islinux() ? @@ -76,6 +100,7 @@ elseif Sys.islinux() || Sys.KERNEL === :FreeBSD end elseif Sys.iswindows() + has_system_clipboard() = true function clipboard(x::AbstractString) if Base.containsnul(x) throw(ArgumentError("Windows clipboard strings cannot contain NUL character")) @@ -142,6 +167,7 @@ elseif Sys.iswindows() end else + has_system_clipboard() = false clipboard(x="") = error("`clipboard` function not implemented for $(Sys.KERNEL)") end diff --git a/stdlib/PCRE2_jll/src/PCRE2_jll.jl b/stdlib/PCRE2_jll/src/PCRE2_jll.jl index c6e32bf3e6672..df53e4d47138a 100644 --- a/stdlib/PCRE2_jll/src/PCRE2_jll.jl +++ b/stdlib/PCRE2_jll/src/PCRE2_jll.jl @@ -16,7 +16,7 @@ artifact_dir::String = "" libpcre2_8_path::String = "" const libpcre2_8 = LazyLibrary( if Sys.iswindows() - BundledLazyLibraryPath("libpcre2-8.dll") + BundledLazyLibraryPath("libpcre2-8-0.dll") elseif Sys.isapple() BundledLazyLibraryPath("libpcre2-8.0.dylib") elseif Sys.islinux() || Sys.isfreebsd() diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 2970f3583bdf3..34fa2f1eacd28 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = release-1.13 -PKG_SHA1 = 4f9884fdb867f2c928ba43dc41da5f150aaec4ab +PKG_SHA1 = 5189ac693ce94be61feb3489763d78fea967c65c PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 diff --git a/stdlib/REPL/Project.toml b/stdlib/REPL/Project.toml index 6b37c892f8aa3..5242f8382bf22 100644 --- a/stdlib/REPL/Project.toml +++ b/stdlib/REPL/Project.toml @@ -3,6 +3,7 @@ uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" version = "1.11.0" [deps] +Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" diff --git a/stdlib/REPL/src/History/History.jl b/stdlib/REPL/src/History/History.jl index 3a7ff97543688..8ecf5b1e4730d 100644 --- a/stdlib/REPL/src/History/History.jl +++ b/stdlib/REPL/src/History/History.jl @@ -8,7 +8,8 @@ using StyledStrings: @styled_str as @S_str, Face, addface!, face!, annotations, using JuliaSyntaxHighlighting: highlight using Base.Threads using Dates -using InteractiveUtils: clipboard +using Base64: base64encode +import InteractiveUtils export HistoryFile, HistEntry, update!, runsearch diff --git a/stdlib/REPL/src/History/prompt.jl b/stdlib/REPL/src/History/prompt.jl index 78b87e7868344..1675280c92764 100644 --- a/stdlib/REPL/src/History/prompt.jl +++ b/stdlib/REPL/src/History/prompt.jl @@ -55,14 +55,19 @@ function select_keymap(events::Channel{Symbol}) end """ - create_prompt(events::Channel{Symbol}, term) + create_prompt(events::Channel{Symbol}, term, prefix, terminal_properties) Initialize a custom REPL prompt tied to `events` using the existing `term`. +The `terminal_properties` from the main REPL's MIState are shared with the +history search's MIState so that DA1/OSC responses parsed by the keymap +update the same object. + Returns a tuple `(term, prompt, istate, pstate)` ready for input handling and display. """ -function create_prompt(events::Channel{Symbol}, term, prefix::String = "\e[90m") +function create_prompt(events::Channel{Symbol}, term, prefix::String = "\e[90m", + terminal_properties::REPL.LineEdit.TerminalProperties = REPL.LineEdit.TerminalProperties()) prompt = REPL.LineEdit.Prompt( PROMPT_TEXT, # prompt prefix, "\e[0m", # prompt_prefix, prompt_suffix @@ -77,19 +82,23 @@ function create_prompt(events::Channel{Symbol}, term, prefix::String = "\e[90m") REPL.StylingPasses.StylingPass[]) # styling_passes interface = REPL.LineEdit.ModalInterface([prompt]) istate = REPL.LineEdit.init_state(term, interface) + # Share the main REPL's terminal_properties so that DA1/OSC responses + # parsed by the keymap during history search update the same object. + istate.terminal_properties = terminal_properties pstate = istate.mode_state[prompt] (; term, prompt, istate, pstate) end """ - runprompt!((; term,prompt,pstate,istate), events::Channel{Symbol}) + runprompt!((; term,prompt,pstate,istate), events::Channel{Symbol}, terminal_properties) Drive the prompt input loop until confirm, save, or abort. Emits `:edit`, `:confirm`, `:save`, or `:abort` into `events`, manages raw mode and bracketed paste, and cleans up on exit. """ -function runprompt!((; term, prompt, pstate, istate), events::Channel{Symbol}) +function runprompt!((; term, prompt, pstate, istate), events::Channel{Symbol}, + terminal_properties::REPL.LineEdit.TerminalProperties) Base.reseteof(term) REPL.LineEdit.raw!(term, true) REPL.LineEdit.enable_bracketed_paste(term) @@ -115,7 +124,7 @@ function runprompt!((; term, prompt, pstate, istate), events::Channel{Symbol}) break elseif status === :save print("\e[1G\e[J") - dest = savedest(term) + dest = savedest(term, terminal_properties) if isnothing(dest) push!(events, :redraw) else @@ -133,31 +142,94 @@ function runprompt!((; term, prompt, pstate, istate), events::Channel{Symbol}) end end -function savedest(term::Base.Terminals.TTYTerminal) +function savedest(term::Base.Terminals.TTYTerminal, props::REPL.LineEdit.TerminalProperties) out = term.out_stream + inp = term.in_stream print(out, "\e[1G\e[J") - clipsave = true + has_clip = clipboard_available(props) + # If DA1 already received and no clipboard available, skip straight to file save + if !has_clip && props.da1 !== nothing + return :filesave + end + clipsave = has_clip + # State machine for escape sequence parsing: + # :none - normal input + # :esc - read \e, waiting for next byte + # :csi - read \e[, consuming CSI sequence + esc_state = :none try print(out, get(Base.current_terminfo(), :cursor_invisible, "")) - fclip, ffile = [:emphasis, :bold], [:grey] - char = '\0' while true - print(out, S"\e[1G\e[2K{bold,grey:history>} {bold,emphasis:save to} {$fclip,inverse: Clipboard } {$ffile,inverse: File } {shadow:Tab to toggle ⋅ ⏎ to select}") - ichar = read(term.in_stream, Char) - if ichar ∈ ('\x03', '\x18', '\a') || char == ichar == '\e' - return - elseif ichar == '\r' - break + # Re-check clipboard availability (may have changed via DA1) + new_has_clip = clipboard_available(props) + if new_has_clip != has_clip + has_clip = new_has_clip + clipsave = has_clip + end + # Only render when not mid-escape-sequence + if esc_state === :none + if has_clip + fclip = clipsave ? [:emphasis, :bold] : [:grey] + ffile = clipsave ? [:grey] : [:emphasis, :bold] + print(out, S"\e[1G\e[2K{bold,grey:history>} {bold,emphasis:save to} {$fclip,inverse: Clipboard } {$ffile,inverse: File } {shadow:Tab to toggle ⋅ ⏎ to select}") + else + # DA1 hasn't arrived yet, show file-only with detection message + print(out, S"\e[1G\e[2K{bold,grey:history>} {bold,emphasis:save to} {emphasis,bold,inverse: File } {shadow:Detecting clipboard…}") + end + end + ichar = read(inp, Char) + if esc_state === :none + if ichar == '\e' + esc_state = :esc + elseif ichar ∈ ('\x03', '\x18', '\a') + return nothing + elseif ichar == '\r' + if !has_clip + clipsave = false + end + break + elseif has_clip + clipsave = !clipsave + end + elseif esc_state === :esc + if ichar == '\e' + return nothing # double-escape = abort + elseif ichar == '[' + esc_state = :csi + elseif ichar == ']' + REPL.LineEdit.receive_osc!(props, inp) + esc_state = :none + else + # Unknown escape sequence byte; treat as regular key + esc_state = :none + if ichar ∈ ('\x03', '\x18', '\a') + return nothing + elseif ichar == '\r' + if !has_clip + clipsave = false + end + break + elseif has_clip + clipsave = !clipsave + end + end + elseif esc_state === :csi + if ichar == '?' + # DA1 response (\e[?...c): blocking read until 'c' + REPL.LineEdit.receive_da1!(props, inp) + esc_state = :none + elseif UInt8(ichar) >= 0x40 && UInt8(ichar) <= 0x7e + # CSI final byte, sequence complete + esc_state = :none + end + # else: CSI parameter/intermediate byte, stay in :csi end - char = ichar - fclip, ffile = ffile, fclip - clipsave = !clipsave end finally # NOTE: While it may look like `:cursor_visible` would be the # appropriate choice to reverse `:cursor_invisible`, unfortunately # tmux-256color declares a sequence that doesn't actually make - # the cursor become visible again 😑. + # the cursor become visible again. print(out, get(Base.current_terminfo(), :cursor_normal, "")) print(out, "\e[1G\e[2K") end diff --git a/stdlib/REPL/src/History/resumablefiltering.jl b/stdlib/REPL/src/History/resumablefiltering.jl index 21c1239a60388..8d1f377266a2e 100644 --- a/stdlib/REPL/src/History/resumablefiltering.jl +++ b/stdlib/REPL/src/History/resumablefiltering.jl @@ -24,7 +24,7 @@ const FILTER_SEPARATOR = ';' List of single-character prefixes that set search modes. """ -const FILTER_PREFIXES = ('!', '`', '=', '/', '~', '>') +const FILTER_PREFIXES = ('!', '`', '=', '/', '~') """ FILTER_SHORTHELP_QUERY @@ -53,20 +53,22 @@ const FILTER_SHORTHELP = S""" See more information on behaviour and keybindings with '{REPL_History_search_prefix:??}'. + By default, each word in the search string is looked for in any order. + Should the search string start with {REPL_History_search_prefix:xyz>}, then only xyz-mode entries are considered. + Different search modes are available via prefixes, as follows: {emphasis:•} {REPL_History_search_prefix:=} looks for exact matches {emphasis:•} {REPL_History_search_prefix:!} {italic:excludes} exact matches {emphasis:•} {REPL_History_search_prefix:/} performs a regexp search {emphasis:•} {REPL_History_search_prefix:~} looks for fuzzy matches - {emphasis:•} {REPL_History_search_prefix:>} looks for a particular REPL mode {emphasis:•} {REPL_History_search_prefix:`} looks for an initialism (text with matching initials) You can also apply multiple restrictions with the separator '{REPL_History_search_separator:$FILTER_SEPARATOR}'. For example, {region:{REPL_History_search_prefix:/}^foo{REPL_History_search_separator:$FILTER_SEPARATOR}\ {REPL_History_search_prefix:`}bar{REPL_History_search_separator:$FILTER_SEPARATOR}\ -{REPL_History_search_prefix:>}shell} will look for history entries that start with "{code:foo}", - contains "{code:b... a... r...}", {italic:and} is a shell history entry. +{REPL_History_search_prefix:shell>}} will look for history entries that start with "{code:foo}", + contains "{code:b... a... r...}", {italic:and} are a shell history entry. """ const FILTER_LONGHELP = S""" @@ -108,9 +110,9 @@ function ConditionSet(spec::S) where {S <: AbstractString} function addcond!(condset::ConditionSet, cond::SubString) isempty(cond) && return kind = first(cond) - if kind ∈ ('!', '=', '`', '/', '>', '~') + if kind ∈ ('!', '=', '`', '/', '~') value = @view cond[2:end] - if kind ∈ ('`', '>', '~') + if kind ∈ ('`', '~') value = strip(value) elseif !all(isspace, value) value = if kind == '/' @@ -128,16 +130,23 @@ function ConditionSet(spec::S) where {S <: AbstractString} push!(condset.initialisms, value) elseif startswith(cond, '/') push!(condset.regexps, value) - elseif startswith(cond, '>') - push!(condset.modes, SubString(lowercase(value))) elseif startswith(cond, '~') push!(condset.fuzzy, value) end else if startswith(cond, '\\') && !(length(cond) > 1 && cond[2] == '\\') cond = @view cond[2:end] + else + rang = something(findfirst('>', cond), typemax(Int)) + if rang == something(findfirst(isspace, cond), ncodeunits(cond) + 1) - 1 + mode = @view cond[1:prevind(cond, rang)] + push!(condset.modes, SubString(lowercase(mode))) + cond = @view cond[rang + 1:end] + end end - push!(condset.words, strip(cond)) + cond = strip(cond) + isempty(cond) && return + push!(condset.words, cond) end nothing end @@ -155,10 +164,12 @@ function ConditionSet(spec::S) where {S <: AbstractString} elseif chr == '\\' escaped = true elseif chr == FILTER_SEPARATOR - str = SubString(spec, mark:pos - 1) - if !isempty(dropbytes) - str = SubString(convert(S, String(deleteat!(collect(codeunits(str)), dropbytes)))) + str = if isempty(dropbytes) + SubString(spec, mark:prevind(spec, pos)) + else + subbytes = deleteat!(codeunits(spec)[mark:pos-1], dropbytes) empty!(dropbytes) + SubString(convert(S, String(subbytes))) end addcond!(cset, lstrip(str)) mark = pos + 1 @@ -166,11 +177,14 @@ function ConditionSet(spec::S) where {S <: AbstractString} pos = nextind(spec, pos) end if mark <= lastind - str = SubString(spec, mark:pos - 1) - if !isempty(dropbytes) - str = SubString(convert(S, String(deleteat!(collect(codeunits(str)), dropbytes)))) + str = if isempty(dropbytes) + SubString(spec, mark) + else + subbytes = deleteat!(codeunits(spec)[mark:end], dropbytes) + empty!(dropbytes) + SubString(convert(S, String(subbytes))) end - addcond!(cset, lstrip(SubString(spec, mark:lastind))) + addcond!(cset, lstrip(str)) end cset end diff --git a/stdlib/REPL/src/History/search.jl b/stdlib/REPL/src/History/search.jl index a8351cc17157b..aeb60fe786b5d 100644 --- a/stdlib/REPL/src/History/search.jl +++ b/stdlib/REPL/src/History/search.jl @@ -8,12 +8,13 @@ Launch the interactive REPL history search interface. Spawns prompt and display tasks, waits for user confirm or abort, and returns the final selection (if any). """ -function runsearch(histfile::HistoryFile, term, prefix::String = "\e[90m") +function runsearch(histfile::HistoryFile, term, prefix::String = "\e[90m", + terminal_properties::REPL.LineEdit.TerminalProperties = REPL.LineEdit.TerminalProperties()) update!(histfile) events = Channel{Symbol}(Inf) - pspec = create_prompt(events, term, prefix) - ptask = @spawn runprompt!(pspec, events) - dtask = @spawn run_display!(pspec, events, histfile.records) + pspec = create_prompt(events, term, prefix, terminal_properties) + ptask = @spawn runprompt!(pspec, events, terminal_properties) + dtask = @spawn run_display!(pspec, events, histfile.records, terminal_properties) wait(ptask) fullselection(fetch(dtask)) end @@ -46,7 +47,8 @@ Drive the display event loop until confirm or abort. Listens for navigation, edits, save, and abort events, re-filters history incrementally, and re-renders via `redisplay_all`. """ -function run_display!((; term, pstate), events::Channel{Symbol}, hist::Vector{HistEntry}) +function run_display!((; term, pstate), events::Channel{Symbol}, hist::Vector{HistEntry}, + terminal_properties::REPL.LineEdit.TerminalProperties) # Output-related variables out = term.out_stream outsize = displaysize(out) @@ -177,7 +179,7 @@ function run_display!((; term, pstate), events::Channel{Symbol}, hist::Vector{Hi continue elseif event === :copy content = strip(fullselection(state).text) - isempty(content) || saveclipboard(term.out_stream, content) + isempty(content) || saveclipboard(term, content, terminal_properties) return EMPTY_STATE elseif event === :filesave content = strip(fullselection(state).text) @@ -376,13 +378,50 @@ function savefile(term::Base.Terminals.TTYTerminal, content::AbstractString) end """ - saveclipboard(term::Base.Terminals.TTYTerminal, content::AbstractString) + osc52_copy(io::IO, content::AbstractString) -Save `content` to the clipboard and record the action. +Write the OSC 52 escape sequence to set the system clipboard via the terminal. """ -function saveclipboard(msgio::IO, content::AbstractString) +function osc52_copy(io::IO, content::AbstractString) + write(io, "\e]52;c;", base64encode(content), "\e\\") +end + +""" + _has_osc52(props::REPL.LineEdit.TerminalProperties) -> Bool + +Return `true` if the terminal advertises OSC 52 clipboard support via DA1. +""" +_has_osc52(props::REPL.LineEdit.TerminalProperties) = + props.da1 !== nothing && 52 in props.da1 + +""" + clipboard_available(props::REPL.LineEdit.TerminalProperties) -> Bool + +Return `true` if clipboard saving is possible, either via system clipboard +commands or via OSC 52 terminal escape sequences. +""" +clipboard_available(props::REPL.LineEdit.TerminalProperties) = + InteractiveUtils.has_system_clipboard() || _has_osc52(props) + +""" + saveclipboard(term::Base.Terminals.TTYTerminal, content::AbstractString, + props::REPL.LineEdit.TerminalProperties) + +Save `content` to the clipboard and record the action. Falls back to OSC 52 +terminal escape sequences when no system clipboard command is available. +""" +function saveclipboard(term::Base.Terminals.TTYTerminal, content::AbstractString, + props::REPL.LineEdit.TerminalProperties) + out = term.out_stream nlines = count('\n', content) + 1 - clipboard(content) - println(msgio, S"\e[1G\e[2K{grey,bold:history>} {shadow:Copied $nlines \ + if _has_osc52(props) + osc52_copy(out, content) + elseif InteractiveUtils.has_system_clipboard() + InteractiveUtils.clipboard(content) + else + println(out, S"\e[1G\e[2K{grey,bold:history>} {red:No clipboard mechanism available}\n") + return + end + println(out, S"\e[1G\e[2K{grey,bold:history>} {shadow:Copied $nlines \ $(ifelse(nlines == 1, \"line\", \"lines\")) to clipboard}\n") end diff --git a/stdlib/REPL/src/LineEdit.jl b/stdlib/REPL/src/LineEdit.jl index b619002c2a1be..6786860f94b70 100644 --- a/stdlib/REPL/src/LineEdit.jl +++ b/stdlib/REPL/src/LineEdit.jl @@ -28,6 +28,43 @@ export run_interface, Prompt, ModalInterface, transition, reset_state, edit_inse const StringLike = Union{Char,String,SubString{String}} +mutable struct TerminalProperties + da1::Union{Nothing, Vector{Int}} # DA1 feature parameters + TerminalProperties() = new(nothing) +end + +""" + receive_da1!(props::TerminalProperties, io::IO) + +Read and parse a DA1 (Device Attributes) response from `io` +(after `\\e[?` has been consumed by the keymap). +Reads until `c` (DA1 terminator) or `^C` (bail-out), parses the semicolon-separated +parameters as integers, and stores them in `props.da1`. +""" +function receive_da1!(props::TerminalProperties, io::IO) + buf = IOBuffer() + while !eof(io) + b = read(io, UInt8) + if b == UInt8('c') # DA1 terminator + break + elseif b == 0x03 # ^C bail-out + break + else + write(buf, b) + end + end + body = String(take!(buf)) + params = Int[] + for part in split(body, ';') + n = tryparse(Int, part) + if n !== nothing + push!(params, n) + end + end + props.da1 = params + return +end + # interface for TextInterface function Base.getproperty(ti::TextInterface, name::Symbol) if name === :hp @@ -85,9 +122,10 @@ mutable struct MIState line_modify_lock::Base.ReentrantLock hint_generation_lock::Base.ReentrantLock n_keys_pressed::Int + terminal_properties::TerminalProperties end -MIState(i, mod, c, a, m) = MIState(i, mod, mod, c, a, m, String[], 0, Char[], 0, :none, :none, Channel{Function}(), Base.ReentrantLock(), Base.ReentrantLock(), 0) +MIState(i, mod, c, a, m) = MIState(i, mod, mod, c, a, m, String[], 0, Char[], 0, :none, :none, Channel{Function}(), Base.ReentrantLock(), Base.ReentrantLock(), 0, TerminalProperties()) const BufferLike = Union{MIState,ModeState,IOBuffer} const State = Union{MIState,ModeState} @@ -1829,7 +1867,7 @@ function add_nested_key!(keymap::Dict{Char, Any}, key::Union{String, Char}, valu c, i = y y = iterate(key, i) if !override && c in keys(keymap) && (y === nothing || !isa(keymap[c], Dict)) - error("Conflicting definitions for keyseq " * escape_string(key) * + error("Conflicting definitions for keyseq " * escape_string(string(key)) * " within one keymap") end if y === nothing @@ -2070,6 +2108,8 @@ const escape_defaults = merge!( "\e*" => nothing, "\e[*" => nothing, "\eO*" => nothing, + # Intercept DA1 responses + "\e[?" => (s::MIState, o...) -> receive_da1!(s.terminal_properties, terminal(s)), # Also ignore extended escape sequences # TODO: Support ranges of characters "\e[1**" => nothing, @@ -2714,7 +2754,12 @@ function history_search(mistate::MIState) else mimode.prompt_prefix end - result = histsearch(mimode.hist.history, term, prefix) + # Issue a DA1 query if we haven't received one yet, so that the + # terminal's OSC 52 clipboard capability can be detected. + if mistate.terminal_properties.da1 === nothing + write(term, "\e[c") + end + result = histsearch(mimode.hist.history, term, prefix, mistate.terminal_properties) mimode = if isnothing(result.mode) mistate.current_mode else @@ -2727,7 +2772,12 @@ function history_search(mistate::MIState) mistate.current_mode = mimode activate(mimode, state(mistate, mimode), termbuf, term) commit_changes(term, termbuf) - edit_insert(pstate, result.text) +if !isempty(result.text) + pstate.input_buffer.ptr = 1 + pstate.input_buffer.size = 0 + write(pstate.input_buffer, result.text) + seekend(pstate.input_buffer) +end refresh_multi_line(mistate) nothing end @@ -2753,6 +2803,7 @@ const prefix_history_keymap = merge!( "\e*" => "*", "\e[*" => "*", "\eO*" => "*", + "\e[?" => "*", "\e[1;5*" => "*", # Ctrl-Arrow "\e[1;2*" => "*", # Shift-Arrow "\e[1;3*" => "*", # Meta-Arrow diff --git a/stdlib/REPL/src/docview.jl b/stdlib/REPL/src/docview.jl index 9e56afc7f9f77..05b6629a38ac0 100644 --- a/stdlib/REPL/src/docview.jl +++ b/stdlib/REPL/src/docview.jl @@ -910,7 +910,7 @@ function accessible(mod::Module) return collect(bindings) end -function doc_completions(name, mod::Module=Main) +function doc_completions(name::AbstractString, mod::Module=Main) res = fuzzysort(name, accessible(mod)) # to insert an entry like `raw""` for `"@raw_str"` in `res` @@ -924,7 +924,7 @@ function doc_completions(name, mod::Module=Main) end res end -doc_completions(name::Symbol) = doc_completions(string(name), mod) +doc_completions(name::Symbol, mod::Module=Main) = doc_completions(string(name), mod) # Searching and apropos diff --git a/stdlib/REPL/src/precompile.jl b/stdlib/REPL/src/precompile.jl index 9d925a7c3a0f9..e0290cdecd680 100644 --- a/stdlib/REPL/src/precompile.jl +++ b/stdlib/REPL/src/precompile.jl @@ -223,6 +223,7 @@ let precompile(Tuple{typeof(Base.setindex!), Base.Dict{Any, Any}, Any, Int}) precompile(Tuple{typeof(Base.delete!), Base.Set{Any}, String}) precompile(Tuple{typeof(Base.:(==)), Char, String}) + precompile(Tuple{typeof(Base.convert), Type{Array{String, 1}}, Array{String, 1}}) finally ccall(:jl_tag_newly_inferred_disable, Cvoid, ()) end diff --git a/stdlib/REPL/test/history.jl b/stdlib/REPL/test/history.jl index 59abbebb8326c..8c130cd0c0ee6 100644 --- a/stdlib/REPL/test/history.jl +++ b/stdlib/REPL/test/history.jl @@ -206,7 +206,7 @@ end @test cset.regexps == [SubString("foo.*bar")] end @testset "Mode" begin - cset = ConditionSet(">shell") + cset = ConditionSet("shell>") @test cset.modes == [SubString("shell")] end @testset "Fuzzy" begin @@ -231,9 +231,11 @@ end cset = ConditionSet("hello\\;world;=exact") @test cset.words == [SubString("hello;world")] @test cset.exacts == [SubString("exact")] + cset = ConditionSet("1 \\; 2") + @test cset.words == [SubString("1 ; 2")] end @testset "Complex query" begin - cset = ConditionSet("some = words ;; !error ;> julia;/^def.*;") + cset = ConditionSet("some = words ;; !error ; julia> ;/^def.*;") @test cset.words == [SubString("some = words")] @test cset.negatives == [SubString("error")] @test cset.modes == [SubString("julia")] @@ -254,7 +256,7 @@ end @test isempty(spec2.regexps) end @testset "Complex query" begin - cset = ConditionSet("=exact;!neg;/foo.*bar;>julia") + cset = ConditionSet("=exact;!neg;/foo.*bar;julia>") spec = FilterSpec(cset) @test spec.exacts == ["exact"] @test spec.negatives == ["neg"] @@ -341,7 +343,7 @@ end end @testset "Mode" begin empty!(results) - cset = ConditionSet(">shell") + cset = ConditionSet("shell>") spec = FilterSpec(cset) seen = Set{Tuple{Symbol,String}}() @test filterchunkrev!(results, entries, spec, seen) == 0 diff --git a/stdlib/REPL/test/lineedit.jl b/stdlib/REPL/test/lineedit.jl index d86a6c6b5642a..4b641e687b751 100644 --- a/stdlib/REPL/test/lineedit.jl +++ b/stdlib/REPL/test/lineedit.jl @@ -1166,3 +1166,65 @@ end @test content(s) == "\"()\"" @test position(buffer(s)) == 2 end + +@testset "Conflicting definitions for keyseq" begin + @testset "string" begin + keymap = Dict{Char, Any}() + REPL.LineEdit.add_nested_key!(keymap, "a", "abc") + @test keymap == Dict('a' => "abc") + expected_msg = "Conflicting definitions for keyseq a within one keymap" + @test_throws ErrorException(expected_msg) REPL.LineEdit.add_nested_key!(keymap, "a", "abdef") + @test keymap == Dict('a' => "abc") + end + @testset "char" begin + keymap = Dict{Char, Any}() + REPL.LineEdit.add_nested_key!(keymap, 'a', "abc") + @test keymap == Dict('a' => "abc") + expected_msg = "Conflicting definitions for keyseq a within one keymap" + @test_throws ErrorException(expected_msg) REPL.LineEdit.add_nested_key!(keymap, 'a', "abdef") + @test keymap == Dict('a' => "abc") + end +end + +# Test TerminalProperties and DA1 parsing +@testset "TerminalProperties" begin + @testset "receive_da1!" begin + # Typical DA1 response body (after \e[? already consumed): "64;1;2;6;22c" + props = LineEdit.TerminalProperties() + io = IOBuffer("64;1;2;6;22c") + LineEdit.receive_da1!(props, io) + @test props.da1 == [64, 1, 2, 6, 22] + + # Single parameter + props2 = LineEdit.TerminalProperties() + io = IOBuffer("1c") + LineEdit.receive_da1!(props2, io) + @test props2.da1 == [1] + end + + @testset "receive_da1! with ^C bail-out" begin + props = LineEdit.TerminalProperties() + io = IOBuffer("64;1\x03") + LineEdit.receive_da1!(props, io) + @test props.da1 == [64, 1] + end + + @testset "receive_da1! with empty response" begin + # Empty response should store empty vector, not remain nothing + props = LineEdit.TerminalProperties() + io = IOBuffer("c") + LineEdit.receive_da1!(props, io) + @test props.da1 == Int[] + end + + @testset "TerminalProperties default initialization" begin + props = LineEdit.TerminalProperties() + @test props.da1 === nothing + end + + @testset "MIState has terminal_properties" begin + s = new_state() + @test s.terminal_properties isa LineEdit.TerminalProperties + @test s.terminal_properties.da1 === nothing + end +end diff --git a/stdlib/Serialization/src/Serialization.jl b/stdlib/Serialization/src/Serialization.jl index 6f4894dcc059f..ecfabbb9e454b 100644 --- a/stdlib/Serialization/src/Serialization.jl +++ b/stdlib/Serialization/src/Serialization.jl @@ -460,7 +460,7 @@ function serialize(s::AbstractSerializer, meth::Method) serialize(s, nothing) end if isdefined(meth, :recursion_relation) - serialize(s, method.recursion_relation) + serialize(s, meth.recursion_relation) else serialize(s, nothing) end @@ -1440,7 +1440,7 @@ end function deserialize(s::AbstractSerializer, X::Type{MemoryRef{T}} where T) x = Core.memoryref(deserialize(s))::X i = deserialize(s)::Int - i == 2 || (x = Core.memoryref(x, i, true)) + i == 2 || (x = Core.memoryrefnew(x, i, true)) return x::X end diff --git a/stdlib/Serialization/test/runtests.jl b/stdlib/Serialization/test/runtests.jl index e341c6e3eb9ec..1d87c13c037a5 100644 --- a/stdlib/Serialization/test/runtests.jl +++ b/stdlib/Serialization/test/runtests.jl @@ -673,3 +673,34 @@ if Int === Int64 @test @invokelatest(Main.f111_to_112(16)) == 256 end end + +@testset "MemoryRef" begin + old_m = Memory{Int}(undef, 10) + for i in 1:10 + old_m[i] = i^2 + end + old_x = memoryref(old_m, 5) + @test old_x[] == 25 + old_d = Dict(:x => old_x) + + old_str = sprint(serialize, old_d) + new_d = deserialize(IOBuffer(old_str)) + + @test new_d[:x] isa MemoryRef + @test new_d[:x][] == 25 +end + +@testset "Memory" begin + old_m = Memory{Int}(undef, 10) + for i in 1:10 + old_m[i] = i^3 + end + @test old_m[5] == 125 + old_d = Dict(:m => old_m) + + old_str = sprint(serialize, old_d) + new_d = deserialize(IOBuffer(old_str)) + + @test new_d[:m] isa Memory + @test new_d[:m][5] == 125 +end diff --git a/stdlib/TOML/src/TOML.jl b/stdlib/TOML/src/TOML.jl index 575f78be779d2..95e35dca92ae0 100644 --- a/stdlib/TOML/src/TOML.jl +++ b/stdlib/TOML/src/TOML.jl @@ -11,17 +11,11 @@ using Dates module Internals # The parser is defined in Base - using Base.TOML: Parser, parse, tryparse, ParserError, isvalid_barekey_char, reinit! + using Base.TOML: Parser, Printer, parse, tryparse, ParserError, reinit! # Put the error instances in this module for errtype in instances(Base.TOML.ErrorType) @eval using Base.TOML: $(Symbol(errtype)) end - # We put the printing functionality in a separate module since It - # defines a function `print` and we don't want that to collide with normal - # usage of `(Base.)print` in other files - module Printer - include("print.jl") - end end # https://github.com/JuliaLang/julia/issues/36605 diff --git a/stdlib/Unicode/src/Unicode.jl b/stdlib/Unicode/src/Unicode.jl index 5126f59325410..fbe856b6e85ce 100644 --- a/stdlib/Unicode/src/Unicode.jl +++ b/stdlib/Unicode/src/Unicode.jl @@ -225,7 +225,7 @@ end # because of the bitfields. combining_class(uc::Integer) = 0x000301 ≤ uc ≤ 0x10ffff ? unsafe_load(ccall(:utf8proc_get_property, Ptr{UInt16}, (UInt32,), uc), 2) : 0x0000 -combining_class(c::AbstractChar) = ismalformed(c) ? 0x0000 : combining_class(UInt32(c)) +combining_class(c::AbstractChar) = Base.ismalformed(c) ? 0x0000 : combining_class(UInt32(c)) """ isequal_normalized(s1::AbstractString, s2::AbstractString; casefold=false, stripmark=false, chartransform=identity) diff --git a/stdlib/Unicode/test/runtests.jl b/stdlib/Unicode/test/runtests.jl index 2af7015afa249..869e9f0522561 100644 --- a/stdlib/Unicode/test/runtests.jl +++ b/stdlib/Unicode/test/runtests.jl @@ -537,6 +537,11 @@ isequal_normalized_naive(s1, s2; kws...) = normalize(s1; kws...) == normalize(s2 end end +@testset "combining_class" begin + @test Unicode.combining_class('\u0302') === 0x00e6 # combining class "Above" + @test Unicode.combining_class(reinterpret(Char, UInt32(0xc0) << 24)) === 0x0000 # malformed +end + @testset "Docstrings" begin @test isempty(Docs.undocumented_names(Unicode)) end diff --git a/test/core.jl b/test/core.jl index 2702c34879523..823da2e6ff47c 100644 --- a/test/core.jl +++ b/test/core.jl @@ -8516,6 +8516,45 @@ let load_path = mktempdir() end end +# Deduplication of method tables in jl_foreach_reachable_mtable: +# when a method table is imported, it should not be visited multiple times. +let load_path = mktempdir() + depot_path = mkdepottempdir() + try + pushfirst!(LOAD_PATH, load_path) + pushfirst!(DEPOT_PATH, depot_path) + + write(joinpath(load_path, "MtDef.jl"), + """ + module MtDef + Base.Experimental.@MethodTable(mt) + end + """) + + MtDef = Base.require(Main, :MtDef) + @test length(MtDef.mt) == 0 + + write(joinpath(load_path, "MtUser.jl"), + """ + module MtUser + using MtDef: mt + Base.Experimental.@overlay mt sin(x::Int) = 42 + end + """) + + # MtUser imports mt from MtDef, making it reachable from both modules' + # bindings during precompilation. Without deduplication in + # jl_foreach_reachable_mtable, the overlay method would be serialized + # twice, causing an assertion failure when activating methods on load. + MtUser = Base.require(Main, :MtUser) + @test length(MtDef.mt) == 1 + finally + filter!((≠)(load_path), LOAD_PATH) + filter!((≠)(depot_path), DEPOT_PATH) + rm(load_path, recursive=true, force=true) + end +end + # merging va tuple unions @test Tuple === Union{Tuple{},Tuple{Any,Vararg}} @test Tuple{Any,Vararg} === Union{Tuple{Any},Tuple{Any,Any,Vararg}} diff --git a/test/docs.jl b/test/docs.jl index 148c0cf8ca649..3c8d17b6e3073 100644 --- a/test/docs.jl +++ b/test/docs.jl @@ -1296,6 +1296,10 @@ let x = Binding(Main, :+) @test Meta.parse(string(x)) == :(Base.:+) end +let x = Binding(Main, :(:)) + @test Meta.parse(string(x)) == :(Base.:(:)) +end + let x = Binding(Meta, :parse) @test Meta.parse(string(x)) == :(Base.Meta.parse) end diff --git a/test/intrinsics.jl b/test/intrinsics.jl index 5e18c1fb3672a..b73cf3e3b3aab 100644 --- a/test/intrinsics.jl +++ b/test/intrinsics.jl @@ -564,3 +564,26 @@ end)() f(gws) = passthrough(Core.bitcast(Core.LLVMPtr{UInt32,1}, gws)) f(C_NULL) end + +# Test bitcast on union values with inline_roots (split representation) +@testset "bitcast union with inline_roots" begin + struct BitcastMixedGC + a::Vector{Int} + b::Vector{Int} + c::Vector{Float64} + d::Int + end + @noinline function _bitcast_returns_union(x::Int) + x == 0 && return BitcastMixedGC(Int[], Int[], Float64[], 0) + x == 1 && return UInt(0) + x == 2 && return Int(0) + x == 3 && return C_NULL + return nothing + end + function _bitcast_trigger(x::Int) + val = _bitcast_returns_union(x) + return Core.Intrinsics.bitcast(Ptr{Nothing}, val) + end + @test _bitcast_trigger(1) === Ptr{Nothing}(0) + @test _bitcast_trigger(3) === Ptr{Nothing}(0) +end diff --git a/test/loading.jl b/test/loading.jl index 4baf979cf128a..b203d77761c0d 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -874,6 +874,34 @@ pop!(Base.active_project_callbacks) Base.set_active_project(saved_active_project) @test watcher_counter[] == 3 +# pkgversion should return cached version even after source is deleted +@testset "pkgversion after source deletion" begin + mktempdir() do tmp + pkg_dir = joinpath(tmp, "DeletedPkg") + src_dir = joinpath(pkg_dir, "src") + mkpath(src_dir) + write(joinpath(pkg_dir, "Project.toml"), """ + name = "DeletedPkg" + uuid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + version = "4.5.6" + """) + write(joinpath(src_dir, "DeletedPkg.jl"), "module DeletedPkg end") + old_act_proj = Base.ACTIVE_PROJECT[] + pushfirst!(LOAD_PATH, "@") + try + Base.set_active_project(pkg_dir) + @eval using DeletedPkg + m = @__MODULE__ + @test pkgversion(@invokelatest(m.DeletedPkg)) == v"4.5.6" + rm(pkg_dir; recursive=true) + @test pkgversion(@invokelatest(m.DeletedPkg)) == v"4.5.6" + finally + Base.set_active_project(old_act_proj) + popfirst!(LOAD_PATH) + end + end +end + # issue #28190 module Foo28190; import Libdl; end import .Foo28190.Libdl; import Libdl diff --git a/test/precompile.jl b/test/precompile.jl index 2bba1fcac011e..127ec7a17e312 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1887,6 +1887,7 @@ end dir = @__DIR__ @test success(pipeline(Cmd(`$(Base.julia_cmd()) --startup-file=no precompile_absint1.jl`; dir); stdout, stderr)) @test success(pipeline(Cmd(`$(Base.julia_cmd()) --startup-file=no precompile_absint2.jl`; dir); stdout, stderr)) + @test success(pipeline(Cmd(`$(Base.julia_cmd()) --startup-file=no precompile_extmi.jl`; dir); stdout, stderr)) end precompile_test_harness("Recursive types") do load_path diff --git a/test/precompile_absint1.jl b/test/precompile_absint1.jl index 08ec6788a356f..000ba116bae95 100644 --- a/test/precompile_absint1.jl +++ b/test/precompile_absint1.jl @@ -43,39 +43,35 @@ precompile_test_harness() do load_path let m = only(methods(TestAbsIntPrecompile1.basic_callee)) mi = only(Base.specializations(m)) ci = mi.cache - @test isdefined(ci, :next) - @test ci.owner === cache_owner + ci = check_presence(mi, nothing) + @test ci !== nothing + @test ci.owner === nothing @test ci.max_world == typemax(UInt) @test Base.module_build_id(TestAbsIntPrecompile1) == Base.object_build_id(ci) - ci = ci.next - @test !isdefined(ci, :next) - @test ci.owner === nothing + ci = check_presence(mi, cache_owner) + @test ci !== nothing + @test ci.owner === cache_owner @test ci.max_world == typemax(UInt) @test Base.module_build_id(TestAbsIntPrecompile1) == Base.object_build_id(ci) end let m = only(methods(sum, (Vector{Float64},))) - found = false for mi in Base.specializations(m) if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}} - ci = mi.cache - @test isdefined(ci, :next) - @test ci.owner === cache_owner - @test ci.max_world == typemax(UInt) - @test Base.module_build_id(TestAbsIntPrecompile1) == - Base.object_build_id(ci) - ci = ci.next - @test !isdefined(ci, :next) + ci = check_presence(mi, nothing) + @test ci !== nothing @test ci.owner === nothing @test ci.max_world == typemax(UInt) + @test Base.module_build_id(TestAbsIntPrecompile1) == Base.object_build_id(ci) + ci = check_presence(mi, cache_owner) + @test ci !== nothing + @test ci.owner === cache_owner + @test ci.max_world == typemax(UInt) @test Base.module_build_id(TestAbsIntPrecompile1) == Base.object_build_id(ci) - found = true - break end end - @test found end end end diff --git a/test/precompile_absint2.jl b/test/precompile_absint2.jl index 0ec03d802fa82..566d8a6b354c3 100644 --- a/test/precompile_absint2.jl +++ b/test/precompile_absint2.jl @@ -62,40 +62,34 @@ precompile_test_harness() do load_path TestAbsIntPrecompile2.Custom.PrecompileInterpreter()) let m = only(methods(TestAbsIntPrecompile2.basic_callee)) mi = only(Base.specializations(m)) - ci = mi.cache - @test isdefined(ci, :next) - @test ci.owner === cache_owner - @test ci.max_world == typemax(UInt) - @test Base.module_build_id(TestAbsIntPrecompile2) == - Base.object_build_id(ci) - ci = ci.next - @test !isdefined(ci, :next) + + ci = check_presence(mi, nothing) + @test ci !== nothing @test ci.owner === nothing @test ci.max_world == typemax(UInt) @test Base.module_build_id(TestAbsIntPrecompile2) == Base.object_build_id(ci) + ci = check_presence(mi, cache_owner) + @test ci !== nothing + @test ci.owner === cache_owner + @test ci.max_world == typemax(UInt) + @test Base.module_build_id(TestAbsIntPrecompile2) == Base.object_build_id(ci) end let m = only(methods(sum, (Vector{Float64},))) - found = false for mi = Base.specializations(m) if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}} - ci = mi.cache - @test isdefined(ci, :next) - @test ci.owner === cache_owner - @test ci.max_world == typemax(UInt) - @test Base.module_build_id(TestAbsIntPrecompile2) == - Base.object_build_id(ci) - ci = ci.next - @test !isdefined(ci, :next) + ci = check_presence(mi, nothing) + @test ci !== nothing @test ci.owner === nothing @test ci.max_world == typemax(UInt) - @test Base.module_build_id(TestAbsIntPrecompile2) == - Base.object_build_id(ci) - found = true - break + @test Base.module_build_id(TestAbsIntPrecompile2) == Base.object_build_id(ci) + ci = check_presence(mi, cache_owner) + @test ci !== nothing + @test ci.owner === cache_owner + @test ci.max_world == typemax(UInt) + @test Base.module_build_id(TestAbsIntPrecompile2) == Base.object_build_id(ci) end end - @test found end end end diff --git a/test/precompile_extmi.jl b/test/precompile_extmi.jl new file mode 100644 index 0000000000000..65856e98feb93 --- /dev/null +++ b/test/precompile_extmi.jl @@ -0,0 +1,100 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +using Test + +include("precompile_utils.jl") + +precompile_test_harness() do load_path + write(joinpath(load_path, "ExampleCompiler.jl"), :( + module ExampleCompiler + const CC = Core.Compiler + + struct ExampleInterpreter <: CC.AbstractInterpreter + world::UInt + inf_cache::Vector{CC.InferenceResult} + end + ExampleInterpreter(world::UInt) = + ExampleInterpreter(world, CC.InferenceResult[]) + + CC.InferenceParams(::ExampleInterpreter) = CC.InferenceParams() + CC.OptimizationParams(::ExampleInterpreter) = CC.OptimizationParams() + CC.get_inference_cache(interp::ExampleInterpreter) = interp.inf_cache + CC.cache_owner(interp::ExampleInterpreter) = :ExampleInterpreter + + CC.get_inference_world(interp::ExampleInterpreter) = interp.world + CC.lock_mi_inference(::ExampleInterpreter, ::Core.MethodInstance) = nothing + CC.unlock_mi_inference(::ExampleInterpreter, ::Core.MethodInstance) = nothing + + function infer(mi, world) + interp = ExampleInterpreter(world) + ci = CC.typeinf_ext(interp, mi, CC.SOURCE_MODE_GET_SOURCE) + @assert ci !== nothing "Inference of $mi failed" + + return ci + end + + function precompile(f, tt; world = Base.get_world_counter()) + mi = Base.method_instance(f, tt; world) + infer(mi, world) + end + end) |> string) + Base.compilecache(Base.PkgId("ExampleCompiler")) + + write(joinpath(load_path, "ExampleUser.jl"), :( + module ExampleUser + import ExampleCompiler + + function square(x) + return x * x + end + ExampleCompiler.precompile(square, (Float64,)) + + # Stubbed together version of PrecompileTools + ccall(:jl_tag_newly_inferred_enable, Cvoid, ()) + try + # Important `identity(::Any) mi does not belong to us` + ExampleCompiler.precompile(identity, (Float64,)) + finally + ccall(:jl_tag_newly_inferred_disable, Cvoid, ()) + end + end) |> string) + Base.compilecache(Base.PkgId("ExampleUser")) + + @eval let + using ExampleUser + cache_owner = :ExampleInterpreter + + mi_square = Base.method_instance(ExampleUser.square, (Float64,)) + @test check_presence(mi_square, :ExampleInterpreter) !== nothing + + mi_identity = Base.method_instance(identity, (Float64,)) + @test check_presence(mi_identity, :ExampleInterpreter) !== nothing + end +end + +# Test that methods with non-standard (foreign) source survive precompilation round-trip +precompile_test_harness() do load_path + write(joinpath(load_path, "ForeignIR.jl"), """ + module ForeignIR + struct MyIR + data::Vector{UInt8} + end + function foreign_func(x) + return x + 1 + end + let m = first(methods(foreign_func)) + m.source = MyIR(UInt8[1,2,3]) + end + end + """) + Base.compilecache(Base.PkgId("ForeignIR")) + + @eval let + using ForeignIR + m = first(methods(ForeignIR.foreign_func)) + @test m.source isa ForeignIR.MyIR + @test !Base.has_image_globalref(m) + end +end + +finish_precompile_test!() diff --git a/test/precompile_utils.jl b/test/precompile_utils.jl index c9a7c98d262e0..9c5d734431b2a 100644 --- a/test/precompile_utils.jl +++ b/test/precompile_utils.jl @@ -29,3 +29,18 @@ let original_depot_path = copy(Base.DEPOT_PATH) append!(Base.LOAD_PATH, original_load_path) end end + +function check_presence(mi, token) + ci = isdefined(mi, :cache) ? mi.cache : nothing + while ci !== nothing + # CI should have been validated + @test ci.max_world != Base.ReinferUtils.WORLD_AGE_REVALIDATION_SENTINEL + @test ci.min_world != ~zero(UInt) + # Chose a CI with the right owner and current validity. + if ci.owner === token && ci.max_world == typemax(UInt) + return ci + end + ci = isdefined(ci, :next) ? ci.next : nothing + end + return nothing +end diff --git a/test/syntax.jl b/test/syntax.jl index cec389be057ad..eef4e58bb626d 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -3730,7 +3730,7 @@ end @test p("public() = 6") == Expr(:(=), Expr(:call, :public), Expr(:block, 6)) end -@testset "removing argument sideeffects" begin +@testset "removing argument side effects" begin # Allow let blocks in broadcasted LHSes, but only evaluate them once: execs = 0 array = [1] @@ -3746,6 +3746,15 @@ end let; execs += 1; array; end::Vector{Int} .= 7 @test array == [7] @test execs == 4 + + # remove argument side effects on lhs kwcall + pa_execs = 0 + kw_execs = 0 + f60152(v, pa; kw) = copy(v) + @test (f60152([1, 2, 3], 0; kw=0) .*= 2) == [2,4,6] + @test (f60152([1, 2, 3], (pa_execs+=1); kw=(kw_execs+=1)) .*= 2) == [2,4,6] + @test pa_execs === 1 + @test kw_execs === 1 end # Allow GlobalRefs in macro definition diff --git a/test/trimming/basic_jll.jl b/test/trimming/basic_jll.jl index 748bc2585c050..87b409405dbfd 100644 --- a/test/trimming/basic_jll.jl +++ b/test/trimming/basic_jll.jl @@ -9,6 +9,8 @@ function print_string(fptr::Ptr{Cvoid}) println(Core.stdout, unsafe_string(ccall(fptr, Cstring, ()))) end +version_str::String = "1.2.3" + function @main(args::Vector{String})::Cint # Test the basic "Hello, world!" println(Core.stdout, "Julia! Hello, world!") @@ -18,6 +20,9 @@ function @main(args::Vector{String})::Cint println(Core.stdout, ver) @assert ver == build_ver + parsed_ver = VersionNumber(version_str) + @assert parsed_ver == v"1.2.3" + sleep(0.01) # Add an indirection via `@cfunction` / 1-arg ccall diff --git a/test/trimming/trimmability.jl b/test/trimming/trimmability.jl index 209a27343d18d..0fd9eedc6999d 100644 --- a/test/trimming/trimmability.jl +++ b/test/trimming/trimmability.jl @@ -19,6 +19,13 @@ area(c::Circle) = pi*c.radius^2 sum_areas(v::Vector{Shape}) = sum(area, v) +mutable struct Foo; x::Int; end +const storage = Foo[] +function add_one(x::Cint)::Cint + push!(storage, Foo(x)) + return x + 1 +end + function @main(args::Vector{String})::Cint println(Core.stdout, str()) println(Core.stdout, PROGRAM_FILE) @@ -38,6 +45,12 @@ function @main(args::Vector{String})::Cint d = mapreduce(x -> x^2, +, sorted_arr) # e = reduce(xor, rand(Int, 10)) + for i = 1:10 + # https://github.com/JuliaLang/julia/issues/60846 + add_one(Cint(i)) + GC.gc() + end + try sock = connect("localhost", 4900) if isopen(sock)