From 13308f88076a86f7718881a021f7f9bc3d1858fc Mon Sep 17 00:00:00 2001 From: MarcMush Date: Mon, 25 May 2020 10:27:59 +0200 Subject: [PATCH 01/39] WIP --- src/ProgressMeter.jl | 2 + src/parallel_progress.jl | 157 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 src/parallel_progress.jl diff --git a/src/ProgressMeter.jl b/src/ProgressMeter.jl index c268969..75fedd2 100644 --- a/src/ProgressMeter.jl +++ b/src/ProgressMeter.jl @@ -854,4 +854,6 @@ function ncalls(mapfun::Function, map_args) end end +include("parallel_progress.jl") + end diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl new file mode 100644 index 0000000..5467bdf --- /dev/null +++ b/src/parallel_progress.jl @@ -0,0 +1,157 @@ +""" +`prog = ParallelProgress(n; kw...)` + +works like `Progress` but can be used from other workers +Extra arguments after `update` or `cancel` are ignored + +# Example: +```jldoctest +julia> using Distributed +julia> addprocs() +julia> @everywhere using ProgressMeter +julia> prog = ParallelProgress(10; desc="test ") +julia> pmap(1:10) do + sleep(rand()) + next!(prog) + return myid() + end +``` +""" +struct ParallelProgress{C} + channel::C + n::Int +end + +const PP_NEXT = -1 +const PP_FINISH = -2 +const PP_CANCEL = -3 + +next!(pp::ParallelProgress) = put!(pp.channel, PP_NEXT) +finish!(pp::ParallelProgress) = put!(pp.channel, PP_FINISH) +cancel(pp::ParallelProgress, args...; kw...) = put!(pp.channel, PP_CANCEL) +update!(pp::ParallelProgress, counter, color = nothing) = put!(pp.channel, counter) + +function ParallelProgress(n::Int; kw...) + channel = RemoteChannel(() -> Channel{Int}(n)) + progress = Progress(n; kw...) + + @async while progress.counter < progress.n + f = take!(channel) + if f == PP_NEXT + next!(progress) + elseif f == PP_FINISH + finish!(progress) + break + elseif f == PP_CANCEL + cancel(progress) + break + elseif f >= 0 + update!(progress, f) + end + end + return ParallelProgress(channel, n) +end + +struct MultipleChannel{C} + channel::C + id +end +Distributed.put!(mc::MultipleChannel, x) = put!(mc.channel, (mc.id, x)) + + +struct MultipleProgress{C} + channel::C + amount::Int + lengths::Vector{Int} +end + +Base.getindex(mp::MultipleProgress, n::Integer) = ParallelProgress(MultipleChannel(mp.channel, n), mp.lengths[n]) +finish!(mp::MultipleProgress) = put!.([mp.channel], [(p, PP_FINISH) for p in 1:mp.amount]) + + +""" + prog = MultipleProgress(amount, lengths; kw...) + +equivalent to + + MultipleProgress(lengths*ones(T,amount); kw...) + +""" +function MultipleProgress(amount::Integer, lengths::T; kw...) where T <: Integer + MultipleProgress(lengths*ones(T,amount); kw...) +end + + +""" + prog = MultipleProgress(lengths; kws, kw...) + +generates one progressbar for each value in `lengths` and one for a global progressbar +`kw` arguments are applied on all progressbars +kws[i] arguments are applied on the i-th progressbar + +# Example +""" +function MultipleProgress(lengths::AbstractVector{<:Integer}; + kws = [() for _ in lengths], + kw...) + @assert length(lengths) == length(kws) "`length(lengths)` must be equal to `length(kws)`" + amount = length(lengths) + + total_length = sum(lengths) + main_progress = Progress(total_length; offset=0, kw...) + progresses = Union{Progress,Nothing}[nothing for _ in 1:amount] + taken_offsets = Set(Int[]) + channel = RemoteChannel(() -> Channel{Tuple{Int,Int}}(max(2amount, 64))) + + max_offsets = 1 + + # we must make sure that 2 progresses aren't updated at the same time + @async begin + while true + + (p, value) = take!(channel) + + # first time calling progress p + if isnothing(progresses[p]) + # find first available offset + offset = 1 + while offset in taken_offsets + offset += 1 + end + max_offsets = max(max_offsets, offset) + progresses[p] = Progress(lengths[p]; offset=offset, kw..., kws[p]...) + push!(taken_offsets, offset) + end + + if value == PP_NEXT + next!(progresses[p]) + next!(main_progress) + else + prev_p_value = progresses[p].counter + + if value == PP_FINISH + finish!(progresses[p]) + elseif value == PP_CANCEL + cancel(progresses[p]) + elseif value >= 0 + update!(progresses[p], value) + end + + update!(main_progress, + main_progress.counter - prev_p_value + progresses[p].counter) + end + + if progresses[p].counter >= lengths[p] + delete!(taken_offsets, progresses[p].offset) + end + + main_progress.counter >= total_length && break + end + + print("\n" ^ max_offsets) + end + + return MultipleProgress(channel, amount, collect(lengths)) +end + + From ea86d043957ecaee9ebd7c23a875d295f535dd21 Mon Sep 17 00:00:00 2001 From: MarcMush Date: Mon, 25 May 2020 16:15:07 +0200 Subject: [PATCH 02/39] [WIP] working version --- src/ProgressMeter.jl | 4 +++- src/parallel_progress.jl | 21 ++++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/ProgressMeter.jl b/src/ProgressMeter.jl index 75fedd2..6da9aef 100644 --- a/src/ProgressMeter.jl +++ b/src/ProgressMeter.jl @@ -3,7 +3,9 @@ module ProgressMeter using Printf: @sprintf using Distributed -export Progress, ProgressThresh, ProgressUnknown, BarGlyphs, next!, update!, cancel, finish!, @showprogress, progress_map, progress_pmap, ijulia_behavior +export Progress, ProgressThresh, ProgressUnknown, BarGlyphs, next!, update!, cancel, + finish!, @showprogress, progress_map, progress_pmap, ijulia_behavior, + MultipleProgress, ParallelProgress """ `ProgressMeter` contains a suite of utilities for displaying progress diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 5467bdf..11dee18 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -86,10 +86,24 @@ end prog = MultipleProgress(lengths; kws, kw...) generates one progressbar for each value in `lengths` and one for a global progressbar -`kw` arguments are applied on all progressbars -kws[i] arguments are applied on the i-th progressbar + - `kw` arguments are applied on all progressbars + - `kws[i]` arguments are applied on the i-th progressbar # Example +```jldoctest +julia> using Distributed +julia> addprocs(2) +julia> @everywhere using ProgressMeter +julia> p = MultipleProgress(5,10; desc="global ", kws=[(desc="task \$i ",) for i in 1:5]) + pmap(1:5) do x + for i in 1:10 + sleep(rand()) + next!(p[x]) + end + sleep(0.01) + myid() + end +``` """ function MultipleProgress(lengths::AbstractVector{<:Integer}; kws = [() for _ in lengths], @@ -105,7 +119,8 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; max_offsets = 1 - # we must make sure that 2 progresses aren't updated at the same time + # we must make sure that 2 progresses aren't updated at the same time, + # that's why we use only one Channel @async begin while true From e9e10cf61fe6f22241ffc662c074330b969f779a Mon Sep 17 00:00:00 2001 From: MarcMush Date: Thu, 25 Jun 2020 15:12:07 +0200 Subject: [PATCH 03/39] add test_parallel.jl --- src/parallel_progress.jl | 74 +++++++++++------ test/runtests.jl | 2 +- test/test_parallel.jl | 172 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 221 insertions(+), 27 deletions(-) create mode 100644 test/test_parallel.jl diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 11dee18..c734e3b 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -17,9 +17,8 @@ julia> pmap(1:10) do end ``` """ -struct ParallelProgress{C} - channel::C - n::Int +mutable struct ParallelProgress + channel end const PP_NEXT = -1 @@ -31,27 +30,38 @@ finish!(pp::ParallelProgress) = put!(pp.channel, PP_FINISH) cancel(pp::ParallelProgress, args...; kw...) = put!(pp.channel, PP_CANCEL) update!(pp::ParallelProgress, counter, color = nothing) = put!(pp.channel, counter) -function ParallelProgress(n::Int; kw...) +function ParallelProgress(n::Integer; kw...) channel = RemoteChannel(() -> Channel{Int}(n)) progress = Progress(n; kw...) + pp = ParallelProgress(channel) - @async while progress.counter < progress.n - f = take!(channel) - if f == PP_NEXT - next!(progress) - elseif f == PP_FINISH - finish!(progress) - break - elseif f == PP_CANCEL - cancel(progress) - break - elseif f >= 0 - update!(progress, f) + @async begin + while progress.counter < progress.n + f = take!(channel) + if f == PP_NEXT + next!(progress) + elseif f == PP_FINISH + finish!(progress) + break + elseif f == PP_CANCEL + cancel(progress) + break + elseif f >= 0 + update!(progress, f) + end end + while isready(pp.channel) + take!(pp.channel) + end + pp.channel = FakeChannel() end - return ParallelProgress(channel, n) + + return pp end +struct FakeChannel end +Distributed.put!(::FakeChannel, x) = nothing + struct MultipleChannel{C} channel::C id @@ -59,15 +69,15 @@ end Distributed.put!(mc::MultipleChannel, x) = put!(mc.channel, (mc.id, x)) -struct MultipleProgress{C} - channel::C +mutable struct MultipleProgress + channel amount::Int lengths::Vector{Int} end -Base.getindex(mp::MultipleProgress, n::Integer) = ParallelProgress(MultipleChannel(mp.channel, n), mp.lengths[n]) +Base.getindex(mp::MultipleProgress, n::Integer) = ParallelProgress(MultipleChannel(mp.channel, n)) finish!(mp::MultipleProgress) = put!.([mp.channel], [(p, PP_FINISH) for p in 1:mp.amount]) - +cancel(mp::MultipleProgress) = put!.([mp.channel], [(p, PP_CANCEL) for p in 1:mp.amount]) """ prog = MultipleProgress(amount, lengths; kw...) @@ -106,6 +116,7 @@ julia> p = MultipleProgress(5,10; desc="global ", kws=[(desc="task \$i ",) for i ``` """ function MultipleProgress(lengths::AbstractVector{<:Integer}; + count_overshoot::Bool = false, kws = [() for _ in lengths], kw...) @assert length(lengths) == length(kws) "`length(lengths)` must be equal to `length(kws)`" @@ -119,6 +130,8 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; max_offsets = 1 + mp = MultipleProgress(channel, amount, collect(lengths)) + # we must make sure that 2 progresses aren't updated at the same time, # that's why we use only one Channel @async begin @@ -138,17 +151,24 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; push!(taken_offsets, offset) end + if value == PP_NEXT - next!(progresses[p]) - next!(main_progress) + if count_overshoot || progresses[p].counter < lengths[p] + next!(progresses[p]) + next!(main_progress) + end else prev_p_value = progresses[p].counter if value == PP_FINISH finish!(progresses[p]) elseif value == PP_CANCEL + finish!(progresses[p]) cancel(progresses[p]) elseif value >= 0 + if !count_overshoot + value = min(value, lengths[n]) + end update!(progresses[p], value) end @@ -162,11 +182,13 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; main_progress.counter >= total_length && break end - + while isready(mp.channel) + take!(mp.channel) + end + mp.channel = FakeChannel() print("\n" ^ max_offsets) end - return MultipleProgress(channel, amount, collect(lengths)) + return mp end - diff --git a/test/runtests.jl b/test/runtests.jl index f843a87..b52c2b4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,7 +7,7 @@ include("test_showvalues.jl") include("test_map.jl") include("test_float.jl") include("test_threads.jl") - +include("test_parallel.jl") println("") println("All tests complete") diff --git a/test/test_parallel.jl b/test/test_parallel.jl new file mode 100644 index 0000000..ce2df05 --- /dev/null +++ b/test/test_parallel.jl @@ -0,0 +1,172 @@ +using Distributed + +@testset "ParallelProgress() tests" begin + if workers() != [1] + rmworkers(workers()) + end + addprocs(4) + @everywhere import ProgressMeter + + procs = nworkers() + (procs == 1) && @info "incomplete tests: nworkers() == 1" + @test all([@fetchfrom w @isdefined(ProgressMeter) for w in workers()]) + + println("Testing simultaneous updates...") + p = ProgressMeter.ParallelProgress(100) + @sync for _ in 1:10 + @async for _ in 1:10 + sleep(0.1) + ProgressMeter.next!(p) + end + end + sleep(0.01) + + println("Testing over-shooting...") + p = ProgressMeter.ParallelProgress(10) + for _ in 1:100 + sleep(0.01) + ProgressMeter.next!(p) + end + sleep(0.01) + @test p.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + + println("Testing under-shooting...") + p = ProgressMeter.ParallelProgress(200) + for _ in 1:100 + sleep(0.01) + ProgressMeter.next!(p) + end + ProgressMeter.finish!(p) + sleep(0.01) + @test p.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + + println("Testing rapid over-shooting...") + p = ProgressMeter.ParallelProgress(100) + ProgressMeter.next!(p) + sleep(0.1) + for _ in 1:10000 + ProgressMeter.next!(p) + end + sleep(0.01) + @test p.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + + + println("Testing early cancel...") + p = ProgressMeter.ParallelProgress(100) + for _ in 1:50 + sleep(0.02) + ProgressMeter.next!(p) + end + ProgressMeter.cancel(p) + sleep(0.01) + @test p.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + + + + println("Testing across $procs workers with @distributed...") + n = 20 #per core + p = ProgressMeter.ParallelProgress(n*procs) + @sync @distributed for _ in 1:n*procs + sleep(0.05) + ProgressMeter.next!(p) + end + sleep(0.01) + @test p.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + + + println("Testing across $procs workers with pmap...") + n = 20 + p = ProgressMeter.ParallelProgress(n*procs) + ids = pmap(1:n*procs) do i + sleep(0.05) + ProgressMeter.next!(p) + return myid() + end + sleep(0.01) + @test p.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + @test length(unique(ids)) == procs + +end + +@testset "MultipleProgress() tests" begin + + procs = nworkers() + @test all([@fetchfrom w @isdefined(ProgressMeter) for w in workers()]) + + function test_MP_finished(lengths, finish; kwargs...) + n = length(lengths) + p = ProgressMeter.MultipleProgress(lengths; kwargs...) + for _ in 1:100 + sleep(0.01) + for i in 1:n + ProgressMeter.next!(p[i]) + end + end + finish && ProgressMeter.finish!(p) + sleep(0.01) + for i in 1:n + @test p[i].channel.channel isa ProgressMeter.FakeChannel + end + end + + println("Testing custom titles and color...") + test_MP_finished([100, 100], false; + desc="default ", color=:yellow, + kws=[(desc=" task A ",), (desc=" task B ",)]) + + println("Testing over-shooting and under-shooting...") + test_MP_finished([10, 110], true; dt=0.01) + + println("Testing over-shooting with count_overshoot...") + test_MP_finished([10, 190], false; count_overshoot=true, dt=0.01) + + + println("Testing rapid over-shooting...") + p = ProgressMeter.MultipleProgress([100]; dt=0.01, count_overshoot=true) + ProgressMeter.next!(p[1]) + sleep(0.1) + for _ in 1:10000 + ProgressMeter.next!(p[1]) + end + sleep(0.01) + @test p[1].channel.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + + + println("Testing early cancel...") + p = ProgressMeter.MultipleProgress([100, 80]) + for _ in 1:50 + sleep(0.02) + ProgressMeter.next!(p[1]) + ProgressMeter.next!(p[2]) + end + ProgressMeter.cancel(p[1]) + ProgressMeter.finish!(p[2]) + sleep(0.1) + @test p[1].channel.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + + println("Testing early global cancel...") + p = ProgressMeter.MultipleProgress([100, 80]) + for _ in 1:50 + sleep(0.02) + ProgressMeter.next!(p[1]) + ProgressMeter.next!(p[2]) + end + ProgressMeter.cancel(p) + sleep(0.1) + @test p[2].channel.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + + println("Testing bar remplacement with $procs workers and pmap...") + lengths = rand(20:40, 2*procs) + p = ProgressMeter.MultipleProgress(lengths; dt=0.01, kws=[(desc=" task $i ",) for i in 1:2*procs]) + ids = pmap(1:2*procs) do ip + for _ in 1:lengths[ip] + sleep(0.05) + ProgressMeter.next!(p[ip]) + end + myid() + end + sleep(0.01) + @test length(unique(ids)) == procs + @test p[1].channel.channel isa ProgressMeter.FakeChannel # finished + +end From 68d4de7e390e0f0074df6d9796de5026714d383d Mon Sep 17 00:00:00 2001 From: MarcMush Date: Thu, 25 Jun 2020 17:04:34 +0200 Subject: [PATCH 04/39] (WIP) add parallel update! --- src/parallel_progress.jl | 75 ++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index c734e3b..206126e 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -21,35 +21,39 @@ mutable struct ParallelProgress channel end -const PP_NEXT = -1 -const PP_FINISH = -2 -const PP_CANCEL = -3 +const PP_NEXT = :next +const PP_CANCEL = :cancel +const PP_FINISH = :finish +const PP_UPDATE = :update -next!(pp::ParallelProgress) = put!(pp.channel, PP_NEXT) -finish!(pp::ParallelProgress) = put!(pp.channel, PP_FINISH) -cancel(pp::ParallelProgress, args...; kw...) = put!(pp.channel, PP_CANCEL) -update!(pp::ParallelProgress, counter, color = nothing) = put!(pp.channel, counter) +next!(pp::ParallelProgress, args...; kw...) = put!(pp.channel, (PP_NEXT, args, kw)) +cancel(pp::ParallelProgress, args...; kw...) = put!(pp.channel, (PP_CANCEL, args, kw)) +finish!(pp::ParallelProgress, args...; kw...) = put!(pp.channel, (PP_FINISH, args, kw)) +update!(pp::ParallelProgress, args...; kw...) = put!(pp.channel, (PP_UPDATE, args, kw)) function ParallelProgress(n::Integer; kw...) - channel = RemoteChannel(() -> Channel{Int}(n)) + channel = RemoteChannel(() -> Channel{Tuple}(n)) progress = Progress(n; kw...) pp = ParallelProgress(channel) @async begin while progress.counter < progress.n - f = take!(channel) + f, args, kw = take!(channel) if f == PP_NEXT - next!(progress) - elseif f == PP_FINISH - finish!(progress) - break + next!(progress, args...; kw...) elseif f == PP_CANCEL - cancel(progress) + cancel(progress, args...; kw...) + break + elseif f == PP_FINISH + finish!(progress, args...; kw...) break - elseif f >= 0 - update!(progress, f) + elseif f == PP_UPDATE + update!(progress, args...; kw...) + else + error("not recognized: $f") end end + # empty channel before ending it while isready(pp.channel) take!(pp.channel) end @@ -59,14 +63,15 @@ function ParallelProgress(n::Integer; kw...) return pp end +# fake channel to allow over-shoot struct FakeChannel end Distributed.put!(::FakeChannel, x) = nothing -struct MultipleChannel{C} - channel::C +struct MultipleChannel + channel id end -Distributed.put!(mc::MultipleChannel, x) = put!(mc.channel, (mc.id, x)) +Distributed.put!(mc::MultipleChannel, x) = put!(mc.channel, (mc.id, x...)) mutable struct MultipleProgress @@ -76,8 +81,9 @@ mutable struct MultipleProgress end Base.getindex(mp::MultipleProgress, n::Integer) = ParallelProgress(MultipleChannel(mp.channel, n)) -finish!(mp::MultipleProgress) = put!.([mp.channel], [(p, PP_FINISH) for p in 1:mp.amount]) -cancel(mp::MultipleProgress) = put!.([mp.channel], [(p, PP_CANCEL) for p in 1:mp.amount]) +Base.lastindex(mp::MultipleProgress) = mp.amount +finish!(mp::MultipleProgress, args...; kw...) = finish!.(MultipleProgress[:]) +cancel(mp::MultipleProgress, args...; kw...) = cancel.(MultipleProgress[:]) """ prog = MultipleProgress(amount, lengths; kw...) @@ -126,7 +132,7 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; main_progress = Progress(total_length; offset=0, kw...) progresses = Union{Progress,Nothing}[nothing for _ in 1:amount] taken_offsets = Set(Int[]) - channel = RemoteChannel(() -> Channel{Tuple{Int,Int}}(max(2amount, 64))) + channel = RemoteChannel(() -> Channel{Tuple}(max(2amount, 64))) max_offsets = 1 @@ -137,7 +143,7 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; @async begin while true - (p, value) = take!(channel) + p, f, args, kw = take!(channel) # first time calling progress p if isnothing(progresses[p]) @@ -152,24 +158,27 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; end - if value == PP_NEXT + if f == PP_NEXT if count_overshoot || progresses[p].counter < lengths[p] - next!(progresses[p]) + next!(progresses[p], args...; kw...) next!(main_progress) end else prev_p_value = progresses[p].counter - if value == PP_FINISH - finish!(progresses[p]) - elseif value == PP_CANCEL + if f == PP_FINISH + finish!(progresses[p], args...; kw...) + elseif f == PP_CANCEL finish!(progresses[p]) - cancel(progresses[p]) - elseif value >= 0 - if !count_overshoot - value = min(value, lengths[n]) + cancel(progresses[p], args...; kw...) + elseif f == PP_UPDATE + if !count_overshoot && !isempty(args) + value = min(args[1], lengths[n]) + update!(progresses[p], value, args[2:end]...; kw...) + else + update!(progresses[p], args...; kw...) end - update!(progresses[p], value) + end update!(main_progress, From 3284c083406a3c1a8cf9857e7c9f4082c009cde1 Mon Sep 17 00:00:00 2001 From: MarcMush Date: Fri, 10 Jul 2020 18:07:48 +0200 Subject: [PATCH 05/39] first working version --- src/parallel_progress.jl | 95 +++++++++++++++++++++++----------------- test/runtests.jl | 13 +++--- test/test_parallel.jl | 11 ++--- 3 files changed, 67 insertions(+), 52 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 206126e..e2d0c2a 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -80,10 +80,10 @@ mutable struct MultipleProgress lengths::Vector{Int} end -Base.getindex(mp::MultipleProgress, n::Integer) = ParallelProgress(MultipleChannel(mp.channel, n)) +Base.getindex(mp::MultipleProgress, n) = ParallelProgress.(MultipleChannel.(fill(mp.channel), n)) Base.lastindex(mp::MultipleProgress) = mp.amount -finish!(mp::MultipleProgress, args...; kw...) = finish!.(MultipleProgress[:]) -cancel(mp::MultipleProgress, args...; kw...) = cancel.(MultipleProgress[:]) +finish!(mp::MultipleProgress, args...; kw...) = finish!.(mp[1:end]) +cancel(mp::MultipleProgress, args...; kw...) = cancel.(mp[1:end]) """ prog = MultipleProgress(amount, lengths; kw...) @@ -141,55 +141,68 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; # we must make sure that 2 progresses aren't updated at the same time, # that's why we use only one Channel @async begin - while true + while main_progress.counter < total_length p, f, args, kw = take!(channel) - # first time calling progress p - if isnothing(progresses[p]) - # find first available offset - offset = 1 - while offset in taken_offsets - offset += 1 + # main progressbar + if p == 0 + if f == PP_CANCEL + cancel(main_progress, args...; kw...) + break + elseif f == PP_UPDATE + update!(main_progress, args...; kw...) + elseif f == PP_NEXT + next!(main_progress, args...; kw...) + elseif f == PP_FINISH + finish!(main_progress, args...; kw...) end - max_offsets = max(max_offsets, offset) - progresses[p] = Progress(lengths[p]; offset=offset, kw..., kws[p]...) - push!(taken_offsets, offset) - end - + else - if f == PP_NEXT - if count_overshoot || progresses[p].counter < lengths[p] - next!(progresses[p], args...; kw...) - next!(main_progress) + # first time calling progress p + if isnothing(progresses[p]) + # find first available offset + offset = 1 + while offset in taken_offsets + offset += 1 + end + max_offsets = max(max_offsets, offset) + progresses[p] = Progress(lengths[p]; offset=offset, kw..., kws[p]...) + push!(taken_offsets, offset) end - else - prev_p_value = progresses[p].counter - - if f == PP_FINISH - finish!(progresses[p], args...; kw...) - elseif f == PP_CANCEL - finish!(progresses[p]) - cancel(progresses[p], args...; kw...) - elseif f == PP_UPDATE - if !count_overshoot && !isempty(args) - value = min(args[1], lengths[n]) - update!(progresses[p], value, args[2:end]...; kw...) - else - update!(progresses[p], args...; kw...) + + + if f == PP_NEXT + if count_overshoot || progresses[p].counter < lengths[p] + next!(progresses[p], args...; kw...) + next!(main_progress) end + else + prev_p_value = progresses[p].counter - end + if f == PP_FINISH + finish!(progresses[p], args...; kw...) + elseif f == PP_CANCEL + finish!(progresses[p]) + cancel(progresses[p], args...; kw...) + elseif f == PP_UPDATE + if !count_overshoot && !isempty(args) + value = min(args[1], lengths[n]) + update!(progresses[p], value, args[2:end]...; kw...) + else + update!(progresses[p], args...; kw...) + end + + end - update!(main_progress, - main_progress.counter - prev_p_value + progresses[p].counter) - end + update!(main_progress, + main_progress.counter - prev_p_value + progresses[p].counter) + end - if progresses[p].counter >= lengths[p] - delete!(taken_offsets, progresses[p].offset) + if progresses[p].counter >= lengths[p] + delete!(taken_offsets, progresses[p].offset) + end end - - main_progress.counter >= total_length && break end while isready(mp.channel) take!(mp.channel) diff --git a/test/runtests.jl b/test/runtests.jl index b52c2b4..37059c7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,13 +1,14 @@ import ProgressMeter using Test -include("core.jl") -include("test.jl") -include("test_showvalues.jl") -include("test_map.jl") -include("test_float.jl") -include("test_threads.jl") +# include("core.jl") +# include("test.jl") +# include("test_showvalues.jl") +# include("test_map.jl") +# include("test_float.jl") +# include("test_threads.jl") include("test_parallel.jl") +include("test_parallel_update.jl") println("") println("All tests complete") diff --git a/test/test_parallel.jl b/test/test_parallel.jl index ce2df05..42840cb 100644 --- a/test/test_parallel.jl +++ b/test/test_parallel.jl @@ -1,11 +1,12 @@ using Distributed +if workers() != [1] + rmprocs(workers()) +end +addprocs(4) +@everywhere import ProgressMeter + @testset "ParallelProgress() tests" begin - if workers() != [1] - rmworkers(workers()) - end - addprocs(4) - @everywhere import ProgressMeter procs = nworkers() (procs == 1) && @info "incomplete tests: nworkers() == 1" From eea66f2b82814618684910bdfd4ee6a62488e534 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Thu, 12 Aug 2021 11:08:44 +0200 Subject: [PATCH 06/39] some tidying --- src/parallel_progress.jl | 15 +++-- test/runtests.jl | 2 +- test/test_parallel.jl | 115 ++++++++++++++++++++------------------- 3 files changed, 68 insertions(+), 64 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index e2d0c2a..4e10c75 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -17,7 +17,7 @@ julia> pmap(1:10) do end ``` """ -mutable struct ParallelProgress +mutable struct ParallelProgress <: AbstractProgress channel end @@ -32,8 +32,11 @@ finish!(pp::ParallelProgress, args...; kw...) = put!(pp.channel, (PP_FINISH, arg update!(pp::ParallelProgress, args...; kw...) = put!(pp.channel, (PP_UPDATE, args, kw)) function ParallelProgress(n::Integer; kw...) - channel = RemoteChannel(() -> Channel{Tuple}(n)) - progress = Progress(n; kw...) + return ParallelProgress(Progress(n; kw...); kw...) +end + +function ParallelProgress(progress::Progress) + channel = RemoteChannel(() -> Channel{Tuple}()) pp = ParallelProgress(channel) @async begin @@ -80,7 +83,7 @@ mutable struct MultipleProgress lengths::Vector{Int} end -Base.getindex(mp::MultipleProgress, n) = ParallelProgress.(MultipleChannel.(fill(mp.channel), n)) +Base.getindex(mp::MultipleProgress, n) = ParallelProgress.(MultipleChannel.(Ref(mp.channel), n)) Base.lastindex(mp::MultipleProgress) = mp.amount finish!(mp::MultipleProgress, args...; kw...) = finish!.(mp[1:end]) cancel(mp::MultipleProgress, args...; kw...) = cancel.(mp[1:end]) @@ -131,8 +134,8 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; total_length = sum(lengths) main_progress = Progress(total_length; offset=0, kw...) progresses = Union{Progress,Nothing}[nothing for _ in 1:amount] - taken_offsets = Set(Int[]) - channel = RemoteChannel(() -> Channel{Tuple}(max(2amount, 64))) + taken_offsets = Set{Int}() + channel = RemoteChannel(() -> Channel{Tuple}()) max_offsets = 1 diff --git a/test/runtests.jl b/test/runtests.jl index 37059c7..4d9455f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -8,7 +8,7 @@ using Test # include("test_float.jl") # include("test_threads.jl") include("test_parallel.jl") -include("test_parallel_update.jl") +#include("test_parallel_update.jl") println("") println("All tests complete") diff --git a/test/test_parallel.jl b/test/test_parallel.jl index 42840cb..7281b45 100644 --- a/test/test_parallel.jl +++ b/test/test_parallel.jl @@ -1,10 +1,11 @@ using Distributed +using ProgressMeter: FakeChannel if workers() != [1] rmprocs(workers()) end addprocs(4) -@everywhere import ProgressMeter +@everywhere using ProgressMeter @testset "ParallelProgress() tests" begin @@ -13,78 +14,78 @@ addprocs(4) @test all([@fetchfrom w @isdefined(ProgressMeter) for w in workers()]) println("Testing simultaneous updates...") - p = ProgressMeter.ParallelProgress(100) + p = ParallelProgress(100) @sync for _ in 1:10 @async for _ in 1:10 sleep(0.1) - ProgressMeter.next!(p) + next!(p) end end - sleep(0.01) + sleep(0.1) println("Testing over-shooting...") - p = ProgressMeter.ParallelProgress(10) + p = ParallelProgress(10) for _ in 1:100 sleep(0.01) - ProgressMeter.next!(p) + next!(p) end - sleep(0.01) - @test p.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + sleep(0.1) + @test p.channel isa FakeChannel #ParallelProgress finished println("Testing under-shooting...") - p = ProgressMeter.ParallelProgress(200) + p = ParallelProgress(200) for _ in 1:100 sleep(0.01) - ProgressMeter.next!(p) + next!(p) end - ProgressMeter.finish!(p) - sleep(0.01) - @test p.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + finish!(p) + sleep(0.1) + @test p.channel isa FakeChannel #ParallelProgress finished println("Testing rapid over-shooting...") - p = ProgressMeter.ParallelProgress(100) - ProgressMeter.next!(p) + p = ParallelProgress(100) + next!(p) sleep(0.1) for _ in 1:10000 - ProgressMeter.next!(p) + next!(p) end - sleep(0.01) - @test p.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + sleep(0.1) + @test p.channel isa FakeChannel #ParallelProgress finished println("Testing early cancel...") - p = ProgressMeter.ParallelProgress(100) + p = ParallelProgress(100) for _ in 1:50 sleep(0.02) - ProgressMeter.next!(p) + next!(p) end - ProgressMeter.cancel(p) - sleep(0.01) - @test p.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + cancel(p) + sleep(0.1) + @test p.channel isa FakeChannel #ParallelProgress finished println("Testing across $procs workers with @distributed...") n = 20 #per core - p = ProgressMeter.ParallelProgress(n*procs) + p = ParallelProgress(n*procs) @sync @distributed for _ in 1:n*procs sleep(0.05) - ProgressMeter.next!(p) + next!(p) end - sleep(0.01) - @test p.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + sleep(0.1) + @test p.channel isa FakeChannel #ParallelProgress finished println("Testing across $procs workers with pmap...") n = 20 - p = ProgressMeter.ParallelProgress(n*procs) + p = ParallelProgress(n*procs) ids = pmap(1:n*procs) do i sleep(0.05) - ProgressMeter.next!(p) + next!(p) return myid() end - sleep(0.01) - @test p.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + sleep(0.1) + @test p.channel isa FakeChannel #ParallelProgress finished @test length(unique(ids)) == procs end @@ -96,23 +97,23 @@ end function test_MP_finished(lengths, finish; kwargs...) n = length(lengths) - p = ProgressMeter.MultipleProgress(lengths; kwargs...) + p = MultipleProgress(lengths; kwargs...) for _ in 1:100 sleep(0.01) for i in 1:n - ProgressMeter.next!(p[i]) + next!(p[i]) end end - finish && ProgressMeter.finish!(p) - sleep(0.01) + finish && finish!(p) + sleep(0.1) for i in 1:n - @test p[i].channel.channel isa ProgressMeter.FakeChannel + @test p[i].channel.channel isa FakeChannel end end println("Testing custom titles and color...") test_MP_finished([100, 100], false; - desc="default ", color=:yellow, + desc="yellow ", color=:yellow, kws=[(desc=" task A ",), (desc=" task B ",)]) println("Testing over-shooting and under-shooting...") @@ -123,51 +124,51 @@ end println("Testing rapid over-shooting...") - p = ProgressMeter.MultipleProgress([100]; dt=0.01, count_overshoot=true) - ProgressMeter.next!(p[1]) + p = MultipleProgress([100]; dt=0.01, count_overshoot=true) + next!(p[1]) sleep(0.1) for _ in 1:10000 - ProgressMeter.next!(p[1]) + next!(p[1]) end - sleep(0.01) - @test p[1].channel.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + sleep(0.1) + @test p[1].channel.channel isa FakeChannel #ParallelProgress finished println("Testing early cancel...") - p = ProgressMeter.MultipleProgress([100, 80]) + p = MultipleProgress([100, 80]) for _ in 1:50 sleep(0.02) - ProgressMeter.next!(p[1]) - ProgressMeter.next!(p[2]) + next!(p[1]) + next!(p[2]) end - ProgressMeter.cancel(p[1]) - ProgressMeter.finish!(p[2]) + cancel(p[1]) + finish!(p[2]) sleep(0.1) - @test p[1].channel.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + @test p[1].channel.channel isa FakeChannel #ParallelProgress finished println("Testing early global cancel...") - p = ProgressMeter.MultipleProgress([100, 80]) + p = MultipleProgress([100, 80]) for _ in 1:50 sleep(0.02) - ProgressMeter.next!(p[1]) - ProgressMeter.next!(p[2]) + next!(p[1]) + next!(p[2]) end - ProgressMeter.cancel(p) + cancel(p) sleep(0.1) - @test p[2].channel.channel isa ProgressMeter.FakeChannel #ParallelProgress finished + @test p[2].channel.channel isa FakeChannel #ParallelProgress finished println("Testing bar remplacement with $procs workers and pmap...") lengths = rand(20:40, 2*procs) - p = ProgressMeter.MultipleProgress(lengths; dt=0.01, kws=[(desc=" task $i ",) for i in 1:2*procs]) + p = MultipleProgress(lengths; dt=0.01, kws=[(desc=" task $i ",) for i in 1:2*procs]) ids = pmap(1:2*procs) do ip for _ in 1:lengths[ip] sleep(0.05) - ProgressMeter.next!(p[ip]) + next!(p[ip]) end myid() end - sleep(0.01) + sleep(0.1) @test length(unique(ids)) == procs - @test p[1].channel.channel isa ProgressMeter.FakeChannel # finished + @test p[1].channel.channel isa FakeChannel # finished end From 5f2b17cb9b3cb80edb596a03d6190728fdc97e8e Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Thu, 12 Aug 2021 16:00:46 +0200 Subject: [PATCH 07/39] add parallel support for ProgressTresh and ProgressUnknown --- src/parallel_progress.jl | 166 +++++++++++++++++++++------------------ test/runtests.jl | 34 ++++---- test/test_parallel.jl | 138 ++++++++++++++++---------------- 3 files changed, 177 insertions(+), 161 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 4e10c75..05baf6d 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -26,21 +26,21 @@ const PP_CANCEL = :cancel const PP_FINISH = :finish const PP_UPDATE = :update -next!(pp::ParallelProgress, args...; kw...) = put!(pp.channel, (PP_NEXT, args, kw)) -cancel(pp::ParallelProgress, args...; kw...) = put!(pp.channel, (PP_CANCEL, args, kw)) -finish!(pp::ParallelProgress, args...; kw...) = put!(pp.channel, (PP_FINISH, args, kw)) -update!(pp::ParallelProgress, args...; kw...) = put!(pp.channel, (PP_UPDATE, args, kw)) +next!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_NEXT, args, kw)); nothing) +cancel(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_CANCEL, args, kw)); nothing) +finish!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_FINISH, args, kw)); nothing) +update!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_UPDATE, args, kw)); nothing) function ParallelProgress(n::Integer; kw...) return ParallelProgress(Progress(n; kw...); kw...) end -function ParallelProgress(progress::Progress) - channel = RemoteChannel(() -> Channel{Tuple}()) +function ParallelProgress(progress::AbstractProgress) + channel = RemoteChannel(() -> Channel{NTuple{3,Any}}(1024)) pp = ParallelProgress(channel) - @async begin - while progress.counter < progress.n + @async begin + while !has_finished(progress) f, args, kw = take!(channel) if f == PP_NEXT next!(progress, args...; kw...) @@ -53,19 +53,23 @@ function ParallelProgress(progress::Progress) elseif f == PP_UPDATE update!(progress, args...; kw...) else - error("not recognized: $f") + error("not recognized: $(repr(f))") end end - # empty channel before ending it - while isready(pp.channel) - take!(pp.channel) - end + pp.channel = FakeChannel() + while isready(channel) + take!(channel) + end + close!(channel) end - return pp end +has_finished(p::Progress) = p.counter >= p.n +has_finished(p::ProgressThresh) = p.triggered +has_finished(p::ProgressUnknown) = p.done + # fake channel to allow over-shoot struct FakeChannel end Distributed.put!(::FakeChannel, x) = nothing @@ -85,8 +89,6 @@ end Base.getindex(mp::MultipleProgress, n) = ParallelProgress.(MultipleChannel.(Ref(mp.channel), n)) Base.lastindex(mp::MultipleProgress) = mp.amount -finish!(mp::MultipleProgress, args...; kw...) = finish!.(mp[1:end]) -cancel(mp::MultipleProgress, args...; kw...) = cancel.(mp[1:end]) """ prog = MultipleProgress(amount, lengths; kw...) @@ -125,92 +127,106 @@ julia> p = MultipleProgress(5,10; desc="global ", kws=[(desc="task \$i ",) for i ``` """ function MultipleProgress(lengths::AbstractVector{<:Integer}; - count_overshoot::Bool = false, + count_overshoot = false, + mainprogress = true, kws = [() for _ in lengths], kw...) @assert length(lengths) == length(kws) "`length(lengths)` must be equal to `length(kws)`" amount = length(lengths) total_length = sum(lengths) - main_progress = Progress(total_length; offset=0, kw...) + mainprogress && (main_progress = Progress(total_length; offset=0, kw...)) progresses = Union{Progress,Nothing}[nothing for _ in 1:amount] taken_offsets = Set{Int}() - channel = RemoteChannel(() -> Channel{Tuple}()) + mainprogress && push!(taken_offsets, 0) + channel = RemoteChannel(() -> Channel{NTuple{4,Any}}()) - max_offsets = 1 + max_offsets = 0 mp = MultipleProgress(channel, amount, collect(lengths)) # we must make sure that 2 progresses aren't updated at the same time, # that's why we use only one Channel @async begin - while main_progress.counter < total_length - - p, f, args, kw = take!(channel) - - # main progressbar - if p == 0 - if f == PP_CANCEL - cancel(main_progress, args...; kw...) - break - elseif f == PP_UPDATE - update!(main_progress, args...; kw...) - elseif f == PP_NEXT - next!(main_progress, args...; kw...) - elseif f == PP_FINISH - finish!(main_progress, args...; kw...) - end - else + try + while main_progress.counter < total_length + + p, f, args, kw = take!(channel) + + # main progressbar + if p == 0 + mainprogress || continue + if f == PP_CANCEL + cancel(main_progress, args...; kw...) + break + elseif f == PP_UPDATE + if !isempty(args) && args[1] == (:) + update!(main_progress, main_progress.counter, args[2:end]...; kw...) + else + update!(main_progress, args...; kw...) + end + elseif f == PP_NEXT + next!(main_progress, args...; kw...) + elseif f == PP_FINISH + finish!(main_progress, args...; kw...) + end + else - # first time calling progress p - if isnothing(progresses[p]) - # find first available offset - offset = 1 - while offset in taken_offsets - offset += 1 + # first time calling progress p + if isnothing(progresses[p]) + # find first available offset + offset = 0 + while offset in taken_offsets + offset += 1 + end + max_offsets = max(max_offsets, offset) + progresses[p] = Progress(lengths[p]; offset=offset, kws[p]...) + push!(taken_offsets, offset) end - max_offsets = max(max_offsets, offset) - progresses[p] = Progress(lengths[p]; offset=offset, kw..., kws[p]...) - push!(taken_offsets, offset) - end - if f == PP_NEXT - if count_overshoot || progresses[p].counter < lengths[p] - next!(progresses[p], args...; kw...) - next!(main_progress) - end - else - prev_p_value = progresses[p].counter - - if f == PP_FINISH - finish!(progresses[p], args...; kw...) - elseif f == PP_CANCEL - finish!(progresses[p]) - cancel(progresses[p], args...; kw...) - elseif f == PP_UPDATE - if !count_overshoot && !isempty(args) - value = min(args[1], lengths[n]) - update!(progresses[p], value, args[2:end]...; kw...) - else - update!(progresses[p], args...; kw...) + if f == PP_NEXT + if count_overshoot || progresses[p].counter < lengths[p] + next!(progresses[p], args...; kw...) + mainprogress && next!(main_progress) end + else + prev_p_value = progresses[p].counter - end + if f == PP_FINISH + finish!(progresses[p], args...; kw...) + elseif f == PP_CANCEL + finish!(progresses[p]) + cancel(progresses[p], args...; kw...) + elseif f == PP_UPDATE + if !isempty(args) + value = args[1] + value == (:) && (value = progresses[p].counter) + !count_overshoot && (value = min(value, lengths[p])) + update!(progresses[p], value, args[2:end]...; kw...) + else + update!(progresses[p]; kw...) + end + end - update!(main_progress, - main_progress.counter - prev_p_value + progresses[p].counter) - end + mainprogress && update!(main_progress, + main_progress.counter - prev_p_value + progresses[p].counter) + end - if progresses[p].counter >= lengths[p] - delete!(taken_offsets, progresses[p].offset) + if progresses[p].counter >= lengths[p] + delete!(taken_offsets, progresses[p].offset) + end end end - end - while isready(mp.channel) - take!(mp.channel) + catch e + println("ERROR") + println(e) end mp.channel = FakeChannel() + while isready(channel) + take!(channel) + end + close(channel) print("\n" ^ max_offsets) end diff --git a/test/runtests.jl b/test/runtests.jl index da9b9f7..76b8840 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,23 +6,23 @@ if get(ENV, "CI", "false") == "true" display(versioninfo()) # among other things, this shows the number of threads end -# @testset "Core" begin -# include("core.jl") -# include("test.jl") -# end -# @testset "Show Values" begin -# include("test_showvalues.jl") -# end -# @testset "Mapping" begin -# include("test_map.jl") -# end -# @testset "Float" begin -# include("test_float.jl") -# end -# @testset "Threading" begin -# include("test_threads.jl") -# end +@testset "Core" begin + include("core.jl") + include("test.jl") +end +@testset "Show Values" begin + include("test_showvalues.jl") +end +@testset "Mapping" begin + include("test_map.jl") +end +@testset "Float" begin + include("test_float.jl") +end +@testset "Threading" begin + include("test_threads.jl") +end @testset "Parallel" begin include("test_parallel.jl") - #include("test_parallel_update.jl") + include("test_multiple.jl") end diff --git a/test/test_parallel.jl b/test/test_parallel.jl index 36a5ddb..8ce9336 100644 --- a/test/test_parallel.jl +++ b/test/test_parallel.jl @@ -1,16 +1,13 @@ using Distributed using ProgressMeter: FakeChannel -if workers() != [1] - rmprocs(workers()) -end -addprocs(4) +nworkers() == 1 && addprocs(4) @everywhere using ProgressMeter @testset "ParallelProgress() tests" begin procs = nworkers() - (procs == 1) && @info "incomplete tests: nworkers() == 1" + procs == 1 && @info "incomplete tests: nworkers() == 1" @test all([@fetchfrom w @isdefined(ProgressMeter) for w in workers()]) println("Testing simultaneous updates") @@ -22,6 +19,22 @@ addprocs(4) end end sleep(0.1) + @test p.channel isa FakeChannel #ParallelProgress finished + + println("Testing update!") + prog = Progress(100) + p = ParallelProgress(prog) + for _ in 1:25 + sleep(0.05) + next!(p) + end + update!(p, 75) + for _ in 76:100 + sleep(0.05) + next!(p) + end + sleep(0.1) + @test p.channel isa FakeChannel #ParallelProgress finished println("Testing over-shooting") p = ParallelProgress(10) @@ -83,84 +96,71 @@ addprocs(4) sleep(0.1) @test p.channel isa FakeChannel #ParallelProgress finished @test length(unique(ids)) == procs -end -@testset "MultipleProgress() tests" begin - - procs = nworkers() - @test all([@fetchfrom w @isdefined(ProgressMeter) for w in workers()]) - - function test_MP_finished(lengths, finish; kwargs...) - n = length(lengths) - p = MultipleProgress(lengths; kwargs...) - for _ in 1:100 - sleep(0.01) - for i in 1:n - next!(p[i]) - end - end - finish && finish!(p) - sleep(0.1) - for i in 1:n - @test p[i].channel.channel isa FakeChannel + println("Testing changing color with next! and update!") + p = ParallelProgress(100) + for i in 1:100 + sleep(0.05) + if i == 25 + next!(p, :red) + elseif i == 50 + update!(p, 51, :blue) + else + next!(p) end end - - println("Testing custom titles and color") - test_MP_finished([100, 100], false; - desc="yellow ", color=:yellow, - kws=[(desc=" task A ",), (desc=" task B ",)]) - - println("Testing over-shooting and under-shooting") - test_MP_finished([10, 110], true; dt=0.01) - - println("Testing over-shooting with count_overshoot") - test_MP_finished([10, 190], false; count_overshoot=true, dt=0.01) - - println("Testing rapid over-shooting") - p = MultipleProgress([100]; dt=0.01, count_overshoot=true) - next!(p[1]) sleep(0.1) - for _ in 1:10000 - next!(p[1]) + @test p.channel isa FakeChannel #ParallelProgress finished + + println("Testing changing desc with next! and update!") + p = ParallelProgress(100) + for i in 1:100 + sleep(0.05) + if i == 25 + next!(p, desc="25% done ") + elseif i == 50 + update!(p, 51, desc="50% done ") + else + next!(p) + end end sleep(0.1) - @test p[1].channel.channel isa FakeChannel #ParallelProgress finished + @test p.channel isa FakeChannel # ParallelProgress finished - println("Testing early cancel") - p = MultipleProgress([100, 80]) - for _ in 1:50 + println("Testing with showvalues") + p = ParallelProgress(100) + for i in 1:100 sleep(0.02) - next!(p[1]) - next!(p[2]) + if i < 50 + next!(p; showvalues=Dict(:i => i, "longstring" => "ABCD"^i)) + else + next!(p; showvalues=() -> [(:i, "$i"), ("halfdone", true)]) + end end - cancel(p[1]) - finish!(p[2]) sleep(0.1) - @test p[1].channel.channel isa FakeChannel #ParallelProgress finished + @test p.channel isa FakeChannel # ParallelProgress finished - println("Testing early global cancel") - p = MultipleProgress([100, 80]) - for _ in 1:50 + println("Testing with ProgressUnknown") + p = ParallelProgress(ProgressUnknown()) + for i in 1:100 sleep(0.02) - next!(p[1]) - next!(p[2]) + next!(p) end - cancel(p) + sleep(0.5) + update!(p, 200) + sleep(0.5) + @test !(p.channel isa FakeChannel) # ParallelProgress not finished + finish!(p) sleep(0.1) - @test p[2].channel.channel isa FakeChannel #ParallelProgress finished - - println("Testing bar remplacement with $procs workers and pmap") - lengths = rand(20:40, 2*procs) - p = MultipleProgress(lengths; dt=0.01, kws=[(desc=" task $i ",) for i in 1:2*procs]) - ids = pmap(1:2*procs) do ip - for _ in 1:lengths[ip] - sleep(0.05) - next!(p[ip]) - end - myid() + @test p.channel isa FakeChannel # ParallelProgress finished + + println("Testing with ProgressThresh") + p = ParallelProgress(ProgressThresh(10)) + for i in 100:-1:0 + sleep(0.02) + update!(p, i) end sleep(0.1) - @test length(unique(ids)) == procs - @test p[1].channel.channel isa FakeChannel # finished + @test p.channel isa FakeChannel # ParallelProgress finished + end From f522aec87e916eafd6cf33348de4ca35e8a10494 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Thu, 12 Aug 2021 16:43:20 +0200 Subject: [PATCH 08/39] add support for no main progressbar in MultipleProgress --- src/parallel_progress.jl | 153 +++++++++++++++++++++------------------ test/runtests.jl | 32 ++++---- test/test_parallel.jl | 20 ++--- 3 files changed, 109 insertions(+), 96 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 05baf6d..9b77690 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -1,8 +1,22 @@ + +mutable struct ParallelProgress <: AbstractProgress + channel +end + +const PP_NEXT = :next +const PP_CANCEL = :cancel +const PP_FINISH = :finish +const PP_UPDATE = :update + +next!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_NEXT, args, kw)); nothing) +cancel(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_CANCEL, args, kw)); nothing) +finish!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_FINISH, args, kw)); nothing) +update!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_UPDATE, args, kw)); nothing) + """ -`prog = ParallelProgress(n; kw...)` +`ParallelProgress(n; kw...)` works like `Progress` but can be used from other workers -Extra arguments after `update` or `cancel` are ignored # Example: ```jldoctest @@ -17,51 +31,42 @@ julia> pmap(1:10) do end ``` """ -mutable struct ParallelProgress <: AbstractProgress - channel -end - -const PP_NEXT = :next -const PP_CANCEL = :cancel -const PP_FINISH = :finish -const PP_UPDATE = :update - -next!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_NEXT, args, kw)); nothing) -cancel(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_CANCEL, args, kw)); nothing) -finish!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_FINISH, args, kw)); nothing) -update!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_UPDATE, args, kw)); nothing) - function ParallelProgress(n::Integer; kw...) return ParallelProgress(Progress(n; kw...); kw...) end +""" +`ParallelProgress(p::AbstractProgress)` + +wrapper around any `Progress`, `ProgressThresh` and `ProgressUnknown` that can be used +from other workers + +""" function ParallelProgress(progress::AbstractProgress) channel = RemoteChannel(() -> Channel{NTuple{3,Any}}(1024)) pp = ParallelProgress(channel) @async begin - while !has_finished(progress) - f, args, kw = take!(channel) - if f == PP_NEXT - next!(progress, args...; kw...) - elseif f == PP_CANCEL - cancel(progress, args...; kw...) - break - elseif f == PP_FINISH - finish!(progress, args...; kw...) - break - elseif f == PP_UPDATE - update!(progress, args...; kw...) - else - error("not recognized: $(repr(f))") + try + while !has_finished(progress) + f, args, kw = take!(channel) + if f == PP_NEXT + next!(progress, args...; kw...) + elseif f == PP_CANCEL + cancel(progress, args...; kw...) + break + elseif f == PP_FINISH + finish!(progress, args...; kw...) + break + elseif f == PP_UPDATE + update!(progress, args...; kw...) + else + error("not recognized: $(repr(f))") + end end + finally + close(pp) end - - pp.channel = FakeChannel() - while isready(channel) - take!(channel) - end - close!(channel) end return pp end @@ -70,7 +75,11 @@ has_finished(p::Progress) = p.counter >= p.n has_finished(p::ProgressThresh) = p.triggered has_finished(p::ProgressUnknown) = p.done -# fake channel to allow over-shoot +""" + FakeChannel() + +fake RemoteChannel that doesn't put anything anywhere (for allowing overshoot) +""" struct FakeChannel end Distributed.put!(::FakeChannel, x) = nothing @@ -80,8 +89,7 @@ struct MultipleChannel end Distributed.put!(mc::MultipleChannel, x) = put!(mc.channel, (mc.id, x...)) - -mutable struct MultipleProgress +mutable struct MultipleProgress <: AbstractProgress channel amount::Int lengths::Vector{Int} @@ -91,31 +99,34 @@ Base.getindex(mp::MultipleProgress, n) = ParallelProgress.(MultipleChannel.(Ref( Base.lastindex(mp::MultipleProgress) = mp.amount """ - prog = MultipleProgress(amount, lengths; kw...) + MultipleProgress(amount, lengths; kw...) equivalent to - MultipleProgress(lengths*ones(T,amount); kw...) + MultipleProgress(fill(lengths, amount); kw...) """ -function MultipleProgress(amount::Integer, lengths::T; kw...) where T <: Integer - MultipleProgress(lengths*ones(T,amount); kw...) +function MultipleProgress(amount::Integer, lengths::Integer; kw...) + MultipleProgress(fill(lengths, amount); kw...) end - """ - prog = MultipleProgress(lengths; kws, kw...) + MultipleProgress(lengths; mainprogress=true, count_overshoot=false, kws, kw...) -generates one progressbar for each value in `lengths` and one for a global progressbar +generates one progressbar for each value in `lengths` - `kw` arguments are applied on all progressbars - `kws[i]` arguments are applied on the i-th progressbar + - `mainprogress` adds a main progressmeter that sums the other ones + - `count_overshoot`: overshooting progressmeters will be counted in the main progressmeter + +use p[i] to access the i-th progressmeter, and p[0] to access the main one # Example ```jldoctest julia> using Distributed julia> addprocs(2) julia> @everywhere using ProgressMeter -julia> p = MultipleProgress(5,10; desc="global ", kws=[(desc="task \$i ",) for i in 1:5]) +julia> p = MultipleProgress(5, 10; desc="global ", kws=[(desc="task \$i ",) for i in 1:5]) pmap(1:5) do x for i in 1:10 sleep(rand()) @@ -151,24 +162,24 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; try while main_progress.counter < total_length - p, f, args, kw = take!(channel) + p, f, args, kwt = take!(channel) # main progressbar if p == 0 - mainprogress || continue if f == PP_CANCEL - cancel(main_progress, args...; kw...) + mainprogress || cancel(main_progress, args...; kwt...) break - elseif f == PP_UPDATE + elseif mainprogress && f == PP_UPDATE if !isempty(args) && args[1] == (:) - update!(main_progress, main_progress.counter, args[2:end]...; kw...) + update!(main_progress, main_progress.counter, args[2:end]...; kwt...) else - update!(main_progress, args...; kw...) + update!(main_progress, args...; kwt...) end - elseif f == PP_NEXT - next!(main_progress, args...; kw...) + elseif mainprogress && f == PP_NEXT + next!(main_progress, args...; kwt...) elseif f == PP_FINISH - finish!(main_progress, args...; kw...) + mainprogress || finish!(main_progress, args...; kwt...) + break end else @@ -180,32 +191,32 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; offset += 1 end max_offsets = max(max_offsets, offset) - progresses[p] = Progress(lengths[p]; offset=offset, kws[p]...) + progresses[p] = Progress(lengths[p]; offset=offset, kw..., kws[p]...) push!(taken_offsets, offset) end if f == PP_NEXT if count_overshoot || progresses[p].counter < lengths[p] - next!(progresses[p], args...; kw...) + next!(progresses[p], args...; kwt...) mainprogress && next!(main_progress) end else prev_p_value = progresses[p].counter if f == PP_FINISH - finish!(progresses[p], args...; kw...) + finish!(progresses[p], args...; kwt...) elseif f == PP_CANCEL finish!(progresses[p]) - cancel(progresses[p], args...; kw...) + cancel(progresses[p], args...; kwt...) elseif f == PP_UPDATE if !isempty(args) value = args[1] value == (:) && (value = progresses[p].counter) !count_overshoot && (value = min(value, lengths[p])) - update!(progresses[p], value, args[2:end]...; kw...) + update!(progresses[p], value, args[2:end]...; kwt...) else - update!(progresses[p]; kw...) + update!(progresses[p]; kwt...) end end @@ -218,18 +229,20 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; end end end - catch e - println("ERROR") - println(e) + finally + close(mp) end - mp.channel = FakeChannel() - while isready(channel) - take!(channel) - end - close(channel) print("\n" ^ max_offsets) end return mp end +function Base.close(p::Union{ParallelProgress,MultipleProgress}) + channel = p.channel + p.channel = FakeChannel() + while isready(channel) + take!(channel) + end + close(channel) +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 76b8840..0f39356 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,22 +6,22 @@ if get(ENV, "CI", "false") == "true" display(versioninfo()) # among other things, this shows the number of threads end -@testset "Core" begin - include("core.jl") - include("test.jl") -end -@testset "Show Values" begin - include("test_showvalues.jl") -end -@testset "Mapping" begin - include("test_map.jl") -end -@testset "Float" begin - include("test_float.jl") -end -@testset "Threading" begin - include("test_threads.jl") -end +# @testset "Core" begin +# include("core.jl") +# include("test.jl") +# end +# @testset "Show Values" begin +# include("test_showvalues.jl") +# end +# @testset "Mapping" begin +# include("test_map.jl") +# end +# @testset "Float" begin +# include("test_float.jl") +# end +# @testset "Threading" begin +# include("test_threads.jl") +# end @testset "Parallel" begin include("test_parallel.jl") include("test_multiple.jl") diff --git a/test/test_parallel.jl b/test/test_parallel.jl index 8ce9336..d3642ca 100644 --- a/test/test_parallel.jl +++ b/test/test_parallel.jl @@ -10,6 +10,7 @@ nworkers() == 1 && addprocs(4) procs == 1 && @info "incomplete tests: nworkers() == 1" @test all([@fetchfrom w @isdefined(ProgressMeter) for w in workers()]) + println("Testing ParallelProgress") println("Testing simultaneous updates") p = ParallelProgress(100) @sync for _ in 1:10 @@ -19,7 +20,7 @@ nworkers() == 1 && addprocs(4) end end sleep(0.1) - @test p.channel isa FakeChannel #ParallelProgress finished + @test p.channel isa FakeChannel # ParallelProgress finished println("Testing update!") prog = Progress(100) @@ -34,7 +35,7 @@ nworkers() == 1 && addprocs(4) next!(p) end sleep(0.1) - @test p.channel isa FakeChannel #ParallelProgress finished + @test p.channel isa FakeChannel # ParallelProgress finished println("Testing over-shooting") p = ParallelProgress(10) @@ -43,7 +44,7 @@ nworkers() == 1 && addprocs(4) next!(p) end sleep(0.1) - @test p.channel isa FakeChannel #ParallelProgress finished + @test p.channel isa FakeChannel # ParallelProgress finished println("Testing under-shooting") p = ParallelProgress(200) @@ -53,7 +54,7 @@ nworkers() == 1 && addprocs(4) end finish!(p) sleep(0.1) - @test p.channel isa FakeChannel #ParallelProgress finished + @test p.channel isa FakeChannel # ParallelProgress finished println("Testing rapid over-shooting") p = ParallelProgress(100) @@ -63,7 +64,7 @@ nworkers() == 1 && addprocs(4) next!(p) end sleep(0.1) - @test p.channel isa FakeChannel #ParallelProgress finished + @test p.channel isa FakeChannel # ParallelProgress finished println("Testing early cancel") p = ParallelProgress(100) @@ -73,7 +74,7 @@ nworkers() == 1 && addprocs(4) end cancel(p) sleep(0.1) - @test p.channel isa FakeChannel #ParallelProgress finished + @test p.channel isa FakeChannel # ParallelProgress finished println("Testing across $procs workers with @distributed") n = 20 #per core @@ -83,7 +84,7 @@ nworkers() == 1 && addprocs(4) next!(p) end sleep(0.1) - @test p.channel isa FakeChannel #ParallelProgress finished + @test p.channel isa FakeChannel # ParallelProgress finished println("Testing across $procs workers with pmap") n = 20 @@ -94,7 +95,7 @@ nworkers() == 1 && addprocs(4) return myid() end sleep(0.1) - @test p.channel isa FakeChannel #ParallelProgress finished + @test p.channel isa FakeChannel # ParallelProgress finished @test length(unique(ids)) == procs println("Testing changing color with next! and update!") @@ -110,7 +111,7 @@ nworkers() == 1 && addprocs(4) end end sleep(0.1) - @test p.channel isa FakeChannel #ParallelProgress finished + @test p.channel isa FakeChannel # ParallelProgress finished println("Testing changing desc with next! and update!") p = ParallelProgress(100) @@ -162,5 +163,4 @@ nworkers() == 1 && addprocs(4) end sleep(0.1) @test p.channel isa FakeChannel # ParallelProgress finished - end From f48fb98d60ccf024b6020343ee5198d4d4d53fb1 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Thu, 12 Aug 2021 16:56:46 +0200 Subject: [PATCH 09/39] fixes --- src/parallel_progress.jl | 14 +++++++------- test/runtests.jl | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 9b77690..b6b725c 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -146,7 +146,7 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; amount = length(lengths) total_length = sum(lengths) - mainprogress && (main_progress = Progress(total_length; offset=0, kw...)) + main_progress = Progress(total_length; offset=0, enabled=mainprogress, kw...) progresses = Union{Progress,Nothing}[nothing for _ in 1:amount] taken_offsets = Set{Int}() mainprogress && push!(taken_offsets, 0) @@ -169,16 +169,16 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; if f == PP_CANCEL mainprogress || cancel(main_progress, args...; kwt...) break - elseif mainprogress && f == PP_UPDATE + elseif f == PP_UPDATE if !isempty(args) && args[1] == (:) update!(main_progress, main_progress.counter, args[2:end]...; kwt...) else update!(main_progress, args...; kwt...) end - elseif mainprogress && f == PP_NEXT + elseif f == PP_NEXT next!(main_progress, args...; kwt...) elseif f == PP_FINISH - mainprogress || finish!(main_progress, args...; kwt...) + finish!(main_progress, args...; kwt...) break end else @@ -199,7 +199,7 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; if f == PP_NEXT if count_overshoot || progresses[p].counter < lengths[p] next!(progresses[p], args...; kwt...) - mainprogress && next!(main_progress) + next!(main_progress) end else prev_p_value = progresses[p].counter @@ -220,7 +220,7 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; end end - mainprogress && update!(main_progress, + update!(main_progress, main_progress.counter - prev_p_value + progresses[p].counter) end @@ -231,8 +231,8 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; end finally close(mp) + print("\n" ^ max_offsets) end - print("\n" ^ max_offsets) end return mp diff --git a/test/runtests.jl b/test/runtests.jl index 0f39356..a95cb84 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -23,6 +23,6 @@ end # include("test_threads.jl") # end @testset "Parallel" begin - include("test_parallel.jl") + #include("test_parallel.jl") include("test_multiple.jl") end From 5838982a02679e99067ec39fae3564ab9c53b949 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Thu, 12 Aug 2021 16:59:34 +0200 Subject: [PATCH 10/39] add tests for MultipleProgress --- test/test_multiple.jl | 187 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 test/test_multiple.jl diff --git a/test/test_multiple.jl b/test/test_multiple.jl new file mode 100644 index 0000000..2154ef4 --- /dev/null +++ b/test/test_multiple.jl @@ -0,0 +1,187 @@ +using Distributed +using ProgressMeter: FakeChannel + +nworkers() == 1 && addprocs(4) +@everywhere using ProgressMeter + +@testset "MultipleProgress() tests" begin + + procs = nworkers() + procs == 1 && @info "incomplete tests: nworkers() == 1" + @test all([@fetchfrom w @isdefined(ProgressMeter) for w in workers()]) + + println("Testing MultipleProgress") + println("Testing update!") + p = MultipleProgress([100]) + for _ in 1:25 + sleep(0.05) + next!(p[1]) + end + update!(p[1], 75) + for _ in 76:99 + sleep(0.05) + next!(p[1]) + end + sleep(0.1) + @test !(p.channel isa FakeChannel) # ParallelProgress not finished yet + next!(p[1]) + sleep(0.1) + @test p.channel isa FakeChannel # ParallelProgress finished + + function test_MP_finished(lengths, finish; kwargs...) + n = length(lengths) + p = MultipleProgress(lengths; kwargs...) + for _ in 1:100 + sleep(0.01) + for i in 1:n + next!(p[i]) + end + end + finish && finish!(p[0]) + sleep(0.1) + for i in 1:n + @test p[i].channel.channel isa FakeChannel + end + end + + println("Testing MultipleProgress with custom titles and color") + test_MP_finished([100, 100], false; + desc="yellow ", color=:yellow, + kws=[(desc="red " , color=:red ), + (desc="yellow too ", )]) + + println("Testing over-shooting and under-shooting") + test_MP_finished([10, 110], true; dt=0.01) + + println("Testing over-shooting with count_overshoot") + test_MP_finished([10, 190], false; count_overshoot=true, dt=0.01) + + println("Testing rapid over-shooting") + p = MultipleProgress([10]; dt=0.01, count_overshoot=true) + next!(p[1]) + sleep(0.1) + for i in 1:10000 + next!(p[1]) + end + sleep(0.1) + @test p[1].channel.channel isa FakeChannel # ParallelProgress finished + + println("Testing early cancel") + p = MultipleProgress([100, 80]) + for _ in 1:50 + sleep(0.02) + next!(p[1]) + next!(p[2]) + end + cancel(p[1]) + finish!(p[2]) + sleep(0.1) + @test p[1].channel.channel isa FakeChannel # ParallelProgress finished + + println("Testing early global cancel") + p = MultipleProgress([100, 80]) + for _ in 1:50 + sleep(0.02) + next!(p[1]) + next!(p[2]) + end + cancel(p[0]) + sleep(0.1) + @test p[2].channel.channel isa FakeChannel # ParallelProgress finished + + println("Testing bar remplacement with $procs workers and pmap") + lengths = rand(20:40, 2*procs) + p = MultipleProgress(lengths; dt=0.01, kws=[(desc=" task $i ",) for i in 1:2*procs]) + ids = pmap(1:2*procs) do ip + for _ in 1:lengths[ip] + sleep(0.05) + next!(p[ip]) + end + myid() + end + sleep(0.1) + @test length(unique(ids)) == procs + @test p[1].channel.channel isa FakeChannel # finished + + println("Testing changing color with next! and update!") + p = MultipleProgress([100,100]) + for i in 1:100 + sleep(0.01) + if i == 25 + next!(p[1], :red) + next!(p[2]) + elseif i == 50 + next!(p[1]) + update!(p[2], 51, :blue) + else + if i == 75 + update!(p[0], :, :yellow) + end + next!(p[1]) + next!(p[2]) + end + end + sleep(0.1) + @test p[1].channel.channel isa FakeChannel + + println("Testing changing desc with next! and update!") + p = MultipleProgress([100,100]) + for i in 1:100 + sleep(0.05) + if i == 25 + next!(p[1], desc="25% done ") + next!(p[2]) + elseif i == 50 + next!(p[1]) + update!(p[2], 51, desc="50% done ") + else + if i == 75 + update!(p[0], desc="75% done ") + end + next!(p[1]) + next!(p[2]) + end + end + sleep(0.1) + @test p[1].channel.channel isa FakeChannel # ParallelProgress finished + + println("Update and finish all") + p = MultipleProgress([100,100,100]) + for i in 1:100 + rand() < 0.5 && next!(p[1]) + rand() < 0.3 && next!(p[2]) + rand() < 0.1 && next!(p[3]) + sleep(0.02) + end + update!.(p[0:end], :, :red) + sleep(0.5) + finish!.(p[1:end]) + sleep(0.1) + @test p[1].channel.channel isa FakeChannel # ParallelProgress finished + + println("Testing without main progressmeter and closing without finish") + p = MultipleProgress([100,100,100], mainprogress=false) + update!(p[0], 10, :red, desc="I shouldn't exist ") + for i in 1:100 + rand() < 0.5 && next!(p[1], desc="task a ") + rand() < 0.3 && next!(p[2], desc="task b ") + rand() < 0.1 && next!(p[3], desc="task c ") + sleep(0.02) + end + sleep(0.1) + @test !(p.channel isa FakeChannel) # ParallelProgress not finished yet + close(p) + @test p.channel isa FakeChannel # ParallelProgress finished + print("\n"^3) + + + println("Testing with showvalues (doesn't really work)") + p = MultipleProgress([100,100]) + for i in 1:100 + sleep(0.02) + next!(p[1]; showvalues = Dict(:i=>i)) + next!(p[2]; showvalues = [i=>i, "longstring"=>"WXYZ"^i]) + end + sleep(0.1) + @test p.channel isa FakeChannel # ParallelProgress finished +end From 83123b104e288323237a467dc2d7a286cae86bb6 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Thu, 12 Aug 2021 17:38:43 +0200 Subject: [PATCH 11/39] restore tests --- test/runtests.jl | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index a95cb84..76b8840 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,23 +6,23 @@ if get(ENV, "CI", "false") == "true" display(versioninfo()) # among other things, this shows the number of threads end -# @testset "Core" begin -# include("core.jl") -# include("test.jl") -# end -# @testset "Show Values" begin -# include("test_showvalues.jl") -# end -# @testset "Mapping" begin -# include("test_map.jl") -# end -# @testset "Float" begin -# include("test_float.jl") -# end -# @testset "Threading" begin -# include("test_threads.jl") -# end +@testset "Core" begin + include("core.jl") + include("test.jl") +end +@testset "Show Values" begin + include("test_showvalues.jl") +end +@testset "Mapping" begin + include("test_map.jl") +end +@testset "Float" begin + include("test_float.jl") +end +@testset "Threading" begin + include("test_threads.jl") +end @testset "Parallel" begin - #include("test_parallel.jl") + include("test_parallel.jl") include("test_multiple.jl") end From fa22215686f0c264bceec930b23e241e04aaf5d4 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Thu, 12 Aug 2021 18:11:57 +0200 Subject: [PATCH 12/39] fixes for julia 0.7 --- src/parallel_progress.jl | 8 ++++++-- test/test_multiple.jl | 14 +++++++------- test/test_parallel.jl | 18 +++++++++--------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index b6b725c..4a93960 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -64,6 +64,10 @@ function ParallelProgress(progress::AbstractProgress) error("not recognized: $(repr(f))") end end + catch e + println("ERROR") + println(e) + println() finally close(pp) end @@ -150,7 +154,7 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; progresses = Union{Progress,Nothing}[nothing for _ in 1:amount] taken_offsets = Set{Int}() mainprogress && push!(taken_offsets, 0) - channel = RemoteChannel(() -> Channel{NTuple{4,Any}}()) + channel = RemoteChannel(() -> Channel{NTuple{4,Any}}(1024)) max_offsets = 0 @@ -184,7 +188,7 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; else # first time calling progress p - if isnothing(progresses[p]) + if progresses[p] === nothing # find first available offset offset = 0 while offset in taken_offsets diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 2154ef4..eb1ea49 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -6,8 +6,8 @@ nworkers() == 1 && addprocs(4) @testset "MultipleProgress() tests" begin - procs = nworkers() - procs == 1 && @info "incomplete tests: nworkers() == 1" + np = nworkers() + np == 1 && @info "incomplete tests: nworkers() == 1" @test all([@fetchfrom w @isdefined(ProgressMeter) for w in workers()]) println("Testing MultipleProgress") @@ -89,10 +89,10 @@ nworkers() == 1 && addprocs(4) sleep(0.1) @test p[2].channel.channel isa FakeChannel # ParallelProgress finished - println("Testing bar remplacement with $procs workers and pmap") - lengths = rand(20:40, 2*procs) - p = MultipleProgress(lengths; dt=0.01, kws=[(desc=" task $i ",) for i in 1:2*procs]) - ids = pmap(1:2*procs) do ip + println("Testing bar remplacement with $np workers and pmap") + lengths = rand(20:40, 2*np) + p = MultipleProgress(lengths; dt=0.01, kws=[(desc=" task $i ",) for i in 1:2*np]) + ids = pmap(1:2*np) do ip for _ in 1:lengths[ip] sleep(0.05) next!(p[ip]) @@ -100,7 +100,7 @@ nworkers() == 1 && addprocs(4) myid() end sleep(0.1) - @test length(unique(ids)) == procs + @test length(unique(ids)) == np @test p[1].channel.channel isa FakeChannel # finished println("Testing changing color with next! and update!") diff --git a/test/test_parallel.jl b/test/test_parallel.jl index d3642ca..5a2a116 100644 --- a/test/test_parallel.jl +++ b/test/test_parallel.jl @@ -6,8 +6,8 @@ nworkers() == 1 && addprocs(4) @testset "ParallelProgress() tests" begin - procs = nworkers() - procs == 1 && @info "incomplete tests: nworkers() == 1" + np = nworkers() + np == 1 && @info "incomplete tests: nworkers() == 1" @test all([@fetchfrom w @isdefined(ProgressMeter) for w in workers()]) println("Testing ParallelProgress") @@ -76,27 +76,27 @@ nworkers() == 1 && addprocs(4) sleep(0.1) @test p.channel isa FakeChannel # ParallelProgress finished - println("Testing across $procs workers with @distributed") + println("Testing across $np workers with @distributed") n = 20 #per core - p = ParallelProgress(n*procs) - @sync @distributed for _ in 1:n*procs + p = ParallelProgress(n*np) + @sync @distributed for _ in 1:n*np sleep(0.05) next!(p) end sleep(0.1) @test p.channel isa FakeChannel # ParallelProgress finished - println("Testing across $procs workers with pmap") + println("Testing across $np workers with pmap") n = 20 - p = ParallelProgress(n*procs) - ids = pmap(1:n*procs) do i + p = ParallelProgress(n*np) + ids = pmap(1:n*np) do i sleep(0.05) next!(p) return myid() end sleep(0.1) @test p.channel isa FakeChannel # ParallelProgress finished - @test length(unique(ids)) == procs + @test length(unique(ids)) == np println("Testing changing color with next! and update!") p = ParallelProgress(100) From 7c64cd20777e063c160cb99cf9f525539458a4d5 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Fri, 13 Aug 2021 12:43:48 +0200 Subject: [PATCH 13/39] better error handling and hopefully better codecov --- src/parallel_progress.jl | 82 +++++++++++++++---------- test/test_multiple.jl | 125 +++++++++++++++++++++++++++++---------- test/test_parallel.jl | 52 +++++++++++----- 3 files changed, 181 insertions(+), 78 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 4a93960..be4fc1c 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -32,7 +32,7 @@ julia> pmap(1:10) do ``` """ function ParallelProgress(n::Integer; kw...) - return ParallelProgress(Progress(n; kw...); kw...) + return ParallelProgress(Progress(n; kw...)) end """ @@ -48,7 +48,7 @@ function ParallelProgress(progress::AbstractProgress) @async begin try - while !has_finished(progress) + while !has_finished(progress) && !has_finished(pp) f, args, kw = take!(channel) if f == PP_NEXT next!(progress, args...; kw...) @@ -60,14 +60,16 @@ function ParallelProgress(progress::AbstractProgress) break elseif f == PP_UPDATE update!(progress, args...; kw...) - else - error("not recognized: $(repr(f))") end end - catch e - println("ERROR") - println(e) + catch err println() + # channel closed should only happen from Base.close(pp), which isn't an error + if err != Base.closed_exception() + bt = catch_backtrace() + showerror(stderr, err, bt) + println() + end finally close(pp) end @@ -75,9 +77,6 @@ function ParallelProgress(progress::AbstractProgress) return pp end -has_finished(p::Progress) = p.counter >= p.n -has_finished(p::ProgressThresh) = p.triggered -has_finished(p::ProgressUnknown) = p.done """ FakeChannel() @@ -85,7 +84,9 @@ has_finished(p::ProgressUnknown) = p.done fake RemoteChannel that doesn't put anything anywhere (for allowing overshoot) """ struct FakeChannel end -Distributed.put!(::FakeChannel, x) = nothing +Base.close(::FakeChannel) = nothing +Base.isready(::FakeChannel) = false +Distributed.put!(::FakeChannel, _...) = nothing struct MultipleChannel channel @@ -102,18 +103,6 @@ end Base.getindex(mp::MultipleProgress, n) = ParallelProgress.(MultipleChannel.(Ref(mp.channel), n)) Base.lastindex(mp::MultipleProgress) = mp.amount -""" - MultipleProgress(amount, lengths; kw...) - -equivalent to - - MultipleProgress(fill(lengths, amount); kw...) - -""" -function MultipleProgress(amount::Integer, lengths::Integer; kw...) - MultipleProgress(fill(lengths, amount); kw...) -end - """ MultipleProgress(lengths; mainprogress=true, count_overshoot=false, kws, kw...) @@ -130,7 +119,7 @@ use p[i] to access the i-th progressmeter, and p[0] to access the main one julia> using Distributed julia> addprocs(2) julia> @everywhere using ProgressMeter -julia> p = MultipleProgress(5, 10; desc="global ", kws=[(desc="task \$i ",) for i in 1:5]) +julia> p = MultipleProgress(fill(10,5); desc="global ", kws=[(desc="task \$i ",) for i in 1:5]) pmap(1:5) do x for i in 1:10 sleep(rand()) @@ -164,14 +153,15 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; # that's why we use only one Channel @async begin try - while main_progress.counter < total_length + while !has_finished(main_progress) p, f, args, kwt = take!(channel) # main progressbar if p == 0 if f == PP_CANCEL - mainprogress || cancel(main_progress, args...; kwt...) + main_progress.counter = main_progress.n + cancel(main_progress, args...; kwt...) break elseif f == PP_UPDATE if !isempty(args) && args[1] == (:) @@ -191,7 +181,7 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; if progresses[p] === nothing # find first available offset offset = 0 - while offset in taken_offsets + while offset ∈ taken_offsets offset += 1 end max_offsets = max(max_offsets, offset) @@ -211,8 +201,9 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; if f == PP_FINISH finish!(progresses[p], args...; kwt...) elseif f == PP_CANCEL - finish!(progresses[p]) + #finish!(progresses[p]) cancel(progresses[p], args...; kwt...) + progresses[p].counter = progresses[p].n elseif f == PP_UPDATE if !isempty(args) value = args[1] @@ -233,20 +224,49 @@ function MultipleProgress(lengths::AbstractVector{<:Integer}; end end end + catch err + # channel closed should only happen from Base.close(mp), which isn't an error + if err != Base.closed_exception() + bt = catch_backtrace() + println() + showerror(stderr, err, bt) + println() + end finally - close(mp) print("\n" ^ max_offsets) + # progress with offset 0 adds automatically a line break when finished (#215) + if !mainprogress || !has_finished(main_progress) + println() + end + close(mp) end end return mp end + + +""" + close(p::Union{ParallelProgress,MultipleProgress}) + +empties and closes the channel of the progress and replaces it with a `FakeChannel` to allow overshoot + +""" function Base.close(p::Union{ParallelProgress,MultipleProgress}) channel = p.channel p.channel = FakeChannel() - while isready(channel) + while isready(channel) # empty channel to avoid waiting `put!` take!(channel) end close(channel) -end \ No newline at end of file +end + +has_finished(p::Progress) = p.counter >= p.n +has_finished(p::ProgressThresh) = p.triggered +has_finished(p::ProgressUnknown) = p.done +has_finished(p::ParallelProgress) = isfakechannel(p.channel) +has_finished(p::MultipleProgress) = isfakechannel(p.channel) +isfakechannel(_) = false +isfakechannel(::FakeChannel) = true +isfakechannel(mc::MultipleChannel) = isfakechannel(mc.channel) diff --git a/test/test_multiple.jl b/test/test_multiple.jl index eb1ea49..80ddf8f 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -1,5 +1,5 @@ using Distributed -using ProgressMeter: FakeChannel +using ProgressMeter: has_finished nworkers() == 1 && addprocs(4) @everywhere using ProgressMeter @@ -23,10 +23,10 @@ nworkers() == 1 && addprocs(4) next!(p[1]) end sleep(0.1) - @test !(p.channel isa FakeChannel) # ParallelProgress not finished yet + @test !has_finished(p) next!(p[1]) sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) function test_MP_finished(lengths, finish; kwargs...) n = length(lengths) @@ -40,7 +40,7 @@ nworkers() == 1 && addprocs(4) finish && finish!(p[0]) sleep(0.1) for i in 1:n - @test p[i].channel.channel isa FakeChannel + @test has_finished(p[i]) end end @@ -64,7 +64,7 @@ nworkers() == 1 && addprocs(4) next!(p[1]) end sleep(0.1) - @test p[1].channel.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) println("Testing early cancel") p = MultipleProgress([100, 80]) @@ -76,9 +76,9 @@ nworkers() == 1 && addprocs(4) cancel(p[1]) finish!(p[2]) sleep(0.1) - @test p[1].channel.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) - println("Testing early global cancel") + println("Testing early cancel main progerss") p = MultipleProgress([100, 80]) for _ in 1:50 sleep(0.02) @@ -87,7 +87,31 @@ nworkers() == 1 && addprocs(4) end cancel(p[0]) sleep(0.1) - @test p[2].channel.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) + + + println("Testing early finish main progress") + p = MultipleProgress([100, 80]) + for _ in 1:50 + sleep(0.02) + next!(p[1]) + next!(p[2]) + end + finish!(p[0]) + sleep(0.1) + @test has_finished(p) + + println("Testing next! on main progress") + p = MultipleProgress([100]) + for _ in 1:99 + sleep(0.02) + next!(p[1]) + end + sleep(0.1) + @test !has_finished(p) + next!(p[0]) + sleep(0.1) + @test has_finished(p) println("Testing bar remplacement with $np workers and pmap") lengths = rand(20:40, 2*np) @@ -101,7 +125,7 @@ nworkers() == 1 && addprocs(4) end sleep(0.1) @test length(unique(ids)) == np - @test p[1].channel.channel isa FakeChannel # finished + @test has_finished(p) println("Testing changing color with next! and update!") p = MultipleProgress([100,100]) @@ -122,28 +146,29 @@ nworkers() == 1 && addprocs(4) end end sleep(0.1) - @test p[1].channel.channel isa FakeChannel + @test has_finished(p) println("Testing changing desc with next! and update!") - p = MultipleProgress([100,100]) + p = MultipleProgress([100,100,100]) for i in 1:100 sleep(0.05) - if i == 25 - next!(p[1], desc="25% done ") - next!(p[2]) - elseif i == 50 - next!(p[1]) - update!(p[2], 51, desc="50% done ") + if i == 20 + next!(p[1], desc="20% done ") + next!.(p[2:3]) + elseif i == 40 + next!.(p[[1,3]]) + update!(p[2], 41, desc="40% done ") else - if i == 75 - update!(p[0], desc="75% done ") + if i == 60 + update!(p[0], desc="60% done ") + elseif i == 80 + update!(p[3], desc="80% done ") end - next!(p[1]) - next!(p[2]) + next!.(p[1:3]) end end sleep(0.1) - @test p[1].channel.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) println("Update and finish all") p = MultipleProgress([100,100,100]) @@ -157,23 +182,59 @@ nworkers() == 1 && addprocs(4) sleep(0.5) finish!.(p[1:end]) sleep(0.1) - @test p[1].channel.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) - println("Testing without main progressmeter and closing without finish") + println("Testing without main progressmeter and offset 0 finishes last") p = MultipleProgress([100,100,100], mainprogress=false) update!(p[0], 10, :red, desc="I shouldn't exist ") - for i in 1:100 - rand() < 0.5 && next!(p[1], desc="task a ") - rand() < 0.3 && next!(p[2], desc="task b ") - rand() < 0.1 && next!(p[3], desc="task c ") + for i in 1:80 + next!(p[1], desc="task a ") + rand() < 0.7 && next!(p[2], desc="task b ") + rand() < 0.4 && next!(p[3], desc="task c ") + sleep(0.02) + end + sleep(0.1) + @test !has_finished(p) + finish!.(p[end:-1:1]) + sleep(0.1) + @test has_finished(p) + + println("Testing without main progressmeter and offset 0 finishes first (#215)") + p = MultipleProgress([100,100,100], mainprogress=false) + update!(p[0], 10, :red, desc="I shouldn't exist ") + for i in 1:80 + next!(p[1], desc="task a ") + rand() < 0.7 && next!(p[2], desc="task b ") + rand() < 0.4 && next!(p[3], desc="task c ") sleep(0.02) end sleep(0.1) - @test !(p.channel isa FakeChannel) # ParallelProgress not finished yet + @test !has_finished(p) + finish!.(p[1:end]) + sleep(0.1) + @test has_finished(p) + + println("Testing early close (should not display error)") + p = MultipleProgress([100], desc="Close test") + for i in 1:30 + sleep(0.01) + next!(p[1]) + end + sleep(0.1) + @test !has_finished(p) close(p) - @test p.channel isa FakeChannel # ParallelProgress finished - print("\n"^3) + sleep(0.1) + @test has_finished(p) + println("Testing errors in MultipleProgress (should display error)") + p = MultipleProgress([100], desc="Error test", color=:red) + for i in 1:30 + sleep(0.01) + next!(p[1]) + end + next!(p[1], 1) + sleep(2) + @test has_finished(p) println("Testing with showvalues (doesn't really work)") p = MultipleProgress([100,100]) @@ -183,5 +244,5 @@ nworkers() == 1 && addprocs(4) next!(p[2]; showvalues = [i=>i, "longstring"=>"WXYZ"^i]) end sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) end diff --git a/test/test_parallel.jl b/test/test_parallel.jl index 5a2a116..6a0a8d4 100644 --- a/test/test_parallel.jl +++ b/test/test_parallel.jl @@ -1,5 +1,5 @@ using Distributed -using ProgressMeter: FakeChannel +using ProgressMeter: has_finished nworkers() == 1 && addprocs(4) @everywhere using ProgressMeter @@ -20,7 +20,7 @@ nworkers() == 1 && addprocs(4) end end sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) println("Testing update!") prog = Progress(100) @@ -35,7 +35,7 @@ nworkers() == 1 && addprocs(4) next!(p) end sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) println("Testing over-shooting") p = ParallelProgress(10) @@ -44,7 +44,7 @@ nworkers() == 1 && addprocs(4) next!(p) end sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) println("Testing under-shooting") p = ParallelProgress(200) @@ -54,7 +54,7 @@ nworkers() == 1 && addprocs(4) end finish!(p) sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) println("Testing rapid over-shooting") p = ParallelProgress(100) @@ -64,7 +64,7 @@ nworkers() == 1 && addprocs(4) next!(p) end sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) println("Testing early cancel") p = ParallelProgress(100) @@ -74,7 +74,7 @@ nworkers() == 1 && addprocs(4) end cancel(p) sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) println("Testing across $np workers with @distributed") n = 20 #per core @@ -84,7 +84,7 @@ nworkers() == 1 && addprocs(4) next!(p) end sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) println("Testing across $np workers with pmap") n = 20 @@ -95,7 +95,7 @@ nworkers() == 1 && addprocs(4) return myid() end sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) @test length(unique(ids)) == np println("Testing changing color with next! and update!") @@ -111,7 +111,7 @@ nworkers() == 1 && addprocs(4) end end sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) println("Testing changing desc with next! and update!") p = ParallelProgress(100) @@ -126,7 +126,7 @@ nworkers() == 1 && addprocs(4) end end sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) println("Testing with showvalues") p = ParallelProgress(100) @@ -139,7 +139,7 @@ nworkers() == 1 && addprocs(4) end end sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) println("Testing with ProgressUnknown") p = ParallelProgress(ProgressUnknown()) @@ -150,10 +150,10 @@ nworkers() == 1 && addprocs(4) sleep(0.5) update!(p, 200) sleep(0.5) - @test !(p.channel isa FakeChannel) # ParallelProgress not finished + @test !has_finished(p) finish!(p) sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) println("Testing with ProgressThresh") p = ParallelProgress(ProgressThresh(10)) @@ -162,5 +162,27 @@ nworkers() == 1 && addprocs(4) update!(p, i) end sleep(0.1) - @test p.channel isa FakeChannel # ParallelProgress finished + @test has_finished(p) + + println("Testing early close (should not display error)") + p = ParallelProgress(100, desc="Close test") + for i in 1:30 + sleep(0.01) + next!(p) + end + sleep(0.1) + @test !has_finished(p) + close(p) + sleep(0.1) + @test has_finished(p) + + println("Testing errors in ParallelProgress (should display error)") + p = ParallelProgress(100, desc="Error test", color=:red) + for i in 1:30 + sleep(0.01) + next!(p) + end + next!(p, 1) + sleep(2) + @test has_finished(p) end From 03f8b1c185211f1ca3c658181c3255c6524b334d Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Fri, 13 Aug 2021 13:40:04 +0200 Subject: [PATCH 14/39] fix typo --- test/test_multiple.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 80ddf8f..6c65a37 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -78,7 +78,7 @@ nworkers() == 1 && addprocs(4) sleep(0.1) @test has_finished(p) - println("Testing early cancel main progerss") + println("Testing early cancel main progress") p = MultipleProgress([100, 80]) for _ in 1:50 sleep(0.02) From a2a08d9fed7a831b980394dc59f28d654082d0a2 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Fri, 13 Aug 2021 16:33:25 +0200 Subject: [PATCH 15/39] first working version of multiple progresses from a vector --- src/parallel_progress.jl | 218 +++++++++++++++++++++------------------ test/runtests.jl | 34 +++--- test/test_multiple.jl | 91 +++++++++------- 3 files changed, 188 insertions(+), 155 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index be4fc1c..ec7e6bb 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -94,10 +94,9 @@ struct MultipleChannel end Distributed.put!(mc::MultipleChannel, x) = put!(mc.channel, (mc.id, x...)) -mutable struct MultipleProgress <: AbstractProgress +mutable struct MultipleProgress channel amount::Int - lengths::Vector{Int} end Base.getindex(mp::MultipleProgress, n) = ParallelProgress.(MultipleChannel.(Ref(mp.channel), n)) @@ -130,119 +129,136 @@ julia> p = MultipleProgress(fill(10,5); desc="global ", kws=[(desc="task \$i ",) end ``` """ -function MultipleProgress(lengths::AbstractVector{<:Integer}; - count_overshoot = false, - mainprogress = true, - kws = [() for _ in lengths], - kw...) - @assert length(lengths) == length(kws) "`length(lengths)` must be equal to `length(kws)`" - amount = length(lengths) - - total_length = sum(lengths) - main_progress = Progress(total_length; offset=0, enabled=mainprogress, kw...) - progresses = Union{Progress,Nothing}[nothing for _ in 1:amount] - taken_offsets = Set{Int}() - mainprogress && push!(taken_offsets, 0) +function MultipleProgress(progresses::AbstractVector{Progress}; + count_finishes=false, kwmain=(), kw...) + main_length = count_finishes ? length(progresses) : sum(p->p.n, progresses) + mainprogress = Progress(main_length; kwmain...) + return MultipleProgress(progresses, mainprogress; count_finishes=count_finishes, kw...) +end + +function MultipleProgress( + progresses::AbstractVector{<:AbstractProgress}, + mainprogress::AbstractProgress; + count_finishes = false, + count_overshoot = false, + auto_reset_timer = true + ) channel = RemoteChannel(() -> Channel{NTuple{4,Any}}(1024)) + mp = MultipleProgress(channel, length(progresses)) + @async runMultipleProgress(progresses, mainprogress, mp; + count_finishes=count_finishes, + count_overshoot=count_overshoot, + auto_reset_timer=auto_reset_timer) + return mp +end - max_offsets = 0 +function runMultipleProgress( + progresses::AbstractVector{<:AbstractProgress}, + mainprogress::AbstractProgress, + mp::MultipleProgress; + count_finishes = false, + count_overshoot = false, + auto_reset_timer = true + ) + for p in progresses + p.offset = -1 + end - mp = MultipleProgress(channel, amount, collect(lengths)) + channel = mp.channel + taken_offsets = Set{Int}() + max_offsets = 1 + try + # we must make sure that 2 progresses aren't updated at the same time, + # that's why we use only one Channel + while !has_finished(mainprogress) && !all(has_finished, progresses) + + p, f, args, kwt = take!(channel) + + # main progressbar + if p == 0 + if f == PP_CANCEL + mainprogress.counter = mainprogress.n + cancel(mainprogress, args...; kwt...) + break + elseif f == PP_UPDATE + if !isempty(args) && args[1] == (:) + update!(mainprogress, mainprogress.counter, args[2:end]...; kwt...) + else + update!(mainprogress, args...; kwt...) + end + elseif f == PP_NEXT + next!(mainprogress, args...; kwt...) + elseif f == PP_FINISH + finish!(mainprogress, args...; kwt...) + break + end + else + + # first time calling progress p + if progresses[p].offset == -1 + # find first available offset + offset = 1 + while offset ∈ taken_offsets + offset += 1 + end + max_offsets = max(max_offsets, offset) + progresses[p].offset = offset + if auto_reset_timer + progresses[p].tinit = progresses[p].tsecond = progresses[p].tlast = time() + end + push!(taken_offsets, offset) + end - # we must make sure that 2 progresses aren't updated at the same time, - # that's why we use only one Channel - @async begin - try - while !has_finished(main_progress) - - p, f, args, kwt = take!(channel) - - # main progressbar - if p == 0 - if f == PP_CANCEL - main_progress.counter = main_progress.n - cancel(main_progress, args...; kwt...) - break - elseif f == PP_UPDATE - if !isempty(args) && args[1] == (:) - update!(main_progress, main_progress.counter, args[2:end]...; kwt...) - else - update!(main_progress, args...; kwt...) - end - elseif f == PP_NEXT - next!(main_progress, args...; kwt...) - elseif f == PP_FINISH - finish!(main_progress, args...; kwt...) - break + if f == PP_NEXT + if count_overshoot || !has_finished(progresses[p]) + next!(progresses[p], args...; kwt...) + next!(mainprogress) end else - - # first time calling progress p - if progresses[p] === nothing - # find first available offset - offset = 0 - while offset ∈ taken_offsets - offset += 1 + prev_p_value = progresses[p].counter + + if f == PP_FINISH + finish!(progresses[p], args...; kwt...) + elseif f == PP_CANCEL + #finish!(progresses[p]) + cancel(progresses[p], args...; kwt...) + progresses[p].counter = progresses[p].n + elseif f == PP_UPDATE + if !isempty(args) + value = args[1] + value == (:) && (value = progresses[p].counter) + !count_overshoot && (value = min(value, progresses[p].n)) + update!(progresses[p], value, args[2:end]...; kwt...) + else + update!(progresses[p]; kwt...) end - max_offsets = max(max_offsets, offset) - progresses[p] = Progress(lengths[p]; offset=offset, kw..., kws[p]...) - push!(taken_offsets, offset) end + update!(mainprogress, + mainprogress.counter - prev_p_value + progresses[p].counter) + end - if f == PP_NEXT - if count_overshoot || progresses[p].counter < lengths[p] - next!(progresses[p], args...; kwt...) - next!(main_progress) - end - else - prev_p_value = progresses[p].counter - - if f == PP_FINISH - finish!(progresses[p], args...; kwt...) - elseif f == PP_CANCEL - #finish!(progresses[p]) - cancel(progresses[p], args...; kwt...) - progresses[p].counter = progresses[p].n - elseif f == PP_UPDATE - if !isempty(args) - value = args[1] - value == (:) && (value = progresses[p].counter) - !count_overshoot && (value = min(value, lengths[p])) - update!(progresses[p], value, args[2:end]...; kwt...) - else - update!(progresses[p]; kwt...) - end - end - - update!(main_progress, - main_progress.counter - prev_p_value + progresses[p].counter) - end - - if progresses[p].counter >= lengths[p] - delete!(taken_offsets, progresses[p].offset) - end + if has_finished(progresses[p]) + delete!(taken_offsets, progresses[p].offset) end end - catch err - # channel closed should only happen from Base.close(mp), which isn't an error - if err != Base.closed_exception() - bt = catch_backtrace() - println() - showerror(stderr, err, bt) - println() - end - finally - print("\n" ^ max_offsets) - # progress with offset 0 adds automatically a line break when finished (#215) - if !mainprogress || !has_finished(main_progress) - println() - end - close(mp) end + catch err + # channel closed should only happen from Base.close(mp), which isn't an error + if err != Base.closed_exception() + bt = catch_backtrace() + println() + showerror(stderr, err, bt) + println() + end + finally + print("\n" ^ max_offsets) + # progress with offset 0 adds automatically a line break when finished (#215) + if !has_finished(mainprogress) || mainprogress.enabled == false + println() + end + close(mp) end - - return mp end diff --git a/test/runtests.jl b/test/runtests.jl index 76b8840..a95cb84 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,23 +6,23 @@ if get(ENV, "CI", "false") == "true" display(versioninfo()) # among other things, this shows the number of threads end -@testset "Core" begin - include("core.jl") - include("test.jl") -end -@testset "Show Values" begin - include("test_showvalues.jl") -end -@testset "Mapping" begin - include("test_map.jl") -end -@testset "Float" begin - include("test_float.jl") -end -@testset "Threading" begin - include("test_threads.jl") -end +# @testset "Core" begin +# include("core.jl") +# include("test.jl") +# end +# @testset "Show Values" begin +# include("test_showvalues.jl") +# end +# @testset "Mapping" begin +# include("test_map.jl") +# end +# @testset "Float" begin +# include("test_float.jl") +# end +# @testset "Threading" begin +# include("test_threads.jl") +# end @testset "Parallel" begin - include("test_parallel.jl") + #include("test_parallel.jl") include("test_multiple.jl") end diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 6c65a37..879ab0f 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -12,7 +12,7 @@ nworkers() == 1 && addprocs(4) println("Testing MultipleProgress") println("Testing update!") - p = MultipleProgress([100]) + p = MultipleProgress([Progress(100)]) for _ in 1:25 sleep(0.05) next!(p[1]) @@ -28,36 +28,52 @@ nworkers() == 1 && addprocs(4) sleep(0.1) @test has_finished(p) - function test_MP_finished(lengths, finish; kwargs...) - n = length(lengths) - p = MultipleProgress(lengths; kwargs...) - for _ in 1:100 - sleep(0.01) - for i in 1:n - next!(p[i]) - end - end - finish && finish!(p[0]) - sleep(0.1) - for i in 1:n - @test has_finished(p[i]) - end - end println("Testing MultipleProgress with custom titles and color") - test_MP_finished([100, 100], false; - desc="yellow ", color=:yellow, - kws=[(desc="red " , color=:red ), - (desc="yellow too ", )]) + p = MultipleProgress( + [Progress(100; color=:red, desc=" red "), + Progress(100; desc=" default color ")], + kwmain=(desc="yellow ", color=:yellow) + ) + for _ in 1:99 + sleep(0.01) + next!.(p[1:2]) + end + sleep(0.1) + @test !has_finished(p) + next!.(p[1:2]) + sleep(0.1) + @test has_finished(p) println("Testing over-shooting and under-shooting") - test_MP_finished([10, 110], true; dt=0.01) + p = MultipleProgress(Progress.([50, 140]), count_overshoot=false) + for _ in 1:100 + sleep(0.01) + next!.(p[1:2]) + end + sleep(0.1) + @test !has_finished(p) + finish!(p[2]) + sleep(0.1) + @test has_finished(p) println("Testing over-shooting with count_overshoot") - test_MP_finished([10, 190], false; count_overshoot=true, dt=0.01) + p = MultipleProgress(Progress.([52, 153]), count_overshoot=true) + next!.(p[1:2]) + sleep(0.1) + next!.(p[1:2]) + for _ in 1:200 + sleep(0.01) + next!(p[2]) + end + sleep(0.1) + @test !has_finished(p) + next!(p[2]) + sleep(0.1) + @test has_finished(p) println("Testing rapid over-shooting") - p = MultipleProgress([10]; dt=0.01, count_overshoot=true) + p = MultipleProgress([Progress(10)], count_overshoot=true) next!(p[1]) sleep(0.1) for i in 1:10000 @@ -67,7 +83,7 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing early cancel") - p = MultipleProgress([100, 80]) + p = MultipleProgress(Progress.([100, 80])) for _ in 1:50 sleep(0.02) next!(p[1]) @@ -79,7 +95,7 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing early cancel main progress") - p = MultipleProgress([100, 80]) + p = MultipleProgress(Progress.([100, 80])) for _ in 1:50 sleep(0.02) next!(p[1]) @@ -91,7 +107,7 @@ nworkers() == 1 && addprocs(4) println("Testing early finish main progress") - p = MultipleProgress([100, 80]) + p = MultipleProgress(Progress.([100, 80])) for _ in 1:50 sleep(0.02) next!(p[1]) @@ -102,7 +118,7 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing next! on main progress") - p = MultipleProgress([100]) + p = MultipleProgress([Progress(100)]) for _ in 1:99 sleep(0.02) next!(p[1]) @@ -114,8 +130,9 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing bar remplacement with $np workers and pmap") - lengths = rand(20:40, 2*np) - p = MultipleProgress(lengths; dt=0.01, kws=[(desc=" task $i ",) for i in 1:2*np]) + lengths = rand(20:50, 2*np) + progresses = [Progress(lengths[i], desc=" task $i ") for i in 1:2np] + p = MultipleProgress(progresses) ids = pmap(1:2*np) do ip for _ in 1:lengths[ip] sleep(0.05) @@ -128,7 +145,7 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing changing color with next! and update!") - p = MultipleProgress([100,100]) + p = MultipleProgress(Progress.([100,100])) for i in 1:100 sleep(0.01) if i == 25 @@ -149,7 +166,7 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing changing desc with next! and update!") - p = MultipleProgress([100,100,100]) + p = MultipleProgress(Progress.([100,100,100])) for i in 1:100 sleep(0.05) if i == 20 @@ -171,7 +188,7 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Update and finish all") - p = MultipleProgress([100,100,100]) + p = MultipleProgress(Progress.([100,100,100])) for i in 1:100 rand() < 0.5 && next!(p[1]) rand() < 0.3 && next!(p[2]) @@ -185,7 +202,7 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing without main progressmeter and offset 0 finishes last") - p = MultipleProgress([100,100,100], mainprogress=false) + p = MultipleProgress(Progress.([100,100,100]), kwmain=(enabled=false,)) update!(p[0], 10, :red, desc="I shouldn't exist ") for i in 1:80 next!(p[1], desc="task a ") @@ -200,7 +217,7 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing without main progressmeter and offset 0 finishes first (#215)") - p = MultipleProgress([100,100,100], mainprogress=false) + p = MultipleProgress(Progress.([100,100,100]), kwmain=(enabled=false,)) update!(p[0], 10, :red, desc="I shouldn't exist ") for i in 1:80 next!(p[1], desc="task a ") @@ -215,7 +232,7 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing early close (should not display error)") - p = MultipleProgress([100], desc="Close test") + p = MultipleProgress([Progress(100, desc="Close test")]) for i in 1:30 sleep(0.01) next!(p[1]) @@ -227,7 +244,7 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing errors in MultipleProgress (should display error)") - p = MultipleProgress([100], desc="Error test", color=:red) + p = MultipleProgress(Progress.([100]), kwmain=(desc="Error test", color=:red)) for i in 1:30 sleep(0.01) next!(p[1]) @@ -237,7 +254,7 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing with showvalues (doesn't really work)") - p = MultipleProgress([100,100]) + p = MultipleProgress(Progress.([100,100])) for i in 1:100 sleep(0.02) next!(p[1]; showvalues = Dict(:i=>i)) From e90a6a040830e497c1cbb11729d77f40cf6fb361 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Mon, 16 Aug 2021 10:56:17 +0200 Subject: [PATCH 16/39] on hold --- src/parallel_progress.jl | 49 ++--- test/test_multiple.jl | 427 ++++++++++++++++++++------------------- 2 files changed, 245 insertions(+), 231 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index ec7e6bb..280d829 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -177,19 +177,19 @@ function runMultipleProgress( # main progressbar if p == 0 if f == PP_CANCEL - mainprogress.counter = mainprogress.n - cancel(mainprogress, args...; kwt...) + finish!(mainprogress; keep=false) + cancel(mainprogress, args...; kwt..., keep=false) break elseif f == PP_UPDATE if !isempty(args) && args[1] == (:) - update!(mainprogress, mainprogress.counter, args[2:end]...; kwt...) + update!(mainprogress, valueorcounter(mainprogress), args[2:end]...; kwt..., keep=false) else - update!(mainprogress, args...; kwt...) + update!(mainprogress, args...; kwt..., keep=false) end elseif f == PP_NEXT - next!(mainprogress, args...; kwt...) + next!(mainprogress, args...; kwt..., keep=false) elseif f == PP_FINISH - finish!(mainprogress, args...; kwt...) + finish!(mainprogress, args...; kwt..., keep=false) break end else @@ -211,35 +211,35 @@ function runMultipleProgress( if f == PP_NEXT if count_overshoot || !has_finished(progresses[p]) - next!(progresses[p], args...; kwt...) - next!(mainprogress) + next!(progresses[p], args...; kwt..., keep=false) + !count_finishes && next!(mainprogress; keep=false) end else - prev_p_value = progresses[p].counter + prev_p_value = valueorcounter(progresses[p]) if f == PP_FINISH - finish!(progresses[p], args...; kwt...) + finish!(progresses[p], args...; kwt..., keep=false) elseif f == PP_CANCEL - #finish!(progresses[p]) - cancel(progresses[p], args...; kwt...) - progresses[p].counter = progresses[p].n + finish!(progresses[p]; keep=false) + cancel(progresses[p], args...; kwt..., keep=false) elseif f == PP_UPDATE if !isempty(args) value = args[1] - value == (:) && (value = progresses[p].counter) - !count_overshoot && (value = min(value, progresses[p].n)) - update!(progresses[p], value, args[2:end]...; kwt...) + value == (:) && (value = valueorcounter(progresses[p])) + !count_overshoot && progresses[p] isa Progress && (value = min(value, progresses[p].n)) + update!(progresses[p], value, args[2:end]...; kwt..., keep=false) else - update!(progresses[p]; kwt...) + update!(progresses[p]; kwt..., keep=false) end end - update!(mainprogress, - mainprogress.counter - prev_p_value + progresses[p].counter) + !count_finishes && update!(mainprogress, + mainprogress.counter - prev_p_value + progresses[p].counter; keep=false) end if has_finished(progresses[p]) delete!(taken_offsets, progresses[p].offset) + count_finishes && next!(mainprogress; keep=false) end end end @@ -252,11 +252,7 @@ function runMultipleProgress( println() end finally - print("\n" ^ max_offsets) - # progress with offset 0 adds automatically a line break when finished (#215) - if !has_finished(mainprogress) || mainprogress.enabled == false - println() - end + print("\n" ^ (max_offsets+1)) close(mp) end end @@ -283,6 +279,11 @@ has_finished(p::ProgressThresh) = p.triggered has_finished(p::ProgressUnknown) = p.done has_finished(p::ParallelProgress) = isfakechannel(p.channel) has_finished(p::MultipleProgress) = isfakechannel(p.channel) + isfakechannel(_) = false isfakechannel(::FakeChannel) = true isfakechannel(mc::MultipleChannel) = isfakechannel(mc.channel) + +valueorcounter(p::Progress) = p.counter +valueorcounter(p::ProgressThresh) = p.val +valueorcounter(p::ProgressUnknown) = p.counter diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 879ab0f..3cd9423 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -29,229 +29,229 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) - println("Testing MultipleProgress with custom titles and color") - p = MultipleProgress( - [Progress(100; color=:red, desc=" red "), - Progress(100; desc=" default color ")], - kwmain=(desc="yellow ", color=:yellow) - ) - for _ in 1:99 - sleep(0.01) - next!.(p[1:2]) - end - sleep(0.1) - @test !has_finished(p) - next!.(p[1:2]) - sleep(0.1) - @test has_finished(p) + # println("Testing MultipleProgress with custom titles and color") + # p = MultipleProgress( + # [Progress(100; color=:red, desc=" red "), + # Progress(100; desc=" default color ")], + # kwmain=(desc="yellow ", color=:yellow) + # ) + # for _ in 1:99 + # sleep(0.01) + # next!.(p[1:2]) + # end + # sleep(0.1) + # @test !has_finished(p) + # next!.(p[1:2]) + # sleep(0.1) + # @test has_finished(p) - println("Testing over-shooting and under-shooting") - p = MultipleProgress(Progress.([50, 140]), count_overshoot=false) - for _ in 1:100 - sleep(0.01) - next!.(p[1:2]) - end - sleep(0.1) - @test !has_finished(p) - finish!(p[2]) - sleep(0.1) - @test has_finished(p) + # println("Testing over-shooting and under-shooting") + # p = MultipleProgress(Progress.([50, 140]), count_overshoot=false) + # for _ in 1:100 + # sleep(0.01) + # next!.(p[1:2]) + # end + # sleep(0.1) + # @test !has_finished(p) + # finish!(p[2]) + # sleep(0.1) + # @test has_finished(p) - println("Testing over-shooting with count_overshoot") - p = MultipleProgress(Progress.([52, 153]), count_overshoot=true) - next!.(p[1:2]) - sleep(0.1) - next!.(p[1:2]) - for _ in 1:200 - sleep(0.01) - next!(p[2]) - end - sleep(0.1) - @test !has_finished(p) - next!(p[2]) - sleep(0.1) - @test has_finished(p) + # println("Testing over-shooting with count_overshoot") + # p = MultipleProgress(Progress.([52, 153]), count_overshoot=true) + # next!.(p[1:2]) + # sleep(0.1) + # next!.(p[1:2]) + # for _ in 1:200 + # sleep(0.01) + # next!(p[2]) + # end + # sleep(0.1) + # @test !has_finished(p) + # next!(p[2]) + # sleep(0.1) + # @test has_finished(p) - println("Testing rapid over-shooting") - p = MultipleProgress([Progress(10)], count_overshoot=true) - next!(p[1]) - sleep(0.1) - for i in 1:10000 - next!(p[1]) - end - sleep(0.1) - @test has_finished(p) + # println("Testing rapid over-shooting") + # p = MultipleProgress([Progress(10)], count_overshoot=true) + # next!(p[1]) + # sleep(0.1) + # for i in 1:10000 + # next!(p[1]) + # end + # sleep(0.1) + # @test has_finished(p) - println("Testing early cancel") - p = MultipleProgress(Progress.([100, 80])) - for _ in 1:50 - sleep(0.02) - next!(p[1]) - next!(p[2]) - end - cancel(p[1]) - finish!(p[2]) - sleep(0.1) - @test has_finished(p) + # println("Testing early cancel") + # p = MultipleProgress(Progress.([100, 80])) + # for _ in 1:50 + # sleep(0.02) + # next!(p[1]) + # next!(p[2]) + # end + # cancel(p[1]) + # finish!(p[2]) + # sleep(0.1) + # @test has_finished(p) - println("Testing early cancel main progress") - p = MultipleProgress(Progress.([100, 80])) - for _ in 1:50 - sleep(0.02) - next!(p[1]) - next!(p[2]) - end - cancel(p[0]) - sleep(0.1) - @test has_finished(p) + # println("Testing early cancel main progress") + # p = MultipleProgress(Progress.([100, 80])) + # for _ in 1:50 + # sleep(0.02) + # next!(p[1]) + # next!(p[2]) + # end + # cancel(p[0]) + # sleep(0.1) + # @test has_finished(p) - println("Testing early finish main progress") - p = MultipleProgress(Progress.([100, 80])) - for _ in 1:50 - sleep(0.02) - next!(p[1]) - next!(p[2]) - end - finish!(p[0]) - sleep(0.1) - @test has_finished(p) + # println("Testing early finish main progress") + # p = MultipleProgress(Progress.([100, 80])) + # for _ in 1:50 + # sleep(0.02) + # next!(p[1]) + # next!(p[2]) + # end + # finish!(p[0]) + # sleep(0.1) + # @test has_finished(p) - println("Testing next! on main progress") - p = MultipleProgress([Progress(100)]) - for _ in 1:99 - sleep(0.02) - next!(p[1]) - end - sleep(0.1) - @test !has_finished(p) - next!(p[0]) - sleep(0.1) - @test has_finished(p) + # println("Testing next! on main progress") + # p = MultipleProgress([Progress(100)]) + # for _ in 1:99 + # sleep(0.02) + # next!(p[1]) + # end + # sleep(0.1) + # @test !has_finished(p) + # next!(p[0]) + # sleep(0.1) + # @test has_finished(p) - println("Testing bar remplacement with $np workers and pmap") - lengths = rand(20:50, 2*np) - progresses = [Progress(lengths[i], desc=" task $i ") for i in 1:2np] - p = MultipleProgress(progresses) - ids = pmap(1:2*np) do ip - for _ in 1:lengths[ip] - sleep(0.05) - next!(p[ip]) - end - myid() - end - sleep(0.1) - @test length(unique(ids)) == np - @test has_finished(p) + # println("Testing bar remplacement with $np workers and pmap") + # lengths = rand(20:50, 2*np) + # progresses = [Progress(lengths[i], desc=" task $i ") for i in 1:2np] + # p = MultipleProgress(progresses) + # ids = pmap(1:2*np) do ip + # for _ in 1:lengths[ip] + # sleep(0.05) + # next!(p[ip]) + # end + # myid() + # end + # sleep(0.1) + # @test length(unique(ids)) == np + # @test has_finished(p) - println("Testing changing color with next! and update!") - p = MultipleProgress(Progress.([100,100])) - for i in 1:100 - sleep(0.01) - if i == 25 - next!(p[1], :red) - next!(p[2]) - elseif i == 50 - next!(p[1]) - update!(p[2], 51, :blue) - else - if i == 75 - update!(p[0], :, :yellow) - end - next!(p[1]) - next!(p[2]) - end - end - sleep(0.1) - @test has_finished(p) + # println("Testing changing color with next! and update!") + # p = MultipleProgress(Progress.([100,100])) + # for i in 1:100 + # sleep(0.01) + # if i == 25 + # next!(p[1], :red) + # next!(p[2]) + # elseif i == 50 + # next!(p[1]) + # update!(p[2], 51, :blue) + # else + # if i == 75 + # update!(p[0], :, :yellow) + # end + # next!(p[1]) + # next!(p[2]) + # end + # end + # sleep(0.1) + # @test has_finished(p) - println("Testing changing desc with next! and update!") - p = MultipleProgress(Progress.([100,100,100])) - for i in 1:100 - sleep(0.05) - if i == 20 - next!(p[1], desc="20% done ") - next!.(p[2:3]) - elseif i == 40 - next!.(p[[1,3]]) - update!(p[2], 41, desc="40% done ") - else - if i == 60 - update!(p[0], desc="60% done ") - elseif i == 80 - update!(p[3], desc="80% done ") - end - next!.(p[1:3]) - end - end - sleep(0.1) - @test has_finished(p) + # println("Testing changing desc with next! and update!") + # p = MultipleProgress(Progress.([100,100,100])) + # for i in 1:100 + # sleep(0.05) + # if i == 20 + # next!(p[1], desc="20% done ") + # next!.(p[2:3]) + # elseif i == 40 + # next!.(p[[1,3]]) + # update!(p[2], 41, desc="40% done ") + # else + # if i == 60 + # update!(p[0], desc="60% done ") + # elseif i == 80 + # update!(p[3], desc="80% done ") + # end + # next!.(p[1:3]) + # end + # end + # sleep(0.1) + # @test has_finished(p) - println("Update and finish all") - p = MultipleProgress(Progress.([100,100,100])) - for i in 1:100 - rand() < 0.5 && next!(p[1]) - rand() < 0.3 && next!(p[2]) - rand() < 0.1 && next!(p[3]) - sleep(0.02) - end - update!.(p[0:end], :, :red) - sleep(0.5) - finish!.(p[1:end]) - sleep(0.1) - @test has_finished(p) + # println("Update and finish all") + # p = MultipleProgress(Progress.([100,100,100])) + # for i in 1:100 + # rand() < 0.5 && next!(p[1]) + # rand() < 0.3 && next!(p[2]) + # rand() < 0.1 && next!(p[3]) + # sleep(0.02) + # end + # update!.(p[0:end], :, :red) + # sleep(0.5) + # finish!.(p[1:end]) + # sleep(0.1) + # @test has_finished(p) - println("Testing without main progressmeter and offset 0 finishes last") - p = MultipleProgress(Progress.([100,100,100]), kwmain=(enabled=false,)) - update!(p[0], 10, :red, desc="I shouldn't exist ") - for i in 1:80 - next!(p[1], desc="task a ") - rand() < 0.7 && next!(p[2], desc="task b ") - rand() < 0.4 && next!(p[3], desc="task c ") - sleep(0.02) - end - sleep(0.1) - @test !has_finished(p) - finish!.(p[end:-1:1]) - sleep(0.1) - @test has_finished(p) + # println("Testing without main progressmeter and offset 0 finishes last") + # p = MultipleProgress(Progress.([100,100,100]), kwmain=(enabled=false,)) + # update!(p[0], 10, :red, desc="I shouldn't exist ") + # for i in 1:80 + # next!(p[1], desc="task a ") + # rand() < 0.7 && next!(p[2], desc="task b ") + # rand() < 0.4 && next!(p[3], desc="task c ") + # sleep(0.02) + # end + # sleep(0.1) + # @test !has_finished(p) + # finish!.(p[end:-1:1]) + # sleep(0.1) + # @test has_finished(p) - println("Testing without main progressmeter and offset 0 finishes first (#215)") - p = MultipleProgress(Progress.([100,100,100]), kwmain=(enabled=false,)) - update!(p[0], 10, :red, desc="I shouldn't exist ") - for i in 1:80 - next!(p[1], desc="task a ") - rand() < 0.7 && next!(p[2], desc="task b ") - rand() < 0.4 && next!(p[3], desc="task c ") - sleep(0.02) - end - sleep(0.1) - @test !has_finished(p) - finish!.(p[1:end]) - sleep(0.1) - @test has_finished(p) + # println("Testing without main progressmeter and offset 0 finishes first (#215)") + # p = MultipleProgress(Progress.([100,100,100]), kwmain=(enabled=false,)) + # update!(p[0], 10, :red, desc="I shouldn't exist ") + # for i in 1:80 + # next!(p[1], desc="task a ") + # rand() < 0.7 && next!(p[2], desc="task b ") + # rand() < 0.4 && next!(p[3], desc="task c ") + # sleep(0.02) + # end + # sleep(0.1) + # @test !has_finished(p) + # finish!.(p[1:end]) + # sleep(0.1) + # @test has_finished(p) - println("Testing early close (should not display error)") - p = MultipleProgress([Progress(100, desc="Close test")]) - for i in 1:30 - sleep(0.01) - next!(p[1]) - end - sleep(0.1) - @test !has_finished(p) - close(p) - sleep(0.1) - @test has_finished(p) + # println("Testing early close (should not display error)") + # p = MultipleProgress([Progress(100, desc="Close test")]) + # for i in 1:30 + # sleep(0.01) + # next!(p[1]) + # end + # sleep(0.1) + # @test !has_finished(p) + # close(p) + # sleep(0.1) + # @test has_finished(p) - println("Testing errors in MultipleProgress (should display error)") - p = MultipleProgress(Progress.([100]), kwmain=(desc="Error test", color=:red)) - for i in 1:30 - sleep(0.01) - next!(p[1]) - end - next!(p[1], 1) - sleep(2) - @test has_finished(p) + # println("Testing errors in MultipleProgress (should display error)") + # p = MultipleProgress(Progress.([100]), kwmain=(desc="Error test", color=:red)) + # for i in 1:30 + # sleep(0.01) + # next!(p[1]) + # end + # next!(p[1], 1) + # sleep(2) + # @test has_finished(p) println("Testing with showvalues (doesn't really work)") p = MultipleProgress(Progress.([100,100])) @@ -262,4 +262,17 @@ nworkers() == 1 && addprocs(4) end sleep(0.1) @test has_finished(p) + + println("Testing MultipleProgress with ProgressUnknown") + p = MultipleProgress([Progress(100), Progress(100)], ProgressUnknown(); count_finishes=false) + for i in 1:100 + sleep(0.01) + next!(p[1]) + rand() < 0.5 && next!(p[2]) + end + sleep(0.1) + @test !has_finished(p) + finish!(p[2]) + sleep(0.1) + @test has_finished(p) end From cfa53bc866e685ac18e6d618e437a5e161d129f3 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Mon, 16 Aug 2021 11:42:07 +0200 Subject: [PATCH 17/39] add support for offset in ProgressUnknown --- src/ProgressMeter.jl | 29 ++++++++++++++++++++++++----- test/test.jl | 12 ++++++++++++ test/test_float.jl | 12 ++++++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/ProgressMeter.jl b/src/ProgressMeter.jl index 641bfb8..e95aa2c 100644 --- a/src/ProgressMeter.jl +++ b/src/ProgressMeter.jl @@ -194,6 +194,7 @@ mutable struct ProgressUnknown <: AbstractProgress spinner::Bool # show a spinner output::IO # output stream into which the progress is written numprintedvalues::Int # num values printed below progress in last iteration + offset::Int # position offset of progress bar (default is 0) enabled::Bool # is the output enabled showspeed::Bool # should the output include average time per iteration check_iterations::Int @@ -201,13 +202,21 @@ mutable struct ProgressUnknown <: AbstractProgress threads_used::Vector{Int} end -function ProgressUnknown(;dt::Real=0.1, desc::AbstractString="Progress: ", color::Symbol=:green, spinner::Bool=false, output::IO=stderr, enabled::Bool = true, showspeed::Bool = false) +function ProgressUnknown(; + dt::Real=0.1, + desc::AbstractString="Progress: ", + color::Symbol=:green, + spinner::Bool=false, + output::IO=stderr, + offset::Integer=0, + enabled::Bool = true, + showspeed::Bool = false) RUNNING_IJULIA_KERNEL[] = running_ijulia_kernel() CLEAR_IJULIA[] = clear_ijulia() reentrantlocker = Threads.ReentrantLock() tinit = tlast = time() printed = false - ProgressUnknown(false, reentrantlocker, dt, 0, 0, false, tinit, tlast, printed, desc, color, spinner, output, 0, enabled, showspeed, 1, 1, Int[]) + ProgressUnknown(false, reentrantlocker, dt, 0, 0, false, tinit, tlast, printed, desc, color, spinner, output, 0, offset, enabled, showspeed, 1, 1, Int[]) end ProgressUnknown(dt::Real, desc::AbstractString="Progress: ", @@ -402,9 +411,12 @@ spinner_char(p::ProgressUnknown, spinner::AbstractVector{<:AbstractChar}) = spinner_char(p::ProgressUnknown, spinner::AbstractString) = p.done ? spinner_done : spinner[nextind(spinner, 1, p.spincounter % length(spinner))] -function updateProgress!(p::ProgressUnknown; showvalues = (), truncate_lines = false, valuecolor = :blue, desc = p.desc, - ignore_predictor = false, spinner::Union{AbstractChar,AbstractString,AbstractVector{<:AbstractChar}} = spinner_chars) +function updateProgress!(p::ProgressUnknown; showvalues = (), truncate_lines = false, + valuecolor = :blue, desc = p.desc, ignore_predictor = false, + spinner::Union{AbstractChar,AbstractString,AbstractVector{<:AbstractChar}} = spinner_chars, + offset::Integer = p.offset, keep = (offset == 0)) (!RUNNING_IJULIA_KERNEL[] & !p.enabled) && return + p.offset = offset p.desc = desc if p.done if p.printed @@ -421,10 +433,15 @@ function updateProgress!(p::ProgressUnknown; showvalues = (), truncate_lines = f sec_per_iter = elapsed_time / p.counter msg = @sprintf "%s (%s)" msg speedstring(sec_per_iter) end + print(p.output, "\n" ^ (p.offset + p.numprintedvalues)) move_cursor_up_while_clearing_lines(p.output, p.numprintedvalues) printover(p.output, msg, p.color) printvalues!(p, showvalues; color = valuecolor, truncate = truncate_lines) - println(p.output) + if keep + println(p.output) + else + print(p.output, "\r\u1b[A" ^ (p.offset + p.numprintedvalues)) + end flush(p.output) end return @@ -447,9 +464,11 @@ function updateProgress!(p::ProgressUnknown; showvalues = (), truncate_lines = f sec_per_iter = elapsed_time / p.counter msg = @sprintf "%s (%s)" msg speedstring(sec_per_iter) end + print(p.output, "\n" ^ (p.offset + p.numprintedvalues)) move_cursor_up_while_clearing_lines(p.output, p.numprintedvalues) printover(p.output, msg, p.color) printvalues!(p, showvalues; color = valuecolor, truncate = truncate_lines) + print(p.output, "\r\u1b[A" ^ (p.offset + p.numprintedvalues)) flush(p.output) # Compensate for any overhead of printing. This can be # especially important if you're running over a slow network diff --git a/test/test.jl b/test/test.jl index e4c4ecc..b09a333 100644 --- a/test/test.jl +++ b/test/test.jl @@ -416,3 +416,15 @@ function testfunc19() end println("Testing speed display with no update") testfunc19() + +function testfunc20(r, p) + for i in r + sleep(0.05) + update!(p, i) + end + cancel(p) +end +println("Testing early cancel") +testfunc20(1:50, Progress(100)) +testfunc20(1:50, ProgressUnknown()) +testfunc20(50:-1:1, ProgressThresh(0)) \ No newline at end of file diff --git a/test/test_float.jl b/test/test_float.jl index a1445c4..f9d7d28 100644 --- a/test/test_float.jl +++ b/test/test_float.jl @@ -61,3 +61,15 @@ function testfunc5(n, tsleep) print("\n" ^ 2) end testfunc5(5, 0.1) + +println("Testing floating unknown progress bar (offset 2)") +function testfunc6(desc, n, tsleep, offset) + p = ProgressMeter.ProgressUnknown(desc; offset=offset) + for i = 1:n + sleep(tsleep) + ProgressMeter.next!(p) + end + ProgressMeter.finish!(p) + print("\n" ^ 3) +end +testfunc6("Computing... ", 100, 0.02, 2) From 2c83e237c7e115d01ba498e586d45376e59d1792 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Mon, 16 Aug 2021 11:44:07 +0200 Subject: [PATCH 18/39] add support for offset in ProgressUnknown --- test/test.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.jl b/test/test.jl index b09a333..768e539 100644 --- a/test/test.jl +++ b/test/test.jl @@ -427,4 +427,4 @@ end println("Testing early cancel") testfunc20(1:50, Progress(100)) testfunc20(1:50, ProgressUnknown()) -testfunc20(50:-1:1, ProgressThresh(0)) \ No newline at end of file +testfunc20(50:-1:1, ProgressThresh(0)) From e27702c4b6011381b3bc08178403565a5698430e Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Mon, 16 Aug 2021 11:56:36 +0200 Subject: [PATCH 19/39] add test for cancel with offset and keep --- test/test.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/test.jl b/test/test.jl index 768e539..289f3dd 100644 --- a/test/test.jl +++ b/test/test.jl @@ -419,12 +419,16 @@ testfunc19() function testfunc20(r, p) for i in r - sleep(0.05) + sleep(0.03) update!(p, i) end - cancel(p) + cancel(p; keep=true) end println("Testing early cancel") testfunc20(1:50, Progress(100)) testfunc20(1:50, ProgressUnknown()) testfunc20(50:-1:1, ProgressThresh(0)) +println("Testing early cancel with offset 1 and keep") +testfunc20(1:50, Progress(100, offset=1)) +testfunc20(1:50, ProgressUnknown(offset=1)) +testfunc20(50:-1:1, ProgressThresh(0, offset=1)) From 81608b2cff6310d3b47beb6f8ce5ba3cb7502432 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Mon, 16 Aug 2021 12:35:27 +0200 Subject: [PATCH 20/39] allow any AbstractProgress in MultipleProgress --- src/parallel_progress.jl | 68 ++++++++++++++++++++++------------------ test/test_multiple.jl | 26 ++++++++++++--- 2 files changed, 59 insertions(+), 35 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 280d829..d831ad5 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -129,20 +129,11 @@ julia> p = MultipleProgress(fill(10,5); desc="global ", kws=[(desc="task \$i ",) end ``` """ -function MultipleProgress(progresses::AbstractVector{Progress}; - count_finishes=false, kwmain=(), kw...) - main_length = count_finishes ? length(progresses) : sum(p->p.n, progresses) - mainprogress = Progress(main_length; kwmain...) - return MultipleProgress(progresses, mainprogress; count_finishes=count_finishes, kw...) -end - -function MultipleProgress( - progresses::AbstractVector{<:AbstractProgress}, - mainprogress::AbstractProgress; - count_finishes = false, - count_overshoot = false, - auto_reset_timer = true - ) +function MultipleProgress(progresses::AbstractVector{<:AbstractProgress}, + mainprogress::AbstractProgress; + count_finishes = false, + count_overshoot = false, + auto_reset_timer = true) channel = RemoteChannel(() -> Channel{NTuple{4,Any}}(1024)) mp = MultipleProgress(channel, length(progresses)) @async runMultipleProgress(progresses, mainprogress, mp; @@ -152,14 +143,28 @@ function MultipleProgress( return mp end -function runMultipleProgress( - progresses::AbstractVector{<:AbstractProgress}, - mainprogress::AbstractProgress, - mp::MultipleProgress; - count_finishes = false, - count_overshoot = false, - auto_reset_timer = true - ) +function MultipleProgress(progresses::AbstractVector{Progress}; + count_finishes=false, kwmain=(), kw...) + main_length = count_finishes ? length(progresses) : sum(p->p.n, progresses) + mainprogress = Progress(main_length; kwmain...) + return MultipleProgress(progresses, mainprogress; count_finishes=count_finishes, kw...) +end + +function MultipleProgress(progresses::AbstractVector{<:AbstractProgress}; + count_finishes=false, kwmain=(), kw...) + if count_finishes + MultipleProgress(progresses, Progress(length(progresses); kwmain...); count_finishes=count_finishes, kw...) + else + MultipleProgress(progresses, ProgressUnknown(;kwmain...); count_finishes=count_finishes, kw...) + end +end + +function runMultipleProgress(progresses::AbstractVector{<:AbstractProgress}, + mainprogress::AbstractProgress, + mp::MultipleProgress; + count_finishes = false, + count_overshoot = false, + auto_reset_timer = true) for p in progresses p.offset = -1 end @@ -170,7 +175,7 @@ function runMultipleProgress( try # we must make sure that 2 progresses aren't updated at the same time, # that's why we use only one Channel - while !has_finished(mainprogress) && !all(has_finished, progresses) + while !has_finished(mainprogress) && any(!has_finished, progresses) p, f, args, kwt = take!(channel) @@ -180,8 +185,8 @@ function runMultipleProgress( finish!(mainprogress; keep=false) cancel(mainprogress, args...; kwt..., keep=false) break - elseif f == PP_UPDATE - if !isempty(args) && args[1] == (:) + elseif f == PP_UPDATE + if !isempty(args) && args[1] == (:) # allow to update without knowing the counter with (:) update!(mainprogress, valueorcounter(mainprogress), args[2:end]...; kwt..., keep=false) else update!(mainprogress, args...; kwt..., keep=false) @@ -204,11 +209,13 @@ function runMultipleProgress( max_offsets = max(max_offsets, offset) progresses[p].offset = offset if auto_reset_timer - progresses[p].tinit = progresses[p].tsecond = progresses[p].tlast = time() + progresses[p].tinit = time() end push!(taken_offsets, offset) end + already_finished = has_finished(progresses[p]) + if f == PP_NEXT if count_overshoot || !has_finished(progresses[p]) next!(progresses[p], args...; kwt..., keep=false) @@ -216,6 +223,7 @@ function runMultipleProgress( end else prev_p_value = valueorcounter(progresses[p]) + prev_p_counter = progresses[p].counter if f == PP_FINISH finish!(progresses[p], args...; kwt..., keep=false) @@ -223,7 +231,7 @@ function runMultipleProgress( finish!(progresses[p]; keep=false) cancel(progresses[p], args...; kwt..., keep=false) elseif f == PP_UPDATE - if !isempty(args) + if !isempty(args) # allow to update without knowing the counter with (:) value = args[1] value == (:) && (value = valueorcounter(progresses[p])) !count_overshoot && progresses[p] isa Progress && (value = min(value, progresses[p].n)) @@ -234,10 +242,10 @@ function runMultipleProgress( end !count_finishes && update!(mainprogress, - mainprogress.counter - prev_p_value + progresses[p].counter; keep=false) + mainprogress.counter - prev_p_counter + progresses[p].counter; keep=false) end - if has_finished(progresses[p]) + if has_finished(progresses[p]) && !already_finished delete!(taken_offsets, progresses[p].offset) count_finishes && next!(mainprogress; keep=false) end @@ -257,8 +265,6 @@ function runMultipleProgress( end end - - """ close(p::Union{ParallelProgress,MultipleProgress}) diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 3cd9423..133540f 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -262,17 +262,35 @@ nworkers() == 1 && addprocs(4) end sleep(0.1) @test has_finished(p) + print("\n"^5) - println("Testing MultipleProgress with ProgressUnknown") - p = MultipleProgress([Progress(100), Progress(100)], ProgressUnknown(); count_finishes=false) + println("Testing MultipleProgress with ProgressUnknown as mainprogress") + p = MultipleProgress([Progress(100), ProgressUnknown(),ProgressThresh(0.01)]; count_finishes=false) for i in 1:100 - sleep(0.01) + sleep(0.02) next!(p[1]) - rand() < 0.5 && next!(p[2]) + next!(p[2]) + update!(p[3], 1/i) end sleep(0.1) @test !has_finished(p) finish!(p[2]) sleep(0.1) @test has_finished(p) + + println("Testing MultipleProgress with count_finishes") + p = MultipleProgress([ProgressUnknown(), Progress(50), ProgressThresh(0.05)]; + count_finishes=true, kwmain=(dt=0, desc="Tasks finished ")) + update!(p[0], 0) + for i in 1:100 + sleep(0.02) + next!(p[1]) + next!(p[2]) + update!(p[3], 1/i) + end + sleep(0.1) + @test !has_finished(p) + finish!(p[1]) + sleep(0.1) + @test has_finished(p) end From dac02ddfbbcb610a341d9450581c6561f0152502 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Mon, 16 Aug 2021 15:29:27 +0200 Subject: [PATCH 21/39] fixes and doc --- src/parallel_progress.jl | 138 ++++++++++-- test/runtests.jl | 2 +- test/test_multiple.jl | 462 +++++++++++++++++++++------------------ 3 files changed, 369 insertions(+), 233 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index d831ad5..2c7de08 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -3,10 +3,15 @@ mutable struct ParallelProgress <: AbstractProgress channel end -const PP_NEXT = :next -const PP_CANCEL = :cancel -const PP_FINISH = :finish -const PP_UPDATE = :update +@enum ProgressAction begin + PP_NEXT + PP_CANCEL + PP_FINISH + PP_UPDATE + MP_ADD_THRESH + MP_ADD_UNKNOWN + MP_ADD_PROGRESS +end next!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_NEXT, args, kw)); nothing) cancel(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_CANCEL, args, kw)); nothing) @@ -19,16 +24,16 @@ update!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_UPDATE, ar works like `Progress` but can be used from other workers # Example: -```jldoctest -julia> using Distributed -julia> addprocs() -julia> @everywhere using ProgressMeter -julia> prog = ParallelProgress(10; desc="test ") -julia> pmap(1:10) do - sleep(rand()) - next!(prog) - return myid() - end +```julia +using Distributed +addprocs() +@everywhere using ProgressMeter +prog = ParallelProgress(10; desc="test ") +pmap(1:10) do + sleep(rand()) + next!(prog) + return myid() +end ``` """ function ParallelProgress(n::Integer; kw...) @@ -103,13 +108,24 @@ Base.getindex(mp::MultipleProgress, n) = ParallelProgress.(MultipleChannel.(Ref( Base.lastindex(mp::MultipleProgress) = mp.amount """ - MultipleProgress(lengths; mainprogress=true, count_overshoot=false, kws, kw...) - -generates one progressbar for each value in `lengths` - - `kw` arguments are applied on all progressbars - - `kws[i]` arguments are applied on the i-th progressbar - - `mainprogress` adds a main progressmeter that sums the other ones + MultipleProgress(progresses::AbstractVector{<:AbstractProgress}, + mainprogress::AbstractProgress; + enabled = true, + auto_close = true, + count_finishes = false, + count_overshoot = false, + auto_reset_timer = true) + +allows to call the `progresses` and `mainprogress` from different workers + - `progresses`: contains the different progressbars + - `mainprogress`: main progressbar + - `enabled`: `enabled == false` doesn't show anything and doesn't open a channel + - `auto_close`: if true, the channel will close when all progresses are finished, otherwise, + when mainprogress finishes or with `close(p)` + - `count_finishes`: if false, main_progress will be the sum of the individual progressbars, + if true, it will be equal to the number of finished progressbars - `count_overshoot`: overshooting progressmeters will be counted in the main progressmeter + - `auto_reset_timer`: tinit in progresses will be reset at first call use p[i] to access the i-th progressmeter, and p[0] to access the main one @@ -131,18 +147,30 @@ julia> p = MultipleProgress(fill(10,5); desc="global ", kws=[(desc="task \$i ",) """ function MultipleProgress(progresses::AbstractVector{<:AbstractProgress}, mainprogress::AbstractProgress; + enabled = true, + auto_close = true, count_finishes = false, count_overshoot = false, auto_reset_timer = true) + !enabled && return MultipleProgress(FakeChannel(), length(progresses)) + channel = RemoteChannel(() -> Channel{NTuple{4,Any}}(1024)) mp = MultipleProgress(channel, length(progresses)) @async runMultipleProgress(progresses, mainprogress, mp; + auto_close=auto_close, count_finishes=count_finishes, count_overshoot=count_overshoot, auto_reset_timer=auto_reset_timer) return mp end +""" + MultipleProgress(progresses::AbstractVector{Progress}; count_finishes=false, kwmain=(), kw...) + +is equivalent to: + + MultipleProgress(progresses, Progress(main_length; kwmain...); count_finishes, kw...) +""" function MultipleProgress(progresses::AbstractVector{Progress}; count_finishes=false, kwmain=(), kw...) main_length = count_finishes ? length(progresses) : sum(p->p.n, progresses) @@ -150,6 +178,16 @@ function MultipleProgress(progresses::AbstractVector{Progress}; return MultipleProgress(progresses, mainprogress; count_finishes=count_finishes, kw...) end +""" + MultipleProgress(progresses::AbstractVector{<:AbstractProgress}; count_finishes=false, kwmain=(), kw...) + +is equivalent to: + + MultipleProgress(progresses, prog; count_finishes, kw...) + +with `prog` a `Progress` of the same length as `progresses` if `count_finishes == true`, +otherwise, `prog` is a `ProgressUnknown` +""" function MultipleProgress(progresses::AbstractVector{<:AbstractProgress}; count_finishes=false, kwmain=(), kw...) if count_finishes @@ -159,9 +197,21 @@ function MultipleProgress(progresses::AbstractVector{<:AbstractProgress}; end end +""" + MultipleProgress(mainprogress::AbstractProgress; auto_close=false, kw...) + +is equivalent to + + MultipleProgress(AbstractProgress[], mainprogress; auto_close, kw...) +""" +function MultipleProgress(mainprogress::AbstractProgress; auto_close=false, kw...) + return MultipleProgress(AbstractProgress[], mainprogress; auto_close=auto_close, kw...) +end + function runMultipleProgress(progresses::AbstractVector{<:AbstractProgress}, mainprogress::AbstractProgress, mp::MultipleProgress; + auto_close = true, count_finishes = false, count_overshoot = false, auto_reset_timer = true) @@ -175,12 +225,11 @@ function runMultipleProgress(progresses::AbstractVector{<:AbstractProgress}, try # we must make sure that 2 progresses aren't updated at the same time, # that's why we use only one Channel - while !has_finished(mainprogress) && any(!has_finished, progresses) + while !has_finished(mainprogress) && !(auto_close && all(has_finished, progresses)) p, f, args, kwt = take!(channel) - # main progressbar - if p == 0 + if p == 0 # main progressbar if f == PP_CANCEL finish!(mainprogress; keep=false) cancel(mainprogress, args...; kwt..., keep=false) @@ -198,6 +247,23 @@ function runMultipleProgress(progresses::AbstractVector{<:AbstractProgress}, break end else + # add progress + if f == MP_ADD_PROGRESS + resize!(progresses, max(length(progresses), p)) + mp.amount = length(progresses) + progresses[p] = Progress(args...; kwt..., offset=-1) + continue + elseif f == MP_ADD_UNKNOWN + resize!(progresses, max(length(progresses), p)) + mp.amount = length(progresses) + progresses[p] = ProgressUnknown(args...; kwt..., offset=-1) + continue + elseif f == MP_ADD_THRESH + resize!(progresses, max(length(progresses), p)) + mp.amount = length(progresses) + progresses[p] = ProgressThresh(args...; kwt..., offset=-1) + continue + end # first time calling progress p if progresses[p].offset == -1 @@ -222,7 +288,6 @@ function runMultipleProgress(progresses::AbstractVector{<:AbstractProgress}, !count_finishes && next!(mainprogress; keep=false) end else - prev_p_value = valueorcounter(progresses[p]) prev_p_counter = progresses[p].counter if f == PP_FINISH @@ -293,3 +358,28 @@ isfakechannel(mc::MultipleChannel) = isfakechannel(mc.channel) valueorcounter(p::Progress) = p.counter valueorcounter(p::ProgressThresh) = p.val valueorcounter(p::ProgressUnknown) = p.counter + +""" + addprogress!(mp[i], T::Type{<:AbstractProgress}, args...; kw...) + +will add the progressbar `T(args..., kw...)` to the MultipleProgress `mp` at index `i` + +# Example + +```julia +p = MultipleProgress(Progress(N, "tasks done "); count_finishes=true) +sleep(0.1) +pmap(1:N) do i + L = rand(20:50) + ProgressMeter.addprogress!(p[i], Progress, L, desc=" task \$i ") + for _ in 1:L + sleep(0.05) + next!(p[i]) + end +end + +``` +""" +addprogress!(p::ParallelProgress, ::Type{Progress}, args...; kw...) = (put!(p.channel, (MP_ADD_PROGRESS, args, kw)); nothing) +addprogress!(p::ParallelProgress, ::Type{ProgressThresh}, args...; kw...) = (put!(p.channel, (MP_ADD_THRESH, args, kw)); nothing) +addprogress!(p::ParallelProgress, ::Type{ProgressUnknown}, args...; kw...) = (put!(p.channel, (MP_ADD_UNKNOWN, args, kw)); nothing) diff --git a/test/runtests.jl b/test/runtests.jl index a95cb84..0f39356 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -23,6 +23,6 @@ end # include("test_threads.jl") # end @testset "Parallel" begin - #include("test_parallel.jl") + include("test_parallel.jl") include("test_multiple.jl") end diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 133540f..ca12564 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -29,229 +29,229 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) - # println("Testing MultipleProgress with custom titles and color") - # p = MultipleProgress( - # [Progress(100; color=:red, desc=" red "), - # Progress(100; desc=" default color ")], - # kwmain=(desc="yellow ", color=:yellow) - # ) - # for _ in 1:99 - # sleep(0.01) - # next!.(p[1:2]) - # end - # sleep(0.1) - # @test !has_finished(p) - # next!.(p[1:2]) - # sleep(0.1) - # @test has_finished(p) + println("Testing MultipleProgress with custom titles and color") + p = MultipleProgress( + [Progress(100; color=:red, desc=" red "), + Progress(100; desc=" default color ")], + kwmain=(desc="yellow ", color=:yellow) + ) + for _ in 1:99 + sleep(0.01) + next!.(p[1:2]) + end + sleep(0.1) + @test !has_finished(p) + next!.(p[1:2]) + sleep(0.1) + @test has_finished(p) - # println("Testing over-shooting and under-shooting") - # p = MultipleProgress(Progress.([50, 140]), count_overshoot=false) - # for _ in 1:100 - # sleep(0.01) - # next!.(p[1:2]) - # end - # sleep(0.1) - # @test !has_finished(p) - # finish!(p[2]) - # sleep(0.1) - # @test has_finished(p) + println("Testing over-shooting and under-shooting") + p = MultipleProgress(Progress.([50, 140]), count_overshoot=false) + for _ in 1:100 + sleep(0.01) + next!.(p[1:2]) + end + sleep(0.1) + @test !has_finished(p) + finish!(p[2]) + sleep(0.1) + @test has_finished(p) - # println("Testing over-shooting with count_overshoot") - # p = MultipleProgress(Progress.([52, 153]), count_overshoot=true) - # next!.(p[1:2]) - # sleep(0.1) - # next!.(p[1:2]) - # for _ in 1:200 - # sleep(0.01) - # next!(p[2]) - # end - # sleep(0.1) - # @test !has_finished(p) - # next!(p[2]) - # sleep(0.1) - # @test has_finished(p) + println("Testing over-shooting with count_overshoot") + p = MultipleProgress(Progress.([52, 153]), count_overshoot=true) + next!.(p[1:2]) + sleep(0.1) + next!.(p[1:2]) + for _ in 1:200 + sleep(0.01) + next!(p[2]) + end + sleep(0.1) + @test !has_finished(p) + next!(p[2]) + sleep(0.1) + @test has_finished(p) - # println("Testing rapid over-shooting") - # p = MultipleProgress([Progress(10)], count_overshoot=true) - # next!(p[1]) - # sleep(0.1) - # for i in 1:10000 - # next!(p[1]) - # end - # sleep(0.1) - # @test has_finished(p) + println("Testing rapid over-shooting") + p = MultipleProgress([Progress(10)], count_overshoot=true) + next!(p[1]) + sleep(0.1) + for i in 1:10000 + next!(p[1]) + end + sleep(0.1) + @test has_finished(p) - # println("Testing early cancel") - # p = MultipleProgress(Progress.([100, 80])) - # for _ in 1:50 - # sleep(0.02) - # next!(p[1]) - # next!(p[2]) - # end - # cancel(p[1]) - # finish!(p[2]) - # sleep(0.1) - # @test has_finished(p) + println("Testing early cancel") + p = MultipleProgress(Progress.([100, 80])) + for _ in 1:50 + sleep(0.02) + next!(p[1]) + next!(p[2]) + end + cancel(p[1]) + finish!(p[2]) + sleep(0.1) + @test has_finished(p) - # println("Testing early cancel main progress") - # p = MultipleProgress(Progress.([100, 80])) - # for _ in 1:50 - # sleep(0.02) - # next!(p[1]) - # next!(p[2]) - # end - # cancel(p[0]) - # sleep(0.1) - # @test has_finished(p) + println("Testing early cancel main progress") + p = MultipleProgress(Progress.([100, 80])) + for _ in 1:50 + sleep(0.02) + next!(p[1]) + next!(p[2]) + end + cancel(p[0]) + sleep(0.1) + @test has_finished(p) - # println("Testing early finish main progress") - # p = MultipleProgress(Progress.([100, 80])) - # for _ in 1:50 - # sleep(0.02) - # next!(p[1]) - # next!(p[2]) - # end - # finish!(p[0]) - # sleep(0.1) - # @test has_finished(p) + println("Testing early finish main progress") + p = MultipleProgress(Progress.([100, 80])) + for _ in 1:50 + sleep(0.02) + next!(p[1]) + next!(p[2]) + end + finish!(p[0]) + sleep(0.1) + @test has_finished(p) - # println("Testing next! on main progress") - # p = MultipleProgress([Progress(100)]) - # for _ in 1:99 - # sleep(0.02) - # next!(p[1]) - # end - # sleep(0.1) - # @test !has_finished(p) - # next!(p[0]) - # sleep(0.1) - # @test has_finished(p) + println("Testing next! on main progress") + p = MultipleProgress([Progress(100)]) + for _ in 1:99 + sleep(0.02) + next!(p[1]) + end + sleep(0.1) + @test !has_finished(p) + next!(p[0]) + sleep(0.1) + @test has_finished(p) - # println("Testing bar remplacement with $np workers and pmap") - # lengths = rand(20:50, 2*np) - # progresses = [Progress(lengths[i], desc=" task $i ") for i in 1:2np] - # p = MultipleProgress(progresses) - # ids = pmap(1:2*np) do ip - # for _ in 1:lengths[ip] - # sleep(0.05) - # next!(p[ip]) - # end - # myid() - # end - # sleep(0.1) - # @test length(unique(ids)) == np - # @test has_finished(p) + println("Testing bar remplacement with $np workers and pmap") + lengths = rand(20:50, 2*np) + progresses = [Progress(lengths[i], desc=" task $i ") for i in 1:2np] + p = MultipleProgress(progresses) + ids = pmap(1:2*np) do ip + for _ in 1:lengths[ip] + sleep(0.05) + next!(p[ip]) + end + myid() + end + sleep(0.1) + @test length(unique(ids)) == np + @test has_finished(p) - # println("Testing changing color with next! and update!") - # p = MultipleProgress(Progress.([100,100])) - # for i in 1:100 - # sleep(0.01) - # if i == 25 - # next!(p[1], :red) - # next!(p[2]) - # elseif i == 50 - # next!(p[1]) - # update!(p[2], 51, :blue) - # else - # if i == 75 - # update!(p[0], :, :yellow) - # end - # next!(p[1]) - # next!(p[2]) - # end - # end - # sleep(0.1) - # @test has_finished(p) + println("Testing changing color with next! and update!") + p = MultipleProgress(Progress.([100,100])) + for i in 1:100 + sleep(0.01) + if i == 25 + next!(p[1], :red) + next!(p[2]) + elseif i == 50 + next!(p[1]) + update!(p[2], 51, :blue) + else + if i == 75 + update!(p[0], :, :yellow) + end + next!(p[1]) + next!(p[2]) + end + end + sleep(0.1) + @test has_finished(p) - # println("Testing changing desc with next! and update!") - # p = MultipleProgress(Progress.([100,100,100])) - # for i in 1:100 - # sleep(0.05) - # if i == 20 - # next!(p[1], desc="20% done ") - # next!.(p[2:3]) - # elseif i == 40 - # next!.(p[[1,3]]) - # update!(p[2], 41, desc="40% done ") - # else - # if i == 60 - # update!(p[0], desc="60% done ") - # elseif i == 80 - # update!(p[3], desc="80% done ") - # end - # next!.(p[1:3]) - # end - # end - # sleep(0.1) - # @test has_finished(p) + println("Testing changing desc with next! and update!") + p = MultipleProgress(Progress.([100,100,100])) + for i in 1:100 + sleep(0.05) + if i == 20 + next!(p[1], desc="20% done ") + next!.(p[2:3]) + elseif i == 40 + next!.(p[[1,3]]) + update!(p[2], 41, desc="40% done ") + else + if i == 60 + update!(p[0], desc="60% done ") + elseif i == 80 + update!(p[3], desc="80% done ") + end + next!.(p[1:3]) + end + end + sleep(0.1) + @test has_finished(p) - # println("Update and finish all") - # p = MultipleProgress(Progress.([100,100,100])) - # for i in 1:100 - # rand() < 0.5 && next!(p[1]) - # rand() < 0.3 && next!(p[2]) - # rand() < 0.1 && next!(p[3]) - # sleep(0.02) - # end - # update!.(p[0:end], :, :red) - # sleep(0.5) - # finish!.(p[1:end]) - # sleep(0.1) - # @test has_finished(p) + println("Update and finish all") + p = MultipleProgress(Progress.([100,100,100])) + for i in 1:100 + rand() < 0.5 && next!(p[1]) + rand() < 0.3 && next!(p[2]) + rand() < 0.1 && next!(p[3]) + sleep(0.02) + end + update!.(p[0:end], :, :red) + sleep(0.5) + finish!.(p[1:end]) + sleep(0.1) + @test has_finished(p) - # println("Testing without main progressmeter and offset 0 finishes last") - # p = MultipleProgress(Progress.([100,100,100]), kwmain=(enabled=false,)) - # update!(p[0], 10, :red, desc="I shouldn't exist ") - # for i in 1:80 - # next!(p[1], desc="task a ") - # rand() < 0.7 && next!(p[2], desc="task b ") - # rand() < 0.4 && next!(p[3], desc="task c ") - # sleep(0.02) - # end - # sleep(0.1) - # @test !has_finished(p) - # finish!.(p[end:-1:1]) - # sleep(0.1) - # @test has_finished(p) + println("Testing without main progressmeter and offset 0 finishes last") + p = MultipleProgress(Progress.([100,100,100]), kwmain=(enabled=false,)) + update!(p[0], 10, :red, desc="I shouldn't exist ") + for i in 1:80 + next!(p[1], desc="task a ") + rand() < 0.7 && next!(p[2], desc="task b ") + rand() < 0.4 && next!(p[3], desc="task c ") + sleep(0.02) + end + sleep(0.1) + @test !has_finished(p) + finish!.(p[end:-1:1]) + sleep(0.1) + @test has_finished(p) - # println("Testing without main progressmeter and offset 0 finishes first (#215)") - # p = MultipleProgress(Progress.([100,100,100]), kwmain=(enabled=false,)) - # update!(p[0], 10, :red, desc="I shouldn't exist ") - # for i in 1:80 - # next!(p[1], desc="task a ") - # rand() < 0.7 && next!(p[2], desc="task b ") - # rand() < 0.4 && next!(p[3], desc="task c ") - # sleep(0.02) - # end - # sleep(0.1) - # @test !has_finished(p) - # finish!.(p[1:end]) - # sleep(0.1) - # @test has_finished(p) + println("Testing without main progressmeter and offset 0 finishes first (#215)") + p = MultipleProgress(Progress.([100,100,100]), kwmain=(enabled=false,)) + update!(p[0], 10, :red, desc="I shouldn't exist ") + for i in 1:80 + next!(p[1], desc="task a ") + rand() < 0.7 && next!(p[2], desc="task b ") + rand() < 0.4 && next!(p[3], desc="task c ") + sleep(0.02) + end + sleep(0.1) + @test !has_finished(p) + finish!.(p[1:end]) + sleep(0.1) + @test has_finished(p) - # println("Testing early close (should not display error)") - # p = MultipleProgress([Progress(100, desc="Close test")]) - # for i in 1:30 - # sleep(0.01) - # next!(p[1]) - # end - # sleep(0.1) - # @test !has_finished(p) - # close(p) - # sleep(0.1) - # @test has_finished(p) + println("Testing early close (should not display error)") + p = MultipleProgress([Progress(100, desc="Close test")]) + for i in 1:30 + sleep(0.01) + next!(p[1]) + end + sleep(0.1) + @test !has_finished(p) + close(p) + sleep(0.1) + @test has_finished(p) - # println("Testing errors in MultipleProgress (should display error)") - # p = MultipleProgress(Progress.([100]), kwmain=(desc="Error test", color=:red)) - # for i in 1:30 - # sleep(0.01) - # next!(p[1]) - # end - # next!(p[1], 1) - # sleep(2) - # @test has_finished(p) + println("Testing errors in MultipleProgress (should display error)") + p = MultipleProgress(Progress.([100]), kwmain=(desc="Error test", color=:red)) + for i in 1:30 + sleep(0.01) + next!(p[1]) + end + next!(p[1], 1) + sleep(2) + @test has_finished(p) println("Testing with showvalues (doesn't really work)") p = MultipleProgress(Progress.([100,100])) @@ -293,4 +293,50 @@ nworkers() == 1 && addprocs(4) finish!(p[1]) sleep(0.1) @test has_finished(p) -end + + N = 4*np + println("Testing adding $N progresses with $np workers") + p = MultipleProgress(Progress(N, "tasks done "); count_finishes=true) + sleep(0.1) + @test !has_finished(p) + pmap(1:N) do ip + L = rand(20:50) + ProgressMeter.addprogress!(p[ip], Progress, L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") + for _ in 1:L + sleep(0.05) + next!(p[ip]) + end + end + sleep(0.1) + @test has_finished(p) + + N = 4*np + println("Testing adding $N mixed progresses with $np workers") + p = MultipleProgress(ProgressUnknown(spinner=true)) + sleep(0.1) + @test !has_finished(p) + pmap(1:N) do ip + L = rand(20:50) + if ip%3 == 0 + ProgressMeter.addprogress!(p[ip], Progress, L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") + elseif ip%3 == 1 + ProgressMeter.addprogress!(p[ip], ProgressUnknown, desc=" $(string(ip,pad=2)) (id=$(myid())) ", spinner=true) + else + ProgressMeter.addprogress!(p[ip], ProgressThresh, 1/L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") + end + for i in 1:L + sleep(0.05) + if ip%3 == 2 + update!(p[ip], 1/i) + else + next!(p[ip]) + end + end + ip%3 == 1 && finish!(p[ip]) + end + sleep(0.1) + @test !has_finished(p) + finish!(p[0]) + sleep(0.1) + @test has_finished(p) +end \ No newline at end of file From 191737fc84ee9081960091ec32f833b1cee750b9 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Mon, 16 Aug 2021 15:48:31 +0200 Subject: [PATCH 22/39] fixes --- src/ProgressMeter.jl | 2 +- src/parallel_progress.jl | 17 ++++------------- test/runtests.jl | 32 ++++++++++++++++---------------- test/test_multiple.jl | 24 ++++++++++++++++-------- 4 files changed, 37 insertions(+), 38 deletions(-) diff --git a/src/ProgressMeter.jl b/src/ProgressMeter.jl index c941033..fbcf630 100644 --- a/src/ProgressMeter.jl +++ b/src/ProgressMeter.jl @@ -5,7 +5,7 @@ using Distributed export Progress, ProgressThresh, ProgressUnknown, BarGlyphs, next!, update!, cancel, finish!, @showprogress, progress_map, progress_pmap, ijulia_behavior, - MultipleProgress, ParallelProgress + MultipleProgress, ParallelProgress, addprogress! """ `ProgressMeter` contains a suite of utilities for displaying progress diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 2c7de08..7de8373 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -235,11 +235,7 @@ function runMultipleProgress(progresses::AbstractVector{<:AbstractProgress}, cancel(mainprogress, args...; kwt..., keep=false) break elseif f == PP_UPDATE - if !isempty(args) && args[1] == (:) # allow to update without knowing the counter with (:) - update!(mainprogress, valueorcounter(mainprogress), args[2:end]...; kwt..., keep=false) - else - update!(mainprogress, args...; kwt..., keep=false) - end + update!(mainprogress, args...; kwt..., keep=false) elseif f == PP_NEXT next!(mainprogress, args...; kwt..., keep=false) elseif f == PP_FINISH @@ -296,9 +292,8 @@ function runMultipleProgress(progresses::AbstractVector{<:AbstractProgress}, finish!(progresses[p]; keep=false) cancel(progresses[p], args...; kwt..., keep=false) elseif f == PP_UPDATE - if !isempty(args) # allow to update without knowing the counter with (:) + if !isempty(args) value = args[1] - value == (:) && (value = valueorcounter(progresses[p])) !count_overshoot && progresses[p] isa Progress && (value = min(value, progresses[p].n)) update!(progresses[p], value, args[2:end]...; kwt..., keep=false) else @@ -310,7 +305,7 @@ function runMultipleProgress(progresses::AbstractVector{<:AbstractProgress}, mainprogress.counter - prev_p_counter + progresses[p].counter; keep=false) end - if has_finished(progresses[p]) && !already_finished + if !already_finished && has_finished(progresses[p]) delete!(taken_offsets, progresses[p].offset) count_finishes && next!(mainprogress; keep=false) end @@ -355,10 +350,6 @@ isfakechannel(_) = false isfakechannel(::FakeChannel) = true isfakechannel(mc::MultipleChannel) = isfakechannel(mc.channel) -valueorcounter(p::Progress) = p.counter -valueorcounter(p::ProgressThresh) = p.val -valueorcounter(p::ProgressUnknown) = p.counter - """ addprogress!(mp[i], T::Type{<:AbstractProgress}, args...; kw...) @@ -371,7 +362,7 @@ p = MultipleProgress(Progress(N, "tasks done "); count_finishes=true) sleep(0.1) pmap(1:N) do i L = rand(20:50) - ProgressMeter.addprogress!(p[i], Progress, L, desc=" task \$i ") + addprogress!(p[i], Progress, L, desc=" task \$i ") for _ in 1:L sleep(0.05) next!(p[i]) diff --git a/test/runtests.jl b/test/runtests.jl index 0f39356..76b8840 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,22 +6,22 @@ if get(ENV, "CI", "false") == "true" display(versioninfo()) # among other things, this shows the number of threads end -# @testset "Core" begin -# include("core.jl") -# include("test.jl") -# end -# @testset "Show Values" begin -# include("test_showvalues.jl") -# end -# @testset "Mapping" begin -# include("test_map.jl") -# end -# @testset "Float" begin -# include("test_float.jl") -# end -# @testset "Threading" begin -# include("test_threads.jl") -# end +@testset "Core" begin + include("core.jl") + include("test.jl") +end +@testset "Show Values" begin + include("test_showvalues.jl") +end +@testset "Mapping" begin + include("test_map.jl") +end +@testset "Float" begin + include("test_float.jl") +end +@testset "Threading" begin + include("test_threads.jl") +end @testset "Parallel" begin include("test_parallel.jl") include("test_multiple.jl") diff --git a/test/test_multiple.jl b/test/test_multiple.jl index ca12564..33d3e09 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -28,7 +28,6 @@ nworkers() == 1 && addprocs(4) sleep(0.1) @test has_finished(p) - println("Testing MultipleProgress with custom titles and color") p = MultipleProgress( [Progress(100; color=:red, desc=" red "), @@ -105,7 +104,6 @@ nworkers() == 1 && addprocs(4) sleep(0.1) @test has_finished(p) - println("Testing early finish main progress") p = MultipleProgress(Progress.([100, 80])) for _ in 1:50 @@ -156,7 +154,7 @@ nworkers() == 1 && addprocs(4) update!(p[2], 51, :blue) else if i == 75 - update!(p[0], :, :yellow) + next!(p[0], :yellow; step=0) end next!(p[1]) next!(p[2]) @@ -195,7 +193,7 @@ nworkers() == 1 && addprocs(4) rand() < 0.1 && next!(p[3]) sleep(0.02) end - update!.(p[0:end], :, :red) + next!.(p[0:end], :red; step=0) sleep(0.5) finish!.(p[1:end]) sleep(0.1) @@ -264,6 +262,16 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) print("\n"^5) + println("Testing with enabled=false") + p = MultipleProgress(Progress.([100, 100]), Progress(200); enabled = false) + @test has_finished(p) + next!.(p[0:end]) + update!.(p[0:end]) + finish!.(p[0:end]) + cancel.(p[0:end]) + close(p) + addprogress!(p[3], Progress, 100) + println("Testing MultipleProgress with ProgressUnknown as mainprogress") p = MultipleProgress([Progress(100), ProgressUnknown(),ProgressThresh(0.01)]; count_finishes=false) for i in 1:100 @@ -301,7 +309,7 @@ nworkers() == 1 && addprocs(4) @test !has_finished(p) pmap(1:N) do ip L = rand(20:50) - ProgressMeter.addprogress!(p[ip], Progress, L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") + addprogress!(p[ip], Progress, L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") for _ in 1:L sleep(0.05) next!(p[ip]) @@ -318,11 +326,11 @@ nworkers() == 1 && addprocs(4) pmap(1:N) do ip L = rand(20:50) if ip%3 == 0 - ProgressMeter.addprogress!(p[ip], Progress, L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") + addprogress!(p[ip], Progress, L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") elseif ip%3 == 1 - ProgressMeter.addprogress!(p[ip], ProgressUnknown, desc=" $(string(ip,pad=2)) (id=$(myid())) ", spinner=true) + addprogress!(p[ip], ProgressUnknown, desc=" $(string(ip,pad=2)) (id=$(myid())) ", spinner=true) else - ProgressMeter.addprogress!(p[ip], ProgressThresh, 1/L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") + addprogress!(p[ip], ProgressThresh, 1/L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") end for i in 1:L sleep(0.05) From 91a0abf8b19894157fee958739c97f06782edaa4 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Mon, 16 Aug 2021 16:31:08 +0200 Subject: [PATCH 23/39] fix CI hopefully --- test/test_multiple.jl | 149 +++++++++++++++++++++--------------------- test/test_parallel.jl | 88 +++++++++++++------------ 2 files changed, 120 insertions(+), 117 deletions(-) diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 33d3e09..dab4097 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -13,13 +13,13 @@ nworkers() == 1 && addprocs(4) println("Testing MultipleProgress") println("Testing update!") p = MultipleProgress([Progress(100)]) - for _ in 1:25 - sleep(0.05) + for _ in 1:5 + sleep(0.2) next!(p[1]) end - update!(p[1], 75) - for _ in 76:99 - sleep(0.05) + update!(p[1], 95) + for _ in 96:99 + sleep(0.2) next!(p[1]) end sleep(0.1) @@ -30,12 +30,12 @@ nworkers() == 1 && addprocs(4) println("Testing MultipleProgress with custom titles and color") p = MultipleProgress( - [Progress(100; color=:red, desc=" red "), - Progress(100; desc=" default color ")], + [Progress(10; color=:red, desc=" red "), + Progress(10; desc=" default color ")], kwmain=(desc="yellow ", color=:yellow) ) - for _ in 1:99 - sleep(0.01) + for _ in 1:9 + sleep(0.1) next!.(p[1:2]) end sleep(0.1) @@ -45,9 +45,9 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing over-shooting and under-shooting") - p = MultipleProgress(Progress.([50, 140]), count_overshoot=false) - for _ in 1:100 - sleep(0.01) + p = MultipleProgress(Progress.([5, 14]), count_overshoot=false) + for _ in 1:10 + sleep(0.1) next!.(p[1:2]) end sleep(0.1) @@ -57,12 +57,12 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing over-shooting with count_overshoot") - p = MultipleProgress(Progress.([52, 153]), count_overshoot=true) + p = MultipleProgress(Progress.([5, 15]), count_overshoot=true) next!.(p[1:2]) sleep(0.1) next!.(p[1:2]) - for _ in 1:200 - sleep(0.01) + for _ in 1:15 + sleep(0.1) next!(p[2]) end sleep(0.1) @@ -82,9 +82,9 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing early cancel") - p = MultipleProgress(Progress.([100, 80])) - for _ in 1:50 - sleep(0.02) + p = MultipleProgress(Progress.([10, 8])) + for _ in 1:5 + sleep(0.2) next!(p[1]) next!(p[2]) end @@ -94,9 +94,9 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing early cancel main progress") - p = MultipleProgress(Progress.([100, 80])) - for _ in 1:50 - sleep(0.02) + p = MultipleProgress(Progress.([10, 8])) + for _ in 1:5 + sleep(0.2) next!(p[1]) next!(p[2]) end @@ -105,9 +105,9 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing early finish main progress") - p = MultipleProgress(Progress.([100, 80])) - for _ in 1:50 - sleep(0.02) + p = MultipleProgress(Progress.([10, 8])) + for _ in 1:5 + sleep(0.2) next!(p[1]) next!(p[2]) end @@ -116,8 +116,8 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing next! on main progress") - p = MultipleProgress([Progress(100)]) - for _ in 1:99 + p = MultipleProgress([Progress(10)]) + for _ in 1:9 sleep(0.02) next!(p[1]) end @@ -128,12 +128,12 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing bar remplacement with $np workers and pmap") - lengths = rand(20:50, 2*np) + lengths = rand(6:10, 2*np) progresses = [Progress(lengths[i], desc=" task $i ") for i in 1:2np] p = MultipleProgress(progresses) ids = pmap(1:2*np) do ip for _ in 1:lengths[ip] - sleep(0.05) + sleep(0.2) next!(p[ip]) end myid() @@ -143,17 +143,17 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing changing color with next! and update!") - p = MultipleProgress(Progress.([100,100])) - for i in 1:100 + p = MultipleProgress(Progress.([12,12])) + for i in 1:12 sleep(0.01) - if i == 25 + if i == 3 next!(p[1], :red) next!(p[2]) - elseif i == 50 + elseif i == 6 next!(p[1]) update!(p[2], 51, :blue) else - if i == 75 + if i == 9 next!(p[0], :yellow; step=0) end next!(p[1]) @@ -164,19 +164,19 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing changing desc with next! and update!") - p = MultipleProgress(Progress.([100,100,100])) - for i in 1:100 - sleep(0.05) - if i == 20 + p = MultipleProgress(Progress.([10,10,10])) + for i in 1:10 + sleep(0.5) + if i == 2 next!(p[1], desc="20% done ") next!.(p[2:3]) - elseif i == 40 + elseif i == 4 next!.(p[[1,3]]) - update!(p[2], 41, desc="40% done ") + update!(p[2], 5, desc="40% done ") else - if i == 60 + if i == 6 update!(p[0], desc="60% done ") - elseif i == 80 + elseif i == 8 update!(p[3], desc="80% done ") end next!.(p[1:3]) @@ -186,12 +186,12 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Update and finish all") - p = MultipleProgress(Progress.([100,100,100])) + p = MultipleProgress(Progress.([10,10,10])) for i in 1:100 - rand() < 0.5 && next!(p[1]) - rand() < 0.3 && next!(p[2]) - rand() < 0.1 && next!(p[3]) - sleep(0.02) + rand() < 0.9 && next!(p[1]) + rand() < 0.8 && next!(p[2]) + rand() < 0.7 && next!(p[3]) + sleep(0.2) end next!.(p[0:end], :red; step=0) sleep(0.5) @@ -200,13 +200,13 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing without main progressmeter and offset 0 finishes last") - p = MultipleProgress(Progress.([100,100,100]), kwmain=(enabled=false,)) - update!(p[0], 10, :red, desc="I shouldn't exist ") - for i in 1:80 + p = MultipleProgress(Progress.([10,10,10]), kwmain=(enabled=false,)) + update!(p[0], 1, :red, desc="I shouldn't exist ") + for i in 1:8 next!(p[1], desc="task a ") - rand() < 0.7 && next!(p[2], desc="task b ") - rand() < 0.4 && next!(p[3], desc="task c ") - sleep(0.02) + rand() < 0.9 && next!(p[2], desc="task b ") + rand() < 0.8 && next!(p[3], desc="task c ") + sleep(0.2) end sleep(0.1) @test !has_finished(p) @@ -215,13 +215,13 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing without main progressmeter and offset 0 finishes first (#215)") - p = MultipleProgress(Progress.([100,100,100]), kwmain=(enabled=false,)) - update!(p[0], 10, :red, desc="I shouldn't exist ") - for i in 1:80 + p = MultipleProgress(Progress.([10,10,10]), kwmain=(enabled=false,)) + update!(p[0], 1, :red, desc="I shouldn't exist ") + for i in 1:9 next!(p[1], desc="task a ") - rand() < 0.7 && next!(p[2], desc="task b ") - rand() < 0.4 && next!(p[3], desc="task c ") - sleep(0.02) + rand() < 0.9 && next!(p[2], desc="task b ") + rand() < 0.8 && next!(p[3], desc="task c ") + sleep(0.2) end sleep(0.1) @test !has_finished(p) @@ -230,9 +230,9 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing early close (should not display error)") - p = MultipleProgress([Progress(100, desc="Close test")]) - for i in 1:30 - sleep(0.01) + p = MultipleProgress([Progress(10, desc="Close test")]) + for i in 1:3 + sleep(0.1) next!(p[1]) end sleep(0.1) @@ -242,19 +242,20 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing errors in MultipleProgress (should display error)") - p = MultipleProgress(Progress.([100]), kwmain=(desc="Error test", color=:red)) - for i in 1:30 - sleep(0.01) + p = MultipleProgress(Progress.([10]), kwmain=(desc="Error test", color=:red)) + for i in 1:3 + sleep(0.1) next!(p[1]) end next!(p[1], 1) - sleep(2) + sleep(3) + get(ENV, "GITHUB_ACTIONS", "false") == "true" && sleep(10) @test has_finished(p) println("Testing with showvalues (doesn't really work)") - p = MultipleProgress(Progress.([100,100])) - for i in 1:100 - sleep(0.02) + p = MultipleProgress(Progress.([10,10])) + for i in 1:10 + sleep(0.2) next!(p[1]; showvalues = Dict(:i=>i)) next!(p[2]; showvalues = [i=>i, "longstring"=>"WXYZ"^i]) end @@ -273,9 +274,9 @@ nworkers() == 1 && addprocs(4) addprogress!(p[3], Progress, 100) println("Testing MultipleProgress with ProgressUnknown as mainprogress") - p = MultipleProgress([Progress(100), ProgressUnknown(),ProgressThresh(0.01)]; count_finishes=false) - for i in 1:100 - sleep(0.02) + p = MultipleProgress([Progress(10), ProgressUnknown(),ProgressThresh(0.1)]; count_finishes=false) + for i in 1:10 + sleep(0.2) next!(p[1]) next!(p[2]) update!(p[3], 1/i) @@ -287,11 +288,11 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing MultipleProgress with count_finishes") - p = MultipleProgress([ProgressUnknown(), Progress(50), ProgressThresh(0.05)]; + p = MultipleProgress([ProgressUnknown(), Progress(5), ProgressThresh(0.5)]; count_finishes=true, kwmain=(dt=0, desc="Tasks finished ")) update!(p[0], 0) - for i in 1:100 - sleep(0.02) + for i in 1:10 + sleep(0.2) next!(p[1]) next!(p[2]) update!(p[3], 1/i) diff --git a/test/test_parallel.jl b/test/test_parallel.jl index 6a0a8d4..dbf047d 100644 --- a/test/test_parallel.jl +++ b/test/test_parallel.jl @@ -25,13 +25,13 @@ nworkers() == 1 && addprocs(4) println("Testing update!") prog = Progress(100) p = ParallelProgress(prog) - for _ in 1:25 - sleep(0.05) + for _ in 1:5 + sleep(0.3) next!(p) end - update!(p, 75) - for _ in 76:100 - sleep(0.05) + update!(p, 95) + for _ in 96:100 + sleep(0.3) next!(p) end sleep(0.1) @@ -48,8 +48,8 @@ nworkers() == 1 && addprocs(4) println("Testing under-shooting") p = ParallelProgress(200) - for _ in 1:100 - sleep(0.01) + for _ in 1:10 + sleep(0.1) next!(p) end finish!(p) @@ -57,7 +57,7 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing rapid over-shooting") - p = ParallelProgress(100) + p = ParallelProgress(10) next!(p) sleep(0.1) for _ in 1:10000 @@ -67,9 +67,9 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing early cancel") - p = ParallelProgress(100) - for _ in 1:50 - sleep(0.02) + p = ParallelProgress(10) + for _ in 1:5 + sleep(0.2) next!(p) end cancel(p) @@ -77,20 +77,20 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing across $np workers with @distributed") - n = 20 #per core + n = 10 #per core p = ParallelProgress(n*np) @sync @distributed for _ in 1:n*np - sleep(0.05) + sleep(0.2) next!(p) end sleep(0.1) @test has_finished(p) println("Testing across $np workers with pmap") - n = 20 + n = 10 p = ParallelProgress(n*np) ids = pmap(1:n*np) do i - sleep(0.05) + sleep(0.2) next!(p) return myid() end @@ -99,13 +99,13 @@ nworkers() == 1 && addprocs(4) @test length(unique(ids)) == np println("Testing changing color with next! and update!") - p = ParallelProgress(100) - for i in 1:100 - sleep(0.05) - if i == 25 + p = ParallelProgress(10) + for i in 1:10 + sleep(0.5) + if i == 3 next!(p, :red) - elseif i == 50 - update!(p, 51, :blue) + elseif i == 6 + update!(p, 7, :blue) else next!(p) end @@ -114,13 +114,13 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing changing desc with next! and update!") - p = ParallelProgress(100) - for i in 1:100 - sleep(0.05) - if i == 25 - next!(p, desc="25% done ") - elseif i == 50 - update!(p, 51, desc="50% done ") + p = ParallelProgress(10) + for i in 1:10 + sleep(0.5) + if i == 3 + next!(p, desc="30% done ") + elseif i == 6 + update!(p, 7, desc="60% done ") else next!(p) end @@ -129,10 +129,10 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing with showvalues") - p = ParallelProgress(100) - for i in 1:100 - sleep(0.02) - if i < 50 + p = ParallelProgress(20) + for i in 1:20 + sleep(0.1) + if i < 10 next!(p; showvalues=Dict(:i => i, "longstring" => "ABCD"^i)) else next!(p; showvalues=() -> [(:i, "$i"), ("halfdone", true)]) @@ -143,8 +143,8 @@ nworkers() == 1 && addprocs(4) println("Testing with ProgressUnknown") p = ParallelProgress(ProgressUnknown()) - for i in 1:100 - sleep(0.02) + for i in 1:10 + sleep(0.2) next!(p) end sleep(0.5) @@ -157,17 +157,17 @@ nworkers() == 1 && addprocs(4) println("Testing with ProgressThresh") p = ParallelProgress(ProgressThresh(10)) - for i in 100:-1:0 - sleep(0.02) + for i in 20:-1:0 + sleep(0.2) update!(p, i) end sleep(0.1) @test has_finished(p) println("Testing early close (should not display error)") - p = ParallelProgress(100, desc="Close test") - for i in 1:30 - sleep(0.01) + p = ParallelProgress(10, desc="Close test") + for i in 1:3 + sleep(0.1) next!(p) end sleep(0.1) @@ -177,12 +177,14 @@ nworkers() == 1 && addprocs(4) @test has_finished(p) println("Testing errors in ParallelProgress (should display error)") - p = ParallelProgress(100, desc="Error test", color=:red) - for i in 1:30 - sleep(0.01) + @test_throws MethodError next!(Progress(10), 1) + p = ParallelProgress(10, desc="Error test", color=:red) + for i in 1:3 + sleep(0.1) next!(p) end next!(p, 1) - sleep(2) + sleep(3) + get(ENV, "GITHUB_ACTIONS", "false") == "true" && sleep(10) @test has_finished(p) end From 56827f7e5498f7b4211fcd1fb99c50c1b7ace2a3 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Tue, 17 Aug 2021 16:07:32 +0200 Subject: [PATCH 24/39] doc changes [skip-ci] --- src/parallel_progress.jl | 58 ++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 35 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 7de8373..e766470 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -109,7 +109,7 @@ Base.lastindex(mp::MultipleProgress) = mp.amount """ MultipleProgress(progresses::AbstractVector{<:AbstractProgress}, - mainprogress::AbstractProgress; + [mainprogress::AbstractProgress]; enabled = true, auto_close = true, count_finishes = false, @@ -118,7 +118,8 @@ Base.lastindex(mp::MultipleProgress) = mp.amount allows to call the `progresses` and `mainprogress` from different workers - `progresses`: contains the different progressbars - - `mainprogress`: main progressbar + - `mainprogress`: main progressbar, defaults to `Progress` or `ProgressUnknown`, + according to `count_finishes` and whether all progresses have known length or not - `enabled`: `enabled == false` doesn't show anything and doesn't open a channel - `auto_close`: if true, the channel will close when all progresses are finished, otherwise, when mainprogress finishes or with `close(p)` @@ -130,19 +131,19 @@ allows to call the `progresses` and `mainprogress` from different workers use p[i] to access the i-th progressmeter, and p[0] to access the main one # Example -```jldoctest -julia> using Distributed -julia> addprocs(2) -julia> @everywhere using ProgressMeter -julia> p = MultipleProgress(fill(10,5); desc="global ", kws=[(desc="task \$i ",) for i in 1:5]) - pmap(1:5) do x - for i in 1:10 - sleep(rand()) - next!(p[x]) - end - sleep(0.01) - myid() - end +```julia +using Distributed +addprocs(2) +@everywhere using ProgressMeter +p = MultipleProgress([Progress(10; desc="task \$i ") for i in 1:5], Progress(50; desc="global ")) +pmap(1:5) do x + for i in 1:10 + sleep(rand()) + next!(p[x]) + end + sleep(0.01) + myid() +end ``` """ function MultipleProgress(progresses::AbstractVector{<:AbstractProgress}, @@ -164,13 +165,6 @@ function MultipleProgress(progresses::AbstractVector{<:AbstractProgress}, return mp end -""" - MultipleProgress(progresses::AbstractVector{Progress}; count_finishes=false, kwmain=(), kw...) - -is equivalent to: - - MultipleProgress(progresses, Progress(main_length; kwmain...); count_finishes, kw...) -""" function MultipleProgress(progresses::AbstractVector{Progress}; count_finishes=false, kwmain=(), kw...) main_length = count_finishes ? length(progresses) : sum(p->p.n, progresses) @@ -178,33 +172,27 @@ function MultipleProgress(progresses::AbstractVector{Progress}; return MultipleProgress(progresses, mainprogress; count_finishes=count_finishes, kw...) end -""" - MultipleProgress(progresses::AbstractVector{<:AbstractProgress}; count_finishes=false, kwmain=(), kw...) - -is equivalent to: - - MultipleProgress(progresses, prog; count_finishes, kw...) - -with `prog` a `Progress` of the same length as `progresses` if `count_finishes == true`, -otherwise, `prog` is a `ProgressUnknown` -""" function MultipleProgress(progresses::AbstractVector{<:AbstractProgress}; count_finishes=false, kwmain=(), kw...) if count_finishes MultipleProgress(progresses, Progress(length(progresses); kwmain...); count_finishes=count_finishes, kw...) else - MultipleProgress(progresses, ProgressUnknown(;kwmain...); count_finishes=count_finishes, kw...) + MultipleProgress(progresses, ProgressUnknown(; kwmain...); count_finishes=count_finishes, kw...) end end """ - MultipleProgress(mainprogress::AbstractProgress; auto_close=false, kw...) + MultipleProgress(mainprogress=ProgressUnknown(); auto_close=false, kw...) is equivalent to MultipleProgress(AbstractProgress[], mainprogress; auto_close, kw...) + +See also: `addprogress!` + +Close the underlying channel with `finish!(p[0])` (finishes `mainprogress`) or `close(p)`. """ -function MultipleProgress(mainprogress::AbstractProgress; auto_close=false, kw...) +function MultipleProgress(mainprogress::AbstractProgress=ProgressUnknown(); auto_close=false, kw...) return MultipleProgress(AbstractProgress[], mainprogress; auto_close=auto_close, kw...) end From a5c5b2ccfac1435b00f31683647fc17697a07742 Mon Sep 17 00:00:00 2001 From: Marc Ittel Date: Mon, 20 Sep 2021 16:48:54 +0200 Subject: [PATCH 25/39] additional sleep time for CI --- test/test_multiple.jl | 78 +++++++++++++++++++++++-------------------- test/test_parallel.jl | 42 +++++++++++++---------- 2 files changed, 66 insertions(+), 54 deletions(-) diff --git a/test/test_multiple.jl b/test/test_multiple.jl index dab4097..2023e84 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -4,6 +4,13 @@ using ProgressMeter: has_finished nworkers() == 1 && addprocs(4) @everywhere using ProgressMeter +# additional time before checking if progressbar has finished during CI +if get(ENV, "CI", "false") == "true" + s = 0.1 +else + s = 1.0 +end + @testset "MultipleProgress() tests" begin np = nworkers() @@ -22,10 +29,10 @@ nworkers() == 1 && addprocs(4) sleep(0.2) next!(p[1]) end - sleep(0.1) + sleep(s) @test !has_finished(p) next!(p[1]) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing MultipleProgress with custom titles and color") @@ -38,10 +45,10 @@ nworkers() == 1 && addprocs(4) sleep(0.1) next!.(p[1:2]) end - sleep(0.1) + sleep(s) @test !has_finished(p) next!.(p[1:2]) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing over-shooting and under-shooting") @@ -50,10 +57,10 @@ nworkers() == 1 && addprocs(4) sleep(0.1) next!.(p[1:2]) end - sleep(0.1) + sleep(s) @test !has_finished(p) finish!(p[2]) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing over-shooting with count_overshoot") @@ -65,10 +72,10 @@ nworkers() == 1 && addprocs(4) sleep(0.1) next!(p[2]) end - sleep(0.1) + sleep(s) @test !has_finished(p) next!(p[2]) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing rapid over-shooting") @@ -78,7 +85,7 @@ nworkers() == 1 && addprocs(4) for i in 1:10000 next!(p[1]) end - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing early cancel") @@ -90,7 +97,7 @@ nworkers() == 1 && addprocs(4) end cancel(p[1]) finish!(p[2]) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing early cancel main progress") @@ -101,7 +108,7 @@ nworkers() == 1 && addprocs(4) next!(p[2]) end cancel(p[0]) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing early finish main progress") @@ -112,7 +119,7 @@ nworkers() == 1 && addprocs(4) next!(p[2]) end finish!(p[0]) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing next! on main progress") @@ -121,10 +128,10 @@ nworkers() == 1 && addprocs(4) sleep(0.02) next!(p[1]) end - sleep(0.1) + sleep(s) @test !has_finished(p) next!(p[0]) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing bar remplacement with $np workers and pmap") @@ -138,7 +145,7 @@ nworkers() == 1 && addprocs(4) end myid() end - sleep(0.1) + sleep(s) @test length(unique(ids)) == np @test has_finished(p) @@ -160,7 +167,7 @@ nworkers() == 1 && addprocs(4) next!(p[2]) end end - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing changing desc with next! and update!") @@ -182,7 +189,7 @@ nworkers() == 1 && addprocs(4) next!.(p[1:3]) end end - sleep(0.1) + sleep(s) @test has_finished(p) println("Update and finish all") @@ -196,7 +203,7 @@ nworkers() == 1 && addprocs(4) next!.(p[0:end], :red; step=0) sleep(0.5) finish!.(p[1:end]) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing without main progressmeter and offset 0 finishes last") @@ -208,10 +215,10 @@ nworkers() == 1 && addprocs(4) rand() < 0.8 && next!(p[3], desc="task c ") sleep(0.2) end - sleep(0.1) + sleep(s) @test !has_finished(p) finish!.(p[end:-1:1]) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing without main progressmeter and offset 0 finishes first (#215)") @@ -223,10 +230,10 @@ nworkers() == 1 && addprocs(4) rand() < 0.8 && next!(p[3], desc="task c ") sleep(0.2) end - sleep(0.1) + sleep(s) @test !has_finished(p) finish!.(p[1:end]) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing early close (should not display error)") @@ -235,10 +242,10 @@ nworkers() == 1 && addprocs(4) sleep(0.1) next!(p[1]) end - sleep(0.1) + sleep(s) @test !has_finished(p) close(p) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing errors in MultipleProgress (should display error)") @@ -248,8 +255,7 @@ nworkers() == 1 && addprocs(4) next!(p[1]) end next!(p[1], 1) - sleep(3) - get(ENV, "GITHUB_ACTIONS", "false") == "true" && sleep(10) + sleep(30s) @test has_finished(p) println("Testing with showvalues (doesn't really work)") @@ -259,7 +265,7 @@ nworkers() == 1 && addprocs(4) next!(p[1]; showvalues = Dict(:i=>i)) next!(p[2]; showvalues = [i=>i, "longstring"=>"WXYZ"^i]) end - sleep(0.1) + sleep(s) @test has_finished(p) print("\n"^5) @@ -281,10 +287,10 @@ nworkers() == 1 && addprocs(4) next!(p[2]) update!(p[3], 1/i) end - sleep(0.1) + sleep(s) @test !has_finished(p) finish!(p[2]) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing MultipleProgress with count_finishes") @@ -297,16 +303,16 @@ nworkers() == 1 && addprocs(4) next!(p[2]) update!(p[3], 1/i) end - sleep(0.1) + sleep(s) @test !has_finished(p) finish!(p[1]) - sleep(0.1) + sleep(s) @test has_finished(p) N = 4*np println("Testing adding $N progresses with $np workers") p = MultipleProgress(Progress(N, "tasks done "); count_finishes=true) - sleep(0.1) + sleep(s) @test !has_finished(p) pmap(1:N) do ip L = rand(20:50) @@ -316,13 +322,13 @@ nworkers() == 1 && addprocs(4) next!(p[ip]) end end - sleep(0.1) + sleep(s) @test has_finished(p) N = 4*np println("Testing adding $N mixed progresses with $np workers") p = MultipleProgress(ProgressUnknown(spinner=true)) - sleep(0.1) + sleep(s) @test !has_finished(p) pmap(1:N) do ip L = rand(20:50) @@ -343,9 +349,9 @@ nworkers() == 1 && addprocs(4) end ip%3 == 1 && finish!(p[ip]) end - sleep(0.1) + sleep(s) @test !has_finished(p) finish!(p[0]) - sleep(0.1) + sleep(s) @test has_finished(p) end \ No newline at end of file diff --git a/test/test_parallel.jl b/test/test_parallel.jl index dbf047d..24454be 100644 --- a/test/test_parallel.jl +++ b/test/test_parallel.jl @@ -4,6 +4,13 @@ using ProgressMeter: has_finished nworkers() == 1 && addprocs(4) @everywhere using ProgressMeter +# additional time before checking if progressbar has finished during CI +if get(ENV, "CI", "false") == "true" + s = 0.1 +else + s = 1.0 +end + @testset "ParallelProgress() tests" begin np = nworkers() @@ -19,7 +26,7 @@ nworkers() == 1 && addprocs(4) next!(p) end end - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing update!") @@ -34,7 +41,7 @@ nworkers() == 1 && addprocs(4) sleep(0.3) next!(p) end - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing over-shooting") @@ -43,7 +50,7 @@ nworkers() == 1 && addprocs(4) sleep(0.01) next!(p) end - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing under-shooting") @@ -53,7 +60,7 @@ nworkers() == 1 && addprocs(4) next!(p) end finish!(p) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing rapid over-shooting") @@ -63,7 +70,7 @@ nworkers() == 1 && addprocs(4) for _ in 1:10000 next!(p) end - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing early cancel") @@ -73,7 +80,7 @@ nworkers() == 1 && addprocs(4) next!(p) end cancel(p) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing across $np workers with @distributed") @@ -83,7 +90,7 @@ nworkers() == 1 && addprocs(4) sleep(0.2) next!(p) end - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing across $np workers with pmap") @@ -94,7 +101,7 @@ nworkers() == 1 && addprocs(4) next!(p) return myid() end - sleep(0.1) + sleep(s) @test has_finished(p) @test length(unique(ids)) == np @@ -110,7 +117,7 @@ nworkers() == 1 && addprocs(4) next!(p) end end - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing changing desc with next! and update!") @@ -125,7 +132,7 @@ nworkers() == 1 && addprocs(4) next!(p) end end - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing with showvalues") @@ -138,7 +145,7 @@ nworkers() == 1 && addprocs(4) next!(p; showvalues=() -> [(:i, "$i"), ("halfdone", true)]) end end - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing with ProgressUnknown") @@ -149,10 +156,10 @@ nworkers() == 1 && addprocs(4) end sleep(0.5) update!(p, 200) - sleep(0.5) + sleep(5s) @test !has_finished(p) finish!(p) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing with ProgressThresh") @@ -161,7 +168,7 @@ nworkers() == 1 && addprocs(4) sleep(0.2) update!(p, i) end - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing early close (should not display error)") @@ -170,10 +177,10 @@ nworkers() == 1 && addprocs(4) sleep(0.1) next!(p) end - sleep(0.1) + sleep(s) @test !has_finished(p) close(p) - sleep(0.1) + sleep(s) @test has_finished(p) println("Testing errors in ParallelProgress (should display error)") @@ -184,7 +191,6 @@ nworkers() == 1 && addprocs(4) next!(p) end next!(p, 1) - sleep(3) - get(ENV, "GITHUB_ACTIONS", "false") == "true" && sleep(10) + sleep(30s) @test has_finished(p) end From ce8ff7510cee8fa509de4c065a0ea2d66b87ac44 Mon Sep 17 00:00:00 2001 From: MarcMush <35898736+MarcMush@users.noreply.github.com> Date: Sat, 3 Jun 2023 14:37:51 +0200 Subject: [PATCH 26/39] fix ParallelProgress example --- src/parallel_progress.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index e766470..90fc354 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -29,7 +29,7 @@ using Distributed addprocs() @everywhere using ProgressMeter prog = ParallelProgress(10; desc="test ") -pmap(1:10) do +pmap(1:10) do i sleep(rand()) next!(prog) return myid() From 4cbbf857258037279a0088cf6da2f3143e5d853f Mon Sep 17 00:00:00 2001 From: MarcMush Date: Thu, 31 Aug 2023 21:21:52 +0200 Subject: [PATCH 27/39] change from vector to dict to store and reference progresses --- src/parallel_progress.jl | 152 +++++++++++++++++++++++---------------- test/test_multiple.jl | 16 ++--- test/test_parallel.jl | 8 +-- 3 files changed, 102 insertions(+), 74 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 90fc354..31658fa 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -101,15 +101,17 @@ Distributed.put!(mc::MultipleChannel, x) = put!(mc.channel, (mc.id, x...)) mutable struct MultipleProgress channel - amount::Int + main + keys::Vector end Base.getindex(mp::MultipleProgress, n) = ParallelProgress.(MultipleChannel.(Ref(mp.channel), n)) -Base.lastindex(mp::MultipleProgress) = mp.amount +Base.keys(mp::MultipleProgress) = mp.keys """ - MultipleProgress(progresses::AbstractVector{<:AbstractProgress}, + MultipleProgress(progresses::AbstractDict{T, <:AbstractProgress}, [mainprogress::AbstractProgress]; + main = T<:Number ? 0 : :main, enabled = true, auto_close = true, count_finishes = false, @@ -117,18 +119,18 @@ Base.lastindex(mp::MultipleProgress) = mp.amount auto_reset_timer = true) allows to call the `progresses` and `mainprogress` from different workers - - `progresses`: contains the different progressbars - - `mainprogress`: main progressbar, defaults to `Progress` or `ProgressUnknown`, + - `progresses`: contains the different progressbars, can also be an `AbstractVector` + - `mainprogress`: main progressbar, defaults to `Progress` or `ProgressUnknown`, \ according to `count_finishes` and whether all progresses have known length or not + - `main`: how `mainprogress` should be called. Defaults to `0` or `:main` - `enabled`: `enabled == false` doesn't show anything and doesn't open a channel - - `auto_close`: if true, the channel will close when all progresses are finished, otherwise, + - `auto_close`: if true, the channel will close when all progresses are finished, otherwise, \ when mainprogress finishes or with `close(p)` - - `count_finishes`: if false, main_progress will be the sum of the individual progressbars, + - `count_finishes`: if false, main_progress will be the sum of the individual progressbars, \ if true, it will be equal to the number of finished progressbars - `count_overshoot`: overshooting progressmeters will be counted in the main progressmeter - `auto_reset_timer`: tinit in progresses will be reset at first call -use p[i] to access the i-th progressmeter, and p[0] to access the main one # Example ```julia @@ -148,15 +150,48 @@ end """ function MultipleProgress(progresses::AbstractVector{<:AbstractProgress}, mainprogress::AbstractProgress; + main = 0, + kw...) + return MultipleProgress(Dict(pairs(progresses)), mainprogress; main=main, kw...) +end + +function MultipleProgress(progresses::AbstractVector{<:AbstractProgress}; main=0, kw...) + return MultipleProgress(Dict(pairs(progresses)); main=main, kw...) +end + +function MultipleProgress(progresses::AbstractDict{T,<:AbstractProgress}; + kwmain = (), + count_finishes = false, + kw...) where T + if count_finishes + mainprogress = Progress(length(progresses); kwmain...) + elseif valtype(progresses) <: Progress + mainprogress = Progress(sum(p->p.n, values(progresses)); kwmain...) + else + mainprogress = ProgressUnknown(; kwmain...) + end + return MultipleProgress(progresses, mainprogress; count_finishes=count_finishes, kw...) +end + +function MultipleProgress(progresses::AbstractDict{T,<:AbstractProgress}, + mainprogress::AbstractProgress; + main = T<:Number ? 0 : :main, + kwmain = (), enabled = true, auto_close = true, count_finishes = false, count_overshoot = false, - auto_reset_timer = true) - !enabled && return MultipleProgress(FakeChannel(), length(progresses)) + auto_reset_timer = true) where T + + !enabled && return MultipleProgress(FakeChannel(), main, collect(keys(progresses))) + + if main ∈ keys(progresses) + error("`main=$main` cannot be used in `progresses`") + end channel = RemoteChannel(() -> Channel{NTuple{4,Any}}(1024)) - mp = MultipleProgress(channel, length(progresses)) + + mp = MultipleProgress(channel, main, collect(keys(progresses))) @async runMultipleProgress(progresses, mainprogress, mp; auto_close=auto_close, count_finishes=count_finishes, @@ -165,59 +200,45 @@ function MultipleProgress(progresses::AbstractVector{<:AbstractProgress}, return mp end -function MultipleProgress(progresses::AbstractVector{Progress}; - count_finishes=false, kwmain=(), kw...) - main_length = count_finishes ? length(progresses) : sum(p->p.n, progresses) - mainprogress = Progress(main_length; kwmain...) - return MultipleProgress(progresses, mainprogress; count_finishes=count_finishes, kw...) -end - -function MultipleProgress(progresses::AbstractVector{<:AbstractProgress}; - count_finishes=false, kwmain=(), kw...) - if count_finishes - MultipleProgress(progresses, Progress(length(progresses); kwmain...); count_finishes=count_finishes, kw...) - else - MultipleProgress(progresses, ProgressUnknown(; kwmain...); count_finishes=count_finishes, kw...) - end -end - """ - MultipleProgress(mainprogress=ProgressUnknown(); auto_close=false, kw...) + MultipleProgress(mainprogress=ProgressUnknown(); main=0, auto_close=false, kw...) is equivalent to - MultipleProgress(AbstractProgress[], mainprogress; auto_close, kw...) + progresses = Dict{typeof(main), AbstractProgress}() + return MultipleProgress(progresses, mainprogress; main, auto_close, kw...) See also: `addprogress!` - -Close the underlying channel with `finish!(p[0])` (finishes `mainprogress`) or `close(p)`. """ -function MultipleProgress(mainprogress::AbstractProgress=ProgressUnknown(); auto_close=false, kw...) - return MultipleProgress(AbstractProgress[], mainprogress; auto_close=auto_close, kw...) +function MultipleProgress(mainprogress::AbstractProgress=ProgressUnknown(); main=0, auto_close=false, kw...) + progresses = Dict{typeof(main), AbstractProgress}() + return MultipleProgress(progresses, mainprogress; main=main, auto_close=auto_close, kw...) end -function runMultipleProgress(progresses::AbstractVector{<:AbstractProgress}, +function runMultipleProgress(progresses::AbstractDict{T,<:AbstractProgress}, mainprogress::AbstractProgress, mp::MultipleProgress; auto_close = true, count_finishes = false, count_overshoot = false, - auto_reset_timer = true) - for p in progresses - p.offset = -1 - end - - channel = mp.channel - taken_offsets = Set{Int}() + auto_reset_timer = true) where T max_offsets = 1 try + for p in values(progresses) + p.offset = -1 + end + + channel = mp.channel + taken_offsets = Set{Int}() + + mainprogress.offset = 0 # we must make sure that 2 progresses aren't updated at the same time, # that's why we use only one Channel - while !has_finished(mainprogress) && !(auto_close && all(has_finished, progresses)) + while !has_finished(mainprogress) p, f, args, kwt = take!(channel) - if p == 0 # main progressbar + if p == mp.main # main progressbar if f == PP_CANCEL finish!(mainprogress; keep=false) cancel(mainprogress, args...; kwt..., keep=false) @@ -229,27 +250,27 @@ function runMultipleProgress(progresses::AbstractVector{<:AbstractProgress}, elseif f == PP_FINISH finish!(mainprogress, args...; kwt..., keep=false) break + else + error("action `$f` not applicable to main progressmeter") end else # add progress - if f == MP_ADD_PROGRESS - resize!(progresses, max(length(progresses), p)) - mp.amount = length(progresses) - progresses[p] = Progress(args...; kwt..., offset=-1) - continue - elseif f == MP_ADD_UNKNOWN - resize!(progresses, max(length(progresses), p)) - mp.amount = length(progresses) - progresses[p] = ProgressUnknown(args...; kwt..., offset=-1) - continue - elseif f == MP_ADD_THRESH - resize!(progresses, max(length(progresses), p)) - mp.amount = length(progresses) - progresses[p] = ProgressThresh(args...; kwt..., offset=-1) + if f ∈ (MP_ADD_PROGRESS, MP_ADD_THRESH, MP_ADD_UNKNOWN) + if p ∈ keys(progresses) + error("key `$p` already in use") + end + if f == MP_ADD_PROGRESS + progresses[p] = Progress(args...; kwt..., offset=-1) + elseif f == MP_ADD_UNKNOWN + progresses[p] = ProgressUnknown(args...; kwt..., offset=-1) + else + progresses[p] = ProgressThresh(args...; kwt..., offset=-1) + end + push!(mp.keys, p) continue end - # first time calling progress p + # if first time calling progress p if progresses[p].offset == -1 # find first available offset offset = 1 @@ -282,20 +303,27 @@ function runMultipleProgress(progresses::AbstractVector{<:AbstractProgress}, elseif f == PP_UPDATE if !isempty(args) value = args[1] - !count_overshoot && progresses[p] isa Progress && (value = min(value, progresses[p].n)) + if !count_overshoot && progresses[p] isa Progress + value = min(value, progresses[p].n) # avoid overshoot + end update!(progresses[p], value, args[2:end]...; kwt..., keep=false) else update!(progresses[p]; kwt..., keep=false) end end - !count_finishes && update!(mainprogress, - mainprogress.counter - prev_p_counter + progresses[p].counter; keep=false) + if !count_finishes + update!(mainprogress, mainprogress.counter-prev_p_counter+progresses[p].counter; keep=false) + end end if !already_finished && has_finished(progresses[p]) delete!(taken_offsets, progresses[p].offset) count_finishes && next!(mainprogress; keep=false) + + if auto_close && all(has_finished, values(progresses)) + break + end end end end @@ -308,8 +336,8 @@ function runMultipleProgress(progresses::AbstractVector{<:AbstractProgress}, println() end finally - print("\n" ^ (max_offsets+1)) close(mp) + println("\n"^max_offsets) end end diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 2023e84..c8389ec 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -200,9 +200,9 @@ end rand() < 0.7 && next!(p[3]) sleep(0.2) end - next!.(p[0:end], :red; step=0) + next!.(p[0:3], :red; step=0) sleep(0.5) - finish!.(p[1:end]) + finish!.(p[1:3]) sleep(s) @test has_finished(p) @@ -217,7 +217,7 @@ end end sleep(s) @test !has_finished(p) - finish!.(p[end:-1:1]) + finish!.(p[3:-1:1]) sleep(s) @test has_finished(p) @@ -232,7 +232,7 @@ end end sleep(s) @test !has_finished(p) - finish!.(p[1:end]) + finish!.(p[1:3]) sleep(s) @test has_finished(p) @@ -272,10 +272,10 @@ end println("Testing with enabled=false") p = MultipleProgress(Progress.([100, 100]), Progress(200); enabled = false) @test has_finished(p) - next!.(p[0:end]) - update!.(p[0:end]) - finish!.(p[0:end]) - cancel.(p[0:end]) + next!.(p[0:2]) + update!.(p[0:2]) + finish!.(p[0:2]) + cancel.(p[0:2]) close(p) addprogress!(p[3], Progress, 100) diff --git a/test/test_parallel.jl b/test/test_parallel.jl index 24454be..4ebaba9 100644 --- a/test/test_parallel.jl +++ b/test/test_parallel.jl @@ -139,11 +139,11 @@ end p = ParallelProgress(20) for i in 1:20 sleep(0.1) - if i < 10 + # if i < 10 next!(p; showvalues=Dict(:i => i, "longstring" => "ABCD"^i)) - else - next!(p; showvalues=() -> [(:i, "$i"), ("halfdone", true)]) - end + # else # lazy broken ? + # next!(p; showvalues=() -> [(:i, "$i"), ("halfdone", true)]) + # end end sleep(s) @test has_finished(p) From c6721e4394f20308464abeed24745a73d6f67b95 Mon Sep 17 00:00:00 2001 From: MarcMush Date: Thu, 31 Aug 2023 21:34:56 +0200 Subject: [PATCH 28/39] add a test with dicts --- test/runtests.jl | 34 +++++++++++++++++----------------- test/test_multiple.jl | 24 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 76b8840..13f9945 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,23 +6,23 @@ if get(ENV, "CI", "false") == "true" display(versioninfo()) # among other things, this shows the number of threads end -@testset "Core" begin - include("core.jl") - include("test.jl") -end -@testset "Show Values" begin - include("test_showvalues.jl") -end -@testset "Mapping" begin - include("test_map.jl") -end -@testset "Float" begin - include("test_float.jl") -end -@testset "Threading" begin - include("test_threads.jl") -end +# @testset "Core" begin +# include("core.jl") +# include("test.jl") +# end +# @testset "Show Values" begin +# include("test_showvalues.jl") +# end +# @testset "Mapping" begin +# include("test_map.jl") +# end +# @testset "Float" begin +# include("test_float.jl") +# end +# @testset "Threading" begin +# include("test_threads.jl") +# end @testset "Parallel" begin - include("test_parallel.jl") + # include("test_parallel.jl") include("test_multiple.jl") end diff --git a/test/test_multiple.jl b/test/test_multiple.jl index c8389ec..68c743c 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -51,6 +51,30 @@ end sleep(s) @test has_finished(p) + println("Testing with Dicts, :main changes to red") + p = MultipleProgress( + Dict(:a => Progress(10, desc="task :a "), + :b => Progress(10, desc="task :b ")), + kwmain=(desc=":main ",) + ) + @test isempty(setdiff(p.keys, [:a, :b])) + @test p.main == :main + for _ in 1:9 + sleep(0.1) + next!(p[:a]) + next!(p[:b]) + end + update!(p[:main], 18, :red) + sleep(s) + @test !has_finished(p) + next!(p[:a]) + next!(p[:b]) + sleep(s) + @test has_finished(p) + + + + println("Testing over-shooting and under-shooting") p = MultipleProgress(Progress.([5, 14]), count_overshoot=false) for _ in 1:10 From 99df2918228a823218695c1100cbfd57a4c1ff6c Mon Sep 17 00:00:00 2001 From: MarcMush Date: Thu, 31 Aug 2023 21:48:54 +0200 Subject: [PATCH 29/39] add tests for main key error --- test/runtests.jl | 34 +++++++++++++++++----------------- test/test_multiple.jl | 15 ++++++++++++++- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 13f9945..76b8840 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,23 +6,23 @@ if get(ENV, "CI", "false") == "true" display(versioninfo()) # among other things, this shows the number of threads end -# @testset "Core" begin -# include("core.jl") -# include("test.jl") -# end -# @testset "Show Values" begin -# include("test_showvalues.jl") -# end -# @testset "Mapping" begin -# include("test_map.jl") -# end -# @testset "Float" begin -# include("test_float.jl") -# end -# @testset "Threading" begin -# include("test_threads.jl") -# end +@testset "Core" begin + include("core.jl") + include("test.jl") +end +@testset "Show Values" begin + include("test_showvalues.jl") +end +@testset "Mapping" begin + include("test_map.jl") +end +@testset "Float" begin + include("test_float.jl") +end +@testset "Threading" begin + include("test_threads.jl") +end @testset "Parallel" begin - # include("test_parallel.jl") + include("test_parallel.jl") include("test_multiple.jl") end diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 68c743c..1c2be7f 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -72,7 +72,20 @@ end sleep(s) @test has_finished(p) - + println("Testing forbidden main keys") + @test_throws ErrorException MultipleProgress( + Dict(:main => Progress(10, desc="task :a "), + :b => Progress(10, desc="task :b ")), + ) + @test_throws ErrorException MultipleProgress( + Dict(0 => Progress(10, desc="task :a "), + 1 => Progress(10, desc="task :b ")), + ) + @test_throws ErrorException MultipleProgress( + Dict(:a => Progress(10, desc="task :a "), + :b => Progress(10, desc="task :b ")); + main=:a + ) println("Testing over-shooting and under-shooting") From be0826acb573623b0dd6a02527b7870b2f5fd188 Mon Sep 17 00:00:00 2001 From: MarcMush Date: Thu, 31 Aug 2023 22:10:05 +0200 Subject: [PATCH 30/39] add test for duplicate key --- test/test_multiple.jl | 15 +++++++++++++-- test/test_parallel.jl | 4 ++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 1c2be7f..43a8207 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -6,9 +6,9 @@ nworkers() == 1 && addprocs(4) # additional time before checking if progressbar has finished during CI if get(ENV, "CI", "false") == "true" - s = 0.1 -else s = 1.0 +else + s = 0.1 end @testset "MultipleProgress() tests" begin @@ -87,6 +87,17 @@ end main=:a ) + println("Testing adding same key twice (should display error)") + p = MultipleProgress(; main="main") + addprogress!(p["a"], Progress, 10; color=:red) + for i in 1:5 + next!(p["a"]) + sleep(0.1) + end + println() + addprogress!(p["a"], Progress, 100) + sleep(5s) + @test has_finished(p) println("Testing over-shooting and under-shooting") p = MultipleProgress(Progress.([5, 14]), count_overshoot=false) diff --git a/test/test_parallel.jl b/test/test_parallel.jl index 4ebaba9..46fa04f 100644 --- a/test/test_parallel.jl +++ b/test/test_parallel.jl @@ -6,9 +6,9 @@ nworkers() == 1 && addprocs(4) # additional time before checking if progressbar has finished during CI if get(ENV, "CI", "false") == "true" - s = 0.1 -else s = 1.0 +else + s = 0.1 end @testset "ParallelProgress() tests" begin From f9ba27a25f87d39cd0403651d6a68a4c97745eb3 Mon Sep 17 00:00:00 2001 From: MarcMush Date: Thu, 31 Aug 2023 22:17:50 +0200 Subject: [PATCH 31/39] tweak test --- test/test_multiple.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 43a8207..8e0c70c 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -88,12 +88,14 @@ end ) println("Testing adding same key twice (should display error)") - p = MultipleProgress(; main="main") + p = MultipleProgress(; main="main", auto_close=false) addprogress!(p["a"], Progress, 10; color=:red) - for i in 1:5 - next!(p["a"]) + for i in 1:10 sleep(0.1) + next!(p["a"]) end + sleep(s) + @test !has_finished(p) println() addprogress!(p["a"], Progress, 100) sleep(5s) From 5437ce6132cc9c9ac301951091054f747dd3b26f Mon Sep 17 00:00:00 2001 From: MarcMush Date: Thu, 31 Aug 2023 22:30:55 +0200 Subject: [PATCH 32/39] 0.7 compat --- src/parallel_progress.jl | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 31658fa..7bb56b1 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -120,14 +120,11 @@ Base.keys(mp::MultipleProgress) = mp.keys allows to call the `progresses` and `mainprogress` from different workers - `progresses`: contains the different progressbars, can also be an `AbstractVector` - - `mainprogress`: main progressbar, defaults to `Progress` or `ProgressUnknown`, \ - according to `count_finishes` and whether all progresses have known length or not + - `mainprogress`: main progressbar, defaults to `Progress` or `ProgressUnknown`, according to `count_finishes` and whether all progresses have known length or not - `main`: how `mainprogress` should be called. Defaults to `0` or `:main` - `enabled`: `enabled == false` doesn't show anything and doesn't open a channel - - `auto_close`: if true, the channel will close when all progresses are finished, otherwise, \ - when mainprogress finishes or with `close(p)` - - `count_finishes`: if false, main_progress will be the sum of the individual progressbars, \ - if true, it will be equal to the number of finished progressbars + - `auto_close`: if true, the channel will close when all progresses are finished, otherwise, when mainprogress finishes or with `close(p)` + - `count_finishes`: if false, main_progress will be the sum of the individual progressbars, if true, it will be equal to the number of finished progressbars - `count_overshoot`: overshooting progressmeters will be counted in the main progressmeter - `auto_reset_timer`: tinit in progresses will be reset at first call From 74f3b95e5f0a654ad9078cc24d3507274459c1a6 Mon Sep 17 00:00:00 2001 From: MarcMush Date: Sat, 2 Sep 2023 12:13:20 +0200 Subject: [PATCH 33/39] update README.md --- README.md | 89 +++++++++++++++++++++++++++++++++---------- test/test_parallel.jl | 14 ++++++- 2 files changed, 81 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index c59a31b..2e9c053 100644 --- a/README.md +++ b/README.md @@ -359,44 +359,91 @@ remove **all** output from the cell. You can restore previous behavior by callin `ProgressMeter.ijulia_behavior(:append)`. You can enable it again by calling `ProgressMeter.ijulia_behavior(:clear)`, which will also disable the warning message. -### Tips for parallel programming +### Parallel programming with `ParallelProgress` -For remote parallelization, when multiple processes or tasks are being used for a computation, -the workers should communicate back to a single task for displaying the progress bar. This -can be accomplished with a `RemoteChannel`: +`ParallelProgress` wraps an `AbstractProgress` to allow updating from other workers, +by going through a `RemoteChannel` ```julia -using ProgressMeter using Distributed +@everywhere using ProgressMeter n_steps = 20 -p = Progress(n_steps) -channel = RemoteChannel(()->Channel{Bool}(), 1) +p = ParallelProgress(n_steps) # introduce a long-running dummy task to all workers @everywhere long_task() = sum([ 1/x for x in 1:100_000_000 ]) @time long_task() # a single execution is about 0.3 seconds -@sync begin # start two tasks which will be synced in the very end - # the first task updates the progress bar - @async while take!(channel) - next!(p) +@distributed (+) for i in 1:n_steps + long_task() + next!(p) + i^2 +end + +finish!(p) +``` + +Here, returning some number `i^2` and reducing it somehow `(+)`is necessary to make the distribution happen. +`finish!(p)` or `close(p)` makes sure that the underlying `Channel` is closed + +```julia +pu = ParallelProgress(ProgressUnknown(color=:red)) +pt = ParallelProgress(ProgressThresh(0.1, desc="Optimizing...")) +close(pu) +close(pt) +``` + +### Mutliple progressbars across multiple workers with `MutlipleProgress` + +`MultipleProgress` combines multiple progressbars, allowing them to update simultaneously across multiple workers + +```julia +using Distributed +addprocs(2) +@everywhere using ProgressMeter + +p = MultipleProgress([Progress(10; desc="task $i ") for i in 1:5], Progress(50; desc="global ")) +res = pmap(1:5) do i + for _ in 1:10 + sleep(rand()) + next!(p[i]) end + sleep(0.01) + myid() +end +close(p) +``` - # the second task does the computation - @async begin - @distributed (+) for i in 1:n_steps - long_task() - put!(channel, true) # trigger a progress bar update - i^2 - end - put!(channel, false) # this tells the printing task to finish +``` +global 100%|██████████████████████████████████████████████| Time: 0:00:24 +task 4 100%|██████████████████████████████████████████████| Time: 0:00:05 +task 5 100%|██████████████████████████████████████████████| Time: 0:00:05 +``` + +the progress bars can be passed in an `Vector` or a `Dict`. In the first case the main progress bar can be +updated with `p[0]`, otherwise `p[:main]` or by specifying the kwarg `main` when building the `MultipleProgress` + +If no main progress is given, one will be automatically generated. See `?MultipleProgress` for all available options. + +Additional progress bars can be added with `addprogress!` from another worker + +```julia +p = MultipleProgress(Progress(10; desc="tasks done "); count_finishes=true) +sleep(0.1) +pmap(1:10) do i + N = rand(20:50) + addprogress!(p[i], Progress, N; desc=" task $i ") + for _ in 1:N + next!(p[i]) + sleep(0.05) end end +close(p) ``` -Here, returning some number `i^2` and reducing it somehow `(+)` -is necessary to make the distribution happen. +the main progress bar can be a `Progress` or a `ProgessUnknown`, +while the other progresses can be any `AbstractProgress` ### `progress_map` diff --git a/test/test_parallel.jl b/test/test_parallel.jl index 46fa04f..eff0a4d 100644 --- a/test/test_parallel.jl +++ b/test/test_parallel.jl @@ -93,6 +93,18 @@ end sleep(s) @test has_finished(p) + println("Testing across $np workers with @distributed and reduce") + n = 10 #per core + p = ParallelProgress(n*np) + res = @distributed (+) for i in 1:n*np + sleep(0.2) + next!(p) + i^2 + end + @test res == sum(i->i^2, 1:n*np) + sleep(s) + @test has_finished(p) + println("Testing across $np workers with pmap") n = 10 p = ParallelProgress(n*np) @@ -141,7 +153,7 @@ end sleep(0.1) # if i < 10 next!(p; showvalues=Dict(:i => i, "longstring" => "ABCD"^i)) - # else # lazy broken ? + # else #? lazy broken? # next!(p; showvalues=() -> [(:i, "$i"), ("halfdone", true)]) # end end From d8977efdec08d424aeebb67dcdac456e920f54f4 Mon Sep 17 00:00:00 2001 From: MarcMush Date: Sun, 3 Sep 2023 13:01:54 +0200 Subject: [PATCH 34/39] improve codecov --- test/test_multiple.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 8e0c70c..15d66e6 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -30,9 +30,11 @@ end next!(p[1]) end sleep(s) + @test !ProgressMeter.isfakechannel(p[1]) @test !has_finished(p) next!(p[1]) sleep(s) + @test ProgressMeter.isfakechannel(p[1]) @test has_finished(p) println("Testing MultipleProgress with custom titles and color") @@ -57,7 +59,7 @@ end :b => Progress(10, desc="task :b ")), kwmain=(desc=":main ",) ) - @test isempty(setdiff(p.keys, [:a, :b])) + @test issetequal(keys(p), [:a, :b]) @test p.main == :main for _ in 1:9 sleep(0.1) From 639bceb5e8037fd12f3d1a1b17172718080e50f0 Mon Sep 17 00:00:00 2001 From: MarcMush Date: Sun, 3 Sep 2023 15:00:20 +0200 Subject: [PATCH 35/39] fix tests --- src/parallel_progress.jl | 4 +++- test/test_multiple.jl | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 7bb56b1..eaa78d5 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -358,8 +358,10 @@ has_finished(p::ProgressThresh) = p.triggered has_finished(p::ProgressUnknown) = p.done has_finished(p::ParallelProgress) = isfakechannel(p.channel) has_finished(p::MultipleProgress) = isfakechannel(p.channel) +has_finished(mc::MultipleChannel) = isfakechannel(mc) -isfakechannel(_) = false +isfakechannel(::AbstractChannel) = false +isfakechannel(::RemoteChannel) = false isfakechannel(::FakeChannel) = true isfakechannel(mc::MultipleChannel) = isfakechannel(mc.channel) diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 15d66e6..21c1ec0 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -30,11 +30,13 @@ end next!(p[1]) end sleep(s) - @test !ProgressMeter.isfakechannel(p[1]) + @test !ProgressMeter.isfakechannel(p.channel) + @test !ProgressMeter.isfakechannel(p[1].channel) @test !has_finished(p) next!(p[1]) sleep(s) - @test ProgressMeter.isfakechannel(p[1]) + @test ProgressMeter.isfakechannel(p.channel) + @test ProgressMeter.isfakechannel(p[1].channel) @test has_finished(p) println("Testing MultipleProgress with custom titles and color") From 9a1df84acd0b89879a81cf07d226f89c1a9c5b18 Mon Sep 17 00:00:00 2001 From: MarcMush Date: Sun, 3 Sep 2023 15:27:08 +0200 Subject: [PATCH 36/39] tests --- src/parallel_progress.jl | 1 - test/test_multiple.jl | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index eaa78d5..08a4ebc 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -358,7 +358,6 @@ has_finished(p::ProgressThresh) = p.triggered has_finished(p::ProgressUnknown) = p.done has_finished(p::ParallelProgress) = isfakechannel(p.channel) has_finished(p::MultipleProgress) = isfakechannel(p.channel) -has_finished(mc::MultipleChannel) = isfakechannel(mc) isfakechannel(::AbstractChannel) = false isfakechannel(::RemoteChannel) = false diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 21c1ec0..9987695 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -18,6 +18,11 @@ end @test all([@fetchfrom w @isdefined(ProgressMeter) for w in workers()]) println("Testing MultipleProgress") + + c = Channel() + @test !ProgressMeter.isfakechannel(c) + close(c) + println("Testing update!") p = MultipleProgress([Progress(100)]) for _ in 1:5 From 69b13a12e99378af6afe23a97da916f40ce0a39c Mon Sep 17 00:00:00 2001 From: MarcMush Date: Sun, 3 Sep 2023 15:48:26 +0200 Subject: [PATCH 37/39] fix --- test/test_multiple.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 9987695..6d07187 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -19,7 +19,7 @@ end println("Testing MultipleProgress") - c = Channel() + c = Channel(10) @test !ProgressMeter.isfakechannel(c) close(c) From 7094dd30f8e2bd8d6e424b7e0d3a553918031f26 Mon Sep 17 00:00:00 2001 From: MarcMush Date: Sat, 13 Jul 2024 10:40:47 +0200 Subject: [PATCH 38/39] way too many changes for one commit - `addprogress!` replaced with `setindex!` - add `ping(::ParallelProgress)` for testing - use `try_put!` for not erroring when over-shooting - remove `keys(::MultipleProgress)` - use of `waitfor` in tests for more intelligent waiting - use new keyword-only syntax --- src/ProgressMeter.jl | 2 +- src/parallel_progress.jl | 84 ++++++++++----------- test/runtests.jl | 1 + test/test_multiple.jl | 155 +++++++++++++-------------------------- test/test_parallel.jl | 80 ++++++++++---------- 5 files changed, 125 insertions(+), 197 deletions(-) diff --git a/src/ProgressMeter.jl b/src/ProgressMeter.jl index d285c37..3f5a8f1 100644 --- a/src/ProgressMeter.jl +++ b/src/ProgressMeter.jl @@ -5,7 +5,7 @@ using Distributed export Progress, ProgressThresh, ProgressUnknown, BarGlyphs, next!, update!, cancel, finish!, @showprogress, progress_map, progress_pmap, ijulia_behavior, - MultipleProgress, ParallelProgress, addprogress! + MultipleProgress, ParallelProgress """ `ProgressMeter` contains a suite of utilities for displaying progress diff --git a/src/parallel_progress.jl b/src/parallel_progress.jl index 08a4ebc..bdf32be 100644 --- a/src/parallel_progress.jl +++ b/src/parallel_progress.jl @@ -4,19 +4,35 @@ mutable struct ParallelProgress <: AbstractProgress end @enum ProgressAction begin + PP_ADD + PP_PING PP_NEXT PP_CANCEL PP_FINISH PP_UPDATE - MP_ADD_THRESH - MP_ADD_UNKNOWN - MP_ADD_PROGRESS end -next!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_NEXT, args, kw)); nothing) -cancel(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_CANCEL, args, kw)); nothing) -finish!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_FINISH, args, kw)); nothing) -update!(pp::ParallelProgress, args...; kw...) = (put!(pp.channel, (PP_UPDATE, args, kw)); nothing) +function try_put!(pp::ParallelProgress, x) + # try to put! in channel, but don't error if closed + try + put!(pp.channel, x) + catch e + if e == Base.closed_exception() || + e isa RemoteException && e.captured isa CapturedException && e.captured.ex == Base.closed_exception() + # progress has finished or been closed + pp.channel = FakeChannel() + else + rethrow() + end + end + return nothing +end + +ping(pp::ParallelProgress, args...; kw...) = try_put!(pp, (PP_PING, args, kw)) +next!(pp::ParallelProgress, args...; kw...) = try_put!(pp, (PP_NEXT, args, kw)) +cancel(pp::ParallelProgress, args...; kw...) = try_put!(pp, (PP_CANCEL, args, kw)) +finish!(pp::ParallelProgress, args...; kw...) = try_put!(pp, (PP_FINISH, args, kw)) +update!(pp::ParallelProgress, args...; kw...) = try_put!(pp, (PP_UPDATE, args, kw)) """ `ParallelProgress(n; kw...)` @@ -55,7 +71,9 @@ function ParallelProgress(progress::AbstractProgress) try while !has_finished(progress) && !has_finished(pp) f, args, kw = take!(channel) - if f == PP_NEXT + if f == PP_PING + print(args...) + elseif f == PP_NEXT next!(progress, args...; kw...) elseif f == PP_CANCEL cancel(progress, args...; kw...) @@ -102,11 +120,10 @@ Distributed.put!(mc::MultipleChannel, x) = put!(mc.channel, (mc.id, x...)) mutable struct MultipleProgress channel main - keys::Vector end Base.getindex(mp::MultipleProgress, n) = ParallelProgress.(MultipleChannel.(Ref(mp.channel), n)) -Base.keys(mp::MultipleProgress) = mp.keys +Base.setindex!(mp::MultipleProgress, p::AbstractProgress, n) = (put!(mp.channel, (n, PP_ADD, (p,), ())); nothing) """ MultipleProgress(progresses::AbstractDict{T, <:AbstractProgress}, @@ -180,7 +197,7 @@ function MultipleProgress(progresses::AbstractDict{T,<:AbstractProgress}, count_overshoot = false, auto_reset_timer = true) where T - !enabled && return MultipleProgress(FakeChannel(), main, collect(keys(progresses))) + !enabled && return MultipleProgress(FakeChannel(), main) if main ∈ keys(progresses) error("`main=$main` cannot be used in `progresses`") @@ -188,7 +205,7 @@ function MultipleProgress(progresses::AbstractDict{T,<:AbstractProgress}, channel = RemoteChannel(() -> Channel{NTuple{4,Any}}(1024)) - mp = MultipleProgress(channel, main, collect(keys(progresses))) + mp = MultipleProgress(channel, main) @async runMultipleProgress(progresses, mainprogress, mp; auto_close=auto_close, count_finishes=count_finishes, @@ -235,7 +252,9 @@ function runMultipleProgress(progresses::AbstractDict{T,<:AbstractProgress}, p, f, args, kwt = take!(channel) - if p == mp.main # main progressbar + if f == PP_PING + println(args...) + elseif p == mp.main # main progressbar if f == PP_CANCEL finish!(mainprogress; keep=false) cancel(mainprogress, args...; kwt..., keep=false) @@ -252,18 +271,14 @@ function runMultipleProgress(progresses::AbstractDict{T,<:AbstractProgress}, end else # add progress - if f ∈ (MP_ADD_PROGRESS, MP_ADD_THRESH, MP_ADD_UNKNOWN) + if f == PP_ADD if p ∈ keys(progresses) error("key `$p` already in use") end - if f == MP_ADD_PROGRESS - progresses[p] = Progress(args...; kwt..., offset=-1) - elseif f == MP_ADD_UNKNOWN - progresses[p] = ProgressUnknown(args...; kwt..., offset=-1) - else - progresses[p] = ProgressThresh(args...; kwt..., offset=-1) - end - push!(mp.keys, p) + newprog = args[1] + newprog.output = mainprogress.output + newprog.offset = -1 + progresses[p] = newprog continue end @@ -363,28 +378,3 @@ isfakechannel(::AbstractChannel) = false isfakechannel(::RemoteChannel) = false isfakechannel(::FakeChannel) = true isfakechannel(mc::MultipleChannel) = isfakechannel(mc.channel) - -""" - addprogress!(mp[i], T::Type{<:AbstractProgress}, args...; kw...) - -will add the progressbar `T(args..., kw...)` to the MultipleProgress `mp` at index `i` - -# Example - -```julia -p = MultipleProgress(Progress(N, "tasks done "); count_finishes=true) -sleep(0.1) -pmap(1:N) do i - L = rand(20:50) - addprogress!(p[i], Progress, L, desc=" task \$i ") - for _ in 1:L - sleep(0.05) - next!(p[i]) - end -end - -``` -""" -addprogress!(p::ParallelProgress, ::Type{Progress}, args...; kw...) = (put!(p.channel, (MP_ADD_PROGRESS, args, kw)); nothing) -addprogress!(p::ParallelProgress, ::Type{ProgressThresh}, args...; kw...) = (put!(p.channel, (MP_ADD_THRESH, args, kw)); nothing) -addprogress!(p::ParallelProgress, ::Type{ProgressUnknown}, args...; kw...) = (put!(p.channel, (MP_ADD_UNKNOWN, args, kw)); nothing) diff --git a/test/runtests.jl b/test/runtests.jl index 7b591b6..78c8b40 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,5 @@ using ProgressMeter +using Distributed using Test if get(ENV, "CI", "false") == "true" diff --git a/test/test_multiple.jl b/test/test_multiple.jl index 6d07187..56fe579 100644 --- a/test/test_multiple.jl +++ b/test/test_multiple.jl @@ -1,15 +1,5 @@ using Distributed -using ProgressMeter: has_finished - -nworkers() == 1 && addprocs(4) -@everywhere using ProgressMeter - -# additional time before checking if progressbar has finished during CI -if get(ENV, "CI", "false") == "true" - s = 1.0 -else - s = 0.1 -end +using ProgressMeter: has_finished, isfakechannel @testset "MultipleProgress() tests" begin @@ -20,7 +10,7 @@ end println("Testing MultipleProgress") c = Channel(10) - @test !ProgressMeter.isfakechannel(c) + @test !isfakechannel(c) close(c) println("Testing update!") @@ -34,14 +24,12 @@ end sleep(0.2) next!(p[1]) end - sleep(s) - @test !ProgressMeter.isfakechannel(p.channel) - @test !ProgressMeter.isfakechannel(p[1].channel) + @test !waitfor(()->isfakechannel(p.channel)) + @test !isfakechannel(p[1].channel) @test !has_finished(p) next!(p[1]) - sleep(s) - @test ProgressMeter.isfakechannel(p.channel) - @test ProgressMeter.isfakechannel(p[1].channel) + @test waitfor(()->isfakechannel(p.channel)) + @test isfakechannel(p[1].channel) @test has_finished(p) println("Testing MultipleProgress with custom titles and color") @@ -54,11 +42,9 @@ end sleep(0.1) next!.(p[1:2]) end - sleep(s) - @test !has_finished(p) + @test !waitfor(()->has_finished(p)) next!.(p[1:2]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing with Dicts, :main changes to red") p = MultipleProgress( @@ -66,20 +52,17 @@ end :b => Progress(10, desc="task :b ")), kwmain=(desc=":main ",) ) - @test issetequal(keys(p), [:a, :b]) @test p.main == :main for _ in 1:9 sleep(0.1) next!(p[:a]) next!(p[:b]) end - update!(p[:main], 18, :red) - sleep(s) + update!(p[:main], 18, color=:red) @test !has_finished(p) next!(p[:a]) next!(p[:b]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing forbidden main keys") @test_throws ErrorException MultipleProgress( @@ -98,17 +81,15 @@ end println("Testing adding same key twice (should display error)") p = MultipleProgress(; main="main", auto_close=false) - addprogress!(p["a"], Progress, 10; color=:red) + p["a"] = Progress(10; color=:red) for i in 1:10 sleep(0.1) next!(p["a"]) end - sleep(s) - @test !has_finished(p) + @test !waitfor(()->has_finished(p)) println() - addprogress!(p["a"], Progress, 100) - sleep(5s) - @test has_finished(p) + p["a"] = Progress(100) + @test waitfor(()->has_finished(p); tmax=5tmax) println("Testing over-shooting and under-shooting") p = MultipleProgress(Progress.([5, 14]), count_overshoot=false) @@ -116,11 +97,8 @@ end sleep(0.1) next!.(p[1:2]) end - sleep(s) - @test !has_finished(p) finish!(p[2]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing over-shooting with count_overshoot") p = MultipleProgress(Progress.([5, 15]), count_overshoot=true) @@ -131,21 +109,17 @@ end sleep(0.1) next!(p[2]) end - sleep(s) - @test !has_finished(p) next!(p[2]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing rapid over-shooting") p = MultipleProgress([Progress(10)], count_overshoot=true) next!(p[1]) sleep(0.1) - for i in 1:10000 + pmap(1:10000) do _ next!(p[1]) end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing early cancel") p = MultipleProgress(Progress.([10, 8])) @@ -156,8 +130,7 @@ end end cancel(p[1]) finish!(p[2]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing early cancel main progress") p = MultipleProgress(Progress.([10, 8])) @@ -167,8 +140,7 @@ end next!(p[2]) end cancel(p[0]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing early finish main progress") p = MultipleProgress(Progress.([10, 8])) @@ -178,8 +150,7 @@ end next!(p[2]) end finish!(p[0]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing next! on main progress") p = MultipleProgress([Progress(10)]) @@ -187,11 +158,8 @@ end sleep(0.02) next!(p[1]) end - sleep(s) - @test !has_finished(p) next!(p[0]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing bar remplacement with $np workers and pmap") lengths = rand(6:10, 2*np) @@ -204,30 +172,28 @@ end end myid() end - sleep(s) + @test waitfor(()->has_finished(p)) @test length(unique(ids)) == np - @test has_finished(p) println("Testing changing color with next! and update!") p = MultipleProgress(Progress.([12,12])) for i in 1:12 sleep(0.01) if i == 3 - next!(p[1], :red) + next!(p[1], color=:red) next!(p[2]) elseif i == 6 next!(p[1]) - update!(p[2], 51, :blue) + update!(p[2], 51, color=:blue) else if i == 9 - next!(p[0], :yellow; step=0) + next!(p[0]; color=:yellow, step=0) end next!(p[1]) next!(p[2]) end end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing changing desc with next! and update!") p = MultipleProgress(Progress.([10,10,10])) @@ -248,8 +214,7 @@ end next!.(p[1:3]) end end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Update and finish all") p = MultipleProgress(Progress.([10,10,10])) @@ -259,41 +224,35 @@ end rand() < 0.7 && next!(p[3]) sleep(0.2) end - next!.(p[0:3], :red; step=0) + next!.(p[0:3], color=:red, step=0) sleep(0.5) finish!.(p[1:3]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing without main progressmeter and offset 0 finishes last") p = MultipleProgress(Progress.([10,10,10]), kwmain=(enabled=false,)) - update!(p[0], 1, :red, desc="I shouldn't exist ") + update!(p[0], 1, color=:red, desc="I shouldn't exist ") for i in 1:8 next!(p[1], desc="task a ") rand() < 0.9 && next!(p[2], desc="task b ") rand() < 0.8 && next!(p[3], desc="task c ") sleep(0.2) end - sleep(s) - @test !has_finished(p) + @test !waitfor(()->has_finished(p)) finish!.(p[3:-1:1]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing without main progressmeter and offset 0 finishes first (#215)") p = MultipleProgress(Progress.([10,10,10]), kwmain=(enabled=false,)) - update!(p[0], 1, :red, desc="I shouldn't exist ") + update!(p[0], 1, color=:red, desc="I shouldn't exist ") for i in 1:9 next!(p[1], desc="task a ") rand() < 0.9 && next!(p[2], desc="task b ") rand() < 0.8 && next!(p[3], desc="task c ") sleep(0.2) end - sleep(s) - @test !has_finished(p) finish!.(p[1:3]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing early close (should not display error)") p = MultipleProgress([Progress(10, desc="Close test")]) @@ -301,11 +260,8 @@ end sleep(0.1) next!(p[1]) end - sleep(s) - @test !has_finished(p) close(p) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing errors in MultipleProgress (should display error)") p = MultipleProgress(Progress.([10]), kwmain=(desc="Error test", color=:red)) @@ -314,8 +270,7 @@ end next!(p[1]) end next!(p[1], 1) - sleep(30s) - @test has_finished(p) + @test waitfor(()->has_finished(p);tmax=10tmax) println("Testing with showvalues (doesn't really work)") p = MultipleProgress(Progress.([10,10])) @@ -324,8 +279,7 @@ end next!(p[1]; showvalues = Dict(:i=>i)) next!(p[2]; showvalues = [i=>i, "longstring"=>"WXYZ"^i]) end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) print("\n"^5) println("Testing with enabled=false") @@ -336,7 +290,7 @@ end finish!.(p[0:2]) cancel.(p[0:2]) close(p) - addprogress!(p[3], Progress, 100) + p[3] = Progress(100) println("Testing MultipleProgress with ProgressUnknown as mainprogress") p = MultipleProgress([Progress(10), ProgressUnknown(),ProgressThresh(0.1)]; count_finishes=false) @@ -346,11 +300,9 @@ end next!(p[2]) update!(p[3], 1/i) end - sleep(s) - @test !has_finished(p) + @test !waitfor(()->has_finished(p)) finish!(p[2]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing MultipleProgress with count_finishes") p = MultipleProgress([ProgressUnknown(), Progress(5), ProgressThresh(0.5)]; @@ -362,41 +314,35 @@ end next!(p[2]) update!(p[3], 1/i) end - sleep(s) @test !has_finished(p) finish!(p[1]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) N = 4*np println("Testing adding $N progresses with $np workers") - p = MultipleProgress(Progress(N, "tasks done "); count_finishes=true) - sleep(s) + p = MultipleProgress(Progress(N, desc="tasks done "); count_finishes=true) @test !has_finished(p) pmap(1:N) do ip L = rand(20:50) - addprogress!(p[ip], Progress, L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") + p[ip] = Progress(L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") for _ in 1:L sleep(0.05) next!(p[ip]) end end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) N = 4*np println("Testing adding $N mixed progresses with $np workers") p = MultipleProgress(ProgressUnknown(spinner=true)) - sleep(s) - @test !has_finished(p) pmap(1:N) do ip L = rand(20:50) if ip%3 == 0 - addprogress!(p[ip], Progress, L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") + p[ip] = Progress(L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") elseif ip%3 == 1 - addprogress!(p[ip], ProgressUnknown, desc=" $(string(ip,pad=2)) (id=$(myid())) ", spinner=true) + p[ip] = ProgressUnknown(desc=" $(string(ip,pad=2)) (id=$(myid())) ", spinner=true) else - addprogress!(p[ip], ProgressThresh, 1/L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") + p[ip] = ProgressThresh(1/L, desc=" $(string(ip,pad=2)) (id=$(myid())) ") end for i in 1:L sleep(0.05) @@ -408,9 +354,6 @@ end end ip%3 == 1 && finish!(p[ip]) end - sleep(s) - @test !has_finished(p) finish!(p[0]) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) end \ No newline at end of file diff --git a/test/test_parallel.jl b/test/test_parallel.jl index eff0a4d..4680c6f 100644 --- a/test/test_parallel.jl +++ b/test/test_parallel.jl @@ -6,9 +6,20 @@ nworkers() == 1 && addprocs(4) # additional time before checking if progressbar has finished during CI if get(ENV, "CI", "false") == "true" - s = 1.0 + dt = 0.1 + tmax = 5 else - s = 0.1 + dt = 0.1 + tmax = 1 +end + +function waitfor(f; tmax=tmax, dt=dt) + t0 = time() + while time() - t0 < tmax + f() && return true + sleep(dt) + end + return false end @testset "ParallelProgress() tests" begin @@ -26,8 +37,7 @@ end next!(p) end end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing update!") prog = Progress(100) @@ -41,8 +51,7 @@ end sleep(0.3) next!(p) end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing over-shooting") p = ParallelProgress(10) @@ -50,8 +59,7 @@ end sleep(0.01) next!(p) end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing under-shooting") p = ParallelProgress(200) @@ -60,8 +68,7 @@ end next!(p) end finish!(p) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing rapid over-shooting") p = ParallelProgress(10) @@ -70,8 +77,7 @@ end for _ in 1:10000 next!(p) end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing early cancel") p = ParallelProgress(10) @@ -80,8 +86,7 @@ end next!(p) end cancel(p) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing across $np workers with @distributed") n = 10 #per core @@ -90,8 +95,7 @@ end sleep(0.2) next!(p) end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing across $np workers with @distributed and reduce") n = 10 #per core @@ -102,8 +106,7 @@ end i^2 end @test res == sum(i->i^2, 1:n*np) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing across $np workers with pmap") n = 10 @@ -113,8 +116,7 @@ end next!(p) return myid() end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) @test length(unique(ids)) == np println("Testing changing color with next! and update!") @@ -122,30 +124,28 @@ end for i in 1:10 sleep(0.5) if i == 3 - next!(p, :red) + next!(p; color=:red) elseif i == 6 - update!(p, 7, :blue) + update!(p, 7; color=:blue) else next!(p) end end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing changing desc with next! and update!") p = ParallelProgress(10) for i in 1:10 sleep(0.5) if i == 3 - next!(p, desc="30% done ") + next!(p; desc="30% done ") elseif i == 6 - update!(p, 7, desc="60% done ") + update!(p, 7; desc="60% done ") else next!(p) end end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing with showvalues") p = ParallelProgress(20) @@ -157,8 +157,7 @@ end # next!(p; showvalues=() -> [(:i, "$i"), ("halfdone", true)]) # end end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing with ProgressUnknown") p = ParallelProgress(ProgressUnknown()) @@ -168,11 +167,9 @@ end end sleep(0.5) update!(p, 200) - sleep(5s) - @test !has_finished(p) + @test !waitfor(()->has_finished(p)) finish!(p) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing with ProgressThresh") p = ParallelProgress(ProgressThresh(10)) @@ -180,20 +177,17 @@ end sleep(0.2) update!(p, i) end - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing early close (should not display error)") - p = ParallelProgress(10, desc="Close test") + p = ParallelProgress(10; desc="Close test") for i in 1:3 sleep(0.1) next!(p) end - sleep(s) - @test !has_finished(p) + @test !waitfor(()->has_finished(p)) close(p) - sleep(s) - @test has_finished(p) + @test waitfor(()->has_finished(p)) println("Testing errors in ParallelProgress (should display error)") @test_throws MethodError next!(Progress(10), 1) @@ -203,6 +197,6 @@ end next!(p) end next!(p, 1) - sleep(30s) - @test has_finished(p) + @test waitfor(()->has_finished(p); tmax=10tmax) + sleep(1) end From 5c21f5deebb19b64f64e2da605c3fca2166e58c8 Mon Sep 17 00:00:00 2001 From: MarcMush Date: Sat, 13 Jul 2024 10:54:10 +0200 Subject: [PATCH 39/39] update readme --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0914616..592b522 100644 --- a/README.md +++ b/README.md @@ -373,6 +373,7 @@ by going through a `RemoteChannel` ```julia using Distributed +addprocs(2) @everywhere using ProgressMeter n_steps = 20 @@ -410,7 +411,9 @@ using Distributed addprocs(2) @everywhere using ProgressMeter -p = MultipleProgress([Progress(10; desc="task $i ") for i in 1:5], Progress(50; desc="global ")) +progs = [Progress(10; desc="task $i ") for i in 1:5] +mainprog = Progress(50; desc="global ") +p = MultipleProgress(progs, mainprog) res = pmap(1:5) do i for _ in 1:10 sleep(rand()) @@ -433,14 +436,14 @@ updated with `p[0]`, otherwise `p[:main]` or by specifying the kwarg `main` when If no main progress is given, one will be automatically generated. See `?MultipleProgress` for all available options. -Additional progress bars can be added with `addprogress!` from another worker +Additional progress bars can be added from another worker: ```julia p = MultipleProgress(Progress(10; desc="tasks done "); count_finishes=true) sleep(0.1) pmap(1:10) do i N = rand(20:50) - addprogress!(p[i], Progress, N; desc=" task $i ") + p[i] = Progress(N; desc=" task $i ") for _ in 1:N next!(p[i]) sleep(0.05)