Lots of improvements

Project.toml

@@ -1,6 +1,6 @@
 name = "ParallelProcessingTools"
 uuid = "8e8a01fc-6193-5ca1-a2f1-20776dae4199"
-version = "0.4.10"
+version = "0.4.11"
 
 [deps]
 ArgCheck = "dce04be8-c92d-5529-be00-80e4d2c0e197"

ext/ParallelProcessingToolsThreadPinningExt.jl

@@ -200,4 +200,4 @@ ThreadPinning.pinthreads(::ParallelProcessingTools.AutoThreadPinning) = nothing
 end # if _threadpinning_supported
 
 
-end # module ChangesOfVariablesInverseFunctionsExt
+end # module ParallelProcessingToolsThreadPinningExt

src/display.jl

@@ -24,18 +24,18 @@ const _g_unicode_occupancy = (
 
 
 """
-    ParallelProcessingTools.in_vscode_notebook():Bool
+    ParallelProcessingTools.in_vscode_notebook()::Bool
 
 Test if running within a Visual Studio Code notebook.
 """
 in_vscode_notebook() = haskey(ENV, "VSCODE_CWD")
 
 
 """
-    ParallelProcessingTools.printover(f_show::Function, io::IOBuffer)
+    ParallelProcessingTools.printover(f_show::Function, io::IO)
 
-Runs `f_show(tmpio)` with an IO buffer, then clears the required number of
-lines on `io` (typically `stdout`) and prints the output over them.
+Runs `f_show(tmpio)` with a temporary IO buffer, then clears the required
+number of lines on `io` (typically `stdout`) and prints the output over them.
 """
 function printover(f_show, io)
     vscode_nb_mode = in_vscode_notebook()

src/exceptions.jl

@@ -34,9 +34,8 @@ original_exception(err::RemoteException) = original_exception(err.captured.ex)
 """
     ParallelProcessingTools.onlyfirst_exception(err)
 
-Replaces `CompositeException`s with their first exception.
-
-Also employs `inner_exception` if `simplify` is `true`.
+Replaces `CompositeException`s with their first exception. Leaves other
+exceptions unchanged.
 """
 function onlyfirst_exception end
 export onlyfirst_exception
@@ -54,7 +53,7 @@ If multiple exceptions originate from parallel code in `expr`, only one
 is rethrown, and `TaskFailedException`s and `RemoteException`s are replaced
 by the original exceptions that caused them.
 
-See [`inner_exception`] and [`onlyfirst_exception`](@ref).
+See [`inner_exception`](@ref) and [`onlyfirst_exception`](@ref).
 """
 macro userfriendly_exceptions(expr)
     quote

src/fileio.jl

@@ -87,7 +87,7 @@ end
     ParallelProcessingTools.default_cache_dir()::String
 
 Returns the default cache directory, e.g. for [`write_files`](@ref) and
-`read_files`(@ref).
+[`read_files`](@ref).
 
 See also [`default_cache_dir!`](@ref).
 """
@@ -114,7 +114,7 @@ end
 
 Sets the default cache directory to `dir` and returns it.
 
-See also [`default_cache_dir!`](@ref).
+See also [`default_cache_dir`](@ref).
 """
 function default_cache_dir!(dir::AbstractString)
     lock(_g_default_cachedir_lock) do
@@ -180,8 +180,8 @@ _should_overwrite_if_necessary(::CreateOrIgnore) = false
 """
     CreateNew() isa WriteMode
 
-Indicates that new files should be created and to throw and eror if the files
-already exist.
+Indicates that new files should be created and that an error should be thrown
+if the files already exist.
 
 See [`WriteMode`](@ref) and [`write_files`](@ref).
 """
@@ -190,7 +190,7 @@ export CreateNew
 
 function _already_done(::CreateNew, target_fnames::AbstractVector{<:String}, any_pre_existing::Bool, all_pre_existing::Bool, loglevel::LogLevel)
     if any_pre_existing
-        throw(ErrorException("Some, but not all of $target_fnames exist, but not allowed to replace files"))
+        throw(ErrorException("Some or all of $target_fnames exist already, but not allowed to replace files"))
     else
         return false
     end
@@ -220,7 +220,7 @@ _should_overwrite_if_necessary(::CreateOrReplace) = true
 
 
 """
-    CreateOrIgnore() isa WriteMode
+    CreateOrModify() isa WriteMode
 
 Indicates that either new files should be created, or that existing files
 should be modified.
@@ -279,7 +279,7 @@ write to.
 With `ftw::FilesToWrite`, use `collect(ftw)` or `iterate(ftw)` to access the
 filenames to write to. Use `close(ftw)` or `close(ftw, true)` to close things
 in good order, indicating success, and use `close(ftw, false)` or
-`close(ftw, err:Exception)` to abort, indicating failure.
+`close(ftw, err::Exception)` to abort, indicating failure.
 
 See [`write_files`](@ref) for example code.
 
@@ -436,9 +436,9 @@ on the OS and file-system used).
 [`ModifyExisting()`](@ref).
 
 If a writing function `f_write` is given, calls
-`f_create(temporary_filenames...)`. If `f_create` doesn't throw an exception,
+`f_write(temporary_filenames...)`. If `f_write` doesn't throw an exception,
 the files `temporary_filenames` are renamed to `filenames`, otherwise
-the temporary files are are either deleted (if `delete_tmp_onerror` is `true)
+the temporary files are either deleted (if `delete_tmp_onerror` is `true`)
 or left in place (e.g. for debugging purposes).
 
 Set `ENV["JULIA_DEBUG"] = "ParallelProcessingTools"` to see a log of all
@@ -457,7 +457,7 @@ end
 files were (re-)written or `nothing` if there was nothing to do (depending
 on `mode`).
 
-If no writing funcion `f_write` is given then, `write_files` returns an object
+If no writing function `f_write` is given then, `write_files` returns an object
 of type [`FilesToWrite`](@ref) that holds the temporary filenames. Closing it
 will, like above, either rename temporary files to `filenames` or remove them.
 So
@@ -524,15 +524,15 @@ end
 function write_files(
     @nospecialize(filenames::AbstractString...);
     mode::WriteMode = CreateOrIgnore(),
-    use_cache::Bool = false, @nospecialize(cache_dirname::AbstractString = default_cache_dir()),
+    use_cache::Bool = false, @nospecialize(cache_dir::AbstractString = default_cache_dir()),
     create_dirs::Bool = true, delete_tmp_onerror::Bool=true,
     verbose::Bool = false
 )
     isempty(filenames) && return nothing
 
     loglevel = verbose ? Info : Debug
 
-    cache_dir = String(cache_dirname) # Fix type
+    cache_dir = String(cache_dir) # Fix type
     target_fnames = String[filenames...] # Fix type
     staging_fnames = String[]
     cache_fnames = String[]
@@ -643,9 +643,9 @@ function Base.close(ftr::FilesToRead, @nospecialize(reason::Union{Bool,Exception
     ftr._isopen[] = false
 
     if reason == true
-        # @debug "Reading from to $(ftr._source_fnames) was indicated to have succeeded."
+        # @debug "Reading from $(ftr._source_fnames) was indicated to have succeeded."
     elseif reason == false
-        @debug "Reading from to $(ftr._source_fnames) was indicated to have failed."
+        @debug "Reading from $(ftr._source_fnames) was indicated to have failed."
     else
         @debug "Aborted reading from $(ftr._source_fnames) due to exception:" reason
     end
@@ -704,7 +704,7 @@ result = read_files("foo.txt", "bar.txt", use_cache = true) do foo, bar
 end
 ```
 
-If no reading funcion `f_read` is given, then `read_files` returns an object
+If no reading function `f_read` is given, then `read_files` returns an object
 of type [`FilesToRead`](@ref) that holds the temporary filenames. Closing it
 will clean up temporary files, like described above. So
 

src/htcondor.jl

@@ -83,7 +83,7 @@ function _generate_condor_worker_script(filename, runmode::OnHTCondor, manager::
     additional_julia_flags = `$jl_threads_flag $jl_heap_size_hint_flag $julia_flags`
     worker_cmd = worker_local_startcmd(
         manager;
-        julia_flags = `$julia_flags $additional_julia_flags`,
+        julia_flags = additional_julia_flags,
         redirect_output = runmode.redirect_output, env = runmode.env
     )
     depot_path = join(runmode.julia_depot, ":")

src/localhost.jl

@@ -16,12 +16,12 @@ Example:
 runmode = OnLocalhost(n = 4)
 task, n = runworkers(runmode)
 
-Threads.@async begin
+@async begin
     wait(task)
-    @info "SLURM workers have terminated."
+    @info "Local workers have terminated."
 end
 
-@wait_while nprocs()-1 < n)
+@wait_while nprocs()-1 < n
 ```
 
 Workers can also be started manually, use
@@ -56,7 +56,7 @@ end
 function runworkers(runmode::OnLocalhost, manager::ElasticManager)
     start_cmd, m, n = worker_start_command(runmode, manager)
 
-    task = Threads.@async begin
+    task = @async begin
         processes = Base.Process[]
         for _ in 1:m
             push!(processes, open(start_cmd))

src/memory.jl

@@ -21,7 +21,7 @@ Values of `-1` mean unlimited.
 
 !!! note
     Currently only works on Linux, simply returns `(Int64(-1), Int64(-1))` on
-    other operationg systems.
+    other operating systems.
 """
 function memory_limit end
 export memory_limit
@@ -75,5 +75,5 @@ export memory_limit!
         return rlim[].cur, rlim[].max
     end
 else
-    memory_limit!(soft_limit::Integer, hard_limit::Integer) = Int64(-1), Int64(-1)
+    memory_limit!(soft_limit::Integer, hard_limit::Integer = Int64(-1)) = Int64(-1), Int64(-1)
 end

src/onthreads.jl

@@ -19,9 +19,8 @@ end
 
 # Adapted from Julia PR 32477:
 function _threading_run(func, threadsel::AbstractVector{<:Integer})
-    tasks = Vector{Task}(undef, length(eachindex(threadsel)))
-    for tid in threadsel
-        i = firstindex(tasks) + (tid - first(threadsel))
+    tasks = Vector{Task}(undef, length(threadsel))
+    for (i, tid) in enumerate(threadsel)
         tasks[i] = _run_on(Task(func), tid)
     end
     foreach(wait, tasks)
@@ -52,9 +51,12 @@ end
 """
     allthreads()
 
-Convencience function, returns an equivalent of `1:Base.Threads.nthreads()`.
+Returns the thread IDs of all threads in the default threadpool.
+
+Returns an equivalent of `1:Base.Threads.nthreads()` if no interactive
+threads are present.
 """
-allthreads() = Base.OneTo(Base.Threads.nthreads())
+allthreads() = Threads.threadpooltids(:default)
 export allthreads
 
 
@@ -65,11 +67,11 @@ Execute code in `expr` in parallel on the threads in `threadsel`.
 
 `threadsel` should be a single thread-ID or a range (or array) of thread-ids.
 If `threadsel == Base.Threads.threadid()`, `expr` is run on the current
-tread with only minimal overhead.
+thread with only minimal overhead.
 
 Example 1:
 
-```juliaexpr
+```julia
 tlsum = ThreadLocal(0.0)
 data = rand(100)
 @onthreads allthreads() begin
@@ -81,13 +83,13 @@ sum(getallvalues(tlsum)) ≈ sum(data)
 Example 2:
 
 ```julia
-# Assuming 4 threads:
+# Assuming 4 threads in the default threadpool:
 tl = ThreadLocal(42)
-threadsel = 2:3
+threadsel = allthreads()[2:3]
 @onthreads threadsel begin
     tl[] = Base.Threads.threadid()
 end
-getallvalues(tl)[threadsel] == [2, 3]
+getallvalues(tl)[2:3] == threadsel
 getallvalues(tl)[[1,4]] == [42, 42]
 ```
 """
@@ -99,7 +101,6 @@ export @onthreads
 
 function ThreadLocal{T}(f::Base.Callable) where {T}
     result = ThreadLocal{T}(undef)
-    result.value
     @onthreads allthreads() result.value[threadid()] = f()
     result
 end
@@ -115,12 +116,13 @@ multi-threaded tasks.
 
 Example:
 
-```
+```julia
 @mt_out_of_order begin
     a = foo()
     bar()
     c = baz()
 end
+```
 
 will run `a = foo()`, `bar()` and `c = baz()` in parallel and in arbitrary
 order, results of assignments will appear in the outside scope.
@@ -145,7 +147,6 @@ macro mt_out_of_order(ex)
                 exprs[i] = esc(exprs[i])
             end
         elseif exprs[i] isa Expr
-            ftvar = gensym()
             exprs[i] = :(push!($tasks, Base.Threads.@spawn($(esc(exprs[i])))))
             push!(handle_results, :(wait(popfirst!($tasks))))
         else