From dc3b54a028507e6237031e13c6ad387ff1ded19b Mon Sep 17 00:00:00 2001 From: DrChainsaw Date: Wed, 10 Jun 2026 22:45:58 +0200 Subject: [PATCH 1/2] tree-ops optimizations for large trees --- Project.toml | 18 ++-- src/FileTrees.jl | 2 + src/datastructure.jl | 106 +++++++++++++++++++---- src/parallelism.jl | 2 +- src/patterns.jl | 4 +- src/tree-ops.jl | 194 ++++++++++++++++++++++++++----------------- test/basics.jl | 7 +- test/taxi.jl | 2 +- 8 files changed, 228 insertions(+), 107 deletions(-) diff --git a/Project.toml b/Project.toml index b802a9c..1060649 100644 --- a/Project.toml +++ b/Project.toml @@ -1,18 +1,28 @@ name = "FileTrees" uuid = "72696420-646e-6120-6e77-6f6420746567" -authors = ["Shashi Gowda ", "Julian Samaroo "] version = "0.4.3" +authors = ["Shashi Gowda ", "Julian Samaroo "] [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f" Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" +OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +TaskLocalValues = "ed4db957-447d-4319-bfb6-7fa9ae7ecf34" + +[weakdeps] +Dagger = "d58978e5-989f-55fb-8d15-ea34adc7bf54" + +[extensions] +DaggerFileTreesExt = "Dagger" [compat] AbstractTrees = "0.3, 0.4" Dagger = "0.19" FilePathsBase = "0.9" Glob = "1.3" +OrderedCollections = "1.8" +TaskLocalValues = "0.1.3" julia = "1" [extras] @@ -25,9 +35,3 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Test", "Dates", "DataFrames", "CSV", "Distributed", "Dagger"] - -[weakdeps] -Dagger = "d58978e5-989f-55fb-8d15-ea34adc7bf54" - -[extensions] -DaggerFileTreesExt = "Dagger" diff --git a/src/FileTrees.jl b/src/FileTrees.jl index 9378dff..6c28b17 100644 --- a/src/FileTrees.jl +++ b/src/FileTrees.jl @@ -2,6 +2,8 @@ module FileTrees using FilePathsBase using AbstractTrees +using TaskLocalValues +using OrderedCollections: OrderedDict import AbstractTrees: children import Base: parent, getindex diff --git a/src/datastructure.jl b/src/datastructure.jl index 9a84c4b..eb4673f 100644 --- a/src/datastructure.jl +++ b/src/datastructure.jl @@ -20,11 +20,14 @@ struct NoValue end Copy over all fields from `tree`, but use any fields provided as keyword arguments. - FileTree(dirname::String; [sort]) + FileTree(dirname::String; [sort], [follow_symlinks]) Construct a `FileTree` to reflect directory from disk in the current working directory. If `sort=true` (the default) then each level of the tree will be lexicographically sorted. + +If `follow_symlinks=false` (the default) then symbolic links will not be followed. Setting this to `true` can dramatically speed up the construction, +even if no symbolic links are present as it elides a stat check inside `walkdir``. """ struct FileTree parent::Union{FileTree, Nothing} @@ -57,11 +60,11 @@ end FileTree(dir; kwargs...) = FileTree(nothing, dir; kwargs...) -function FileTree(parent, dir; sort=true, root="") +function FileTree(parent, dir; sort=true, root="", follow_symlinks=false) parent′ = FileTree(parent, dir, []) - for (path, dirs, files) in walkdir(joinpath(root, dir)) + for (path, dirs, files) in walkdir(joinpath(root, dir); follow_symlinks) for dir in dirs - push!(parent′.children, FileTree(parent′, dir; sort, root=path)) + push!(parent′.children, FileTree(parent′, dir; sort, root=path, follow_symlinks)) end for file in files push!(parent′.children, File(parent′, file)) @@ -119,13 +122,25 @@ Return a copy of node with `val` set as the value. setvalue(x::FileTree, val) = FileTree(x; value=val) """ - setparent(node::Union{FileTree, File}, parent) + setparent(node::Union{FileTree, File}, parent; [deep]) Return a copy of node with `parent` set as the parent. """ function setparent(x::FileTree, parent=parent(x)) - p = FileTree(x, parent=parent, children=copy(x.children)) - copy!(p.children, setparent.(x.children, (p,))) + p = FileTree(x, parent=parent, children=similar(x.children)) + @inbounds for i in eachindex(p.children, x.children) + p.children[i] = setparent(x.children[i], p) + end + p +end + +# Update children without copying them. Only for when we are dead certain we have +# created the whole tree from scratch +function setparent!(x::FileTree, parent=parent(x)) + p = FileTree(x, parent=parent, children=x.children) + @inbounds for i in eachindex(p.children, x.children) + p.children[i] = setparent!(x.children[i], p) + end p end @@ -138,7 +153,7 @@ Base.show(io::IO, d::FileTree) = AbstractTrees.print_tree(io, d) - `parent::Union{FileTree, Nothing}` -- The parent node. `nothing` if it's the root node. - `name::String` -- Name of the root. -- `value::Any` -- the value at the node, if no value is present, a `NoValue()` sentinal value. +- `value::Any` -- the value at the node, if no value is present, a `NoValue()` sentinel value. """ struct File parent::Union{Nothing, FileTree} @@ -184,6 +199,9 @@ setvalue(x::File, val) = File(x, value=val) setparent(x::File, parent=parent(x)) = File(x, parent=parent) +setparent!(f::File, args...;kws...) = setparent(f, args...) + + Base.getindex(tree::FileTree, i::Int) = tree.children[i] function Base.getindex(tree::FileTree, ix::Vector) @@ -279,22 +297,80 @@ maketree(node::Vector) = maketree("."=>node) Base.basename(d::Node) = Path(d.name) +# Internal struct for collecting the path from a Node with minimal overhead +# We reuse it so we don't need to allocate the data array each time we use it +# TaskLocalValue is used to thread safety +struct PathStringBuffer + data::Vector{UInt8} +end +PathStringBuffer() = PathStringBuffer(Vector{UInt8}[]) +const _PATH_STRING_BUFFER = TaskLocalValue{PathStringBuffer}(PathStringBuffer) + + +function _rpath(buf::PathStringBuffer, f::FileTrees.Node, delim) + pos = _rpath!(buf, f, 0, delim) + res = GC.@preserve buf unsafe_string(pointer(buf.data), pos) + empty!(buf.data) + res +end + +function _rpath!(buf, f::FileTrees.Node, pos::Int, delim::Vector{UInt8}) + if !isnothing(parent(f)) + pos = _rpath!(buf, parent(f), pos, delim) + append!(buf.data, delim) + pos += length(delim) + end + n = codeunits(name(f)) + append!(buf.data, n) + return pos + length(n) +end + +# Internal struct for collecting the Path segments from a Node with minimal overhead +# We reuse it so we don't need to allocate the path array each time we use it +# TaskLocalValue is used to thread safety +struct PathStringArray + path::Vector{String} +end +PathStringArray() = PathStringArray(sizehint!(String[], 4)) +const _PATH_STRING_ARRAY = TaskLocalValue{PathStringArray}(PathStringArray) + +function _rpath(arr::PathStringArray, f::FileTrees.Node) + len = _rpath!(arr, f, 1)-1 + ntuple(i -> arr.path[i], len) +end +function _rpath!(arr::PathStringArray, f::FileTrees.Node, n) + if isnothing(parent(f)) + if length(arr.path) < n + resize!(arr.path, n) + end + arr.path[1] = name(f) + return 2 + end + + pos = _rpath!(arr, parent(f), n+1) + arr.path[pos] = name(f) + pos+1 +end + """ Path(file::Union{File, FileTree) Returns an [`AbstractPath`](https://rofinn.github.io/FilePathsBase.jl/stable/design/#Path-Types-1) object which is the Path of the file from the root node leading up to this file. """ -function Path(d::Node) - parent(d) === nothing && return Path(d.name) - # Prevent that empty names become ".". Not 100 that this is always right, but at least is prevents mv from silently deleting them - isempty(d.name) && return Path(parent(d)) - Path(parent(d)) / Path(d.name) +function Path(d::Node) + arr =_PATH_STRING_ARRAY[] + len = _rpath!(arr, d, 1)-1 + Path(ntuple(i -> arr.path[i], len)) end -path(x::Node) = string(Path(x)) +const _PATH_SEPARATOR = Vector{UInt8}(Base.Filesystem.path_separator) +const _CANONICAL_PATH_SEPARATOR = [UInt8('/')] + +path(x::Node) = _rpath(_PATH_STRING_BUFFER[], x, _PATH_SEPARATOR) +canonical_path(x::Node) = _rpath(_PATH_STRING_BUFFER[], x, _CANONICAL_PATH_SEPARATOR) -Base.dirname(d::Node) = dirname(Path(d)) +Base.dirname(d::Node) = Path(parent(d)) Base.getindex(d::Node) = d.value diff --git a/src/parallelism.jl b/src/parallelism.jl index 1408891..fbe6407 100644 --- a/src/parallelism.jl +++ b/src/parallelism.jl @@ -96,7 +96,7 @@ baremodule Executor Use Julia's standard library `Threads` to spawn each computation in a separate task when passed as first argument to `exec`. - If the keyword argument `unwrap_exceptions` is set to `true`` (the default), any `TaskFailedExceptions` will be unwrapped, + If the keyword argument `unwrap_exceptions` is set to `true` (the default), any `TaskFailedExceptions` will be unwrapped, which typically results in less visual noise in case the computation throws an exception. The keyword argument `pool` (default `:default`) is given as first argument to `Threads.@spawn`. diff --git a/src/patterns.jl b/src/patterns.jl index 06d45f6..660dc1d 100644 --- a/src/patterns.jl +++ b/src/patterns.jl @@ -57,7 +57,7 @@ function Base.getindex(x::Union{FileTree, File}, regex::Regex) end function _regex_map(yes, no, t::FileTree, regex::Regex, toplevel=true) - if !toplevel && !isnothing(match(regex, canonical_path(Path(t)))) + if !toplevel && !isnothing(match(regex, canonical_path(t))) return yes(t) end @@ -78,7 +78,7 @@ function _regex_map(yes, no, t::FileTree, regex::Regex, toplevel=true) end function _regex_map(yes, no, t::File, regex::Regex, toplevel=false) - !isnothing(match(regex, canonical_path(Path(t)))) ? yes(t) : no(t) + !isnothing(match(regex, canonical_path(t))) ? yes(t) : no(t) end diff --git a/src/tree-ops.jl b/src/tree-ops.jl index a96befd..4423984 100644 --- a/src/tree-ops.jl +++ b/src/tree-ops.jl @@ -20,53 +20,67 @@ end # attach a node file at `path`; use `combine` to overwrite. function attach(t, path::PathLike, t′; combine=_merge_error) spath = Path(path).segments - t1 = foldl((x, acc) -> acc => [x], [t′; reverse(spath)...]) |> maketree - merge(t, maketree(name(t)=>[t1]); combine=combine) -end + + t1 = t′ + for parentname in Iterators.reverse(spath) + t1 = FileTree(nothing, parentname, [t1]) + end + t1 = setparent!(FileTree(nothing, name(t), [t1])) -# Attach all paths_and_ts to t assuming combine is associative to enable more parllelism -function assocattach(t, paths_and_ts::AbstractVector{<:Pair{<:PathLike, <:Node}}; combine=_merge_error) - isempty(paths_and_ts) && return t - length(paths_and_ts) == 1 && return attach(t, first(paths_and_ts)...; combine) - l = length(paths_and_ts) - m = div(l, 2) - t1 = assocattach(t, @view(paths_and_ts[begin:begin+m-1]); combine) - t2 = assocattach(t, @view(paths_and_ts[begin+m:end]); combine) - merge(t1, t2; combine=combine) + isempty(t) && return t1 + _merge(t, t1; combine=combine) end # rewrite `tree` according by performing string search and replace on every # path use `combine` to overwrite multiple files which map to the same path +# Note, callers may assume the returned tree has no references to anything in +# input and is therefore safe to mutate (e.g. with setparent!) function regex_rewrite_tree(tree, from_path, to_path, combine; associative=false) newtree = maketree(name(tree)=>[]) # Exclude the root from the matching because # 1) the public API where the pattern is relative to the root and # 2) attach wants a path relative to the root - rootoffset = length(canonical_path(Path(tree)))+2 + rootoffset = length(canonical_path(tree))+2 + + combinetree = OrderedDict{String, Vector}() + for x in Leaves(tree) + newpath = replace(@view(canonical_path(x)[rootoffset:end]), from_path => to_path) + push!(get!(() -> [], combinetree, newpath), x[]) + end + + buildcache = Dict{String, Node}("" => newtree) + cfun = maybe_lazy(combine) if associative - paths_and_ts = map(Leaves(tree)) do x - newname = replace(canonical_path(Path(x))[rootoffset:end], from_path => to_path) - dir = dirname(newname) - dir = isempty(dir) ? "." : dir - dir => rename(x, basename(newname)) + for (path, vals) in combinetree + val = length(vals) > 1 ? assocreduce(cfun, vals) : vals[1] + _append_tree!(buildcache, newtree, path, val) end - assocattach(newtree, paths_and_ts; combine) else + for (path, vals) in combinetree + val = length(vals) > 1 ? reduce(cfun, vals) : vals[1] + _append_tree!(buildcache, newtree, path, val) + end + end + newtree +end - for x in Leaves(tree) - newname = replace(canonical_path(Path(x))[rootoffset:end], from_path => to_path) - dir = dirname(newname) - dir = isempty(dir) ? "." : dir - newtree = attach(newtree, - dir, - rename(x, basename(newname)); - combine=combine) +function _append_tree!(buildcache, root, path, val) + parentpath, name = splitdir(path) + child = File(nothing, name, val) + while root !== child + nextpath, parentname = splitdir(parentpath) + parent = get!(buildcache, parentpath) do + FileTree(nothing, parentname, Node[]) end - newtree + push!(parent.children, child) + length(parent.children) == 1 || break # We have seen this node before + child = parent + parentpath = nextpath end end + # getindex but return the rooted tree function _getsubtree(x, path::PathLike) _getsubtree(x, GlobMatch(string(path))) @@ -99,15 +113,15 @@ function _combine(cs, combine) seen[name(c)] = length(out) end end - map(identity, out) + out end # flatten folders named "." into the parent directory. function normdots(x::FileTree; combine=_merge_error) - c2 = map(children(x)) do y + c2 = Iterators.flatmap(children(x)) do y z=normdots(y; combine=combine) name(z) == "." ? children(z) : [z] - end |> Iterators.flatten |> collect + end |> collect FileTree(x; children=_combine(c2, combine)) |> setparent end @@ -122,26 +136,32 @@ normdots(x::File; kw...) = x For each node in `t2` remove a node in `t1` at the same path if it exists. Returns the difference tree. """ -function Base.diff(t1::FileTree, t2::FileTree) +Base.diff(t1::FileTree, t2::FileTree) = setparent(_diff(t1, t2)) + +function _diff(t1::FileTree, t2::FileTree) if name(t1) == name(t2) - t2_names = name.(children(t2)) - cs = [] + + cs = sizehint!([], length(children(t1))) for x in children(t1) - idx = findfirst(==(name(x)), t2_names) - if !isnothing(idx) - if t2[idx] isa File - @assert x isa File - elseif x isa FileTree && t2[idx] isa FileTree - d = diff(x, t2[idx]) - if !isempty(d) - push!(cs, d) + nomatch=true + for y in children(t2) + if name(x) === name(y) + nomatch = false + if y isa File + @assert x isa File + elseif x isa FileTree && y isa FileTree + d = _diff(x, y) + if !isempty(d) + push!(cs, d) + end end end - else + end + if nomatch push!(cs, x) end end - FileTree(t1; children=cs) |> setparent + FileTree(t1; children=cs) else t1 end @@ -206,45 +226,45 @@ dir/ ``` """ function cp(t::FileTree, from_path::Regex, to_path::SubstitutionString; combine=_merge_error, associative=false) - matches = t[from_path] - isempty(matches) && return t - - newtree = regex_rewrite_tree(matches, from_path, to_path, combine; associative=associative) + newtree = regex_rewrite_tree(t, from_path, to_path, combine; associative=associative) merge(t, newtree; combine=combine) end -""" - merge(t1::FileTree, t2::FileTree; combine) - -Merge two FileTrees. If files at the same path contain values, the `combine` callback will -be called with their values to result in a new value. - -If one of the dirs does not have a value, its corresponding argument will be `NoValue()` -If any of the values is lazy, the output value is lazy as well. -""" -function Base.merge(t1::FileTree, t2::FileTree; combine=_merge_error, dotnorm=true) +function _merge(t1::FileTree, t2::FileTree; combine=_merge_error, dotnorm=true) bigt = if name(t1) == name(t2) - t2_names = name.(children(t2)) - t2_merged = zeros(Bool, length(t2_names)) - cs = [] + t2_merged = falses(length(children(t2))) + cs = sizehint!(Node[], length(children(t1)) + length(children(t2))) for x in children(t1) - idx = findfirst(==(name(x)), t2_names) - if !isnothing(idx) + nomatch=true + for idx in eachindex(children(t2)) y = t2[idx] - if y isa FileTree - push!(cs, merge(x, y; combine=combine, dotnorm=false)) - else - push!(cs, apply_combine(combine, x, y)) + if name(x) === name(y) + if y isa FileTree + push!(cs, _merge(x, y; combine=combine, dotnorm=false)) + else + push!(cs, apply_combine(combine, x, y)) + end + t2_merged[idx] = true + nomatch=false + break end - t2_merged[idx] = true - else + end + if nomatch push!(cs, x) end end - FileTree(t1; children=vcat(cs, children(t2)[map(!, t2_merged)])) + @inbounds for i in eachindex(children(t2)) + if !t2_merged[i] + push!(cs, children(t2)[i]) + end + end + FileTree(t1; children=cs) else FileTree(nothing, ".", [t1, t2], NoValue()) - end |> setparent + end + # dotnorm seems to mostly be for the old mv usecase where nodes targeted for deletion were renamed to ".". + # It is however (maybe unintentioally) in the public API and might play a role in the outermost else clause + # above dotnorm ? normdots(bigt; combine=combine) : bigt end @@ -295,13 +315,31 @@ dir/ ``` """ function mv(t::FileTree, from_path::Regex, to_path::SubstitutionString; combine=_merge_error, associative=false) - matches = t[from_path] - isempty(matches) && return t - - newtree = regex_rewrite_tree(matches, from_path, to_path, combine; associative=associative) - merge(diff(t, matches), newtree; combine=combine) + setparent!(regex_rewrite_tree(t, from_path, to_path, combine; associative=associative)) end -function Base.merge(x::Node, y::Node; combine=_merge_error) + +""" + mergewith(combine, t1::FileTree, t2::FileTree) + +Merge two FileTrees. If files at the same path contain values, the `combine` callback will +be called with their values to result in a new value. + +Equivalent to calling `merge(t1, t2; combine=combine)` +""" +Base.mergewith(combine, t1::Node, t2::Node; kwargs...) = merge(t1, t2; combine, kwargs...) + +""" + merge(t1::FileTree, t2::FileTree; combine) + +Merge two FileTrees. If files at the same path contain values, the `combine` callback will +be called with their values to result in a new value. + +If one of the dirs does not have a value, its corresponding argument will be `NoValue()` +If any of the values is lazy, the output value is lazy as well. +""" +Base.merge(x::Node, y::Node; kws...) = setparent(_merge(x, y; kws...)) # We can't be 100% sure _merge returns a copy here... + +function _merge(x::Node, y::Node; combine=_merge_error, kws...) name(x) == name(y) ? apply_combine(combine, x, y) : FileTree(nothing, ".", [x,y], NoValue()) end @@ -330,7 +368,7 @@ end Create an file node at `path` in the tree. Does not contain any value by default. """ function touch(t::FileTree, path::PathLike; value=NoValue()) - _mknode(File, t, path, value) + setparent(_mknode(File, t, path, value)) end diff --git a/test/basics.jl b/test/basics.jl index 83987c4..4b49786 100644 --- a/test/basics.jl +++ b/test/basics.jl @@ -85,6 +85,7 @@ import FileTrees: attach File(nothing, "data.csv", (yr, mo)) ) end end + FileTrees.setparent!(t1) @test isconsistent(t1) @@ -101,6 +102,7 @@ import FileTrees: attach File(nothing, string("0", mo) * ".csv", (yr, mo))) end end + FileTrees.setparent!(t5) @test isconsistent(t5) @@ -149,9 +151,8 @@ import FileTrees: attach # Should be a noop tmv = mv(t, r"(.*)", s"\1") - # The file named "." gets swallowed due to normdots which might be surprising. Maybe worth fixing in the future - @test name.(files(tmv)) == ["", "..", "a"] - @test reducevalues(vcat, tmv) == [1, 3, 4] + @test name.(files(tmv)) == ["", ".", "..", "a"] + @test reducevalues(vcat, tmv) == 1:4 end end diff --git a/test/taxi.jl b/test/taxi.jl index 9bbfe2d..0b99219 100644 --- a/test/taxi.jl +++ b/test/taxi.jl @@ -44,7 +44,7 @@ end @test isempty(setdiff(name.(children(t2)), ["yellow", "green"])) # this should throw - @test_throws ErrorException mv(dfs, r"^([^/]*)/([^/]*)/yellow.csv$", s"yellow.csv") + @test_throws FileTrees.UndefMergeError mv(dfs, r"^([^/]*)/([^/]*)/yellow.csv$", s"yellow.csv") yellow = mv(dfs, r"^([^/]*)/([^/]*)/yellow.csv$", s"yellow.csv", combine=vcat, associative=false)["yellow.csv"] @test get(yellow) isa DataFrame From 6cf91620e58668965287aea347a451d6f38dda6b Mon Sep 17 00:00:00 2001 From: DrChainsaw Date: Thu, 11 Jun 2026 11:30:20 +0200 Subject: [PATCH 2/2] Add parallel FileTree constructor and set follow_symlinks=true by default --- src/datastructure.jl | 32 ++++++++++++++++++++------------ test/taxi.jl | 7 +++++++ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/datastructure.jl b/src/datastructure.jl index eb4673f..b7da03a 100644 --- a/src/datastructure.jl +++ b/src/datastructure.jl @@ -14,20 +14,24 @@ struct NoValue end - `parent::Union{FileTree, Nothing}` -- The parent node. `nothing` if it's the root node. - `name::String` -- Name of the root. - `children::Vector` -- children -- `value::Any` -- the value at the node, if no value is present, a `NoValue()` sentinal value. +- `value::Any` -- the value at the node, if no value is present, a `NoValue()` sentinel value. FileTree(tree::FileTree; parent, name, children, value) Copy over all fields from `tree`, but use any fields provided as keyword arguments. - FileTree(dirname::String; [sort], [follow_symlinks]) + FileTree(dirname::String; [sort], [follow_symlinks], [paralleldepth]) Construct a `FileTree` to reflect directory from disk in the current working directory. If `sort=true` (the default) then each level of the tree will be lexicographically sorted. -If `follow_symlinks=false` (the default) then symbolic links will not be followed. Setting this to `true` can dramatically speed up the construction, -even if no symbolic links are present as it elides a stat check inside `walkdir``. +If `follow_symlinks=true` (the default) then symbolic links will be followed. Setting this to `true` actually speeds up the construction, +even if no symbolic links are present as it elides a stat check inside `walkdir`. Only reason to set this to `false` is if there exist +symbolic links that should not be followed. + +Subdirectories down to `paralleldepth` (default `0`) levels will be read in separate tasks, which can give significant speedups when the directory tree is +wide and I/O latency is high (e.g. NFS). """ struct FileTree parent::Union{FileTree, Nothing} @@ -60,23 +64,27 @@ end FileTree(dir; kwargs...) = FileTree(nothing, dir; kwargs...) -function FileTree(parent, dir; sort=true, root="", follow_symlinks=false) +function FileTree(parent, dir; sort=true, follow_symlinks=true, root="", paralleldepth=0) parent′ = FileTree(parent, dir, []) - for (path, dirs, files) in walkdir(joinpath(root, dir); follow_symlinks) - for dir in dirs - push!(parent′.children, FileTree(parent′, dir; sort, root=path, follow_symlinks)) + for (path, dirs, files) in walkdir(joinpath(root, dir); follow_symlinks) + + if paralleldepth > 0 && length(dirs) > 1 + tasks = [Threads.@spawn FileTree(parent′, d; sort, follow_symlinks, root=path, paralleldepth=paralleldepth-1) for d in dirs] + append!(parent′.children, fetch.(tasks)) + else + for d in dirs + push!(parent′.children, FileTree(parent′, d; sort, follow_symlinks, root=path, paralleldepth=0)) + end end for file in files push!(parent′.children, File(parent′, file)) end - if sort + if sort sort!(parent′.children; by=name) end - # this might look a bit silly, but we don't care much for walkdir recursing down the tree, only that it is - # much faster than readdir + isdir/isfile when it comes to separating files from directories break end - parent′ + parent′ end """ diff --git a/test/taxi.jl b/test/taxi.jl index 0b99219..4c74066 100644 --- a/test/taxi.jl +++ b/test/taxi.jl @@ -32,6 +32,13 @@ end lazy_dfs, lazy=false)) == 8 end + @testset "Parallel loading" begin + dfsp = FileTrees.load(FileTree(name(taxi_dir); paralleldepth=Inf)) do file + DataFrame(CSV.File(path(file))) + end + @test isequal(dfs, dfsp) + end + end @testset "mv" begin