Add parallel_bisect.jl debugging tool

debugging/Project.toml

@@ -2,11 +2,13 @@ uuid = "21676e69-6767-7562-6564-696365736162"
 
 [deps]
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JLLPrefixes = "afc68a34-7891-4c5a-9da1-1c62935e7b0d"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 Sandbox = "9307e30f-c43e-9ca7-d17c-c2dc59df670d"

debugging/julia_checkout.jl

@@ -1,6 +1,7 @@
-using LibGit2, Scratch, SHA
+using Git, Scratch, SHA
 using Base: SHA1
 
+Base.SHA1(x::SHA1) = x
 struct GitCheckout
     repo_url::String
     commit::SHA1
@@ -15,41 +16,66 @@ struct GitCheckout
     end
 end
 
-function cached_git_clone(url::AbstractString;
-                          hash::Union{Nothing, SHA1} = nothing,
-                          downloads_dir::String = @get_scratch!("git-clones"))
-    repo_path = joinpath(downloads_dir, string(basename(url), "-", bytes2hex(sha256(url))))
+iscommit(repo::String, commit::String) = success(git(["-C", repo, "cat-file", "-e", commit]))
+default_clones_dir() = @get_scratch!("git_clones")
+
+"""
+    cached_git_clone(url::String; hash = nothing, verbose = false)
+
+Return the path to a local git clone of the given `url`.  If `hash` is given,
+then a cached git repository will not be updated if the commit already exists locally.
+"""
+function cached_git_clone(url::String;
+                          hash::Union{Nothing,String} = nothing,
+                          clones_dir::String = default_clones_dir(),
+                          verbose::Bool = false)
+    quiet_args = String[]
+    if !verbose
+        push!(quiet_args, "-q")
+    end
+
+    repo_path = joinpath(clones_dir, string(basename(url), "-", bytes2hex(sha256(url))))
     if isdir(repo_path)
-        LibGit2.with(LibGit2.GitRepo(repo_path)) do repo
-            # In some cases, we know the hash we're looking for, so only fetch() if
-            # this git repository doesn't contain the hash we're seeking
-            # this is not only faster, it avoids race conditions when we have
-            # multiple builders on the same machine all fetching at once.
-            if hash === nothing || !LibGit2.iscommit(bytes2hex(hash.bytes), repo)
-                LibGit2.fetch(repo)
-            end
+        if verbose
+            @info("Using cached git repository", url, repo_path)
+        end
+
+        # If we didn't just mercilessly obliterate the cached git repo, use it!
+        # In some cases, we know the hash we're looking for, so only fetch() if
+        # this git repository doesn't contain the hash we're seeking.
+        # this is not only faster, it avoids race conditions when we have
+        # multiple builders on the same machine all fetching at once.
+        if hash === nothing || !iscommit(repo_path, hash)
+            run(git(["-C", repo_path, "fetch", "-a", quiet_args...]))
         end
     else
+        if verbose
+            @info("Cloning git repository", url, repo_path)
+        end
         # If there is no repo_path yet, clone it down into a bare repository
-        LibGit2.clone(url, repo_path; isbare=true)
+        run(git(["clone", "--mirror", url, repo_path, quiet_args...]))
     end
     return repo_path
 end
 
 function get_checkout(repo_url::String,
                       hash::SHA1,
                       checkout_dir::String;
-                      downloads_dir::String = @get_scratch!("git-clones"))
+                      clones_dir::String = default_clones_dir())
     # Clone down (or verify that we've cached) a repository that contains the requested commit
-    repo_path = cached_git_clone(repo_url; hash, downloads_dir)
+    repo_path = cached_git_clone(repo_url; hash=bytes2hex(hash.bytes), clones_dir)
 
-    # Checkout the desired commit to a temporary directory that `reman` will clean up:
-    LibGit2.with(LibGit2.clone(repo_path, checkout_dir)) do cloned_repo
-        LibGit2.checkout!(cloned_repo, bytes2hex(hash.bytes))
-    end
-    return checkout_dir
+    run(git(["clone", "--shared", repo_path, checkout_dir, "-q"]))
+    run(git(["-C", checkout_dir, "checkout", bytes2hex(hash.bytes), "-q"]))
 end
 
 function get_checkout(gc::GitCheckout, checkout_prefix::String; kwargs...)
     return get_checkout(gc.repo_url, gc.commit, joinpath(checkout_prefix, gc.checkout_path); kwargs...)
 end
+
+function get_commits_between(repo_url::String, before::SHA1, after::SHA1;
+                             clones_dir::String = default_clones_dir())
+    repo_path = cached_git_clone(repo_url; hash=bytes2hex(after.bytes), clones_dir)
+    lines = readchomp(git(["-C", repo_path, "log", "--reverse", "--pretty=format:%H", string(bytes2hex(before.bytes), "^!"), bytes2hex(after.bytes)]))
+    return [parse(SHA1, line) for line in split(lines)]
+end

debugging/parallel_bisect/README.md

@@ -0,0 +1,12 @@
+# parallel_bisect.jl
+
+Our bootstrap process is single-threaded and slow.
+When bisecting an issue, it sure would be nice if we could make use of all of those extra cores, wouldn't it?
+
+```
+julia -t5 --project parallel_bisect.jl <good_sha> <bad_sha> script_to_test_issue.jl
+```
+
+Build errors get skipped.
+The first run will verify your script on the given good and bad gitsha's to ensure that it reacts properly.
+Use the `-t` argument to Julia to specify how many jobs should run (each job will use enuogh threads to hopefully saturate your machine without completely destroying it).