Prune excluded directories from the index walk

CHANGELOG.md

@@ -18,6 +18,9 @@ Format: `<description> (by <contributor>, <pr number>)`
 - Indexing made significantly more performant (by @Keluaa, 735)
 - Support filtering by date and time with `"<date> <time>"` instead of
   `<date>T<time>` only (by @tjex, 743)
+- Exclude globs now also prune matching directories from the index walk instead
+  of descending into them, so large excluded trees (e.g. asset or vendored
+  directories) no longer slow down indexing (by @ehsash, 728)
 
 ## 0.15.5
 

docs/config/config-note.md

@@ -15,7 +15,8 @@ The `[note]` section from the [configuration file](config.md) is used to set the
     * Path to the [template](../notes/template.md) used to generate the note content.
     * Either an absolute path, or relative to `.zk/templates/`.
 * `exclude` (list of strings)
-    * List of [path globs](https://en.wikipedia.org/wiki/Glob_\(programming\)) excluded during note indexing.
+    * [Path globs](https://en.wikipedia.org/wiki/Glob_\(programming\)) to exclude during note indexing.
+    * Entire directories can be excluded (e.g.,  `your-directory` or `your-directory/**`).
 * `id-charset` (string)
     * Characters set used to [generate random IDs](../notes/note-id.md).
     * You can use:

internal/core/note_index.go

@@ -113,7 +113,7 @@ func (t *indexTask) execute(callback func(change paths.DiffChange)) (NoteIndexin
 	}
 	ignoredFiles := []IgnoredFile{}
 
-	shouldIgnorePath := func(path string) (bool, error) {
+	shouldIgnorePath := func(path string, isDir bool) (bool, error) {
 		notifyIgnored := func(reason string) {
 			ignoredFiles = append(ignoredFiles, IgnoredFile{
 				Path:   path,
@@ -126,13 +126,22 @@ func (t *indexTask) execute(callback func(change paths.DiffChange)) (NoteIndexin
 			return true, err
 		}
 
-		if filepath.Ext(path) != "."+group.Note.Extension {
+		// The note extension only applies to files. A directory is matched
+		// against the exclude globs alone, so that an excluded directory can be
+		// pruned from the walk instead of being traversed file by file.
+		if !isDir && filepath.Ext(path) != "."+group.Note.Extension {
 			notifyIgnored("expected extension \"" + group.Note.Extension + "\"")
 			return true, nil
 		}
 
 		for _, ignoreGlob := range group.ExcludeGlobs() {
-			matches, err := doublestar.PathMatch(ignoreGlob, path)
+			var matches bool
+			var err error
+			if isDir {
+				matches, err = globPrunesDir(ignoreGlob, path)
+			} else {
+				matches, err = doublestar.PathMatch(ignoreGlob, path)
+			}
 			if err != nil {
 				return true, fmt.Errorf("failed to match exclude glob %s to %s: %w", ignoreGlob, path, err)
 			}
@@ -217,3 +226,26 @@ func (t *indexTask) execute(callback func(change paths.DiffChange)) (NoteIndexin
 	}
 	return stats, nil
 }
+
+// globPrunesDir reports whether the exclude glob lets the whole directory be
+// pruned from the index walk, as opposed to only matching some of its entries.
+//
+// A directory is pruned only when the glob excludes its entire subtree: either
+// by naming the directory itself (e.g. "dir") or by covering its descendants
+// (e.g. "dir/**"). A glob that only matches the directory's immediate children
+// (e.g. "dir/*") must not prune it, or notes nested deeper — which the glob
+// never matches — would be dropped along with it.
+func globPrunesDir(glob, dir string) (bool, error) {
+	matches, err := doublestar.PathMatch(glob, dir)
+	if err != nil || !matches {
+		return false, err
+	}
+	// The glob names the directory exactly.
+	if glob == dir {
+		return true, nil
+	}
+	// The glob also matches a path below the directory, so it covers the whole
+	// subtree (as "dir/**" does) rather than only the immediate children (as
+	// "dir/*" does, which never matches a grandchild).
+	return doublestar.PathMatch(glob, dir+"/x/y")
+}

internal/util/paths/walk.go

@@ -9,8 +9,10 @@ import (
 )
 
 // Walk emits the metadata of each file stored in the directory if they pass
-// the given shouldIgnorePath closure. Hidden files and directories are ignored.
-func Walk(basePath string, logger util.Logger, notebookRoot string, shouldIgnorePath func(string) (bool, error)) <-chan Metadata {
+// the given shouldIgnorePath closure. Hidden files and directories are ignored,
+// as are directories rejected by shouldIgnorePath, which are pruned from the
+// walk instead of being traversed file by file.
+func Walk(basePath string, logger util.Logger, notebookRoot string, shouldIgnorePath func(string, bool) (bool, error)) <-chan Metadata {
 	c := make(chan Metadata, 50)
 	go func() {
 		defer close(c)
@@ -24,18 +26,30 @@ func Walk(basePath string, logger util.Logger, notebookRoot string, shouldIgnore
 			isHidden := strings.HasPrefix(filename, ".")
 			isNotebookRoot := filename == notebookRoot
 
+			path, err := filepath.Rel(basePath, abs)
+			if err != nil {
+				logger.Println(err)
+				return nil
+			}
+
 			if info.IsDir() {
 				if isHidden && !isNotebookRoot {
 					return filepath.SkipDir
 				}
+				// Prune excluded directories.
+				if !isNotebookRoot {
+					shouldIgnore, err := shouldIgnorePath(path, true)
+					if err != nil {
+						logger.Println(err)
+						return nil
+					}
+					if shouldIgnore {
+						return filepath.SkipDir
+					}
+				}
 
 			} else {
-				path, err := filepath.Rel(basePath, abs)
-				if err != nil {
-					logger.Println(err)
-					return nil
-				}
-				shouldIgnore, err := shouldIgnorePath(path)
+				shouldIgnore, err := shouldIgnorePath(path, false)
 				if err != nil {
 					logger.Println(err)
 					return nil

internal/util/paths/walk_test.go

@@ -2,6 +2,7 @@ package paths
 
 import (
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/zk-org/zk/internal/util"
@@ -12,7 +13,10 @@ import (
 func TestWalk(t *testing.T) {
 	var path = fixtures.Path("walk")
 
-	shouldIgnore := func(path string) (bool, error) {
+	shouldIgnore := func(path string, isDir bool) (bool, error) {
+		if isDir {
+			return false, nil
+		}
 		return filepath.Ext(path) != ".md", nil
 	}
 
@@ -41,7 +45,10 @@ func TestWalk(t *testing.T) {
 func TestWalkHidden(t *testing.T) {
 	var path = fixtures.Path(".walk-hidden")
 
-	shouldIgnore := func(path string) (bool, error) {
+	shouldIgnore := func(path string, isDir bool) (bool, error) {
+		if isDir {
+			return false, nil
+		}
 		return filepath.Ext(path) != ".md", nil
 	}
 
@@ -63,3 +70,43 @@ func TestWalkHidden(t *testing.T) {
 		"dir2/a.md",
 	})
 }
+
+// Walk should prune directories rejected by shouldIgnorePath, so an excluded
+// subtree is never traversed (rather than being filtered file by file).
+func TestWalkExcludedDirsArePruned(t *testing.T) {
+	var path = fixtures.Path("walk")
+
+	// Record every path the walker asks about, to prove it never descends into
+	// the excluded directory.
+	queried := make([]string, 0)
+	shouldIgnore := func(path string, isDir bool) (bool, error) {
+		queried = append(queried, path)
+		if isDir {
+			return path == "dir1", nil
+		}
+		return filepath.Ext(path) != ".md", nil
+	}
+
+	notebookRoot := filepath.Base(path)
+	actual := make([]string, 0)
+	for m := range Walk(path, &util.NullLogger, notebookRoot, shouldIgnore) {
+		actual = append(actual, m.Path)
+	}
+
+	// No note is emitted from the pruned dir1 subtree.
+	assert.Equal(t, actual, []string{
+		"Dir3/a.md",
+		"a.md",
+		"b.md",
+		"dir1 a space/a.md",
+		"dir2/a.md",
+	})
+
+	// The walker never even queried anything inside dir1 — proof it pruned the
+	// directory rather than walking and filtering each file.
+	for _, p := range queried {
+		if strings.HasPrefix(p, "dir1"+string(filepath.Separator)) {
+			t.Errorf("walker descended into pruned directory: %q", p)
+		}
+	}
+}

tests/issue-728.tesh

@@ -0,0 +1,36 @@
+# An exclude glob matching a directory prunes the whole directory from
+# indexing, instead of walking it and filtering each file individually.
+# https://github.com/zk-org/zk/issues/728
+
+$ cd blank
+
+$ mkdir dir
+$ mkdir dir/subdir
+$ touch banana.md
+$ touch orange.md
+$ touch dir/orange.md
+$ touch dir/subdir/apple.md
+
+# A bare directory name excludes the directory and its whole subtree.
+$ echo -e "[note]\n exclude = ['dir']" > .zk/config.toml
+$ zk index -v
+>- added banana.md
+>- added orange.md
+>- ignored dir: matched exclude glob "dir"
+>
+>Indexed 2 notes in 0s
+>  + 2 added
+>  ~ 0 modified
+>  - 0 removed
+
+# The "dir/**" subtree form prunes the directory as well.
+$ echo -e "[note]\n exclude = ['dir/**']" > .zk/config.toml
+$ zk index -vf
+>- modified banana.md
+>- modified orange.md
+>- ignored dir: matched exclude glob "dir/**"
+>
+>Indexed 2 notes in 0s
+>  + 0 added
+>  ~ 2 modified
+>  - 0 removed