tempodb: tolerate blocks removed during local listing

tempodb/backend/local/local.go

@@ -2,6 +2,7 @@ package local
 
 import (
 	"context"
+	"errors"
 	"io"
 	"io/fs"
 	"os"
@@ -199,6 +200,12 @@ func (rw *Backend) ListBlocks(_ context.Context, tenant string) (metas []uuid.UU
 	fff := os.DirFS(rootPath)
 	err = fs.WalkDir(fff, ".", func(path string, _ fs.DirEntry, err error) error {
 		if err != nil {
+			// Blocks can be deleted by retention while the blocklist is being
+			// walked. A vanished child is no longer part of the blocklist, so
+			// skip it instead of failing the entire tenant poll.
+			if path != "." && errors.Is(err, fs.ErrNotExist) {
+				return nil
+			}
 			return err
 		}
 

tempodb/backend/local/local_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"path/filepath"
 	"sync"
 	"testing"
 
@@ -81,6 +82,39 @@ func TestReadWrite(t *testing.T) {
 	assert.Len(t, cm, 1)
 }
 
+func TestListBlocksAllowsConcurrentBlockDeletion(t *testing.T) {
+	path := t.TempDir()
+	r, _, _, err := New(&Config{Path: path})
+	require.NoError(t, err)
+
+	tenant := "tenant"
+	blockID := uuid.New()
+	blockPath := filepath.Join(path, tenant, blockID.String())
+	require.NoError(t, os.MkdirAll(blockPath, 0o700))
+	require.NoError(t, os.WriteFile(filepath.Join(blockPath, backend.MetaName), []byte("meta"), 0o600))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for ctx.Err() == nil {
+			_ = os.RemoveAll(blockPath)
+			_ = os.MkdirAll(blockPath, 0o700)
+			_ = os.WriteFile(filepath.Join(blockPath, backend.MetaName), []byte("meta"), 0o600)
+		}
+	}()
+	t.Cleanup(func() {
+		cancel()
+		wg.Wait()
+	})
+
+	for range 1_000 {
+		_, _, err = r.ListBlocks(context.Background(), tenant)
+		require.NoError(t, err)
+	}
+}
+
 func TestShutdownLeavesTenantsWithBlocks(t *testing.T) {
 	r, w, _, err := New(&Config{
 		Path: t.TempDir(),