diff --git a/.gc/settings.json b/.gc/settings.json new file mode 100644 index 000000000..de6cb7ec1 --- /dev/null +++ b/.gc/settings.json @@ -0,0 +1,45 @@ +{ + "awaySummaryEnabled": false, + "editorMode": "normal", + "enableAllProjectMcpServers": true, + "hooks": { + "PreCompact": [ + { + "hooks": [ + { + "command": "export PATH=\"$HOME/go/bin:$HOME/.local/bin:$PATH\" && gc handoff --auto \"context cycle\"", + "type": "command" + } + ], + "matcher": "" + } + ], + "SessionStart": [ + { + "hooks": [ + { + "command": "export PATH=\"$HOME/go/bin:$HOME/.local/bin:$PATH\" && GC_MANAGED_SESSION_HOOK=1 GC_HOOK_EVENT_NAME=SessionStart gc prime --hook --hook-format codex", + "type": "command" + } + ], + "matcher": "startup" + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "command": "export PATH=\"$HOME/go/bin:$HOME/.local/bin:$PATH\" && gc hook run --timeout 15s --timeout-exit-code 0 -- nudge drain --inject", + "type": "command" + }, + { + "command": "export PATH=\"$HOME/go/bin:$HOME/.local/bin:$PATH\" && gc hook run --timeout 15s --timeout-exit-code 0 -- mail check --inject", + "type": "command" + } + ], + "matcher": "" + } + ] + }, + "skipDangerousModePermissionPrompt": true +} diff --git a/.github/workflows/doltlite-linked.yml b/.github/workflows/doltlite-linked.yml new file mode 100644 index 000000000..d888269f3 --- /dev/null +++ b/.github/workflows/doltlite-linked.yml @@ -0,0 +1,117 @@ +name: DoltLite linked backend + +on: + pull_request: + branches: [main] + paths: + - ".github/workflows/doltlite-linked.yml" + - "cmd/bd/**" + - "internal/configfile/**" + - "internal/storage/**" + - "go.mod" + - "go.sum" + - "Makefile" + - ".buildflags" + push: + branches: + - "gascity-doltlite-pin" + - "doltlite/**" + - "feat/doltlite/**" + - "fix/doltlite/**" + workflow_dispatch: + inputs: + doltlite_ref: + description: "DoltLite ref to build" + required: false + default: "master" + +permissions: + contents: read + +concurrency: + group: doltlite-linked-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref || github.run_id }} + cancel-in-progress: true + +env: + BD_DISABLE_METRICS: "1" + BD_DISABLE_EVENT_FLUSH: "1" + DOLTLITE_REF: ${{ inputs.doltlite_ref || 'master' }} + +jobs: + source-built-doltlite: + name: Source-built DoltLite backend + runs-on: ubuntu-latest + steps: + - name: Checkout beads + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6 + with: + fetch-depth: 0 + + - name: Checkout DoltLite + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6 + with: + repository: dolthub/doltlite + ref: ${{ env.DOLTLITE_REF }} + path: doltlite-src + + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version-file: "go.mod" + + - name: Install native build dependencies + run: sudo apt-get update && sudo apt-get install -y build-essential tcl zlib1g-dev + + - name: Build DoltLite from source + run: | + set -euo pipefail + mkdir -p doltlite-src/build + cd doltlite-src/build + ../configure + make -j"$(nproc)" doltlite-lib doltlite + printf 'SELECT doltlite_engine();\n' | ./doltlite :memory: + + - name: Test native DoltLite storage package + env: + DOLTLITE_LIB: ${{ github.workspace }}/doltlite-src/build + LD_LIBRARY_PATH: ${{ github.workspace }}/doltlite-src/build + run: make test-doltlite + + - name: Build linked bd binary + env: + DOLTLITE_LIB: ${{ github.workspace }}/doltlite-src/build + LD_LIBRARY_PATH: ${{ github.workspace }}/doltlite-src/build + run: | + set -euo pipefail + mkdir -p artifacts + source ./.buildflags + CGO_ENABLED=1 \ + CGO_LDFLAGS="-L${DOLTLITE_LIB} -Wl,-rpath,${DOLTLITE_LIB} -ldoltlite" \ + go build -tags "libsqlite3 ${BEADS_BUILD_TAGS}" -o artifacts/bd-doltlite ./cmd/bd + ./artifacts/bd-doltlite version + + - name: Smoke test bd with DoltLite backend + env: + LD_LIBRARY_PATH: ${{ github.workspace }}/doltlite-src/build + BD_NON_INTERACTIVE: "1" + BD_DISABLE_METRICS: "1" + BD_DISABLE_EVENT_FLUSH: "1" + run: | + set -euo pipefail + workdir="$(mktemp -d)" + cd "$workdir" + git init -q + git config beads.role maintainer + export BEADS_DIR="$workdir/.beads" + "${GITHUB_WORKSPACE}/artifacts/bd-doltlite" init --backend doltlite --prefix dl --role maintainer --quiet --skip-hooks --skip-agents + "${GITHUB_WORKSPACE}/artifacts/bd-doltlite" create "DoltLite workflow smoke" --json + "${GITHUB_WORKSPACE}/artifacts/bd-doltlite" list --json + find .beads/doltlite -maxdepth 1 -name '*.db' -type f | grep -q . + + - name: Upload linked bd artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: bd-doltlite-linux + path: artifacts/bd-doltlite + retention-days: 3 + if-no-files-found: error diff --git a/.github/workflows/fork-doltlite-release.yml b/.github/workflows/fork-doltlite-release.yml new file mode 100644 index 000000000..67962ebfc --- /dev/null +++ b/.github/workflows/fork-doltlite-release.yml @@ -0,0 +1,218 @@ +name: Fork DoltLite Release + +on: + push: + branches: + - "fork/doltlite-release-workflow" + - "fork/release/**" + workflow_dispatch: + inputs: + tag_name: + description: "Fork release tag to create or publish, e.g. v1.0.5-doltlite.2" + required: true + type: string + target_ref: + description: "Commit SHA, branch, or tag to build" + required: true + default: "main" + type: string + doltlite_version: + description: "DoltLite release library version" + required: true + default: "0.11.23" + type: string + +concurrency: + group: fork-doltlite-release-${{ inputs.tag_name || github.ref_name }} + cancel-in-progress: false + +permissions: {} + +jobs: + fork-doltlite-release: + name: Fork DoltLite release + if: ${{ github.repository == 'duncan4123/beads-doltlite' }} + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout target + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6 + with: + ref: ${{ inputs.target_ref || github.ref }} + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version-file: go.mod + + - name: Resolve fork release tag + id: release + env: + REQUESTED_TAG: ${{ inputs.tag_name || '' }} + TARGET_REF: ${{ inputs.target_ref || github.ref_name }} + run: | + set -euo pipefail + + if [ -n "$REQUESTED_TAG" ]; then + TAG_NAME="$REQUESTED_TAG" + else + TAG_NAME="v0.0.0-doltlite.${GITHUB_RUN_NUMBER}" + fi + + if ! [[ "$TAG_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-doltlite\.[0-9]+$ ]]; then + echo "ERROR: tag must match vMAJOR.MINOR.PATCH-doltlite.N, got: $TAG_NAME" >&2 + exit 1 + fi + + commit="$(git rev-parse HEAD)" + git fetch --force --tags origin + + if git rev-parse -q --verify "refs/tags/$TAG_NAME" >/dev/null; then + tagged_commit="$(git rev-list -n 1 "$TAG_NAME")" + if [ "$tagged_commit" != "$commit" ]; then + echo "ERROR: tag $TAG_NAME points at $tagged_commit, not checked-out commit $commit from $TARGET_REF" >&2 + exit 1 + fi + echo "Using existing tag $TAG_NAME at $commit" + else + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -a "$TAG_NAME" -m "Release $TAG_NAME" "$commit" + git push origin "refs/tags/$TAG_NAME" + echo "Created tag $TAG_NAME at $commit" + fi + + echo "tag=$TAG_NAME" >> "$GITHUB_OUTPUT" + echo "commit=$commit" >> "$GITHUB_OUTPUT" + + - name: Download DoltLite release library + id: doltlite + env: + REQUESTED_DOLTLITE_VERSION: ${{ inputs.doltlite_version || '' }} + run: | + set -euo pipefail + DOLTLITE_VERSION="${REQUESTED_DOLTLITE_VERSION:-0.11.23}" + version="${DOLTLITE_VERSION#v}" + asset="doltlite-lib-linux-x64-${version}.zip" + base_url="https://github.com/dolthub/doltlite/releases/download/v${version}" + work_dir="${RUNNER_TEMP}/doltlite-lib/${version}" + zip_path="${work_dir}/${asset}" + lib_dir="${work_dir}/lib" + mkdir -p "$work_dir" + + python3 - "$base_url/$asset" "$zip_path" <<'PY' + import os + import sys + import tempfile + import urllib.request + + url, dest = sys.argv[1], sys.argv[2] + directory = os.path.dirname(dest) + fd, tmp = tempfile.mkstemp(prefix=".download-", dir=directory) + os.close(fd) + try: + with urllib.request.urlopen(url, timeout=120) as response: + with open(tmp, "wb") as out: + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + out.write(chunk) + os.replace(tmp, dest) + except Exception: + try: + os.unlink(tmp) + except OSError: + pass + raise + PY + + rm -rf "$lib_dir" + python3 - "$zip_path" "$lib_dir" <<'PY' + import os + import shutil + import sys + import tempfile + import zipfile + + zip_path, dest = sys.argv[1], sys.argv[2] + tmp = tempfile.mkdtemp(prefix="doltlite-lib-") + try: + with zipfile.ZipFile(zip_path) as archive: + archive.extractall(tmp) + entries = [os.path.join(tmp, name) for name in os.listdir(tmp)] + src = entries[0] if len(entries) == 1 and os.path.isdir(entries[0]) else tmp + os.makedirs(os.path.dirname(dest), exist_ok=True) + shutil.copytree(src, dest) + finally: + shutil.rmtree(tmp, ignore_errors=True) + PY + + test -r "$lib_dir/doltlite.h" + test -r "$lib_dir/libdoltlite.a" + echo "lib_dir=$lib_dir" >> "$GITHUB_OUTPUT" + + - name: Build bd-doltlite + env: + DOLTLITE_LIB: ${{ steps.doltlite.outputs.lib_dir }} + run: | + set -euo pipefail + mkdir -p artifacts + source ./.buildflags + CGO_ENABLED=1 \ + CGO_CFLAGS="-I${DOLTLITE_LIB}" \ + CGO_LDFLAGS="-L${DOLTLITE_LIB} ${DOLTLITE_LIB}/libdoltlite.a -lz -lpthread -lm" \ + go build -tags "libsqlite3 ${BEADS_BUILD_TAGS}" -o artifacts/bd-doltlite-linux-amd64 ./cmd/bd + ./artifacts/bd-doltlite-linux-amd64 version + + - name: Smoke test bd with DoltLite backend + env: + BD_NON_INTERACTIVE: "1" + BD_DISABLE_METRICS: "1" + BD_DISABLE_EVENT_FLUSH: "1" + run: | + set -euo pipefail + workdir="$(mktemp -d)" + cd "$workdir" + git init -q + git config beads.role maintainer + export BEADS_DIR="$workdir/.beads" + "${GITHUB_WORKSPACE}/artifacts/bd-doltlite-linux-amd64" init --backend doltlite --prefix dl --role maintainer --quiet --skip-hooks --skip-agents + "${GITHUB_WORKSPACE}/artifacts/bd-doltlite-linux-amd64" create "DoltLite fork release smoke" --json + "${GITHUB_WORKSPACE}/artifacts/bd-doltlite-linux-amd64" list --json + find .beads/doltlite -maxdepth 1 -name '*.db' -type f | grep -q . + + - name: Write checksums + run: | + set -euo pipefail + cd artifacts + sha256sum bd-doltlite-linux-amd64 > checksums.txt + + - name: Publish fork release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG_NAME: ${{ steps.release.outputs.tag }} + run: | + set -euo pipefail + if ! gh release view "$TAG_NAME" >/dev/null 2>&1; then + gh release create "$TAG_NAME" --title "bd $TAG_NAME" --notes "Fork DoltLite-linked bd release." + fi + gh release upload "$TAG_NAME" artifacts/bd-doltlite-linux-amd64 artifacts/checksums.txt --clobber + + - name: Summarize fork release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG_NAME: ${{ steps.release.outputs.tag }} + TARGET_COMMIT: ${{ steps.release.outputs.commit }} + run: | + set -euo pipefail + url="$(gh release view "$TAG_NAME" --json url --jq .url)" + { + echo "### Fork DoltLite release" + echo + echo "- Tag: \`$TAG_NAME\`" + echo "- Commit: \`$TARGET_COMMIT\`" + echo "- Release: $url" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/Makefile b/Makefile index 22ac5f452..cc84895fa 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ SHELL := $(subst cmd,bin,$(subst git.exe,bash.exe,$(GIT_BASH))) endif endif -.PHONY: all build test test-icu-path test-full-cgo test-regression test-upgrade test-cross-version test-migration bench bench-quick clean clean-test-tmp install install-force help check-up-to-date fmt fmt-check check-testing-short +.PHONY: all build test test-doltlite test-icu-path test-full-cgo test-regression test-upgrade test-cross-version test-migration bench bench-quick clean clean-test-tmp install install-force help check-up-to-date fmt fmt-check check-testing-short .PHONY: ci-pr-core ci-pr-policy ci-pr-lint ci-package-mcp ci-package-npm ci-website # Default target @@ -48,6 +48,8 @@ endif # opt-in ICU regex path). BUILD_TAGS := gms_pure_go REGRESSION_TIMEOUT ?= 20m +DOLTLITE_LIB ?= $(abspath ../doltlite-work/build) +DOLTLITE_TEST_FLAGS ?= # Build the bd binary build: @@ -67,6 +69,18 @@ test: @echo "Running tests..." @TEST_COVER=1 ./scripts/test.sh +# Run the embedded DoltLite package tests against a libdoltlite-linked sqlite3 +# driver. Plain `go test ./internal/storage/doltlite` skips when these native +# SQL functions are not linked. +test-doltlite: + @if [ ! -f "$(DOLTLITE_LIB)/libdoltlite.so" ]; then \ + echo "ERROR: libdoltlite.so not found at $(DOLTLITE_LIB)"; \ + echo "Set DOLTLITE_LIB=/path/to/doltlite/build or build DoltLite first."; \ + exit 1; \ + fi + CGO_LDFLAGS="-L$(DOLTLITE_LIB) -Wl,-rpath,$(DOLTLITE_LIB) -ldoltlite" \ + go test -tags "libsqlite3 $(BUILD_TAGS)" ./internal/storage/doltlite $(DOLTLITE_TEST_FLAGS) + # Run the opt-in ICU regex path test suite (no skip list). # This is a local developer workflow for intentionally exercising the leftover # ICU path; it is not part of normal validation. diff --git a/cmd/bd/backup_auto_test.go b/cmd/bd/backup_auto_test.go index 9ed8415da..3b6e35942 100644 --- a/cmd/bd/backup_auto_test.go +++ b/cmd/bd/backup_auto_test.go @@ -67,10 +67,17 @@ func TestIsBackupAutoEnabled(t *testing.T) { // Set env var: "\x00" = unset, anything else = set to that value if tt.envVal == "\x00" { os.Unsetenv("BD_BACKUP_ENABLED") - t.Cleanup(func() { os.Unsetenv("BD_BACKUP_ENABLED") }) + os.Unsetenv("BEADS_BACKUP_ENABLED") + t.Cleanup(func() { + os.Unsetenv("BD_BACKUP_ENABLED") + os.Unsetenv("BEADS_BACKUP_ENABLED") + }) } else { t.Setenv("BD_BACKUP_ENABLED", tt.envVal) + os.Unsetenv("BEADS_BACKUP_ENABLED") + t.Cleanup(func() { os.Unsetenv("BEADS_BACKUP_ENABLED") }) } + t.Setenv("BEADS_DIR", filepath.Join(t.TempDir(), ".beads")) config.ResetForTesting() t.Cleanup(func() { config.ResetForTesting() }) @@ -80,7 +87,9 @@ func TestIsBackupAutoEnabled(t *testing.T) { got := isBackupAutoEnabled() if got != tt.wantResult { - t.Errorf("isBackupAutoEnabled() = %v, want %v", got, tt.wantResult) + t.Errorf("isBackupAutoEnabled() = %v, want %v (source=%v, value=%v, config=%q)", + got, tt.wantResult, config.GetValueSource("backup.enabled"), config.GetBool("backup.enabled"), + config.ConfigFileUsed()) } }) } diff --git a/cmd/bd/config.go b/cmd/bd/config.go index 2efa72098..d097d191a 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -866,7 +866,7 @@ Examples: // Keys under custom.* are always accepted (user-extensible). var recognizedConfigPrefixes = []string{ "export.", "import.", "dolt.", "jira.", "linear.", "github.", "custom.", - "status.", "doctor.suppress.", "routing.", "sync.", "git.", + "status.", "types.", "doctor.suppress.", "routing.", "sync.", "git.", "directory.", "repos.", "external_projects.", "validation.", "hierarchy.", "ai.", "backup.", "federation.", "metrics.", } diff --git a/cmd/bd/config_validate_key_test.go b/cmd/bd/config_validate_key_test.go index 6b077118a..42bd52870 100644 --- a/cmd/bd/config_validate_key_test.go +++ b/cmd/bd/config_validate_key_test.go @@ -9,7 +9,7 @@ func TestIsRecognizedConfigKey(t *testing.T) { recognized := []string{ "export.auto", "dolt.auto-push", "jira.url", "custom.anything", "doctor.suppress.git-hooks", "no-git-ops", "beads.role", - "status.custom", "ai.model", "backup.enabled", "import.path", + "status.custom", "types.custom", "ai.model", "backup.enabled", "import.path", "dolt.local-only", } for _, key := range recognized { diff --git a/cmd/bd/context_cmd.go b/cmd/bd/context_cmd.go index e4fae354d..9a716625b 100644 --- a/cmd/bd/context_cmd.go +++ b/cmd/bd/context_cmd.go @@ -98,6 +98,7 @@ Examples: } info.DoltMode = cfg.GetDoltMode() + info.Backend = cfg.GetBackend() info.Database = cfg.GetDoltDatabase() info.ProjectID = cfg.ProjectID diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 563b51fa9..6a02ddcbc 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -7,6 +7,7 @@ import ( "io" "os" "sort" + "strings" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/storage" @@ -639,6 +640,12 @@ func executeGraphApply(ctx context.Context, plan *GraphApplyPlan, opts GraphAppl return nil }); err != nil { + if result, recovered, recoverErr := recoverGraphApplyResultAfterPostCommitError(plan, keyToID, err); recovered { + if recoverErr != nil { + return nil, recoverErr + } + return result, nil + } return nil, err } @@ -858,6 +865,54 @@ func graphApplyDepPairIDs(pair string) (string, string, bool) { return "", "", false } +func recoverGraphApplyResultAfterPostCommitError(plan *GraphApplyPlan, keyToID map[string]string, err error) (*GraphApplyResult, bool, error) { + postCommitErr, ok := storage.AsPostTransactionCommitError(err) + if !ok || !isRetryableGraphPostCommitError(postCommitErr.Unwrap()) { + return nil, false, nil + } + + result := &GraphApplyResult{IDs: keyToID} + if validateErr := validateGraphApplyResultIDs(plan, result); validateErr != nil { + return nil, true, fmt.Errorf("graph apply SQL rows were committed, but result recovery failed after post-commit error: %w; original error: %v", validateErr, err) + } + return result, true, nil +} + +func isRetryableGraphPostCommitError(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + for _, needle := range []string{ + "database is locked", + "database is busy", + "database table is locked", + "sqlite_busy", + "another connection committed", + "please retry your transaction", + } { + if strings.Contains(msg, needle) { + return true + } + } + return false +} + +func validateGraphApplyResultIDs(plan *GraphApplyPlan, result *GraphApplyResult) error { + if result == nil { + return fmt.Errorf("missing result") + } + for _, node := range plan.Nodes { + if result.IDs[node.Key] == "" { + return fmt.Errorf("missing ID for node %q", node.Key) + } + } + if len(result.IDs) != len(plan.Nodes) { + return fmt.Errorf("result has %d IDs, want %d", len(result.IDs), len(plan.Nodes)) + } + return nil +} + func resolveEdgeRef(key, id string, keyToID map[string]string) string { if id != "" { return id diff --git a/cmd/bd/graph_apply_test.go b/cmd/bd/graph_apply_test.go index da73f5e63..a78dfd4c9 100644 --- a/cmd/bd/graph_apply_test.go +++ b/cmd/bd/graph_apply_test.go @@ -2,9 +2,12 @@ package main import ( "bytes" + "errors" "reflect" "strings" "testing" + + "github.com/steveyegge/beads/internal/storage" ) func TestValidateGraphApplyPlanAcceptsCustomTypes(t *testing.T) { @@ -413,3 +416,100 @@ func TestGraphApplyParentDepPairs(t *testing.T) { t.Fatal("unexpected reverse parent dep pair") } } + +func TestGraphApplyRecoversIDsAfterDoltlitePostCommitDependencyLock(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "root", Title: "Root"}, + {Key: "child", Title: "Child"}, + }, + Edges: []GraphApplyEdge{ + {FromKey: "child", ToKey: "root", Type: "blocks"}, + }, + } + keyToID := map[string]string{ + "root": "bd-root", + "child": "bd-child", + } + err := storage.NewPostTransactionCommitError( + "bd: graph-apply 2 nodes", + errors.New("doltlite add dependencies: database is locked"), + ) + + got, recovered, recoverErr := recoverGraphApplyResultAfterPostCommitError(plan, keyToID, err) + if recoverErr != nil { + t.Fatalf("recover error: %v", recoverErr) + } + if !recovered { + t.Fatal("expected recovery for retryable post-commit lock") + } + if got == nil || got.IDs["root"] != "bd-root" || got.IDs["child"] != "bd-child" { + t.Fatalf("recovered result = %#v", got) + } +} + +func TestGraphApplyDoesNotRecoverPlainTransactionError(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{{Key: "root", Title: "Root"}}, + } + keyToID := map[string]string{"root": "bd-root"} + + got, recovered, recoverErr := recoverGraphApplyResultAfterPostCommitError( + plan, + keyToID, + errors.New("doltlite add dependencies: database is locked"), + ) + if recoverErr != nil { + t.Fatalf("recover error: %v", recoverErr) + } + if recovered { + t.Fatalf("unexpected recovery: %#v", got) + } +} + +func TestGraphApplyDoesNotRecoverNonRetryablePostCommitError(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{{Key: "root", Title: "Root"}}, + } + keyToID := map[string]string{"root": "bd-root"} + err := storage.NewPostTransactionCommitError( + "bd: graph-apply 1 nodes", + errors.New("doltlite add dependencies: permission denied"), + ) + + got, recovered, recoverErr := recoverGraphApplyResultAfterPostCommitError(plan, keyToID, err) + if recoverErr != nil { + t.Fatalf("recover error: %v", recoverErr) + } + if recovered { + t.Fatalf("unexpected recovery: %#v", got) + } +} + +func TestGraphApplyRejectsIncompleteRecoveredIDs(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "root", Title: "Root"}, + {Key: "child", Title: "Child"}, + }, + } + err := storage.NewPostTransactionCommitError( + "bd: graph-apply 2 nodes", + errors.New("doltlite add dependencies: database is locked"), + ) + + got, recovered, recoverErr := recoverGraphApplyResultAfterPostCommitError( + plan, + map[string]string{"root": "bd-root"}, + err, + ) + if !recovered { + t.Fatal("expected recovery path to be selected") + } + if got != nil { + t.Fatalf("unexpected recovered result: %#v", got) + } + if recoverErr == nil || !strings.Contains(recoverErr.Error(), `missing ID for node "child"`) { + t.Fatalf("recoverErr = %v, want missing child ID", recoverErr) + } +} diff --git a/cmd/bd/heartbeat.go b/cmd/bd/heartbeat.go new file mode 100644 index 000000000..43b23877a --- /dev/null +++ b/cmd/bd/heartbeat.go @@ -0,0 +1,93 @@ +package main + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/ui" +) + +var heartbeatCmd = &cobra.Command{ + Use: "heartbeat ", + Aliases: []string{"hb"}, + GroupID: "issues", + Short: "Refresh the lease on an issue you hold in_progress", + Long: `Refresh the lease on an issue you currently hold in_progress. + +A claim carries a lease that expires after a TTL. A worker keeps its claim alive +by heartbeating faster than the TTL; once it stops (because it died), the lease +goes stale and 'bd reclaim' reverts the issue to ready so another worker can pick +it up. Heartbeat pushes lease_expires_at forward and stamps heartbeat_at = now. + +Only the current owner may heartbeat. If the lease has already been reclaimed or +the issue closed, heartbeat fails so the worker learns to stop. + +Heartbeat writes a Dolt commit, so heartbeat well below the TTL but not so fast +it bloats history — cadence should be a small fraction of the TTL, not per-op. + +Examples: + bd heartbeat bd-123 + bd hb bd-123`, + Args: cobra.ExactArgs(1), + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + CheckReadonly("heartbeat") + + evt := metrics.NewCommandEvent("heartbeat") + defer func() { + if c := metrics.Global(); c != nil { + c.CloseEventAndAdd(evt) + } + }() + + ctx := rootCtx + id := args[0] + + result, err := resolveAndGetIssueForMutation(ctx, store, id) + if err != nil { + if result != nil { + result.Close() + } + return HandleErrorRespectJSON("resolving %s: %v", id, err) + } + if result == nil || result.Issue == nil { + if result != nil { + result.Close() + } + return HandleErrorRespectJSON("issue %s not found", id) + } + defer result.Close() + + issueStore := result.Store + if err := issueStore.HeartbeatIssue(ctx, result.ResolvedID, actor); err != nil { + return HandleErrorRespectJSON("heartbeat %s: %v", result.ResolvedID, err) + } + + if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + Command: "heartbeat", + IssueIDs: []string{result.ResolvedID}, + }); err != nil { + return HandleErrorRespectJSON("failed to commit: %v", err) + } + + SetLastTouchedID(result.ResolvedID) + + if jsonOutput { + return outputJSON(map[string]string{ + "id": result.ResolvedID, + "status": "heartbeat", + "owner": actor, + }) + } + fmt.Printf("%s Heartbeat %s (lease refreshed)\n", ui.RenderPass("✓"), formatFeedbackID(result.ResolvedID, result.Issue.Title)) + return nil + }, +} + +func init() { + heartbeatCmd.ValidArgsFunction = issueIDCompletion + rootCmd.AddCommand(heartbeatCmd) +} diff --git a/cmd/bd/init.go b/cmd/bd/init.go index d0ebe2c3b..a6f34d436 100644 --- a/cmd/bd/init.go +++ b/cmd/bd/init.go @@ -40,8 +40,9 @@ var initCmd = &cobra.Command{ Long: `Initialize bd in the current directory by creating a .beads/ directory and Dolt database. Optionally specify a custom issue prefix. -Dolt is the default (and only supported) storage backend. The legacy SQLite -backend has been removed. Use --backend=sqlite to see migration instructions. +Dolt is the default storage backend. The legacy SQLite backend has been +removed. Use --backend=sqlite to see migration instructions. Use +--backend=doltlite for the embedded DoltLite backend. Use --database to specify an existing server database name, overriding the default prefix-based naming. This is useful when an external tool (e.g. an orchestrator) @@ -211,20 +212,31 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): externalConfig = &cfg } - // Handle --backend flag: "dolt" is the only supported backend. + // Handle --backend flag. Dolt remains the default; DoltLite is an + // explicit local backend for embedded SQLite-compatible storage. // "sqlite" is accepted for backward compatibility but prints a // deprecation notice and exits with an error. + backend := configfile.BackendDolt if backendFlag == "sqlite" { fmt.Fprintf(os.Stderr, "%s The SQLite backend has been removed.\n\n", ui.RenderWarn("⚠ DEPRECATED:")) - fmt.Fprintf(os.Stderr, "Dolt is now the default (and only) storage backend for beads.\n") + fmt.Fprintf(os.Stderr, "Dolt is now the default storage backend for beads.\n") fmt.Fprintf(os.Stderr, "To initialize with Dolt:\n") fmt.Fprintf(os.Stderr, " bd init\n\n") fmt.Fprintf(os.Stderr, "To import issues from an existing JSONL export:\n") fmt.Fprintf(os.Stderr, " bd init --from-jsonl\n\n") fmt.Fprintf(os.Stderr, "See: https://github.com/gastownhall/beads/blob/main/docs/DOLT.md\n") return fmt.Errorf("--backend=sqlite is no longer supported") - } else if backendFlag != "" && backendFlag != "dolt" { - return fmt.Errorf("unknown backend %q: only \"dolt\" is supported", backendFlag) + } else if backendFlag == configfile.BackendDoltlite { + backend = configfile.BackendDoltlite + } else if backendFlag != "" && backendFlag != configfile.BackendDolt { + return fmt.Errorf("unknown backend %q: supported backends are \"dolt\" and \"doltlite\"", backendFlag) + } + useDoltlite := backend == configfile.BackendDoltlite + if useDoltlite && (initServerMode || sharedServer || externalServer || initProxiedServer) { + return fmt.Errorf("--backend=doltlite is local-only and cannot be combined with --server, --shared-server, --external, or --proxied-server") + } + if useDoltlite && initRemoteChanged && initRemote != "" { + return fmt.Errorf("--backend=doltlite does not support Dolt remote bootstrap") } // Validate --database format early, before any side effects. @@ -256,9 +268,6 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): return fmt.Errorf("--team requires interactive prompts and cannot be used with --non-interactive") } - // Dolt is the only supported backend - backend := configfile.BackendDolt - // Also treat BEADS_DOLT_SERVER_MODE=1 env var as --server. if os.Getenv("BEADS_DOLT_SERVER_MODE") == "1" { initServerMode = true @@ -271,6 +280,9 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): if sharedServer || strings.EqualFold(os.Getenv("BEADS_DOLT_SHARED_SERVER"), "true") || os.Getenv("BEADS_DOLT_SHARED_SERVER") == "1" { initServerMode = true } + if useDoltlite && (initServerMode || sharedServer) { + return fmt.Errorf("--backend=doltlite is local-only and cannot be combined with Dolt server environment settings") + } // Set serverMode so !usesSQLServer() returns the correct value. // Both the global and cmdCtx must be set because PersistentPreRun @@ -949,7 +961,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): doltCfg.ServerUser = serverUser } - initLock, err := acquireEmbeddedLock(beadsDir, initServerMode || initProxiedServer) + initLock, err := acquireEmbeddedLock(beadsDir, initServerMode || initProxiedServer || useDoltlite) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) return &exitError{Code: 1} @@ -1009,7 +1021,12 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): } } - store, err := newDoltStore(ctx, doltCfg) + var store storage.DoltStorage + if useDoltlite { + store, err = newDoltliteStore(ctx, beadsDir, dbName) + } else { + store, err = newDoltStore(ctx, doltCfg) + } if err != nil { // #4259: the remote-migrate gate refused to auto-apply pending // migrations. When init just bootstrapped the clone, the REMOTE is @@ -1156,6 +1173,15 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): // Always store backend explicitly in metadata.json cfg.Backend = backend + if backend == configfile.BackendDoltlite { + cfg.Database = "doltlite" + if database != "" { + cfg.DoltDatabase = database + } else if cfg.DoltDatabase == "" && prefix != "" { + cfg.DoltDatabase = strings.ReplaceAll(prefix, "-", "_") + } + cfg.DoltMode = configfile.DoltModeEmbedded + } // Metadata.json.database should point to the Dolt directory (not beads.db). // Backward-compat: older dolt setups left this as "beads.db", which is misleading. if backend == configfile.BackendDolt { @@ -1749,8 +1775,8 @@ func init() { initCmd.Flags().Bool("non-interactive", false, "Skip all interactive prompts (auto-detected in CI or non-TTY environments)") initCmd.Flags().String("role", "", "Set beads role without prompting: \"maintainer\" or \"contributor\"") - // Backend selection (dolt is the only supported backend; sqlite accepted for deprecation notice) - initCmd.Flags().String("backend", "", "Storage backend (default: dolt). --backend=sqlite prints deprecation notice.") + // Backend selection (dolt is the default; sqlite accepted for deprecation notice) + initCmd.Flags().String("backend", "", "Storage backend (default: dolt; supported: dolt, doltlite). --backend=sqlite prints deprecation notice.") // Dolt server connection flags initCmd.Flags().Bool("server", false, "Use external dolt sql-server instead of embedded engine") @@ -1991,6 +2017,39 @@ Aborting.`, ui.RenderWarn("⚠"), location, ui.RenderAccent("bd list"), prefix) // database. Skip the SQLite checks below and allow init to proceed. return nil } + // Check for existing doltlite database + if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil && cfg.IsDoltliteBackend() { + doltliteDir := filepath.Join(beadsDir, "doltlite") + entries, err := os.ReadDir(doltliteDir) + if err != nil { + if os.IsNotExist(err) { + return nil // No doltlite directory — fresh clone, safe to init + } + return fmt.Errorf("failed to read doltlite directory %s: %w", doltliteDir, err) + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + if strings.HasSuffix(entry.Name(), ".db") { + location := filepath.Join(doltliteDir, entry.Name()) + return fmt.Errorf(` +%s Found existing doltlite database: %s + +This workspace is already initialized. + +To use the existing database: + Just run bd commands normally (e.g., %s) + +If the database is genuinely corrupt and unrecoverable: + bd export > backup.jsonl # Back up first! + bd init --force --prefix %s # Then reinitialize + +Aborting.`, ui.RenderWarn("⚠"), location, ui.RenderAccent("bd list"), prefix) + } + } + return nil // doltlite directory exists but no .db files — safe to init + } // Check for redirect file - if present, check the redirect target redirectTarget := beads.FollowRedirect(beadsDir) diff --git a/cmd/bd/list.go b/cmd/bd/list.go index 218972b54..cdcf72be0 100644 --- a/cmd/bd/list.go +++ b/cmd/bd/list.go @@ -723,6 +723,7 @@ func init() { // Infra type filtering: exclude agent/role/message by default listCmd.Flags().Bool("include-infra", false, "Include infrastructure beads (agent/role/message) in output") + listCmd.Flags().Bool("include-ephemeral", false, "Include ephemeral issues (wisps) in results") // Explicit type exclusion listCmd.Flags().StringSlice("exclude-type", nil, "Exclude issue types from results (comma-separated or repeatable, e.g., --exclude-type=convoy,epic)") diff --git a/cmd/bd/list_filter.go b/cmd/bd/list_filter.go index 54d3c650b..80010122a 100644 --- a/cmd/bd/list_filter.go +++ b/cmd/bd/list_filter.go @@ -314,7 +314,11 @@ func buildListFilter(in listInput, cfg listFilterConfig) (types.IssueFilter, err filter.HasMetadataKey = in.hasMetadataKey } - if !in.includeInfra && (in.issueType == "" || !cfg.isInfra(in.issueType)) { + if in.includeEphemeral { + ephemeral := true + filter.Ephemeral = &ephemeral + filter.SkipWisps = false + } else if !in.includeInfra && (in.issueType == "" || !cfg.isInfra(in.issueType)) { filter.SkipWisps = true } diff --git a/cmd/bd/list_input.go b/cmd/bd/list_input.go index fc3112cb6..c8c585477 100644 --- a/cmd/bd/list_input.go +++ b/cmd/bd/list_input.go @@ -61,6 +61,7 @@ type listInput struct { includeTemplates bool includeGates bool includeInfra bool + includeEphemeral bool excludeTypeStrs []string parentID string @@ -185,6 +186,7 @@ func gatherListInput(cmd *cobra.Command) (listInput, error) { in.includeTemplates, _ = cmd.Flags().GetBool("include-templates") in.includeGates, _ = cmd.Flags().GetBool("include-gates") in.includeInfra, _ = cmd.Flags().GetBool("include-infra") + in.includeEphemeral, _ = cmd.Flags().GetBool("include-ephemeral") in.excludeTypeStrs, _ = cmd.Flags().GetStringSlice("exclude-type") in.parentID, _ = cmd.Flags().GetString("parent") diff --git a/cmd/bd/list_test.go b/cmd/bd/list_test.go index e607829f0..ccfef2b6a 100644 --- a/cmd/bd/list_test.go +++ b/cmd/bd/list_test.go @@ -1739,4 +1739,14 @@ func TestListCommandInit(t *testing.T) { if excludeLabelFlag.DefValue != "[]" { t.Errorf("--exclude-label default should be '[]', got %q", excludeLabelFlag.DefValue) } + + for _, name := range []string{"skip-labels", "include-ephemeral"} { + flag := listCmd.Flags().Lookup(name) + if flag == nil { + t.Fatalf("--%s flag should exist on bd list", name) + } + if flag.DefValue != "false" { + t.Errorf("--%s default should be false, got %q", name, flag.DefValue) + } + } } diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 1d86b23d4..e3f151304 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1111,7 +1111,11 @@ var rootCmd = &cobra.Command{ // Removing them WILL cause unrecoverable data corruption and data loss. // Dolt manages these files itself; external interference is never safe. - store, err = newDoltStore(rootCtx, doltCfg) + if cfg != nil && cfg.IsDoltliteBackend() { + store, err = newDoltliteStore(rootCtx, beadsDir, doltCfg.Database) + } else { + store, err = newDoltStore(rootCtx, doltCfg) + } // Track final read-only state for staleness checks (GH#1089) storeIsReadOnly = doltCfg.ReadOnly diff --git a/cmd/bd/reclaim.go b/cmd/bd/reclaim.go new file mode 100644 index 000000000..9c5161d00 --- /dev/null +++ b/cmd/bd/reclaim.go @@ -0,0 +1,96 @@ +package main + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/ui" +) + +var reclaimCmd = &cobra.Command{ + Use: "reclaim", + GroupID: "issues", + Short: "Revert stale-lease in_progress issues back to ready (dead-worker recovery)", + Long: `Revert in_progress issues whose lease has gone stale back to ready. + +When a worker claims an issue it takes a lease that expires after a TTL, kept +alive by 'bd heartbeat'. A worker that dies stops heartbeating, so its lease +expires and its issue would otherwise stay in_progress forever. reclaim is the +reaper: it finds in_progress issues whose lease expired more than --older-than +ago, clears the assignee, and sets them back to open so another worker can +claim them. The previous owner's stale lease is recorded as a recovery event. + +--older-than is a grace window past lease expiry: only leases that expired at +least this long ago are reclaimed, so a worker briefly paused (GC, clock skew) +is not robbed of live work. Run it from a supervisor on a timer with a window +of roughly 2× the claim TTL. + +Examples: + bd reclaim # default grace window (2× the lease TTL) + bd reclaim --older-than 10m # reclaim leases expired >10m ago + bd reclaim --older-than 0s # reclaim every currently-expired lease`, + Args: cobra.NoArgs, + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + CheckReadonly("reclaim") + + evt := metrics.NewCommandEvent("reclaim") + defer func() { + if c := metrics.Global(); c != nil { + c.CloseEventAndAdd(evt) + } + }() + + olderThan, _ := cmd.Flags().GetDuration("older-than") + if olderThan < 0 { + return HandleErrorRespectJSON("--older-than must not be negative") + } + + ctx := rootCtx + reclaimed, err := store.ReclaimExpiredLeases(ctx, olderThan, actor) + if err != nil { + return HandleErrorRespectJSON("reclaim: %v", err) + } + + ids := make([]string, 0, len(reclaimed)) + for _, r := range reclaimed { + ids = append(ids, r.ID) + } + if err := commitPendingIfEmbedded(ctx, store, actor, doltAutoCommitParams{ + Command: "reclaim", + IssueIDs: ids, + }); err != nil { + return HandleErrorRespectJSON("failed to commit: %v", err) + } + + if jsonOutput { + return outputJSON(map[string]interface{}{ + "reclaimed": reclaimed, + "count": len(reclaimed), + }) + } + if len(reclaimed) == 0 { + fmt.Printf("%s No stale leases to reclaim\n", ui.RenderPass("✓")) + return nil + } + fmt.Printf("%s Reclaimed %d stale-lease issue(s):\n", ui.RenderPass("✓"), len(reclaimed)) + for _, r := range reclaimed { + owner := r.PreviousOwner + if owner == "" { + owner = "(unassigned)" + } + fmt.Printf(" %s (was held by %s)\n", r.ID, owner) + } + return nil + }, +} + +func init() { + reclaimCmd.Flags().Duration("older-than", 2*issueops.DefaultLeaseTTL, + "Only reclaim leases that expired at least this long ago (grace window)") + rootCmd.AddCommand(reclaimCmd) +} diff --git a/cmd/bd/sql.go b/cmd/bd/sql.go index 71495b30e..11c47d3fb 100644 --- a/cmd/bd/sql.go +++ b/cmd/bd/sql.go @@ -41,9 +41,6 @@ WARNING: Direct database access bypasses the storage layer. Use with caution.`, } }() - if !usesSQLServer() { - return HandleError("'bd sql' is not yet supported in embedded mode") - } query := args[0] csvOutput, _ := cmd.Flags().GetBool("csv") diff --git a/cmd/bd/store_factory.go b/cmd/bd/store_factory.go index 5858c7ff0..7a28ae528 100644 --- a/cmd/bd/store_factory.go +++ b/cmd/bd/store_factory.go @@ -14,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/util" "github.com/steveyegge/beads/internal/storage/dolt" + "github.com/steveyegge/beads/internal/storage/doltlite" "github.com/steveyegge/beads/internal/storage/embeddeddolt" ) @@ -63,6 +64,13 @@ func newDoltStore(ctx context.Context, cfg *dolt.Config) (storage.DoltStorage, e return embeddeddolt.Open(ctx, cfg.BeadsDir, cfg.Database, "main") } +func newDoltliteStore(ctx context.Context, beadsDir, database string) (storage.DoltStorage, error) { + if database == "" { + database = configfile.DefaultDoltDatabase + } + return doltlite.New(ctx, beadsDir, database, "main") +} + // acquireEmbeddedLock acquires an exclusive flock on the embeddeddolt data // directory derived from beadsDir. The caller must defer lock.Unlock(). // Returns a no-op lock when serverMode is true (the server handles its own @@ -92,6 +100,9 @@ func acquireEmbeddedLock(beadsDir string, serverMode bool) (util.Unlocker, error // auto-sanitized to underscores and the fix is persisted to metadata.json. func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { cfg, err := configfile.Load(beadsDir) + if err == nil && cfg != nil && cfg.IsDoltliteBackend() { + return newDoltliteStore(ctx, beadsDir, cfg.GetDoltDatabase()) + } if err == nil && cfg != nil && cfg.IsDoltProxiedServerMode() { // TODO: this needs to be uow provider return nil, fmt.Errorf("proxy server store should be uow provider") @@ -165,6 +176,9 @@ func migrateHyphenatedDB(beadsDir string, cfg *configfile.Config, oldName, newNa // hydration from mutating foreign projects (GH#3231). func newReadOnlyStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { cfg, err := configfile.Load(beadsDir) + if err == nil && cfg != nil && cfg.IsDoltliteBackend() { + return newDoltliteStore(ctx, beadsDir, cfg.GetDoltDatabase()) + } if err == nil && cfg != nil && cfg.IsDoltProxiedServerMode() { // TODO: this needs to be uow provider return nil, fmt.Errorf("proxy server store needs to be uow provider") diff --git a/cmd/bd/store_factory_nocgo.go b/cmd/bd/store_factory_nocgo.go index e1ab9fdc3..757cf3731 100644 --- a/cmd/bd/store_factory_nocgo.go +++ b/cmd/bd/store_factory_nocgo.go @@ -40,6 +40,10 @@ func newDoltStore(ctx context.Context, cfg *dolt.Config) (storage.DoltStorage, e return dolt.New(ctx, cfg) } +func newDoltliteStore(_ context.Context, _, _ string) (storage.DoltStorage, error) { + return nil, fmt.Errorf("%s", nocgoDoltliteErrMsg) +} + // acquireEmbeddedLock returns a no-op lock in non-CGO builds. func acquireEmbeddedLock(_ string, _ bool) (util.Unlocker, error) { return util.NoopLock{}, nil @@ -101,3 +105,7 @@ Three options: curl -fsSL https://raw.githubusercontent.com/steveyegge/beads/main/scripts/install.sh | bash See docs/INSTALLING.md for the full comparison.` + +const nocgoDoltliteErrMsg = `DoltLite requires a CGO build, but this bd binary was built with CGO_ENABLED=0. + +Build bd with CGO_ENABLED=1 and libdoltlite available, or use the Dolt backend for non-CGO builds.` diff --git a/cmd/bd/store_factory_nocgo_test.go b/cmd/bd/store_factory_nocgo_test.go index 384078acb..6b03ce31f 100644 --- a/cmd/bd/store_factory_nocgo_test.go +++ b/cmd/bd/store_factory_nocgo_test.go @@ -27,6 +27,19 @@ func TestNocgoNewDoltStore_ErrorSuggestsCorrectFlag(t *testing.T) { } } +func TestNocgoNewDoltliteStore_ErrorRequiresCGO(t *testing.T) { + _, err := newDoltliteStore(t.Context(), t.TempDir(), "beads") + if err == nil { + t.Fatal("expected error for DoltLite in a non-CGO build") + } + msg := err.Error() + for _, want := range []string{"DoltLite requires a CGO build", "CGO_ENABLED=0", "CGO_ENABLED=1", "libdoltlite"} { + if !strings.Contains(msg, want) { + t.Errorf("error should contain %q, got: %s", want, msg) + } + } +} + // TestNocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag verifies that // newDoltStoreFromConfig suggests "bd init --server" when no server-mode // config exists. diff --git a/default.nix b/default.nix index d0449b48c..178de0660 100644 --- a/default.nix +++ b/default.nix @@ -19,7 +19,7 @@ buildGoModule { # proxyVendor avoids vendor/modules.txt consistency checks when the vendored # tree lags go.mod/go.sum. proxyVendor = true; - vendorHash = "sha256-pNGXUkKrV8olLYE7EecLHPxiiytorJbgBsYLKCV0o7Y="; + vendorHash = "sha256-F1yCmLRESzCWrnsRPqyNzlSQcWmCtO6mXEJibXxLmKE="; # Match go.mod to the selected Nix Go toolchain. buildGoModule also builds # vendored dependencies in the Nix sandbox, where toolchain downloads are not diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index de9565a5b..0a6620a2d 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -25,6 +25,7 @@ Reference for bd Latest. Generated from `bd help --all`. - [bd gate list](#bd-gate-list) — List gate issues - [bd gate resolve](#bd-gate-resolve) — Manually resolve (close) a gate - [bd gate show](#bd-gate-show) — Show a gate issue +- [bd heartbeat](#bd-heartbeat) — Refresh the lease on an issue you hold in_progress - [bd label](#bd-label) — Manage issue labels - [bd label add](#bd-label-add) — Add a label to one or more issues - [bd label list](#bd-label-list) — List labels for an issue @@ -43,6 +44,7 @@ Reference for bd Latest. Generated from `bd help --all`. - [bd promote](#bd-promote) — Promote a wisp to a permanent bead - [bd q](#bd-q) — Quick capture: create issue and output only ID - [bd query](#bd-query) — Query issues using a simple query language +- [bd reclaim](#bd-reclaim) — Revert stale-lease in_progress issues back to ready (dead-worker recovery) - [bd reopen](#bd-reopen) — Reopen one or more closed issues - [bd search](#bd-search) — Search issues by text query - [bd set-state](#bd-set-state) — Set operational state (creates event + updates label) @@ -813,6 +815,31 @@ This is similar to 'bd show' but validates that the issue is a gate. bd gate show ``` +### bd heartbeat + +Refresh the lease on an issue you currently hold in_progress. + +A claim carries a lease that expires after a TTL. A worker keeps its claim alive +by heartbeating faster than the TTL; once it stops (because it died), the lease +goes stale and 'bd reclaim' reverts the issue to ready so another worker can pick +it up. Heartbeat pushes lease_expires_at forward and stamps heartbeat_at = now. + +Only the current owner may heartbeat. If the lease has already been reclaimed or +the issue closed, heartbeat fails so the worker learns to stop. + +Heartbeat writes a Dolt commit, so heartbeat well below the TTL but not so fast +it bloats history — cadence should be a small fraction of the TTL, not per-op. + +Examples: + bd heartbeat bd-123 + bd hb bd-123 + +``` +bd heartbeat +``` + +**Aliases:** hb + ### bd label Manage issue labels @@ -913,6 +940,7 @@ bd list [flags] --format string Output format: 'digraph' (for golang.org/x/tools/cmd/digraph), 'dot' (Graphviz), or Go template --has-metadata-key string Filter issues that have this metadata key set --id string Filter by specific issue IDs (comma-separated, e.g., bd-1,bd-5,bd-10) + --include-ephemeral Include ephemeral issues (wisps) in results --include-gates Include gate issues in output (normally hidden) --include-infra Include infrastructure beads (agent/role/message) in output --include-templates Include template molecules in output @@ -1210,6 +1238,37 @@ bd query [expression] [flags] --sort string Sort by field: priority, created, updated, closed, status, id, title, type, assignee ``` +### bd reclaim + +Revert in_progress issues whose lease has gone stale back to ready. + +When a worker claims an issue it takes a lease that expires after a TTL, kept +alive by 'bd heartbeat'. A worker that dies stops heartbeating, so its lease +expires and its issue would otherwise stay in_progress forever. reclaim is the +reaper: it finds in_progress issues whose lease expired more than --older-than +ago, clears the assignee, and sets them back to open so another worker can +claim them. The previous owner's stale lease is recorded as a recovery event. + +--older-than is a grace window past lease expiry: only leases that expired at +least this long ago are reclaimed, so a worker briefly paused (GC, clock skew) +is not robbed of live work. Run it from a supervisor on a timer with a window +of roughly 2× the claim TTL. + +Examples: + bd reclaim # default grace window (2× the lease TTL) + bd reclaim --older-than 10m # reclaim leases expired >10m ago + bd reclaim --older-than 0s # reclaim every currently-expired lease + +``` +bd reclaim [flags] +``` + +**Flags:** + +``` + --older-than duration Only reclaim leases that expired at least this long ago (grace window) (default 10m0s) +``` + ### bd reopen Reopen closed issues by setting status to 'open' and clearing the closed_at timestamp. @@ -3382,8 +3441,9 @@ bd info [flags] Initialize bd in the current directory by creating a .beads/ directory and Dolt database. Optionally specify a custom issue prefix. -Dolt is the default (and only supported) storage backend. The legacy SQLite -backend has been removed. Use --backend=sqlite to see migration instructions. +Dolt is the default storage backend. The legacy SQLite backend has been +removed. Use --backend=sqlite to see migration instructions. Use +--backend=doltlite for the embedded DoltLite backend. Use --database to specify an existing server database name, overriding the default prefix-based naming. This is useful when an external tool (e.g. an orchestrator) @@ -3423,7 +3483,7 @@ bd init [flags] --agents-file string Custom filename for agent instructions (default: AGENTS.md) --agents-profile string AGENTS.md profile: 'minimal' (default, pointer to bd prime) or 'full' (complete command reference) --agents-template string Path to custom AGENTS.md template (overrides embedded default) - --backend string Storage backend (default: dolt). --backend=sqlite prints deprecation notice. + --backend string Storage backend (default: dolt; supported: dolt, doltlite). --backend=sqlite prints deprecation notice. --contributor Run OSS contributor setup wizard --database string Use existing server database name (overrides prefix-based naming) --debug Run the managed Dolt sql-server with --loglevel=debug and CPU profiling (--prof cpu). Persisted to config.yaml as dolt.debug. No effect on externally-managed servers. diff --git a/go.mod b/go.mod index d1a1f7177..09adab41e 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/dolthub/driver/v2 v2.1.4 github.com/dolthub/eventkit v0.0.0-20260611184414-99f5693e696a github.com/go-sql-driver/mysql v1.9.3 + github.com/mattn/go-sqlite3 v1.14.8 github.com/olebedev/when v1.1.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 diff --git a/internal/beads/beads.go b/internal/beads/beads.go index baf010799..772856cb5 100644 --- a/internal/beads/beads.go +++ b/internal/beads/beads.go @@ -428,6 +428,17 @@ func findLocalBeadsDir() string { func findDatabaseInBeadsDir(beadsDir string, _ bool) string { // Check for metadata.json first (single source of truth) if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil { + if cfg.IsDoltliteBackend() { + doltlitePath := filepath.Join(beadsDir, "doltlite", cfg.GetDoltDatabase()+".db") + if info, err := os.Stat(doltlitePath); err == nil && !info.IsDir() { + return doltlitePath + } + doltliteDir := filepath.Join(beadsDir, "doltlite") + if info, err := os.Stat(doltliteDir); err == nil && info.IsDir() { + return doltliteDir + } + return "" + } // For Dolt server mode, database is on the server - no local directory required if cfg.IsDoltServerMode() { return cfg.DatabasePath(beadsDir) @@ -643,8 +654,9 @@ func hasBeadsProjectFiles(beadsDir string) bool { // hasBeadsDatabase is the strict counterpart to hasBeadsProjectFiles: it // returns true only when beadsDir contains an actual database — a dolt/ -// directory, an embeddeddolt/ directory, or a non-backup *.db file. Mere -// presence of metadata.json / config.yaml / issues.jsonl does not count. +// directory, an embeddeddolt/ directory, a doltlite/*.db file, or a +// non-backup *.db file. Mere presence of metadata.json / config.yaml / +// issues.jsonl does not count. // // Used by FindBeadsDir's worktree-separate-DB branch to distinguish a // genuine separate-database worktree (which owns its own Dolt data) from @@ -659,6 +671,13 @@ func hasBeadsDatabase(beadsDir string) bool { if info, err := os.Stat(filepath.Join(beadsDir, "embeddeddolt")); err == nil && info.IsDir() { return true } + doltliteMatches, _ := filepath.Glob(filepath.Join(beadsDir, "doltlite", "*.db")) + for _, match := range doltliteMatches { + baseName := filepath.Base(match) + if !strings.HasPrefix(baseName, ".") && !strings.Contains(baseName, ".backup") { + return true + } + } dbMatches, _ := filepath.Glob(filepath.Join(beadsDir, "*.db")) for _, match := range dbMatches { baseName := filepath.Base(match) diff --git a/internal/beads/context.go b/internal/beads/context.go index 33cffc255..65dfb714c 100644 --- a/internal/beads/context.go +++ b/internal/beads/context.go @@ -129,11 +129,12 @@ func buildRepoContext() (*RepoContext, error) { // Beads dir is in a different repo - use that repo's root repoRoot = repoRootForBeadsDir(beadsDir) } else { - // Normal case - find repo root via git - var err error - repoRoot, err = git.GetMainRepoRoot() - if err != nil { - return nil, fmt.Errorf("cannot determine repository root: %w", err) + // Normal case - prefer the VCS root, but keep diagnostics usable in + // non-git workspaces that already contain a .beads directory. + if root, err := git.GetMainRepoRoot(); err == nil && root != "" { + repoRoot = root + } else { + repoRoot = filepath.Dir(beadsDir) } } @@ -471,7 +472,7 @@ func buildRepoContextForWorkspace(workspacePath string) (*RepoContext, error) { isWorktree = false repoRoot = git.GetRepoRoot() if repoRoot == "" { - return nil, fmt.Errorf("workspace %s is not in a git repository", workspacePath) + repoRoot = workspacePath } } diff --git a/internal/beads/context_test.go b/internal/beads/context_test.go index bd27cd44a..d96d7b1a6 100644 --- a/internal/beads/context_test.go +++ b/internal/beads/context_test.go @@ -142,6 +142,42 @@ func TestGetRepoContextForWorkspace_NonGitDirectory(t *testing.T) { } } +// TestGetRepoContextForWorkspace_NonGitDirectoryWithBeads verifies that a +// workspace-local .beads directory is enough for context diagnostics. Commands +// such as `bd context` should not require git just to report backend identity. +func TestGetRepoContextForWorkspace_NonGitDirectoryWithBeads(t *testing.T) { + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0750); err != nil { + t.Fatalf("failed to create .beads dir: %v", err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"doltlite"}`), 0644); err != nil { + t.Fatalf("failed to create metadata.json: %v", err) + } + + t.Cleanup(func() { + ResetCaches() + git.ResetCaches() + }) + + rc, err := GetRepoContextForWorkspace(tmpDir) + if err != nil { + t.Fatalf("GetRepoContextForWorkspace failed: %v", err) + } + if rc.RepoRoot != tmpDir { + t.Errorf("RepoRoot mismatch: expected %s, got %s", tmpDir, rc.RepoRoot) + } + if rc.BeadsDir != beadsDir { + t.Errorf("BeadsDir mismatch: expected %s, got %s", beadsDir, rc.BeadsDir) + } + if rc.CWDRepoRoot != "" { + t.Errorf("CWDRepoRoot should be empty outside git, got %q", rc.CWDRepoRoot) + } + if rc.IsWorktree { + t.Error("IsWorktree should be false outside git") + } +} + // TestGetRepoContextForWorkspace_MissingBeadsDir tests error when .beads doesn't exist func TestGetRepoContextForWorkspace_MissingBeadsDir(t *testing.T) { tmpDir := t.TempDir() diff --git a/internal/configfile/configfile.go b/internal/configfile/configfile.go index d187d9858..9fe360cdf 100644 --- a/internal/configfile/configfile.go +++ b/internal/configfile/configfile.go @@ -169,7 +169,8 @@ func (c *Config) GetStaleClosedIssuesDays() int { // Backend constants const ( - BackendDolt = "dolt" + BackendDolt = "dolt" + BackendDoltlite = "doltlite" ) // BackendCapabilities describes behavioral constraints for a storage backend. @@ -204,11 +205,19 @@ func (c *Config) GetCapabilities() BackendCapabilities { return CapabilitiesForBackend(backend) } -// GetBackend returns the backend type. Always returns "dolt". +// GetBackend returns the backend type. Legacy or unknown values are normalized +// to Dolt, but DoltLite is preserved for embedded DoltLite workspaces. func (c *Config) GetBackend() string { + if c != nil && c.Backend == BackendDoltlite { + return BackendDoltlite + } return BackendDolt } +func (c *Config) IsDoltliteBackend() bool { + return c != nil && c.GetBackend() == BackendDoltlite +} + // Dolt mode constants const ( DoltModeEmbedded = "embedded" diff --git a/internal/configfile/configfile_test.go b/internal/configfile/configfile_test.go index 3911970d6..37e83e75a 100644 --- a/internal/configfile/configfile_test.go +++ b/internal/configfile/configfile_test.go @@ -626,28 +626,38 @@ func TestProxiedServerClientInfo_ResolvedPaths(t *testing.T) { }) } -// TestGetBackendAlwaysDolt tests that GetBackend always returns "dolt". -func TestGetBackendAlwaysDolt(t *testing.T) { +func TestGetBackendNormalizesLegacyValues(t *testing.T) { tests := []struct { name string cfg *Config + want string }{ - {name: "explicit dolt", cfg: &Config{Backend: BackendDolt}}, - {name: "empty backend", cfg: &Config{Backend: ""}}, - {name: "legacy config", cfg: &Config{}}, - {name: "stale sqlite value", cfg: &Config{Backend: "sqlite"}}, - {name: "unknown backend", cfg: &Config{Backend: "postgres"}}, + {name: "explicit dolt", cfg: &Config{Backend: BackendDolt}, want: BackendDolt}, + {name: "explicit doltlite", cfg: &Config{Backend: BackendDoltlite}, want: BackendDoltlite}, + {name: "empty backend", cfg: &Config{Backend: ""}, want: BackendDolt}, + {name: "legacy config", cfg: &Config{}, want: BackendDolt}, + {name: "stale sqlite value", cfg: &Config{Backend: "sqlite"}, want: BackendDolt}, + {name: "unknown backend", cfg: &Config{Backend: "postgres"}, want: BackendDolt}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := tt.cfg.GetBackend(); got != BackendDolt { - t.Errorf("GetBackend() = %q, want %q", got, BackendDolt) + if got := tt.cfg.GetBackend(); got != tt.want { + t.Errorf("GetBackend() = %q, want %q", got, tt.want) } }) } } +func TestIsDoltliteBackend(t *testing.T) { + if !(&Config{Backend: BackendDoltlite}).IsDoltliteBackend() { + t.Fatal("IsDoltliteBackend() = false, want true") + } + if (&Config{Backend: BackendDolt}).IsDoltliteBackend() { + t.Fatal("IsDoltliteBackend() = true for dolt backend") + } +} + // TestDatabasePathAlwaysDolt tests that DatabasePath always returns the dolt path. func TestDatabasePathAlwaysDolt(t *testing.T) { beadsDir := "/home/user/project/.beads" diff --git a/internal/storage/bulk_issues.go b/internal/storage/bulk_issues.go index 3baeef97d..d5c75265d 100644 --- a/internal/storage/bulk_issues.go +++ b/internal/storage/bulk_issues.go @@ -2,6 +2,7 @@ package storage import ( "context" + "time" "github.com/steveyegge/beads/internal/types" ) @@ -14,6 +15,14 @@ type BulkIssueStore interface { UpdateIssueID(ctx context.Context, oldID, newID string, issue *types.Issue, actor string) error ClaimIssue(ctx context.Context, id string, actor string) error ClaimReadyIssue(ctx context.Context, filter types.WorkFilter, actor string) (*types.Issue, error) + // HeartbeatIssue refreshes the lease on an in_progress issue held by actor, + // pushing its expiry forward so a reaper won't reclaim it. Returns + // ErrNotClaimable/ErrAlreadyClaimed if actor no longer holds the lease. + HeartbeatIssue(ctx context.Context, id, actor string) error + // ReclaimExpiredLeases reverts in_progress issues whose lease expired more + // than olderThan ago back to ready (clearing the assignee), recovering work + // stranded by dead workers. Returns the issues it reclaimed. + ReclaimExpiredLeases(ctx context.Context, olderThan time.Duration, actor string) ([]types.ReclaimedLease, error) PromoteFromEphemeral(ctx context.Context, id string, actor string) error GetNextChildID(ctx context.Context, parentID string) (string, error) } diff --git a/internal/storage/dolt/concurrent_test.go b/internal/storage/dolt/concurrent_test.go index 5ee336923..48155e9b5 100644 --- a/internal/storage/dolt/concurrent_test.go +++ b/internal/storage/dolt/concurrent_test.go @@ -955,3 +955,123 @@ func TestSerializationConflictRetry(t *testing.T) { t.Fatalf("expected %d labels, got %d: %v", numGoroutines, len(labels), labels) } } + +// ============================================================================= +// Test: Concurrent Work-Queue Drain (the "Gas Station" scenario) +// +// N workers concurrently dequeue from ONE shared ready-front via +// ClaimReadyIssue until it returns nil, exactly as N agent clones draining a +// shared Beads work queue would. This is the core Gas Station claim-queue +// invariant and the scenario the multi-agent port harness depends on: +// +// - every issue is claimed by EXACTLY ONE worker (no double-claim / lost work) +// - every issue is claimed (no stranded ready work left behind) +// - the count of distinct claims equals the number of issues +// +// Dolt has no SKIP LOCKED, so the safety comes from the claim CAS (UPDATE ... +// SET assignee WHERE assignee IS NULL) colliding on the same cell, surfacing as +// a 1213/1205 serialization conflict, which ClaimReadyIssue's withRetryTx +// re-scans and retries. This test is the regression guard for that guarantee. +// ============================================================================= + +func TestConcurrentWorkQueueDrain(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + + ctx, cancel := concurrentTestContext(t) + defer cancel() + + // Seed a ready-front: all open, unassigned, no blockers => all ready. + const numIssues = 40 + want := make(map[string]bool, numIssues) + for i := 0; i < numIssues; i++ { + id := fmt.Sprintf("queue-%03d", i) + issue := &types.Issue{ + ID: id, + Title: fmt.Sprintf("Queue item %d", i), + Description: "ready work for the shared drain", + Status: types.StatusOpen, + Priority: (i % 4) + 1, + IssueType: types.TypeTask, + } + if err := store.CreateIssue(ctx, issue, "seeder"); err != nil { + t.Fatalf("seed issue %s: %v", id, err) + } + want[id] = true + } + + const numWorkers = 6 + + var mu sync.Mutex + claimedBy := make(map[string]string, numIssues) // issueID -> worker that claimed it + var doubleClaim atomic.Int32 + var claimErrs atomic.Int32 + var wg sync.WaitGroup + + for w := 0; w < numWorkers; w++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + actor := fmt.Sprintf("worker-%d", workerID) + for { + issue, err := store.ClaimReadyIssue(ctx, types.WorkFilter{}, actor) + if err != nil { + claimErrs.Add(1) + return + } + if issue == nil { + return // ready-front drained from this worker's snapshot + } + mu.Lock() + if prev, ok := claimedBy[issue.ID]; ok { + t.Errorf("issue %s double-claimed: first by %s, then by %s", issue.ID, prev, actor) + doubleClaim.Add(1) + } else { + claimedBy[issue.ID] = actor + } + mu.Unlock() + } + }(w) + } + + done := make(chan struct{}) + go func() { wg.Wait(); close(done) }() + select { + case <-done: + case <-ctx.Done(): + t.Fatal("work-queue drain timeout — possible deadlock or claim livelock") + } + + if n := claimErrs.Load(); n != 0 { + t.Errorf("got %d claim errors; ClaimReadyIssue should retry serialization conflicts internally and never surface one", n) + } + if n := doubleClaim.Load(); n != 0 { + t.Errorf("got %d double-claims; the claim CAS must give each issue to exactly one worker", n) + } + + // No stranded work: every seeded issue claimed exactly once. + if len(claimedBy) != numIssues { + t.Errorf("claimed %d distinct issues, want %d (stranded ready work)", len(claimedBy), numIssues) + } + for id := range want { + if _, ok := claimedBy[id]; !ok { + t.Errorf("issue %s was never claimed (stranded)", id) + } + } + + // Cross-check against the store: nothing should remain ready/open. + remaining, err := store.ClaimReadyIssue(ctx, types.WorkFilter{}, "final-sweeper") + if err != nil { + t.Fatalf("final sweep: %v", err) + } + if remaining != nil { + t.Errorf("ready-front not fully drained: %s still claimable after all workers finished", remaining.ID) + } + + // Distribution sanity (informational): how evenly work spread across workers. + perWorker := make(map[string]int) + for _, actor := range claimedBy { + perWorker[actor]++ + } + t.Logf("drain complete: %d issues across %d workers: %v", len(claimedBy), numWorkers, perWorker) +} diff --git a/internal/storage/dolt/issues.go b/internal/storage/dolt/issues.go index f819006ee..87a746231 100644 --- a/internal/storage/dolt/issues.go +++ b/internal/storage/dolt/issues.go @@ -175,30 +175,27 @@ func (s *DoltStore) UpdateIssue(ctx context.Context, id string, updates map[stri return s.DemoteToWisp(ctx, id, updates, actor) } - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - _, err = issueops.UpdateIssueInTx(ctx, tx, id, updates, actor) - if err != nil { - return err - } - - for _, table := range []string{"issues", "events"} { - _, _ = tx.ExecContext(ctx, "CALL DOLT_ADD(?)", table) - } - commitMsg := fmt.Sprintf("bd: update %s", id) - if _, err := tx.ExecContext(ctx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)", - commitMsg, s.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) { - return fmt.Errorf("dolt commit: %w", err) - } + // Wrap in withRetryTx so a concurrent writer that loses Dolt's optimistic + // commit-time merge (MySQL 1213/1205, guaranteed server-side rollback) is + // retried rather than surfaced as a hard failure. Dolt has no real row + // locking — FOR UPDATE / SKIP LOCKED are parse-only no-ops + // (https://www.dolthub.com/blog/2023-10-23-hold-my-beer/) — so retry is the + // only safety net. withRetryTx owns BeginTx and the final Commit. + return s.withRetryTx(ctx, func(tx *sql.Tx) error { + if _, err := issueops.UpdateIssueInTx(ctx, tx, id, updates, actor); err != nil { + return err + } - if err := tx.Commit(); err != nil { - return wrapTransactionError("commit update issue", err) - } - return nil + for _, table := range []string{"issues", "events"} { + _, _ = tx.ExecContext(ctx, "CALL DOLT_ADD(?)", table) + } + commitMsg := fmt.Sprintf("bd: update %s", id) + if _, err := tx.ExecContext(ctx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)", + commitMsg, s.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) { + return fmt.Errorf("dolt commit: %w", err) + } + return nil + }) } // ClaimIssue atomically claims an issue using compare-and-swap semantics. @@ -213,62 +210,125 @@ func (s *DoltStore) ClaimIssue(ctx context.Context, id string, actor string) err return s.claimWisp(ctx, id, actor) } - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - if _, err := issueops.ClaimIssueInTx(ctx, tx, id, actor); err != nil { - return err - } - - // Dolt versioning for permanent issues. - // GH#2455: Stage only the tables we modified, then commit without -A. - for _, table := range []string{"issues", "events"} { - _, _ = tx.ExecContext(ctx, "CALL DOLT_ADD(?)", table) - } - commitMsg := fmt.Sprintf("bd: claim %s", id) - if _, err := tx.ExecContext(ctx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)", - commitMsg, s.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) { - return fmt.Errorf("dolt commit: %w", err) - } + // Wrap in withRetryTx so a concurrent claim that loses Dolt's optimistic + // commit-time merge (MySQL 1213/1205, guaranteed server-side rollback) is + // retried instead of surfaced as a hard failure. Dolt has no real row + // locking — FOR UPDATE / SKIP LOCKED are parse-only no-ops + // (https://www.dolthub.com/blog/2023-10-23-hold-my-beer/) — so retry is the + // only safety net under concurrent claimants. The body stays a single tx + // (CAS + DOLT_COMMIT); withRetryTx owns BeginTx and the final Commit. + return s.withRetryTx(ctx, func(tx *sql.Tx) error { + if _, err := issueops.ClaimIssueInTx(ctx, tx, id, actor); err != nil { + return err + } - if err := tx.Commit(); err != nil { - return wrapTransactionError("commit claim issue", err) - } - return nil + // Dolt versioning for permanent issues. + // GH#2455: Stage only the tables we modified, then commit without -A. + for _, table := range []string{"issues", "events"} { + _, _ = tx.ExecContext(ctx, "CALL DOLT_ADD(?)", table) + } + commitMsg := fmt.Sprintf("bd: claim %s", id) + if _, err := tx.ExecContext(ctx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)", + commitMsg, s.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) { + return fmt.Errorf("dolt commit: %w", err) + } + return nil + }) } // ClaimReadyIssue atomically claims the first ready issue matching filter. func (s *DoltStore) ClaimReadyIssue(ctx context.Context, filter types.WorkFilter, actor string) (*types.Issue, error) { - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return nil, fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() + // Wrap in withRetryTx: under concurrent workers the loser of Dolt's + // optimistic commit-time merge gets MySQL 1213/1205 (guaranteed server-side + // rollback). Retrying re-scans the ready front from a fresh snapshot and + // claims the next available issue instead of failing the dequeue. Dolt has + // no real row locking — FOR UPDATE / SKIP LOCKED are parse-only no-ops + // (https://www.dolthub.com/blog/2023-10-23-hold-my-beer/) — so retry is the + // safety net. withRetryTx owns BeginTx and the final Commit. + var claimed *types.Issue + err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + var err error + claimed, err = issueops.ClaimReadyIssueInTx(ctx, tx, filter, actor) + if err != nil { + return err + } + if claimed == nil { + return nil + } - claimed, err := issueops.ClaimReadyIssueInTx(ctx, tx, filter, actor) + for _, table := range []string{"issues", "events"} { + _, _ = tx.ExecContext(ctx, "CALL DOLT_ADD(?)", table) + } + commitMsg := fmt.Sprintf("bd: claim ready %s", claimed.ID) + if _, err := tx.ExecContext(ctx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)", + commitMsg, s.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) { + return fmt.Errorf("dolt commit: %w", err) + } + return nil + }) if err != nil { return nil, err } - if claimed == nil { - return nil, nil - } + return claimed, nil +} - for _, table := range []string{"issues", "events"} { - _, _ = tx.ExecContext(ctx, "CALL DOLT_ADD(?)", table) - } - commitMsg := fmt.Sprintf("bd: claim ready %s", claimed.ID) - if _, err := tx.ExecContext(ctx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)", - commitMsg, s.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) { - return nil, fmt.Errorf("dolt commit: %w", err) +// HeartbeatIssue refreshes the lease on an issue actor holds in_progress, +// pushing lease_expires_at forward and rewriting row_lock (see issueops.lease). +// Wrapped in withRetryTx so a heartbeat that loses Dolt's optimistic merge to a +// concurrent reclaim/close on the same row is replayed against a fresh snapshot +// rather than surfaced — the row_lock collision is what forces that retry. +func (s *DoltStore) HeartbeatIssue(ctx context.Context, id, actor string) error { + if s.isActiveWisp(ctx, id) { + // Wisps are ephemeral and never leased; nothing to heartbeat. + return fmt.Errorf("%w: %s is ephemeral", storage.ErrNotClaimable, id) } + return s.withRetryTx(ctx, func(tx *sql.Tx) error { + if err := issueops.HeartbeatIssueInTx(ctx, tx, id, actor); err != nil { + return err + } + // GH#2455: stage only the tables we touched, then commit without -A. + for _, table := range []string{"issues"} { + _, _ = tx.ExecContext(ctx, "CALL DOLT_ADD(?)", table) + } + commitMsg := fmt.Sprintf("bd: heartbeat %s", id) + if _, err := tx.ExecContext(ctx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)", + commitMsg, s.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) { + return fmt.Errorf("dolt commit: %w", err) + } + return nil + }) +} - if err := tx.Commit(); err != nil { - return nil, wrapTransactionError("commit claim ready issue", err) +// ReclaimExpiredLeases reverts in_progress issues whose lease expired more than +// olderThan ago back to ready, recovering work stranded by dead workers. The +// reclaim rewrites row_lock so it conflicts with any racing heartbeat/close on +// the same row; withRetryTx replays the loser. Returns the reclaimed issues. +func (s *DoltStore) ReclaimExpiredLeases(ctx context.Context, olderThan time.Duration, actor string) ([]types.ReclaimedLease, error) { + cutoff := time.Now().UTC().Add(-olderThan) + var reclaimed []types.ReclaimedLease + err := s.withRetryTx(ctx, func(tx *sql.Tx) error { + var err error + reclaimed, err = issueops.ReclaimExpiredLeasesInTx(ctx, tx, cutoff, actor) + if err != nil { + return err + } + if len(reclaimed) == 0 { + return nil + } + for _, table := range []string{"issues", "events"} { + _, _ = tx.ExecContext(ctx, "CALL DOLT_ADD(?)", table) + } + commitMsg := fmt.Sprintf("bd: reclaim %d expired lease(s)", len(reclaimed)) + if _, err := tx.ExecContext(ctx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)", + commitMsg, s.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) { + return fmt.Errorf("dolt commit: %w", err) + } + return nil + }) + if err != nil { + return nil, err } - return claimed, nil + return reclaimed, nil } // ReopenIssue reopens a closed issue, setting status to open and clearing @@ -306,31 +366,29 @@ func (s *DoltStore) CloseIssue(ctx context.Context, id string, reason string, ac return s.closeWisp(ctx, id, reason, actor, session) } - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - - if _, err := issueops.CloseIssueInTx(ctx, tx, id, reason, actor, session); err != nil { - return err - } - - // Dolt versioning for permanent issues. - // GH#2455: Stage only the tables we modified, then commit without -A. - for _, table := range []string{"issues", "events"} { - _, _ = tx.ExecContext(ctx, "CALL DOLT_ADD(?)", table) - } - commitMsg := fmt.Sprintf("bd: close %s", id) - if _, err := tx.ExecContext(ctx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)", - commitMsg, s.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) { - return fmt.Errorf("dolt commit: %w", err) - } + // Wrap in withRetryTx so a concurrent writer that loses Dolt's optimistic + // commit-time merge (MySQL 1213/1205, guaranteed server-side rollback) is + // retried rather than surfaced as a hard failure. Dolt has no real row + // locking — FOR UPDATE / SKIP LOCKED are parse-only no-ops + // (https://www.dolthub.com/blog/2023-10-23-hold-my-beer/) — so retry is the + // only safety net. withRetryTx owns BeginTx and the final Commit. + return s.withRetryTx(ctx, func(tx *sql.Tx) error { + if _, err := issueops.CloseIssueInTx(ctx, tx, id, reason, actor, session); err != nil { + return err + } - if err := tx.Commit(); err != nil { - return wrapTransactionError("commit close issue", err) - } - return nil + // Dolt versioning for permanent issues. + // GH#2455: Stage only the tables we modified, then commit without -A. + for _, table := range []string{"issues", "events"} { + _, _ = tx.ExecContext(ctx, "CALL DOLT_ADD(?)", table) + } + commitMsg := fmt.Sprintf("bd: close %s", id) + if _, err := tx.ExecContext(ctx, "CALL DOLT_COMMIT('-m', ?, '--author', ?)", + commitMsg, s.commitAuthorString()); err != nil && !isDoltNothingToCommit(err) { + return fmt.Errorf("dolt commit: %w", err) + } + return nil + }) } // DeleteIssue permanently removes an issue diff --git a/internal/storage/dolt/lease_test.go b/internal/storage/dolt/lease_test.go new file mode 100644 index 000000000..28fbe0550 --- /dev/null +++ b/internal/storage/dolt/lease_test.go @@ -0,0 +1,413 @@ +package dolt + +import ( + "context" + "database/sql" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// leaseState reads the lease columns for an issue directly, for assertions. +type leaseState struct { + status string + assignee sql.NullString + leaseExpires sql.NullTime + heartbeatAt sql.NullTime + rowLock int64 + startedAtNull bool +} + +func readLeaseState(t *testing.T, ctx context.Context, store *DoltStore, id string) leaseState { + t.Helper() + var ls leaseState + var startedAt sql.NullTime + err := store.db.QueryRowContext(ctx, ` + SELECT status, assignee, lease_expires_at, heartbeat_at, row_lock, started_at + FROM issues WHERE id = ? + `, id).Scan(&ls.status, &ls.assignee, &ls.leaseExpires, &ls.heartbeatAt, &ls.rowLock, &startedAt) + if err != nil { + t.Fatalf("read lease state for %s: %v", id, err) + } + ls.startedAtNull = !startedAt.Valid + return ls +} + +func seedClaimedIssue(t *testing.T, ctx context.Context, store *DoltStore, id, owner string, ttl time.Duration) { + t.Helper() + issue := &types.Issue{ + ID: id, + Title: "lease " + id, + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + } + if err := store.CreateIssue(ctx, issue, "seeder"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + claimCtx := issueops.WithLeaseTTL(ctx, ttl) + if err := store.ClaimIssue(claimCtx, id, owner); err != nil { + t.Fatalf("claim %s by %s: %v", id, owner, err) + } +} + +// TestClaimStampsLease verifies a claim sets a future lease, a heartbeat +// timestamp, and a non-zero row_lock. +func TestClaimStampsLease(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + seedClaimedIssue(t, ctx, store, "lease-claim", "alice", time.Minute) + ls := readLeaseState(t, ctx, store, "lease-claim") + + if ls.status != "in_progress" { + t.Errorf("status = %q, want in_progress", ls.status) + } + if ls.assignee.String != "alice" { + t.Errorf("assignee = %q, want alice", ls.assignee.String) + } + if !ls.leaseExpires.Valid || !ls.leaseExpires.Time.After(time.Now()) { + t.Errorf("lease_expires_at = %v, want a future time", ls.leaseExpires) + } + if !ls.heartbeatAt.Valid { + t.Error("heartbeat_at is NULL, want set on claim") + } + if ls.rowLock == 0 { + t.Error("row_lock = 0, want a fresh non-zero value on claim") + } +} + +// TestHeartbeatExtendsLeaseAndGuardsOwnership verifies heartbeat pushes the +// lease forward and rewrites row_lock, that only the owner may heartbeat, and +// that a heartbeat on a closed issue fails. +func TestHeartbeatExtendsLeaseAndGuardsOwnership(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + seedClaimedIssue(t, ctx, store, "lease-hb", "alice", time.Minute) + before := readLeaseState(t, ctx, store, "lease-hb") + + time.Sleep(1100 * time.Millisecond) // DATETIME is second-granular; ensure a tick + if err := store.HeartbeatIssue(ctx, "lease-hb", "alice"); err != nil { + t.Fatalf("owner heartbeat: %v", err) + } + after := readLeaseState(t, ctx, store, "lease-hb") + if !after.leaseExpires.Time.After(before.leaseExpires.Time) { + t.Errorf("heartbeat did not extend lease: before=%v after=%v", before.leaseExpires.Time, after.leaseExpires.Time) + } + if after.rowLock == before.rowLock { + t.Error("heartbeat did not rewrite row_lock") + } + + // A non-owner cannot heartbeat. + if err := store.HeartbeatIssue(ctx, "lease-hb", "mallory"); !errors.Is(err, storage.ErrAlreadyClaimed) { + t.Errorf("non-owner heartbeat err = %v, want ErrAlreadyClaimed", err) + } + + // Once closed, the lease is gone — heartbeat fails. + if err := store.CloseIssue(ctx, "lease-hb", "done", "alice", ""); err != nil { + t.Fatalf("close: %v", err) + } + if err := store.HeartbeatIssue(ctx, "lease-hb", "alice"); !errors.Is(err, storage.ErrNotClaimable) { + t.Errorf("heartbeat after close err = %v, want ErrNotClaimable", err) + } + closed := readLeaseState(t, ctx, store, "lease-hb") + if closed.leaseExpires.Valid || closed.heartbeatAt.Valid { + t.Errorf("close did not clear lease columns: %+v", closed) + } +} + +// TestReclaimRevertsExpiredOnly verifies reclaim reverts an expired lease to +// ready (clearing the owner) and leaves a still-valid lease untouched, and that +// the grace window is honored. +func TestReclaimRevertsExpiredOnly(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + // "dead" holds a 1s lease and never heartbeats; "live" holds a long lease. + seedClaimedIssue(t, ctx, store, "lease-dead", "dead-worker", time.Second) + seedClaimedIssue(t, ctx, store, "lease-live", "live-worker", time.Hour) + + time.Sleep(1500 * time.Millisecond) // let dead's lease expire + + // Grace window larger than how long the lease has been expired: nothing yet. + reclaimed, err := store.ReclaimExpiredLeases(ctx, time.Hour, "reaper") + if err != nil { + t.Fatalf("reclaim with big grace: %v", err) + } + if len(reclaimed) != 0 { + t.Errorf("reclaimed %v with a 1h grace window, want none", reclaimed) + } + + // Zero grace: the dead lease (expired) is reclaimed, the live one is not. + reclaimed, err = store.ReclaimExpiredLeases(ctx, 0, "reaper") + if err != nil { + t.Fatalf("reclaim grace=0: %v", err) + } + if len(reclaimed) != 1 || reclaimed[0].ID != "lease-dead" { + t.Fatalf("reclaimed = %+v, want exactly [lease-dead]", reclaimed) + } + if reclaimed[0].PreviousOwner != "dead-worker" { + t.Errorf("PreviousOwner = %q, want dead-worker", reclaimed[0].PreviousOwner) + } + + dead := readLeaseState(t, ctx, store, "lease-dead") + if dead.status != "open" || dead.assignee.Valid && dead.assignee.String != "" { + t.Errorf("reclaimed issue state = %+v, want open + unassigned", dead) + } + if dead.leaseExpires.Valid || dead.heartbeatAt.Valid || !dead.startedAtNull { + t.Errorf("reclaim did not clear lease/started columns: %+v", dead) + } + live := readLeaseState(t, ctx, store, "lease-live") + if live.status != "in_progress" || live.assignee.String != "live-worker" { + t.Errorf("live issue disturbed by reclaim: %+v", live) + } + + // A recovery event was recorded for the reclaimed issue. + var events int + if err := store.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM events WHERE issue_id = ? AND event_type = 'lease_reclaimed'`, "lease-dead").Scan(&events); err != nil { + t.Fatalf("count reclaim events: %v", err) + } + if events != 1 { + t.Errorf("lease_reclaimed events = %d, want 1", events) + } +} + +// TestRowLockForcesConflictOnDisjointCellWrites is the regression guard for the +// row_lock trick. It shows, against real Dolt, that two concurrent transactions +// writing DISJOINT cells of the same issue row silently cell-merge into a +// corrupt "zombie" state — UNLESS both also rewrite the shared row_lock cell, +// which forces the second commit to conflict (1213) so withRetryTx can replay. +// +// The dangerous pair: a worker's heartbeat (writes heartbeat_at) racing a reaper +// reverting the row to ready (writes status/assignee). Without a shared lock +// cell Dolt merges them, producing an open+unassigned issue that still carries +// the worker's fresh heartbeat — the worker believes it owns work that another +// worker can now also claim. +func TestRowLockForcesConflictOnDisjointCellWrites(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + // runRace commits two concurrent disjoint-cell writers. withLock adds the + // shared row_lock cell to both. Returns the error from the second commit. + runRace := func(id string, withLock bool) error { + seedClaimedIssue(t, ctx, store, id, "owner", time.Hour) + + tx1, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin tx1: %v", err) + } + tx2, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin tx2: %v", err) + } + + hbSQL := "UPDATE issues SET heartbeat_at = ? WHERE id = ?" + reclaimSQL := "UPDATE issues SET status = 'open', assignee = NULL WHERE id = ?" + hbArgs := []any{time.Now().UTC(), id} + reclaimArgs := []any{id} + if withLock { + hbSQL = "UPDATE issues SET heartbeat_at = ?, row_lock = ? WHERE id = ?" + hbArgs = []any{time.Now().UTC(), int64(111111), id} + reclaimSQL = "UPDATE issues SET status = 'open', assignee = NULL, row_lock = ? WHERE id = ?" + reclaimArgs = []any{int64(222222), id} + } + + if _, err := tx1.ExecContext(ctx, hbSQL, hbArgs...); err != nil { + t.Fatalf("tx1 heartbeat exec: %v", err) + } + if _, err := tx2.ExecContext(ctx, reclaimSQL, reclaimArgs...); err != nil { + t.Fatalf("tx2 reclaim exec: %v", err) + } + // Commit the heartbeat first, then the reclaim. The reclaim is the loser + // that must conflict when both touch row_lock. + if err := tx1.Commit(); err != nil { + t.Fatalf("tx1 commit (heartbeat): %v", err) + } + return tx2.Commit() + } + + // WITHOUT row_lock: the disjoint writes merge with no error, leaving a zombie. + if err := runRace("zombie-norowlock", false); err != nil { + // A conflict here would also be acceptable (still safe), but the point of + // the test is to demonstrate the silent merge, so surface if Dolt's + // behavior changed. + t.Skipf("expected silent merge without row_lock, got conflict %v (Dolt merge semantics changed)", err) + } + z := readLeaseState(t, ctx, store, "zombie-norowlock") + zombie := z.status == "open" && z.heartbeatAt.Valid && (!z.assignee.Valid || z.assignee.String == "") + if !zombie { + t.Fatalf("without row_lock expected a merged zombie (open+unassigned+heartbeat), got %+v", z) + } + t.Logf("without row_lock: cell-merge produced zombie state %+v", z) + + // WITH row_lock: both writers touch the shared cell, so the second commit + // conflicts (1213/1205) instead of silently merging. + err := runRace("zombie-rowlock", true) + if err == nil { + t.Fatal("with row_lock expected the second commit to conflict, but it succeeded (silent merge not prevented)") + } + if !isSerializationError(err) { + t.Fatalf("with row_lock got err %v, want a serialization conflict (1213/1205)", err) + } + t.Logf("with row_lock: second commit correctly conflicted: %v", err) +} + +// TestConcurrentHeartbeatReclaimClose is the integration race (the lease analog +// of TestConcurrentWorkQueueDrain). Half the workers are "live" (heartbeat, then +// close their issue); half are "dead" (claim, then go silent). A reaper runs +// continuously with a zero grace window. The store-level withRetryTx + row_lock +// must keep every issue in a consistent terminal state: +// +// - a live worker's close is never lost and never reverted by a racing reclaim +// - a dead worker's issue is recovered to ready (open, unassigned, no lease) +// - no issue is ever a zombie: open with a lingering owner/lease, or closed +// with a lingering owner/lease +func TestConcurrentHeartbeatReclaimClose(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + const numWorkers = 8 // even split live/dead + // lease_expires_at/heartbeat_at are second-granular DATETIME columns (and + // Dolt ROUNDS, not truncates), so a sub-second TTL is meaningless — it can + // round to a whole second in the future. Use second-scale timings: a 2s + // lease that live workers refresh every 500ms (always ≥1.5s in the future, + // so the reaper never touches a live claim), while dead claims expire after + // 2s and get reclaimed. + const ttl = 2 * time.Second + type job struct { + id string + owner string + live bool + } + jobs := make([]job, numWorkers) + for i := 0; i < numWorkers; i++ { + j := job{ + id: fmt.Sprintf("lease-race-%02d", i), + owner: fmt.Sprintf("worker-%02d", i), + live: i%2 == 0, + } + jobs[i] = j + seedClaimedIssue(t, ctx, store, j.id, j.owner, ttl) + } + + raceCtx, stopReaper := context.WithCancel(ctx) + var reaperErrs atomic.Int32 + var reaperDone sync.WaitGroup + reaperDone.Add(1) + go func() { + defer reaperDone.Done() + for { + select { + case <-raceCtx.Done(): + return + default: + } + if _, err := store.ReclaimExpiredLeases(raceCtx, 0, "reaper"); err != nil { + if raceCtx.Err() == nil { + reaperErrs.Add(1) + } + return + } + time.Sleep(15 * time.Millisecond) + } + }() + + var wg sync.WaitGroup + var lostClose atomic.Int32 + for _, j := range jobs { + if !j.live { + continue // dead workers do nothing; the reaper recovers them + } + wg.Add(1) + go func(j job) { + defer wg.Done() + hbCtx := issueops.WithLeaseTTL(ctx, ttl) + // Heartbeat every 500ms (well within the 2s TTL) to keep the lease + // alive while the reaper scans concurrently, then close. Each heartbeat + // and close rewrites row_lock, so they race the reaper's row_lock + // rewrites; withRetryTx must absorb any conflict. + for k := 0; k < 4; k++ { + if err := store.HeartbeatIssue(hbCtx, j.id, j.owner); err != nil { + // A live worker keeps its lease well in the future, so the + // reaper must never preempt it — a failure is a real defect. + t.Errorf("live worker %s heartbeat failed: %v", j.owner, err) + } + time.Sleep(500 * time.Millisecond) + } + if err := store.CloseIssue(ctx, j.id, "done", j.owner, ""); err != nil { + // A close can only fail here if the reaper reverted the issue to + // open and another path re-touched it; with row_lock the close + // retries and wins, so a hard failure is a real defect. + t.Errorf("live worker %s close failed: %v", j.owner, err) + lostClose.Add(1) + } + }(j) + } + wg.Wait() + stopReaper() + reaperDone.Wait() + + if n := reaperErrs.Load(); n != 0 { + t.Errorf("reaper surfaced %d errors; ReclaimExpiredLeases should retry conflicts internally", n) + } + if n := lostClose.Load(); n != 0 { + t.Errorf("%d live-worker closes were lost to a racing reclaim", n) + } + + // Final consistency sweep: wait past the TTL (plus the ≤1s DATETIME rounding + // slop) so every dead claim's lease is unambiguously expired, then drain so + // dead workers' issues reach their terminal open state regardless of reaper + // timing. + time.Sleep(ttl + 1500*time.Millisecond) + if _, err := store.ReclaimExpiredLeases(ctx, 0, "final-reaper"); err != nil { + t.Fatalf("final reclaim sweep: %v", err) + } + + closed, open := 0, 0 + for _, j := range jobs { + ls := readLeaseState(t, ctx, store, j.id) + hasOwner := ls.assignee.Valid && ls.assignee.String != "" + switch ls.status { + case "closed": + closed++ + // A closed issue must carry no live lease (close clears it). + if ls.leaseExpires.Valid || ls.heartbeatAt.Valid { + t.Errorf("issue %s closed but still leased: %+v", j.id, ls) + } + case "open": + open++ + // A reclaimed issue must be fully released: no owner, no lease. + if hasOwner || ls.leaseExpires.Valid || ls.heartbeatAt.Valid { + t.Errorf("issue %s open but still owned/leased (zombie): %+v", j.id, ls) + } + case "in_progress": + t.Errorf("issue %s still in_progress after race settled: %+v", j.id, ls) + default: + t.Errorf("issue %s unexpected status %q", j.id, ls.status) + } + } + t.Logf("lease race settled: %d closed (live), %d open (reclaimed)", closed, open) + // Every live issue should have ended closed; every dead one open. + if closed == 0 || open == 0 { + t.Errorf("expected a mix of closed (live) and open (reclaimed) issues, got closed=%d open=%d", closed, open) + } +} diff --git a/internal/storage/dolt/schema_version_test.go b/internal/storage/dolt/schema_version_test.go index 5073299d9..4f72d3f69 100644 --- a/internal/storage/dolt/schema_version_test.go +++ b/internal/storage/dolt/schema_version_test.go @@ -161,7 +161,13 @@ func TestMigration0053PromotesRigWisps(t *testing.T) { t.Fatalf("commit seed fixture: %v", err) } - if _, err := store.db.ExecContext(ctx, "DELETE FROM schema_migrations WHERE version = ?", schema.LatestVersion()); err != nil { + // This test exercises the rig-wisp promotion in migration 0053 specifically. + // MigrateUp only re-applies versions strictly greater than MAX(applied), so to + // replay 0053 we delete every row >= 53 (not just LatestVersion(), which now + // points past 0053 as later migrations like 0054 land). 0053 re-runs the + // promotion; any later migration replays as a guarded no-op. + const rigWispsMigrationVersion = 53 + if _, err := store.db.ExecContext(ctx, "DELETE FROM schema_migrations WHERE version >= ?", rigWispsMigrationVersion); err != nil { t.Fatalf("mark 0053 pending: %v", err) } if _, err := schema.MigrateUp(ctx, store.db); err != nil { diff --git a/internal/storage/doltlite/backfill_custom_tables.go b/internal/storage/doltlite/backfill_custom_tables.go new file mode 100644 index 000000000..0ccbf02ff --- /dev/null +++ b/internal/storage/doltlite/backfill_custom_tables.go @@ -0,0 +1,143 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + + "github.com/steveyegge/beads/internal/types" +) + +// backfillCustomTables populates empty custom_types and custom_statuses +// tables from the corresponding config values. This mirrors the Dolt +// backend's BackfillCustomTables migration (016) and repairs databases +// where the schema migration (0024) created the tables but did not +// backfill them from config. +// +// Must be called after the persistent DB is open (s.db != nil). +func (s *DoltliteStore) backfillCustomTables(ctx context.Context) error { + db, _, err := s.activeDB(ctx) + if err != nil { + return fmt.Errorf("backfill: open db: %w", err) + } + // Don't close the returned db — activeDB returns the persistent handle + // or a clone; we use it for one-shot queries and let the connection + // pool reuse it. + + if err := backfillCustomTypesSQLite(ctx, db); err != nil { + return fmt.Errorf("custom_types: %w", err) + } + if err := backfillCustomStatusesSQLite(ctx, db); err != nil { + return fmt.Errorf("custom_statuses: %w", err) + } + return nil +} + +func backfillCustomTypesSQLite(ctx context.Context, db *sql.DB) error { + // Check table exists + var hasTable int + if err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='custom_types'", + ).Scan(&hasTable); err != nil || hasTable == 0 { + return err + } + + // Skip if already populated + var count int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM custom_types").Scan(&count); err != nil { + return err + } + if count > 0 { + return nil + } + + // Read types.custom from config + var value string + err := db.QueryRowContext(ctx, + "SELECT value FROM config WHERE `key` = ?", "types.custom", + ).Scan(&value) + if err != nil || value == "" { + return nil // No config to backfill from + } + + for _, name := range parseTypesValue(value) { + _, err = db.ExecContext(ctx, + "INSERT OR IGNORE INTO custom_types (name) VALUES (?)", name, + ) + if err != nil { + return fmt.Errorf("inserting type %q: %w", name, err) + } + } + return nil +} + +func backfillCustomStatusesSQLite(ctx context.Context, db *sql.DB) error { + var hasTable int + if err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='custom_statuses'", + ).Scan(&hasTable); err != nil || hasTable == 0 { + return err + } + + var count int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM custom_statuses").Scan(&count); err != nil { + return err + } + if count > 0 { + return nil + } + + var value string + err := db.QueryRowContext(ctx, + "SELECT value FROM config WHERE `key` = ?", "status.custom", + ).Scan(&value) + if err != nil || value == "" { + return nil + } + + parsed, parseErr := types.ParseCustomStatusConfig(value) + if parseErr != nil { + // Invalid config value: log and skip (same behavior as Dolt migration) + return nil + } + for _, s := range parsed { + _, err = db.ExecContext(ctx, + "INSERT OR IGNORE INTO custom_statuses (name, category) VALUES (?, ?)", + s.Name, string(s.Category), + ) + if err != nil { + return fmt.Errorf("inserting status %q: %w", s.Name, err) + } + } + return nil +} + +// parseTypesValue tries JSON array first, then falls back to comma-separated. +// Mirrors dolt/migrations/015_custom_status_type_tables.go:parseTypesValue. +func parseTypesValue(value string) []string { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + var jsonTypes []string + if err := json.Unmarshal([]byte(value), &jsonTypes); err == nil { + return jsonTypes + } + return splitCommaSeparated(value) +} + +func splitCommaSeparated(value string) []string { + parts := strings.Split(value, ",") + result := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + result = append(result, p) + } + } + return result +} diff --git a/internal/storage/doltlite/backfill_custom_tables_test.go b/internal/storage/doltlite/backfill_custom_tables_test.go new file mode 100644 index 000000000..692d831d8 --- /dev/null +++ b/internal/storage/doltlite/backfill_custom_tables_test.go @@ -0,0 +1,56 @@ +//go:build cgo + +package doltlite_test + +import ( + "context" + "path/filepath" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/storage/doltlite" +) + +// TestBackfillCustomTablesOnNew verifies that creating a new doltlite store +// leaves normalized custom config tables stable across reopen. Fresh upstream +// config does not seed types.custom, so the tables may legitimately be empty. +func TestBackfillCustomTablesOnNew(t *testing.T) { + ctx := context.Background() + + dir := filepath.Join(t.TempDir(), ".beads") + + // First open: backfill should succeed even when config has no custom types. + store1, err := doltlite.New(ctx, dir, "beads", "main") + if err != nil { + t.Fatalf("New (first): %v", err) + } + + types1, err := store1.GetCustomTypes(ctx) + if err != nil { + store1.Close() + t.Fatalf("GetCustomTypes (first): %v", err) + } + if !slices.Equal(types1, []string{}) { + store1.Close() + t.Fatalf("fresh custom types = %v, want empty", types1) + } + + if err := store1.Close(); err != nil { + t.Fatalf("Close (first): %v", err) + } + + // Re-open: backfill should be a no-op (table already populated). + store2, err := doltlite.New(ctx, dir, "beads", "main") + if err != nil { + t.Fatalf("New (second): %v", err) + } + defer store2.Close() + + types2, err := store2.GetCustomTypes(ctx) + if err != nil { + t.Fatalf("GetCustomTypes (second): %v", err) + } + if !slices.Equal(types1, types2) { + t.Errorf("custom types changed across re-open:\n first: %v\n second: %v", types1, types2) + } +} diff --git a/internal/storage/doltlite/cache_cleanup.go b/internal/storage/doltlite/cache_cleanup.go new file mode 100644 index 000000000..70a21ba36 --- /dev/null +++ b/internal/storage/doltlite/cache_cleanup.go @@ -0,0 +1,94 @@ +//go:build cgo + +package doltlite + +import ( + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// cleanGitRemoteCacheGarbage removes orphaned tmp_pack_* files from the +// Dolt git-remote-cache. These files are created by `git fetch` (invoked +// by Dolt's GitBlobstore) and should be renamed to final .pack/.idx files +// on success or deleted on failure. In practice, failed or interrupted +// fetches leave them behind indefinitely, and Dolt's built-in periodic +// git gc (maybeRunGC, gated to once per 24h) either never runs or cannot +// keep up with the accumulation rate. +// +// On a real machine with normal beads usage, this leak consumed 102 GB +// (412 files) in 7 days. See https://github.com/gastownhall/beads/issues/3354 +// +// This function is safe to call concurrently and is rate-limited to avoid +// unnecessary filesystem walks on hot paths. +func (s *DoltliteStore) cleanGitRemoteCacheGarbage() { + if !cacheCleanupThrottle.shouldRun() { + return + } + + cacheBase := filepath.Join(s.dataDir, s.database, ".dolt", "git-remote-cache") + if _, err := os.Stat(cacheBase); os.IsNotExist(err) { + return + } + + cutoff := time.Now().Add(-tmpPackMinAge) + + _ = filepath.WalkDir(cacheBase, func(path string, d os.DirEntry, err error) error { + if err != nil { + return nil // best-effort: skip unreadable entries + } + if d.IsDir() { + return nil + } + name := d.Name() + if !strings.HasPrefix(name, "tmp_pack_") && !strings.HasPrefix(name, "tmp_idx_") { + return nil + } + info, err := d.Info() + if err != nil { + return nil + } + if info.ModTime().Before(cutoff) { + // #nosec G122 -- path is under .dolt/git-remote-cache/ which is + // owned by the user running bd. A TOCTOU symlink swap would + // require write access to that directory; in that case the + // attacker already controls the Dolt data. The tmp_pack_/tmp_idx_ + // prefix check further narrows the scope to files Dolt itself writes. + _ = os.Remove(path) + } + return nil + }) +} + +const ( + // tmpPackMinAge is the minimum age before a tmp_pack file is considered + // garbage. Files younger than this may belong to an in-progress fetch. + tmpPackMinAge = 5 * time.Minute + + // cacheCleanupInterval is how often cleanGitRemoteCacheGarbage actually + // walks the filesystem when called repeatedly. + cacheCleanupInterval = 10 * time.Minute +) + +// throttle gates a function to run at most once per interval. +type throttle struct { + mu sync.Mutex + interval time.Duration + lastRun time.Time +} + +func (t *throttle) shouldRun() bool { + t.mu.Lock() + defer t.mu.Unlock() + if time.Since(t.lastRun) < t.interval { + return false + } + t.lastRun = time.Now() + return true +} + +// cacheCleanupThrottle is a package-level throttle shared across all +// DoltliteStore instances in the same process. +var cacheCleanupThrottle = &throttle{interval: cacheCleanupInterval} diff --git a/internal/storage/doltlite/child_id.go b/internal/storage/doltlite/child_id.go new file mode 100644 index 000000000..1b21647b7 --- /dev/null +++ b/internal/storage/doltlite/child_id.go @@ -0,0 +1,63 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "fmt" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +func (s *DoltliteStore) GetNextChildID(ctx context.Context, parentID string) (string, error) { + var childID string + err := s.withConn(ctx, true, func(tx *sql.Tx) error { + counterTable, issueTable := "child_counters", "issues" + if issueops.IsActiveWispInTx(ctx, tx, parentID) { + counterTable, issueTable = "wisp_child_counters", "wisps" + } + + var lastChild int + err := tx.QueryRowContext(ctx, fmt.Sprintf("SELECT last_child FROM %s WHERE parent_id = ?", counterTable), parentID).Scan(&lastChild) + if err == sql.ErrNoRows { + lastChild = 0 + } else if err != nil { + return fmt.Errorf("get next child ID: read counter: %w", err) + } + + rows, err := tx.QueryContext(ctx, fmt.Sprintf(` + SELECT id FROM %s + WHERE id LIKE ? + AND id NOT LIKE ? + `, issueTable), parentID+".%", parentID+".%.%") + if err != nil { + return fmt.Errorf("get next child ID: query existing children: %w", err) + } + defer rows.Close() + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return fmt.Errorf("get next child ID: scan child row: %w", err) + } + _, childNum, ok := issueops.ParseHierarchicalID(id) + if ok && childNum > lastChild { + lastChild = childNum + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("get next child ID: iterate children: %w", err) + } + + nextChild := lastChild + 1 + if _, err := tx.ExecContext(ctx, fmt.Sprintf(` + INSERT INTO %s (parent_id, last_child) VALUES (?, ?) + ON CONFLICT(parent_id) DO UPDATE SET last_child = excluded.last_child + `, counterTable), parentID, nextChild); err != nil { + return fmt.Errorf("get next child ID: update counter: %w", err) + } + childID = fmt.Sprintf("%s.%d", parentID, nextChild) + return nil + }) + return childID, err +} diff --git a/internal/storage/doltlite/commit_pending.go b/internal/storage/doltlite/commit_pending.go new file mode 100644 index 000000000..d2af36b52 --- /dev/null +++ b/internal/storage/doltlite/commit_pending.go @@ -0,0 +1,77 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "fmt" + "strings" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +func buildDoltliteBatchCommitMessage(ctx context.Context, db issueops.SQLQuerier, actor string) string { + if actor == "" { + actor = "bd" + } + + var added, modified, removed int + rows, err := db.QueryContext(ctx, ` + SELECT diff_type, COUNT(*) as cnt + FROM dolt_diff_issues('HEAD', 'WORKING') + GROUP BY diff_type + `) + if err == nil { + defer rows.Close() + for rows.Next() { + var diffType string + var count int + if scanErr := rows.Scan(&diffType, &count); scanErr == nil { + switch diffType { + case "added": + added = count + case "modified": + modified = count + case "removed": + removed = count + } + } + } + _ = rows.Err() + } + + var otherTables []string + statusRows, statusErr := db.QueryContext(ctx, "SELECT table_name FROM dolt_status WHERE table_name != 'issues' ORDER BY table_name") + if statusErr == nil { + defer statusRows.Close() + for statusRows.Next() { + var table string + if scanErr := statusRows.Scan(&table); scanErr == nil { + if isDoltliteRuntimeTable(table) { + continue + } + otherTables = append(otherTables, table) + } + } + _ = statusRows.Err() + } + + msg := fmt.Sprintf("bd: batch commit by %s", actor) + var parts []string + if added > 0 { + parts = append(parts, fmt.Sprintf("%d created", added)) + } + if modified > 0 { + parts = append(parts, fmt.Sprintf("%d updated", modified)) + } + if removed > 0 { + parts = append(parts, fmt.Sprintf("%d deleted", removed)) + } + if len(parts) > 0 { + msg += " - " + strings.Join(parts, ", ") + } + if len(otherTables) > 0 { + msg += fmt.Sprintf(" (+ %s)", strings.Join(otherTables, ", ")) + } + return msg +} diff --git a/internal/storage/doltlite/config_metadata.go b/internal/storage/doltlite/config_metadata.go new file mode 100644 index 000000000..8072fe7fe --- /dev/null +++ b/internal/storage/doltlite/config_metadata.go @@ -0,0 +1,115 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "fmt" + + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) SetConfig(ctx context.Context, key, value string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + if err := issueops.SetConfigInTx(ctx, tx, key, value); err != nil { + return err + } + // Sync normalized tables when config keys change + switch key { + case "status.custom": + if err := issueops.SyncCustomStatusesTable(ctx, tx, value); err != nil { + return fmt.Errorf("syncing custom_statuses table: %w", err) + } + case "types.custom": + if err := issueops.SyncCustomTypesTable(ctx, tx, value); err != nil { + return fmt.Errorf("syncing custom_types table: %w", err) + } + } + return nil + }) +} + +func (s *DoltliteStore) GetConfig(ctx context.Context, key string) (string, error) { + var value string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + value, err = issueops.GetConfigInTx(ctx, tx, key) + return err + }) + return value, err +} + +func (s *DoltliteStore) GetAllConfig(ctx context.Context) (map[string]string, error) { + var result map[string]string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetAllConfigInTx(ctx, tx) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetMetadata(ctx context.Context, key string) (string, error) { + var value string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + value, err = issueops.GetMetadataInTx(ctx, tx, key) + return err + }) + return value, err +} + +func (s *DoltliteStore) SetMetadata(ctx context.Context, key, value string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.SetMetadataInTx(ctx, tx, key, value) + }) +} + +func (s *DoltliteStore) SetLocalMetadata(ctx context.Context, key, value string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.SetLocalMetadataInTx(ctx, tx, key, value) + }) +} + +func (s *DoltliteStore) GetLocalMetadata(ctx context.Context, key string) (string, error) { + var value string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + value, err = issueops.GetLocalMetadataInTx(ctx, tx, key) + return err + }) + return value, err +} + +// GetInfraTypes returns the set of infrastructure types that should be routed +// to the wisps table. Reads from DB config "types.infra", falls back to YAML, +// then to hardcoded defaults (agent, role, message). +func (s *DoltliteStore) GetInfraTypes(ctx context.Context) map[string]bool { + var result map[string]bool + if err := s.withConn(ctx, false, func(tx *sql.Tx) error { + result = issueops.ResolveInfraTypesInTx(ctx, tx) + return nil + }); err != nil || result == nil { + // DB unavailable — fall back to YAML then defaults. + var typeList []string + if yamlTypes := config.GetInfraTypesFromYAML(); len(yamlTypes) > 0 { + typeList = yamlTypes + } else { + typeList = domain.DefaultInfraTypes() + } + result = make(map[string]bool, len(typeList)) + for _, t := range typeList { + result[t] = true + } + } + return result +} + +// IsInfraTypeCtx returns true if the issue type is an infrastructure type. +func (s *DoltliteStore) IsInfraTypeCtx(ctx context.Context, t types.IssueType) bool { + return s.GetInfraTypes(ctx)[string(t)] +} diff --git a/internal/storage/doltlite/conformance_test.go b/internal/storage/doltlite/conformance_test.go new file mode 100644 index 000000000..cbde8b20d --- /dev/null +++ b/internal/storage/doltlite/conformance_test.go @@ -0,0 +1,38 @@ +//go:build cgo + +package doltlite_test + +import ( + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/conformance" + "github.com/steveyegge/beads/internal/storage/doltlite" +) + +// TestConformance runs the backend-agnostic storage conformance suite +// (internal/storage/conformance) against the DoltLite backend. TestMain in this +// package self-skips when the native libdoltlite-backed sqlite driver is not +// linked, so `make test-doltlite` is the intended entry point. +func TestConformance(t *testing.T) { + conformance.RunAll(t, func(t *testing.T) storage.DoltStorage { + ctx := t.Context() + beadsDir := filepath.Join(t.TempDir(), ".beads") + store, err := doltlite.New(ctx, beadsDir, "beads", "main") + if err != nil { + t.Fatalf("New DoltLite store: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + // Match the post-`bd init` contract expected by the shared conformance + // factory: an initialized store with issue_prefix configured. + if err := store.SetConfig(ctx, "issue_prefix", "test"); err != nil { + t.Fatalf("SetConfig(issue_prefix): %v", err) + } + if err := store.Commit(ctx, "bd init"); err != nil { + t.Fatalf("Commit: %v", err) + } + return store + }) +} diff --git a/internal/storage/doltlite/counts.go b/internal/storage/doltlite/counts.go new file mode 100644 index 000000000..dcaeeb1ff --- /dev/null +++ b/internal/storage/doltlite/counts.go @@ -0,0 +1,154 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// depTargetExpr resolves a dependency row's target id from the split physical +// columns instead of the STORED generated `depends_on_id` column. Both +// `dependencies` and `wisp_dependencies` define depends_on_id as +// GENERATED ALWAYS AS (COALESCE(depends_on_issue_id, depends_on_wisp_id, +// depends_on_external)). Count queries must filter on the base columns: inside +// a count(*) (which projects no real columns) the pure-Go GMS analyzer can +// prune the base columns the generated column derives from and then fail with +// "column depends_on_id could not be found in any table in scope". The slice +// path projects real columns, so it can use depends_on_id directly. This +// matches issueops.DepTargetExpr on main. +const depTargetExpr = "COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external)" + +// CountIssues returns the number of issues matching query and filter. +// Filter.Limit and Filter.Offset are ignored; all other fields apply. +// Wisps-merge semantics follow SearchIssues: SkipWisps=true counts the +// durable issues table only, otherwise the wisps tier is merged in (GH#4387). +func (s *DoltliteStore) CountIssues(ctx context.Context, query string, filter types.IssueFilter) (int64, error) { + var n int64 + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + count, err := issueops.CountIssuesSQLiteInTx(ctx, tx, query, filter) + if err != nil { + return err + } + n = int64(count) + return nil + }) + return n, err +} + +// CountIssuesByGroup returns per-group issue counts. groupBy is one of: +// status, priority, type, assignee, label. +func (s *DoltliteStore) CountIssuesByGroup(ctx context.Context, filter types.IssueFilter, groupBy string) (map[string]int, error) { + var result map[string]int + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.CountIssuesByGroupSQLiteInTx(ctx, tx, filter, groupBy) + return err + }) + return result, err +} + +// CountDependents counts both dependency tables so the total matches +// GetDependentsWithMetadata: a dependent may be a permanent issue (edge in +// `dependencies`) or a wisp (edge in `wisp_dependencies`). Counted in separate +// top-level queries and summed in Go. +// +// Both tables' targets are resolved via depTargetExpr (the split physical +// columns) rather than the STORED generated depends_on_id, which a count(*) +// can fail to resolve under the pure-Go GMS analyzer. +func (s *DoltliteStore) CountDependents(ctx context.Context, issueID string) (int64, error) { + var n int64 + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var perm, wisp int64 + if err := tx.QueryRowContext(ctx, + `SELECT count(*) FROM dependencies WHERE `+depTargetExpr+` = ?`, issueID).Scan(&perm); err != nil { + return err + } + if err := tx.QueryRowContext(ctx, + `SELECT count(*) FROM wisp_dependencies WHERE `+depTargetExpr+` = ?`, issueID).Scan(&wisp); err != nil { + return err + } + n = perm + wisp + return nil + }) + return n, err +} + +// CountDependencies counts both dependency tables so the total matches +// GetDependenciesWithMetadata: a wisp's outgoing edges live in +// `wisp_dependencies`, a permanent issue's in `dependencies`. Counted as two +// separate queries summed in Go (see CountDependents for why a single combined +// query is avoided). +func (s *DoltliteStore) CountDependencies(ctx context.Context, issueID string) (int64, error) { + var n int64 + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var perm, wisp int64 + if err := tx.QueryRowContext(ctx, + `SELECT count(*) FROM dependencies WHERE issue_id = ?`, issueID).Scan(&perm); err != nil { + return err + } + if err := tx.QueryRowContext(ctx, + `SELECT count(*) FROM wisp_dependencies WHERE issue_id = ?`, issueID).Scan(&wisp); err != nil { + return err + } + n = perm + wisp + return nil + }) + return n, err +} + +func (s *DoltliteStore) CountIssueComments(ctx context.Context, issueID string) (int64, error) { + var n int64 + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + return tx.QueryRowContext(ctx, + `SELECT count(*) FROM comments WHERE issue_id = ?`, issueID).Scan(&n) + }) + return n, err +} + +func (s *DoltliteStore) CountEvents(ctx context.Context, issueID string, limit int) (int64, error) { + var n int64 + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + return tx.QueryRowContext(ctx, + `SELECT count(*) FROM events WHERE issue_id = ?`, issueID).Scan(&n) + }) + if err != nil { + return 0, err + } + if limit > 0 && n > int64(limit) { + n = int64(limit) + } + return n, nil +} + +// CountDependentsByStatus counts both dependency tables, joining each to its +// home issue table (dependencies→issues, wisp_dependencies→wisps), so wisp +// dependents are included the same way GetDependentsWithMetadata includes them. +// Counted as two separate queries summed in Go (see CountDependents for why a +// single combined query is avoided). +func (s *DoltliteStore) CountDependentsByStatus(ctx context.Context, issueID string, status types.Status) (int64, error) { + var n int64 + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var perm, wisp int64 + if err := tx.QueryRowContext(ctx, + `SELECT count(*) FROM dependencies d + JOIN issues i ON i.id = d.issue_id + WHERE COALESCE(d.depends_on_issue_id, d.depends_on_wisp_id, d.depends_on_external) = ? AND i.status = ?`, + issueID, string(status)).Scan(&perm); err != nil { + return err + } + if err := tx.QueryRowContext(ctx, + `SELECT count(*) FROM wisp_dependencies d + JOIN wisps w ON w.id = d.issue_id + WHERE COALESCE(d.depends_on_issue_id, d.depends_on_wisp_id, d.depends_on_external) = ? AND w.status = ?`, + issueID, string(status)).Scan(&wisp); err != nil { + return err + } + n = perm + wisp + return nil + }) + return n, err +} diff --git a/internal/storage/doltlite/create_issue.go b/internal/storage/doltlite/create_issue.go new file mode 100644 index 000000000..9c5a22438 --- /dev/null +++ b/internal/storage/doltlite/create_issue.go @@ -0,0 +1,197 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "fmt" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) CreateIssue(ctx context.Context, issue *types.Issue, actor string) error { + if issue == nil { + return fmt.Errorf("issue must not be nil") + } + // Route infra types to wisps, matching DoltStore.CreateIssue behavior. + if s.IsInfraTypeCtx(ctx, issue.IssueType) { + issue.Ephemeral = true + } + + return s.withConn(ctx, true, func(tx *sql.Tx) error { + // SkipPrefixValidation matches DoltStore.CreateIssue, which does not + // validate prefixes for explicit IDs on the single-issue path. + bc, err := issueops.NewBatchContext(ctx, tx, storage.BatchCreateOptions{ + SkipPrefixValidation: true, + }) + if err != nil { + return err + } + return createIssueSQLite(ctx, tx, bc, issue, actor) + }) +} + +func (s *DoltliteStore) CreateIssues(ctx context.Context, issues []*types.Issue, actor string) error { + return s.CreateIssuesWithFullOptions(ctx, issues, actor, storage.BatchCreateOptions{ + OrphanHandling: storage.OrphanAllow, + SkipPrefixValidation: false, + }) +} + +func (s *DoltliteStore) CreateIssuesWithFullOptions(ctx context.Context, issues []*types.Issue, actor string, opts storage.BatchCreateOptions) error { + if len(issues) == 0 { + return nil + } + + // All-wisps fast path: create each wisp/no-history issue individually within + // its own transaction, threading opts through so that callers' + // SkipPrefixValidation / OrphanHandling settings are respected. + if issueops.AllWisps(issues) { + for _, issue := range issues { + if !issue.NoHistory { + issue.Ephemeral = true + } + if err := s.withConn(ctx, true, func(tx *sql.Tx) error { + bc, err := issueops.NewBatchContext(ctx, tx, opts) + if err != nil { + return err + } + return createIssueSQLite(ctx, tx, bc, issue, actor) + }); err != nil { + return err + } + } + return nil + } + + return s.withConn(ctx, true, func(tx *sql.Tx) error { + bc, err := issueops.NewBatchContext(ctx, tx, opts) + if err != nil { + return err + } + for _, issue := range issues { + if err := createIssueSQLite(ctx, tx, bc, issue, actor); err != nil { + return err + } + } + return nil + }) +} + +func createIssueSQLite(ctx context.Context, tx *sql.Tx, bc *issueops.BatchContext, issue *types.Issue, actor string) error { + if err := issueops.PrepareIssueForInsert(issue, bc.CustomStatuses, bc.CustomTypes); err != nil { + return err + } + issueTable, eventTable := issueops.TableRouting(issue) + if issue.ID == "" { + prefix := bc.ConfigPrefix + if issue.PrefixOverride != "" { + prefix = issue.PrefixOverride + } else if issue.IDPrefix != "" { + prefix = bc.ConfigPrefix + "-" + issue.IDPrefix + } else if issueops.IsWisp(issue) { + prefix = bc.ConfigPrefix + "-wisp" + } + var err error + issue.ID, err = issueops.GenerateIssueIDInTable(ctx, tx, issueTable, prefix, issue, actor) + if err != nil { + return fmt.Errorf("failed to generate issue ID: %w", err) + } + } else if !bc.Opts.SkipPrefixValidation { + if err := issueops.ValidateIssueIDPrefix(issue.ID, bc.ConfigPrefix, bc.AllowedPrefixes); err != nil { + return fmt.Errorf("prefix validation failed for %s: %w", issue.ID, err) + } + } + if skip, err := issueops.CheckOrphan(ctx, tx, issue, issueTable, bc.Opts.OrphanHandling); err != nil { + return err + } else if skip { + return nil + } + var existingCount int + if err := tx.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE id = ?", issueTable), issue.ID).Scan(&existingCount); err != nil { + return fmt.Errorf("failed to check issue existence for %s: %w", issue.ID, err) + } + if err := insertIssueSQLite(ctx, tx, issueTable, issue); err != nil { + return err + } + if existingCount == 0 { + if err := issueops.RecordEventInTable(ctx, tx, eventTable, issue.ID, types.EventCreated, actor, ""); err != nil { + return fmt.Errorf("failed to record event for %s: %w", issue.ID, err) + } + } + if err := persistLabelsSQLite(ctx, tx, issue, actor, eventTable); err != nil { + return err + } + if _, err := issueops.PersistComments(ctx, tx, issue); err != nil { + return err + } + return nil +} + +func insertIssueSQLite(ctx context.Context, tx *sql.Tx, table string, issue *types.Issue) error { + _, err := tx.ExecContext(ctx, fmt.Sprintf(` + INSERT OR REPLACE INTO %s ( + id, content_hash, title, description, design, acceptance_criteria, notes, + status, priority, issue_type, assignee, estimated_minutes, + created_at, created_by, owner, updated_at, started_at, closed_at, external_ref, spec_id, + compaction_level, compacted_at, compacted_at_commit, original_size, + sender, ephemeral, no_history, wisp_type, pinned, is_template, + mol_type, work_type, source_system, source_repo, close_reason, + event_kind, actor, target, payload, + await_type, await_id, timeout_ns, waiters, + due_at, defer_until, metadata + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ? + ) + `, table), + issue.ID, issue.ContentHash, issue.Title, issue.Description, issue.Design, issue.AcceptanceCriteria, issue.Notes, + issue.Status, issue.Priority, issue.IssueType, issueops.NullString(issue.Assignee), issueops.NullInt(issue.EstimatedMinutes), + issue.CreatedAt, issue.CreatedBy, issue.Owner, issue.UpdatedAt, issue.StartedAt, issue.ClosedAt, issueops.NullStringPtr(issue.ExternalRef), issue.SpecID, + issue.CompactionLevel, issue.CompactedAt, issueops.NullStringPtr(issue.CompactedAtCommit), issueops.NullIntVal(issue.OriginalSize), + issue.Sender, issue.Ephemeral, issue.NoHistory, issue.WispType, issue.Pinned, issue.IsTemplate, + issue.MolType, issue.WorkType, issue.SourceSystem, issue.SourceRepo, issue.CloseReason, + issue.EventKind, issue.Actor, issue.Target, issue.Payload, + issue.AwaitType, issue.AwaitID, issue.Timeout.Nanoseconds(), issueops.FormatJSONStringArray(issue.Waiters), + issue.DueAt, issue.DeferUntil, issueops.JSONMetadata(issue.Metadata), + ) + if err != nil { + return fmt.Errorf("insert issue into %s: %w", table, err) + } + return nil +} + +func persistLabelsSQLite(ctx context.Context, tx *sql.Tx, issue *types.Issue, actor, eventTable string) error { + if len(issue.Labels) == 0 { + return nil + } + labelTable := "labels" + if issueops.IsWisp(issue) { + labelTable = "wisp_labels" + } + seen := make(map[string]struct{}, len(issue.Labels)) + for _, label := range issue.Labels { + if _, ok := seen[label]; ok { + continue + } + seen[label] = struct{}{} + if _, err := tx.ExecContext(ctx, fmt.Sprintf("INSERT OR IGNORE INTO %s (issue_id, label) VALUES (?, ?)", labelTable), issue.ID, label); err != nil { + return fmt.Errorf("failed to insert label %q for %s: %w", label, issue.ID, err) + } + if _, err := tx.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s (id, issue_id, event_type, actor, comment) VALUES (?, ?, ?, ?, ?)", eventTable), + issueops.NewEventID(), issue.ID, types.EventLabelAdded, actor, "Added label: "+label); err != nil { + return fmt.Errorf("failed to record label event %q for %s: %w", label, issue.ID, err) + } + } + return nil +} diff --git a/internal/storage/doltlite/dependencies.go b/internal/storage/doltlite/dependencies.go new file mode 100644 index 000000000..5e6a1f379 --- /dev/null +++ b/internal/storage/doltlite/dependencies.go @@ -0,0 +1,73 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) AddDependency(ctx context.Context, dep *types.Dependency, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.AddDependencyInTx(ctx, tx, dep, actor, issueops.AddDependencyOpts{ + IsCrossPrefix: types.ExtractPrefix(dep.IssueID) != types.ExtractPrefix(dep.DependsOnID), + UseSQLiteBlockedRecompute: true, + }) + }) +} + +// RemoveDependency removes a dependency between two issues. +func (s *DoltliteStore) RemoveDependency(ctx context.Context, issueID, dependsOnID string, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.RemoveDependencySQLiteInTx(ctx, tx, issueID, dependsOnID) + }) +} + +// GetIssuesByIDs retrieves multiple issues by ID. +func (s *DoltliteStore) GetIssuesByIDs(ctx context.Context, ids []string) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetIssuesByIDsInTx(ctx, tx, ids, nil) + return err + }) + return result, err +} + +// GetDependenciesWithMetadata returns issues that the given issue depends on, +// along with the dependency type. +func (s *DoltliteStore) GetDependenciesWithMetadata(ctx context.Context, issueID string) ([]*types.IssueWithDependencyMetadata, error) { + var result []*types.IssueWithDependencyMetadata + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependenciesWithMetadataInTx(ctx, tx, issueID) + return err + }) + return result, err +} + +// GetDependentsWithMetadata returns issues that depend on the given issue, +// along with the dependency type. +func (s *DoltliteStore) GetDependentsWithMetadata(ctx context.Context, issueID string) ([]*types.IssueWithDependencyMetadata, error) { + var result []*types.IssueWithDependencyMetadata + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependentsWithMetadataInTx(ctx, tx, issueID) + return err + }) + return result, err +} + +// DetectCycles finds dependency cycles across both permanent and wisp dependencies. +func (s *DoltliteStore) DetectCycles(ctx context.Context) ([][]*types.Issue, error) { + var result [][]*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.DetectCyclesInTx(ctx, tx) + return err + }) + return result, err +} diff --git a/internal/storage/doltlite/federation.go b/internal/storage/doltlite/federation.go new file mode 100644 index 000000000..bf92aeaca --- /dev/null +++ b/internal/storage/doltlite/federation.go @@ -0,0 +1,340 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "database/sql" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" +) + +// credentialKeyFile is the filename for the random encryption key. +const credentialKeyFile = ".beads-credential-key" //nolint:gosec // G101: filename, not a credential + +// ensureCredentialKey lazily initializes the credential encryption key. +func (s *DoltliteStore) ensureCredentialKey() error { + if s.credentialKey != nil { + return nil + } + if s.beadsDir == "" { + return fmt.Errorf("beads directory not set; credential encryption unavailable") + } + + keyPath := filepath.Join(s.beadsDir, credentialKeyFile) + + // Try to load existing key. + key, err := os.ReadFile(keyPath) //nolint:gosec // G304: keyPath derived from trusted beadsDir + if err == nil && len(key) == 32 { + s.credentialKey = key + return nil + } + + // Generate new random 32-byte key (AES-256). + key = make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, key); err != nil { + return fmt.Errorf("generate credential key: %w", err) + } + if err := os.WriteFile(keyPath, key, 0600); err != nil { + return fmt.Errorf("write credential key: %w", err) + } + + s.credentialKey = key + return nil +} + +func (s *DoltliteStore) encryptPassword(password string) ([]byte, error) { + if password == "" { + return nil, nil + } + if err := s.ensureCredentialKey(); err != nil { + return nil, err + } + block, err := aes.NewCipher(s.credentialKey) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + return gcm.Seal(nonce, nonce, []byte(password), nil), nil +} + +func (s *DoltliteStore) decryptPassword(encrypted []byte) (string, error) { + if len(encrypted) == 0 { + return "", nil + } + if err := s.ensureCredentialKey(); err != nil { + return "", err + } + block, err := aes.NewCipher(s.credentialKey) + if err != nil { + return "", err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return "", err + } + nonceSize := gcm.NonceSize() + if len(encrypted) < nonceSize { + return "", fmt.Errorf("ciphertext too short") + } + nonce, ciphertext := encrypted[:nonceSize], encrypted[nonceSize:] + plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return "", err + } + return string(plaintext), nil +} + +// --------------------------------------------------------------------------- +// FederationStore implementation +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) AddFederationPeer(ctx context.Context, peer *storage.FederationPeer) error { + encryptedPwd, err := s.encryptPassword(peer.Password) + if err != nil { + return fmt.Errorf("encrypt password: %w", err) + } + + if err := s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.AddFederationPeerInTx(ctx, tx, peer, encryptedPwd) + }); err != nil { + return err + } + if peer.RemoteURL != "" { + if err := s.AddRemote(ctx, peer.Name, peer.RemoteURL); err != nil { + return err + } + } + return nil +} + +func (s *DoltliteStore) GetFederationPeer(ctx context.Context, name string) (*storage.FederationPeer, error) { + var row *issueops.FederationPeerRow + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + row, err = issueops.GetFederationPeerInTx(ctx, tx, name) + return err + }) + if err != nil { + return nil, err + } + + if len(row.EncryptedPwd) > 0 { + row.Peer.Password, err = s.decryptPassword(row.EncryptedPwd) + if err != nil { + return nil, fmt.Errorf("decrypt password: %w", err) + } + } + return &row.Peer, nil +} + +func (s *DoltliteStore) ListFederationPeers(ctx context.Context) ([]*storage.FederationPeer, error) { + var rows []*issueops.FederationPeerRow + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + rows, err = issueops.ListFederationPeersInTx(ctx, tx) + return err + }) + if err != nil { + return nil, err + } + + peers := make([]*storage.FederationPeer, 0, len(rows)) + for _, row := range rows { + if len(row.EncryptedPwd) > 0 { + pwd, err := s.decryptPassword(row.EncryptedPwd) + if err != nil { + return nil, fmt.Errorf("decrypt password for peer %s: %w", row.Peer.Name, err) + } + row.Peer.Password = pwd + } + peers = append(peers, &row.Peer) + } + return peers, nil +} + +func (s *DoltliteStore) RemoveFederationPeer(ctx context.Context, name string) error { + if err := s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.RemoveFederationPeerInTx(ctx, tx, name) + }); err != nil { + return err + } + + // Also remove the Dolt remote (best-effort). + if rmErr := s.RemoveRemote(ctx, name); rmErr != nil { + if !strings.Contains(rmErr.Error(), "not found") { + // Silently ignore "not found" — the remote may not exist. + _ = rmErr + } + } + return nil +} + +// --------------------------------------------------------------------------- +// SyncStore implementation +// --------------------------------------------------------------------------- + +// Sync performs a full bidirectional sync with a peer: +// 1. Fetch from peer +// 2. Merge peer's changes (handling conflicts per strategy) +// 3. Push local changes to peer +func (s *DoltliteStore) Sync(ctx context.Context, peer string, strategy string) (*storage.SyncResult, error) { + result := &storage.SyncResult{ + Peer: peer, + StartTime: time.Now(), + } + + // Step 1: Fetch + if err := s.Fetch(ctx, peer); err != nil { + result.Error = fmt.Errorf("fetch failed: %w", err) + return result, result.Error + } + result.Fetched = true + + // Step 2: Get commit before merge for change detection + beforeCommit, _ := s.GetCurrentCommit(ctx) + + // Step 3: Merge peer's branch + remoteBranch := fmt.Sprintf("%s/%s", peer, s.branch) + conflicts, err := s.Merge(ctx, remoteBranch) + if err != nil { + result.Error = fmt.Errorf("merge failed: %w", err) + return result, result.Error + } + + // Step 4: Handle conflicts + if len(conflicts) > 0 { + result.Conflicts = conflicts + + if strategy == "" { + result.Error = fmt.Errorf("merge conflicts require resolution (use --strategy ours|theirs)") + return result, result.Error + } + + for _, c := range conflicts { + if err := s.ResolveConflicts(ctx, c.Field, strategy); err != nil { + result.Error = fmt.Errorf("conflict resolution failed for %s: %w", c.Field, err) + return result, result.Error + } + } + result.ConflictsResolved = true + + if err := s.Commit(ctx, fmt.Sprintf("Resolve conflicts from %s using %s strategy", peer, strategy)); err != nil { + result.Error = fmt.Errorf("commit conflict resolution: %w", err) + return result, result.Error + } + } + result.Merged = true + + afterCommit, _ := s.GetCurrentCommit(ctx) + if beforeCommit != afterCommit { + result.PulledCommits = 1 + } + + // Step 5: Push + if err := s.PushTo(ctx, peer); err != nil { + result.PushError = err + } else { + result.Pushed = true + } + + // Record last sync time in metadata. + _ = s.setLastSyncTime(ctx, peer) + + result.EndTime = time.Now() + return result, nil +} + +// SyncStatus returns the synchronization status with a peer. +func (s *DoltliteStore) SyncStatus(ctx context.Context, peer string) (*storage.SyncStatus, error) { + status := &storage.SyncStatus{ + Peer: peer, + } + + // Doltlite does not expose a historical dolt_log slice. Report exact zeroes + // only when refs match; otherwise keep counts unknown. + remoteRef := peer + "/" + s.branch + if err := issueops.ValidateRef(remoteRef); err != nil { + status.LocalAhead = -1 + status.LocalBehind = -1 + } else if err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var localHash, remoteHash string + if err := db.QueryRowContext(ctx, "SELECT dolt_hashof('HEAD')").Scan(&localHash); err != nil { + status.LocalAhead = -1 + status.LocalBehind = -1 + return nil + } + if err := db.QueryRowContext(ctx, "SELECT dolt_hashof(?)", remoteRef).Scan(&remoteHash); err != nil { + status.LocalAhead = -1 + status.LocalBehind = -1 + return nil + } + if localHash == remoteHash { + status.LocalAhead = 0 + status.LocalBehind = 0 + } else { + status.LocalAhead = -1 + status.LocalBehind = -1 + } + return nil + }); err != nil { + return nil, err + } + + // Check for conflicts. + conflicts, err := s.GetConflicts(ctx) + if err == nil && len(conflicts) > 0 { + status.HasConflicts = true + } + + // Get last sync time. + status.LastSync = s.getLastSyncTime(ctx, peer) + + return status, nil +} + +// setLastSyncTime records the last sync time for a peer in metadata. +func (s *DoltliteStore) setLastSyncTime(ctx context.Context, peer string) error { + key := "last_sync_" + peer + value := time.Now().Format(time.RFC3339) + return s.withConn(ctx, true, func(tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, + "REPLACE INTO metadata (`key`, value) VALUES (?, ?)", key, value) + return err + }) +} + +// getLastSyncTime retrieves the last sync time for a peer from metadata. +func (s *DoltliteStore) getLastSyncTime(ctx context.Context, peer string) time.Time { + key := "last_sync_" + peer + var value string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + return tx.QueryRowContext(ctx, "SELECT value FROM metadata WHERE `key` = ?", key).Scan(&value) + }) + if err != nil { + return time.Time{} + } + t, err := time.Parse(time.RFC3339, value) + if err != nil { + return time.Time{} + } + return t +} diff --git a/internal/storage/doltlite/flock.go b/internal/storage/doltlite/flock.go new file mode 100644 index 000000000..adda4089f --- /dev/null +++ b/internal/storage/doltlite/flock.go @@ -0,0 +1,114 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + backoff "github.com/cenkalti/backoff/v4" + "github.com/steveyegge/beads/internal/lockfile" +) + +// Unlocker is the interface for releasing an acquired lock. +type Unlocker interface { + Unlock() +} + +// Lock holds an exclusive flock on the doltlite data directory. +// Used by commands that require single-writer access (e.g., bd init). +type Lock struct { + f *os.File +} + +// TryLock attempts to acquire a non-blocking exclusive flock on /.lock. +// dataDir is created if it does not exist. Returns the held lock on success. +// If another process holds the lock, returns an error directing the user to +// the dolt server backend for concurrent access. +func TryLock(dataDir string) (*Lock, error) { + if err := os.MkdirAll(dataDir, 0700); err != nil { + return nil, fmt.Errorf("doltlite: creating data directory for lock: %w", err) + } + + lockPath := filepath.Join(dataDir, ".lock") + f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0600) //nolint:gosec // lockPath is derived from dataDir, not user input + if err != nil { + return nil, fmt.Errorf("doltlite: opening lock file: %w", err) + } + + if err := lockfile.FlockExclusiveNonBlocking(f); err != nil { + _ = f.Close() + if lockfile.IsLocked(err) { + return nil, fmt.Errorf("doltlite: another process holds the exclusive lock on %s; "+ + "the embedded backend supports only one writer at a time — "+ + "use the dolt server backend for concurrent access", dataDir) + } + return nil, fmt.Errorf("doltlite: acquiring lock: %w", err) + } + + return &Lock{f: f}, nil +} + +// WaitLock blocks until an exclusive flock on /.lock can be acquired +// or the context is canceled. It uses exponential backoff with non-blocking +// lock attempts so the wait is interruptible via context cancellation. +// Non-lock filesystem errors are returned immediately without retrying. +func WaitLock(ctx context.Context, dataDir string) (*Lock, error) { + if err := os.MkdirAll(dataDir, 0700); err != nil { + return nil, fmt.Errorf("doltlite: creating data directory for lock: %w", err) + } + + lockPath := filepath.Join(dataDir, ".lock") + f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0600) //nolint:gosec // lockPath is derived from dataDir, not user input + if err != nil { + return nil, fmt.Errorf("doltlite: opening lock file: %w", err) + } + + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = 50 * time.Millisecond + bo.MaxInterval = 2 * time.Second + bo.MaxElapsedTime = 0 // wait until context cancellation + + err = backoff.Retry(func() error { + lockErr := lockfile.FlockExclusiveNonBlocking(f) + if lockErr == nil { + return nil // acquired + } + if lockfile.IsLocked(lockErr) { + return lockErr // retryable + } + // Filesystem error — not retryable. + return backoff.Permanent(lockErr) + }, backoff.WithContext(bo, ctx)) + + if err != nil { + _ = f.Close() + if ctx.Err() != nil { + return nil, fmt.Errorf("doltlite: waiting for lock on %s: %w", dataDir, ctx.Err()) + } + return nil, fmt.Errorf("doltlite: acquiring lock: %w", err) + } + + return &Lock{f: f}, nil +} + +// Unlock releases the flock and closes the underlying file. +// Panics on failure to prevent deadlocks. +func (l *Lock) Unlock() { + if err := lockfile.FlockUnlock(l.f); err != nil { + panic(fmt.Sprintf("doltlite: failed to release lock: %v", err)) + } + if err := l.f.Close(); err != nil { + panic(fmt.Sprintf("doltlite: failed to close lock file: %v", err)) + } +} + +// NoopLock is a lock that does nothing. Used in server mode where the +// external dolt sql-server handles its own concurrency. +type NoopLock struct{} + +// Unlock is a no-op. +func (NoopLock) Unlock() {} diff --git a/internal/storage/doltlite/flock_stub.go b/internal/storage/doltlite/flock_stub.go new file mode 100644 index 000000000..6e10d10e8 --- /dev/null +++ b/internal/storage/doltlite/flock_stub.go @@ -0,0 +1,27 @@ +//go:build !cgo + +package doltlite + +import "errors" + +// Unlocker is the interface for releasing an acquired lock. +type Unlocker interface { + Unlock() +} + +// Lock is a stub for builds without CGO. +type Lock struct{} + +// TryLock returns an error when CGO is not enabled. +func TryLock(_ string) (*Lock, error) { + return nil, errors.New("doltlite: requires CGO (build with CGO_ENABLED=1)") +} + +// Unlock is a no-op stub. +func (l *Lock) Unlock() {} + +// NoopLock is a lock that does nothing. +type NoopLock struct{} + +// Unlock is a no-op. +func (NoopLock) Unlock() {} diff --git a/internal/storage/doltlite/get_issue.go b/internal/storage/doltlite/get_issue.go new file mode 100644 index 000000000..b207728eb --- /dev/null +++ b/internal/storage/doltlite/get_issue.go @@ -0,0 +1,21 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) GetIssue(ctx context.Context, id string) (*types.Issue, error) { + var issue *types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + issue, err = issueops.GetIssueInTx(ctx, tx, id) + return err + }) + return issue, err +} diff --git a/internal/storage/doltlite/issues.go b/internal/storage/doltlite/issues.go new file mode 100644 index 000000000..7fef1ed7e --- /dev/null +++ b/internal/storage/doltlite/issues.go @@ -0,0 +1,133 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// ClaimIssue atomically claims an issue using compare-and-swap semantics. +// Delegates SQL work to issueops; EmbeddedDolt auto-commits the transaction. +func (s *DoltliteStore) ClaimIssue(ctx context.Context, id string, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + _, err := issueops.ClaimIssueInTx(ctx, tx, id, actor) + return err + }) +} + +// ClaimReadyIssue atomically claims the first ready issue matching filter. +func (s *DoltliteStore) ClaimReadyIssue(ctx context.Context, filter types.WorkFilter, actor string) (*types.Issue, error) { + var claimed *types.Issue + err := s.withConn(ctx, true, func(tx *sql.Tx) error { + var err error + claimed, err = issueops.ClaimReadyIssueSQLiteInTx(ctx, tx, filter, actor) + return err + }) + return claimed, err +} + +// UpdateIssue updates fields on an issue. +// Delegates SQL work to issueops; EmbeddedDolt auto-commits the transaction. +func (s *DoltliteStore) UpdateIssue(ctx context.Context, id string, updates map[string]interface{}, actor string) error { + // Validate metadata against schema before routing. + if rawMeta, ok := updates["metadata"]; ok { + metadataStr, err := storage.NormalizeMetadataValue(rawMeta) + if err != nil { + return fmt.Errorf("invalid metadata: %w", err) + } + if err := issueops.ValidateMetadataIfConfigured(json.RawMessage(metadataStr)); err != nil { + return err + } + } + + return s.withConn(ctx, true, func(tx *sql.Tx) error { + _, err := issueops.UpdateIssueSQLiteInTx(ctx, tx, id, updates, actor) + return err + }) +} + +// HeartbeatIssue refreshes the lease on an issue actor holds in_progress. +// Delegates SQL work to issueops; EmbeddedDolt auto-commits the transaction. +func (s *DoltliteStore) HeartbeatIssue(ctx context.Context, id, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.HeartbeatIssueInTx(ctx, tx, id, actor) + }) +} + +// ReclaimExpiredLeases reverts in_progress issues whose lease expired more than +// olderThan ago back to ready, recovering work stranded by dead workers. +func (s *DoltliteStore) ReclaimExpiredLeases(ctx context.Context, olderThan time.Duration, actor string) ([]types.ReclaimedLease, error) { + cutoff := time.Now().UTC().Add(-olderThan) + var reclaimed []types.ReclaimedLease + err := s.withConn(ctx, true, func(tx *sql.Tx) error { + var err error + reclaimed, err = issueops.ReclaimExpiredLeasesInTx(ctx, tx, cutoff, actor) + return err + }) + return reclaimed, err +} + +// ReopenIssue reopens a closed issue, setting status to open and clearing +// closed_at and defer_until. If reason is non-empty, it is recorded as a comment. +// Wraps UpdateIssue; EmbeddedDolt auto-commits the transaction. +func (s *DoltliteStore) ReopenIssue(ctx context.Context, id string, reason string, actor string) error { + updates := map[string]interface{}{ + "status": string(types.StatusOpen), + "defer_until": nil, + } + if err := s.UpdateIssue(ctx, id, updates, actor); err != nil { + return err + } + if reason != "" { + if err := s.AddComment(ctx, id, actor, reason); err != nil { + return fmt.Errorf("reopen comment: %w", err) + } + } + return nil +} + +// UpdateIssueType changes the issue_type field of an issue. +// Wraps UpdateIssue; EmbeddedDolt auto-commits the transaction. +func (s *DoltliteStore) UpdateIssueType(ctx context.Context, id string, issueType string, actor string) error { + return s.UpdateIssue(ctx, id, map[string]interface{}{"issue_type": issueType}, actor) +} + +// CloseIssue closes an issue with a reason. +// Delegates SQL work to issueops; EmbeddedDolt auto-commits the transaction. +func (s *DoltliteStore) CloseIssue(ctx context.Context, id string, reason string, actor string, session string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + _, err := issueops.CloseIssueSQLiteInTx(ctx, tx, id, reason, actor, session) + return err + }) +} + +// IsBlocked checks if an issue is blocked by active dependencies. +func (s *DoltliteStore) IsBlocked(ctx context.Context, issueID string) (bool, []string, error) { + var blocked bool + var blockers []string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + blocked, blockers, err = issueops.IsBlockedInTx(ctx, tx, issueID) + return err + }) + return blocked, blockers, err +} + +// GetNewlyUnblockedByClose finds issues that become unblocked when closedIssueID is closed. +func (s *DoltliteStore) GetNewlyUnblockedByClose(ctx context.Context, closedIssueID string) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetNewlyUnblockedByCloseInTx(ctx, tx, closedIssueID) + return err + }) + return result, err +} diff --git a/internal/storage/doltlite/iter_stubs.go b/internal/storage/doltlite/iter_stubs.go new file mode 100644 index 000000000..1cdc80fa2 --- /dev/null +++ b/internal/storage/doltlite/iter_stubs.go @@ -0,0 +1,137 @@ +//go:build cgo + +// Package embeddeddolt — iter_stubs.go +// +// Slice-wrapping stubs for the Iter* methods. The embedded Dolt backend +// uses a per-method short-lived connection model (`withConn`) which is +// incompatible with the dedicated-conn cursor pattern used by the +// streaming iterators in internal/storage/dolt and internal/storage/postgres. +// The interface ships complete now (be-jaavsb / be-yinl4d); a follow-up +// child of be-yinl4d may add a streaming variant if DoltliteStore +// gains a cursor-conn API. For now every Iter* method materializes the +// slice and wraps it in storage.SliceIter. +package doltlite + +import ( + "context" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// IterIssues streams issues matching the filter (slice-then-walk). +// +// TODO(be-yinl4d-iter): replace with a fully streaming implementation. +func (s *DoltliteStore) IterIssues(ctx context.Context, query string, filter types.IssueFilter) (storage.Iter[types.Issue], error) { + is, err := s.SearchIssues(ctx, query, filter) + if err != nil { + return nil, err + } + return storage.NewSliceIter(is), nil +} + +// IterDependentsWithMetadata streams dependents (slice-then-walk). +// +// TODO(be-yinl4d-iter): replace with a fully streaming implementation. +func (s *DoltliteStore) IterDependentsWithMetadata(ctx context.Context, issueID string) (storage.Iter[types.IssueWithDependencyMetadata], error) { + deps, err := s.GetDependentsWithMetadata(ctx, issueID) + if err != nil { + return nil, err + } + return storage.NewSliceIter(deps), nil +} + +// IterDependenciesWithMetadata streams dependencies (slice-then-walk). +// +// TODO(be-yinl4d-iter): replace with a fully streaming implementation. +func (s *DoltliteStore) IterDependenciesWithMetadata(ctx context.Context, issueID string) (storage.Iter[types.IssueWithDependencyMetadata], error) { + deps, err := s.GetDependenciesWithMetadata(ctx, issueID) + if err != nil { + return nil, err + } + return storage.NewSliceIter(deps), nil +} + +// IterIssueComments streams comments on an issue (slice-then-walk). +// +// TODO(be-yinl4d-iter): replace with a fully streaming implementation. +func (s *DoltliteStore) IterIssueComments(ctx context.Context, issueID string) (storage.Iter[types.Comment], error) { + cs, err := s.GetIssueComments(ctx, issueID) + if err != nil { + return nil, err + } + return storage.NewSliceIter(cs), nil +} + +// IterEvents streams audit-trail events for an issue (slice-then-walk). +// +// TODO(be-yinl4d-iter): replace with a fully streaming implementation. +func (s *DoltliteStore) IterEvents(ctx context.Context, issueID string, limit int) (storage.Iter[types.Event], error) { + ev, err := s.GetEvents(ctx, issueID, limit) + if err != nil { + return nil, err + } + return storage.NewSliceIter(ev), nil +} + +// IterAllEventsSince streams every audit-trail event newer than `since` +// (slice-then-walk). +// +// TODO(be-yinl4d-iter): replace with a fully streaming implementation. +func (s *DoltliteStore) IterAllEventsSince(ctx context.Context, since time.Time) (storage.Iter[types.Event], error) { + ev, err := s.GetAllEventsSince(ctx, since) + if err != nil { + return nil, err + } + return storage.NewSliceIter(ev), nil +} + +// IterReadyWork streams ready-work issues (slice-then-walk). +// +// TODO(be-yinl4d-iter): replace with a fully streaming implementation. +func (s *DoltliteStore) IterReadyWork(ctx context.Context, filter types.WorkFilter) (storage.Iter[types.Issue], error) { + is, err := s.GetReadyWork(ctx, filter) + if err != nil { + return nil, err + } + return storage.NewSliceIter(is), nil +} + +// IterBlockedIssues streams blocked issues (slice-then-walk). +// +// TODO(be-yinl4d-iter): replace with a fully streaming implementation. +func (s *DoltliteStore) IterBlockedIssues(ctx context.Context, filter types.WorkFilter) (storage.Iter[types.BlockedIssue], error) { + bs, err := s.GetBlockedIssues(ctx, filter) + if err != nil { + return nil, err + } + return storage.NewSliceIter(bs), nil +} + +// IterWisps streams ephemeral issues matching the filter (slice-then-walk). +// +// TODO(be-yinl4d-iter): replace with a fully streaming implementation. +func (s *DoltliteStore) IterWisps(ctx context.Context, filter types.WispFilter) (storage.Iter[types.Issue], error) { + ws, err := s.ListWisps(ctx, filter) + if err != nil { + return nil, err + } + return storage.NewSliceIter(ws), nil +} + +// IterAllDependencyRecords streams every dependency edge as a flat +// sequence of *types.Dependency rows (slice-then-walk). +// +// TODO(be-yinl4d-iter): replace with a fully streaming implementation. +func (s *DoltliteStore) IterAllDependencyRecords(ctx context.Context) (storage.Iter[types.Dependency], error) { + all, err := s.GetAllDependencyRecords(ctx) + if err != nil { + return nil, err + } + var flat []*types.Dependency + for _, deps := range all { + flat = append(flat, deps...) + } + return storage.NewSliceIter(flat), nil +} diff --git a/internal/storage/doltlite/labels.go b/internal/storage/doltlite/labels.go new file mode 100644 index 000000000..170f02a5f --- /dev/null +++ b/internal/storage/doltlite/labels.go @@ -0,0 +1,44 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "fmt" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) GetLabels(ctx context.Context, issueID string) ([]string, error) { + var labels []string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + labels, err = issueops.GetLabelsInTx(ctx, tx, "", issueID) + return err + }) + return labels, err +} + +func (s *DoltliteStore) AddLabel(ctx context.Context, issueID, label, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + isWisp := issueops.IsActiveWispInTx(ctx, tx, issueID) + _, labelTable, eventTable, _ := issueops.WispTableRouting(isWisp) + if _, err := tx.ExecContext(ctx, fmt.Sprintf("INSERT OR IGNORE INTO %s (issue_id, label) VALUES (?, ?)", labelTable), issueID, label); err != nil { + return fmt.Errorf("add label: %w", err) + } + if _, err := tx.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s (id, issue_id, event_type, actor, comment) VALUES (?, ?, ?, ?, ?)", eventTable), + issueops.NewEventID(), issueID, types.EventLabelAdded, actor, "Added label: "+label); err != nil { + return fmt.Errorf("add label: record event: %w", err) + } + return nil + }) +} + +// RemoveLabel removes a label from an issue. +func (s *DoltliteStore) RemoveLabel(ctx context.Context, issueID, label, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.RemoveLabelInTx(ctx, tx, "", "", issueID, label, actor) + }) +} diff --git a/internal/storage/doltlite/list_queries.go b/internal/storage/doltlite/list_queries.go new file mode 100644 index 000000000..c234cc6ec --- /dev/null +++ b/internal/storage/doltlite/list_queries.go @@ -0,0 +1,106 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.SearchIssuesSQLiteInTx(ctx, tx, query, filter) + return err + }) + return result, err +} + +func (s *DoltliteStore) SearchIssuesWithCounts(ctx context.Context, query string, filter types.IssueFilter) ([]*types.IssueWithCounts, error) { + var result []*types.IssueWithCounts + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.SearchIssuesWithCountsSQLiteInTx(ctx, tx, query, filter) + return err + }) + return result, err +} + +func (s *DoltliteStore) ListWisps(ctx context.Context, filter types.WispFilter) ([]*types.Issue, error) { + issueFilter := issueops.WispFilterToIssueFilter(filter) + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.SearchIssuesSQLiteInTx(ctx, tx, "", issueFilter) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetLabelsForIssues(ctx context.Context, issueIDs []string) (map[string][]string, error) { + var result map[string][]string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetLabelsForIssuesInTx(ctx, tx, issueIDs) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetCommentCounts(ctx context.Context, issueIDs []string) (map[string]int, error) { + var result map[string]int + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetCommentCountsInTx(ctx, tx, issueIDs) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetAllDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) { + var result map[string][]*types.Dependency + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetAllDependencyRecordsInTx(ctx, tx) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetDependencyRecordsForIssues(ctx context.Context, issueIDs []string) (map[string][]*types.Dependency, error) { + var result map[string][]*types.Dependency + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependencyRecordsForIssuesInTx(ctx, tx, issueIDs) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetDependencyCounts(ctx context.Context, issueIDs []string) (map[string]*types.DependencyCounts, error) { + var result map[string]*types.DependencyCounts + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependencyCountsInTx(ctx, tx, issueIDs) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetBlockingInfoForIssues(ctx context.Context, issueIDs []string) ( + blockedByMap map[string][]string, + blocksMap map[string][]string, + parentMap map[string]string, + err error, +) { + err = s.withConn(ctx, false, func(tx *sql.Tx) error { + var txErr error + blockedByMap, blocksMap, parentMap, txErr = issueops.GetBlockingInfoForIssuesInTx(ctx, tx, issueIDs) + return txErr + }) + return +} diff --git a/internal/storage/doltlite/merge_slot.go b/internal/storage/doltlite/merge_slot.go new file mode 100644 index 000000000..6f58c2b56 --- /dev/null +++ b/internal/storage/doltlite/merge_slot.go @@ -0,0 +1,33 @@ +//go:build cgo + +package doltlite + +import ( + "context" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// MergeSlotCreate creates the merge slot bead for the current rig. +// Idempotent: returns the existing slot if one already exists. +func (s *DoltliteStore) MergeSlotCreate(ctx context.Context, actor string) (*types.Issue, error) { + return storage.MergeSlotCreateImpl(ctx, s, actor) +} + +// MergeSlotCheck returns the current status of the merge slot. +func (s *DoltliteStore) MergeSlotCheck(ctx context.Context) (*storage.MergeSlotStatus, error) { + return storage.MergeSlotCheckImpl(ctx, s) +} + +// MergeSlotAcquire attempts to acquire the merge slot atomically. +// When wait is true and the slot is held, the caller is added to the waiters queue. +func (s *DoltliteStore) MergeSlotAcquire(ctx context.Context, holder, actor string, wait bool) (*storage.MergeSlotResult, error) { + return storage.MergeSlotAcquireImpl(ctx, s, holder, actor, wait) +} + +// MergeSlotRelease releases the merge slot, clearing the holder. +// If holder is non-empty it is verified against the current holder before releasing. +func (s *DoltliteStore) MergeSlotRelease(ctx context.Context, holder, actor string) error { + return storage.MergeSlotReleaseImpl(ctx, s, holder, actor) +} diff --git a/internal/storage/doltlite/multiprocess_test.go b/internal/storage/doltlite/multiprocess_test.go new file mode 100644 index 000000000..38ac736c8 --- /dev/null +++ b/internal/storage/doltlite/multiprocess_test.go @@ -0,0 +1,75 @@ +//go:build cgo + +package doltlite_test + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/doltlite" +) + +func TestConcurrentOpenWhilePeerStoreAlive(t *testing.T) { + beadsDir := filepath.Join(t.TempDir(), ".beads") + readyPath := filepath.Join(t.TempDir(), "ready") + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=TestConcurrentOpenHelper") + cmd.Env = append(os.Environ(), + "BEADS_DOLTLITE_OPEN_HELPER=1", + "BEADS_DOLTLITE_TEST_DIR="+beadsDir, + "BEADS_DOLTLITE_READY="+readyPath, + ) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatalf("start helper: %v", err) + } + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(readyPath); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("helper did not open store") + } + time.Sleep(25 * time.Millisecond) + } + + openCtx, openCancel := context.WithTimeout(t.Context(), time.Second) + defer openCancel() + store, err := doltlite.New(openCtx, beadsDir, "beads", "main") + if err != nil { + t.Fatalf("second open while peer store alive: %v", err) + } + if err := store.Close(); err != nil { + t.Fatalf("close second store: %v", err) + } +} + +func TestConcurrentOpenHelper(t *testing.T) { + if os.Getenv("BEADS_DOLTLITE_OPEN_HELPER") != "1" { + t.Skip("helper only") + } + beadsDir := os.Getenv("BEADS_DOLTLITE_TEST_DIR") + readyPath := os.Getenv("BEADS_DOLTLITE_READY") + store, err := doltlite.New(t.Context(), beadsDir, "beads", "main") + if err != nil { + t.Fatalf("helper open: %v", err) + } + defer func() { _ = store.Close() }() + if err := os.WriteFile(readyPath, []byte("ready\n"), 0o600); err != nil { + t.Fatalf("write ready: %v", err) + } + time.Sleep(2 * time.Second) +} diff --git a/internal/storage/doltlite/native_test.go b/internal/storage/doltlite/native_test.go new file mode 100644 index 000000000..5b423b39b --- /dev/null +++ b/internal/storage/doltlite/native_test.go @@ -0,0 +1,51 @@ +//go:build cgo + +package doltlite_test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/doltlite" +) + +func TestMain(m *testing.M) { + if err := requireNativeDoltliteForTests(); err != nil { + fmt.Fprintf(os.Stderr, "SKIP internal/storage/doltlite: %v\n", err) + fmt.Fprintln(os.Stderr, "Run: make test-doltlite") + os.Exit(0) + } + os.Exit(m.Run()) +} + +func requireNativeDoltliteForTests() error { + ctx := context.Background() + dir, err := os.MkdirTemp("", "bd-doltlite-native-probe.*") + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(dir) }() + dataDir := filepath.Join(dir, "doltlite") + if err := os.MkdirAll(dataDir, 0o755); err != nil { + return err + } + + db, cleanup, err := doltlite.OpenSQL(ctx, dataDir, "beads", "") + if err != nil { + return err + } + defer func() { _ = cleanup() }() + + var version string + if err := db.QueryRowContext(ctx, "SELECT dolt_version()").Scan(&version); err != nil { + if strings.Contains(err.Error(), "no such function") { + return fmt.Errorf("libdoltlite SQL functions are not linked into the sqlite driver: %w", err) + } + return err + } + return nil +} diff --git a/internal/storage/doltlite/open.go b/internal/storage/doltlite/open.go new file mode 100644 index 000000000..ab7810a2d --- /dev/null +++ b/internal/storage/doltlite/open.go @@ -0,0 +1,122 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "crypto/rand" + "database/sql" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/mattn/go-sqlite3" +) + +// validIdentifier matches safe SQL identifiers (letters, digits, underscores). +// Hyphens are excluded because database names are interpolated into system +// variable identifiers (@@_head_ref) where hyphens are invalid. +var validIdentifier = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + +const ( + commitName = "beads" + commitEmail = "beads@local" + defaultBusyTimeout = 10000 + driverName = "sqlite3_doltlite" +) + +func init() { + sql.Register(driverName, &sqlite3.SQLiteDriver{ + ConnectHook: func(conn *sqlite3.SQLiteConn) error { + return conn.RegisterFunc("UUID", newUUID, true) + }, + }) +} + +// OpenSQL opens an doltlite database at dir. The returned cleanup +// function closes the *sql.DB. +func OpenSQL(ctx context.Context, dir, database, branch string) (*sql.DB, func() error, error) { + dbPath, dbFile, err := buildDSN(dir, database) + if err != nil { + return nil, nil, err + } + // v0.11.5 libdoltlite sqlite3_open can fail with NOTADB when the file + // does not exist; pre-creating an empty file avoids the issue. + if _, err := os.Stat(dbFile); os.IsNotExist(err) { + // #nosec G304 -- dbFile is built under the caller-selected DoltLite data dir by buildDSN. + f, err := os.Create(dbFile) + if err != nil { + return nil, nil, fmt.Errorf("doltlite: create db file: %w", err) + } + if err := f.Close(); err != nil { + return nil, nil, fmt.Errorf("doltlite: close db file: %w", err) + } + } + db, err := sql.Open(driverName, dbPath) + if err != nil { + return nil, nil, err + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + db.SetConnMaxIdleTime(0) + db.SetConnMaxLifetime(0) + + cleanup := func() error { + return db.Close() + } + + if err := db.PingContext(ctx); err != nil { + closeErr := cleanup() + if closeErr != nil { + return nil, nil, fmt.Errorf("%w; close: %v", err, closeErr) + } + return nil, nil, err + } + + if branch = strings.TrimSpace(branch); branch != "" { + if _, err := db.ExecContext(ctx, "SELECT dolt_checkout(?)", branch); err != nil { + closeErr := cleanup() + if closeErr != nil { + return nil, nil, fmt.Errorf("%w; close: %v", err, closeErr) + } + return nil, nil, fmt.Errorf("doltlite: checkout branch %s: %w", branch, err) + } + } + + return db, cleanup, nil +} + +func newUUID() (string, error) { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + return fmt.Sprintf( + "%x-%x-%x-%x-%x", + b[0:4], + b[4:6], + b[6:8], + b[8:10], + b[10:16], + ), nil +} + +func buildDSN(dir, database string) (string, string, error) { + if strings.TrimSpace(database) != "" { + if !validIdentifier.MatchString(database) { + return "", "", fmt.Errorf("doltlite: invalid database name: %q", database) + } + } else { + database = "beads" + } + filename := database + ".db" + path := filepath.Join(dir, filename) + if os.PathSeparator == '\\' { + path = strings.ReplaceAll(path, `\`, `/`) + } + return fmt.Sprintf("%s?_busy_timeout=%d", path, defaultBusyTimeout), path, nil +} diff --git a/internal/storage/doltlite/open_stub.go b/internal/storage/doltlite/open_stub.go new file mode 100644 index 000000000..01f017f8d --- /dev/null +++ b/internal/storage/doltlite/open_stub.go @@ -0,0 +1,14 @@ +//go:build !cgo + +package doltlite + +import ( + "context" + "database/sql" + "errors" +) + +// OpenSQL is a stub that returns an error when CGO is not enabled. +func OpenSQL(_ context.Context, _, _, _ string) (*sql.DB, func() error, error) { + return nil, nil, errors.New("doltlite: requires CGO (build with CGO_ENABLED=1)") +} diff --git a/internal/storage/doltlite/queries.go b/internal/storage/doltlite/queries.go new file mode 100644 index 000000000..bb2e9e9b8 --- /dev/null +++ b/internal/storage/doltlite/queries.go @@ -0,0 +1,41 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) GetReadyWork(ctx context.Context, filter types.WorkFilter) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetReadyWorkSQLiteInTx(ctx, tx, filter) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, error) { + var result []*types.IssueWithCounts + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetReadyWorkWithCountsSQLiteInTx(ctx, tx, filter) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetMoleculeProgress(ctx context.Context, moleculeID string) (*types.MoleculeProgressStats, error) { + var result *types.MoleculeProgressStats + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetMoleculeProgressInTx(ctx, tx, moleculeID) + return err + }) + return result, err +} diff --git a/internal/storage/doltlite/remote_validation_test.go b/internal/storage/doltlite/remote_validation_test.go new file mode 100644 index 000000000..e8ed6c545 --- /dev/null +++ b/internal/storage/doltlite/remote_validation_test.go @@ -0,0 +1,74 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "errors" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestValidateDoltliteRemoteSyncURLRejectsGitProtocol(t *testing.T) { + err := validateDoltliteRemoteSyncURL("origin", "git+ssh://git@github.com/org/repo.git") + if !errors.Is(err, errDoltliteUnsupportedRemoteURL) { + t.Fatalf("validateDoltliteRemoteSyncURL error = %v, want errDoltliteUnsupportedRemoteURL", err) + } + for _, want := range []string{"origin", "git+ssh://git@github.com/org/repo.git", "DoltLite", "file://", "http://", "Dolt backend"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("error %q should contain %q", err.Error(), want) + } + } +} + +func TestValidateDoltliteRemoteSyncURLAllowsNativeRemotes(t *testing.T) { + for _, url := range []string{"file:///tmp/beads-remote", "http://127.0.0.1:8080/repo"} { + if err := validateDoltliteRemoteSyncURL("origin", url); err != nil { + t.Fatalf("validateDoltliteRemoteSyncURL(%q) = %v, want nil", url, err) + } + } +} + +func TestGuardDoltliteRemoteSyncURL(t *testing.T) { + t.Run("rejects unsupported configured remote before transfer", func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectQuery(`SELECT url FROM dolt_remotes WHERE name = \?`). + WithArgs("origin"). + WillReturnRows(sqlmock.NewRows([]string{"url"}).AddRow("git+ssh://git@github.com/org/repo.git")) + + err = guardDoltliteRemoteSyncURL(context.Background(), db, "origin") + if !errors.Is(err, errDoltliteUnsupportedRemoteURL) { + t.Fatalf("guardDoltliteRemoteSyncURL error = %v, want errDoltliteUnsupportedRemoteURL", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) + + t.Run("leaves missing remote to transfer path", func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectQuery(`SELECT url FROM dolt_remotes WHERE name = \?`). + WithArgs("origin"). + WillReturnError(sql.ErrNoRows) + + if err := guardDoltliteRemoteSyncURL(context.Background(), db, "origin"); err != nil { + t.Fatalf("guardDoltliteRemoteSyncURL missing remote = %v, want nil", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } + }) +} diff --git a/internal/storage/doltlite/schema.go b/internal/storage/doltlite/schema.go new file mode 100644 index 000000000..f3adb517f --- /dev/null +++ b/internal/storage/doltlite/schema.go @@ -0,0 +1,12 @@ +//go:build cgo + +package doltlite + +import ( + "github.com/steveyegge/beads/internal/storage/schema" +) + +// LatestVersion delegates to the shared schema package. +func LatestVersion() int { + return schema.LatestVersion() +} diff --git a/internal/storage/doltlite/slots.go b/internal/storage/doltlite/slots.go new file mode 100644 index 000000000..bdef6a557 --- /dev/null +++ b/internal/storage/doltlite/slots.go @@ -0,0 +1,96 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "encoding/json" + "fmt" +) + +// SlotSet sets a key-value pair in the issue's metadata JSON. +func (s *DoltliteStore) SlotSet(ctx context.Context, issueID, key, value, actor string) error { + issue, err := s.GetIssue(ctx, issueID) + if err != nil { + return fmt.Errorf("getting issue %s: %w", issueID, err) + } + + metadata := make(map[string]interface{}) + if len(issue.Metadata) > 0 { + if err := json.Unmarshal(issue.Metadata, &metadata); err != nil { + return fmt.Errorf("parsing metadata for %s: %w", issueID, err) + } + } + metadata[key] = value + + raw, err := json.Marshal(metadata) + if err != nil { + return fmt.Errorf("marshaling metadata for %s: %w", issueID, err) + } + + updates := map[string]interface{}{"metadata": string(raw)} + return s.UpdateIssue(ctx, issueID, updates, actor) +} + +// SlotGet retrieves the value of a metadata key from an issue. +func (s *DoltliteStore) SlotGet(ctx context.Context, issueID, key string) (string, error) { + issue, err := s.GetIssue(ctx, issueID) + if err != nil { + return "", fmt.Errorf("getting issue %s: %w", issueID, err) + } + + if len(issue.Metadata) == 0 { + return "", fmt.Errorf("no slot %q on %s: no metadata", key, issueID) + } + + metadata := make(map[string]interface{}) + if err := json.Unmarshal(issue.Metadata, &metadata); err != nil { + return "", fmt.Errorf("parsing metadata for %s: %w", issueID, err) + } + + val, ok := metadata[key] + if !ok { + return "", fmt.Errorf("no slot %q on %s: key not found", key, issueID) + } + + switch v := val.(type) { + case string: + return v, nil + default: + raw, err := json.Marshal(v) + if err != nil { + return "", fmt.Errorf("marshaling slot value for %s.%s: %w", issueID, key, err) + } + return string(raw), nil + } +} + +// SlotClear removes a metadata key from an issue. +func (s *DoltliteStore) SlotClear(ctx context.Context, issueID, key, actor string) error { + issue, err := s.GetIssue(ctx, issueID) + if err != nil { + return fmt.Errorf("getting issue %s: %w", issueID, err) + } + + if len(issue.Metadata) == 0 { + return nil + } + + metadata := make(map[string]interface{}) + if err := json.Unmarshal(issue.Metadata, &metadata); err != nil { + return fmt.Errorf("parsing metadata for %s: %w", issueID, err) + } + + if _, ok := metadata[key]; !ok { + return nil + } + delete(metadata, key) + + raw, err := json.Marshal(metadata) + if err != nil { + return fmt.Errorf("marshaling metadata for %s: %w", issueID, err) + } + + updates := map[string]interface{}{"metadata": string(raw)} + return s.UpdateIssue(ctx, issueID, updates, actor) +} diff --git a/internal/storage/doltlite/smoke_test.go b/internal/storage/doltlite/smoke_test.go new file mode 100644 index 000000000..bb81b6fe5 --- /dev/null +++ b/internal/storage/doltlite/smoke_test.go @@ -0,0 +1,504 @@ +//go:build cgo + +package doltlite_test + +import ( + "path/filepath" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/doltlite" + "github.com/steveyegge/beads/internal/types" +) + +func doltliteCommitCount(t *testing.T, store *doltlite.DoltliteStore) int { + t.Helper() + commits, err := store.Log(t.Context(), 1000) + if err != nil { + t.Fatalf("Log: %v", err) + } + return len(commits) +} + +func requireDoltliteClean(t *testing.T, store *doltlite.DoltliteStore) { + t.Helper() + status, err := store.Status(t.Context()) + if err != nil { + t.Fatalf("Status: %v", err) + } + if len(status.Staged) != 0 || len(status.Unstaged) != 0 { + t.Fatalf("status not clean: %+v", status) + } +} + +func TestSmokeCreateGetCommit(t *testing.T) { + ctx := t.Context() + store, err := doltlite.New(ctx, filepath.Join(t.TempDir(), ".beads"), "beads", "main") + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + + now := time.Now().UTC() + issue := &types.Issue{ + ID: "bd-test", + Title: "doltlite smoke", + Description: "verify doltlite backend", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now, + UpdatedAt: now, + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + + got, err := store.GetIssue(ctx, issue.ID) + if err != nil { + t.Fatalf("GetIssue: %v", err) + } + if got.Title != issue.Title { + t.Fatalf("title = %q, want %q", got.Title, issue.Title) + } + + if err := store.Commit(ctx, "test: doltlite smoke"); err != nil { + t.Fatalf("Commit: %v", err) + } +} + +func TestSmokeLabels(t *testing.T) { + ctx := t.Context() + store, err := doltlite.New(ctx, filepath.Join(t.TempDir(), ".beads"), "beads", "main") + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + + now := time.Now().UTC() + issue := &types.Issue{ + ID: "bd-label", + Title: "doltlite labels", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now, + UpdatedAt: now, + Labels: []string{"gc:session"}, + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + if err := store.AddLabel(ctx, issue.ID, "agent:worker", "test"); err != nil { + t.Fatalf("AddLabel: %v", err) + } + labels, err := store.GetLabels(ctx, issue.ID) + if err != nil { + t.Fatalf("GetLabels: %v", err) + } + got := map[string]bool{} + for _, label := range labels { + got[label] = true + } + for _, want := range []string{"gc:session", "agent:worker"} { + if !got[want] { + t.Fatalf("labels = %v, missing %q", labels, want) + } + } +} + +func TestSmokeChildIDAndDependencyUseSQLiteDialect(t *testing.T) { + ctx := t.Context() + store, err := doltlite.New(ctx, filepath.Join(t.TempDir(), ".beads"), "beads", "main") + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + + now := time.Now().UTC() + parent := &types.Issue{ + ID: "bd-parent", + Title: "parent", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now, + UpdatedAt: now, + } + child := &types.Issue{ + ID: "bd-parent.1", + Title: "child", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now, + UpdatedAt: now, + } + if err := store.CreateIssue(ctx, parent, "test"); err != nil { + t.Fatalf("CreateIssue parent: %v", err) + } + if err := store.CreateIssue(ctx, child, "test"); err != nil { + t.Fatalf("CreateIssue child: %v", err) + } + + next, err := store.GetNextChildID(ctx, parent.ID) + if err != nil { + t.Fatalf("GetNextChildID: %v", err) + } + if next != "bd-parent.2" { + t.Fatalf("next child ID = %q, want bd-parent.2", next) + } + + dep := &types.Dependency{ + IssueID: child.ID, + DependsOnID: parent.ID, + Type: types.DepParentChild, + } + if err := store.AddDependency(ctx, dep, "test"); err != nil { + t.Fatalf("AddDependency: %v", err) + } + deps, err := store.GetDependencyRecords(ctx, child.ID) + if err != nil { + t.Fatalf("GetDependencyRecords: %v", err) + } + if len(deps) != 1 || deps[0].DependsOnID != parent.ID || deps[0].Type != types.DepParentChild { + t.Fatalf("deps = %#v, want parent-child to %s", deps, parent.ID) + } +} + +func TestSmokeCloseUsesSQLiteBlockedRecompute(t *testing.T) { + ctx := t.Context() + store, err := doltlite.New(ctx, filepath.Join(t.TempDir(), ".beads"), "beads", "main") + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + + now := time.Now().UTC() + blocked := &types.Issue{ + ID: "bd-blocked", + Title: "blocked", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now, + UpdatedAt: now, + } + blocker := &types.Issue{ + ID: "bd-blocker", + Title: "blocker", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now, + UpdatedAt: now, + } + if err := store.CreateIssue(ctx, blocked, "test"); err != nil { + t.Fatalf("CreateIssue blocked: %v", err) + } + if err := store.CreateIssue(ctx, blocker, "test"); err != nil { + t.Fatalf("CreateIssue blocker: %v", err) + } + if err := store.AddDependency(ctx, &types.Dependency{ + IssueID: blocked.ID, + DependsOnID: blocker.ID, + Type: types.DepBlocks, + }, "test"); err != nil { + t.Fatalf("AddDependency: %v", err) + } + isBlocked, _, err := store.IsBlocked(ctx, blocked.ID) + if err != nil { + t.Fatalf("IsBlocked before close: %v", err) + } + if !isBlocked { + t.Fatalf("%s should be blocked before closing %s", blocked.ID, blocker.ID) + } + + if err := store.CloseIssue(ctx, blocker.ID, "done", "test", "sess"); err != nil { + t.Fatalf("CloseIssue: %v", err) + } + isBlocked, _, err = store.IsBlocked(ctx, blocked.ID) + if err != nil { + t.Fatalf("IsBlocked after close: %v", err) + } + if isBlocked { + t.Fatalf("%s should be unblocked after closing %s", blocked.ID, blocker.ID) + } +} + +func TestSmokeVersionControl(t *testing.T) { + ctx := t.Context() + store, err := doltlite.New(ctx, filepath.Join(t.TempDir(), ".beads"), "beads", "main") + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + if err := store.CommitWithConfig(ctx, "test: config"); err != nil { + t.Fatalf("Commit config: %v", err) + } + + branch, err := store.CurrentBranch(ctx) + if err != nil { + t.Fatalf("CurrentBranch: %v", err) + } + if branch != "main" { + t.Fatalf("branch = %q, want main", branch) + } + + if err := store.Branch(ctx, "feature"); err != nil { + t.Fatalf("Branch: %v", err) + } + if err := store.Checkout(ctx, "feature"); err != nil { + t.Fatalf("Checkout feature: %v", err) + } + branch, err = store.CurrentBranch(ctx) + if err != nil { + t.Fatalf("CurrentBranch feature: %v", err) + } + if branch != "feature" { + t.Fatalf("branch = %q, want feature", branch) + } + + branches, err := store.ListBranches(ctx) + if err != nil { + t.Fatalf("ListBranches: %v", err) + } + if len(branches) < 2 { + t.Fatalf("branches = %v, want at least main and feature", branches) + } + + if err := store.Checkout(ctx, "main"); err != nil { + t.Fatalf("Checkout main: %v", err) + } + if err := store.DeleteBranch(ctx, "feature"); err != nil { + t.Fatalf("DeleteBranch: %v", err) + } + + if _, err := store.Status(ctx); err != nil { + t.Fatalf("Status: %v", err) + } + if commits, err := store.Log(ctx, 5); err != nil { + t.Fatalf("Log: %v", err) + } else if len(commits) == 0 { + t.Fatal("Log returned no commits") + } + if hash, err := store.GetCurrentCommit(ctx); err != nil { + t.Fatalf("GetCurrentCommit: %v", err) + } else if hash == "" { + t.Fatal("GetCurrentCommit returned empty hash") + } +} + +func TestCommitPendingSkipsWispOnlyChanges(t *testing.T) { + ctx := t.Context() + store, err := doltlite.New(ctx, filepath.Join(t.TempDir(), ".beads"), "beads", "main") + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + if err := store.CommitWithConfig(ctx, "test: config"); err != nil { + t.Fatalf("Commit config: %v", err) + } + requireDoltliteClean(t, store) + before := doltliteCommitCount(t, store) + + now := time.Now().UTC() + wisps := []*types.Issue{ + { + ID: "bd-wisp-ephemeral", + Title: "ephemeral wisp", + Description: "operational state", + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + Ephemeral: true, + CreatedAt: now, + UpdatedAt: now, + }, + { + ID: "bd-wisp-no-history", + Title: "no-history wisp", + Description: "operational state", + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + NoHistory: true, + CreatedAt: now, + UpdatedAt: now, + }, + } + if err := store.CreateIssues(ctx, wisps, "test"); err != nil { + t.Fatalf("CreateIssues: %v", err) + } + + committed, err := store.CommitPending(ctx, "test") + if err != nil { + t.Fatalf("CommitPending: %v", err) + } + if committed { + status, statusErr := store.Status(ctx) + t.Fatalf("CommitPending committed wisp-only changes; status=%+v statusErr=%v", status, statusErr) + } + after := doltliteCommitCount(t, store) + if after != before { + t.Fatalf("commit count changed after wisp-only writes: before=%d after=%d", before, after) + } +} + +func TestCommitPendingCommitsPermanentIssue(t *testing.T) { + ctx := t.Context() + store, err := doltlite.New(ctx, filepath.Join(t.TempDir(), ".beads"), "beads", "main") + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + if err := store.CommitWithConfig(ctx, "test: config"); err != nil { + t.Fatalf("Commit config: %v", err) + } + before := doltliteCommitCount(t, store) + + now := time.Now().UTC() + issue := &types.Issue{ + ID: "bd-permanent", + Title: "permanent issue", + Description: "versioned state", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now, + UpdatedAt: now, + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + + committed, err := store.CommitPending(ctx, "test") + if err != nil { + t.Fatalf("CommitPending: %v", err) + } + if !committed { + t.Fatal("CommitPending did not commit permanent issue") + } + after := doltliteCommitCount(t, store) + if after != before+1 { + t.Fatalf("commit count after permanent write = %d, want %d", after, before+1) + } +} + +func TestCommitPendingRefreshesStaleConnectionAfterConcurrentCommit(t *testing.T) { + ctx := t.Context() + beadsDir := filepath.Join(t.TempDir(), ".beads") + + bootstrap, err := doltlite.New(ctx, beadsDir, "beads", "main") + if err != nil { + t.Fatalf("bootstrap New: %v", err) + } + if err := bootstrap.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatalf("bootstrap SetConfig: %v", err) + } + if err := bootstrap.CommitWithConfig(ctx, "test: bootstrap config"); err != nil { + t.Fatalf("bootstrap CommitWithConfig: %v", err) + } + if err := bootstrap.Close(); err != nil { + t.Fatalf("bootstrap Close: %v", err) + } + + stale, err := doltlite.New(ctx, beadsDir, "beads", "main") + if err != nil { + t.Fatalf("stale New: %v", err) + } + t.Cleanup(func() { _ = stale.Close() }) + + peer, err := doltlite.New(ctx, beadsDir, "beads", "main") + if err != nil { + t.Fatalf("peer New: %v", err) + } + t.Cleanup(func() { _ = peer.Close() }) + + base := doltliteCommitCount(t, stale) + now := time.Now().UTC() + peerIssue := &types.Issue{ + ID: "bd-peer", + Title: "peer commit", + Description: "advance branch from another connection", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now, + UpdatedAt: now, + } + if err := peer.CreateIssue(ctx, peerIssue, "peer"); err != nil { + t.Fatalf("peer CreateIssue: %v", err) + } + committed, err := peer.CommitPending(ctx, "peer") + if err != nil { + t.Fatalf("peer CommitPending: %v", err) + } + if !committed { + t.Fatal("peer CommitPending did not commit") + } + afterPeer := doltliteCommitCount(t, peer) + if afterPeer != base+1 { + t.Fatalf("commit count after peer write = %d, want %d", afterPeer, base+1) + } + + staleIssue := &types.Issue{ + ID: "bd-stale", + Title: "stale commit", + Description: "commit from a connection opened before peer advanced HEAD", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now.Add(time.Second), + UpdatedAt: now.Add(time.Second), + } + if err := stale.CreateIssue(ctx, staleIssue, "stale"); err != nil { + t.Fatalf("stale CreateIssue: %v", err) + } + committed, err = stale.CommitPending(ctx, "stale") + if err != nil { + t.Fatalf("stale CommitPending after peer commit: %v", err) + } + if !committed { + t.Fatal("stale CommitPending did not commit") + } + afterStale := doltliteCommitCount(t, stale) + if afterStale != afterPeer+1 { + t.Fatalf("commit count after stale write = %d, want %d", afterStale, afterPeer+1) + } + + if got, err := stale.GetIssue(ctx, peerIssue.ID); err != nil || got == nil { + t.Fatalf("peer issue missing after stale commit: issue=%+v err=%v", got, err) + } + if got, err := stale.GetIssue(ctx, staleIssue.ID); err != nil || got == nil { + t.Fatalf("stale issue missing after stale commit: issue=%+v err=%v", got, err) + } +} diff --git a/internal/storage/doltlite/statistics.go b/internal/storage/doltlite/statistics.go new file mode 100644 index 000000000..fbbdb3d3d --- /dev/null +++ b/internal/storage/doltlite/statistics.go @@ -0,0 +1,39 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "fmt" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) GetStatistics(ctx context.Context) (*types.Statistics, error) { + stats := &types.Statistics{} + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + if err := issueops.ScanIssueCountsInTx(ctx, tx, stats); err != nil { + return err + } + + var blockedCount int + if err := tx.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM issues + WHERE is_blocked = 1 AND status <> 'closed' AND status <> 'pinned' + `).Scan(&blockedCount); err != nil { + return err + } + stats.BlockedIssues = blockedCount + stats.ReadyIssues = stats.OpenIssues - stats.BlockedIssues + if stats.ReadyIssues < 0 { + stats.ReadyIssues = 0 + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("doltlite: get statistics: %w", err) + } + return stats, nil +} diff --git a/internal/storage/doltlite/store.go b/internal/storage/doltlite/store.go new file mode 100644 index 000000000..a53162479 --- /dev/null +++ b/internal/storage/doltlite/store.go @@ -0,0 +1,1325 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/schema" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" + "github.com/steveyegge/beads/internal/types" +) + +// Compile-time interface checks. +var _ storage.DoltStorage = (*DoltliteStore)(nil) +var _ storage.RawDBAccessor = (*DoltliteStore)(nil) +var _ storage.StoreLocator = (*DoltliteStore)(nil) +var _ storage.GarbageCollector = (*DoltliteStore)(nil) +var _ storage.Flattener = (*DoltliteStore)(nil) +var _ storage.Compactor = (*DoltliteStore)(nil) + +// DoltliteStore implements storage.DoltStorage backed by the doltlite engine. +// Each method call opens a short-lived connection, executes within an explicit +// SQL transaction, and closes the connection immediately. This minimizes the +// time the embedded engine's write lock is held, reducing contention when +// multiple processes access the same database concurrently. +// +// Schema bootstrap is protected by a short exclusive flock. Normal operations +// rely on doltlite's file-level locking and conflict detection so multiple bd +// processes can read concurrently and serialize writes. +type DoltliteStore struct { + dataDir string + beadsDir string + database string + branch string + credentialKey []byte + dbMu sync.Mutex + db *sql.DB + dbCleanup func() error + closed atomic.Bool +} + +// errClosed is returned when a method is called after Close. +var errClosed = errors.New("doltlite: store is closed") + +// Option configures optional behavior for New. +type Option func(*options) + +type options struct { + lock Unlocker // pre-acquired lock; nil means New acquires its own +} + +// WithLock passes a pre-acquired exclusive lock to New for schema bootstrap. +// The caller retains ownership. Normal store operations do not hold this lock. +func WithLock(lock Unlocker) Option { + return func(o *options) { o.lock = lock } +} + +// New creates an DoltliteStore using the doltlite engine. +// beadsDir is the .beads/ root; the data directory is derived as /doltlite/. +// The database is created automatically if it doesn't exist (initSchema handles this). +// +// Schema bootstrap is guarded by a short exclusive flock. After bootstrap, the +// lock is released and normal operations use doltlite's own file-level locks. +func New(ctx context.Context, beadsDir, database, branch string, opts ...Option) (*DoltliteStore, error) { + if database == "" { + return nil, fmt.Errorf("doltlite: database name must not be empty (caller should default to %q)", "beads") + } + + var o options + for _, fn := range opts { + fn(&o) + } + + // Resolve to absolute path so the SQLite database path is stable across + // callers with different working directories. + absBeadsDir, err := filepath.Abs(beadsDir) + if err != nil { + return nil, fmt.Errorf("doltlite: resolving beads dir: %w", err) + } + dataDir := filepath.Join(absBeadsDir, "doltlite") + if err := os.MkdirAll(dataDir, config.BeadsDirPerm); err != nil { + return nil, fmt.Errorf("doltlite: creating data directory: %w", err) + } + + lock := o.lock + ownsLock := lock == nil + if ownsLock { + var err error + lock, err = WaitLock(ctx, dataDir) + if err != nil { + return nil, err + } + } + s := &DoltliteStore{ + dataDir: dataDir, + beadsDir: absBeadsDir, + database: database, + branch: branch, + } + + if err := s.initSchema(ctx); err != nil { + if lock != nil && ownsLock { + lock.Unlock() + } + return nil, fmt.Errorf("doltlite: init schema: %w", err) + } + if lock != nil && ownsLock { + lock.Unlock() + lock = nil + } + if err := s.openPersistentDB(ctx); err != nil { + return nil, fmt.Errorf("doltlite: open database: %w", err) + } + + // Backfill custom_types / custom_statuses from config values, + // fixing databases where schema migration created empty tables. + if err := s.backfillCustomTables(ctx); err != nil { + return nil, fmt.Errorf("doltlite: backfill custom tables: %w", err) + } + + if s.branch == "" { + branch, err := s.CurrentBranch(ctx) + if err != nil { + return nil, fmt.Errorf("doltlite: get current branch: %w", err) + } + s.branch = branch + } + // Ensure dolt_ignore'd wisp tables exist in the working set. + // After a clone or branch switch, these tables are absent because + // dolt_ignore prevents them from being committed. Server mode handles + // this in newServerMode(); embedded mode must do it here. (GH#3270) + if err := s.ensureIgnoredTables(ctx); err != nil { + return nil, fmt.Errorf("doltlite: ensure ignored tables: %w", err) + } + + return s, nil +} + +func (s *DoltliteStore) openPersistentDB(ctx context.Context) error { + s.dbMu.Lock() + defer s.dbMu.Unlock() + if s.db != nil { + return nil + } + var db *sql.DB + var cleanup func() error + if err := s.withRetry(ctx, func() error { + var err error + db, cleanup, err = OpenSQL(ctx, s.dataDir, s.database, "") + return err + }); err != nil { + return err + } + s.db = db + s.dbCleanup = cleanup + return nil +} + +func (s *DoltliteStore) activeDB(ctx context.Context) (*sql.DB, func() error, error) { + s.dbMu.Lock() + db := s.db + s.dbMu.Unlock() + if db != nil { + return db, func() error { return nil }, nil + } + var cleanup func() error + if err := s.withRetry(ctx, func() error { + var err error + db, cleanup, err = OpenSQL(ctx, s.dataDir, s.database, "") + return err + }); err != nil { + return nil, nil, err + } + return db, cleanup, nil +} + +// DB returns the persistent DoltLite SQL connection for direct queries. +// Use sparingly; prefer the store's typed methods for normal operations. +func (s *DoltliteStore) DB() *sql.DB { + s.dbMu.Lock() + defer s.dbMu.Unlock() + return s.db +} + +// UnderlyingDB returns the persistent DoltLite SQL connection for diagnostics +// and raw SQL maintenance commands. +func (s *DoltliteStore) UnderlyingDB() *sql.DB { + return s.DB() +} + +// withRootConn opens a short-lived database connection without selecting any +// database or branch, begins an explicit SQL transaction, and passes it to fn. +// This is used during initialization when the database may not yet exist. +func (s *DoltliteStore) withRootConn(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (err error) { + if commit { + return s.withExclusiveLock(ctx, func() error { + return s.withRetry(ctx, func() error { + return s.withRootConnOnce(ctx, commit, fn) + }) + }) + } + return s.withRootConnOnce(ctx, commit, fn) +} + +func (s *DoltliteStore) withRootConnOnce(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (err error) { + if s.closed.Load() { + err = errClosed + return + } + + var db *sql.DB + var cleanup func() error + db, cleanup, err = OpenSQL(ctx, s.dataDir, "", "") + if err != nil { + return + } + + defer func() { + err = errors.Join(err, cleanup()) + }() + + var tx *sql.Tx + tx, err = db.BeginTx(ctx, nil) + if err != nil { + err = fmt.Errorf("doltlite: begin tx: %w", err) + return + } + + err = fn(tx) + if err != nil { + err = errors.Join(err, tx.Rollback()) + return + } + + if !commit { + return tx.Rollback() + } + + err = tx.Commit() + return +} + +// withConn opens a short-lived database connection configured for the store's +// database and branch, begins an explicit SQL transaction, and passes it to +// fn. If commit is true and fn returns nil, the transaction is committed; +// otherwise it is rolled back. The connection is closed before withConn +// returns regardless of outcome. +// +// The database must already exist (created during initSchema). +func (s *DoltliteStore) withConn(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (err error) { + if commit { + return s.withExclusiveLock(ctx, func() error { + return s.withRetryRefreshingDB(ctx, func() error { + return s.withConnOnce(ctx, commit, fn) + }) + }) + } + return s.withConnOnce(ctx, commit, fn) +} + +func (s *DoltliteStore) withConnOnce(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (err error) { + if s.closed.Load() { + err = errClosed + return + } + + var db *sql.DB + var cleanup func() error + db, cleanup, err = s.activeDB(ctx) + if err != nil { + return + } + + defer func() { + err = errors.Join(err, cleanup()) + }() + + var tx *sql.Tx + tx, err = db.BeginTx(ctx, nil) + if err != nil { + err = fmt.Errorf("doltlite: begin tx: %w", err) + return + } + + err = fn(tx) + if err != nil { + err = errors.Join(err, tx.Rollback()) + return + } + + if !commit { + return tx.Rollback() + } + + err = tx.Commit() + return +} + +func (s *DoltliteStore) withRetry(ctx context.Context, fn func() error) error { + return s.withRetryAfter(ctx, fn, nil) +} + +func (s *DoltliteStore) withRetryRefreshingDB(ctx context.Context, fn func() error) error { + return s.withRetryAfter(ctx, fn, s.resetPersistentDB) +} + +func (s *DoltliteStore) withRetryAfter(ctx context.Context, fn func() error, afterRetryable func()) error { + const maxAttempts = 5 + var err error + for attempt := 0; attempt < maxAttempts; attempt++ { + if err = fn(); err == nil { + return nil + } + if !isRetryableConcurrencyError(err) { + return err + } + if afterRetryable != nil { + afterRetryable() + } + select { + case <-ctx.Done(): + return errors.Join(err, ctx.Err()) + case <-time.After(time.Duration(50*(1< 0 { + if err := commitAllNative(ctx, db, "schema: apply migrations"); err != nil { + return fmt.Errorf("commit migration: %w", err) + } + } + + return nil +} + +// ensureIgnoredTables creates dolt_ignore'd wisp tables if they don't exist. +// Uses withConn (not withRootConn) because the database is already created. +func (s *DoltliteStore) ensureIgnoredTables(ctx context.Context) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + currentVersion, err := schema.CurrentVersion(ctx, tx) + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "no such table") { + currentVersion = 0 + } else { + return err + } + } + if currentVersion == 0 { + _, err = schema.MigrateFreshSQLite(ctx, tx, schema.LatestVersion()) + } else { + _, err = schema.MigrateSQLiteUpTo(ctx, tx, schema.LatestVersion()) + } + if err != nil { + return err + } + return ensureDoltliteLocalSchemaCompat(ctx, tx) + }) +} + +func ensureDoltliteLocalSchemaCompat(ctx context.Context, tx *sql.Tx) error { + if err := ensureDoltliteColumn(ctx, tx, "wisps", "is_blocked", "ALTER TABLE wisps ADD COLUMN is_blocked TINYINT(1) NOT NULL DEFAULT 0"); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "CREATE INDEX IF NOT EXISTS idx_wisps_is_blocked ON wisps(is_blocked, status)"); err != nil { + return fmt.Errorf("creating idx_wisps_is_blocked: %w", err) + } + if err := ensureDoltliteWispDependenciesShape(ctx, tx); err != nil { + return err + } + return nil +} + +func ensureDoltliteWispDependenciesShape(ctx context.Context, tx *sql.Tx) error { + hasSplitTarget, err := doltliteColumnExists(ctx, tx, "wisp_dependencies", "depends_on_issue_id") + if err != nil { + return fmt.Errorf("checking wisp_dependencies shape: %w", err) + } + if hasSplitTarget { + return nil + } + + var rows int + if err := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM wisp_dependencies").Scan(&rows); err != nil { + if !strings.Contains(strings.ToLower(err.Error()), "no such table") { + return fmt.Errorf("counting legacy wisp_dependencies rows: %w", err) + } + } + if rows == 0 { + if _, err := tx.ExecContext(ctx, sqliteWispDependenciesSchema("wisp_dependencies")); err != nil { + return fmt.Errorf("repairing wisp_dependencies schema: %w", err) + } + return nil + } + + if err := migrateLegacyDoltliteWispDependencies(ctx, tx); err != nil { + return fmt.Errorf("repairing legacy wisp_dependencies schema with %d rows: %w", rows, err) + } + return nil +} + +func migrateLegacyDoltliteWispDependencies(ctx context.Context, tx *sql.Tx) error { + columns, err := doltliteColumns(ctx, tx, "wisp_dependencies") + if err != nil { + return err + } + if !columns["depends_on_id"] { + return fmt.Errorf("legacy wisp_dependencies missing depends_on_id") + } + + const tmpTable = "wisp_dependencies_doltlite_repair" + if _, err := tx.ExecContext(ctx, sqliteWispDependenciesSchema(tmpTable)); err != nil { + return fmt.Errorf("creating repair table: %w", err) + } + + target := "NULLIF(depends_on_id, '')" + wispExists := "EXISTS (SELECT 1 FROM wisps w WHERE w.id = " + target + ")" + issueExists := "EXISTS (SELECT 1 FROM issues i WHERE i.id = " + target + ")" + idExpr := doltliteLegacyColumnValue(columns, "id", "issue_id || ':' || "+target) + typeExpr := doltliteLegacyColumnValue(columns, "type", "'blocks'") + createdAtExpr := doltliteLegacyColumnValue(columns, "created_at", "CURRENT_TIMESTAMP") + createdByExpr := doltliteLegacyColumnValue(columns, "created_by", "''") + metadataExpr := doltliteLegacyColumnValue(columns, "metadata", "'{}'") + threadIDExpr := doltliteLegacyColumnValue(columns, "thread_id", "''") + + // #nosec G201 -- interpolated fragments are fixed schema identifiers and expressions from this migration. + copySQL := fmt.Sprintf(` +INSERT OR IGNORE INTO %[1]s ( + id, issue_id, depends_on_issue_id, depends_on_wisp_id, depends_on_external, + type, created_at, created_by, metadata, thread_id +) +SELECT + %[2]s, + issue_id, + CASE WHEN %[3]s NOT LIKE 'external:%%' AND NOT (%[4]s) AND (%[5]s) THEN %[3]s ELSE NULL END, + CASE WHEN %[3]s NOT LIKE 'external:%%' AND (%[4]s) THEN %[3]s ELSE NULL END, + CASE WHEN %[3]s LIKE 'external:%%' OR (NOT (%[4]s) AND NOT (%[5]s)) THEN %[3]s ELSE NULL END, + %[6]s, + %[7]s, + %[8]s, + %[9]s, + %[10]s +FROM wisp_dependencies +WHERE NULLIF(issue_id, '') IS NOT NULL + AND %[3]s IS NOT NULL +`, tmpTable, idExpr, target, wispExists, issueExists, typeExpr, createdAtExpr, createdByExpr, metadataExpr, threadIDExpr) + if _, err := tx.ExecContext(ctx, copySQL); err != nil { + return fmt.Errorf("copying legacy rows: %w", err) + } + if _, err := tx.ExecContext(ctx, "DROP TABLE wisp_dependencies"); err != nil { + return fmt.Errorf("dropping legacy table: %w", err) + } + if _, err := tx.ExecContext(ctx, "ALTER TABLE "+tmpTable+" RENAME TO wisp_dependencies"); err != nil { + return fmt.Errorf("renaming repair table: %w", err) + } + if _, err := tx.ExecContext(ctx, sqliteWispDependenciesIndexes("wisp_dependencies")); err != nil { + return fmt.Errorf("creating repaired indexes: %w", err) + } + return nil +} + +func doltliteLegacyColumnValue(columns map[string]bool, column, fallback string) string { + if !columns[column] { + return fallback + } + return fmt.Sprintf("COALESCE(NULLIF(%s, ''), %s)", column, fallback) +} + +func sqliteWispDependenciesSchema(table string) string { + return fmt.Sprintf(`DROP TABLE IF EXISTS %[1]s; +CREATE TABLE %[1]s ( + id CHAR(36) NOT NULL PRIMARY KEY, + issue_id VARCHAR(255) NOT NULL, + depends_on_issue_id VARCHAR(255) NULL, + depends_on_wisp_id VARCHAR(255) NULL, + depends_on_external VARCHAR(255) NULL, + type VARCHAR(32) NOT NULL DEFAULT 'blocks', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(255) DEFAULT '', + metadata TEXT DEFAULT '{}', + thread_id VARCHAR(255) DEFAULT '', + UNIQUE (issue_id, depends_on_issue_id), + UNIQUE (issue_id, depends_on_wisp_id), + UNIQUE (issue_id, depends_on_external) +); +%[2]s`, table, sqliteWispDependenciesIndexes(table)) +} + +func sqliteWispDependenciesIndexes(table string) string { + return fmt.Sprintf(`CREATE INDEX IF NOT EXISTS idx_wisp_dep_type_issue ON %[1]s (type, depends_on_issue_id); +CREATE INDEX IF NOT EXISTS idx_wisp_dep_type_wisp ON %[1]s (type, depends_on_wisp_id); +CREATE INDEX IF NOT EXISTS idx_wisp_dep_type_external ON %[1]s (type, depends_on_external); +CREATE INDEX IF NOT EXISTS idx_wisp_dep_wisp_target ON %[1]s (depends_on_wisp_id); +CREATE INDEX IF NOT EXISTS idx_wisp_dep_issue_target ON %[1]s (depends_on_issue_id); +CREATE INDEX IF NOT EXISTS idx_wisp_dep_external_target ON %[1]s (depends_on_external);`, table) +} + +func ensureDoltliteColumn(ctx context.Context, tx *sql.Tx, table, column, alterSQL string) error { + hasColumn, err := doltliteColumnExists(ctx, tx, table, column) + if err != nil { + return err + } + if hasColumn { + return nil + } + if _, err := tx.ExecContext(ctx, alterSQL); err != nil { + return fmt.Errorf("adding %s.%s: %w", table, column, err) + } + return nil +} + +func doltliteColumnExists(ctx context.Context, tx *sql.Tx, table, column string) (bool, error) { + columns, err := doltliteColumns(ctx, tx, table) + if err != nil { + return false, err + } + return columns[column], nil +} + +func doltliteColumns(ctx context.Context, tx *sql.Tx, table string) (map[string]bool, error) { + rows, err := tx.QueryContext(ctx, "PRAGMA table_info("+table+")") + if err != nil { + return nil, fmt.Errorf("reading %s columns: %w", table, err) + } + defer rows.Close() + columns := make(map[string]bool) + for rows.Next() { + var cid int + var name, typ string + var notnull int + var dflt sql.NullString + var pk int + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + return nil, fmt.Errorf("scanning %s columns: %w", table, err) + } + columns[name] = true + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("reading %s columns: %w", table, err) + } + return columns, nil +} + +// GetIssue is implemented in get_issue.go. + +func (s *DoltliteStore) GetIssueByExternalRef(ctx context.Context, externalRef string) (*types.Issue, error) { + var id string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + id, err = issueops.GetIssueByExternalRefInTx(ctx, tx, externalRef) + return err + }) + if err != nil { + return nil, err + } + return s.GetIssue(ctx, id) +} + +// GetIssuesByIDs is implemented in dependencies.go. + +// UpdateIssue is implemented in issues.go. + +// CloseIssue is implemented in issues.go. + +func (s *DoltliteStore) DeleteIssue(ctx context.Context, id string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.DeleteIssueSQLiteInTx(ctx, tx, id) + }) +} + +// AddDependency is implemented in dependencies.go. + +// RemoveDependency is implemented in dependencies.go. + +func (s *DoltliteStore) GetDependencies(ctx context.Context, issueID string) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependenciesInTx(ctx, tx, issueID) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetDependents(ctx context.Context, issueID string) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependentsInTx(ctx, tx, issueID) + return err + }) + return result, err +} + +// GetDependenciesWithMetadata is implemented in dependencies.go. + +// GetDependentsWithMetadata is implemented in dependencies.go. + +func (s *DoltliteStore) GetDependencyTree(ctx context.Context, issueID string, maxDepth int, showAllPaths bool, reverse bool) ([]*types.TreeNode, error) { + var result []*types.TreeNode + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependencyTreeInTx(ctx, tx, issueID, maxDepth, showAllPaths, reverse) + return err + }) + return result, err +} + +// AddLabel is implemented in labels.go. + +// RemoveLabel is implemented in labels.go. + +// GetLabels is implemented in labels.go. + +func (s *DoltliteStore) GetIssuesByLabel(ctx context.Context, label string) ([]*types.Issue, error) { + var ids []string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + ids, err = issueops.GetIssuesByLabelInTx(ctx, tx, label) + return err + }) + if err != nil { + return nil, err + } + return s.GetIssuesByIDs(ctx, ids) +} + +// GetReadyWork is implemented in queries.go. + +func (s *DoltliteStore) GetBlockedIssues(ctx context.Context, filter types.WorkFilter) ([]*types.BlockedIssue, error) { + var result []*types.BlockedIssue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetBlockedIssuesInTx(ctx, tx, filter) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetEpicsEligibleForClosure(ctx context.Context) ([]*types.EpicStatus, error) { + var result []*types.EpicStatus + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetEpicsEligibleForClosureInTx(ctx, tx) + return err + }) + return result, err +} + +func (s *DoltliteStore) AddIssueComment(ctx context.Context, issueID, author, text string) (*types.Comment, error) { + var result *types.Comment + err := s.withConn(ctx, true, func(tx *sql.Tx) error { + var err error + result, err = issueops.AddIssueCommentInTx(ctx, tx, issueID, author, text) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetIssueComments(ctx context.Context, issueID string) ([]*types.Comment, error) { + var result []*types.Comment + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetIssueCommentsInTx(ctx, tx, issueID) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetEvents(ctx context.Context, issueID string, limit int) ([]*types.Event, error) { + var result []*types.Event + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetEventsInTx(ctx, tx, issueID, limit) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetAllEventsSince(ctx context.Context, since time.Time) ([]*types.Event, error) { + var result []*types.Event + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetAllEventsSinceInTx(ctx, tx, since) + return err + }) + return result, err +} + +// RunInTransaction is implemented in transaction.go. + +// Close marks the store as closed and cleans up orphaned git-remote-cache +// garbage. Subsequent method calls will return errClosed. +func (s *DoltliteStore) Close() error { + if s.closed.CompareAndSwap(false, true) { + s.dbMu.Lock() + cleanup := s.dbCleanup + s.db = nil + s.dbCleanup = nil + s.dbMu.Unlock() + if cleanup != nil { + _ = cleanup() + } + s.cleanGitRemoteCacheGarbage() + } + return nil +} + +// DoltGC runs Dolt garbage collection to reclaim disk space. +func (s *DoltliteStore) DoltGC(ctx context.Context) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_gc()"); err != nil { + return fmt.Errorf("doltlite gc: %w", err) + } + return nil + }) +} + +// Flatten squashes all doltlite commit history into a single commit. +func (s *DoltliteStore) Flatten(ctx context.Context) error { + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + var initialHash string + if err := db.QueryRowContext(ctx, + "SELECT commit_hash FROM dolt_log ORDER BY date ASC LIMIT 1", + ).Scan(&initialHash); err != nil { + return fmt.Errorf("find initial commit: %w", err) + } + + var commitCount int + if err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM dolt_log", + ).Scan(&commitCount); err != nil { + return fmt.Errorf("count commits: %w", err) + } + if commitCount <= 1 { + return nil + } + + steps := []struct { + name string + query string + args []any + }{ + {"create temp branch", "SELECT dolt_branch('flatten-tmp')", nil}, + {"checkout temp branch", "SELECT dolt_checkout('flatten-tmp')", nil}, + {"soft reset to initial", "SELECT dolt_reset('--soft', ?)", []any{initialHash}}, + {"commit flattened snapshot", "SELECT dolt_commit('-A', '-m', 'flatten: squash all history into single commit')", nil}, + {"checkout main", "SELECT dolt_checkout('main')", nil}, + {"reset main to flattened", "SELECT dolt_reset('--hard', 'flatten-tmp')", nil}, + {"delete temp branch", "SELECT dolt_branch('-D', 'flatten-tmp')", nil}, + } + for _, step := range steps { + if _, err := db.ExecContext(ctx, step.query, step.args...); err != nil { + return fmt.Errorf("flatten step %q: %w", step.name, err) + } + } + return nil + }) +} + +// Compact squashes old doltlite commits while preserving recent ones. +func (s *DoltliteStore) Compact(ctx context.Context, initialHash, boundaryHash string, oldCommits int, recentHashes []string) error { + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) (retErr error) { + branchCreated := false + defer func() { + if retErr != nil && branchCreated { + _, _ = db.ExecContext(ctx, "SELECT dolt_checkout('main')") + _, _ = db.ExecContext(ctx, "SELECT dolt_branch('-D', 'compact-tmp')") + } + }() + + execSQL := func(name, query string, args ...any) error { + if _, err := db.ExecContext(ctx, query, args...); err != nil { + return fmt.Errorf("compact step %q: %w", name, err) + } + return nil + } + + if err := execSQL("create temp branch", "SELECT dolt_branch('compact-tmp', ?)", boundaryHash); err != nil { + return err + } + branchCreated = true + + if err := execSQL("checkout temp", "SELECT dolt_checkout('compact-tmp')"); err != nil { + return err + } + if err := execSQL("soft reset to initial", "SELECT dolt_reset('--soft', ?)", initialHash); err != nil { + return err + } + msg := fmt.Sprintf("compact: squash %d commits into base snapshot", oldCommits) + if err := execSQL("commit squashed base", "SELECT dolt_commit('-A', '-m', ?)", msg); err != nil { + return err + } + + for _, hash := range recentHashes { + label := hash + if len(label) > 8 { + label = label[:8] + } + if err := execSQL("cherry-pick "+label, "SELECT dolt_cherry_pick(?)", hash); err != nil { + return err + } + } + + if err := execSQL("checkout main", "SELECT dolt_checkout('main')"); err != nil { + return err + } + if err := execSQL("reset main to compacted", "SELECT dolt_reset('--hard', 'compact-tmp')"); err != nil { + return err + } + if err := execSQL("delete temp branch", "SELECT dolt_branch('-D', 'compact-tmp')"); err != nil { + return err + } + + return nil + }) +} + +// Path returns the doltlite data directory (.beads/doltlite/). +func (s *DoltliteStore) Path() string { + return s.dataDir +} + +// CLIDir returns the directory for dolt CLI operations (push/pull/remote). +// This is the actual database directory within the data dir. +func (s *DoltliteStore) CLIDir() string { + if s.dataDir == "" { + return "" + } + _, dbFile, err := buildDSN(s.dataDir, s.database) + if err != nil { + return "" + } + return dbFile +} + +// --------------------------------------------------------------------------- +// storage.VersionControl +// --------------------------------------------------------------------------- + +// Branch, Checkout, CurrentBranch, DeleteBranch, ListBranches are +// implemented in version_control.go. + +func (s *DoltliteStore) CommitPending(ctx context.Context, actor string) (bool, error) { + var hasPending bool + var msg string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + hasPending, err = hasPendingChangesDoltlite(ctx, tx) + if err != nil { + return err + } + if hasPending { + msg = buildDoltliteBatchCommitMessage(ctx, tx, actor) + } + return nil + }) + if err != nil { + return false, err + } + if !hasPending { + return false, nil + } + + if err := s.CommitWithConfig(ctx, msg); err != nil { + if issueops.IsNothingToCommitError(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// CommitExists is implemented in version_control.go. + +func (s *DoltliteStore) GetCurrentCommit(ctx context.Context) (string, error) { + var hash string + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + return db.QueryRowContext(ctx, "SELECT dolt_hashof('HEAD')").Scan(&hash) + }) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return hash, err +} + +// Status, Log, Merge, GetConflicts, ResolveConflicts are implemented in +// version_control.go. + +// --------------------------------------------------------------------------- +// storage.HistoryViewer +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) History(ctx context.Context, issueID string) ([]*storage.HistoryEntry, error) { + var result []*storage.HistoryEntry + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = doltliteHistoryInTx(ctx, tx, issueID) + return err + }) + return result, err +} + +func (s *DoltliteStore) AsOf(ctx context.Context, issueID string, ref string) (*types.Issue, error) { + var result *types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = doltliteAsOfInTx(ctx, tx, issueID, ref) + return err + }) + return result, err +} + +func (s *DoltliteStore) Diff(ctx context.Context, fromRef, toRef string) ([]*storage.DiffEntry, error) { + var result []*storage.DiffEntry + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = doltliteDiffInTx(ctx, tx, fromRef, toRef) + return err + }) + return result, err +} + +// --------------------------------------------------------------------------- +// storage.RemoteStore +// --------------------------------------------------------------------------- + +// RemoveRemote, ListRemotes, Push, Pull, ForcePush, Fetch, PushTo, PullFrom +// are implemented in version_control.go. + +// --------------------------------------------------------------------------- +// storage.SyncStore +// --------------------------------------------------------------------------- + +// Sync and SyncStatus are implemented in federation.go. + +// --------------------------------------------------------------------------- +// storage.FederationStore +// --------------------------------------------------------------------------- + +// AddFederationPeer, GetFederationPeer, ListFederationPeers, RemoveFederationPeer +// are implemented in federation.go via issueops. + +// --------------------------------------------------------------------------- +// storage.BulkIssueStore +// --------------------------------------------------------------------------- + +// CreateIssuesWithFullOptions is implemented in create_issue.go. + +func (s *DoltliteStore) DeleteIssues(ctx context.Context, ids []string, cascade bool, force bool, dryRun bool) (*types.DeleteIssuesResult, error) { + var result *types.DeleteIssuesResult + err := s.withConn(ctx, !dryRun, func(tx *sql.Tx) error { + var err error + result, err = issueops.DeleteIssuesInTx(ctx, tx, ids, cascade, force, dryRun) + return err + }) + return result, err +} + +func (s *DoltliteStore) DeleteIssuesBySourceRepo(ctx context.Context, sourceRepo string) (int, error) { + var count int + err := s.withConn(ctx, true, func(tx *sql.Tx) error { + var err error + count, err = issueops.DeleteIssuesBySourceRepoInTx(ctx, tx, sourceRepo) + return err + }) + return count, err +} + +func (s *DoltliteStore) UpdateIssueID(ctx context.Context, oldID, newID string, issue *types.Issue, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.UpdateIssueIDInTx(ctx, tx, oldID, newID, issue, actor) + }) +} + +// ClaimIssue is implemented in issues.go. + +func (s *DoltliteStore) PromoteFromEphemeral(ctx context.Context, id string, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.PromoteFromEphemeralInTx(ctx, tx, id, actor) + }) +} + +// GetNextChildID is implemented in child_id.go. + +func (s *DoltliteStore) RenameCounterPrefix(ctx context.Context, oldPrefix, newPrefix string) error { + return nil // Hash-based IDs don't use counters. +} + +// --------------------------------------------------------------------------- +// storage.DependencyQueryStore +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) GetDependencyRecords(ctx context.Context, issueID string) ([]*types.Dependency, error) { + var result []*types.Dependency + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + m, err := issueops.GetDependencyRecordsForIssuesInTx(ctx, tx, []string{issueID}) + if err != nil { + return err + } + result = m[issueID] + return nil + }) + return result, err +} + +// IsBlocked is implemented in issues.go. + +// GetNewlyUnblockedByClose is implemented in issues.go. + +// DetectCycles is implemented in dependencies.go. + +func (s *DoltliteStore) FindWispDependentsRecursive(ctx context.Context, ids []string) (map[string]bool, error) { + var result map[string]bool + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.FindWispDependentsRecursiveInTx(ctx, tx, ids) + return err + }) + return result, err +} + +func (s *DoltliteStore) RenameDependencyPrefix(ctx context.Context, oldPrefix, newPrefix string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + for _, table := range []string{"dependencies", "wisp_dependencies"} { + for _, column := range []string{"issue_id", "depends_on_issue_id", "depends_on_wisp_id"} { + if _, err := tx.ExecContext(ctx, fmt.Sprintf(` + UPDATE %s + SET %s = ? || substr(%s, ?) + WHERE %s = ? OR %s LIKE ? + `, table, column, column, column, column), + newPrefix, len(oldPrefix)+1, oldPrefix, oldPrefix+".%", + ); err != nil { + return fmt.Errorf("rename dependency prefix in %s.%s: %w", table, column, err) + } + } + } + return nil + }) +} + +// --------------------------------------------------------------------------- +// storage.AnnotationQueryStore +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) AddComment(ctx context.Context, issueID, actor, comment string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.AddCommentEventInTx(ctx, tx, issueID, actor, comment) + }) +} + +func (s *DoltliteStore) ImportIssueComment(ctx context.Context, issueID, author, text string, createdAt time.Time) (*types.Comment, error) { + var result *types.Comment + err := s.withConn(ctx, true, func(tx *sql.Tx) error { + var err error + result, err = issueops.ImportIssueCommentInTx(ctx, tx, issueID, author, text, createdAt) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetCommentsForIssues(ctx context.Context, issueIDs []string) (map[string][]*types.Comment, error) { + var result map[string][]*types.Comment + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetCommentsForIssuesInTx(ctx, tx, issueIDs) + return err + }) + return result, err +} + +// --------------------------------------------------------------------------- +// storage.ConfigMetadataStore +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) DeleteConfig(ctx context.Context, key string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.DeleteConfigInTx(ctx, tx, key) + }) +} + +func (s *DoltliteStore) GetCustomStatuses(ctx context.Context) ([]string, error) { + detailed, err := s.GetCustomStatusesDetailed(ctx) + if err != nil { + return nil, err + } + return types.CustomStatusNames(detailed), nil +} + +func (s *DoltliteStore) GetCustomStatusesDetailed(ctx context.Context) ([]types.CustomStatus, error) { + var result []types.CustomStatus + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var txErr error + result, txErr = issueops.ResolveCustomStatusesDetailedInTx(ctx, tx) + return txErr + }) + if err != nil { + // DB unavailable — fall back to config.yaml. + if yamlStatuses := config.GetCustomStatusesFromYAML(); len(yamlStatuses) > 0 { + return issueops.ParseStatusFallback(yamlStatuses), nil + } + return nil, nil + } + return result, nil +} + +func (s *DoltliteStore) GetCustomTypes(ctx context.Context) ([]string, error) { + var result []string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var txErr error + result, txErr = issueops.ResolveCustomTypesInTx(ctx, tx) + return txErr + }) + if err != nil { + // DB unavailable — fall back to config.yaml. + if yamlTypes := config.GetCustomTypesFromYAML(); len(yamlTypes) > 0 { + return yamlTypes, nil + } + return nil, err + } + return result, nil +} + +// --------------------------------------------------------------------------- +// storage.CompactionStore +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) CheckEligibility(ctx context.Context, issueID string, tier int) (bool, string, error) { + var eligible bool + var reason string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + eligible, reason, err = issueops.CheckEligibilityInTx(ctx, tx, issueID, tier) + return err + }) + return eligible, reason, err +} + +func (s *DoltliteStore) ApplyCompaction(ctx context.Context, issueID string, tier int, originalSize int, _ int, commitHash string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.ApplyCompactionInTx(ctx, tx, issueID, tier, originalSize, commitHash) + }) +} + +func (s *DoltliteStore) SnapshotIssue(ctx context.Context, issueID string, tier int) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.SnapshotIssueInTx(ctx, tx, issueID, tier) + }) +} + +func (s *DoltliteStore) GetCompactionSnapshot(ctx context.Context, issueID string) (*types.IssueSnapshot, error) { + var snap *types.IssueSnapshot + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + snap, err = issueops.GetLatestSnapshotInTx(ctx, tx, issueID) + return err + }) + return snap, err +} + +func (s *DoltliteStore) RestoreFromSnapshot(ctx context.Context, issueID string) (*types.IssueSnapshot, error) { + var snap *types.IssueSnapshot + err := s.withConn(ctx, true, func(tx *sql.Tx) error { + var err error + snap, err = issueops.RestoreFromSnapshotInTx(ctx, tx, issueID) + return err + }) + return snap, err +} + +func (s *DoltliteStore) GetTier1Candidates(ctx context.Context) ([]*types.CompactionCandidate, error) { + var result []*types.CompactionCandidate + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetTier1CandidatesInTx(ctx, tx) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetTier2Candidates(ctx context.Context) ([]*types.CompactionCandidate, error) { + var result []*types.CompactionCandidate + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetTier2CandidatesInTx(ctx, tx) + return err + }) + return result, err +} + +// --------------------------------------------------------------------------- +// storage.AdvancedQueryStore +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) GetRepoMtime(ctx context.Context, repoPath string) (int64, error) { + var result int64 + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetRepoMtimeInTx(ctx, tx, repoPath) + return err + }) + return result, err +} + +func (s *DoltliteStore) SetRepoMtime(ctx context.Context, repoPath, jsonlPath string, mtimeNs int64) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.SetRepoMtimeInTx(ctx, tx, repoPath, jsonlPath, mtimeNs) + }) +} + +func (s *DoltliteStore) ClearRepoMtime(ctx context.Context, repoPath string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.ClearRepoMtimeInTx(ctx, tx, repoPath) + }) +} + +// GetMoleculeProgress is implemented in queries.go. + +func (s *DoltliteStore) GetMoleculeLastActivity(ctx context.Context, moleculeID string) (*types.MoleculeLastActivity, error) { + var result *types.MoleculeLastActivity + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetMoleculeLastActivityInTx(ctx, tx, moleculeID) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetStaleIssues(ctx context.Context, filter types.StaleFilter) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetStaleIssuesInTx(ctx, tx, filter) + return err + }) + return result, err +} diff --git a/internal/storage/doltlite/store_repair_test.go b/internal/storage/doltlite/store_repair_test.go new file mode 100644 index 000000000..0119b8012 --- /dev/null +++ b/internal/storage/doltlite/store_repair_test.go @@ -0,0 +1,113 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "testing" +) + +func TestEnsureDoltliteWispDependenciesShapeMigratesLegacyRows(t *testing.T) { + ctx := context.Background() + db, err := sql.Open(driverName, ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() //nolint:errcheck // test cleanup + + for _, stmt := range []string{ + `CREATE TABLE issues (id TEXT PRIMARY KEY)`, + `CREATE TABLE wisps (id TEXT PRIMARY KEY)`, + `CREATE TABLE wisp_dependencies ( + issue_id TEXT NOT NULL, + depends_on_id TEXT NOT NULL, + type TEXT DEFAULT 'blocks', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + created_by TEXT DEFAULT '', + metadata TEXT DEFAULT '{}', + thread_id TEXT DEFAULT '', + PRIMARY KEY (issue_id, depends_on_id) + )`, + `INSERT INTO issues (id) VALUES ('bd-issue-target')`, + `INSERT INTO wisps (id) VALUES ('bd-wisp-source'), ('bd-wisp-target')`, + `INSERT INTO wisp_dependencies (issue_id, depends_on_id, type) VALUES + ('bd-wisp-source', 'bd-issue-target', 'blocks'), + ('bd-wisp-source', 'bd-wisp-target', 'waits-for'), + ('bd-wisp-source', 'external:ticket-1', 'tracks')`, + } { + if _, err := db.ExecContext(ctx, stmt); err != nil { + t.Fatalf("setup: %v\nstmt: %s", err, stmt) + } + } + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin: %v", err) + } + if err := ensureDoltliteWispDependenciesShape(ctx, tx); err != nil { + _ = tx.Rollback() + t.Fatalf("ensureDoltliteWispDependenciesShape: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit: %v", err) + } + + for _, column := range []string{"depends_on_issue_id", "depends_on_wisp_id", "depends_on_external"} { + ok, err := testDoltliteColumnExists(ctx, db, "wisp_dependencies", column) + if err != nil { + t.Fatalf("column %s: %v", column, err) + } + if !ok { + t.Fatalf("wisp_dependencies missing repaired column %s", column) + } + } + + assertWispDependencyTarget(t, ctx, db, "blocks", "bd-issue-target", "", "") + assertWispDependencyTarget(t, ctx, db, "waits-for", "", "bd-wisp-target", "") + assertWispDependencyTarget(t, ctx, db, "tracks", "", "", "external:ticket-1") +} + +func testDoltliteColumnExists(ctx context.Context, db *sql.DB, table, column string) (bool, error) { + rows, err := db.QueryContext(ctx, "PRAGMA table_info("+table+")") + if err != nil { + return false, err + } + defer rows.Close() + for rows.Next() { + var cid int + var name, typ string + var notnull int + var dflt sql.NullString + var pk int + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + return false, err + } + if name == column { + return true, nil + } + } + return false, rows.Err() +} + +func assertWispDependencyTarget(t *testing.T, ctx context.Context, db *sql.DB, depType, wantIssue, wantWisp, wantExternal string) { + t.Helper() + var issue, wisp, external sql.NullString + err := db.QueryRowContext(ctx, ` + SELECT depends_on_issue_id, depends_on_wisp_id, depends_on_external + FROM wisp_dependencies + WHERE type = ? + `, depType).Scan(&issue, &wisp, &external) + if err != nil { + t.Fatalf("read repaired dependency %s: %v", depType, err) + } + if issue.String != wantIssue || issue.Valid != (wantIssue != "") { + t.Fatalf("%s issue target = %q valid=%v, want %q", depType, issue.String, issue.Valid, wantIssue) + } + if wisp.String != wantWisp || wisp.Valid != (wantWisp != "") { + t.Fatalf("%s wisp target = %q valid=%v, want %q", depType, wisp.String, wisp.Valid, wantWisp) + } + if external.String != wantExternal || external.Valid != (wantExternal != "") { + t.Fatalf("%s external target = %q valid=%v, want %q", depType, external.String, external.Valid, wantExternal) + } +} diff --git a/internal/storage/doltlite/store_stub.go b/internal/storage/doltlite/store_stub.go new file mode 100644 index 000000000..b3394dae1 --- /dev/null +++ b/internal/storage/doltlite/store_stub.go @@ -0,0 +1,28 @@ +//go:build !cgo + +package doltlite + +import ( + "context" + "errors" +) + +// DoltliteStore is a stub for builds without CGO. +type DoltliteStore struct { + dataDir string + database string + branch string +} + +// Option configures optional behavior for New (stub: no-op). +type Option func(*struct{}) + +// WithLock is a no-op in non-CGO builds. +func WithLock(_ Unlocker) Option { + return func(*struct{}) {} +} + +// New returns an error when CGO is not enabled. +func New(_ context.Context, _, _, _ string, _ ...Option) (*DoltliteStore, error) { + return nil, errors.New("doltlite: requires CGO (build with CGO_ENABLED=1)") +} diff --git a/internal/storage/doltlite/time_travel.go b/internal/storage/doltlite/time_travel.go new file mode 100644 index 000000000..1b7e78558 --- /dev/null +++ b/internal/storage/doltlite/time_travel.go @@ -0,0 +1,164 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func doltliteAsOfInTx(ctx context.Context, tx *sql.Tx, issueID string, ref string) (*types.Issue, error) { + if err := issueops.ValidateRef(ref); err != nil { + return nil, fmt.Errorf("invalid ref: %w", err) + } + + query := fmt.Sprintf(` + SELECT %s + FROM dolt_at_issues(?) + WHERE id = ? + `, issueops.IssueSelectColumns) + issue, err := issueops.ScanIssueFrom(tx.QueryRowContext(ctx, query, ref, issueID)) + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: issue %s as of %s", storage.ErrNotFound, issueID, ref) + } + if err != nil { + return nil, fmt.Errorf("get issue as of %s: %w", ref, err) + } + return issue, nil +} + +func doltliteDiffInTx(ctx context.Context, tx *sql.Tx, fromRef, toRef string) ([]*storage.DiffEntry, error) { + if err := issueops.ValidateRef(fromRef); err != nil { + return nil, fmt.Errorf("invalid fromRef: %w", err) + } + if err := issueops.ValidateRef(toRef); err != nil { + return nil, fmt.Errorf("invalid toRef: %w", err) + } + + rows, err := tx.QueryContext(ctx, ` + SELECT + COALESCE(from_id, '') as from_id, + COALESCE(to_id, '') as to_id, + diff_type, + from_title, to_title, + from_description, to_description, + from_status, to_status, + from_priority, to_priority + FROM dolt_diff_issues(?, ?) + `, fromRef, toRef) + if err != nil { + return nil, fmt.Errorf("failed to get diff: %w", err) + } + defer rows.Close() + + var entries []*storage.DiffEntry + for rows.Next() { + var fromID, toID, diffType string + var fromTitle, toTitle, fromDesc, toDesc, fromStatus, toStatus *string + var fromPriority, toPriority *int + + if err := rows.Scan(&fromID, &toID, &diffType, + &fromTitle, &toTitle, + &fromDesc, &toDesc, + &fromStatus, &toStatus, + &fromPriority, &toPriority); err != nil { + return nil, fmt.Errorf("failed to scan diff: %w", err) + } + + entry := &storage.DiffEntry{DiffType: diffType} + if toID != "" { + entry.IssueID = toID + } else { + entry.IssueID = fromID + } + if diffType != "added" && fromID != "" { + entry.OldValue = &types.Issue{ID: fromID} + if fromTitle != nil { + entry.OldValue.Title = *fromTitle + } + if fromDesc != nil { + entry.OldValue.Description = *fromDesc + } + if fromStatus != nil { + entry.OldValue.Status = types.Status(*fromStatus) + } + if fromPriority != nil { + entry.OldValue.Priority = *fromPriority + } + } + if diffType != "removed" && toID != "" { + entry.NewValue = &types.Issue{ID: toID} + if toTitle != nil { + entry.NewValue.Title = *toTitle + } + if toDesc != nil { + entry.NewValue.Description = *toDesc + } + if toStatus != nil { + entry.NewValue.Status = types.Status(*toStatus) + } + if toPriority != nil { + entry.NewValue.Priority = *toPriority + } + } + + entries = append(entries, entry) + } + + return entries, rows.Err() +} + +func doltliteHistoryInTx(ctx context.Context, tx *sql.Tx, issueID string) ([]*storage.HistoryEntry, error) { + rows, err := tx.QueryContext(ctx, ` + SELECT commit_hash, committer, commit_date + FROM dolt_history_issues + WHERE id = ? + ORDER BY commit_date DESC + `, issueID) + if err != nil { + return nil, fmt.Errorf("failed to get issue history: %w", err) + } + + type historyMeta struct { + hash string + committer string + date any + } + var metas []historyMeta + for rows.Next() { + var meta historyMeta + if err := rows.Scan(&meta.hash, &meta.committer, &meta.date); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("failed to scan history: %w", err) + } + metas = append(metas, meta) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return nil, err + } + if err := rows.Close(); err != nil { + return nil, err + } + + entries := make([]*storage.HistoryEntry, 0, len(metas)) + for _, meta := range metas { + issue, err := doltliteAsOfInTx(ctx, tx, issueID, meta.hash) + if err != nil { + return nil, err + } + entries = append(entries, &storage.HistoryEntry{ + CommitHash: meta.hash, + Committer: meta.committer, + CommitDate: parseDoltliteTimeValue(meta.date), + Issue: issue, + }) + } + return entries, nil +} diff --git a/internal/storage/doltlite/transaction.go b/internal/storage/doltlite/transaction.go new file mode 100644 index 000000000..b24eaf4ad --- /dev/null +++ b/internal/storage/doltlite/transaction.go @@ -0,0 +1,249 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" + "github.com/steveyegge/beads/internal/types" +) + +// RunInTransaction executes a function within a database transaction. +// After the SQL transaction commits, dirty tables are selectively staged +// and a Dolt version commit is created with the given message. +func (s *DoltliteStore) RunInTransaction(ctx context.Context, commitMsg string, fn func(tx storage.Transaction) error) error { + var tracker versioncontrolops.DirtyTableTracker + + if err := s.withConn(ctx, true, func(tx *sql.Tx) error { + return fn(&embeddedTransaction{tx: tx, dirty: &tracker}) + }); err != nil { + return err + } + + // Create a Dolt version commit from the working set changes. + if commitMsg != "" && len(tracker.DirtyTables()) > 0 { + if err := s.Commit(ctx, commitMsg); err != nil { + return storage.NewPostTransactionCommitError(commitMsg, err) + } + } + return nil +} + +type embeddedTransaction struct { + tx *sql.Tx + dirty *versioncontrolops.DirtyTableTracker +} + +func (t *embeddedTransaction) CreateIssue(ctx context.Context, issue *types.Issue, actor string) error { + bc, err := issueops.NewBatchContext(ctx, t.tx, storage.BatchCreateOptions{SkipPrefixValidation: true}) + if err != nil { + return err + } + if err := createIssueSQLite(ctx, t.tx, bc, issue, actor); err != nil { + return err + } + t.dirty.MarkDirty("issues") + t.dirty.MarkDirty("wisps") + t.dirty.MarkDirty("events") + t.dirty.MarkDirty("wisp_events") + t.dirty.MarkDirty("labels") + t.dirty.MarkDirty("wisp_labels") + t.dirty.MarkDirty("comments") + t.dirty.MarkDirty("wisp_comments") + return nil +} + +func (t *embeddedTransaction) CreateIssues(ctx context.Context, issues []*types.Issue, actor string) error { + bc, err := issueops.NewBatchContext(ctx, t.tx, storage.BatchCreateOptions{ + OrphanHandling: storage.OrphanAllow, + SkipPrefixValidation: true, + }) + if err != nil { + return err + } + for _, issue := range issues { + if err := createIssueSQLite(ctx, t.tx, bc, issue, actor); err != nil { + return err + } + } + t.dirty.MarkDirty("issues") + t.dirty.MarkDirty("wisps") + t.dirty.MarkDirty("events") + t.dirty.MarkDirty("wisp_events") + t.dirty.MarkDirty("labels") + t.dirty.MarkDirty("wisp_labels") + t.dirty.MarkDirty("comments") + t.dirty.MarkDirty("wisp_comments") + return nil +} + +func (t *embeddedTransaction) UpdateIssue(ctx context.Context, id string, updates map[string]interface{}, actor string) error { + t.dirty.MarkDirty("issues") + t.dirty.MarkDirty("events") + _, err := issueops.UpdateIssueSQLiteInTx(ctx, t.tx, id, updates, actor) + return err +} + +func (t *embeddedTransaction) CloseIssue(ctx context.Context, id string, reason string, actor string, session string) error { + t.dirty.MarkDirty("issues") + t.dirty.MarkDirty("events") + _, err := issueops.CloseIssueSQLiteInTx(ctx, t.tx, id, reason, actor, session) + return err +} + +func (t *embeddedTransaction) DeleteIssue(ctx context.Context, id string) error { + t.dirty.MarkDirty("issues") + t.dirty.MarkDirty("dependencies") + t.dirty.MarkDirty("labels") + t.dirty.MarkDirty("comments") + t.dirty.MarkDirty("events") + return issueops.DeleteIssueSQLiteInTx(ctx, t.tx, id) +} + +func (t *embeddedTransaction) GetIssue(ctx context.Context, id string) (*types.Issue, error) { + return issueops.GetIssueInTx(ctx, t.tx, id) +} + +func (t *embeddedTransaction) SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { + return issueops.SearchIssuesInTx(ctx, t.tx, query, filter) +} + +func (t *embeddedTransaction) AddDependency(ctx context.Context, dep *types.Dependency, actor string) error { + return t.AddDependencyWithOptions(ctx, dep, actor, storage.DependencyAddOptions{}) +} + +func (t *embeddedTransaction) AddDependencyWithOptions(ctx context.Context, dep *types.Dependency, actor string, addOpts storage.DependencyAddOptions) error { + _, _, _, depTable := issueops.WispTableRouting(issueops.IsActiveWispInTx(ctx, t.tx, dep.IssueID)) + if err := issueops.AddDependencyInTx(ctx, t.tx, dep, actor, issueops.AddDependencyOpts{ + IsCrossPrefix: types.ExtractPrefix(dep.IssueID) != types.ExtractPrefix(dep.DependsOnID), + SkipCycleCheck: addOpts.SkipCycleCheck, + UseSQLiteBlockedRecompute: true, + }); err != nil { + return err + } + t.dirty.MarkDirty(depTable) + return nil +} + +// CycleThroughEdges reports a blocking cycle through one of the new edges, +// including the transaction's own uncommitted dependency writes +// (bd-6dnrw.8, bd-578h9.9). +func (t *embeddedTransaction) CycleThroughEdges(ctx context.Context, edges [][2]string) (string, error) { + graph := make(map[string][]string) + if err := issueops.AppendBlockingGraphInTx(ctx, t.tx, []string{"dependencies", "wisp_dependencies"}, graph); err != nil { + return "", err + } + return issueops.CycleThroughEdgesInGraph(graph, edges), nil +} + +func (t *embeddedTransaction) RemoveDependency(ctx context.Context, issueID, dependsOnID string, actor string) error { + t.dirty.MarkDirty("dependencies") + return issueops.RemoveDependencySQLiteInTx(ctx, t.tx, issueID, dependsOnID) +} + +func (t *embeddedTransaction) GetDependencyRecords(ctx context.Context, issueID string) ([]*types.Dependency, error) { + m, err := issueops.GetDependencyRecordsForIssuesInTx(ctx, t.tx, []string{issueID}) + if err != nil { + return nil, err + } + return m[issueID], nil +} + +func (t *embeddedTransaction) AddLabel(ctx context.Context, issueID, label, actor string) error { + isWisp := issueops.IsActiveWispInTx(ctx, t.tx, issueID) + _, labelTable, eventTable, _ := issueops.WispTableRouting(isWisp) + t.dirty.MarkDirty(labelTable) + t.dirty.MarkDirty(eventTable) + if _, err := t.tx.ExecContext(ctx, fmt.Sprintf("INSERT OR IGNORE INTO %s (issue_id, label) VALUES (?, ?)", labelTable), issueID, label); err != nil { + return fmt.Errorf("add label: %w", err) + } + if _, err := t.tx.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s (id, issue_id, event_type, actor, comment) VALUES (?, ?, ?, ?, ?)", eventTable), + issueops.NewEventID(), issueID, types.EventLabelAdded, actor, "Added label: "+label); err != nil { + return fmt.Errorf("add label: record event: %w", err) + } + return nil +} + +func (t *embeddedTransaction) RemoveLabel(ctx context.Context, issueID, label, actor string) error { + t.dirty.MarkDirty("labels") + return issueops.RemoveLabelInTx(ctx, t.tx, "", "", issueID, label, actor) +} + +func (t *embeddedTransaction) GetLabels(ctx context.Context, issueID string) ([]string, error) { + return issueops.GetLabelsInTx(ctx, t.tx, "", issueID) +} + +func (t *embeddedTransaction) SetConfig(ctx context.Context, key, value string) error { + t.dirty.MarkDirty("config") + if err := issueops.SetConfigInTx(ctx, t.tx, key, value); err != nil { + return err + } + // Sync normalized tables when config keys change + switch key { + case "status.custom": + t.dirty.MarkDirty("custom_statuses") + if err := issueops.SyncCustomStatusesTable(ctx, t.tx, value); err != nil { + return fmt.Errorf("syncing custom_statuses table: %w", err) + } + case "types.custom": + t.dirty.MarkDirty("custom_types") + if err := issueops.SyncCustomTypesTable(ctx, t.tx, value); err != nil { + return fmt.Errorf("syncing custom_types table: %w", err) + } + } + return nil +} + +func (t *embeddedTransaction) GetConfig(ctx context.Context, key string) (string, error) { + return issueops.GetConfigInTx(ctx, t.tx, key) +} + +func (t *embeddedTransaction) SetMetadata(ctx context.Context, key, value string) error { + t.dirty.MarkDirty("metadata") + return issueops.SetMetadataInTx(ctx, t.tx, key, value) +} + +func (t *embeddedTransaction) GetMetadata(ctx context.Context, key string) (string, error) { + return issueops.GetMetadataInTx(ctx, t.tx, key) +} + +func (t *embeddedTransaction) SetLocalMetadata(ctx context.Context, key, value string) error { + return issueops.SetLocalMetadataInTx(ctx, t.tx, key, value) +} + +func (t *embeddedTransaction) GetLocalMetadata(ctx context.Context, key string) (string, error) { + return issueops.GetLocalMetadataInTx(ctx, t.tx, key) +} + +func (t *embeddedTransaction) AddComment(ctx context.Context, issueID, actor, comment string) error { + return fmt.Errorf("embeddedTransaction: AddComment not implemented") +} + +func (t *embeddedTransaction) ImportIssueComment(ctx context.Context, issueID, author, text string, createdAt time.Time) (*types.Comment, error) { + return nil, fmt.Errorf("embeddedTransaction: ImportIssueComment not implemented") +} + +func (t *embeddedTransaction) GetIssueComments(ctx context.Context, issueID string) ([]*types.Comment, error) { + return nil, fmt.Errorf("embeddedTransaction: GetIssueComments not implemented") +} + +func (t *embeddedTransaction) CreateIssueImport(ctx context.Context, issue *types.Issue, actor string, skipPrefixValidation bool) error { + bc, err := issueops.NewBatchContext(ctx, t.tx, storage.BatchCreateOptions{SkipPrefixValidation: skipPrefixValidation}) + if err != nil { + return err + } + result, err := issueops.CreateIssueInTxWithResult(ctx, t.tx, bc, issue, actor) + if err != nil { + return err + } + for table := range issueops.CreateIssueDirtyTables(ctx, issue, result) { + t.dirty.MarkDirty(table) + } + return nil +} diff --git a/internal/storage/doltlite/version_control.go b/internal/storage/doltlite/version_control.go new file mode 100644 index 000000000..ae77bdb5d --- /dev/null +++ b/internal/storage/doltlite/version_control.go @@ -0,0 +1,558 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/doltutil" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" +) + +// withDBConn opens a database connection configured for the store's native +// doltlite branch and passes it to fn without starting an explicit SQL +// transaction. Version-control functions manage their own transaction boundary. +func (s *DoltliteStore) withDBConn(ctx context.Context, fn func(db versioncontrolops.DBConn) error) (err error) { + if s.closed.Load() { + return errClosed + } + + var db *sql.DB + var cleanup func() error + db, cleanup, err = s.activeDB(ctx) + if err != nil { + return + } + defer func() { + err = errors.Join(err, cleanup()) + s.cleanGitRemoteCacheGarbage() + }() + + return fn(db) +} + +func (s *DoltliteStore) withDBWrite(ctx context.Context, fn func(db versioncontrolops.DBConn) error) error { + return s.withExclusiveLock(ctx, func() error { + return s.withRetryRefreshingDB(ctx, func() error { + return s.withDBConn(ctx, fn) + }) + }) +} + +// commitAuthor returns the author string for native doltlite commits. +const commitAuthor = commitName + " <" + commitEmail + ">" + +func commitNative(ctx context.Context, db versioncontrolops.DBConn, message string, includeConfig bool) error { + if message == "" { + message = "doltlite: snapshot" + } + + tables, err := pendingTablesDoltlite(ctx, db, includeConfig) + if err != nil { + return err + } + if len(tables) == 0 { + return nil + } + + for _, table := range tables { + if _, err := db.ExecContext(ctx, "SELECT dolt_add(?)", table); err != nil { + return fmt.Errorf("doltlite add %s: %w", table, err) + } + } + + _, err = db.ExecContext(ctx, "SELECT dolt_commit('-m', ?, '--author', ?)", message, commitAuthor) + if err != nil && !issueops.IsNothingToCommitError(err) { + return fmt.Errorf("doltlite commit: %w", err) + } + return nil +} + +func pendingTablesDoltlite(ctx context.Context, db issueops.SQLQuerier, includeConfig bool) ([]string, error) { + rows, err := db.QueryContext(ctx, "SELECT table_name FROM dolt_status ORDER BY table_name") + if err != nil { + return nil, fmt.Errorf("failed to query status: %w", err) + } + defer rows.Close() + + var tables []string + for rows.Next() { + var table string + if err := rows.Scan(&table); err != nil { + return nil, fmt.Errorf("failed to scan status: %w", err) + } + if !includeConfig && table == "config" { + continue + } + if isDoltliteRuntimeTable(table) { + continue + } + tables = append(tables, table) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to iterate status: %w", err) + } + return tables, nil +} + +func hasPendingChangesDoltlite(ctx context.Context, db issueops.SQLQuerier) (bool, error) { + tables, err := pendingTablesDoltlite(ctx, db, true) + if err != nil { + return false, err + } + return len(tables) > 0, nil +} + +func isDoltliteRuntimeTable(table string) bool { + switch table { + case "wisps", "wisp_labels", "wisp_dependencies", "wisp_events", "wisp_comments", + "wisp_child_counters", "repo_mtimes", "local_metadata": + return true + default: + return false + } +} + +func commitAllNative(ctx context.Context, db versioncontrolops.DBConn, message string) error { + if message == "" { + message = "doltlite: snapshot" + } + _, err := db.ExecContext(ctx, "SELECT dolt_commit('-A', '-m', ?, '--author', ?)", message, commitAuthor) + if err != nil && !issueops.IsNothingToCommitError(err) { + return fmt.Errorf("doltlite commit: %w", err) + } + return nil +} + +func (s *DoltliteStore) Commit(ctx context.Context, message string) error { + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + return commitNative(ctx, db, message, false) + }) +} + +// CommitWithConfig commits all working set changes including config. +func (s *DoltliteStore) CommitWithConfig(ctx context.Context, message string) error { + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + return commitAllNative(ctx, db, message) + }) +} + +func (s *DoltliteStore) CommitMergeResolution(ctx context.Context, message string) error { + return s.CommitWithConfig(ctx, message) +} + +func (s *DoltliteStore) AddRemote(ctx context.Context, name, url string) error { + if err := validateDoltliteRemoteSyncURL(name, url); err != nil { + return err + } + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + var existing string + err := db.QueryRowContext(ctx, "SELECT url FROM dolt_remotes WHERE name = ?", name).Scan(&existing) + switch { + case err == nil && existing == url: + return nil + case err == nil: + if _, rmErr := db.ExecContext(ctx, "SELECT dolt_remote('remove', ?)", name); rmErr != nil { + return fmt.Errorf("remove existing remote %s: %w", name, rmErr) + } + case errors.Is(err, sql.ErrNoRows): + default: + return fmt.Errorf("lookup remote %s: %w", name, err) + } + + if _, err := db.ExecContext(ctx, "SELECT dolt_remote('add', ?, ?)", name, url); err != nil { + return fmt.Errorf("add remote %s: %w", name, err) + } + return nil + }) +} + +func (s *DoltliteStore) HasRemote(ctx context.Context, name string) (bool, error) { + var count int + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + return db.QueryRowContext(ctx, "SELECT count(*) FROM dolt_remotes WHERE name = ?", name).Scan(&count) + }) + if err != nil { + return false, err + } + return count > 0, nil +} + +// --------------------------------------------------------------------------- +// Branch operations +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) Branch(ctx context.Context, name string) error { + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_branch(?)", name); err != nil { + return fmt.Errorf("create branch %s: %w", name, err) + } + return nil + }) +} + +func (s *DoltliteStore) Checkout(ctx context.Context, branch string) error { + if err := s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_checkout(?)", branch); err != nil { + return fmt.Errorf("checkout branch %s: %w", branch, err) + } + return nil + }); err != nil { + return err + } + s.branch = branch + return nil +} + +func (s *DoltliteStore) CurrentBranch(ctx context.Context) (string, error) { + var branch string + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + branch, err = versioncontrolops.CurrentBranch(ctx, db) + return err + }) + if err != nil { + return "", err + } + if branch != "" { + s.branch = branch + } + return branch, nil +} + +func (s *DoltliteStore) DeleteBranch(ctx context.Context, branch string) error { + current, err := s.CurrentBranch(ctx) + if err != nil { + return err + } + if branch == current { + return fmt.Errorf("delete branch %s: cannot delete current branch", branch) + } + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_branch('-D', ?)", branch); err != nil { + return fmt.Errorf("delete branch %s: %w", branch, err) + } + return nil + }) +} + +func (s *DoltliteStore) ListBranches(ctx context.Context) ([]string, error) { + var branches []string + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + branches, err = versioncontrolops.ListBranches(ctx, db) + return err + }) + return branches, err +} + +// --------------------------------------------------------------------------- +// Version control operations +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) CommitExists(ctx context.Context, commitHash string) (bool, error) { + var exists bool + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + exists, err = versioncontrolops.CommitExists(ctx, db, commitHash) + return err + }) + return exists, err +} + +func (s *DoltliteStore) Status(ctx context.Context) (*storage.Status, error) { + var status *storage.Status + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + status, err = versioncontrolops.Status(ctx, db) + return err + }) + return status, err +} + +func (s *DoltliteStore) Log(ctx context.Context, limit int) ([]storage.CommitInfo, error) { + query := "SELECT commit_hash, committer, email, date, message FROM dolt_log ORDER BY date DESC" + var args []any + if limit > 0 { + query += " LIMIT ?" + args = append(args, limit) + } + + var commits []storage.CommitInfo + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + rows, err := db.QueryContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("get log: %w", err) + } + defer rows.Close() + + for rows.Next() { + var c storage.CommitInfo + var date any + if err := rows.Scan(&c.Hash, &c.Author, &c.Email, &date, &c.Message); err != nil { + return fmt.Errorf("scan commit: %w", err) + } + c.Date = parseDoltliteTimeValue(date) + commits = append(commits, c) + } + return rows.Err() + }) + return commits, err +} + +func parseDoltliteTimeValue(v any) time.Time { + switch t := v.(type) { + case time.Time: + return t + case string: + return parseDoltliteTime(t) + case []byte: + return parseDoltliteTime(string(t)) + case int64: + return time.Unix(t, 0).UTC() + default: + return time.Time{} + } +} + +func parseDoltliteTime(s string) time.Time { + for _, layout := range []string{ + time.RFC3339Nano, + time.RFC3339, + "2006-01-02 15:04:05.999999999-07:00", + "2006-01-02 15:04:05.999999999Z07:00", + "2006-01-02 15:04:05", + } { + if t, err := time.Parse(layout, s); err == nil { + return t + } + } + return time.Time{} +} + +func (s *DoltliteStore) Merge(ctx context.Context, branch string) ([]storage.Conflict, error) { + var conflicts []storage.Conflict + err := s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + if _, mergeErr := db.ExecContext(ctx, "SELECT dolt_merge(?)", branch); mergeErr != nil { + c, conflictErr := versioncontrolops.GetConflicts(ctx, db) + if conflictErr == nil && len(c) > 0 { + conflicts = c + return nil + } + return fmt.Errorf("merge branch %s: %w", branch, mergeErr) + } + return nil + }) + return conflicts, err +} + +func (s *DoltliteStore) GetConflicts(ctx context.Context) ([]storage.Conflict, error) { + var conflicts []storage.Conflict + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + conflicts, err = versioncontrolops.GetConflicts(ctx, db) + return err + }) + return conflicts, err +} + +func (s *DoltliteStore) ResolveConflicts(ctx context.Context, table string, strategy string) error { + if table == "" || !validIdentifier.MatchString(table) { + return fmt.Errorf("invalid table name: %s", table) + } + var flag string + switch strategy { + case "ours": + flag = "--ours" + case "theirs": + flag = "--theirs" + default: + return fmt.Errorf("unknown conflict resolution strategy: %s", strategy) + } + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_conflicts_resolve(?, ?)", flag, table); err != nil { + return fmt.Errorf("resolve conflicts: %w", err) + } + return nil + }) +} + +// --------------------------------------------------------------------------- +// Remote operations +// --------------------------------------------------------------------------- + +const defaultRemote = "origin" + +var errDoltliteUnsupportedRemoteURL = errors.New("doltlite remote URL unsupported") + +func validateDoltliteRemoteSyncURL(remote, url string) error { + if strings.HasPrefix(url, "file://") || strings.HasPrefix(url, "http://") { + return nil + } + if doltutil.IsGitProtocolURL(url) { + return fmt.Errorf("%w: remote %q uses git protocol URL %q; DoltLite remote sync supports only file:// and http:// remotes. Use the Dolt backend for GitHub/git+ssh sync, or replace this remote with a DoltLite-supported URL", errDoltliteUnsupportedRemoteURL, remote, url) + } + return fmt.Errorf("%w: remote %q uses URL %q; DoltLite remote sync supports only file:// and http:// remotes", errDoltliteUnsupportedRemoteURL, remote, url) +} + +func guardDoltliteRemoteSyncURL(ctx context.Context, db versioncontrolops.DBConn, remote string) error { + var url string + err := db.QueryRowContext(ctx, "SELECT url FROM dolt_remotes WHERE name = ?", remote).Scan(&url) + switch { + case err == nil: + return validateDoltliteRemoteSyncURL(remote, url) + case errors.Is(err, sql.ErrNoRows): + return nil // Preserve existing remote-not-found handling from dolt_push/dolt_pull. + default: + return fmt.Errorf("lookup remote %s: %w", remote, err) + } +} + +func (s *DoltliteStore) RemoveRemote(ctx context.Context, name string) error { + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_remote('remove', ?)", name); err != nil { + return fmt.Errorf("remove remote %s: %w", name, err) + } + return nil + }) +} + +func (s *DoltliteStore) ListRemotes(ctx context.Context) ([]storage.RemoteInfo, error) { + var remotes []storage.RemoteInfo + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + remotes, err = versioncontrolops.ListRemotes(ctx, db) + return err + }) + return remotes, err +} + +func (s *DoltliteStore) Push(ctx context.Context) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if err := guardDoltliteRemoteSyncURL(ctx, db, defaultRemote); err != nil { + return err + } + _, err := db.ExecContext(ctx, "SELECT dolt_push(?, ?)", defaultRemote, s.branch) + return err + }) +} + +func (s *DoltliteStore) Pull(ctx context.Context) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if err := guardDoltliteRemoteSyncURL(ctx, db, defaultRemote); err != nil { + return err + } + _, err := db.ExecContext(ctx, "SELECT dolt_pull(?, ?)", defaultRemote, s.branch) + return err + }) +} + +func (s *DoltliteStore) ForcePush(ctx context.Context) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if err := guardDoltliteRemoteSyncURL(ctx, db, defaultRemote); err != nil { + return err + } + _, err := db.ExecContext(ctx, "SELECT dolt_push(?, ?, '--force')", defaultRemote, s.branch) + return err + }) +} + +func (s *DoltliteStore) PushRemote(ctx context.Context, remote string, force bool) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if err := guardDoltliteRemoteSyncURL(ctx, db, remote); err != nil { + return err + } + if force { + _, err := db.ExecContext(ctx, "SELECT dolt_push(?, ?, '--force')", remote, s.branch) + return err + } + _, err := db.ExecContext(ctx, "SELECT dolt_push(?, ?)", remote, s.branch) + return err + }) +} + +func (s *DoltliteStore) PullRemote(ctx context.Context, remote string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if err := guardDoltliteRemoteSyncURL(ctx, db, remote); err != nil { + return err + } + _, err := db.ExecContext(ctx, "SELECT dolt_pull(?, ?)", remote, s.branch) + return err + }) +} + +func (s *DoltliteStore) Fetch(ctx context.Context, peer string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if err := guardDoltliteRemoteSyncURL(ctx, db, peer); err != nil { + return err + } + _, err := db.ExecContext(ctx, "SELECT dolt_fetch(?)", peer) + return err + }) +} + +func (s *DoltliteStore) PushTo(ctx context.Context, peer string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if err := guardDoltliteRemoteSyncURL(ctx, db, peer); err != nil { + return err + } + _, err := db.ExecContext(ctx, "SELECT dolt_push(?, ?)", peer, s.branch) + return err + }) +} + +func (s *DoltliteStore) PullFrom(ctx context.Context, peer string) ([]storage.Conflict, error) { + if _, err := s.CommitPending(ctx, "beads"); err != nil { + return nil, fmt.Errorf("commit pending before pull: %w", err) + } + + var conflicts []storage.Conflict + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if err := guardDoltliteRemoteSyncURL(ctx, db, peer); err != nil { + return err + } + if _, pullErr := db.ExecContext(ctx, "SELECT dolt_pull(?, ?)", peer, s.branch); pullErr != nil { + c, conflictErr := versioncontrolops.GetConflicts(ctx, db) + if conflictErr == nil && len(c) > 0 { + conflicts = c + return nil + } + return fmt.Errorf("pull from %s: %w", peer, pullErr) + } + return nil + }) + return conflicts, err +} + +// --------------------------------------------------------------------------- +// Backup operations +// --------------------------------------------------------------------------- + +var errDoltliteBackupUnsupported = errors.New("doltlite backup operations unsupported") + +func (s *DoltliteStore) BackupAdd(ctx context.Context, name, url string) error { + return errDoltliteBackupUnsupported +} + +func (s *DoltliteStore) BackupSync(ctx context.Context, name string) error { + return errDoltliteBackupUnsupported +} + +func (s *DoltliteStore) BackupRemove(ctx context.Context, name string) error { + return errDoltliteBackupUnsupported +} + +func (s *DoltliteStore) BackupDatabase(ctx context.Context, dir string) error { + return errDoltliteBackupUnsupported +} + +func (s *DoltliteStore) RestoreDatabase(ctx context.Context, dir string, force bool) error { + return errDoltliteBackupUnsupported +} diff --git a/internal/storage/embeddeddolt/issues.go b/internal/storage/embeddeddolt/issues.go index b8fb930cd..3e3e238ff 100644 --- a/internal/storage/embeddeddolt/issues.go +++ b/internal/storage/embeddeddolt/issues.go @@ -7,6 +7,7 @@ import ( "database/sql" "encoding/json" "fmt" + "time" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/issueops" @@ -53,6 +54,27 @@ func (s *EmbeddedDoltStore) UpdateIssue(ctx context.Context, id string, updates }) } +// HeartbeatIssue refreshes the lease on an issue actor holds in_progress. +// Delegates SQL work to issueops; EmbeddedDolt auto-commits the transaction. +func (s *EmbeddedDoltStore) HeartbeatIssue(ctx context.Context, id, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.HeartbeatIssueInTx(ctx, tx, id, actor) + }) +} + +// ReclaimExpiredLeases reverts in_progress issues whose lease expired more than +// olderThan ago back to ready, recovering work stranded by dead workers. +func (s *EmbeddedDoltStore) ReclaimExpiredLeases(ctx context.Context, olderThan time.Duration, actor string) ([]types.ReclaimedLease, error) { + cutoff := time.Now().UTC().Add(-olderThan) + var reclaimed []types.ReclaimedLease + err := s.withConn(ctx, true, func(tx *sql.Tx) error { + var err error + reclaimed, err = issueops.ReclaimExpiredLeasesInTx(ctx, tx, cutoff, actor) + return err + }) + return reclaimed, err +} + // ReopenIssue reopens a closed issue, setting status to open and clearing // closed_at and defer_until. If reason is non-empty, it is recorded as a comment. // Wraps UpdateIssue; EmbeddedDolt auto-commits the transaction. diff --git a/internal/storage/embeddeddolt/lease_test.go b/internal/storage/embeddeddolt/lease_test.go new file mode 100644 index 000000000..2db4771a2 --- /dev/null +++ b/internal/storage/embeddeddolt/lease_test.go @@ -0,0 +1,77 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "errors" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// TestLeaseLifecycleEmbedded confirms the lease columns exist in the embedded +// backend and that claim → heartbeat → reclaim are wired through EmbeddedDoltStore. +func TestLeaseLifecycleEmbedded(t *testing.T) { + skipUnlessEmbeddedDolt(t) + + te := newTestEnv(t, "lease") + ctx := t.Context() + + issue := &types.Issue{ + ID: "lease-1", + Title: "lease lifecycle", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + } + if err := te.store.CreateIssue(ctx, issue, "seeder"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + + // Claim with a 1s lease so it expires within the test. + claimCtx := issueops.WithLeaseTTL(ctx, time.Second) + if err := te.store.ClaimIssue(claimCtx, "lease-1", "alice"); err != nil { + t.Fatalf("ClaimIssue: %v", err) + } + + // The claim stamped a non-zero row_lock and a lease. + var rowLock int64 + te.queryScalar(t, ctx, "SELECT row_lock FROM issues WHERE id = ?", []any{"lease-1"}, &rowLock) + if rowLock == 0 { + t.Error("row_lock = 0 after claim, want non-zero") + } + + // Owner can heartbeat; a stranger cannot. Heartbeat with the same short TTL + // so the lease still expires within the test (a default-TTL heartbeat would + // push expiry minutes out). + if err := te.store.HeartbeatIssue(claimCtx, "lease-1", "alice"); err != nil { + t.Fatalf("owner HeartbeatIssue: %v", err) + } + if err := te.store.HeartbeatIssue(ctx, "lease-1", "mallory"); !errors.Is(err, storage.ErrAlreadyClaimed) { + t.Errorf("stranger heartbeat err = %v, want ErrAlreadyClaimed", err) + } + + // Let the lease expire, then reclaim it. + time.Sleep(2500 * time.Millisecond) + reclaimed, err := te.store.ReclaimExpiredLeases(ctx, 0, "reaper") + if err != nil { + t.Fatalf("ReclaimExpiredLeases: %v", err) + } + if len(reclaimed) != 1 || reclaimed[0].ID != "lease-1" || reclaimed[0].PreviousOwner != "alice" { + t.Fatalf("reclaimed = %+v, want [{lease-1 alice}]", reclaimed) + } + + got, err := te.store.GetIssue(ctx, "lease-1") + if err != nil { + t.Fatalf("GetIssue: %v", err) + } + if got.Status != types.StatusOpen { + t.Errorf("status = %q after reclaim, want open", got.Status) + } + if got.Assignee != "" { + t.Errorf("assignee = %q after reclaim, want empty", got.Assignee) + } +} diff --git a/internal/storage/issueops/blocked_consistency.go b/internal/storage/issueops/blocked_consistency.go index 1573642d8..4c001f07a 100644 --- a/internal/storage/issueops/blocked_consistency.go +++ b/internal/storage/issueops/blocked_consistency.go @@ -103,12 +103,12 @@ func recomputeIsBlockedCounting(ctx context.Context, tx DBTX, issueIDs, wispIDs var total int64 for { var changed int64 - n, err := recomputeIsBlockedPassForIssuesInTx(ctx, tx, issueIDs) + n, err := recomputeIsBlockedPassForIssuesInTx(ctx, tx, issueIDs, false) if err != nil { return total, err } changed += n - n, err = recomputeIsBlockedPassForWispsInTx(ctx, tx, wispIDs) + n, err = recomputeIsBlockedPassForWispsInTx(ctx, tx, wispIDs, false) if err != nil { return total, err } diff --git a/internal/storage/issueops/blocked_state.go b/internal/storage/issueops/blocked_state.go index 79a37065a..c22c4f9d8 100644 --- a/internal/storage/issueops/blocked_state.go +++ b/internal/storage/issueops/blocked_state.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "github.com/steveyegge/beads/internal/types" ) @@ -58,19 +59,27 @@ const waitsForGateBlockedSQL = ` ` func RecomputeIsBlockedInTx(ctx context.Context, tx DBTX, issueIDs, wispIDs []string) error { + return recomputeIsBlockedInTx(ctx, tx, issueIDs, wispIDs, false) +} + +func RecomputeIsBlockedSQLiteInTx(ctx context.Context, tx DBTX, issueIDs, wispIDs []string) error { + return recomputeIsBlockedInTx(ctx, tx, issueIDs, wispIDs, true) +} + +func recomputeIsBlockedInTx(ctx context.Context, tx DBTX, issueIDs, wispIDs []string, sqlite bool) error { if len(issueIDs) == 0 && len(wispIDs) == 0 { return nil } for { var changed int64 - n, err := recomputeIsBlockedPassForIssuesInTx(ctx, tx, issueIDs) + n, err := recomputeIsBlockedPassForIssuesInTx(ctx, tx, issueIDs, sqlite) if err != nil { return err } changed += n - n, err = recomputeIsBlockedPassForWispsInTx(ctx, tx, wispIDs) + n, err = recomputeIsBlockedPassForWispsInTx(ctx, tx, wispIDs, sqlite) if err != nil { return err } @@ -83,19 +92,27 @@ func RecomputeIsBlockedInTx(ctx context.Context, tx DBTX, issueIDs, wispIDs []st } func MarkIsBlockedInTx(ctx context.Context, tx DBTX, issueIDs, wispIDs []string) error { + return markIsBlockedInTx(ctx, tx, issueIDs, wispIDs, false) +} + +func MarkIsBlockedSQLiteInTx(ctx context.Context, tx DBTX, issueIDs, wispIDs []string) error { + return markIsBlockedInTx(ctx, tx, issueIDs, wispIDs, true) +} + +func markIsBlockedInTx(ctx context.Context, tx DBTX, issueIDs, wispIDs []string, sqlite bool) error { if len(issueIDs) == 0 && len(wispIDs) == 0 { return nil } for { var changed int64 - n, err := markIsBlockedPassForIssuesInTx(ctx, tx, issueIDs) + n, err := markIsBlockedPassForIssuesInTx(ctx, tx, issueIDs, sqlite) if err != nil { return err } changed += n - n, err = markIsBlockedPassForWispsInTx(ctx, tx, wispIDs) + n, err = markIsBlockedPassForWispsInTx(ctx, tx, wispIDs, sqlite) if err != nil { return err } @@ -116,19 +133,29 @@ func RecomputeIsBlockedForWispIDsInTx(ctx context.Context, tx DBTX, ids []string } //nolint:gosec // G201: SQL templates are constant; only IN-clause placeholders are formatted in. -func recomputeIsBlockedPassForIssuesInTx(ctx context.Context, tx DBTX, ids []string) (int64, error) { +func recomputeIsBlockedPassForIssuesInTx(ctx context.Context, tx DBTX, ids []string, sqlite bool) (int64, error) { if len(ids) == 0 { return 0, nil } - return runMarkUnmarkBatchedInTx(ctx, tx, markBlockedTemplateForIssues(), unmarkBlockedTemplateForIssues(), ids) + markTmpl := markBlockedTemplateForIssues() + unmarkTmpl := unmarkBlockedTemplateForIssues() + if sqlite { + markTmpl = sqliteBlockedTemplate(markTmpl) + unmarkTmpl = sqliteBlockedTemplate(unmarkTmpl) + } + return runMarkUnmarkBatchedInTx(ctx, tx, markTmpl, unmarkTmpl, ids) } -func markIsBlockedPassForIssuesInTx(ctx context.Context, tx DBTX, ids []string) (int64, error) { +func markIsBlockedPassForIssuesInTx(ctx context.Context, tx DBTX, ids []string, sqlite bool) (int64, error) { if len(ids) == 0 { return 0, nil } - return runMarkBatchedInTx(ctx, tx, markBlockedTemplateForIssues(), ids) + markTmpl := markBlockedTemplateForIssues() + if sqlite { + markTmpl = sqliteBlockedTemplate(markTmpl) + } + return runMarkBatchedInTx(ctx, tx, markTmpl, ids) } // The mark/unmark templates explicitly assign updated_at to itself: @@ -229,19 +256,29 @@ func unmarkBlockedTemplateForIssues() string { } //nolint:gosec // G201: SQL templates are constant; only IN-clause placeholders are formatted in. -func recomputeIsBlockedPassForWispsInTx(ctx context.Context, tx DBTX, ids []string) (int64, error) { +func recomputeIsBlockedPassForWispsInTx(ctx context.Context, tx DBTX, ids []string, sqlite bool) (int64, error) { if len(ids) == 0 { return 0, nil } - return runMarkUnmarkBatchedInTx(ctx, tx, markBlockedTemplateForWisps(), unmarkBlockedTemplateForWisps(), ids) + markTmpl := markBlockedTemplateForWisps() + unmarkTmpl := unmarkBlockedTemplateForWisps() + if sqlite { + markTmpl = sqliteBlockedTemplate(markTmpl) + unmarkTmpl = sqliteBlockedTemplate(unmarkTmpl) + } + return runMarkUnmarkBatchedInTx(ctx, tx, markTmpl, unmarkTmpl, ids) } -func markIsBlockedPassForWispsInTx(ctx context.Context, tx DBTX, ids []string) (int64, error) { +func markIsBlockedPassForWispsInTx(ctx context.Context, tx DBTX, ids []string, sqlite bool) (int64, error) { if len(ids) == 0 { return 0, nil } - return runMarkBatchedInTx(ctx, tx, markBlockedTemplateForWisps(), ids) + markTmpl := markBlockedTemplateForWisps() + if sqlite { + markTmpl = sqliteBlockedTemplate(markTmpl) + } + return runMarkBatchedInTx(ctx, tx, markTmpl, ids) } func markBlockedTemplateForWisps() string { @@ -334,6 +371,22 @@ func unmarkBlockedTemplateForWisps() string { `, waitsForGateBlockedSQL) } +func sqliteBlockedTemplate(tmpl string) string { + replacer := strings.NewReplacer( + "UPDATE issues i SET i.is_blocked = 1, i.updated_at = i.updated_at", + "UPDATE issues AS i SET is_blocked = 1, updated_at = updated_at", + "UPDATE issues i SET i.is_blocked = 0, i.updated_at = i.updated_at", + "UPDATE issues AS i SET is_blocked = 0, updated_at = updated_at", + "UPDATE wisps w SET w.is_blocked = 1, w.updated_at = w.updated_at", + "UPDATE wisps AS w SET is_blocked = 1, updated_at = updated_at", + "UPDATE wisps w SET w.is_blocked = 0, w.updated_at = w.updated_at", + "UPDATE wisps AS w SET is_blocked = 0, updated_at = updated_at", + "JSON_UNQUOTE(JSON_EXTRACT(d.metadata, '$.gate'))", + "JSON_EXTRACT(d.metadata, '$.gate')", + ) + return replacer.Replace(tmpl) +} + //nolint:gosec // G201: callers pass constant templates; only IN-clause placeholders are formatted in. func runMarkUnmarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl, unmarkTmpl string, ids []string) (int64, error) { var changed int64 diff --git a/internal/storage/issueops/claim.go b/internal/storage/issueops/claim.go index f5e87f5c9..89363d481 100644 --- a/internal/storage/issueops/claim.go +++ b/internal/storage/issueops/claim.go @@ -9,6 +9,7 @@ import ( "time" "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/sqlbuild" "github.com/steveyegge/beads/internal/types" ) @@ -40,6 +41,13 @@ func ClaimIssueInTx(ctx context.Context, tx DBTX, id string, actor string) (*Cla now := time.Now().UTC() + // Stamp a lease on the claim: lease_expires_at = now + TTL, heartbeat_at = + // now, and a fresh row_lock (see lease.go). The lease is what makes a claim + // recoverable — a worker that dies stops heartbeating and bd reclaim later + // reverts the issue. row_lock here also forces a concurrent reclaim/heartbeat + // to conflict rather than silently cell-merge. + leaseClause, leaseArgs := leaseSetClause(now, leaseTTL(ctx)) + // Conditional UPDATE: only succeeds while the issue is still claimable. // Also set started_at on first transition to in_progress (GH#2796); preserve // any existing value so re-claims don't overwrite the original start time. @@ -47,17 +55,21 @@ func ClaimIssueInTx(ctx context.Context, tx DBTX, id string, actor string) (*Cla result sql.Result ) if oldIssue.StartedAt == nil { + args := append([]interface{}{actor, now, now}, leaseArgs...) + args = append(args, id, actor) result, err = tx.ExecContext(ctx, fmt.Sprintf(` UPDATE %s - SET assignee = ?, status = 'in_progress', updated_at = ?, started_at = ? + SET assignee = ?, status = 'in_progress', updated_at = ?, started_at = ?, %s WHERE id = ? AND status = 'open' AND (assignee = '' OR assignee IS NULL OR assignee = ?) - `, issueTable), actor, now, now, id, actor) + `, issueTable, leaseClause), args...) } else { + args := append([]interface{}{actor, now}, leaseArgs...) + args = append(args, id, actor) result, err = tx.ExecContext(ctx, fmt.Sprintf(` UPDATE %s - SET assignee = ?, status = 'in_progress', updated_at = ? + SET assignee = ?, status = 'in_progress', updated_at = ?, %s WHERE id = ? AND status = 'open' AND (assignee = '' OR assignee IS NULL OR assignee = ?) - `, issueTable), actor, now, id, actor) + `, issueTable, leaseClause), args...) } if err != nil { return nil, fmt.Errorf("failed to claim issue: %w", err) @@ -116,6 +128,27 @@ func ClaimReadyIssueInTx( tx DBTX, filter types.WorkFilter, actor string, +) (*types.Issue, error) { + return claimReadyIssueInTx(ctx, tx, filter, actor, sqlbuild.CountsDialectDolt) +} + +// ClaimReadyIssueSQLiteInTx claims the first ready issue using +// SQLite-compatible readiness SQL for embedded DoltLite stores. +func ClaimReadyIssueSQLiteInTx( + ctx context.Context, + tx DBTX, + filter types.WorkFilter, + actor string, +) (*types.Issue, error) { + return claimReadyIssueInTx(ctx, tx, filter, actor, sqlbuild.CountsDialectSQLite) +} + +func claimReadyIssueInTx( + ctx context.Context, + tx DBTX, + filter types.WorkFilter, + actor string, + dialect sqlbuild.CountsDialect, ) (*types.Issue, error) { claimFilter := filter claimFilter.Status = types.StatusOpen @@ -123,7 +156,7 @@ func ClaimReadyIssueInTx( claimFilter.Assignee = nil claimFilter.Limit = 0 - readyIssues, err := GetReadyWorkInTx(ctx, tx, claimFilter) + readyIssues, err := getReadyWorkInTx(ctx, tx, claimFilter, dialect) if err != nil { return nil, err } diff --git a/internal/storage/issueops/close.go b/internal/storage/issueops/close.go index 798fdcc43..de16d1d86 100644 --- a/internal/storage/issueops/close.go +++ b/internal/storage/issueops/close.go @@ -19,15 +19,21 @@ type CloseResult struct { // and recording the close event. Routes to the correct table (issues/wisps) // automatically. The caller is responsible for Dolt versioning if needed. func CloseIssueInTx(ctx context.Context, tx DBTX, id string, reason, actor, session string) (*CloseResult, error) { - return closeIssueInTx(ctx, tx, id, reason, actor, session, true) + return closeIssueInTx(ctx, tx, id, reason, actor, session, true, false) +} + +// CloseIssueSQLiteInTx closes an issue using SQLite-compatible derived-state +// recompute SQL for embedded DoltLite stores. +func CloseIssueSQLiteInTx(ctx context.Context, tx DBTX, id string, reason, actor, session string) (*CloseResult, error) { + return closeIssueInTx(ctx, tx, id, reason, actor, session, true, true) } func CloseIssueWithoutEventInTx(ctx context.Context, tx DBTX, id string, reason, actor, session string) (*CloseResult, error) { - return closeIssueInTx(ctx, tx, id, reason, actor, session, false) + return closeIssueInTx(ctx, tx, id, reason, actor, session, false, false) } //nolint:gosec // G201: table names come from WispTableRouting (hardcoded constants) -func closeIssueInTx(ctx context.Context, tx DBTX, id string, reason, actor, session string, recordEvent bool) (*CloseResult, error) { +func closeIssueInTx(ctx context.Context, tx DBTX, id string, reason, actor, session string, recordEvent bool, sqlite bool) (*CloseResult, error) { isWisp := IsActiveWispInTx(ctx, tx, id) issueTable, _, eventTable, _ := WispTableRouting(isWisp) @@ -44,10 +50,16 @@ func closeIssueInTx(ctx context.Context, tx DBTX, id string, reason, actor, sess now := time.Now().UTC() + // row_lock is rewritten on close so a concurrent reclaim (which also rewrites + // row_lock) collides on this cell and is forced to conflict-and-retry rather + // than silently cell-merging a revert-to-ready over a completed close (see + // lease.go). lease_expires_at/heartbeat_at are cleared: a closed issue holds + // no lease. result, err := tx.ExecContext(ctx, fmt.Sprintf(` - UPDATE %s SET status = ?, closed_at = ?, updated_at = ?, close_reason = ?, closed_by_session = ? + UPDATE %s SET status = ?, closed_at = ?, updated_at = ?, close_reason = ?, closed_by_session = ?, + lease_expires_at = NULL, heartbeat_at = NULL, row_lock = ? WHERE id = ? AND status != ? - `, issueTable), types.StatusClosed, now, now, reason, session, id, types.StatusClosed) + `, issueTable), types.StatusClosed, now, now, reason, session, freshRowLock(), id, types.StatusClosed) if err != nil { return nil, fmt.Errorf("failed to close issue: %w", err) } @@ -79,8 +91,14 @@ func closeIssueInTx(ctx context.Context, tx DBTX, id string, reason, actor, sess } } - if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { - return nil, fmt.Errorf("recompute is_blocked after close for %s: %w", id, err) + var recomputeErr error + if sqlite { + recomputeErr = RecomputeIsBlockedSQLiteInTx(ctx, tx, affectedIssues, affectedWisps) + } else { + recomputeErr = RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps) + } + if recomputeErr != nil { + return nil, fmt.Errorf("recompute is_blocked after close for %s: %w", id, recomputeErr) } return &CloseResult{IsWisp: isWisp}, nil diff --git a/internal/storage/issueops/commit_pending.go b/internal/storage/issueops/commit_pending.go index 373fc350b..189bb2b24 100644 --- a/internal/storage/issueops/commit_pending.go +++ b/internal/storage/issueops/commit_pending.go @@ -13,21 +13,50 @@ type SQLQuerier interface { QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) } -// HasPendingChanges checks whether there are any committable changes in the -// Dolt working set, excluding tables matched by dolt_ignore. -func HasPendingChanges(ctx context.Context, db SQLQuerier) (bool, error) { - var count int - err := db.QueryRowContext(ctx, ` - SELECT COUNT(*) FROM dolt_status s +// PendingTables returns committable dirty tables from dolt_status, excluding +// tables matched by dolt_ignore. When includeConfig is false, config changes +// are also skipped to match DoltStore.Commit's normal auto-commit policy. +func PendingTables(ctx context.Context, db SQLQuerier, includeConfig bool) ([]string, error) { + query := ` + SELECT table_name FROM dolt_status s WHERE NOT EXISTS ( SELECT 1 FROM dolt_ignore di WHERE di.ignored = 1 AND s.table_name LIKE di.pattern - )`).Scan(&count) + )` + if !includeConfig { + query += "\n\t\tAND s.table_name != 'config'" + } + query += "\n\t\tORDER BY table_name" + + rows, err := db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to query status: %w", err) + } + defer rows.Close() + + var tables []string + for rows.Next() { + var table string + if err := rows.Scan(&table); err != nil { + return nil, fmt.Errorf("failed to scan status: %w", err) + } + tables = append(tables, table) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to iterate status: %w", err) + } + return tables, nil +} + +// HasPendingChanges checks whether there are any committable changes in the +// Dolt working set, excluding tables matched by dolt_ignore. +func HasPendingChanges(ctx context.Context, db SQLQuerier) (bool, error) { + tables, err := PendingTables(ctx, db, true) if err != nil { - return false, fmt.Errorf("failed to check status: %w", err) + return false, err } - return count > 0, nil + return len(tables) > 0, nil } // BuildBatchCommitMessage generates a descriptive commit message summarizing diff --git a/internal/storage/issueops/count.go b/internal/storage/issueops/count.go index 5c2ebb7b4..f2489f333 100644 --- a/internal/storage/issueops/count.go +++ b/internal/storage/issueops/count.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" + "github.com/steveyegge/beads/internal/storage/sqlbuild" "github.com/steveyegge/beads/internal/types" ) @@ -15,8 +16,18 @@ import ( // SkipWisps=true counts the durable issues table only, and otherwise the // wisps count is merged in (GH#4387). func CountIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter) (int, error) { + return countIssuesInTx(ctx, tx, query, filter, sqlbuild.CountsDialectDolt) +} + +// CountIssuesSQLiteInTx counts issues using SQLite-compatible SQL fragments for +// embedded DoltLite stores. +func CountIssuesSQLiteInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter) (int, error) { + return countIssuesInTx(ctx, tx, query, filter, sqlbuild.CountsDialectSQLite) +} + +func countIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter, dialect sqlbuild.CountsDialect) (int, error) { if filter.Ephemeral != nil && *filter.Ephemeral { - wispCount, err := countTableInTx(ctx, tx, query, filter, WispsFilterTables) + wispCount, err := countTableInTxDialect(ctx, tx, query, filter, WispsFilterTables, dialect) if err != nil && !isTableNotExistError(err) { return 0, fmt.Errorf("count wisps (ephemeral filter): %w", err) } @@ -32,14 +43,14 @@ func CountIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter types // normal creation never produces (ephemeral/infra beads route to the // wisps table on insert), but which would otherwise be reported as 0 by // count while list returns it. - count, err := countTableInTx(ctx, tx, query, filter, IssuesFilterTables) + count, err := countTableInTxDialect(ctx, tx, query, filter, IssuesFilterTables, dialect) if err != nil { return 0, fmt.Errorf("count issues (ephemeral fall-through): %w", err) } return count, nil } - count, err := countTableInTx(ctx, tx, query, filter, IssuesFilterTables) + count, err := countTableInTxDialect(ctx, tx, query, filter, IssuesFilterTables, dialect) if err != nil { return 0, fmt.Errorf("count issues: %w", err) } @@ -53,7 +64,7 @@ func CountIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter types // wisps row), so the two counts don't double-count. count trusts that disjoint-table // invariant; SearchIssuesInTx is the corruption detector — it errors loudly if an ID // appears in both tables ("id %q exists in both issues and wisps"). - wispCount, wispErr := countTableInTx(ctx, tx, query, filter, WispsFilterTables) + wispCount, wispErr := countTableInTxDialect(ctx, tx, query, filter, WispsFilterTables, dialect) if wispErr != nil && !isTableNotExistError(wispErr) { return 0, fmt.Errorf("count wisps (merge): %w", wispErr) } @@ -68,8 +79,18 @@ func CountIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter types // route to the wisps table, SkipWisps=true counts the durable issues table // only, and otherwise the wisps tier is merged into each group (GH#4387). func CountIssuesByGroupInTx(ctx context.Context, tx *sql.Tx, filter types.IssueFilter, groupBy string) (map[string]int, error) { + return countIssuesByGroupInTx(ctx, tx, filter, groupBy, sqlbuild.CountsDialectDolt) +} + +// CountIssuesByGroupSQLiteInTx counts grouped issues using SQLite-compatible +// SQL fragments for embedded DoltLite stores. +func CountIssuesByGroupSQLiteInTx(ctx context.Context, tx *sql.Tx, filter types.IssueFilter, groupBy string) (map[string]int, error) { + return countIssuesByGroupInTx(ctx, tx, filter, groupBy, sqlbuild.CountsDialectSQLite) +} + +func countIssuesByGroupInTx(ctx context.Context, tx *sql.Tx, filter types.IssueFilter, groupBy string, dialect sqlbuild.CountsDialect) (map[string]int, error) { if filter.Ephemeral != nil && *filter.Ephemeral { - wispCounts, err := countGroupForTablesInTx(ctx, tx, filter, groupBy, WispsFilterTables) + wispCounts, err := countGroupForTablesInTxDialect(ctx, tx, filter, groupBy, WispsFilterTables, dialect) if err != nil && !isTableNotExistError(err) { return nil, fmt.Errorf("count wisps by %s (ephemeral filter): %w", groupBy, err) } @@ -87,14 +108,14 @@ func CountIssuesByGroupInTx(ctx context.Context, tx *sql.Tx, filter types.IssueF // disagree with the sum of the grouped buckets (wisps-only), breaking // the GH#4387 count/list cardinality parity for `bd count // --include-infra --by-*`. - counts, err := countGroupForTablesInTx(ctx, tx, filter, groupBy, IssuesFilterTables) + counts, err := countGroupForTablesInTxDialect(ctx, tx, filter, groupBy, IssuesFilterTables, dialect) if err != nil { return nil, fmt.Errorf("count issues by %s (ephemeral fall-through): %w", groupBy, err) } return counts, nil } - counts, err := countGroupForTablesInTx(ctx, tx, filter, groupBy, IssuesFilterTables) + counts, err := countGroupForTablesInTxDialect(ctx, tx, filter, groupBy, IssuesFilterTables, dialect) if err != nil { return nil, err } @@ -105,7 +126,7 @@ func CountIssuesByGroupInTx(ctx context.Context, tx *sql.Tx, filter types.IssueF // Merge wisps counts when the caller hasn't opted out (same semantics as // CountIssuesInTx / SearchIssuesInTx; the two tables never share an ID). - wispCounts, wispErr := countGroupForTablesInTx(ctx, tx, filter, groupBy, WispsFilterTables) + wispCounts, wispErr := countGroupForTablesInTxDialect(ctx, tx, filter, groupBy, WispsFilterTables, dialect) if wispErr != nil && !isTableNotExistError(wispErr) { return nil, fmt.Errorf("count wisps by %s (merge): %w", groupBy, wispErr) } @@ -118,8 +139,12 @@ func CountIssuesByGroupInTx(ctx context.Context, tx *sql.Tx, filter types.IssueF // countGroupForTablesInTx runs a grouped count against one table set // (issues or wisps) and normalizes keys to bd count's display format. func countGroupForTablesInTx(ctx context.Context, tx *sql.Tx, filter types.IssueFilter, groupBy string, tables FilterTables) (map[string]int, error) { + return countGroupForTablesInTxDialect(ctx, tx, filter, groupBy, tables, sqlbuild.CountsDialectDolt) +} + +func countGroupForTablesInTxDialect(ctx context.Context, tx *sql.Tx, filter types.IssueFilter, groupBy string, tables FilterTables, dialect sqlbuild.CountsDialect) (map[string]int, error) { if groupBy == "label" { - return countByLabelInTx(ctx, tx, filter, tables) + return countByLabelInTxDialect(ctx, tx, filter, tables, dialect) } // Map user-facing groupBy name to SQL column name. @@ -134,7 +159,7 @@ func countGroupForTablesInTx(ctx context.Context, tx *sql.Tx, filter types.Issue return nil, fmt.Errorf("unsupported groupBy: %s", groupBy) } - rawCounts, err := countByColumnInTx(ctx, tx, filter, col, tables) + rawCounts, err := countByColumnInTxDialect(ctx, tx, filter, col, tables, dialect) if err != nil { return nil, err } @@ -157,7 +182,11 @@ func countGroupForTablesInTx(ctx context.Context, tx *sql.Tx, filter types.Issue // countTableInTx runs SELECT COUNT(*) FROM WHERE . func countTableInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter, tables FilterTables) (int, error) { - clauses, args, err := BuildIssueFilterClauses(query, filter, tables) + return countTableInTxDialect(ctx, tx, query, filter, tables, sqlbuild.CountsDialectDolt) +} + +func countTableInTxDialect(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter, tables FilterTables, dialect sqlbuild.CountsDialect) (int, error) { + clauses, args, err := BuildIssueFilterClausesDialect(query, filter, tables, dialect) if err != nil { return 0, err } @@ -177,7 +206,11 @@ func countTableInTx(ctx context.Context, tx *sql.Tx, query string, filter types. // countByColumnInTx runs SELECT , COUNT(*) GROUP BY against a table. // Returns raw column values as keys (callers normalize for display). func countByColumnInTx(ctx context.Context, tx *sql.Tx, filter types.IssueFilter, col string, tables FilterTables) (map[string]int, error) { - clauses, args, err := BuildIssueFilterClauses("", filter, tables) + return countByColumnInTxDialect(ctx, tx, filter, col, tables, sqlbuild.CountsDialectDolt) +} + +func countByColumnInTxDialect(ctx context.Context, tx *sql.Tx, filter types.IssueFilter, col string, tables FilterTables, dialect sqlbuild.CountsDialect) (map[string]int, error) { + clauses, args, err := BuildIssueFilterClausesDialect("", filter, tables, dialect) if err != nil { return nil, err } @@ -208,7 +241,11 @@ func countByColumnInTx(ctx context.Context, tx *sql.Tx, filter types.IssueFilter // Dolt's joinIter panic (join_iters.go:192). Issues with no labels are counted // under "(no labels)". func countByLabelInTx(ctx context.Context, tx *sql.Tx, filter types.IssueFilter, tables FilterTables) (map[string]int, error) { - clauses, args, err := BuildIssueFilterClauses("", filter, tables) + return countByLabelInTxDialect(ctx, tx, filter, tables, sqlbuild.CountsDialectDolt) +} + +func countByLabelInTxDialect(ctx context.Context, tx *sql.Tx, filter types.IssueFilter, tables FilterTables, dialect sqlbuild.CountsDialect) (map[string]int, error) { + clauses, args, err := BuildIssueFilterClausesDialect("", filter, tables, dialect) if err != nil { return nil, err } diff --git a/internal/storage/issueops/delete.go b/internal/storage/issueops/delete.go index 88f8e3f14..bc6ce4374 100644 --- a/internal/storage/issueops/delete.go +++ b/internal/storage/issueops/delete.go @@ -19,6 +19,16 @@ const maxRecursiveResults = 10000 //nolint:gosec // G201: table names come from WispTableRouting (hardcoded constants) func DeleteIssueInTx(ctx context.Context, tx *sql.Tx, id string) error { + return deleteIssueInTx(ctx, tx, id, false) +} + +// DeleteIssueSQLiteInTx deletes an issue using SQLite-compatible derived-state +// recompute SQL for embedded DoltLite stores. +func DeleteIssueSQLiteInTx(ctx context.Context, tx *sql.Tx, id string) error { + return deleteIssueInTx(ctx, tx, id, true) +} + +func deleteIssueInTx(ctx context.Context, tx *sql.Tx, id string, sqlite bool) error { isWisp := IsActiveWispInTx(ctx, tx, id) var deletedIssues, deletedWisps []string @@ -36,8 +46,14 @@ func DeleteIssueInTx(ctx context.Context, tx *sql.Tx, id string) error { return err } - if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { - return fmt.Errorf("recompute is_blocked after delete for %s: %w", id, err) + var recomputeErr error + if sqlite { + recomputeErr = RecomputeIsBlockedSQLiteInTx(ctx, tx, affectedIssues, affectedWisps) + } else { + recomputeErr = RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps) + } + if recomputeErr != nil { + return fmt.Errorf("recompute is_blocked after delete for %s: %w", id, recomputeErr) } return nil diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index 3d6f5d328..348632033 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -87,7 +87,13 @@ type AddDependencyOpts struct { // SkipCycleCheck skips the recursive pre-insert cycle check for callers // that intentionally trade validation cost for bulk graph wiring speed. SkipCycleCheck bool - TargetKind *DepTargetKind + // SkipBlockedRecompute skips derived is_blocked maintenance for backends + // that recompute the state through a backend-specific SQL path. + SkipBlockedRecompute bool + // UseSQLiteBlockedRecompute uses SQLite-compatible UPDATE/JSON syntax for + // derived is_blocked maintenance. + UseSQLiteBlockedRecompute bool + TargetKind *DepTargetKind } // AddDependencyInTx validates and inserts a dependency within an existing @@ -215,10 +221,13 @@ func AddDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, a //nolint:gosec // G201: writeTable from WispTableRouting; targetCol from DepTargetKind.Column() if _, err := tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO %s (id, issue_id, %s, type, created_at, created_by, metadata, thread_id) - VALUES (?, ?, ?, ?, NOW(), ?, ?, ?) + VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?) `, writeTable, targetCol), depid.New(dep.IssueID, dep.DependsOnID), dep.IssueID, dep.DependsOnID, dep.Type, actor, metadata, dep.ThreadID); err != nil { return fmt.Errorf("failed to add dependency: %w", err) } + if opts.SkipBlockedRecompute { + return nil + } srcIsWisp := writeTable == "wisp_dependencies" var affectedIssues, affectedWisps []string @@ -240,12 +249,22 @@ func AddDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, a if dep.Type == types.DepParentChild { // Parent-child adds are not monotonic: adding an already-closed child can // satisfy an any-children waits-for gate and unblock the waiter. - if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { + if opts.UseSQLiteBlockedRecompute { + err = RecomputeIsBlockedSQLiteInTx(ctx, tx, affectedIssues, affectedWisps) + } else { + err = RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps) + } + if err != nil { return fmt.Errorf("recompute is_blocked after add dependency %s -> %s: %w", dep.IssueID, dep.DependsOnID, err) } return nil } - if err := MarkIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { + if opts.UseSQLiteBlockedRecompute { + err = MarkIsBlockedSQLiteInTx(ctx, tx, affectedIssues, affectedWisps) + } else { + err = MarkIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps) + } + if err != nil { return fmt.Errorf("mark is_blocked after add dependency %s -> %s: %w", dep.IssueID, dep.DependsOnID, err) } return nil @@ -297,7 +316,7 @@ func markDirectBlockingDependencySourceInTx(ctx context.Context, tx *sql.Tx, sou // treat it as an independent rowset, satisfying the restriction. Dolt // accepts both forms, so this is a no-op there. _, err := tx.ExecContext(ctx, fmt.Sprintf(` - UPDATE %s s SET s.is_blocked = 1, s.updated_at = s.updated_at + UPDATE %s AS s SET is_blocked = 1, updated_at = updated_at WHERE s.id = ? AND s.is_blocked = 0 AND s.status <> 'closed' AND s.status <> 'pinned' @@ -703,6 +722,16 @@ func checkRenameTargetCollision(ctx context.Context, tx *sql.Tx, table, typedCol // //nolint:gosec // G201: depTable from WispTableRouting (hardcoded constants) func RemoveDependencyInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID string) error { + return removeDependencyInTx(ctx, tx, issueID, dependsOnID, false) +} + +// RemoveDependencySQLiteInTx removes a dependency using SQLite-compatible +// derived-state recompute SQL for embedded DoltLite stores. +func RemoveDependencySQLiteInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID string) error { + return removeDependencyInTx(ctx, tx, issueID, dependsOnID, true) +} + +func removeDependencyInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID string, sqlite bool) error { isWisp := IsActiveWispInTx(ctx, tx, issueID) _, _, _, depTable := WispTableRouting(isWisp) @@ -735,8 +764,14 @@ func RemoveDependencyInTx(ctx context.Context, tx *sql.Tx, issueID, dependsOnID if aerr != nil { return fmt.Errorf("affected by remove dependency %s -> %s: %w", issueID, dependsOnID, aerr) } - if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { - return fmt.Errorf("recompute is_blocked after remove dependency %s -> %s: %w", issueID, dependsOnID, err) + var recomputeErr error + if sqlite { + recomputeErr = RecomputeIsBlockedSQLiteInTx(ctx, tx, affectedIssues, affectedWisps) + } else { + recomputeErr = RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps) + } + if recomputeErr != nil { + return fmt.Errorf("recompute is_blocked after remove dependency %s -> %s: %w", issueID, dependsOnID, recomputeErr) } return nil } diff --git a/internal/storage/issueops/filters.go b/internal/storage/issueops/filters.go index c3f72e916..4277f982e 100644 --- a/internal/storage/issueops/filters.go +++ b/internal/storage/issueops/filters.go @@ -23,6 +23,12 @@ func BuildIssueFilterClauses(query string, filter types.IssueFilter, tables Filt return sqlbuild.BuildIssueFilterClauses(query, filter, tables) } +// BuildIssueFilterClausesDialect builds WHERE fragments using backend-specific +// SQL for JSON metadata predicates. +func BuildIssueFilterClausesDialect(query string, filter types.IssueFilter, tables FilterTables, dialect sqlbuild.CountsDialect) ([]string, []interface{}, error) { + return sqlbuild.BuildIssueFilterClausesDialect(query, filter, tables, dialect) +} + // LooksLikeIssueID returns true if the query string looks like a beads issue ID. func LooksLikeIssueID(query string) bool { return sqlbuild.LooksLikeIssueID(query) diff --git a/internal/storage/issueops/lease.go b/internal/storage/issueops/lease.go new file mode 100644 index 000000000..61e0ba8bf --- /dev/null +++ b/internal/storage/issueops/lease.go @@ -0,0 +1,207 @@ +package issueops + +import ( + "context" + "crypto/rand" + "encoding/binary" + "fmt" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// DefaultLeaseTTL is how long a fresh claim stays valid without a heartbeat. +// A worker is expected to call HeartbeatIssueInTx well within this window +// (heartbeat cadence ≫ claim cadence; see the commit-bloat note on bd heartbeat) +// so a live claim's lease_expires_at always sits in the future. A worker that +// dies stops heartbeating, its lease_expires_at goes stale, and bd reclaim +// reverts the issue to ready. Tunable per-claim via WithLeaseTTL on the +// context, falling back to this default. +const DefaultLeaseTTL = 5 * time.Minute + +// leaseTTLContextKey overrides DefaultLeaseTTL for a single claim. Used by tests +// (short TTLs) and callers that know their work cadence; unset in normal use. +type leaseTTLContextKey struct{} + +// WithLeaseTTL returns a context whose claims use ttl instead of DefaultLeaseTTL. +func WithLeaseTTL(ctx context.Context, ttl time.Duration) context.Context { + return context.WithValue(ctx, leaseTTLContextKey{}, ttl) +} + +// leaseTTL resolves the lease TTL for the current claim/heartbeat. +func leaseTTL(ctx context.Context) time.Duration { + if ttl, ok := ctx.Value(leaseTTLContextKey{}).(time.Duration); ok && ttl > 0 { + return ttl + } + return DefaultLeaseTTL +} + +// freshRowLock returns a random non-zero int64 for the row_lock cell. +// +// row_lock is the keystone of dead-worker recovery on Dolt. Dolt has no real +// row locking and merges concurrent commits cell-by-cell, so two transactions +// that touch DIFFERENT cells of the same issue row (a heartbeat writing +// heartbeat_at, a close writing status) merge silently instead of conflicting — +// which would let a reclaim quietly revert an issue the owner just closed. By +// having EVERY mutating path rewrite this one shared cell to a fresh random +// value, concurrent writers always collide on row_lock, surfacing the 1213/1205 +// serialization conflict that withRetryTx replays. The value's only job is to +// differ from whatever a concurrent writer wrote, so any source of entropy +// works; we use crypto/rand to avoid seeding concerns. Never 0 (the column +// default) so a freshly-claimed row is always distinguishable from a never- +// touched one. +func freshRowLock() int64 { + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + // crypto/rand failing is catastrophic and ~never happens; fall back to a + // timestamp so the row_lock still changes rather than wedging the write. + return time.Now().UnixNano() | 1 + } + v := int64(binary.LittleEndian.Uint64(b[:])) + if v == 0 { + v = 1 + } + return v +} + +// leaseSetClause returns the SET-clause fragment and args that stamp a fresh +// lease onto a row being claimed or heartbeated: a future expiry, a now +// heartbeat, and a fresh row_lock. Append to an existing UPDATE's SET list. +func leaseSetClause(now time.Time, ttl time.Duration) (string, []interface{}) { + return "lease_expires_at = ?, heartbeat_at = ?, row_lock = ?", + []interface{}{now.Add(ttl), now, freshRowLock()} +} + +// HeartbeatIssueInTx proves the lease owner is still alive: it pushes +// lease_expires_at forward by the TTL, stamps heartbeat_at = now, and rewrites +// row_lock so the heartbeat conflicts with any concurrent reclaim/close on the +// same row (see freshRowLock). Only the current owner of an in_progress issue +// may heartbeat — a heartbeat from anyone else, or on an issue that is no longer +// in_progress (already closed or already reclaimed), affects no rows and returns +// storage.ErrNotClaimable so the caller learns its lease is gone. +// +// Routes to the correct table (issues/wisps). The caller owns Dolt versioning. +// +//nolint:gosec // G201: table names come from WispTableRouting (hardcoded constants) +func HeartbeatIssueInTx(ctx context.Context, tx DBTX, id, actor string) error { + isWisp := IsActiveWispInTx(ctx, tx, id) + issueTable, _, _, _ := WispTableRouting(isWisp) + + now := time.Now().UTC() + leaseClause, leaseArgs := leaseSetClause(now, leaseTTL(ctx)) + + args := append([]interface{}{}, leaseArgs...) + args = append(args, id, actor) + result, err := tx.ExecContext(ctx, fmt.Sprintf(` + UPDATE %s SET %s + WHERE id = ? AND status = 'in_progress' AND assignee = ? + `, issueTable, leaseClause), args...) + if err != nil { + return fmt.Errorf("failed to heartbeat issue: %w", err) + } + rows, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("failed to get rows affected: %w", err) + } + if rows == 0 { + // Disambiguate for the caller: gone (closed/reopened/reclaimed), + // not-found, or owned by someone else. + var assignee, status string + qerr := tx.QueryRowContext(ctx, + fmt.Sprintf("SELECT COALESCE(assignee, ''), status FROM %s WHERE id = ?", issueTable), id, + ).Scan(&assignee, &status) + if qerr != nil { + return fmt.Errorf("%w: %s", storage.ErrNotClaimable, id) + } + if assignee != "" && assignee != actor { + return fmt.Errorf("%w by %s", storage.ErrAlreadyClaimed, assignee) + } + return fmt.Errorf("%w: %s status %s", storage.ErrNotClaimable, id, status) + } + return nil +} + +// ReclaimExpiredLeasesInTx reverts in_progress issues whose lease has gone stale +// back to ready: status → open, assignee cleared, started_at cleared, and a +// fresh row_lock so the reclaim conflicts with a racing heartbeat/close on the +// same row (see freshRowLock). An issue is stale when its lease_expires_at is +// non-null and strictly before cutoff. Callers pass cutoff = now - graceWindow +// (the supervisor uses graceWindow = 2×TTL) so only leases that expired a safe +// margin ago — i.e. workers that are almost certainly dead — are reclaimed. +// +// Reclaim only ever touches the permanent issues table: wisps are ephemeral and +// are never leased work. Returns the issues it reverted (id + the owner it took +// the lease from) so the caller can log/emit recovery events. The caller owns +// Dolt versioning. +func ReclaimExpiredLeasesInTx(ctx context.Context, tx DBTX, cutoff time.Time, actor string) ([]types.ReclaimedLease, error) { + // Snapshot the stale set first so we can report exactly which issues we + // reverted and record per-issue recovery events. The UPDATE below repeats + // the predicate, so an issue that a concurrent heartbeat rescued between the + // SELECT and the UPDATE is simply skipped (0 rows) — it never appears as + // reclaimed. + rows, err := tx.QueryContext(ctx, ` + SELECT id, COALESCE(assignee, '') FROM issues + WHERE status = 'in_progress' + AND lease_expires_at IS NOT NULL + AND lease_expires_at < ? + `, cutoff) + if err != nil { + return nil, fmt.Errorf("scan for stale leases: %w", err) + } + var stale []types.ReclaimedLease + for rows.Next() { + var r types.ReclaimedLease + if err := rows.Scan(&r.ID, &r.PreviousOwner); err != nil { + if closeErr := rows.Close(); closeErr != nil { + return nil, fmt.Errorf("scan stale lease row: %w; close stale lease rows: %v", err, closeErr) + } + return nil, fmt.Errorf("scan stale lease row: %w", err) + } + stale = append(stale, r) + } + if err := rows.Err(); err != nil { + if closeErr := rows.Close(); closeErr != nil { + return nil, fmt.Errorf("iterate stale leases: %w; close stale lease rows: %v", err, closeErr) + } + return nil, fmt.Errorf("iterate stale leases: %w", err) + } + if err := rows.Close(); err != nil { + return nil, fmt.Errorf("close stale lease rows: %w", err) + } + if len(stale) == 0 { + return nil, nil + } + + var reclaimed []types.ReclaimedLease + for _, r := range stale { + // Re-check the predicate inside the UPDATE so a heartbeat that landed + // after the snapshot (pushing lease_expires_at back into the future, or + // the row already closed) cannot be clobbered. row_lock makes the racing + // writer conflict; this WHERE makes a winning racer's rescue stick. + res, err := tx.ExecContext(ctx, ` + UPDATE issues + SET status = 'open', assignee = NULL, started_at = NULL, + lease_expires_at = NULL, heartbeat_at = NULL, + updated_at = ?, row_lock = ? + WHERE id = ? AND status = 'in_progress' + AND lease_expires_at IS NOT NULL AND lease_expires_at < ? + `, time.Now().UTC(), freshRowLock(), r.ID, cutoff) + if err != nil { + return nil, fmt.Errorf("reclaim %s: %w", r.ID, err) + } + n, err := res.RowsAffected() + if err != nil { + return nil, fmt.Errorf("reclaim %s rows affected: %w", r.ID, err) + } + if n == 0 { + continue // rescued by a concurrent heartbeat/close — leave it be + } + if err := RecordFullEventInTable(ctx, tx, "events", r.ID, types.EventLeaseReclaimed, actor, + r.PreviousOwner, ""); err != nil { + return nil, fmt.Errorf("record reclaim event for %s: %w", r.ID, err) + } + reclaimed = append(reclaimed, r) + } + return reclaimed, nil +} diff --git a/internal/storage/issueops/ready_work.go b/internal/storage/issueops/ready_work.go index 60ce22fb8..e12f57803 100644 --- a/internal/storage/issueops/ready_work.go +++ b/internal/storage/issueops/ready_work.go @@ -40,9 +40,13 @@ func buildReadyWorkOrder(policy types.SortPolicy) sqlbuild.ReadyWorkOrder { // needs (children of deferred parents, parent descendants), then delegates // the clause text to sqlbuild so both stacks share ready semantics. func buildReadyWorkPredicates(ctx context.Context, tx DBTX, filter types.WorkFilter, tables FilterTables) (*readyWorkPredicates, error) { - var inputs sqlbuild.ReadyWorkWhereInputs + return buildReadyWorkPredicatesDialect(ctx, tx, filter, tables, sqlbuild.CountsDialectDolt) +} + +func buildReadyWorkPredicatesDialect(ctx context.Context, tx DBTX, filter types.WorkFilter, tables FilterTables, dialect sqlbuild.CountsDialect) (*readyWorkPredicates, error) { + inputs := sqlbuild.ReadyWorkWhereInputs{Dialect: dialect} if !filter.IncludeDeferred { - deferredChildIDs, dcErr := getChildrenOfDeferredParentsInTx(ctx, tx) + deferredChildIDs, dcErr := getChildrenOfDeferredParentsInTx(ctx, tx, dialect) if dcErr != nil { return nil, fmt.Errorf("get ready work: compute deferred parent children: %w", dcErr) } @@ -88,7 +92,26 @@ func GetReadyWorkInTx( tx DBTX, filter types.WorkFilter, ) ([]*types.Issue, error) { - preds, err := buildReadyWorkPredicates(ctx, tx, filter, IssuesFilterTables) + return getReadyWorkInTx(ctx, tx, filter, sqlbuild.CountsDialectDolt) +} + +// GetReadyWorkSQLiteInTx returns ready work using SQLite-compatible SQL +// fragments for embedded DoltLite stores. +func GetReadyWorkSQLiteInTx( + ctx context.Context, + tx DBTX, + filter types.WorkFilter, +) ([]*types.Issue, error) { + return getReadyWorkInTx(ctx, tx, filter, sqlbuild.CountsDialectSQLite) +} + +func getReadyWorkInTx( + ctx context.Context, + tx DBTX, + filter types.WorkFilter, + dialect sqlbuild.CountsDialect, +) ([]*types.Issue, error) { + preds, err := buildReadyWorkPredicatesDialect(ctx, tx, filter, IssuesFilterTables, dialect) if err != nil { return nil, err } @@ -121,7 +144,7 @@ func GetReadyWorkInTx( } } - wisps, wErr := getReadyWispsInTx(ctx, tx, filter, preds.deferredChildIDs) + wisps, wErr := getReadyWispsInTx(ctx, tx, filter, preds.deferredChildIDs, dialect) if wErr != nil { return nil, wErr } @@ -152,7 +175,7 @@ func mergeReadyWisps(ordered []*types.Issue, wisps []*types.Issue, filter types. return kept } -func getReadyWispsInTx(ctx context.Context, tx DBTX, filter types.WorkFilter, deferredChildIDs []string) ([]*types.Issue, error) { +func getReadyWispsInTx(ctx context.Context, tx DBTX, filter types.WorkFilter, deferredChildIDs []string, dialect sqlbuild.CountsDialect) ([]*types.Issue, error) { empty, err := wispsTableEmptyOrMissingInTx(ctx, tx) if err != nil { return nil, fmt.Errorf("search wisps (ready work): probe: %w", err) @@ -164,7 +187,7 @@ func getReadyWispsInTx(ctx context.Context, tx DBTX, filter types.WorkFilter, de wispFilter := readyWorkWispIssueFilter(filter) if filter.Limit <= 0 { wispFilter.Limit = 0 - wisps, err := searchTableInTx(ctx, tx, "", wispFilter, WispsFilterTables) + wisps, err := searchTableInTxDialect(ctx, tx, "", wispFilter, WispsFilterTables, dialect) if err != nil { if isTableNotExistError(err) { return nil, nil @@ -178,7 +201,7 @@ func getReadyWispsInTx(ctx context.Context, tx DBTX, filter types.WorkFilter, de orderBy := buildReadyWorkOrder(filter.SortPolicy) ready := make([]*types.Issue, 0, filter.Limit) for offset := 0; len(ready) < filter.Limit; offset += pageSize { - pageIDs, err := queryReadyWispIssueIDPage(ctx, tx, wispFilter, !filter.IncludeDeferred, orderBy, pageSize, offset) + pageIDs, err := queryReadyWispIssueIDPage(ctx, tx, wispFilter, !filter.IncludeDeferred, orderBy, pageSize, offset, dialect) if err != nil { if isTableNotExistError(err) { return nil, nil @@ -210,15 +233,15 @@ func getReadyWispsInTx(ctx context.Context, tx DBTX, filter types.WorkFilter, de return ready, nil } -func queryReadyWispIssueIDPage(ctx context.Context, tx DBTX, filter types.IssueFilter, excludeDeferred bool, orderBy sqlbuild.ReadyWorkOrder, limit, offset int) ([]string, error) { +func queryReadyWispIssueIDPage(ctx context.Context, tx DBTX, filter types.IssueFilter, excludeDeferred bool, orderBy sqlbuild.ReadyWorkOrder, limit, offset int, dialect sqlbuild.CountsDialect) ([]string, error) { plan := sqlbuild.BuildLabelDrivenSearch(filter, WispsFilterTables) - whereClauses, args, err := BuildIssueFilterClauses("", plan.Filter, WispsFilterTables) + whereClauses, args, err := BuildIssueFilterClausesDialect("", plan.Filter, WispsFilterTables, dialect) if err != nil { return nil, err } whereClauses, args = plan.MergeInto(whereClauses, args) if excludeDeferred { - whereClauses = append(whereClauses, "(defer_until IS NULL OR defer_until <= UTC_TIMESTAMP())") + whereClauses = append(whereClauses, fmt.Sprintf("(defer_until IS NULL OR defer_until <= %s)", sqlbuild.ReadyWorkCurrentTimestamp(dialect))) } whereSQL := "" @@ -484,7 +507,8 @@ func queryReadyIssueIDPage(ctx context.Context, tx DBTX, query string, args []in // future defer_until. Works within an existing transaction. // //nolint:gosec // G201: depTable is selected from a hardcoded list below. -func getChildrenOfDeferredParentsInTx(ctx context.Context, tx DBTX) ([]string, error) { +func getChildrenOfDeferredParentsInTx(ctx context.Context, tx DBTX, dialect sqlbuild.CountsDialect) ([]string, error) { + nowSQL := sqlbuild.ReadyWorkCurrentTimestamp(dialect) hasDeferredParent := false for _, issueTable := range []string{"issues", "wisps"} { //nolint:gosec // G201: issueTable is hardcoded to "issues" or "wisps" @@ -492,9 +516,9 @@ func getChildrenOfDeferredParentsInTx(ctx context.Context, tx DBTX) ([]string, e err := tx.QueryRowContext(ctx, fmt.Sprintf(` SELECT 1 FROM %s WHERE defer_until IS NOT NULL - AND defer_until > UTC_TIMESTAMP() + AND defer_until > %s LIMIT 1 - `, issueTable)).Scan(&exists) + `, issueTable, nowSQL)).Scan(&exists) if err == nil { hasDeferredParent = true break @@ -524,8 +548,8 @@ func getChildrenOfDeferredParentsInTx(ctx context.Context, tx DBTX) ([]string, e JOIN %s parent ON parent.id = dep.%s WHERE dep.type = 'parent-child' AND parent.defer_until IS NOT NULL - AND parent.defer_until > UTC_TIMESTAMP() - `, depTable, issueTable, targetCol)) + AND parent.defer_until > %s + `, depTable, issueTable, targetCol, nowSQL)) if err != nil { if depTable == "wisp_dependencies" && isTableNotExistError(err) { break diff --git a/internal/storage/issueops/ready_work_counts.go b/internal/storage/issueops/ready_work_counts.go index 2f1483470..c8c1c52d8 100644 --- a/internal/storage/issueops/ready_work_counts.go +++ b/internal/storage/issueops/ready_work_counts.go @@ -7,20 +7,29 @@ import ( "fmt" "sort" + "github.com/steveyegge/beads/internal/storage/sqlbuild" "github.com/steveyegge/beads/internal/types" ) func GetReadyWorkWithCountsInTx(ctx context.Context, tx *sql.Tx, filter types.WorkFilter) ([]*types.IssueWithCounts, error) { + return getReadyWorkWithCountsInTx(ctx, tx, filter, sqlbuild.CountsDialectDolt) +} + +func GetReadyWorkWithCountsSQLiteInTx(ctx context.Context, tx *sql.Tx, filter types.WorkFilter) ([]*types.IssueWithCounts, error) { + return getReadyWorkWithCountsInTx(ctx, tx, filter, sqlbuild.CountsDialectSQLite) +} + +func getReadyWorkWithCountsInTx(ctx context.Context, tx *sql.Tx, filter types.WorkFilter, dialect sqlbuild.CountsDialect) ([]*types.IssueWithCounts, error) { wispDepsExist, err := optionalTableExistsInTx(ctx, tx, "wisp_dependencies") if err != nil { return nil, fmt.Errorf("get ready work with counts: wisp dependency probe: %w", err) } - issuePreds, err := buildReadyWorkPredicates(ctx, tx, filter, IssuesFilterTables) + issuePreds, err := buildReadyWorkPredicatesDialect(ctx, tx, filter, IssuesFilterTables, dialect) if err != nil { return nil, err } - out, err := runSearchQueryInTx(ctx, tx, IssuesFilterTables, issuePreds.whereSQL, issuePreds.orderBySQL, issuePreds.limitSQL, issuePreds.args, wispDepsExist, false) + out, err := runSearchQueryInTx(ctx, tx, IssuesFilterTables, issuePreds.whereSQL, issuePreds.orderBySQL, issuePreds.limitSQL, issuePreds.args, wispDepsExist, false, dialect) if err != nil { return nil, err } @@ -36,11 +45,11 @@ func GetReadyWorkWithCountsInTx(ctx context.Context, tx *sql.Tx, filter types.Wo return out, nil } - wispPreds, err := buildReadyWorkPredicates(ctx, tx, filter, WispsFilterTables) + wispPreds, err := buildReadyWorkPredicatesDialect(ctx, tx, filter, WispsFilterTables, dialect) if err != nil { return nil, err } - wisps, err := runSearchQueryInTx(ctx, tx, WispsFilterTables, wispPreds.whereSQL, wispPreds.orderBySQL, wispPreds.limitSQL, wispPreds.args, true, false) + wisps, err := runSearchQueryInTx(ctx, tx, WispsFilterTables, wispPreds.whereSQL, wispPreds.orderBySQL, wispPreds.limitSQL, wispPreds.args, true, false, dialect) if err != nil { if isTableNotExistError(err) { return out, nil diff --git a/internal/storage/issueops/ready_work_test.go b/internal/storage/issueops/ready_work_test.go index 3ee4658f3..21fa7f9fb 100644 --- a/internal/storage/issueops/ready_work_test.go +++ b/internal/storage/issueops/ready_work_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/storage/sqlbuild" "github.com/steveyegge/beads/internal/types" ) @@ -179,7 +180,7 @@ func TestGetChildrenOfDeferredParentsInTx_ReturnsChildrenFromBothDependencyTable mock.ExpectQuery(deferredChildrenQueryRegex("wisp_dependencies", "wisps")). WillReturnRows(sqlmock.NewRows([]string{"issue_id"}).AddRow("child-from-wisp-dependencies-wisps")) - got, err := getChildrenOfDeferredParentsInTx(context.Background(), tx) + got, err := getChildrenOfDeferredParentsInTx(context.Background(), tx, sqlbuild.CountsDialectDolt) if err != nil { t.Fatalf("getChildrenOfDeferredParentsInTx: %v", err) } @@ -206,7 +207,7 @@ func TestGetChildrenOfDeferredParentsInTx_NoDeferredParentsExitsAfterProbe(t *te mock.ExpectQuery(deferredParentProbeRegex("wisps")). WillReturnRows(sqlmock.NewRows([]string{"1"})) - got, err := getChildrenOfDeferredParentsInTx(context.Background(), tx) + got, err := getChildrenOfDeferredParentsInTx(context.Background(), tx, sqlbuild.CountsDialectDolt) if err != nil { t.Fatalf("getChildrenOfDeferredParentsInTx: %v", err) } @@ -231,7 +232,7 @@ func TestGetChildrenOfDeferredParentsInTx_IgnoresMissingWispDependenciesTable(t mock.ExpectQuery(deferredChildrenQueryRegex("wisp_dependencies", "issues")). WillReturnError(errors.New("table wisp_dependencies does not exist")) - got, err := getChildrenOfDeferredParentsInTx(context.Background(), tx) + got, err := getChildrenOfDeferredParentsInTx(context.Background(), tx, sqlbuild.CountsDialectDolt) if err != nil { t.Fatalf("getChildrenOfDeferredParentsInTx: %v", err) } diff --git a/internal/storage/issueops/search.go b/internal/storage/issueops/search.go index a79536d97..0d55a27b1 100644 --- a/internal/storage/issueops/search.go +++ b/internal/storage/issueops/search.go @@ -16,9 +16,19 @@ import ( // Set filter.SkipWisps=true for callers that never need ephemeral results; this // avoids the unconditional full-table wisps scan (Q2 perf opt). func SearchIssuesInTx(ctx context.Context, tx DBTX, query string, filter types.IssueFilter) ([]*types.Issue, error) { + return searchIssuesInTx(ctx, tx, query, filter, sqlbuild.CountsDialectDolt) +} + +// SearchIssuesSQLiteInTx executes filtered issue search using SQLite-compatible +// SQL fragments for embedded DoltLite stores. +func SearchIssuesSQLiteInTx(ctx context.Context, tx DBTX, query string, filter types.IssueFilter) ([]*types.Issue, error) { + return searchIssuesInTx(ctx, tx, query, filter, sqlbuild.CountsDialectSQLite) +} + +func searchIssuesInTx(ctx context.Context, tx DBTX, query string, filter types.IssueFilter, dialect sqlbuild.CountsDialect) ([]*types.Issue, error) { // Route ephemeral-only queries to wisps table. if filter.Ephemeral != nil && *filter.Ephemeral { - results, err := searchTableInTx(ctx, tx, query, filter, WispsFilterTables) + results, err := searchTableInTxDialect(ctx, tx, query, filter, WispsFilterTables, dialect) if err != nil && !isTableNotExistError(err) { return nil, fmt.Errorf("search wisps (ephemeral filter): %w", err) } @@ -28,7 +38,7 @@ func SearchIssuesInTx(ctx context.Context, tx DBTX, query string, filter types.I // Fall through: wisps table doesn't exist or returned no results } - results, err := searchTableInTx(ctx, tx, query, filter, IssuesFilterTables) + results, err := searchTableInTxDialect(ctx, tx, query, filter, IssuesFilterTables, dialect) if err != nil { return nil, fmt.Errorf("search issues: %w", err) } @@ -53,7 +63,7 @@ func SearchIssuesInTx(ctx context.Context, tx DBTX, query string, filter types.I if empty { return results, nil } - wispResults, wispErr := searchTableInTx(ctx, tx, query, filter, WispsFilterTables) + wispResults, wispErr := searchTableInTxDialect(ctx, tx, query, filter, WispsFilterTables, dialect) if wispErr != nil && !isTableNotExistError(wispErr) { return nil, fmt.Errorf("search wisps (merge): %w", wispErr) } @@ -85,8 +95,12 @@ func SearchIssuesInTx(ctx context.Context, tx DBTX, query string, filter types.I // Pattern B is equivalent to Pattern A but faster on large corpora where most rows // are never needed (mirrors the pattern in scanIssueIDs and GetStaleIssuesInTx). func searchTableInTx(ctx context.Context, tx DBTX, query string, filter types.IssueFilter, tables FilterTables) ([]*types.Issue, error) { + return searchTableInTxDialect(ctx, tx, query, filter, tables, sqlbuild.CountsDialectDolt) +} + +func searchTableInTxDialect(ctx context.Context, tx DBTX, query string, filter types.IssueFilter, tables FilterTables, dialect sqlbuild.CountsDialect) ([]*types.Issue, error) { plan := sqlbuild.BuildLabelDrivenSearch(filter, tables) - whereClauses, args, err := BuildIssueFilterClauses(query, plan.Filter, tables) + whereClauses, args, err := BuildIssueFilterClausesDialect(query, plan.Filter, tables, dialect) if err != nil { return nil, err } diff --git a/internal/storage/issueops/search_counts.go b/internal/storage/issueops/search_counts.go index b98563497..22b864db5 100644 --- a/internal/storage/issueops/search_counts.go +++ b/internal/storage/issueops/search_counts.go @@ -11,6 +11,14 @@ import ( ) func SearchIssuesWithCountsInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter) ([]*types.IssueWithCounts, error) { + return searchIssuesWithCountsInTx(ctx, tx, query, filter, sqlbuild.CountsDialectDolt) +} + +func SearchIssuesWithCountsSQLiteInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter) ([]*types.IssueWithCounts, error) { + return searchIssuesWithCountsInTx(ctx, tx, query, filter, sqlbuild.CountsDialectSQLite) +} + +func searchIssuesWithCountsInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter, dialect sqlbuild.CountsDialect) ([]*types.IssueWithCounts, error) { wispDepsExist, err := optionalTableExistsInTx(ctx, tx, "wisp_dependencies") if err != nil { return nil, fmt.Errorf("search issues with counts: wisp dependency probe: %w", err) @@ -22,7 +30,7 @@ func SearchIssuesWithCountsInTx(ctx context.Context, tx *sql.Tx, query string, f return nil, fmt.Errorf("search issues with counts: ephemeral wisp probe: %w", probeErr) } if !empty && wispDepsExist { - wisps, err := runFilterSearchQueryInTx(ctx, tx, query, filter, WispsFilterTables, true) + wisps, err := runFilterSearchQueryInTx(ctx, tx, query, filter, WispsFilterTables, true, dialect) if err != nil && !isTableNotExistError(err) { return nil, err } @@ -36,14 +44,14 @@ func SearchIssuesWithCountsInTx(ctx context.Context, tx *sql.Tx, query string, f // dropping it. Use the same IssuesFilterTables query the non-ephemeral // path uses, keeping the GH#4387 count/list cardinality parity for // searches that project counts (e.g. `bd search --counts --include-infra`). - out, err := runFilterSearchQueryInTx(ctx, tx, query, filter, IssuesFilterTables, wispDepsExist) + out, err := runFilterSearchQueryInTx(ctx, tx, query, filter, IssuesFilterTables, wispDepsExist, dialect) if err != nil { return nil, err } return finishSearchIssuesWithCounts(out, filter), nil } - out, err := runFilterSearchQueryInTx(ctx, tx, query, filter, IssuesFilterTables, wispDepsExist) + out, err := runFilterSearchQueryInTx(ctx, tx, query, filter, IssuesFilterTables, wispDepsExist, dialect) if err != nil { return nil, err } @@ -64,7 +72,7 @@ func SearchIssuesWithCountsInTx(ctx context.Context, tx *sql.Tx, query string, f return finishSearchIssuesWithCounts(out, filter), nil } - wisps, err := runFilterSearchQueryInTx(ctx, tx, query, filter, WispsFilterTables, true) + wisps, err := runFilterSearchQueryInTx(ctx, tx, query, filter, WispsFilterTables, true, dialect) if err != nil { if isTableNotExistError(err) { return finishSearchIssuesWithCounts(out, filter), nil @@ -96,8 +104,8 @@ func SearchIssuesWithCountsInTx(ctx context.Context, tx *sql.Tx, query string, f return finishSearchIssuesWithCounts(kept, filter), nil } -func runFilterSearchQueryInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter, tables FilterTables, includeWispReverseDeps bool) ([]*types.IssueWithCounts, error) { - whereClauses, args, err := BuildIssueFilterClauses(query, filter, tables) +func runFilterSearchQueryInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter, tables FilterTables, includeWispReverseDeps bool, dialect sqlbuild.CountsDialect) ([]*types.IssueWithCounts, error) { + whereClauses, args, err := BuildIssueFilterClausesDialect(query, filter, tables, dialect) if err != nil { return nil, err } @@ -110,12 +118,12 @@ func runFilterSearchQueryInTx(ctx context.Context, tx *sql.Tx, query string, fil limitSQL = fmt.Sprintf("LIMIT %d", filter.Limit) } orderBy := sqlbuild.OrderBy(filter.SortBy, filter.SortDesc, "i") - return runSearchQueryInTx(ctx, tx, tables, whereSQL, orderBy, limitSQL, args, includeWispReverseDeps, filter.SkipLabels) + return runSearchQueryInTx(ctx, tx, tables, whereSQL, orderBy, limitSQL, args, includeWispReverseDeps, filter.SkipLabels, dialect) } //nolint:gosec // G201: SQL fragments are caller-built from hardcoded shapes -func runSearchQueryInTx(ctx context.Context, tx *sql.Tx, tables FilterTables, whereSQL, orderBySQL, limitSQL string, args []interface{}, includeWispReverseDeps bool, skipLabels bool) ([]*types.IssueWithCounts, error) { - searchSQL := sqlbuild.SearchCountsSQL(tables, whereSQL, orderBySQL, limitSQL, includeWispReverseDeps, skipLabels) +func runSearchQueryInTx(ctx context.Context, tx *sql.Tx, tables FilterTables, whereSQL, orderBySQL, limitSQL string, args []interface{}, includeWispReverseDeps bool, skipLabels bool, dialect sqlbuild.CountsDialect) ([]*types.IssueWithCounts, error) { + searchSQL := sqlbuild.SearchCountsSQLDialect(tables, whereSQL, orderBySQL, limitSQL, includeWispReverseDeps, skipLabels, dialect) rows, err := tx.QueryContext(ctx, searchSQL, args...) if err != nil { diff --git a/internal/storage/issueops/update.go b/internal/storage/issueops/update.go index 6eed49e9e..d39c9907c 100644 --- a/internal/storage/issueops/update.go +++ b/internal/storage/issueops/update.go @@ -125,17 +125,23 @@ type UpdateResult struct { // //nolint:gosec // G201: table names come from WispTableRouting (hardcoded constants) func UpdateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string]interface{}, actor string) (*UpdateResult, error) { - return updateIssueInTx(ctx, tx, id, updates, actor, true) + return updateIssueInTx(ctx, tx, id, updates, actor, true, false) +} + +// UpdateIssueSQLiteInTx updates an issue using SQLite-compatible derived-state +// recompute SQL for embedded DoltLite stores. +func UpdateIssueSQLiteInTx(ctx context.Context, tx DBTX, id string, updates map[string]interface{}, actor string) (*UpdateResult, error) { + return updateIssueInTx(ctx, tx, id, updates, actor, true, true) } // UpdateIssueWithoutEventInTx applies normal update semantics without recording // an intermediate event. Demotion uses this to preserve the historical event // stream: create/update history is copied, then a single demotion event is added. func UpdateIssueWithoutEventInTx(ctx context.Context, tx DBTX, id string, updates map[string]interface{}, actor string) (*UpdateResult, error) { - return updateIssueInTx(ctx, tx, id, updates, actor, false) + return updateIssueInTx(ctx, tx, id, updates, actor, false, false) } -func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string]interface{}, actor string, recordEvent bool) (*UpdateResult, error) { +func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string]interface{}, actor string, recordEvent bool, sqlite bool) (*UpdateResult, error) { // Route to correct table. isWisp := IsActiveWispInTx(ctx, tx, id) issueTable, _, eventTable, _ := WispTableRouting(isWisp) @@ -215,6 +221,14 @@ func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string // Auto-manage started_at (set on transition to in_progress). (GH#2796) setClauses, args = ManageStartedAt(oldIssue, updates, setClauses, args) + // Rewrite row_lock on every update so a concurrent lease mutation (heartbeat/ + // reclaim) collides on this shared cell and is forced to conflict-and-retry + // rather than silently cell-merging two writes to different columns of the + // same row (see lease.go). This is the "every mutating path writes row_lock" + // invariant the lease scheme depends on. + setClauses = append(setClauses, "row_lock = ?") + args = append(args, freshRowLock()) + args = append(args, id) //nolint:gosec // G201: issueTable comes from WispTableRouting (hardcoded constants) @@ -254,8 +268,14 @@ func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string if aerr != nil { return nil, fmt.Errorf("affected by status change for %s: %w", id, aerr) } - if err := RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps); err != nil { - return nil, fmt.Errorf("recompute is_blocked after status change for %s: %w", id, err) + var recomputeErr error + if sqlite { + recomputeErr = RecomputeIsBlockedSQLiteInTx(ctx, tx, affectedIssues, affectedWisps) + } else { + recomputeErr = RecomputeIsBlockedInTx(ctx, tx, affectedIssues, affectedWisps) + } + if recomputeErr != nil { + return nil, fmt.Errorf("recompute is_blocked after status change for %s: %w", id, recomputeErr) } } } diff --git a/internal/storage/schema/cli_migrations.go b/internal/storage/schema/cli_migrations.go index ca21b9957..6939d3af8 100644 --- a/internal/storage/schema/cli_migrations.go +++ b/internal/storage/schema/cli_migrations.go @@ -49,6 +49,11 @@ func cliCompatibleMigrationSQL(name, sqlText string) string { // bundles already have the base wisp tables, and the Dolt CLI test // path needs direct DML for deterministic fixture repair. return cliMigration0053RepairRigWisps + case "0054_add_lease_columns.up.sql": + // Fresh bundle bakes the lease columns directly: the Dolt CLI does not + // apply the prepared ALTER TABLE statements the runtime migration uses + // for idempotent re-runs on upgraded databases. + return cliMigration0054AddLeaseColumns default: return sqlText } @@ -68,6 +73,14 @@ ALTER TABLE wisps ADD COLUMN started_at DATETIME;` const cliMigration0032DropSchemaMigrationsAppliedAt = `ALTER TABLE schema_migrations DROP COLUMN applied_at;` +const cliMigration0054AddLeaseColumns = `ALTER TABLE issues ADD COLUMN lease_expires_at DATETIME; +ALTER TABLE issues ADD COLUMN heartbeat_at DATETIME; +ALTER TABLE issues ADD COLUMN row_lock BIGINT NOT NULL DEFAULT 0; +CREATE INDEX idx_issues_lease ON issues (status, lease_expires_at); +ALTER TABLE wisps ADD COLUMN lease_expires_at DATETIME; +ALTER TABLE wisps ADD COLUMN heartbeat_at DATETIME; +ALTER TABLE wisps ADD COLUMN row_lock BIGINT NOT NULL DEFAULT 0;` + const cliMigration0041SplitDependenciesTarget = `DELETE FROM dolt_nonlocal_tables; CALL DOLT_COMMIT('-Am', 'disable nonlocal tables for fk migrations'); SET FOREIGN_KEY_CHECKS = 0; diff --git a/internal/storage/schema/content_hash_test.go b/internal/storage/schema/content_hash_test.go index b0da34297..71d6eac72 100644 --- a/internal/storage/schema/content_hash_test.go +++ b/internal/storage/schema/content_hash_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "os" "path/filepath" "strconv" @@ -39,6 +40,34 @@ func TestEnsureContentHashColumnAddsWhenMissing(t *testing.T) { } } +func TestEnsureContentHashColumnFallsBackToSQLitePragma(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.COLUMNS`). + WillReturnError(errors.New("no such table: INFORMATION_SCHEMA.COLUMNS")) + mock.ExpectQuery(`PRAGMA table_info\(schema_migrations\)`). + WillReturnRows(sqlmock.NewRows([]string{"cid", "name", "type", "notnull", "dflt_value", "pk"}). + AddRow(0, "version", "INT", 0, nil, 1). + AddRow(1, "applied_at", "DATETIME", 1, nil, 0)) + mock.ExpectExec(`ALTER TABLE schema_migrations ADD COLUMN content_hash CHAR\(64\)`). + WillReturnResult(sqlmock.NewResult(0, 0)) + + added, err := mainSource.ensureContentHashColumn(context.Background(), db) + if err != nil { + t.Fatalf("ensureContentHashColumn: %v", err) + } + if !added { + t.Fatal("ensureContentHashColumn added = false, want true when SQLite metadata lacks the column") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations: %v", err) + } +} + // TestEnsureContentHashColumnNoOpWhenPresent verifies it issues no ALTER when the // column already exists. func TestEnsureContentHashColumnNoOpWhenPresent(t *testing.T) { diff --git a/internal/storage/schema/migrations/0033_add_wisp_type_column.down.sql b/internal/storage/schema/migrations/0033_add_wisp_type_column.down.sql deleted file mode 100644 index d99797228..000000000 --- a/internal/storage/schema/migrations/0033_add_wisp_type_column.down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE issues DROP COLUMN wisp_type; diff --git a/internal/storage/schema/migrations/0033_add_wisp_type_column.up.sql b/internal/storage/schema/migrations/0033_add_wisp_type_column.up.sql deleted file mode 100644 index 4cf9a8815..000000000 --- a/internal/storage/schema/migrations/0033_add_wisp_type_column.up.sql +++ /dev/null @@ -1,11 +0,0 @@ -SET @needs_add = ( - SELECT IF(COUNT(*) = 0, 1, 0) - FROM INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_SCHEMA = DATABASE() - AND TABLE_NAME = 'issues' - AND COLUMN_NAME = 'wisp_type' -); -SET @sql = IF(@needs_add = 1, - 'ALTER TABLE issues ADD COLUMN wisp_type VARCHAR(32) DEFAULT ''''', - 'SELECT 1'); -PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/schema/migrations/0034_add_spec_id_column.down.sql b/internal/storage/schema/migrations/0034_add_spec_id_column.down.sql deleted file mode 100644 index 0732e2fac..000000000 --- a/internal/storage/schema/migrations/0034_add_spec_id_column.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP INDEX idx_issues_spec_id ON issues; -ALTER TABLE issues DROP COLUMN spec_id; diff --git a/internal/storage/schema/migrations/0034_add_spec_id_column.up.sql b/internal/storage/schema/migrations/0034_add_spec_id_column.up.sql deleted file mode 100644 index c9b9d0002..000000000 --- a/internal/storage/schema/migrations/0034_add_spec_id_column.up.sql +++ /dev/null @@ -1,25 +0,0 @@ --- spec_id column -SET @needs_add = ( - SELECT IF(COUNT(*) = 0, 1, 0) - FROM INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_SCHEMA = DATABASE() - AND TABLE_NAME = 'issues' - AND COLUMN_NAME = 'spec_id' -); -SET @sql = IF(@needs_add = 1, - 'ALTER TABLE issues ADD COLUMN spec_id VARCHAR(1024)', - 'SELECT 1'); -PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; - --- idx_issues_spec_id index -SET @needs_index = ( - SELECT IF(COUNT(*) = 0, 1, 0) - FROM INFORMATION_SCHEMA.STATISTICS - WHERE TABLE_SCHEMA = DATABASE() - AND TABLE_NAME = 'issues' - AND INDEX_NAME = 'idx_issues_spec_id' -); -SET @sql = IF(@needs_index = 1, - 'CREATE INDEX idx_issues_spec_id ON issues(spec_id)', - 'SELECT 1'); -PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/schema/migrations/0054_add_lease_columns.down.sql b/internal/storage/schema/migrations/0054_add_lease_columns.down.sql new file mode 100644 index 000000000..76fa2e05d --- /dev/null +++ b/internal/storage/schema/migrations/0054_add_lease_columns.down.sql @@ -0,0 +1,7 @@ +DROP INDEX idx_issues_lease ON issues; +ALTER TABLE issues DROP COLUMN lease_expires_at; +ALTER TABLE issues DROP COLUMN heartbeat_at; +ALTER TABLE issues DROP COLUMN row_lock; +ALTER TABLE wisps DROP COLUMN lease_expires_at; +ALTER TABLE wisps DROP COLUMN heartbeat_at; +ALTER TABLE wisps DROP COLUMN row_lock; diff --git a/internal/storage/schema/migrations/0054_add_lease_columns.up.sql b/internal/storage/schema/migrations/0054_add_lease_columns.up.sql new file mode 100644 index 000000000..7b01ea3ee --- /dev/null +++ b/internal/storage/schema/migrations/0054_add_lease_columns.up.sql @@ -0,0 +1,114 @@ +-- Dead-worker recovery (Gas Station v1.1, wy-5r9j): give a claim a lease. +-- +-- A claim was previously permanent — a worker that died mid-task stranded its +-- issue in_progress forever. These columns let a claim expire: +-- +-- lease_expires_at the wall-clock instant after which the claim is stale and +-- a reaper (bd reclaim) may revert the issue to ready. +-- heartbeat_at the last time the lease owner proved it was still alive. +-- row_lock a random BIGINT rewritten by EVERY mutating path on the +-- row. Dolt has no real row locking and merges concurrent +-- writes cell-by-cell, so a heartbeat (touching heartbeat_at) +-- and a close (touching status) would otherwise silently +-- cell-merge instead of conflicting. Forcing every writer to +-- also rewrite this one shared cell turns those into a +-- 1213/1205 serialization conflict that withRetryTx replays — +-- the difference between exactly-once and a lost close. +-- +-- Guarded so the migration is idempotent on a schema_migrations row that +-- regressed without its DDL rolled back (see 0052/0046). assignee doubles as the +-- lease owner; no new owner column is needed. + +-- issues.lease_expires_at +SET @needs_add = ( + SELECT IF(COUNT(*) = 0, 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issues' + AND COLUMN_NAME = 'lease_expires_at' +); +SET @sql = IF(@needs_add = 1, + 'ALTER TABLE issues ADD COLUMN lease_expires_at DATETIME', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- issues.heartbeat_at +SET @needs_add = ( + SELECT IF(COUNT(*) = 0, 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issues' + AND COLUMN_NAME = 'heartbeat_at' +); +SET @sql = IF(@needs_add = 1, + 'ALTER TABLE issues ADD COLUMN heartbeat_at DATETIME', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- issues.row_lock +SET @needs_add = ( + SELECT IF(COUNT(*) = 0, 1, 0) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issues' + AND COLUMN_NAME = 'row_lock' +); +SET @sql = IF(@needs_add = 1, + 'ALTER TABLE issues ADD COLUMN row_lock BIGINT NOT NULL DEFAULT 0', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- idx_issues_lease: the reaper scans in_progress issues by lease_expires_at. +SET @needs_index = ( + SELECT IF(COUNT(*) = 0, 1, 0) + FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'issues' + AND INDEX_NAME = 'idx_issues_lease' +); +SET @sql = IF(@needs_index = 1, + 'CREATE INDEX idx_issues_lease ON issues (status, lease_expires_at)', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- wisps mirror the issues lease columns so the shared issueops claim/heartbeat +-- SQL (which routes by table name) works uniformly. Wisps are ephemeral and are +-- never reclaimed, but the columns must exist for the shared UPDATEs to bind. +-- Guarded on the wisps table existing (older workspaces created issues-only). +SET @has_wisps = ( + SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'wisps' +); + +SET @needs_add = IF(@has_wisps > 0 AND + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'wisps' + AND COLUMN_NAME = 'lease_expires_at') = 0, + 1, 0); +SET @sql = IF(@needs_add = 1, + 'ALTER TABLE wisps ADD COLUMN lease_expires_at DATETIME', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @needs_add = IF(@has_wisps > 0 AND + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'wisps' + AND COLUMN_NAME = 'heartbeat_at') = 0, + 1, 0); +SET @sql = IF(@needs_add = 1, + 'ALTER TABLE wisps ADD COLUMN heartbeat_at DATETIME', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @needs_add = IF(@has_wisps > 0 AND + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'wisps' + AND COLUMN_NAME = 'row_lock') = 0, + 1, 0); +SET @sql = IF(@needs_add = 1, + 'ALTER TABLE wisps ADD COLUMN row_lock BIGINT NOT NULL DEFAULT 0', + 'SELECT 1'); +PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index 500f758ed..44194514e 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -737,18 +737,61 @@ func (m migrationSource) bootstrapSQL() string { } // hasContentHashColumn reports whether the cursor table already carries the -// content_hash column. It probes INFORMATION_SCHEMA, so a not-yet-created table -// simply reports false. +// content_hash column. It probes INFORMATION_SCHEMA for Dolt/MySQL-compatible +// engines and falls back to SQLite PRAGMA metadata for DoltLite. func (m migrationSource) hasContentHashColumn(ctx context.Context, db DBConn) (bool, error) { var count int if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = 'content_hash'`, m.cursorTable).Scan(&count); err != nil { - return false, fmt.Errorf("checking %s.content_hash: %w", m.cursorTable, err) + if !isMissingInformationSchemaError(err) { + return false, fmt.Errorf("checking %s.content_hash: %w", m.cursorTable, err) + } + has, pragmaErr := m.hasContentHashColumnSQLite(ctx, db) + if pragmaErr != nil { + return false, fmt.Errorf("checking %s.content_hash: %w", m.cursorTable, pragmaErr) + } + return has, nil } return count > 0, nil } +func isMissingInformationSchemaError(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "information_schema.columns") || + strings.Contains(msg, "information_schema") || + strings.Contains(msg, "no such table") +} + +func (m migrationSource) hasContentHashColumnSQLite(ctx context.Context, db DBConn) (bool, error) { + //nolint:gosec // G201: m.cursorTable is a hardcoded constant. + rows, err := db.QueryContext(ctx, "PRAGMA table_info("+m.cursorTable+")") + if err != nil { + return false, err + } + defer rows.Close() + + for rows.Next() { + var cid int + var name, columnType string + var notNull, pk int + var defaultValue any + if err := rows.Scan(&cid, &name, &columnType, ¬Null, &defaultValue, &pk); err != nil { + return false, err + } + if name == "content_hash" { + return true, nil + } + } + if err := rows.Err(); err != nil { + return false, err + } + return false, nil +} + // ensureContentHashColumn adds the content_hash column to an existing cursor // table that predates it (gastownhall/beads#4259 reporter fix No.2: record a // per-migration content hash so two clones at the same MAX(version) but with diff --git a/internal/storage/schema/schema_test.go b/internal/storage/schema/schema_test.go index dea203985..bfb1f33d0 100644 --- a/internal/storage/schema/schema_test.go +++ b/internal/storage/schema/schema_test.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strconv" "strings" "testing" @@ -64,6 +65,72 @@ func TestIgnoredPendingMigrationDirtyTablesDetectsWispDependencies(t *testing.T) } } +func TestMigrateSQLiteUpToSkipsMySQLOnlyMigration0035(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectExec(regexp.QuoteMeta(mainSource.bootstrapSQL())). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.COLUMNS`). + WithArgs("schema_migrations"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + mock.ExpectQuery(`SELECT COALESCE\(MAX\(version\), 0\) FROM schema_migrations`). + WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(34)) + mock.ExpectExec(regexp.QuoteMeta("INSERT OR IGNORE INTO schema_migrations (version, content_hash) VALUES (?, ?)")). + WithArgs(35, sqlmock.AnyArg()). + WillReturnResult(sqlmock.NewResult(0, 1)) + + applied, err := MigrateSQLiteUpTo(context.Background(), db, 35) + if err != nil { + t.Fatalf("MigrateSQLiteUpTo: %v", err) + } + if applied != 1 { + t.Fatalf("applied = %d, want 1", applied) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + +func TestMigrateSQLiteUpToMigration0046IsIdempotent(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + mock.ExpectExec(regexp.QuoteMeta(mainSource.bootstrapSQL())). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM INFORMATION_SCHEMA\.COLUMNS`). + WithArgs("schema_migrations"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + mock.ExpectQuery(`SELECT COALESCE\(MAX\(version\), 0\) FROM schema_migrations`). + WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(45)) + mock.ExpectQuery(regexp.QuoteMeta("PRAGMA table_info(issues)")). + WillReturnRows(sqlmock.NewRows([]string{"cid", "name", "type", "notnull", "dflt_value", "pk"}). + AddRow(0, "id", "VARCHAR(255)", 1, nil, 1). + AddRow(1, "is_blocked", "TINYINT(1)", 1, "0", 0)) + mock.ExpectExec(regexp.QuoteMeta("CREATE INDEX IF NOT EXISTS idx_issues_is_blocked ON issues(is_blocked, status)")). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(regexp.QuoteMeta("INSERT OR IGNORE INTO schema_migrations (version, content_hash) VALUES (?, ?)")). + WithArgs(46, sqlmock.AnyArg()). + WillReturnResult(sqlmock.NewResult(0, 1)) + + applied, err := MigrateSQLiteUpTo(context.Background(), db, 46) + if err != nil { + t.Fatalf("MigrateSQLiteUpTo: %v", err) + } + if applied != 1 { + t.Fatalf("applied = %d, want 1", applied) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet sql expectations: %v", err) + } +} + func TestMigrationSQLTouchesTableStatementForms(t *testing.T) { tests := []struct { name string @@ -524,8 +591,9 @@ FROM schema_migrations`) if got := rows[0]["max_version"]; got != want { t.Fatalf("MAX(version) = %s, want %s", got, want) } - if got := rows[0]["version_count"]; got != want { - t.Fatalf("COUNT(*) = %s, want %s", got, want) + wantCount := strconv.Itoa(len(mainSource.list())) + if got := rows[0]["version_count"]; got != wantCount { + t.Fatalf("COUNT(*) = %s, want %s", got, wantCount) } requireDoltNoRows(t, dir, ` diff --git a/internal/storage/schema/sqlite_migrations.go b/internal/storage/schema/sqlite_migrations.go new file mode 100644 index 000000000..e9a10d681 --- /dev/null +++ b/internal/storage/schema/sqlite_migrations.go @@ -0,0 +1,326 @@ +package schema + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "regexp" + "strings" +) + +// MigrateFreshSQLite applies the main schema migration stream to a fresh +// SQLite-compatible database. It records the same migration versions and +// content hashes as the Dolt/MySQL path, while executing SQLite-compatible SQL +// for fresh DoltLite stores. +func MigrateFreshSQLite(ctx context.Context, db DBConn, maxVersion int) (int, error) { + if _, err := db.ExecContext(ctx, mainSource.bootstrapSQL()); err != nil { + return 0, fmt.Errorf("creating %s: %w", mainSource.cursorTable, err) + } + if has, err := mainSource.hasContentHashColumn(ctx, db); err != nil { + return 0, err + } else if !has { + if _, err := db.ExecContext(ctx, "ALTER TABLE "+mainSource.cursorTable+" ADD COLUMN content_hash CHAR(64)"); err != nil { + return 0, fmt.Errorf("adding %s.content_hash: %w", mainSource.cursorTable, err) + } + } + + current, err := mainSource.currentVersion(ctx, db) + if err != nil { + return 0, err + } + if current != 0 { + return 0, fmt.Errorf("sqlite fresh migration requires an empty schema cursor, found version %d", current) + } + + target := mainSource.latest() + if maxVersion > 0 && maxVersion < target { + target = maxVersion + } + + count := 0 + for _, mf := range mainSource.list() { + if mf.version > target { + continue + } + data, err := mainSource.files.ReadFile(mainSource.dir + "/" + mf.name) + if err != nil { + return count, fmt.Errorf("reading migration %s: %w", mf.name, err) + } + if mf.name == "0046_add_is_blocked.up.sql" { + if err := applySQLiteMigration0046(ctx, db); err != nil { + return count, fmt.Errorf("migration %s: %w", mf.name, err) + } + } else { + sqlText := sqliteCompatibleMigrationSQL(mf.name, string(data)) + if strings.TrimSpace(sqlText) != "" { + if _, err := db.ExecContext(ctx, sqlText); err != nil { + return count, fmt.Errorf("migration %s: %w", mf.name, err) + } + } + } + sum := sha256.Sum256(data) + if _, err := db.ExecContext(ctx, "INSERT OR IGNORE INTO "+mainSource.cursorTable+" (version, content_hash) VALUES (?, ?)", mf.version, hex.EncodeToString(sum[:])); err != nil { + return count, fmt.Errorf("recording %s in %s: %w", mf.name, mainSource.cursorTable, err) + } + count++ + } + return count, nil +} + +// MigrateSQLiteUpTo applies pending main schema migrations to an existing +// SQLite-compatible database. It records the original migration hashes while +// executing the SQLite-compatible migration body, which may intentionally be a +// no-op for Dolt/MySQL-only migrations. +func MigrateSQLiteUpTo(ctx context.Context, db DBConn, maxVersion int) (int, error) { + if _, err := db.ExecContext(ctx, mainSource.bootstrapSQL()); err != nil { + return 0, fmt.Errorf("creating %s: %w", mainSource.cursorTable, err) + } + if _, err := mainSource.ensureContentHashColumn(ctx, db); err != nil { + return 0, err + } + + target := mainSource.latest() + if maxVersion > 0 && maxVersion < target { + target = maxVersion + } + + current, err := mainSource.currentVersion(ctx, db) + if err != nil { + return 0, err + } + if current >= target { + return 0, nil + } + + count := 0 + for _, mf := range mainSource.list() { + if mf.version <= current || mf.version > target { + continue + } + data, err := mainSource.files.ReadFile(mainSource.dir + "/" + mf.name) + if err != nil { + return count, fmt.Errorf("reading migration %s: %w", mf.name, err) + } + if mf.name == "0046_add_is_blocked.up.sql" { + if err := applySQLiteMigration0046(ctx, db); err != nil { + return count, fmt.Errorf("migration %s: %w", mf.name, err) + } + } else { + sqlText := sqliteCompatibleMigrationSQL(mf.name, string(data)) + if strings.TrimSpace(sqlText) != "" { + if _, err := db.ExecContext(ctx, sqlText); err != nil { + return count, fmt.Errorf("migration %s: %w", mf.name, err) + } + } + } + sum := sha256.Sum256(data) + if _, err := db.ExecContext(ctx, "INSERT OR IGNORE INTO "+mainSource.cursorTable+" (version, content_hash) VALUES (?, ?)", mf.version, hex.EncodeToString(sum[:])); err != nil { + return count, fmt.Errorf("recording %s in %s: %w", mf.name, mainSource.cursorTable, err) + } + count++ + } + return count, nil +} + +func applySQLiteMigration0046(ctx context.Context, db DBConn) error { + hasIsBlocked, err := sqliteColumnExists(ctx, db, "issues", "is_blocked") + if err != nil { + return fmt.Errorf("checking issues.is_blocked: %w", err) + } + if !hasIsBlocked { + if _, err := db.ExecContext(ctx, "ALTER TABLE issues ADD COLUMN is_blocked TINYINT(1) NOT NULL DEFAULT 0"); err != nil { + return err + } + } + if _, err := db.ExecContext(ctx, "CREATE INDEX IF NOT EXISTS idx_issues_is_blocked ON issues(is_blocked, status)"); err != nil { + return fmt.Errorf("creating idx_issues_is_blocked: %w", err) + } + return nil +} + +func sqliteColumnExists(ctx context.Context, db DBConn, table, column string) (bool, error) { + rows, err := db.QueryContext(ctx, "PRAGMA table_info("+table+")") + if err != nil { + return false, err + } + defer rows.Close() + + for rows.Next() { + var cid int + var name, columnType string + var notNull, pk int + var defaultValue any + if err := rows.Scan(&cid, &name, &columnType, ¬Null, &defaultValue, &pk); err != nil { + return false, err + } + if name == column { + return true, nil + } + } + if err := rows.Err(); err != nil { + return false, err + } + return false, nil +} + +func sqliteCompatibleMigrationSQL(name, sqlText string) string { + switch name { + case "0019_wisps_dolt_ignore.up.sql", "0028_local_state_dolt_ignore.up.sql", + "0040_ignored_tables_also_nonlocal_tables.up.sql": + return "" + case "0022_wisp_dep_type_index.up.sql": + return `CREATE INDEX IF NOT EXISTS idx_wisp_dep_type ON wisp_dependencies (type); +CREATE INDEX IF NOT EXISTS idx_wisp_dep_type_issue ON wisp_dependencies (type, depends_on_issue_id); +CREATE INDEX IF NOT EXISTS idx_wisp_dep_type_wisp ON wisp_dependencies (type, depends_on_wisp_id); +CREATE INDEX IF NOT EXISTS idx_wisp_dep_type_external ON wisp_dependencies (type, depends_on_external);` + case "0023_add_no_history_column.up.sql": + return cliMigration0023AddNoHistoryColumn + case "0027_add_started_at.up.sql": + return cliMigration0027AddStartedAt + case "0020_create_wisps.up.sql": + return sqliteNormalizeMigrationSQL(sqlText) + ` +ALTER TABLE wisps ADD COLUMN is_blocked TINYINT(1) NOT NULL DEFAULT 0; +CREATE INDEX IF NOT EXISTS idx_wisps_is_blocked ON wisps(is_blocked, status);` + case "0030_migrate_local_metadata_keys.up.sql": + return "" + case "0031_wisp_events_created_at_index.up.sql": + return `CREATE INDEX IF NOT EXISTS idx_wisp_events_created_at ON wisp_events (created_at);` + case "0032_drop_schema_migrations_applied_at.up.sql": + return "" + case "0035_migrate_infra_to_wisps.up.sql", "0036_cleanup_autopush_metadata.up.sql", + "0037_uuid_primary_keys.up.sql", "0038_drop_hop_columns.up.sql", + "0039_drop_child_counters_fk.up.sql", "0042_add_on_update_cascade.up.sql", "0047_recompute_mixed_is_blocked.up.sql", + "0048_widen_event_value_columns.up.sql", "0049_longtext_large_content_columns.up.sql", + "0050_dependencies_deterministic_id.up.sql", "0051_drop_aux_id_defaults.up.sql", + "0053_repair_rig_wisps.up.sql": + return "" + case "0041_split_dependencies_target.up.sql", "0043_drop_dependencies_generated_column.up.sql": + return sqliteFinalDependenciesSchema + case "0046_add_is_blocked.up.sql": + return cliMigration0046AddIsBlocked + case "0052_add_date_indexes.up.sql": + return `DROP INDEX IF EXISTS idx_issues_status; +CREATE INDEX IF NOT EXISTS idx_issues_status_updated_at ON issues (status, updated_at); +CREATE INDEX IF NOT EXISTS idx_issues_defer_until ON issues (defer_until);` + case "0054_add_lease_columns.up.sql": + return cliMigration0054AddLeaseColumns + default: + return sqliteNormalizeMigrationSQL(sqlText) + } +} + +func sqliteNormalizeMigrationSQL(sqlText string) string { + sqlText = strings.ReplaceAll(sqlText, " ON UPDATE CURRENT_TIMESTAMP", "") + sqlText = strings.ReplaceAll(sqlText, "JSON DEFAULT (JSON_OBJECT())", "JSON DEFAULT '{}'") + sqlText = strings.ReplaceAll(sqlText, "INSERT IGNORE", "INSERT OR IGNORE") + sqlText = strings.ReplaceAll(sqlText, "UTC_TIMESTAMP()", "CURRENT_TIMESTAMP") + sqlText = strings.ReplaceAll(sqlText, "NOW()", "CURRENT_TIMESTAMP") + sqlText = sqliteNormalizeViews(sqlText) + return sqliteExtractInlineIndexes(sqlText) +} + +var createOrReplaceViewRE = regexp.MustCompile(`(?is)CREATE\s+OR\s+REPLACE\s+VIEW\s+([A-Za-z_][A-Za-z0-9_]*)\s+AS`) + +func sqliteNormalizeViews(sqlText string) string { + return createOrReplaceViewRE.ReplaceAllString(sqlText, "DROP VIEW IF EXISTS $1;\nCREATE VIEW $1 AS") +} + +var ( + createTableRE = regexp.MustCompile(`(?is)CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\s+([A-Za-z_][A-Za-z0-9_]*)\s*\((.*?)\)\s*;`) + inlineIndexRE = regexp.MustCompile(`(?i)^INDEX\s+([A-Za-z_][A-Za-z0-9_]*)\s*(\(.*\))$`) + inlineUniqueRE = regexp.MustCompile(`(?i)^UNIQUE\s+KEY\s+([A-Za-z_][A-Za-z0-9_]*)\s*(\(.*\))$`) + inlineKeyRE = regexp.MustCompile(`(?i)^KEY\s+([A-Za-z_][A-Za-z0-9_]*)\s*(\(.*\))$`) + firstColumnRE = regexp.MustCompile(`(?i)\s+FIRST\b`) + defaultUUIDColRE = regexp.MustCompile(`(?i)\s+DEFAULT\s+\(UUID\(\)\)`) +) + +func sqliteExtractInlineIndexes(sqlText string) string { + return createTableRE.ReplaceAllStringFunc(sqlText, func(stmt string) string { + m := createTableRE.FindStringSubmatch(stmt) + if len(m) != 3 { + return stmt + } + table, body := m[1], m[2] + parts := splitTopLevelComma(body) + var columns []string + var indexes []string + for _, part := range parts { + item := strings.TrimSpace(part) + item = strings.TrimSuffix(item, ",") + switch { + case inlineIndexRE.MatchString(item): + im := inlineIndexRE.FindStringSubmatch(item) + indexes = append(indexes, fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s %s;", im[1], table, im[2])) + case inlineKeyRE.MatchString(item): + im := inlineKeyRE.FindStringSubmatch(item) + indexes = append(indexes, fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s %s;", im[1], table, im[2])) + case inlineUniqueRE.MatchString(item): + im := inlineUniqueRE.FindStringSubmatch(item) + indexes = append(indexes, fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s %s;", im[1], table, im[2])) + default: + item = firstColumnRE.ReplaceAllString(item, "") + item = defaultUUIDColRE.ReplaceAllString(item, "") + columns = append(columns, item) + } + } + var b strings.Builder + fmt.Fprintf(&b, "CREATE TABLE IF NOT EXISTS %s (\n %s\n);", table, strings.Join(columns, ",\n ")) + if len(indexes) > 0 { + b.WriteString("\n") + b.WriteString(strings.Join(indexes, "\n")) + } + return b.String() + }) +} + +func splitTopLevelComma(s string) []string { + var parts []string + start, depth := 0, 0 + inSingle := false + for i, r := range s { + switch r { + case '\'': + inSingle = !inSingle + case '(': + if !inSingle { + depth++ + } + case ')': + if !inSingle && depth > 0 { + depth-- + } + case ',': + if !inSingle && depth == 0 { + parts = append(parts, s[start:i]) + start = i + 1 + } + } + } + parts = append(parts, s[start:]) + return parts +} + +const sqliteFinalDependenciesSchema = `DROP TABLE IF EXISTS dependencies; +CREATE TABLE dependencies ( + id CHAR(36) NOT NULL PRIMARY KEY, + issue_id VARCHAR(255) NOT NULL, + depends_on_issue_id VARCHAR(255) NULL, + depends_on_wisp_id VARCHAR(255) NULL, + depends_on_external VARCHAR(255) NULL, + type VARCHAR(32) NOT NULL DEFAULT 'blocks', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(255) NOT NULL, + metadata JSON DEFAULT '{}', + thread_id VARCHAR(255) DEFAULT '', + UNIQUE (issue_id, depends_on_issue_id), + UNIQUE (issue_id, depends_on_wisp_id), + UNIQUE (issue_id, depends_on_external), + CONSTRAINT ck_dep_one_target CHECK ((depends_on_issue_id IS NOT NULL) + (depends_on_wisp_id IS NOT NULL) + (depends_on_external IS NOT NULL) = 1) +); +CREATE INDEX IF NOT EXISTS idx_dep_type_issue ON dependencies (type, depends_on_issue_id); +CREATE INDEX IF NOT EXISTS idx_dep_type_wisp ON dependencies (type, depends_on_wisp_id); +CREATE INDEX IF NOT EXISTS idx_dep_type_external ON dependencies (type, depends_on_external); +CREATE INDEX IF NOT EXISTS idx_dep_wisp_target ON dependencies (depends_on_wisp_id); +CREATE INDEX IF NOT EXISTS idx_dep_issue_target ON dependencies (depends_on_issue_id); +CREATE INDEX IF NOT EXISTS idx_dep_external_target ON dependencies (depends_on_external);` diff --git a/internal/storage/sqlbuild/counts.go b/internal/storage/sqlbuild/counts.go index 3516be0bf..a3a0f010f 100644 --- a/internal/storage/sqlbuild/counts.go +++ b/internal/storage/sqlbuild/counts.go @@ -2,6 +2,13 @@ package sqlbuild import "fmt" +type CountsDialect int + +const ( + CountsDialectDolt CountsDialect = iota + CountsDialectSQLite +) + // ReadyWorkIssueColumns is IssueSelectColumns qualified with the "i." alias // used by the counts mega-query. var ReadyWorkIssueColumns = QualifyColumns(IssueSelectColumns, "i.") @@ -19,6 +26,30 @@ const DepJSONObject = `JSON_OBJECT( 'thread_id', thread_id )` +const depJSONObjectSQLite = `json_object( + 'issue_id', issue_id, + 'depends_on_id', COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external), + 'type', type, + 'created_at', strftime('%Y-%m-%dT%H:%M:%SZ', created_at), + 'created_by', created_by, + 'metadata', CAST(metadata AS TEXT), + 'thread_id', thread_id +)` + +func jsonArrayAgg(dialect CountsDialect, expr string) string { + if dialect == CountsDialectSQLite { + return fmt.Sprintf("json_group_array(%s)", expr) + } + return fmt.Sprintf("JSON_ARRAYAGG(%s)", expr) +} + +func depJSONObject(dialect CountsDialect) string { + if dialect == CountsDialectSQLite { + return depJSONObjectSQLite + } + return DepJSONObject +} + // SearchCountsSQL renders the counts mega-query: full issue rows aliased "i" // plus labels JSON, dep/rdep/comment counts, parent ID, and dependency JSON, // for one table family. whereSQL/orderBySQL/limitSQL may be empty; the @@ -29,6 +60,12 @@ const DepJSONObject = `JSON_OBJECT( // IssueSelectColumns positionally followed by the six extra columns in the // order projected here. func SearchCountsSQL(tables FilterTables, whereSQL, orderBySQL, limitSQL string, includeWispReverseDeps, skipLabels bool) string { + return SearchCountsSQLDialect(tables, whereSQL, orderBySQL, limitSQL, includeWispReverseDeps, skipLabels, CountsDialectDolt) +} + +// SearchCountsSQLDialect is SearchCountsSQL with backend-specific JSON/date +// functions for the count projection fragments. +func SearchCountsSQLDialect(tables FilterTables, whereSQL, orderBySQL, limitSQL string, includeWispReverseDeps, skipLabels bool, dialect CountsDialect) string { reverseBlockerSelect := ` SELECT COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external) AS dep_id FROM dependencies WHERE type = 'blocks' @@ -44,10 +81,10 @@ func SearchCountsSQL(tables FilterTables, whereSQL, orderBySQL, limitSQL string, labelsSelect := "l.labels_json AS labels_json" labelsJoin := fmt.Sprintf(` LEFT JOIN ( - SELECT issue_id, JSON_ARRAYAGG(label) AS labels_json + SELECT issue_id, %s AS labels_json FROM %s GROUP BY issue_id - ) l ON l.issue_id = i.id`, tables.Labels) + ) l ON l.issue_id = i.id`, jsonArrayAgg(dialect, "label"), tables.Labels) if skipLabels { labelsSelect = "NULL AS labels_json" labelsJoin = "" @@ -87,7 +124,7 @@ func SearchCountsSQL(tables FilterTables, whereSQL, orderBySQL, limitSQL string, GROUP BY issue_id ) pc ON pc.issue_id = i.id LEFT JOIN ( - SELECT issue_id, JSON_ARRAYAGG(%s) AS deps_json + SELECT issue_id, %s AS deps_json FROM %s GROUP BY issue_id ) d ON d.issue_id = i.id @@ -103,7 +140,7 @@ func SearchCountsSQL(tables FilterTables, whereSQL, orderBySQL, limitSQL string, reverseBlockerSelect, tables.Comments, tables.Dependencies, - DepJSONObject, + jsonArrayAgg(dialect, depJSONObject(dialect)), tables.Dependencies, whereSQL, orderBySQL, diff --git a/internal/storage/sqlbuild/filter.go b/internal/storage/sqlbuild/filter.go index c68e22221..fe0e7ffb2 100644 --- a/internal/storage/sqlbuild/filter.go +++ b/internal/storage/sqlbuild/filter.go @@ -14,6 +14,12 @@ import ( // string and IssueFilter. The tables parameter controls which table names are // referenced in subqueries (issues vs wisps). func BuildIssueFilterClauses(query string, filter types.IssueFilter, tables FilterTables) ([]string, []any, error) { + return BuildIssueFilterClausesDialect(query, filter, tables, CountsDialectDolt) +} + +// BuildIssueFilterClausesDialect builds WHERE clause fragments and args using +// backend-specific SQL for JSON metadata predicates. +func BuildIssueFilterClausesDialect(query string, filter types.IssueFilter, tables FilterTables, dialect CountsDialect) ([]string, []any, error) { var whereClauses []string var args []any @@ -121,7 +127,7 @@ func BuildIssueFilterClauses(query string, filter types.IssueFilter, tables Filt if filter.ParentID != nil { parentID := *filter.ParentID - whereClauses = append(whereClauses, fmt.Sprintf("(id IN (SELECT issue_id FROM %s WHERE type = 'parent-child' AND %s = ?) OR (id LIKE CONCAT(?, '.%%') AND id NOT IN (SELECT issue_id FROM %s WHERE type = 'parent-child')))", tables.Dependencies, DepTargetExpr, tables.Dependencies)) + whereClauses = append(whereClauses, fmt.Sprintf("(id IN (SELECT issue_id FROM %s WHERE type = 'parent-child' AND %s = ?) OR (%s AND id NOT IN (SELECT issue_id FROM %s WHERE type = 'parent-child')))", tables.Dependencies, DepTargetExpr, ReadyWorkChildIDLikeExpr(dialect), tables.Dependencies)) args = append(args, parentID, parentID) } if filter.NoParent { @@ -229,7 +235,7 @@ func BuildIssueFilterClauses(query string, filter types.IssueFilter, tables Filt } var err error - whereClauses, args, err = AppendMetadataClauses(whereClauses, args, filter.HasMetadataKey, filter.MetadataFields) + whereClauses, args, err = AppendMetadataClausesDialect(whereClauses, args, filter.HasMetadataKey, filter.MetadataFields, dialect) if err != nil { return nil, nil, err } @@ -240,11 +246,24 @@ func BuildIssueFilterClauses(query string, filter types.IssueFilter, tables Filt // AppendMetadataClauses appends JSON metadata predicates (has-key and exact // field matches, keys in sorted order) to an existing clause/arg list. func AppendMetadataClauses(where []string, args []any, hasKey string, fields map[string]string) ([]string, []any, error) { + return AppendMetadataClausesDialect(where, args, hasKey, fields, CountsDialectDolt) +} + +// AppendMetadataClausesDialect appends backend-specific JSON metadata +// predicates (has-key and exact field matches, keys in sorted order) to an +// existing clause/arg list. +func AppendMetadataClausesDialect(where []string, args []any, hasKey string, fields map[string]string, dialect CountsDialect) ([]string, []any, error) { + jsonExtract := "JSON_EXTRACT" + exactMatch := "JSON_UNQUOTE(JSON_EXTRACT(metadata, ?)) = ?" + if dialect == CountsDialectSQLite { + jsonExtract = "json_extract" + exactMatch = "json_extract(metadata, ?) = ?" + } if hasKey != "" { if err := storage.ValidateMetadataKey(hasKey); err != nil { return nil, nil, err } - where = append(where, "JSON_EXTRACT(metadata, ?) IS NOT NULL") + where = append(where, fmt.Sprintf("%s(metadata, ?) IS NOT NULL", jsonExtract)) args = append(args, storage.JSONMetadataPath(hasKey)) } if len(fields) > 0 { @@ -257,7 +276,7 @@ func AppendMetadataClauses(where []string, args []any, hasKey string, fields map if err := storage.ValidateMetadataKey(k); err != nil { return nil, nil, err } - where = append(where, "JSON_UNQUOTE(JSON_EXTRACT(metadata, ?)) = ?") + where = append(where, exactMatch) args = append(args, storage.JSONMetadataPath(k), fields[k]) } } diff --git a/internal/storage/sqlbuild/ready.go b/internal/storage/sqlbuild/ready.go index 08a5b7036..30e838d90 100644 --- a/internal/storage/sqlbuild/ready.go +++ b/internal/storage/sqlbuild/ready.go @@ -72,6 +72,9 @@ func BuildReadyWorkOrder(policy types.SortPolicy, createdCol, priorityCol string // clause folds in. Computing them takes queries, which is execution-context // work each stack does its own way. type ReadyWorkWhereInputs struct { + // Dialect controls backend-specific SQL functions. The zero value keeps the + // Dolt/MySQL behavior for existing callers. + Dialect CountsDialect // DeferredChildIDs are children of future-deferred parents; consulted // only when !filter.IncludeDeferred. DeferredChildIDs []string @@ -84,6 +87,7 @@ type ReadyWorkWhereInputs struct { // family. Both stacks must keep ready semantics identical (Seam A parity // suite); all ready predicates live here. func BuildReadyWorkWhere(filter types.WorkFilter, tables FilterTables, in ReadyWorkWhereInputs) (string, []any, error) { + nowSQL := ReadyWorkCurrentTimestamp(in.Dialect) var statusClause string if filter.Status != "" { statusClause = "status = ?" @@ -123,7 +127,7 @@ func BuildReadyWorkWhere(filter types.WorkFilter, tables FilterTables, in ReadyW } if !filter.IncludeDeferred { - whereClauses = append(whereClauses, "(defer_until IS NULL OR defer_until <= UTC_TIMESTAMP())") + whereClauses = append(whereClauses, fmt.Sprintf("(defer_until IS NULL OR defer_until <= %s)", nowSQL)) for start := 0; start < len(in.DeferredChildIDs); start += QueryBatchSize { end := start + QueryBatchSize if end > len(in.DeferredChildIDs) { @@ -155,7 +159,7 @@ func BuildReadyWorkWhere(filter types.WorkFilter, tables FilterTables, in ReadyW // help text and WorkFilter.ParentID godoc both promising recursion. if filter.ParentID != nil { parentID := *filter.ParentID - parentClauses := []string{fmt.Sprintf("(id LIKE CONCAT(?, '.%%') AND id NOT IN (SELECT issue_id FROM %s WHERE type = 'parent-child'))", tables.Dependencies)} + parentClauses := []string{fmt.Sprintf("(%s AND id NOT IN (SELECT issue_id FROM %s WHERE type = 'parent-child'))", ReadyWorkChildIDLikeExpr(in.Dialect), tables.Dependencies)} args = append(args, parentID) for start := 0; start < len(in.ParentDescendantIDs); start += QueryBatchSize { end := start + QueryBatchSize @@ -170,15 +174,34 @@ func BuildReadyWorkWhere(filter types.WorkFilter, tables FilterTables, in ReadyW } if filter.MoleculeID != "" { - whereClauses = append(whereClauses, fmt.Sprintf("(id IN (SELECT issue_id FROM %s WHERE type = 'parent-child' AND %s = ?) OR (id LIKE CONCAT(?, '.%%') AND id NOT IN (SELECT issue_id FROM %s WHERE type = 'parent-child')))", tables.Dependencies, DepTargetExpr, tables.Dependencies)) + whereClauses = append(whereClauses, fmt.Sprintf("(id IN (SELECT issue_id FROM %s WHERE type = 'parent-child' AND %s = ?) OR (%s AND id NOT IN (SELECT issue_id FROM %s WHERE type = 'parent-child')))", tables.Dependencies, DepTargetExpr, ReadyWorkChildIDLikeExpr(in.Dialect), tables.Dependencies)) args = append(args, filter.MoleculeID, filter.MoleculeID) } var err error - whereClauses, args, err = AppendMetadataClauses(whereClauses, args, filter.HasMetadataKey, filter.MetadataFields) + whereClauses, args, err = AppendMetadataClausesDialect(whereClauses, args, filter.HasMetadataKey, filter.MetadataFields, in.Dialect) if err != nil { return "", nil, err } return "WHERE " + strings.Join(whereClauses, " AND "), args, nil } + +// ReadyWorkCurrentTimestamp returns the backend-specific SQL expression for +// comparing ready-work defer timestamps against the current UTC time. +func ReadyWorkCurrentTimestamp(dialect CountsDialect) string { + if dialect == CountsDialectSQLite { + return "CURRENT_TIMESTAMP" + } + return "UTC_TIMESTAMP()" +} + +// ReadyWorkChildIDLikeExpr returns the backend-specific hierarchical child ID +// fallback predicate. The caller supplies the parent ID as the expression's +// single placeholder argument. +func ReadyWorkChildIDLikeExpr(dialect CountsDialect) string { + if dialect == CountsDialectSQLite { + return "id LIKE (? || '.%')" + } + return "id LIKE CONCAT(?, '.%')" +} diff --git a/internal/storage/sqlbuild/sqlbuild_test.go b/internal/storage/sqlbuild/sqlbuild_test.go index 729a8ed80..e09199ba1 100644 --- a/internal/storage/sqlbuild/sqlbuild_test.go +++ b/internal/storage/sqlbuild/sqlbuild_test.go @@ -116,6 +116,85 @@ func TestBuildReadyWorkWhereBatchesIDSets(t *testing.T) { } } +func TestBuildReadyWorkWhereUsesSQLiteClockDialect(t *testing.T) { + t.Parallel() + + where, _, err := BuildReadyWorkWhere(types.WorkFilter{}, IssuesFilterTables, ReadyWorkWhereInputs{ + Dialect: CountsDialectSQLite, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if strings.Contains(where, "UTC_TIMESTAMP()") { + t.Fatalf("SQLite ready WHERE must not use UTC_TIMESTAMP(): %s", where) + } + if !strings.Contains(where, "defer_until <= CURRENT_TIMESTAMP") { + t.Fatalf("SQLite ready WHERE missing CURRENT_TIMESTAMP defer predicate: %s", where) + } +} + +func TestBuildReadyWorkWhereUsesSQLiteMetadataAndChildIDDialect(t *testing.T) { + t.Parallel() + + parentID := "bd-parent" + where, _, err := BuildReadyWorkWhere(types.WorkFilter{ + ParentID: &parentID, + MoleculeID: "bd-mol", + HasMetadataKey: "gc.routed_to", + MetadataFields: map[string]string{ + "gc.routed_to": "beads-doltlite/gc.implementation-reviewer", + }, + }, IssuesFilterTables, ReadyWorkWhereInputs{ + Dialect: CountsDialectSQLite, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for _, bad := range []string{"JSON_UNQUOTE", "JSON_EXTRACT", "CONCAT("} { + if strings.Contains(where, bad) { + t.Fatalf("SQLite ready WHERE must not use %s: %s", bad, where) + } + } + for _, want := range []string{ + "json_extract(metadata, ?) IS NOT NULL", + "json_extract(metadata, ?) = ?", + "id LIKE (? || '.%')", + } { + if !strings.Contains(where, want) { + t.Fatalf("SQLite ready WHERE missing %q in %s", want, where) + } + } +} + +func TestBuildIssueFilterClausesUsesSQLiteMetadataDialect(t *testing.T) { + t.Parallel() + + clauses, args, err := BuildIssueFilterClausesDialect("", types.IssueFilter{ + HasMetadataKey: "gc.routed_to", + MetadataFields: map[string]string{ + "gc.routed_to": "beads-doltlite/gc.implementation-reviewer", + }, + }, IssuesFilterTables, CountsDialectSQLite) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + sql := strings.Join(clauses, " AND ") + if strings.Contains(sql, "JSON_UNQUOTE") || strings.Contains(sql, "JSON_EXTRACT") { + t.Fatalf("SQLite metadata predicates must not use MySQL JSON functions: %s", sql) + } + for _, want := range []string{ + "json_extract(metadata, ?) IS NOT NULL", + "json_extract(metadata, ?) = ?", + } { + if !strings.Contains(sql, want) { + t.Fatalf("SQLite metadata predicates missing %q in %s", want, sql) + } + } + if got, want := len(args), 3; got != want { + t.Fatalf("args = %d, want %d", got, want) + } +} + func TestSearchCountsSQLShape(t *testing.T) { t.Parallel() diff --git a/internal/storage/transaction_error.go b/internal/storage/transaction_error.go new file mode 100644 index 000000000..cecb288f7 --- /dev/null +++ b/internal/storage/transaction_error.go @@ -0,0 +1,48 @@ +package storage + +import ( + "errors" + "fmt" +) + +// PostTransactionCommitError reports a failure that happened after the SQL +// transaction body committed, while creating the backend version-control commit. +type PostTransactionCommitError struct { + CommitMessage string + Err error +} + +func NewPostTransactionCommitError(commitMessage string, err error) error { + if err == nil { + return nil + } + return &PostTransactionCommitError{ + CommitMessage: commitMessage, + Err: err, + } +} + +func (e *PostTransactionCommitError) Error() string { + if e == nil || e.Err == nil { + return "post-transaction commit failed" + } + if e.CommitMessage == "" { + return fmt.Sprintf("post-transaction commit failed: %v", e.Err) + } + return fmt.Sprintf("post-transaction commit %q failed: %v", e.CommitMessage, e.Err) +} + +func (e *PostTransactionCommitError) Unwrap() error { + if e == nil { + return nil + } + return e.Err +} + +func AsPostTransactionCommitError(err error) (*PostTransactionCommitError, bool) { + var target *PostTransactionCommitError + if errors.As(err, &target) { + return target, true + } + return nil, false +} diff --git a/internal/storage/transaction_error_test.go b/internal/storage/transaction_error_test.go new file mode 100644 index 000000000..3402db329 --- /dev/null +++ b/internal/storage/transaction_error_test.go @@ -0,0 +1,22 @@ +package storage + +import ( + "errors" + "testing" +) + +func TestPostTransactionCommitErrorWrapsCause(t *testing.T) { + cause := errors.New("doltlite add dependencies: database is locked") + err := NewPostTransactionCommitError("bd: graph-apply 2 nodes", cause) + + postCommitErr, ok := AsPostTransactionCommitError(err) + if !ok { + t.Fatalf("AsPostTransactionCommitError did not match %T", err) + } + if postCommitErr.CommitMessage != "bd: graph-apply 2 nodes" { + t.Fatalf("CommitMessage = %q", postCommitErr.CommitMessage) + } + if !errors.Is(err, cause) { + t.Fatalf("wrapped error does not match cause") + } +} diff --git a/internal/types/types.go b/internal/types/types.go index c8b76dd27..8bcb3fc26 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1006,6 +1006,9 @@ const ( EventLabelAdded EventType = "label_added" EventLabelRemoved EventType = "label_removed" EventCompacted EventType = "compacted" + // EventLeaseReclaimed records that a stale lease was reverted to ready by + // bd reclaim (dead-worker recovery). old_value is the previous owner. + EventLeaseReclaimed EventType = "lease_reclaimed" ) // BlockedIssue extends Issue with blocking information @@ -1320,6 +1323,15 @@ func (s SortPolicy) IsValid() bool { return false } +// ReclaimedLease names an issue whose stale lease was reverted to ready by +// bd reclaim, together with the owner the lease was taken from. Returned so +// callers (the CLI, a supervisor) can report which dead workers' work was +// recovered. +type ReclaimedLease struct { + ID string + PreviousOwner string +} + // WorkFilter is used to filter ready work queries type WorkFilter struct { Status Status diff --git a/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/decomposition.md b/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/decomposition.md new file mode 100644 index 000000000..ee1b461a3 --- /dev/null +++ b/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/decomposition.md @@ -0,0 +1,120 @@ +# Decomposition: DoltLite Controller/Formula Query-Write Audit + +## Source + +- Source bead: `bd-5tj` - Critically audit controller/formula DoltLite query-write coverage. +- Plan artifact: `plans/controller-formula-doltlite-query-write-audit-beads-doltlite/plan.md`. +- Implementation convoy: `bd-7po`. +- This is a document-only decomposition for the jj-managed build workflow. The runnable work units are Beads tasks tracked by `bd-7po`. + +## Goal + +Produce a concrete audit matrix proving every controller/formula read and write path used by Gas City formula-driven work under the beads-doltlite backend. Each operation must have either a named proof or a named gap with the exact test file/function and command needed to close it. + +## Work Units + +### `bd-0r3` - Inventory controller and formula DoltLite query/write operations + +Build the exact operation inventory for the audit matrix. + +Scope: + +- Owners: gascity core, gascity-jj-base pack, gc pack, gastown helpers, and beads-doltlite. +- Lifecycle steps: formula materialization, route stamping, controller desired-state build, pool scale demand, session start/restart/resume, worker hook claim, continuation sibling assignment, drain acknowledgement, finalization, teardown, recovery, and document/workspace metadata operations. +- Matrix fields: owner, lifecycle step, exact operation, predicate or mutation fields, proof mode, and initial result. + +Acceptance: + +- Every controller/formula read and write path has a matrix row. +- Rows name exact commands, jq filters, Go store calls, DoltLite SQL queries, or write mutations. +- Rows cover status, assignee, issue_type, is_blocked, dependencies, `gc.routed_to`, `gc.run_target`, `gc.kind`, `gc.session_affinity`, `gc.root_bead_id`, `gc.root_store_ref`, `gc.continuation_group`, drain metadata, and document/workspace metadata. + +### `bd-8x0` - Build DoltLite parity proof fixture for audit operations + +Create or extend the DoltLite-backed fixture and proof harness used by the audit. + +Scope: + +- Compare each named read path against both `bd` and `doltlite-client query`. +- Prove writes are accepted by beads-doltlite, persisted in DoltLite, and visible to the next lifecycle query. +- Include ready, blocked, assigned, unassigned routed, control-dispatcher, drain/no-work, restart/resume, and continuation-group states. + +Acceptance: + +- Fixture or harness can reproduce the operation matrix states. +- Each read path has a bd-vs-doltlite-client comparison or a named gap. +- Each write path has persistence and next-query visibility evidence or a named gap. + +### `bd-9xd` - Prove formula lifecycle writes and route qualification behavior + +Audit formula-created work and lifecycle metadata writes end to end. + +Scope: + +- Formula-created beads and continuation siblings. +- Route stamping and target qualification behavior. +- Metadata fields: `gc.routed_to`, `gc.run_target`, `gc.kind`, `gc.session_affinity`, `gc.root_bead_id`, `gc.root_store_ref`, `gc.continuation_group`, drain metadata, and document/workspace metadata. +- Fully qualified route examples such as `lightjj/gc.requirements-planner`. +- Short `gc.run_target` values only where the formula or controller intentionally consumes short names. + +Acceptance: + +- Formula-created work is covered, not only manually created beads. +- Fully qualified routes and intentional short targets are explicitly distinguished. +- Drain acknowledgement, finalization, teardown, and recovery writes have proof or named gaps. + +### `bd-b26` - Audit pack shell-outs and helper scripts that call bd + +Inspect shell-based formula and helper integration points. + +Scope: + +- gascity core packs. +- gascity-jj-base. +- gc pack. +- gastown helpers. +- beads-doltlite scripts involved in formula-driven controller work. + +Acceptance: + +- Every pack/helper `bd` shell-out used by formula workflows is represented in the matrix. +- Rows include exact command shape, filters, expected fields, and proof mode. +- Missing coverage identifies the proposed test file/function and exact validation command. + +### `bd-iz0` - Publish DoltLite controller/formula query-write audit report + +Consolidate the final audit deliverable after the proof-gathering beads complete. + +Dependencies: + +- Blocks on `bd-0r3`, `bd-8x0`, `bd-9xd`, and `bd-b26`. + +Acceptance: + +- Final report contains the full audit matrix. +- Every controller/formula operation has named proof or a named test gap. +- Each missing proof includes proposed test file/function and exact command with required tags/libs. +- Each behavioral divergence has a linked implementation bead. +- The report classifies the ready-work issue as fast-path visibility, query construction, route qualification, scale/reconciler demand, session materialization, or another evidenced cause. + +## Dependency Shape + +- `bd-7po` tracks all runnable work units: `bd-0r3`, `bd-8x0`, `bd-9xd`, `bd-b26`, and `bd-iz0`. +- `bd-iz0` depends on the four proof-gathering beads. +- The implementation drain should use `bd-7po`, not the launch/source convoy `bd-vva`. + +## Validation Plan + +Expected local validation commands for implementation workers: + +```bash +make test +CGO_ENABLED=1 go test -tags gms_pure_go ./internal/storage/doltlite ./internal/storage/issueops ./internal/storage/sqlbuild ./cmd/bd/... +``` + +Audit-specific proof commands should be recorded in the final report next to each matrix row. DoltLite parity rows must include both the `bd` command and the equivalent `doltlite-client query` or `doltlite-client exec` command. + +## Notes And Ambiguity + +- The workflow root metadata referenced a requirements artifact change ID that was not resolvable in the current jj workspace during decomposition. The current checkout did contain the implementation plan and manifest, so this decomposition is based on `plan.md` plus the source bead `bd-5tj`. +- The decomposition intentionally creates audit/report tasks, not direct source-change tasks, because the approved source bead asks for a critical audit report with proof classification and follow-up beads for any divergences. diff --git a/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/implementation-summary.md b/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/implementation-summary.md new file mode 100644 index 000000000..ec14ccccb --- /dev/null +++ b/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/implementation-summary.md @@ -0,0 +1,72 @@ +--- +schema: gc.build.implementation-summary.v1 +workflow_root: bd-uwa +generated_at: 2026-06-28T08:13:11Z +--- + +# Implementation Summary: DoltLite Controller/Formula Query-Write Audit + +## Summary + +The jj-build workflow prepared the document workspace and planning artifacts for +the DoltLite controller/formula query-write audit. The implementation convoy is +tracked as `bd-7po`; its planned work items are still open and should be treated +as implementation work remaining after this build summary. + +## Source Identity + +- Source workspace: missing +- Source workspace path: missing +- Latest source change ID: missing +- Source bead: `bd-5tj` + +No manifest entry or bead metadata provided `gc.docs.source_workspace`, +`gc.docs.source_workspace_path`, or `gc.docs.source_change_id`. Downstream +review must not use document workspace change IDs as source state. The latest +document workspace change for this summary is recorded separately in the +manifest and root metadata. + +## Document Workspace + +- Document workspace: `default` +- Base revset: `default@` +- Artifact root: + `/data/projects/doltlite-gascity/beads-doltlite/plans/controller-formula-doltlite-query-write-audit-beads-doltlite` +- Manifest: + `/data/projects/doltlite-gascity/beads-doltlite/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/manifest.json` + +## Produced Documents + +| Document | Schema | Status | +| --- | --- | --- | +| Plan | `gc.build.plan.v1` | Present in manifest | +| Decomposition | `gc.build.decomposition.v1` | Present in manifest | +| Review | `gc.build.review.v1` | Present in manifest | +| Implementation summary | `gc.build.implementation-summary.v1` | Present in manifest | + +## Implementation Convoy + +The implementation convoy `bd-7po` is open. The planned work items are: + +- `bd-0r3` - Inventory controller and formula DoltLite query/write operations +- `bd-8x0` - Build DoltLite parity proof fixture for audit operations +- `bd-9xd` - Prove formula lifecycle writes and route qualification behavior +- `bd-b26` - Audit pack shell-outs and helper scripts that call bd +- `bd-iz0` - Publish DoltLite controller/formula query-write audit report + +## Validation + +The workflow root records validation: + +```text +python3 -m pytest tests/test_gascity_jj_base_pack.py (31 passed) +``` + +The root also records changed file metadata for +`gascity-packs/gascity-jj-base/tests/test_gascity_jj_base_pack.py`. + +## Follow-Up + +Implementation workers should complete the open convoy items and record actual +source workspace identity before any downstream review attempts to inspect +source changes. diff --git a/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/manifest.json b/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/manifest.json new file mode 100644 index 000000000..327dd1d4c --- /dev/null +++ b/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/manifest.json @@ -0,0 +1,30 @@ +{ + "artifact_root": "/data/projects/doltlite-gascity/beads-doltlite/plans/controller-formula-doltlite-query-write-audit-beads-doltlite", + "documents": { + "decomposition": { + "path": "/data/projects/doltlite-gascity/beads-doltlite/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/decomposition.md", + "schema": "gc.build.decomposition.v1", + "hash": "sha256:302a21e226b3fd9fff0747137fbcb94179f4378755af1d9b7e104b3e026126e9", + "change_id": "rqnqrpvsklwsmvouqoqmnummsllwqzkk" + }, + "plan": { + "path": "/data/projects/doltlite-gascity/beads-doltlite/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/plan.md", + "schema": "gc.build.plan.v1", + "hash": "sha256:ae1730700c21310d043881cbfafd2594112346db0b0b829449be05c68aba4aa4", + "change_id": "vmxnysrwuokytlyssrmruokymwomyvou" + }, + "review": { + "path": "/data/projects/doltlite-gascity/beads-doltlite/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/review.md", + "schema": "gc.build.review.v1", + "hash": "sha256:bdfcbe6ba946a077643915c5c45aab0fea502cd74ce7bf89c896afc7fe2ff974", + "change_id": "rqnqrpvsklwsmvouqoqmnummsllwqzkk" + }, + "implementation-summary": { + "path": "/data/projects/doltlite-gascity/beads-doltlite/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/implementation-summary.md", + "schema": "gc.build.implementation-summary.v1", + "hash": "sha256:46fd881d8a6186e21ec71de4b36a24b6caf1bbf63ed9d2533d7f1a473589fa7e", + "change_id": "kzytryvkzrmlozurywvtzpxwyxzzznyq" + } + }, + "latest_change_id": "kzytryvkzrmlozurywvtzpxwyxzzznyq" +} diff --git a/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/plan.md b/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/plan.md new file mode 100644 index 000000000..9275f966c --- /dev/null +++ b/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/plan.md @@ -0,0 +1,135 @@ +# Implementation Plan: DoltLite Query/Write Audit + +## Summary + +Audit and harden the DoltLite backend paths used by controller-formula and +Gas City workers so read queries stay compatible with DoltLite's +SQLite-compatible SQL surface and write operations consistently update derived +state, dirty-table tracking, and Dolt version history. + +The implementation should keep Beads inside the existing storage boundary: +Beads routes behavior through `internal/storage` interfaces and shared +`issueops` helpers. DoltLite-specific handling belongs in the DoltLite storage +adapter only when it is about connection lifecycle, SQLite-compatible SQL, native +DoltLite commits, or local schema compatibility. + +## Current System + +- `internal/storage/doltlite` implements the CGO-only DoltLite store. +- `DoltliteStore.withConn` opens a transaction around each operation and now + uses a bounded exclusive lock/retry path for write transactions. +- `RunInTransaction` passes an `embeddedTransaction` that marks dirty logical + tables and commits DoltLite history after successful SQL transaction commit. +- Shared issue mutation logic lives under `internal/storage/issueops`. +- Several issueops paths already have SQLite-compatible variants, including + update, close, delete, dependency add/remove, ready/search/count, and blocked + recomputation helpers. +- Existing Dolt/server SQL paths must remain available for the embedded Dolt + and server-backed storage adapters. + +Important constraints: + +- Do not add orchestration policy to Beads core. +- Do not leak DoltLite internals through public storage return types. +- Do not add Beads-side crash recovery or engine introspection outside the + storage interface. +- Do not rely on a Dolt SQL server, runtime port, or server-state file for + DoltLite behavior. +- Keep maintenance bounded and non-fatal unless correctness requires a hard + failure. + +## Proposed Implementation + +1. Route DoltLite read/write methods through SQLite-compatible issueops helpers. + + Update DoltLite store methods so every query or mutation that touches issue, + dependency, ready, count, search, close, update, delete, or wisp-derived state + uses the helper variant that is valid against DoltLite's SQLite-compatible + engine. Keep generic issueops behavior shared where it is SQL-neutral. + +2. Preserve derived-state correctness for all write paths. + + Ensure each DoltLite mutation recomputes or marks `is_blocked` through the + SQLite-compatible helpers. Cover these mutation classes: + + - issue status changes that can unblock or block dependents + - dependency add/remove, including wisp and cross-prefix targets + - issue delete, including cascade and forced orphaning paths + - close/reopen/update flows that write event history + +3. Maintain DirtyTableTracker coverage inside `embeddedTransaction`. + + For transaction-scoped writes, mark every logical table that can be touched by + the operation. This includes direct tables such as `issues`, `dependencies`, + `events`, `labels`, and their wisp counterparts when a shared helper can route + between regular issues and wisps. Do not overfit dirty tracking to a single + branch of wisp routing when the helper may touch both table families. + +4. Keep DoltLite transaction behavior bounded and retryable. + + Keep write transactions behind `withExclusiveLock` and the bounded retry + wrapper. Refresh the persistent connection after retryable concurrency errors + so stale prepared/catalog state does not poison subsequent attempts. Reads + should avoid the exclusive lock path. + +5. Repair local DoltLite schema compatibility during open. + + During DoltLite store initialization, run the SQLite migration path for fresh + or existing stores, commit native schema changes when needed, and repair local + DoltLite-only table shape where safe. For legacy local wisp dependency tables, + only perform automatic shape repair when the table is empty; otherwise fail + with an explicit error instead of silently rewriting user data. + +6. Keep the shared SQL-builder boundary clean. + + Put SQL expressions that must be shared between issueops and query builders + in `internal/storage/sqlbuild`. Keep DoltLite-specific branch decisions in the + adapter and keep storage-neutral dependency target logic in issueops. + +## Testing + +Required tests: + +- Unit tests for SQLite-compatible issueops helpers covering update, delete, + dependency add/remove, and status-change derived-state recomputation. +- DoltLite smoke or integration tests proving an existing store opens, + migrates, and can run representative ready/search/count/query flows. +- Tests for `wisp_dependencies` compatibility repair: + - no-op when the split target columns already exist + - empty legacy table is repaired + - non-empty legacy table fails with a clear error +- Tests that DoltLite writes create native commits only when tracked dirty tables + changed. +- Regression tests for concurrent write retry behavior around lock/catalog + errors where practical without making the suite flaky. + +Validation commands: + +```bash +make test +CGO_ENABLED=1 go test -tags gms_pure_go ./internal/storage/doltlite ./internal/storage/issueops ./internal/storage/sqlbuild ./cmd/bd/... +``` + +The CGO command is intentionally scoped to the affected DoltLite and CLI storage +surface. Broader shipped-config CGO runs can be added by maintainers if the +review uncovers cross-package risk. + +## Rollout + +- Land behind the existing DoltLite backend selection; no new CLI or config + surface is required. +- Keep behavior compatible for existing `.beads/doltlite/*.db` stores by + running compatible migrations at open. +- Fail fast on unsafe local schema repair and report the table/shape that needs + manual intervention. +- Do not change the default non-DoltLite storage behavior. + +## Open Questions + +- Should the DoltLite smoke tests create a fixture database at an older schema + version, or is constructing the legacy table shape in-test sufficient? +- Should direct `bd sql` coverage be added for the same ready/search/count cases + to catch query-builder drift separately from typed store methods? +- Is the dirty-table set for transaction-scoped import/create operations + intentionally broad, or should a follow-up narrow it after wisp routing is + made explicit in the helper return values? diff --git a/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/review.md b/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/review.md new file mode 100644 index 000000000..415a71426 --- /dev/null +++ b/plans/controller-formula-doltlite-query-write-audit-beads-doltlite/review.md @@ -0,0 +1,22 @@ +# JJ Review Report + +Verdict: changes_required + +Reviewed change: `rqnqrpvsklwsmvouqoqmnummsllwqzkk` (`decomposition: record DoltLite audit work breakdown`) + +## Findings + +### P1: Ordinary DoltLite package tests now silently skip + +The new package-level `TestMain` in `internal/storage/doltlite/native_test.go:16` exits with status 0 when `dolt_version()` is unavailable (`internal/storage/doltlite/native_test.go:17-20`). Because this repository's `Makefile` exports `CGO_ENABLED=1` globally (`Makefile:33`), `native_test.go` is included in ordinary runs such as `go test ./internal/storage/doltlite`, not only in the new `make test-doltlite` target. + +On this host, `go test -count=1 -v ./internal/storage/doltlite` now prints the native-link skip message and exits `ok` without running the existing smoke tests in `smoke_test.go`. That creates a false green for the whole DoltLite package anywhere libdoltlite is not linked into the default sqlite driver, and it reduces coverage for the non-native path while presenting the package as passing. + +The native-link probe should be isolated from the ordinary package suite, for example behind a dedicated build tag used only by `make test-doltlite`, or converted into an explicit test that calls `t.Skip` without a package-wide `os.Exit(0)` gate. The existing smoke tests should continue to fail or pass on their own merits under normal test commands. + +## Validation + +- `go test -count=1 -v ./internal/storage/doltlite` +- Result: exited `ok` after printing `SKIP internal/storage/doltlite: libdoltlite SQL functions are not linked into the sqlite driver: no such function: dolt_version`; no individual smoke tests ran. +- `make test-doltlite` +- Result: passed in 15.172s using the default linked library path. diff --git a/plans/default-line-audit-20260623/review.md b/plans/default-line-audit-20260623/review.md new file mode 100644 index 000000000..ac4e49db4 --- /dev/null +++ b/plans/default-line-audit-20260623/review.md @@ -0,0 +1,21 @@ +# JJ Review Report + +Verdict: changes_required + +Reviewed change: `nwzzkuqzmnynzkzpmppszqnumpnrurqm` (`fix(doltlite): run compatible migrations for existing stores`) + +Note: the bead metadata named source change `owrmnolqmwrxtokxnzyvzlomxmvmqusz`, but that revision was not present in this workspace. The document workspace is an empty review commit whose parent is the change above, so this report reviews that concrete parent change. + +## Findings + +### P1: SQLite upgrade path drops existing dependency rows + +`MigrateSQLiteUpTo` executes the SQLite-compatible body for each pending migration on existing stores (`internal/storage/schema/sqlite_migrations.go:105-114`). For both `0041_split_dependencies_target.up.sql` and `0043_drop_dependencies_generated_column.up.sql`, `sqliteCompatibleMigrationSQL` returns `sqliteFinalDependenciesSchema` (`internal/storage/schema/sqlite_migrations.go:198-199`), and that schema starts with `DROP TABLE IF EXISTS dependencies` (`internal/storage/schema/sqlite_migrations.go:304`). + +That means any existing DoltLite store upgrading from before migration 0041, or from 0041/0042 into 0043, loses all dependency records during startup. This breaks the change's stated goal of making existing stores compatible and can silently unblock or detach beads from their blockers. The compatibility migration needs to preserve rows while reshaping the table, or use an idempotent copy/rename flow with a regression test that seeds dependencies before the migration and verifies they survive after `MigrateSQLiteUpTo`. + +## Validation + +- `go test ./internal/storage/schema ./internal/storage/sqlbuild ./internal/storage/doltlite` +- Result: schema and sqlbuild packages passed; `internal/storage/doltlite` failed in this environment because DoltLite SQL did not provide `dolt_commit` during migration commit. + diff --git a/website/docs/cli-reference/heartbeat.md b/website/docs/cli-reference/heartbeat.md new file mode 100644 index 000000000..754d84980 --- /dev/null +++ b/website/docs/cli-reference/heartbeat.md @@ -0,0 +1,34 @@ +--- +id: heartbeat +title: bd heartbeat +slug: /cli-reference/heartbeat +sidebar_position: 999 +--- + + +Generated from `bd help --doc heartbeat` + +## bd heartbeat + +Refresh the lease on an issue you currently hold in_progress. + +A claim carries a lease that expires after a TTL. A worker keeps its claim alive +by heartbeating faster than the TTL; once it stops (because it died), the lease +goes stale and 'bd reclaim' reverts the issue to ready so another worker can pick +it up. Heartbeat pushes lease_expires_at forward and stamps heartbeat_at = now. + +Only the current owner may heartbeat. If the lease has already been reclaimed or +the issue closed, heartbeat fails so the worker learns to stop. + +Heartbeat writes a Dolt commit, so heartbeat well below the TTL but not so fast +it bloats history — cadence should be a small fraction of the TTL, not per-op. + +Examples: + bd heartbeat bd-123 + bd hb bd-123 + +``` +bd heartbeat [flags] +``` + +**Aliases:** hb diff --git a/website/docs/cli-reference/index.md b/website/docs/cli-reference/index.md index 67d16fb6c..3a1a6043a 100644 --- a/website/docs/cli-reference/index.md +++ b/website/docs/cli-reference/index.md @@ -9,7 +9,7 @@ sidebar_position: 0 Reference for bd Latest. Generated from `bd help --docs-root`. -This reference covers all 108 live top-level `bd` commands. Regenerate it with: +This reference covers all 110 live top-level `bd` commands. Regenerate it with: ```bash ./scripts/generate-cli-docs.sh @@ -59,6 +59,7 @@ This reference covers all 108 live top-level `bd` commands. Regenerate it with: - [`bd github`](./github.md) - [`bd gitlab`](./gitlab.md) - [`bd graph`](./graph.md) +- [`bd heartbeat`](./heartbeat.md) - [`bd history`](./history.md) - [`bd hooks`](./hooks.md) - [`bd human`](./human.md) @@ -95,6 +96,7 @@ This reference covers all 108 live top-level `bd` commands. Regenerate it with: - [`bd quickstart`](./quickstart.md) - [`bd ready`](./ready.md) - [`bd recall`](./recall.md) +- [`bd reclaim`](./reclaim.md) - [`bd recompute-blocked`](./recompute-blocked.md) - [`bd remember`](./remember.md) - [`bd rename`](./rename.md) diff --git a/website/docs/cli-reference/init.md b/website/docs/cli-reference/init.md index 2bb48ddb1..c352b57bf 100644 --- a/website/docs/cli-reference/init.md +++ b/website/docs/cli-reference/init.md @@ -13,8 +13,9 @@ Generated from `bd help --doc init` Initialize bd in the current directory by creating a .beads/ directory and Dolt database. Optionally specify a custom issue prefix. -Dolt is the default (and only supported) storage backend. The legacy SQLite -backend has been removed. Use --backend=sqlite to see migration instructions. +Dolt is the default storage backend. The legacy SQLite backend has been +removed. Use --backend=sqlite to see migration instructions. Use +--backend=doltlite for the embedded DoltLite backend. Use --database to specify an existing server database name, overriding the default prefix-based naming. This is useful when an external tool (e.g. an orchestrator) @@ -54,7 +55,7 @@ bd init [flags] --agents-file string Custom filename for agent instructions (default: AGENTS.md) --agents-profile string AGENTS.md profile: 'minimal' (default, pointer to bd prime) or 'full' (complete command reference) --agents-template string Path to custom AGENTS.md template (overrides embedded default) - --backend string Storage backend (default: dolt). --backend=sqlite prints deprecation notice. + --backend string Storage backend (default: dolt; supported: dolt, doltlite). --backend=sqlite prints deprecation notice. --contributor Run OSS contributor setup wizard --database string Use existing server database name (overrides prefix-based naming) --debug Run the managed Dolt sql-server with --loglevel=debug and CPU profiling (--prof cpu). Persisted to config.yaml as dolt.debug. No effect on externally-managed servers. diff --git a/website/docs/cli-reference/list.md b/website/docs/cli-reference/list.md index b203b8cf4..566d856fb 100644 --- a/website/docs/cli-reference/list.md +++ b/website/docs/cli-reference/list.md @@ -38,6 +38,7 @@ bd list [flags] --format string Output format: 'digraph' (for golang.org/x/tools/cmd/digraph), 'dot' (Graphviz), or Go template --has-metadata-key string Filter issues that have this metadata key set --id string Filter by specific issue IDs (comma-separated, e.g., bd-1,bd-5,bd-10) + --include-ephemeral Include ephemeral issues (wisps) in results --include-gates Include gate issues in output (normally hidden) --include-infra Include infrastructure beads (agent/role/message) in output --include-templates Include template molecules in output diff --git a/website/docs/cli-reference/reclaim.md b/website/docs/cli-reference/reclaim.md new file mode 100644 index 000000000..0213842d4 --- /dev/null +++ b/website/docs/cli-reference/reclaim.md @@ -0,0 +1,40 @@ +--- +id: reclaim +title: bd reclaim +slug: /cli-reference/reclaim +sidebar_position: 999 +--- + + +Generated from `bd help --doc reclaim` + +## bd reclaim + +Revert in_progress issues whose lease has gone stale back to ready. + +When a worker claims an issue it takes a lease that expires after a TTL, kept +alive by 'bd heartbeat'. A worker that dies stops heartbeating, so its lease +expires and its issue would otherwise stay in_progress forever. reclaim is the +reaper: it finds in_progress issues whose lease expired more than --older-than +ago, clears the assignee, and sets them back to open so another worker can +claim them. The previous owner's stale lease is recorded as a recovery event. + +--older-than is a grace window past lease expiry: only leases that expired at +least this long ago are reclaimed, so a worker briefly paused (GC, clock skew) +is not robbed of live work. Run it from a supervisor on a timer with a window +of roughly 2× the claim TTL. + +Examples: + bd reclaim # default grace window (2× the lease TTL) + bd reclaim --older-than 10m # reclaim leases expired >10m ago + bd reclaim --older-than 0s # reclaim every currently-expired lease + +``` +bd reclaim [flags] +``` + +**Flags:** + +``` + --older-than duration Only reclaim leases that expired at least this long ago (grace window) (default 10m0s) +```