From 4b4de3adec84630fceaa68c4855d4414be95ef2b Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Wed, 29 Apr 2026 10:29:19 +1000 Subject: [PATCH 01/15] Add doltlite storage backend --- beads_cgo.go | 14 +- cmd/bd/store_factory.go | 10 +- go.mod | 1 + internal/beads/beads.go | 18 +- internal/storage/doltlite/cache_cleanup.go | 94 ++ internal/storage/doltlite/child_id.go | 20 + internal/storage/doltlite/config_metadata.go | 115 +++ internal/storage/doltlite/create_issue.go | 159 ++++ internal/storage/doltlite/dependencies.go | 72 ++ internal/storage/doltlite/federation.go | 334 +++++++ internal/storage/doltlite/flock.go | 114 +++ internal/storage/doltlite/flock_stub.go | 27 + internal/storage/doltlite/get_issue.go | 21 + internal/storage/doltlite/issues.go | 100 +++ internal/storage/doltlite/labels.go | 33 + internal/storage/doltlite/list_queries.go | 96 +++ internal/storage/doltlite/merge_slot.go | 33 + internal/storage/doltlite/open.go | 75 ++ internal/storage/doltlite/open_stub.go | 14 + internal/storage/doltlite/queries.go | 42 + internal/storage/doltlite/schema.go | 12 + internal/storage/doltlite/slots.go | 96 +++ internal/storage/doltlite/smoke_test.go | 119 +++ internal/storage/doltlite/statistics.go | 36 + internal/storage/doltlite/store.go | 860 +++++++++++++++++++ internal/storage/doltlite/store_stub.go | 28 + internal/storage/doltlite/transaction.go | 194 +++++ internal/storage/doltlite/version_control.go | 426 +++++++++ internal/storage/schema/helpers.go | 14 +- internal/storage/schema/sqlite.go | 162 ++++ 30 files changed, 3320 insertions(+), 19 deletions(-) create mode 100644 internal/storage/doltlite/cache_cleanup.go create mode 100644 internal/storage/doltlite/child_id.go create mode 100644 internal/storage/doltlite/config_metadata.go create mode 100644 internal/storage/doltlite/create_issue.go create mode 100644 internal/storage/doltlite/dependencies.go create mode 100644 internal/storage/doltlite/federation.go create mode 100644 internal/storage/doltlite/flock.go create mode 100644 internal/storage/doltlite/flock_stub.go create mode 100644 internal/storage/doltlite/get_issue.go create mode 100644 internal/storage/doltlite/issues.go create mode 100644 internal/storage/doltlite/labels.go create mode 100644 internal/storage/doltlite/list_queries.go create mode 100644 internal/storage/doltlite/merge_slot.go create mode 100644 internal/storage/doltlite/open.go create mode 100644 internal/storage/doltlite/open_stub.go create mode 100644 internal/storage/doltlite/queries.go create mode 100644 internal/storage/doltlite/schema.go create mode 100644 internal/storage/doltlite/slots.go create mode 100644 internal/storage/doltlite/smoke_test.go create mode 100644 internal/storage/doltlite/statistics.go create mode 100644 internal/storage/doltlite/store.go create mode 100644 internal/storage/doltlite/store_stub.go create mode 100644 internal/storage/doltlite/transaction.go create mode 100644 internal/storage/doltlite/version_control.go create mode 100644 internal/storage/schema/sqlite.go diff --git a/beads_cgo.go b/beads_cgo.go index d230c714e..e9b3c7683 100644 --- a/beads_cgo.go +++ b/beads_cgo.go @@ -4,10 +4,10 @@ package beads import ( "context" - "path/filepath" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage/dolt" + "github.com/steveyegge/beads/internal/storage/doltlite" "github.com/steveyegge/beads/internal/storage/embeddeddolt" ) @@ -40,21 +40,13 @@ func OpenBestAvailable(ctx context.Context, beadsDir string) (Storage, embeddedd return store, embeddeddolt.NoopLock{}, nil } - // Embedded mode: acquire exclusive flock first. - dataDir := filepath.Join(beadsDir, "embeddeddolt") - lock, err := embeddeddolt.TryLock(dataDir) - if err != nil { - return nil, nil, err - } - database := configfile.DefaultDoltDatabase if cfg != nil { database = cfg.GetDoltDatabase() } - store, err := embeddeddolt.New(ctx, beadsDir, database, "main", embeddeddolt.WithLock(lock)) + store, err := doltlite.New(ctx, beadsDir, database, "main") if err != nil { - lock.Unlock() return nil, nil, err } - return store, lock, nil + return store, embeddeddolt.NoopLock{}, nil } diff --git a/cmd/bd/store_factory.go b/cmd/bd/store_factory.go index 59674fcea..91a668437 100644 --- a/cmd/bd/store_factory.go +++ b/cmd/bd/store_factory.go @@ -12,6 +12,7 @@ import ( "github.com/steveyegge/beads/internal/doltserver" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dolt" + "github.com/steveyegge/beads/internal/storage/doltlite" "github.com/steveyegge/beads/internal/storage/embeddeddolt" ) @@ -44,7 +45,7 @@ func newDoltStore(ctx context.Context, cfg *dolt.Config, opts ...embeddeddolt.Op if cfg.ServerMode { return dolt.New(ctx, cfg) } - return embeddeddolt.New(ctx, cfg.BeadsDir, cfg.Database, "main", opts...) + return doltlite.New(ctx, cfg.BeadsDir, cfg.Database, "main") } // acquireEmbeddedLock acquires an exclusive flock on the embeddeddolt data @@ -55,8 +56,7 @@ func acquireEmbeddedLock(beadsDir string, serverMode bool) (embeddeddolt.Unlocke if serverMode { return embeddeddolt.NoopLock{}, nil } - dataDir := filepath.Join(beadsDir, "embeddeddolt") - return embeddeddolt.TryLock(dataDir) + return embeddeddolt.NoopLock{}, nil } // newDoltStoreFromConfig creates a storage backend from the beads directory's @@ -80,7 +80,7 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltS } database = sanitized } - return embeddeddolt.New(ctx, beadsDir, database, "main") + return doltlite.New(ctx, beadsDir, database, "main") } // migrateHyphenatedDB renames a legacy hyphenated database directory and @@ -141,5 +141,5 @@ func newReadOnlyStoreFromConfig(ctx context.Context, beadsDir string) (storage.D if sanitized := sanitizeDBName(database); sanitized != database { database = sanitized } - return embeddeddolt.New(ctx, beadsDir, database, "main") + return doltlite.New(ctx, beadsDir, database, "main") } diff --git a/go.mod b/go.mod index f9e7c9736..af1224407 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/dolthub/driver v1.86.4 github.com/go-sql-driver/mysql v1.9.3 + github.com/mattn/go-sqlite3 v1.14.8 github.com/olebedev/when v1.1.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 diff --git a/internal/beads/beads.go b/internal/beads/beads.go index 08498d57c..78b41a8a3 100644 --- a/internal/beads/beads.go +++ b/internal/beads/beads.go @@ -432,6 +432,10 @@ func findDatabaseInBeadsDir(beadsDir string, _ bool) string { if cfg.IsDoltServerMode() { return cfg.DatabasePath(beadsDir) } + doltlitePath := filepath.Join(beadsDir, "doltlite") + if info, err := os.Stat(doltlitePath); err == nil && info.IsDir() { + return doltlitePath + } // For embedded Dolt, the engine stores data under .beads/embeddeddolt/, // not .beads/dolt/. Check the actual embedded data directory first. embeddedPath := filepath.Join(beadsDir, "embeddeddolt") @@ -446,7 +450,11 @@ func findDatabaseInBeadsDir(beadsDir string, _ bool) string { } } - // Fall back: check if embeddeddolt or dolt directory exists without metadata.json + // Fall back: check if doltlite, embeddeddolt, or dolt directory exists without metadata.json + doltlitePath := filepath.Join(beadsDir, "doltlite") + if info, err := os.Stat(doltlitePath); err == nil && info.IsDir() { + return doltlitePath + } embeddedPath := filepath.Join(beadsDir, "embeddeddolt") if info, err := os.Stat(embeddedPath); err == nil && info.IsDir() { return embeddedPath @@ -594,10 +602,13 @@ func hasBeadsProjectFiles(beadsDir string) bool { return true } - // Check for Dolt database directory (server mode uses dolt/, embedded uses embeddeddolt/) + // Check for storage directories (server mode uses dolt/, embedded uses doltlite/ or embeddeddolt/) if info, err := os.Stat(filepath.Join(beadsDir, "dolt")); err == nil && info.IsDir() { return true } + if info, err := os.Stat(filepath.Join(beadsDir, "doltlite")); err == nil && info.IsDir() { + return true + } if info, err := os.Stat(filepath.Join(beadsDir, "embeddeddolt")); err == nil && info.IsDir() { return true } @@ -629,6 +640,9 @@ func hasBeadsDatabase(beadsDir string) bool { if info, err := os.Stat(filepath.Join(beadsDir, "dolt")); err == nil && info.IsDir() { return true } + if info, err := os.Stat(filepath.Join(beadsDir, "doltlite")); err == nil && info.IsDir() { + return true + } if info, err := os.Stat(filepath.Join(beadsDir, "embeddeddolt")); err == nil && info.IsDir() { return true } diff --git a/internal/storage/doltlite/cache_cleanup.go b/internal/storage/doltlite/cache_cleanup.go new file mode 100644 index 000000000..70a21ba36 --- /dev/null +++ b/internal/storage/doltlite/cache_cleanup.go @@ -0,0 +1,94 @@ +//go:build cgo + +package doltlite + +import ( + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// cleanGitRemoteCacheGarbage removes orphaned tmp_pack_* files from the +// Dolt git-remote-cache. These files are created by `git fetch` (invoked +// by Dolt's GitBlobstore) and should be renamed to final .pack/.idx files +// on success or deleted on failure. In practice, failed or interrupted +// fetches leave them behind indefinitely, and Dolt's built-in periodic +// git gc (maybeRunGC, gated to once per 24h) either never runs or cannot +// keep up with the accumulation rate. +// +// On a real machine with normal beads usage, this leak consumed 102 GB +// (412 files) in 7 days. See https://github.com/gastownhall/beads/issues/3354 +// +// This function is safe to call concurrently and is rate-limited to avoid +// unnecessary filesystem walks on hot paths. +func (s *DoltliteStore) cleanGitRemoteCacheGarbage() { + if !cacheCleanupThrottle.shouldRun() { + return + } + + cacheBase := filepath.Join(s.dataDir, s.database, ".dolt", "git-remote-cache") + if _, err := os.Stat(cacheBase); os.IsNotExist(err) { + return + } + + cutoff := time.Now().Add(-tmpPackMinAge) + + _ = filepath.WalkDir(cacheBase, func(path string, d os.DirEntry, err error) error { + if err != nil { + return nil // best-effort: skip unreadable entries + } + if d.IsDir() { + return nil + } + name := d.Name() + if !strings.HasPrefix(name, "tmp_pack_") && !strings.HasPrefix(name, "tmp_idx_") { + return nil + } + info, err := d.Info() + if err != nil { + return nil + } + if info.ModTime().Before(cutoff) { + // #nosec G122 -- path is under .dolt/git-remote-cache/ which is + // owned by the user running bd. A TOCTOU symlink swap would + // require write access to that directory; in that case the + // attacker already controls the Dolt data. The tmp_pack_/tmp_idx_ + // prefix check further narrows the scope to files Dolt itself writes. + _ = os.Remove(path) + } + return nil + }) +} + +const ( + // tmpPackMinAge is the minimum age before a tmp_pack file is considered + // garbage. Files younger than this may belong to an in-progress fetch. + tmpPackMinAge = 5 * time.Minute + + // cacheCleanupInterval is how often cleanGitRemoteCacheGarbage actually + // walks the filesystem when called repeatedly. + cacheCleanupInterval = 10 * time.Minute +) + +// throttle gates a function to run at most once per interval. +type throttle struct { + mu sync.Mutex + interval time.Duration + lastRun time.Time +} + +func (t *throttle) shouldRun() bool { + t.mu.Lock() + defer t.mu.Unlock() + if time.Since(t.lastRun) < t.interval { + return false + } + t.lastRun = time.Now() + return true +} + +// cacheCleanupThrottle is a package-level throttle shared across all +// DoltliteStore instances in the same process. +var cacheCleanupThrottle = &throttle{interval: cacheCleanupInterval} diff --git a/internal/storage/doltlite/child_id.go b/internal/storage/doltlite/child_id.go new file mode 100644 index 000000000..1d77c298a --- /dev/null +++ b/internal/storage/doltlite/child_id.go @@ -0,0 +1,20 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +func (s *DoltliteStore) GetNextChildID(ctx context.Context, parentID string) (string, error) { + var childID string + err := s.withConn(ctx, true, func(tx *sql.Tx) error { + var err error + childID, err = issueops.GetNextChildIDTx(ctx, tx, parentID) + return err + }) + return childID, err +} diff --git a/internal/storage/doltlite/config_metadata.go b/internal/storage/doltlite/config_metadata.go new file mode 100644 index 000000000..ea6454ace --- /dev/null +++ b/internal/storage/doltlite/config_metadata.go @@ -0,0 +1,115 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "fmt" + + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) SetConfig(ctx context.Context, key, value string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + if err := issueops.SetConfigInTx(ctx, tx, key, value); err != nil { + return err + } + // Sync normalized tables when config keys change + switch key { + case "status.custom": + if err := issueops.SyncCustomStatusesTable(ctx, tx, value); err != nil { + return fmt.Errorf("syncing custom_statuses table: %w", err) + } + case "types.custom": + if err := issueops.SyncCustomTypesTable(ctx, tx, value); err != nil { + return fmt.Errorf("syncing custom_types table: %w", err) + } + } + return nil + }) +} + +func (s *DoltliteStore) GetConfig(ctx context.Context, key string) (string, error) { + var value string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + value, err = issueops.GetConfigInTx(ctx, tx, key) + return err + }) + return value, err +} + +func (s *DoltliteStore) GetAllConfig(ctx context.Context) (map[string]string, error) { + var result map[string]string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetAllConfigInTx(ctx, tx) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetMetadata(ctx context.Context, key string) (string, error) { + var value string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + value, err = issueops.GetMetadataInTx(ctx, tx, key) + return err + }) + return value, err +} + +func (s *DoltliteStore) SetMetadata(ctx context.Context, key, value string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.SetMetadataInTx(ctx, tx, key, value) + }) +} + +func (s *DoltliteStore) SetLocalMetadata(ctx context.Context, key, value string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.SetLocalMetadataInTx(ctx, tx, key, value) + }) +} + +func (s *DoltliteStore) GetLocalMetadata(ctx context.Context, key string) (string, error) { + var value string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + value, err = issueops.GetLocalMetadataInTx(ctx, tx, key) + return err + }) + return value, err +} + +// GetInfraTypes returns the set of infrastructure types that should be routed +// to the wisps table. Reads from DB config "types.infra", falls back to YAML, +// then to hardcoded defaults (agent, rig, role, message). +func (s *DoltliteStore) GetInfraTypes(ctx context.Context) map[string]bool { + var result map[string]bool + if err := s.withConn(ctx, false, func(tx *sql.Tx) error { + result = issueops.ResolveInfraTypesInTx(ctx, tx) + return nil + }); err != nil || result == nil { + // DB unavailable — fall back to YAML then defaults. + var typeList []string + if yamlTypes := config.GetInfraTypesFromYAML(); len(yamlTypes) > 0 { + typeList = yamlTypes + } else { + typeList = storage.DefaultInfraTypes() + } + result = make(map[string]bool, len(typeList)) + for _, t := range typeList { + result[t] = true + } + } + return result +} + +// IsInfraTypeCtx returns true if the issue type is an infrastructure type. +func (s *DoltliteStore) IsInfraTypeCtx(ctx context.Context, t types.IssueType) bool { + return s.GetInfraTypes(ctx)[string(t)] +} diff --git a/internal/storage/doltlite/create_issue.go b/internal/storage/doltlite/create_issue.go new file mode 100644 index 000000000..56ab0a83e --- /dev/null +++ b/internal/storage/doltlite/create_issue.go @@ -0,0 +1,159 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "fmt" + + "github.com/google/uuid" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) CreateIssue(ctx context.Context, issue *types.Issue, actor string) error { + if issue == nil { + return fmt.Errorf("issue must not be nil") + } + // Route infra types to wisps, matching DoltStore.CreateIssue behavior. + if s.IsInfraTypeCtx(ctx, issue.IssueType) { + issue.Ephemeral = true + } + + return s.withConn(ctx, true, func(tx *sql.Tx) error { + bc, err := issueops.NewBatchContext(ctx, tx, storage.BatchCreateOptions{ + SkipPrefixValidation: true, + }) + if err != nil { + return err + } + return createIssueSQLite(ctx, tx, bc, issue, actor) + }) +} + +func (s *DoltliteStore) CreateIssues(ctx context.Context, issues []*types.Issue, actor string) error { + return s.CreateIssuesWithFullOptions(ctx, issues, actor, storage.BatchCreateOptions{ + OrphanHandling: storage.OrphanAllow, + SkipPrefixValidation: false, + }) +} + +func (s *DoltliteStore) CreateIssuesWithFullOptions(ctx context.Context, issues []*types.Issue, actor string, opts storage.BatchCreateOptions) error { + if len(issues) == 0 { + return nil + } + + for _, issue := range issues { + if issueops.IsWisp(issue) && !issue.NoHistory { + issue.Ephemeral = true + } + if err := s.withConn(ctx, true, func(tx *sql.Tx) error { + bc, err := issueops.NewBatchContext(ctx, tx, opts) + if err != nil { + return err + } + return createIssueSQLite(ctx, tx, bc, issue, actor) + }); err != nil { + return err + } + } + return nil +} + +func createIssueSQLite(ctx context.Context, tx *sql.Tx, bc *issueops.BatchContext, issue *types.Issue, actor string) error { + if err := issueops.PrepareIssueForInsert(issue, bc.CustomStatuses, bc.CustomTypes); err != nil { + return err + } + issueTable, eventTable := issueops.TableRouting(issue) + if issue.ID == "" { + prefix := bc.ConfigPrefix + if issue.PrefixOverride != "" { + prefix = issue.PrefixOverride + } else if issue.IDPrefix != "" { + prefix = bc.ConfigPrefix + "-" + issue.IDPrefix + } else if issueops.IsWisp(issue) { + prefix = bc.ConfigPrefix + "-wisp" + } + var err error + issue.ID, err = issueops.GenerateIssueIDInTable(ctx, tx, issueTable, prefix, issue, actor) + if err != nil { + return fmt.Errorf("failed to generate issue ID: %w", err) + } + } + if skip, err := issueops.CheckOrphan(ctx, tx, issue, issueTable, bc.Opts.OrphanHandling); err != nil { + return err + } else if skip { + return nil + } + + var existingCount int + if err := tx.QueryRowContext(ctx, fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE id = ?", issueTable), issue.ID).Scan(&existingCount); err != nil { + return fmt.Errorf("failed to check issue existence for %s: %w", issue.ID, err) + } + if err := insertIssueSQLite(ctx, tx, issueTable, issue); err != nil { + return err + } + if existingCount == 0 { + if err := recordEventSQLite(ctx, tx, eventTable, issue.ID, types.EventCreated, actor, ""); err != nil { + return fmt.Errorf("failed to record event for %s: %w", issue.ID, err) + } + } + if err := issueops.PersistLabels(ctx, tx, issue); err != nil { + return err + } + return issueops.PersistComments(ctx, tx, issue) +} + +func insertIssueSQLite(ctx context.Context, tx *sql.Tx, table string, issue *types.Issue) error { + _, err := tx.ExecContext(ctx, fmt.Sprintf(` + INSERT OR REPLACE INTO %s ( + id, content_hash, title, description, design, acceptance_criteria, notes, + status, priority, issue_type, assignee, estimated_minutes, + created_at, created_by, owner, updated_at, started_at, closed_at, external_ref, spec_id, + compaction_level, compacted_at, compacted_at_commit, original_size, + sender, ephemeral, no_history, wisp_type, pinned, is_template, + mol_type, work_type, source_system, source_repo, close_reason, + event_kind, actor, target, payload, + await_type, await_id, timeout_ns, waiters, + due_at, defer_until, metadata + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ? + ) + `, table), + issue.ID, issue.ContentHash, issue.Title, issue.Description, issue.Design, issue.AcceptanceCriteria, issue.Notes, + issue.Status, issue.Priority, issue.IssueType, issueops.NullString(issue.Assignee), issueops.NullInt(issue.EstimatedMinutes), + issue.CreatedAt, issue.CreatedBy, issue.Owner, issue.UpdatedAt, issue.StartedAt, issue.ClosedAt, issueops.NullStringPtr(issue.ExternalRef), issue.SpecID, + issue.CompactionLevel, issue.CompactedAt, issueops.NullStringPtr(issue.CompactedAtCommit), issueops.NullIntVal(issue.OriginalSize), + issue.Sender, issue.Ephemeral, issue.NoHistory, issue.WispType, issue.Pinned, issue.IsTemplate, + issue.MolType, issue.WorkType, issue.SourceSystem, issue.SourceRepo, issue.CloseReason, + issue.EventKind, issue.Actor, issue.Target, issue.Payload, + issue.AwaitType, issue.AwaitID, issue.Timeout.Nanoseconds(), issueops.FormatJSONStringArray(issue.Waiters), + issue.DueAt, issue.DeferUntil, issueops.JSONMetadata(issue.Metadata), + ) + if err != nil { + return fmt.Errorf("insert issue into %s: %w", table, err) + } + return nil +} + +func recordEventSQLite(ctx context.Context, tx *sql.Tx, table, issueID string, eventType types.EventType, actor, newValue string) error { + id := uuid.Must(uuid.NewV7()).String() + _, err := tx.ExecContext(ctx, fmt.Sprintf(` + INSERT INTO %s (id, issue_id, event_type, actor, old_value, new_value) + VALUES (?, ?, ?, ?, ?, ?) + `, table), id, issueID, eventType, actor, "", newValue) + if err != nil { + return fmt.Errorf("record event in %s: %w", table, err) + } + return nil +} diff --git a/internal/storage/doltlite/dependencies.go b/internal/storage/doltlite/dependencies.go new file mode 100644 index 000000000..6e54699ab --- /dev/null +++ b/internal/storage/doltlite/dependencies.go @@ -0,0 +1,72 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) AddDependency(ctx context.Context, dep *types.Dependency, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.AddDependencyInTx(ctx, tx, dep, actor, issueops.AddDependencyOpts{ + IsCrossPrefix: types.ExtractPrefix(dep.IssueID) != types.ExtractPrefix(dep.DependsOnID), + }) + }) +} + +// RemoveDependency removes a dependency between two issues. +func (s *DoltliteStore) RemoveDependency(ctx context.Context, issueID, dependsOnID string, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.RemoveDependencyInTx(ctx, tx, issueID, dependsOnID) + }) +} + +// GetIssuesByIDs retrieves multiple issues by ID. +func (s *DoltliteStore) GetIssuesByIDs(ctx context.Context, ids []string) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetIssuesByIDsInTx(ctx, tx, ids, nil) + return err + }) + return result, err +} + +// GetDependenciesWithMetadata returns issues that the given issue depends on, +// along with the dependency type. +func (s *DoltliteStore) GetDependenciesWithMetadata(ctx context.Context, issueID string) ([]*types.IssueWithDependencyMetadata, error) { + var result []*types.IssueWithDependencyMetadata + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependenciesWithMetadataInTx(ctx, tx, issueID) + return err + }) + return result, err +} + +// GetDependentsWithMetadata returns issues that depend on the given issue, +// along with the dependency type. +func (s *DoltliteStore) GetDependentsWithMetadata(ctx context.Context, issueID string) ([]*types.IssueWithDependencyMetadata, error) { + var result []*types.IssueWithDependencyMetadata + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependentsWithMetadataInTx(ctx, tx, issueID) + return err + }) + return result, err +} + +// DetectCycles finds dependency cycles across both permanent and wisp dependencies. +func (s *DoltliteStore) DetectCycles(ctx context.Context) ([][]*types.Issue, error) { + var result [][]*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.DetectCyclesInTx(ctx, tx) + return err + }) + return result, err +} diff --git a/internal/storage/doltlite/federation.go b/internal/storage/doltlite/federation.go new file mode 100644 index 000000000..4a0ec7edc --- /dev/null +++ b/internal/storage/doltlite/federation.go @@ -0,0 +1,334 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "database/sql" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" +) + +// credentialKeyFile is the filename for the random encryption key. +const credentialKeyFile = ".beads-credential-key" //nolint:gosec // G101: filename, not a credential + +// ensureCredentialKey lazily initializes the credential encryption key. +func (s *DoltliteStore) ensureCredentialKey() error { + if s.credentialKey != nil { + return nil + } + if s.beadsDir == "" { + return fmt.Errorf("beads directory not set; credential encryption unavailable") + } + + keyPath := filepath.Join(s.beadsDir, credentialKeyFile) + + // Try to load existing key. + key, err := os.ReadFile(keyPath) //nolint:gosec // G304: keyPath derived from trusted beadsDir + if err == nil && len(key) == 32 { + s.credentialKey = key + return nil + } + + // Generate new random 32-byte key (AES-256). + key = make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, key); err != nil { + return fmt.Errorf("generate credential key: %w", err) + } + if err := os.WriteFile(keyPath, key, 0600); err != nil { + return fmt.Errorf("write credential key: %w", err) + } + + s.credentialKey = key + return nil +} + +func (s *DoltliteStore) encryptPassword(password string) ([]byte, error) { + if password == "" { + return nil, nil + } + if err := s.ensureCredentialKey(); err != nil { + return nil, err + } + block, err := aes.NewCipher(s.credentialKey) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + return gcm.Seal(nonce, nonce, []byte(password), nil), nil +} + +func (s *DoltliteStore) decryptPassword(encrypted []byte) (string, error) { + if len(encrypted) == 0 { + return "", nil + } + if err := s.ensureCredentialKey(); err != nil { + return "", err + } + block, err := aes.NewCipher(s.credentialKey) + if err != nil { + return "", err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return "", err + } + nonceSize := gcm.NonceSize() + if len(encrypted) < nonceSize { + return "", fmt.Errorf("ciphertext too short") + } + nonce, ciphertext := encrypted[:nonceSize], encrypted[nonceSize:] + plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return "", err + } + return string(plaintext), nil +} + +// --------------------------------------------------------------------------- +// FederationStore implementation +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) AddFederationPeer(ctx context.Context, peer *storage.FederationPeer) error { + encryptedPwd, err := s.encryptPassword(peer.Password) + if err != nil { + return fmt.Errorf("encrypt password: %w", err) + } + + if err := s.withConn(ctx, true, func(tx *sql.Tx) error { + if err := issueops.AddFederationPeerInTx(ctx, tx, peer, encryptedPwd); err != nil { + return err + } + // Also add the Dolt remote. + return issueops.AddRemoteIfNotExists(ctx, tx, peer.Name, peer.RemoteURL) + }); err != nil { + return err + } + return nil +} + +func (s *DoltliteStore) GetFederationPeer(ctx context.Context, name string) (*storage.FederationPeer, error) { + var row *issueops.FederationPeerRow + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + row, err = issueops.GetFederationPeerInTx(ctx, tx, name) + return err + }) + if err != nil { + return nil, err + } + + if len(row.EncryptedPwd) > 0 { + row.Peer.Password, err = s.decryptPassword(row.EncryptedPwd) + if err != nil { + return nil, fmt.Errorf("decrypt password: %w", err) + } + } + return &row.Peer, nil +} + +func (s *DoltliteStore) ListFederationPeers(ctx context.Context) ([]*storage.FederationPeer, error) { + var rows []*issueops.FederationPeerRow + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + rows, err = issueops.ListFederationPeersInTx(ctx, tx) + return err + }) + if err != nil { + return nil, err + } + + peers := make([]*storage.FederationPeer, 0, len(rows)) + for _, row := range rows { + if len(row.EncryptedPwd) > 0 { + pwd, err := s.decryptPassword(row.EncryptedPwd) + if err != nil { + return nil, fmt.Errorf("decrypt password for peer %s: %w", row.Peer.Name, err) + } + row.Peer.Password = pwd + } + peers = append(peers, &row.Peer) + } + return peers, nil +} + +func (s *DoltliteStore) RemoveFederationPeer(ctx context.Context, name string) error { + if err := s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.RemoveFederationPeerInTx(ctx, tx, name) + }); err != nil { + return err + } + + // Also remove the Dolt remote (best-effort). + if rmErr := s.RemoveRemote(ctx, name); rmErr != nil { + if !strings.Contains(rmErr.Error(), "not found") { + // Silently ignore "not found" — the remote may not exist. + _ = rmErr + } + } + return nil +} + +// --------------------------------------------------------------------------- +// SyncStore implementation +// --------------------------------------------------------------------------- + +// Sync performs a full bidirectional sync with a peer: +// 1. Fetch from peer +// 2. Merge peer's changes (handling conflicts per strategy) +// 3. Push local changes to peer +func (s *DoltliteStore) Sync(ctx context.Context, peer string, strategy string) (*storage.SyncResult, error) { + result := &storage.SyncResult{ + Peer: peer, + StartTime: time.Now(), + } + + // Step 1: Fetch + if err := s.Fetch(ctx, peer); err != nil { + result.Error = fmt.Errorf("fetch failed: %w", err) + return result, result.Error + } + result.Fetched = true + + // Step 2: Get commit before merge for change detection + beforeCommit, _ := s.GetCurrentCommit(ctx) + + // Step 3: Merge peer's branch + remoteBranch := fmt.Sprintf("%s/%s", peer, s.branch) + conflicts, err := s.Merge(ctx, remoteBranch) + if err != nil { + result.Error = fmt.Errorf("merge failed: %w", err) + return result, result.Error + } + + // Step 4: Handle conflicts + if len(conflicts) > 0 { + result.Conflicts = conflicts + + if strategy == "" { + result.Error = fmt.Errorf("merge conflicts require resolution (use --strategy ours|theirs)") + return result, result.Error + } + + for _, c := range conflicts { + if err := s.ResolveConflicts(ctx, c.Field, strategy); err != nil { + result.Error = fmt.Errorf("conflict resolution failed for %s: %w", c.Field, err) + return result, result.Error + } + } + result.ConflictsResolved = true + + if err := s.Commit(ctx, fmt.Sprintf("Resolve conflicts from %s using %s strategy", peer, strategy)); err != nil { + result.Error = fmt.Errorf("commit conflict resolution: %w", err) + return result, result.Error + } + } + result.Merged = true + + afterCommit, _ := s.GetCurrentCommit(ctx) + if beforeCommit != afterCommit { + result.PulledCommits = 1 + } + + // Step 5: Push + if err := s.PushTo(ctx, peer); err != nil { + result.PushError = err + } else { + result.Pushed = true + } + + // Record last sync time in metadata. + _ = s.setLastSyncTime(ctx, peer) + + result.EndTime = time.Now() + return result, nil +} + +// SyncStatus returns the synchronization status with a peer. +func (s *DoltliteStore) SyncStatus(ctx context.Context, peer string) (*storage.SyncStatus, error) { + status := &storage.SyncStatus{ + Peer: peer, + } + + // Get ahead/behind counts by comparing refs. + // Dolt's AS OF requires a literal ref, not a parameterized expression. + remoteRef := peer + "/" + s.branch + if err := issueops.ValidateRef(remoteRef); err != nil { + status.LocalAhead = -1 + status.LocalBehind = -1 + } else if err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + query := fmt.Sprintf(` + SELECT + (SELECT COUNT(*) FROM dolt_log WHERE commit_hash NOT IN + (SELECT commit_hash FROM dolt_log AS OF '%s')) as ahead, + (SELECT COUNT(*) FROM dolt_log AS OF '%s' WHERE commit_hash NOT IN + (SELECT commit_hash FROM dolt_log)) as behind + `, remoteRef, remoteRef) + if err := db.QueryRowContext(ctx, query). + Scan(&status.LocalAhead, &status.LocalBehind); err != nil { + // Remote branch may not exist locally yet. + status.LocalAhead = -1 + status.LocalBehind = -1 + } + return nil + }); err != nil { + return nil, err + } + + // Check for conflicts. + conflicts, err := s.GetConflicts(ctx) + if err == nil && len(conflicts) > 0 { + status.HasConflicts = true + } + + // Get last sync time. + status.LastSync = s.getLastSyncTime(ctx, peer) + + return status, nil +} + +// setLastSyncTime records the last sync time for a peer in metadata. +func (s *DoltliteStore) setLastSyncTime(ctx context.Context, peer string) error { + key := "last_sync_" + peer + value := time.Now().Format(time.RFC3339) + return s.withConn(ctx, true, func(tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, + "REPLACE INTO metadata (`key`, value) VALUES (?, ?)", key, value) + return err + }) +} + +// getLastSyncTime retrieves the last sync time for a peer from metadata. +func (s *DoltliteStore) getLastSyncTime(ctx context.Context, peer string) time.Time { + key := "last_sync_" + peer + var value string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + return tx.QueryRowContext(ctx, "SELECT value FROM metadata WHERE `key` = ?", key).Scan(&value) + }) + if err != nil { + return time.Time{} + } + t, err := time.Parse(time.RFC3339, value) + if err != nil { + return time.Time{} + } + return t +} diff --git a/internal/storage/doltlite/flock.go b/internal/storage/doltlite/flock.go new file mode 100644 index 000000000..adda4089f --- /dev/null +++ b/internal/storage/doltlite/flock.go @@ -0,0 +1,114 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + backoff "github.com/cenkalti/backoff/v4" + "github.com/steveyegge/beads/internal/lockfile" +) + +// Unlocker is the interface for releasing an acquired lock. +type Unlocker interface { + Unlock() +} + +// Lock holds an exclusive flock on the doltlite data directory. +// Used by commands that require single-writer access (e.g., bd init). +type Lock struct { + f *os.File +} + +// TryLock attempts to acquire a non-blocking exclusive flock on /.lock. +// dataDir is created if it does not exist. Returns the held lock on success. +// If another process holds the lock, returns an error directing the user to +// the dolt server backend for concurrent access. +func TryLock(dataDir string) (*Lock, error) { + if err := os.MkdirAll(dataDir, 0700); err != nil { + return nil, fmt.Errorf("doltlite: creating data directory for lock: %w", err) + } + + lockPath := filepath.Join(dataDir, ".lock") + f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0600) //nolint:gosec // lockPath is derived from dataDir, not user input + if err != nil { + return nil, fmt.Errorf("doltlite: opening lock file: %w", err) + } + + if err := lockfile.FlockExclusiveNonBlocking(f); err != nil { + _ = f.Close() + if lockfile.IsLocked(err) { + return nil, fmt.Errorf("doltlite: another process holds the exclusive lock on %s; "+ + "the embedded backend supports only one writer at a time — "+ + "use the dolt server backend for concurrent access", dataDir) + } + return nil, fmt.Errorf("doltlite: acquiring lock: %w", err) + } + + return &Lock{f: f}, nil +} + +// WaitLock blocks until an exclusive flock on /.lock can be acquired +// or the context is canceled. It uses exponential backoff with non-blocking +// lock attempts so the wait is interruptible via context cancellation. +// Non-lock filesystem errors are returned immediately without retrying. +func WaitLock(ctx context.Context, dataDir string) (*Lock, error) { + if err := os.MkdirAll(dataDir, 0700); err != nil { + return nil, fmt.Errorf("doltlite: creating data directory for lock: %w", err) + } + + lockPath := filepath.Join(dataDir, ".lock") + f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0600) //nolint:gosec // lockPath is derived from dataDir, not user input + if err != nil { + return nil, fmt.Errorf("doltlite: opening lock file: %w", err) + } + + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = 50 * time.Millisecond + bo.MaxInterval = 2 * time.Second + bo.MaxElapsedTime = 0 // wait until context cancellation + + err = backoff.Retry(func() error { + lockErr := lockfile.FlockExclusiveNonBlocking(f) + if lockErr == nil { + return nil // acquired + } + if lockfile.IsLocked(lockErr) { + return lockErr // retryable + } + // Filesystem error — not retryable. + return backoff.Permanent(lockErr) + }, backoff.WithContext(bo, ctx)) + + if err != nil { + _ = f.Close() + if ctx.Err() != nil { + return nil, fmt.Errorf("doltlite: waiting for lock on %s: %w", dataDir, ctx.Err()) + } + return nil, fmt.Errorf("doltlite: acquiring lock: %w", err) + } + + return &Lock{f: f}, nil +} + +// Unlock releases the flock and closes the underlying file. +// Panics on failure to prevent deadlocks. +func (l *Lock) Unlock() { + if err := lockfile.FlockUnlock(l.f); err != nil { + panic(fmt.Sprintf("doltlite: failed to release lock: %v", err)) + } + if err := l.f.Close(); err != nil { + panic(fmt.Sprintf("doltlite: failed to close lock file: %v", err)) + } +} + +// NoopLock is a lock that does nothing. Used in server mode where the +// external dolt sql-server handles its own concurrency. +type NoopLock struct{} + +// Unlock is a no-op. +func (NoopLock) Unlock() {} diff --git a/internal/storage/doltlite/flock_stub.go b/internal/storage/doltlite/flock_stub.go new file mode 100644 index 000000000..6e10d10e8 --- /dev/null +++ b/internal/storage/doltlite/flock_stub.go @@ -0,0 +1,27 @@ +//go:build !cgo + +package doltlite + +import "errors" + +// Unlocker is the interface for releasing an acquired lock. +type Unlocker interface { + Unlock() +} + +// Lock is a stub for builds without CGO. +type Lock struct{} + +// TryLock returns an error when CGO is not enabled. +func TryLock(_ string) (*Lock, error) { + return nil, errors.New("doltlite: requires CGO (build with CGO_ENABLED=1)") +} + +// Unlock is a no-op stub. +func (l *Lock) Unlock() {} + +// NoopLock is a lock that does nothing. +type NoopLock struct{} + +// Unlock is a no-op. +func (NoopLock) Unlock() {} diff --git a/internal/storage/doltlite/get_issue.go b/internal/storage/doltlite/get_issue.go new file mode 100644 index 000000000..b207728eb --- /dev/null +++ b/internal/storage/doltlite/get_issue.go @@ -0,0 +1,21 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) GetIssue(ctx context.Context, id string) (*types.Issue, error) { + var issue *types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + issue, err = issueops.GetIssueInTx(ctx, tx, id) + return err + }) + return issue, err +} diff --git a/internal/storage/doltlite/issues.go b/internal/storage/doltlite/issues.go new file mode 100644 index 000000000..a36ab8a1d --- /dev/null +++ b/internal/storage/doltlite/issues.go @@ -0,0 +1,100 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// ClaimIssue atomically claims an issue using compare-and-swap semantics. +// Delegates SQL work to issueops; EmbeddedDolt auto-commits the transaction. +func (s *DoltliteStore) ClaimIssue(ctx context.Context, id string, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + _, err := issueops.ClaimIssueInTx(ctx, tx, id, actor) + return err + }) +} + +// UpdateIssue updates fields on an issue. +// Delegates SQL work to issueops; EmbeddedDolt auto-commits the transaction. +func (s *DoltliteStore) UpdateIssue(ctx context.Context, id string, updates map[string]interface{}, actor string) error { + // Validate metadata against schema before routing. + if rawMeta, ok := updates["metadata"]; ok { + metadataStr, err := storage.NormalizeMetadataValue(rawMeta) + if err != nil { + return fmt.Errorf("invalid metadata: %w", err) + } + if err := issueops.ValidateMetadataIfConfigured(json.RawMessage(metadataStr)); err != nil { + return err + } + } + + return s.withConn(ctx, true, func(tx *sql.Tx) error { + _, err := issueops.UpdateIssueInTx(ctx, tx, id, updates, actor) + return err + }) +} + +// ReopenIssue reopens a closed issue, setting status to open and clearing +// closed_at and defer_until. If reason is non-empty, it is recorded as a comment. +// Wraps UpdateIssue; EmbeddedDolt auto-commits the transaction. +func (s *DoltliteStore) ReopenIssue(ctx context.Context, id string, reason string, actor string) error { + updates := map[string]interface{}{ + "status": string(types.StatusOpen), + "defer_until": nil, + } + if err := s.UpdateIssue(ctx, id, updates, actor); err != nil { + return err + } + if reason != "" { + if err := s.AddComment(ctx, id, actor, reason); err != nil { + return fmt.Errorf("reopen comment: %w", err) + } + } + return nil +} + +// UpdateIssueType changes the issue_type field of an issue. +// Wraps UpdateIssue; EmbeddedDolt auto-commits the transaction. +func (s *DoltliteStore) UpdateIssueType(ctx context.Context, id string, issueType string, actor string) error { + return s.UpdateIssue(ctx, id, map[string]interface{}{"issue_type": issueType}, actor) +} + +// CloseIssue closes an issue with a reason. +// Delegates SQL work to issueops; EmbeddedDolt auto-commits the transaction. +func (s *DoltliteStore) CloseIssue(ctx context.Context, id string, reason string, actor string, session string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + _, err := issueops.CloseIssueInTx(ctx, tx, id, reason, actor, session) + return err + }) +} + +// IsBlocked checks if an issue is blocked by active dependencies. +func (s *DoltliteStore) IsBlocked(ctx context.Context, issueID string) (bool, []string, error) { + var blocked bool + var blockers []string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + blocked, blockers, err = issueops.IsBlockedInTx(ctx, tx, issueID) + return err + }) + return blocked, blockers, err +} + +// GetNewlyUnblockedByClose finds issues that become unblocked when closedIssueID is closed. +func (s *DoltliteStore) GetNewlyUnblockedByClose(ctx context.Context, closedIssueID string) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetNewlyUnblockedByCloseInTx(ctx, tx, closedIssueID) + return err + }) + return result, err +} diff --git a/internal/storage/doltlite/labels.go b/internal/storage/doltlite/labels.go new file mode 100644 index 000000000..c9987d5a2 --- /dev/null +++ b/internal/storage/doltlite/labels.go @@ -0,0 +1,33 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +func (s *DoltliteStore) GetLabels(ctx context.Context, issueID string) ([]string, error) { + var labels []string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + labels, err = issueops.GetLabelsInTx(ctx, tx, "", issueID) + return err + }) + return labels, err +} + +func (s *DoltliteStore) AddLabel(ctx context.Context, issueID, label, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.AddLabelInTx(ctx, tx, "", "", issueID, label, actor) + }) +} + +// RemoveLabel removes a label from an issue. +func (s *DoltliteStore) RemoveLabel(ctx context.Context, issueID, label, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.RemoveLabelInTx(ctx, tx, "", "", issueID, label, actor) + }) +} diff --git a/internal/storage/doltlite/list_queries.go b/internal/storage/doltlite/list_queries.go new file mode 100644 index 000000000..e5c3209eb --- /dev/null +++ b/internal/storage/doltlite/list_queries.go @@ -0,0 +1,96 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.SearchIssuesInTx(ctx, tx, query, filter) + return err + }) + return result, err +} + +func (s *DoltliteStore) ListWisps(ctx context.Context, filter types.WispFilter) ([]*types.Issue, error) { + issueFilter := issueops.WispFilterToIssueFilter(filter) + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.SearchIssuesInTx(ctx, tx, "", issueFilter) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetLabelsForIssues(ctx context.Context, issueIDs []string) (map[string][]string, error) { + var result map[string][]string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetLabelsForIssuesInTx(ctx, tx, issueIDs, nil) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetCommentCounts(ctx context.Context, issueIDs []string) (map[string]int, error) { + var result map[string]int + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetCommentCountsInTx(ctx, tx, issueIDs) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetAllDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) { + var result map[string][]*types.Dependency + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetAllDependencyRecordsInTx(ctx, tx) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetDependencyRecordsForIssues(ctx context.Context, issueIDs []string) (map[string][]*types.Dependency, error) { + var result map[string][]*types.Dependency + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependencyRecordsForIssuesInTx(ctx, tx, issueIDs) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetDependencyCounts(ctx context.Context, issueIDs []string) (map[string]*types.DependencyCounts, error) { + var result map[string]*types.DependencyCounts + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependencyCountsInTx(ctx, tx, issueIDs) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetBlockingInfoForIssues(ctx context.Context, issueIDs []string) ( + blockedByMap map[string][]string, + blocksMap map[string][]string, + parentMap map[string]string, + err error, +) { + err = s.withConn(ctx, false, func(tx *sql.Tx) error { + var txErr error + blockedByMap, blocksMap, parentMap, txErr = issueops.GetBlockingInfoForIssuesInTx(ctx, tx, issueIDs) + return txErr + }) + return +} diff --git a/internal/storage/doltlite/merge_slot.go b/internal/storage/doltlite/merge_slot.go new file mode 100644 index 000000000..6f58c2b56 --- /dev/null +++ b/internal/storage/doltlite/merge_slot.go @@ -0,0 +1,33 @@ +//go:build cgo + +package doltlite + +import ( + "context" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// MergeSlotCreate creates the merge slot bead for the current rig. +// Idempotent: returns the existing slot if one already exists. +func (s *DoltliteStore) MergeSlotCreate(ctx context.Context, actor string) (*types.Issue, error) { + return storage.MergeSlotCreateImpl(ctx, s, actor) +} + +// MergeSlotCheck returns the current status of the merge slot. +func (s *DoltliteStore) MergeSlotCheck(ctx context.Context) (*storage.MergeSlotStatus, error) { + return storage.MergeSlotCheckImpl(ctx, s) +} + +// MergeSlotAcquire attempts to acquire the merge slot atomically. +// When wait is true and the slot is held, the caller is added to the waiters queue. +func (s *DoltliteStore) MergeSlotAcquire(ctx context.Context, holder, actor string, wait bool) (*storage.MergeSlotResult, error) { + return storage.MergeSlotAcquireImpl(ctx, s, holder, actor, wait) +} + +// MergeSlotRelease releases the merge slot, clearing the holder. +// If holder is non-empty it is verified against the current holder before releasing. +func (s *DoltliteStore) MergeSlotRelease(ctx context.Context, holder, actor string) error { + return storage.MergeSlotReleaseImpl(ctx, s, holder, actor) +} diff --git a/internal/storage/doltlite/open.go b/internal/storage/doltlite/open.go new file mode 100644 index 000000000..25f7d5ccd --- /dev/null +++ b/internal/storage/doltlite/open.go @@ -0,0 +1,75 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + _ "github.com/mattn/go-sqlite3" +) + +// validIdentifier matches safe SQL identifiers (letters, digits, underscores). +// Hyphens are excluded because database names are interpolated into system +// variable identifiers (@@_head_ref) where hyphens are invalid. +var validIdentifier = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + +const ( + commitName = "beads" + commitEmail = "beads@local" +) + +// OpenSQL opens an doltlite database at dir. The returned cleanup +// function closes the *sql.DB. +func OpenSQL(ctx context.Context, dir, database, branch string) (*sql.DB, func() error, error) { + dbPath, err := buildDSN(dir, database) + if err != nil { + return nil, nil, err + } + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + return nil, nil, err + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + db.SetConnMaxIdleTime(0) + db.SetConnMaxLifetime(0) + + cleanup := func() error { + return db.Close() + } + + if err := db.PingContext(ctx); err != nil { + closeErr := cleanup() + if closeErr != nil { + return nil, nil, fmt.Errorf("%w; close: %v", err, closeErr) + } + return nil, nil, err + } + + return db, cleanup, nil +} + +func buildDSN(dir, database string) (string, error) { + if strings.TrimSpace(database) != "" { + if !validIdentifier.MatchString(database) { + return "", fmt.Errorf("doltlite: invalid database name: %q", database) + } + } else { + database = "beads" + } + path := filepath.Join(dir, database+".db") + if os.PathSeparator == '\\' { + path = strings.ReplaceAll(path, `\`, `/`) + } + return path, nil +} + +func sqlStringLiteral(s string) string { + return "'" + strings.ReplaceAll(strings.TrimSpace(s), "'", "''") + "'" +} diff --git a/internal/storage/doltlite/open_stub.go b/internal/storage/doltlite/open_stub.go new file mode 100644 index 000000000..01f017f8d --- /dev/null +++ b/internal/storage/doltlite/open_stub.go @@ -0,0 +1,14 @@ +//go:build !cgo + +package doltlite + +import ( + "context" + "database/sql" + "errors" +) + +// OpenSQL is a stub that returns an error when CGO is not enabled. +func OpenSQL(_ context.Context, _, _, _ string) (*sql.DB, func() error, error) { + return nil, nil, errors.New("doltlite: requires CGO (build with CGO_ENABLED=1)") +} diff --git a/internal/storage/doltlite/queries.go b/internal/storage/doltlite/queries.go new file mode 100644 index 000000000..f74054052 --- /dev/null +++ b/internal/storage/doltlite/queries.go @@ -0,0 +1,42 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// GetReadyWork returns issues that are ready to work on (not blocked). +// Delegates to issueops.GetReadyWorkInTx with the shared blocked-ID computation. +func (s *DoltliteStore) GetReadyWork(ctx context.Context, filter types.WorkFilter) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetReadyWorkInTx(ctx, tx, filter, computeBlockedIDsWrapper) + return err + }) + return result, err +} + +// computeBlockedIDsWrapper adapts ComputeBlockedIDsInTx to the callback +// signature expected by GetReadyWorkInTx. +func computeBlockedIDsWrapper(ctx context.Context, tx *sql.Tx, includeWisps bool) ([]string, error) { + ids, _, err := issueops.ComputeBlockedIDsInTx(ctx, tx, includeWisps) + return ids, err +} + +// GetMoleculeProgress returns progress stats for a molecule. +// Delegates to issueops.GetMoleculeProgressInTx. +func (s *DoltliteStore) GetMoleculeProgress(ctx context.Context, moleculeID string) (*types.MoleculeProgressStats, error) { + var result *types.MoleculeProgressStats + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetMoleculeProgressInTx(ctx, tx, moleculeID) + return err + }) + return result, err +} diff --git a/internal/storage/doltlite/schema.go b/internal/storage/doltlite/schema.go new file mode 100644 index 000000000..f3adb517f --- /dev/null +++ b/internal/storage/doltlite/schema.go @@ -0,0 +1,12 @@ +//go:build cgo + +package doltlite + +import ( + "github.com/steveyegge/beads/internal/storage/schema" +) + +// LatestVersion delegates to the shared schema package. +func LatestVersion() int { + return schema.LatestVersion() +} diff --git a/internal/storage/doltlite/slots.go b/internal/storage/doltlite/slots.go new file mode 100644 index 000000000..bdef6a557 --- /dev/null +++ b/internal/storage/doltlite/slots.go @@ -0,0 +1,96 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "encoding/json" + "fmt" +) + +// SlotSet sets a key-value pair in the issue's metadata JSON. +func (s *DoltliteStore) SlotSet(ctx context.Context, issueID, key, value, actor string) error { + issue, err := s.GetIssue(ctx, issueID) + if err != nil { + return fmt.Errorf("getting issue %s: %w", issueID, err) + } + + metadata := make(map[string]interface{}) + if len(issue.Metadata) > 0 { + if err := json.Unmarshal(issue.Metadata, &metadata); err != nil { + return fmt.Errorf("parsing metadata for %s: %w", issueID, err) + } + } + metadata[key] = value + + raw, err := json.Marshal(metadata) + if err != nil { + return fmt.Errorf("marshaling metadata for %s: %w", issueID, err) + } + + updates := map[string]interface{}{"metadata": string(raw)} + return s.UpdateIssue(ctx, issueID, updates, actor) +} + +// SlotGet retrieves the value of a metadata key from an issue. +func (s *DoltliteStore) SlotGet(ctx context.Context, issueID, key string) (string, error) { + issue, err := s.GetIssue(ctx, issueID) + if err != nil { + return "", fmt.Errorf("getting issue %s: %w", issueID, err) + } + + if len(issue.Metadata) == 0 { + return "", fmt.Errorf("no slot %q on %s: no metadata", key, issueID) + } + + metadata := make(map[string]interface{}) + if err := json.Unmarshal(issue.Metadata, &metadata); err != nil { + return "", fmt.Errorf("parsing metadata for %s: %w", issueID, err) + } + + val, ok := metadata[key] + if !ok { + return "", fmt.Errorf("no slot %q on %s: key not found", key, issueID) + } + + switch v := val.(type) { + case string: + return v, nil + default: + raw, err := json.Marshal(v) + if err != nil { + return "", fmt.Errorf("marshaling slot value for %s.%s: %w", issueID, key, err) + } + return string(raw), nil + } +} + +// SlotClear removes a metadata key from an issue. +func (s *DoltliteStore) SlotClear(ctx context.Context, issueID, key, actor string) error { + issue, err := s.GetIssue(ctx, issueID) + if err != nil { + return fmt.Errorf("getting issue %s: %w", issueID, err) + } + + if len(issue.Metadata) == 0 { + return nil + } + + metadata := make(map[string]interface{}) + if err := json.Unmarshal(issue.Metadata, &metadata); err != nil { + return fmt.Errorf("parsing metadata for %s: %w", issueID, err) + } + + if _, ok := metadata[key]; !ok { + return nil + } + delete(metadata, key) + + raw, err := json.Marshal(metadata) + if err != nil { + return fmt.Errorf("marshaling metadata for %s: %w", issueID, err) + } + + updates := map[string]interface{}{"metadata": string(raw)} + return s.UpdateIssue(ctx, issueID, updates, actor) +} diff --git a/internal/storage/doltlite/smoke_test.go b/internal/storage/doltlite/smoke_test.go new file mode 100644 index 000000000..459684c75 --- /dev/null +++ b/internal/storage/doltlite/smoke_test.go @@ -0,0 +1,119 @@ +//go:build cgo + +package doltlite_test + +import ( + "path/filepath" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/doltlite" + "github.com/steveyegge/beads/internal/types" +) + +func TestSmokeCreateGetCommit(t *testing.T) { + ctx := t.Context() + store, err := doltlite.New(ctx, filepath.Join(t.TempDir(), ".beads"), "beads", "main") + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + + now := time.Now().UTC() + issue := &types.Issue{ + ID: "bd-test", + Title: "doltlite smoke", + Description: "verify doltlite backend", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now, + UpdatedAt: now, + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + + got, err := store.GetIssue(ctx, issue.ID) + if err != nil { + t.Fatalf("GetIssue: %v", err) + } + if got.Title != issue.Title { + t.Fatalf("title = %q, want %q", got.Title, issue.Title) + } + + if err := store.Commit(ctx, "test: doltlite smoke"); err != nil { + t.Fatalf("Commit: %v", err) + } +} + +func TestSmokeVersionControl(t *testing.T) { + ctx := t.Context() + store, err := doltlite.New(ctx, filepath.Join(t.TempDir(), ".beads"), "beads", "main") + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + if err := store.Commit(ctx, "test: config"); err != nil { + t.Fatalf("Commit config: %v", err) + } + + branch, err := store.CurrentBranch(ctx) + if err != nil { + t.Fatalf("CurrentBranch: %v", err) + } + if branch != "main" { + t.Fatalf("branch = %q, want main", branch) + } + + if err := store.Branch(ctx, "feature"); err != nil { + t.Fatalf("Branch: %v", err) + } + if err := store.Checkout(ctx, "feature"); err != nil { + t.Fatalf("Checkout feature: %v", err) + } + branch, err = store.CurrentBranch(ctx) + if err != nil { + t.Fatalf("CurrentBranch feature: %v", err) + } + if branch != "feature" { + t.Fatalf("branch = %q, want feature", branch) + } + + branches, err := store.ListBranches(ctx) + if err != nil { + t.Fatalf("ListBranches: %v", err) + } + if len(branches) < 2 { + t.Fatalf("branches = %v, want at least main and feature", branches) + } + + if err := store.Checkout(ctx, "main"); err != nil { + t.Fatalf("Checkout main: %v", err) + } + if err := store.DeleteBranch(ctx, "feature"); err != nil { + t.Fatalf("DeleteBranch: %v", err) + } + + if _, err := store.Status(ctx); err != nil { + t.Fatalf("Status: %v", err) + } + if commits, err := store.Log(ctx, 5); err != nil { + t.Fatalf("Log: %v", err) + } else if len(commits) == 0 { + t.Fatal("Log returned no commits") + } + if hash, err := store.GetCurrentCommit(ctx); err != nil { + t.Fatalf("GetCurrentCommit: %v", err) + } else if hash == "" { + t.Fatal("GetCurrentCommit returned empty hash") + } +} diff --git a/internal/storage/doltlite/statistics.go b/internal/storage/doltlite/statistics.go new file mode 100644 index 000000000..cd9e64997 --- /dev/null +++ b/internal/storage/doltlite/statistics.go @@ -0,0 +1,36 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "fmt" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func (s *DoltliteStore) GetStatistics(ctx context.Context) (*types.Statistics, error) { + stats := &types.Statistics{} + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + if err := issueops.ScanIssueCountsInTx(ctx, tx, stats); err != nil { + return err + } + + blockedIDs, _, err := issueops.ComputeBlockedIDsInTx(ctx, tx, true) + if err != nil { + return err + } + stats.BlockedIssues = len(blockedIDs) + stats.ReadyIssues = stats.OpenIssues - stats.BlockedIssues + if stats.ReadyIssues < 0 { + stats.ReadyIssues = 0 + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("doltlite: get statistics: %w", err) + } + return stats, nil +} diff --git a/internal/storage/doltlite/store.go b/internal/storage/doltlite/store.go new file mode 100644 index 000000000..935f92da1 --- /dev/null +++ b/internal/storage/doltlite/store.go @@ -0,0 +1,860 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync/atomic" + "time" + + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/schema" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" + "github.com/steveyegge/beads/internal/types" +) + +// Compile-time interface checks. +var _ storage.DoltStorage = (*DoltliteStore)(nil) +var _ storage.StoreLocator = (*DoltliteStore)(nil) +var _ storage.GarbageCollector = (*DoltliteStore)(nil) +var _ storage.Flattener = (*DoltliteStore)(nil) +var _ storage.Compactor = (*DoltliteStore)(nil) + +// DoltliteStore implements storage.DoltStorage backed by the doltlite engine. +// Each method call opens a short-lived connection, executes within an explicit +// SQL transaction, and closes the connection immediately. This minimizes the +// time the embedded engine's write lock is held, reducing contention when +// multiple processes access the same database concurrently. +// +// The store holds an exclusive flock on the data directory for its entire +// lifetime. This prevents concurrent processes from initializing the embedded +// Dolt engine on the same directory, which causes a nil-pointer panic in +// DoltDB.SetCrashOnFatalError (GH#2571). +type DoltliteStore struct { + dataDir string + beadsDir string + database string + branch string + credentialKey []byte + closed atomic.Bool + lock Unlocker // exclusive flock held for the store's lifetime + ownsLock bool // true when New acquired the lock (false when caller supplied it via WithLock) +} + +// errClosed is returned when a method is called after Close. +var errClosed = errors.New("doltlite: store is closed") + +// Option configures optional behavior for New. +type Option func(*options) + +type options struct { + lock Unlocker // pre-acquired lock; nil means New acquires its own +} + +// WithLock passes a pre-acquired exclusive lock to New so it does not attempt +// to acquire a second one. The caller retains ownership — Close will NOT +// release a caller-supplied lock. This is used by bd init, which acquires the +// lock earlier to protect pre-initialization steps. +func WithLock(lock Unlocker) Option { + return func(o *options) { o.lock = lock } +} + +// New creates an DoltliteStore using the doltlite engine. +// beadsDir is the .beads/ root; the data directory is derived as /doltlite/. +// The database is created automatically if it doesn't exist (initSchema handles this). +// +// An exclusive flock is held on the data directory for the store's entire +// lifetime. If another process already holds the lock, New queues with +// exponential backoff until the lock becomes available or the context is +// canceled, instead of panicking during concurrent engine initialization +// (GH#2571). The lock is released when Close is called, unless a pre-acquired +// lock was supplied via WithLock (in which case the caller is responsible for it). +func New(ctx context.Context, beadsDir, database, branch string, opts ...Option) (*DoltliteStore, error) { + if database == "" { + return nil, fmt.Errorf("doltlite: database name must not be empty (caller should default to %q)", "beads") + } + + var o options + for _, fn := range opts { + fn(&o) + } + + // Resolve to absolute path so the SQLite database path is stable across + // callers with different working directories. + absBeadsDir, err := filepath.Abs(beadsDir) + if err != nil { + return nil, fmt.Errorf("doltlite: resolving beads dir: %w", err) + } + dataDir := filepath.Join(absBeadsDir, "doltlite") + if err := os.MkdirAll(dataDir, config.BeadsDirPerm); err != nil { + return nil, fmt.Errorf("doltlite: creating data directory: %w", err) + } + + // Acquire an exclusive flock before initializing the embedded engine. + // Without this, concurrent processes race through NewConnector → + // DoltDB.SetCrashOnFatalError → newDatabase → CollectDBs and one of + // them panics with a nil-pointer dereference (GH#2571). + lock := o.lock + ownsLock := lock == nil + if ownsLock { + lock, err = WaitLock(ctx, dataDir) + if err != nil { + return nil, err + } + } + + s := &DoltliteStore{ + dataDir: dataDir, + beadsDir: absBeadsDir, + database: database, + branch: branch, + lock: lock, + ownsLock: ownsLock, + } + + if err := s.initSchema(ctx); err != nil { + if ownsLock { + lock.Unlock() + } + return nil, fmt.Errorf("doltlite: init schema: %w", err) + } + + // Ensure dolt_ignore'd wisp tables exist in the working set. + // After a clone or branch switch, these tables are absent because + // dolt_ignore prevents them from being committed. Server mode handles + // this in newServerMode(); embedded mode must do it here. (GH#3270) + if err := s.ensureIgnoredTables(ctx); err != nil { + if ownsLock { + lock.Unlock() + } + return nil, fmt.Errorf("doltlite: ensure ignored tables: %w", err) + } + + return s, nil +} + +// withRootConn opens a short-lived database connection without selecting any +// database or branch, begins an explicit SQL transaction, and passes it to fn. +// This is used during initialization when the database may not yet exist. +func (s *DoltliteStore) withRootConn(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (err error) { + if s.closed.Load() { + err = errClosed + return + } + + var db *sql.DB + var cleanup func() error + db, cleanup, err = OpenSQL(ctx, s.dataDir, "", "") + if err != nil { + return + } + + defer func() { + err = errors.Join(err, cleanup()) + }() + + var tx *sql.Tx + tx, err = db.BeginTx(ctx, nil) + if err != nil { + err = fmt.Errorf("doltlite: begin tx: %w", err) + return + } + + err = fn(tx) + if err != nil { + err = errors.Join(err, tx.Rollback()) + return + } + + if !commit { + return tx.Rollback() + } + + err = tx.Commit() + return +} + +// withConn opens a short-lived database connection configured for the store's +// database and branch, begins an explicit SQL transaction, and passes it to +// fn. If commit is true and fn returns nil, the transaction is committed; +// otherwise it is rolled back. The connection is closed before withConn +// returns regardless of outcome. +// +// The database must already exist (created during initSchema). +func (s *DoltliteStore) withConn(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (err error) { + if s.closed.Load() { + err = errClosed + return + } + + var db *sql.DB + var cleanup func() error + db, cleanup, err = OpenSQL(ctx, s.dataDir, s.database, s.branch) + if err != nil { + return + } + + defer func() { + err = errors.Join(err, cleanup()) + }() + + var tx *sql.Tx + tx, err = db.BeginTx(ctx, nil) + if err != nil { + err = fmt.Errorf("doltlite: begin tx: %w", err) + return + } + + err = fn(tx) + if err != nil { + err = errors.Join(err, tx.Rollback()) + return + } + + if !commit { + return tx.Rollback() + } + + err = tx.Commit() + return +} + +// initSchema creates the database (if needed) and runs all pending migrations, +// committing them to Dolt history. Uses withRootConn so the database can be +// created before USE; this avoids running CREATE DATABASE inside withConn, +// which is not safe for concurrent use in the doltlite engine. +// +// After the schema-migration transaction commits, a fresh *sql.DB is opened +// and used to drive the idempotent compat-migration runner. Mirrors the +// server-mode open path in dolt/store.go:initSchemaOnDB and repairs +// pre-existing embedded databases that predate the embedded migration +// system's full coverage (GH#3412). +func (s *DoltliteStore) initSchema(ctx context.Context) error { + if s.database != "" && !validIdentifier.MatchString(s.database) { + return fmt.Errorf("doltlite: invalid database name: %q", s.database) + } + + db, cleanup, err := OpenSQL(ctx, s.dataDir, s.database, s.branch) + if err != nil { + return fmt.Errorf("doltlite: open for schema init: %w", err) + } + defer func() { _ = cleanup() }() + + if err := schema.CreateIgnoredTablesSQLite(ctx, db); err != nil { + return fmt.Errorf("ensure ignored tables before migration: %w", err) + } + + applied, err := schema.MigrateUpSQLite(ctx, db) + if err != nil { + return err + } + if applied > 0 { + if _, err := db.ExecContext(ctx, "SELECT dolt_add('-A')"); err != nil { + return fmt.Errorf("dolt add after migrations: %w", err) + } + if _, err := db.ExecContext(ctx, "SELECT dolt_commit('-m', 'schema: apply migrations')"); err != nil { + if !strings.Contains(err.Error(), "nothing to commit") { + return fmt.Errorf("dolt commit after migrations: %w", err) + } + } + } + + return nil +} + +// ensureIgnoredTables creates dolt_ignore'd wisp tables if they don't exist. +// Uses withConn (not withRootConn) because the database is already created. +func (s *DoltliteStore) ensureIgnoredTables(ctx context.Context) error { + return s.withConn(ctx, false, func(tx *sql.Tx) error { + return schema.CreateIgnoredTablesSQLite(ctx, tx) + }) +} + +// GetIssue is implemented in get_issue.go. + +func (s *DoltliteStore) GetIssueByExternalRef(ctx context.Context, externalRef string) (*types.Issue, error) { + var id string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + id, err = issueops.GetIssueByExternalRefInTx(ctx, tx, externalRef) + return err + }) + if err != nil { + return nil, err + } + return s.GetIssue(ctx, id) +} + +// GetIssuesByIDs is implemented in dependencies.go. + +// UpdateIssue is implemented in issues.go. + +// CloseIssue is implemented in issues.go. + +func (s *DoltliteStore) DeleteIssue(ctx context.Context, id string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.DeleteIssueInTx(ctx, tx, id) + }) +} + +// AddDependency is implemented in dependencies.go. + +// RemoveDependency is implemented in dependencies.go. + +func (s *DoltliteStore) GetDependencies(ctx context.Context, issueID string) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependenciesInTx(ctx, tx, issueID) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetDependents(ctx context.Context, issueID string) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependentsInTx(ctx, tx, issueID) + return err + }) + return result, err +} + +// GetDependenciesWithMetadata is implemented in dependencies.go. + +// GetDependentsWithMetadata is implemented in dependencies.go. + +func (s *DoltliteStore) GetDependencyTree(ctx context.Context, issueID string, maxDepth int, showAllPaths bool, reverse bool) ([]*types.TreeNode, error) { + var result []*types.TreeNode + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetDependencyTreeInTx(ctx, tx, issueID, maxDepth, showAllPaths, reverse) + return err + }) + return result, err +} + +// AddLabel is implemented in labels.go. + +// RemoveLabel is implemented in labels.go. + +// GetLabels is implemented in labels.go. + +func (s *DoltliteStore) GetIssuesByLabel(ctx context.Context, label string) ([]*types.Issue, error) { + var ids []string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + ids, err = issueops.GetIssuesByLabelInTx(ctx, tx, label) + return err + }) + if err != nil { + return nil, err + } + return s.GetIssuesByIDs(ctx, ids) +} + +// GetReadyWork is implemented in queries.go. + +func (s *DoltliteStore) GetBlockedIssues(ctx context.Context, filter types.WorkFilter) ([]*types.BlockedIssue, error) { + var result []*types.BlockedIssue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetBlockedIssuesInTx(ctx, tx, filter) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetEpicsEligibleForClosure(ctx context.Context) ([]*types.EpicStatus, error) { + var result []*types.EpicStatus + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetEpicsEligibleForClosureInTx(ctx, tx) + return err + }) + return result, err +} + +func (s *DoltliteStore) AddIssueComment(ctx context.Context, issueID, author, text string) (*types.Comment, error) { + var result *types.Comment + err := s.withConn(ctx, true, func(tx *sql.Tx) error { + var err error + result, err = issueops.AddIssueCommentInTx(ctx, tx, issueID, author, text) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetIssueComments(ctx context.Context, issueID string) ([]*types.Comment, error) { + var result []*types.Comment + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetIssueCommentsInTx(ctx, tx, issueID) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetEvents(ctx context.Context, issueID string, limit int) ([]*types.Event, error) { + var result []*types.Event + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetEventsInTx(ctx, tx, issueID, limit) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetAllEventsSince(ctx context.Context, since time.Time) ([]*types.Event, error) { + var result []*types.Event + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetAllEventsSinceInTx(ctx, tx, since) + return err + }) + return result, err +} + +// RunInTransaction is implemented in transaction.go. + +// Close marks the store as closed, cleans up orphaned git-remote-cache +// garbage, and releases the exclusive flock on the data directory (if the +// store owns it). Subsequent method calls will return errClosed. +// It is safe to call multiple times. When the lock was supplied by the caller +// via WithLock, Close does NOT release it — the caller retains ownership. +func (s *DoltliteStore) Close() error { + // Use CompareAndSwap so we only unlock once even if Close is called + // multiple times (the Lock.Unlock method panics on double-unlock). + if s.closed.CompareAndSwap(false, true) { + s.cleanGitRemoteCacheGarbage() + if s.lock != nil && s.ownsLock { + s.lock.Unlock() + } + } + return nil +} + +// DoltGC runs Dolt garbage collection to reclaim disk space. +func (s *DoltliteStore) DoltGC(ctx context.Context) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + _, err := db.ExecContext(ctx, "SELECT dolt_gc()") + return err + }) +} + +// Flatten squashes all Dolt commit history into a single commit. +// Pins a single *sql.Conn for session-scoped stored procedures. +func (s *DoltliteStore) Flatten(ctx context.Context) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if pooled, ok := db.(*sql.DB); ok { + conn, err := pooled.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + return versioncontrolops.Flatten(ctx, conn) + } + return versioncontrolops.Flatten(ctx, db) + }) +} + +// Compact squashes old Dolt commits while preserving recent ones. +// Pins a single *sql.Conn for session-scoped stored procedures. +func (s *DoltliteStore) Compact(ctx context.Context, initialHash, boundaryHash string, oldCommits int, recentHashes []string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + // withDBConn returns *sql.DB; pin a single connection for + // session-scoped operations (checkout, reset, cherry-pick). + if pooled, ok := db.(*sql.DB); ok { + conn, err := pooled.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + return versioncontrolops.Compact(ctx, conn, initialHash, boundaryHash, oldCommits, recentHashes) + } + return versioncontrolops.Compact(ctx, db, initialHash, boundaryHash, oldCommits, recentHashes) + }) +} + +// Path returns the doltlite data directory (.beads/doltlite/). +func (s *DoltliteStore) Path() string { + return s.dataDir +} + +// CLIDir returns the directory for dolt CLI operations (push/pull/remote). +// This is the actual database directory within the data dir. +func (s *DoltliteStore) CLIDir() string { + if s.dataDir == "" { + return "" + } + return filepath.Join(s.dataDir, s.database) +} + +// --------------------------------------------------------------------------- +// storage.VersionControl +// --------------------------------------------------------------------------- + +// Branch, Checkout, CurrentBranch, DeleteBranch, ListBranches are +// implemented in version_control.go via versioncontrolops. + +func (s *DoltliteStore) CommitPending(ctx context.Context, actor string) (bool, error) { + var hasPending bool + var msg string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + hasPending, err = issueops.HasPendingChanges(ctx, tx) + if err != nil { + return err + } + if hasPending { + msg = issueops.BuildBatchCommitMessage(ctx, tx, actor) + } + return nil + }) + if err != nil { + return false, err + } + if !hasPending { + return false, nil + } + + if err := s.Commit(ctx, msg); err != nil { + if issueops.IsNothingToCommitError(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// CommitExists is implemented in version_control.go via versioncontrolops. + +func (s *DoltliteStore) GetCurrentCommit(ctx context.Context) (string, error) { + var hash string + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + return db.QueryRowContext(ctx, "SELECT dolt_hashof('HEAD')").Scan(&hash) + }) + return hash, err +} + +// Status, Log, Merge, GetConflicts, ResolveConflicts are implemented in +// version_control.go via versioncontrolops. + +// --------------------------------------------------------------------------- +// storage.HistoryViewer +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) History(ctx context.Context, issueID string) ([]*storage.HistoryEntry, error) { + var result []*storage.HistoryEntry + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.HistoryInTx(ctx, tx, issueID) + return err + }) + return result, err +} + +func (s *DoltliteStore) AsOf(ctx context.Context, issueID string, ref string) (*types.Issue, error) { + var result *types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.AsOfInTx(ctx, tx, issueID, ref) + return err + }) + return result, err +} + +func (s *DoltliteStore) Diff(ctx context.Context, fromRef, toRef string) ([]*storage.DiffEntry, error) { + var result []*storage.DiffEntry + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.DiffInTx(ctx, tx, fromRef, toRef) + return err + }) + return result, err +} + +// --------------------------------------------------------------------------- +// storage.RemoteStore +// --------------------------------------------------------------------------- + +// RemoveRemote, ListRemotes, Push, Pull, ForcePush, Fetch, PushTo, PullFrom +// are implemented in version_control.go via versioncontrolops. + +// --------------------------------------------------------------------------- +// storage.SyncStore +// --------------------------------------------------------------------------- + +// Sync and SyncStatus are implemented in federation.go. + +// --------------------------------------------------------------------------- +// storage.FederationStore +// --------------------------------------------------------------------------- + +// AddFederationPeer, GetFederationPeer, ListFederationPeers, RemoveFederationPeer +// are implemented in federation.go via issueops. + +// --------------------------------------------------------------------------- +// storage.BulkIssueStore +// --------------------------------------------------------------------------- + +// CreateIssuesWithFullOptions is implemented in create_issue.go. + +func (s *DoltliteStore) DeleteIssues(ctx context.Context, ids []string, cascade bool, force bool, dryRun bool) (*types.DeleteIssuesResult, error) { + var result *types.DeleteIssuesResult + err := s.withConn(ctx, !dryRun, func(tx *sql.Tx) error { + var err error + result, err = issueops.DeleteIssuesInTx(ctx, tx, ids, cascade, force, dryRun) + return err + }) + return result, err +} + +func (s *DoltliteStore) DeleteIssuesBySourceRepo(ctx context.Context, sourceRepo string) (int, error) { + var count int + err := s.withConn(ctx, true, func(tx *sql.Tx) error { + var err error + count, err = issueops.DeleteIssuesBySourceRepoInTx(ctx, tx, sourceRepo) + return err + }) + return count, err +} + +func (s *DoltliteStore) UpdateIssueID(ctx context.Context, oldID, newID string, issue *types.Issue, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.UpdateIssueIDInTx(ctx, tx, oldID, newID, issue, actor) + }) +} + +// ClaimIssue is implemented in issues.go. + +func (s *DoltliteStore) PromoteFromEphemeral(ctx context.Context, id string, actor string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.PromoteFromEphemeralInTx(ctx, tx, id, actor) + }) +} + +// GetNextChildID is implemented in child_id.go. + +func (s *DoltliteStore) RenameCounterPrefix(ctx context.Context, oldPrefix, newPrefix string) error { + return nil // Hash-based IDs don't use counters. +} + +// --------------------------------------------------------------------------- +// storage.DependencyQueryStore +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) GetDependencyRecords(ctx context.Context, issueID string) ([]*types.Dependency, error) { + var result []*types.Dependency + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + m, err := issueops.GetDependencyRecordsForIssuesInTx(ctx, tx, []string{issueID}) + if err != nil { + return err + } + result = m[issueID] + return nil + }) + return result, err +} + +// IsBlocked is implemented in issues.go. + +// GetNewlyUnblockedByClose is implemented in issues.go. + +// DetectCycles is implemented in dependencies.go. + +func (s *DoltliteStore) FindWispDependentsRecursive(ctx context.Context, ids []string) (map[string]bool, error) { + var result map[string]bool + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.FindWispDependentsRecursiveInTx(ctx, tx, ids) + return err + }) + return result, err +} + +func (s *DoltliteStore) RenameDependencyPrefix(ctx context.Context, oldPrefix, newPrefix string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.RenameDependencyPrefixInTx(ctx, tx, oldPrefix, newPrefix) + }) +} + +// --------------------------------------------------------------------------- +// storage.AnnotationQueryStore +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) AddComment(ctx context.Context, issueID, actor, comment string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.AddCommentEventInTx(ctx, tx, issueID, actor, comment) + }) +} + +func (s *DoltliteStore) ImportIssueComment(ctx context.Context, issueID, author, text string, createdAt time.Time) (*types.Comment, error) { + var result *types.Comment + err := s.withConn(ctx, true, func(tx *sql.Tx) error { + var err error + result, err = issueops.ImportIssueCommentInTx(ctx, tx, issueID, author, text, createdAt) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetCommentsForIssues(ctx context.Context, issueIDs []string) (map[string][]*types.Comment, error) { + var result map[string][]*types.Comment + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetCommentsForIssuesInTx(ctx, tx, issueIDs) + return err + }) + return result, err +} + +// --------------------------------------------------------------------------- +// storage.ConfigMetadataStore +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) DeleteConfig(ctx context.Context, key string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.DeleteConfigInTx(ctx, tx, key) + }) +} + +func (s *DoltliteStore) GetCustomStatuses(ctx context.Context) ([]string, error) { + detailed, err := s.GetCustomStatusesDetailed(ctx) + if err != nil { + return nil, err + } + return types.CustomStatusNames(detailed), nil +} + +func (s *DoltliteStore) GetCustomStatusesDetailed(ctx context.Context) ([]types.CustomStatus, error) { + var result []types.CustomStatus + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var txErr error + result, txErr = issueops.ResolveCustomStatusesDetailedInTx(ctx, tx) + return txErr + }) + if err != nil { + // DB unavailable — fall back to config.yaml. + if yamlStatuses := config.GetCustomStatusesFromYAML(); len(yamlStatuses) > 0 { + return issueops.ParseStatusFallback(yamlStatuses), nil + } + return nil, nil + } + return result, nil +} + +func (s *DoltliteStore) GetCustomTypes(ctx context.Context) ([]string, error) { + var result []string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var txErr error + result, txErr = issueops.ResolveCustomTypesInTx(ctx, tx) + return txErr + }) + if err != nil { + // DB unavailable — fall back to config.yaml. + if yamlTypes := config.GetCustomTypesFromYAML(); len(yamlTypes) > 0 { + return yamlTypes, nil + } + return nil, err + } + return result, nil +} + +// --------------------------------------------------------------------------- +// storage.CompactionStore +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) CheckEligibility(ctx context.Context, issueID string, tier int) (bool, string, error) { + var eligible bool + var reason string + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + eligible, reason, err = issueops.CheckEligibilityInTx(ctx, tx, issueID, tier) + return err + }) + return eligible, reason, err +} + +func (s *DoltliteStore) ApplyCompaction(ctx context.Context, issueID string, tier int, originalSize int, _ int, commitHash string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.ApplyCompactionInTx(ctx, tx, issueID, tier, originalSize, commitHash) + }) +} + +func (s *DoltliteStore) GetTier1Candidates(ctx context.Context) ([]*types.CompactionCandidate, error) { + var result []*types.CompactionCandidate + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetTier1CandidatesInTx(ctx, tx) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetTier2Candidates(ctx context.Context) ([]*types.CompactionCandidate, error) { + var result []*types.CompactionCandidate + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetTier2CandidatesInTx(ctx, tx) + return err + }) + return result, err +} + +// --------------------------------------------------------------------------- +// storage.AdvancedQueryStore +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) GetRepoMtime(ctx context.Context, repoPath string) (int64, error) { + var result int64 + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetRepoMtimeInTx(ctx, tx, repoPath) + return err + }) + return result, err +} + +func (s *DoltliteStore) SetRepoMtime(ctx context.Context, repoPath, jsonlPath string, mtimeNs int64) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.SetRepoMtimeInTx(ctx, tx, repoPath, jsonlPath, mtimeNs) + }) +} + +func (s *DoltliteStore) ClearRepoMtime(ctx context.Context, repoPath string) error { + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return issueops.ClearRepoMtimeInTx(ctx, tx, repoPath) + }) +} + +// GetMoleculeProgress is implemented in queries.go. + +func (s *DoltliteStore) GetMoleculeLastActivity(ctx context.Context, moleculeID string) (*types.MoleculeLastActivity, error) { + var result *types.MoleculeLastActivity + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetMoleculeLastActivityInTx(ctx, tx, moleculeID) + return err + }) + return result, err +} + +func (s *DoltliteStore) GetStaleIssues(ctx context.Context, filter types.StaleFilter) ([]*types.Issue, error) { + var result []*types.Issue + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetStaleIssuesInTx(ctx, tx, filter) + return err + }) + return result, err +} diff --git a/internal/storage/doltlite/store_stub.go b/internal/storage/doltlite/store_stub.go new file mode 100644 index 000000000..b3394dae1 --- /dev/null +++ b/internal/storage/doltlite/store_stub.go @@ -0,0 +1,28 @@ +//go:build !cgo + +package doltlite + +import ( + "context" + "errors" +) + +// DoltliteStore is a stub for builds without CGO. +type DoltliteStore struct { + dataDir string + database string + branch string +} + +// Option configures optional behavior for New (stub: no-op). +type Option func(*struct{}) + +// WithLock is a no-op in non-CGO builds. +func WithLock(_ Unlocker) Option { + return func(*struct{}) {} +} + +// New returns an error when CGO is not enabled. +func New(_ context.Context, _, _, _ string, _ ...Option) (*DoltliteStore, error) { + return nil, errors.New("doltlite: requires CGO (build with CGO_ENABLED=1)") +} diff --git a/internal/storage/doltlite/transaction.go b/internal/storage/doltlite/transaction.go new file mode 100644 index 000000000..08345403a --- /dev/null +++ b/internal/storage/doltlite/transaction.go @@ -0,0 +1,194 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" + "github.com/steveyegge/beads/internal/types" +) + +// RunInTransaction executes a function within a database transaction. +// After the SQL transaction commits, dirty tables are selectively staged +// and a Dolt version commit is created with the given message. +func (s *DoltliteStore) RunInTransaction(ctx context.Context, commitMsg string, fn func(tx storage.Transaction) error) error { + var tracker versioncontrolops.DirtyTableTracker + + if err := s.withConn(ctx, true, func(sqlTx *sql.Tx) error { + tx := &embeddedTransaction{tx: sqlTx, dirty: &tracker} + return fn(tx) + }); err != nil { + return err + } + + // Create a Dolt version commit from the working set changes. + if commitMsg != "" && len(tracker.DirtyTables()) > 0 { + return s.Commit(ctx, commitMsg) + } + return nil +} + +// embeddedTransaction implements storage.Transaction for DoltliteStore. +type embeddedTransaction struct { + tx *sql.Tx + dirty *versioncontrolops.DirtyTableTracker +} + +func (t *embeddedTransaction) CreateIssue(ctx context.Context, issue *types.Issue, actor string) error { + bc, err := issueops.NewBatchContext(ctx, t.tx, storage.BatchCreateOptions{SkipPrefixValidation: true}) + if err != nil { + return err + } + t.dirty.MarkDirty("issues") + t.dirty.MarkDirty("events") + return createIssueSQLite(ctx, t.tx, bc, issue, actor) +} + +func (t *embeddedTransaction) CreateIssues(ctx context.Context, issues []*types.Issue, actor string) error { + for _, issue := range issues { + if err := t.CreateIssue(ctx, issue, actor); err != nil { + return err + } + } + return nil +} + +func (t *embeddedTransaction) UpdateIssue(ctx context.Context, id string, updates map[string]interface{}, actor string) error { + t.dirty.MarkDirty("issues") + t.dirty.MarkDirty("events") + _, err := issueops.UpdateIssueInTx(ctx, t.tx, id, updates, actor) + return err +} + +func (t *embeddedTransaction) CloseIssue(ctx context.Context, id string, reason string, actor string, session string) error { + t.dirty.MarkDirty("issues") + t.dirty.MarkDirty("events") + _, err := issueops.CloseIssueInTx(ctx, t.tx, id, reason, actor, session) + return err +} + +func (t *embeddedTransaction) DeleteIssue(ctx context.Context, id string) error { + t.dirty.MarkDirty("issues") + t.dirty.MarkDirty("dependencies") + t.dirty.MarkDirty("labels") + t.dirty.MarkDirty("comments") + t.dirty.MarkDirty("events") + return issueops.DeleteIssueInTx(ctx, t.tx, id) +} + +func (t *embeddedTransaction) GetIssue(ctx context.Context, id string) (*types.Issue, error) { + return issueops.GetIssueInTx(ctx, t.tx, id) +} + +func (t *embeddedTransaction) SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { + return issueops.SearchIssuesInTx(ctx, t.tx, query, filter) +} + +func (t *embeddedTransaction) AddDependency(ctx context.Context, dep *types.Dependency, actor string) error { + return t.AddDependencyWithOptions(ctx, dep, actor, storage.DependencyAddOptions{}) +} + +func (t *embeddedTransaction) AddDependencyWithOptions(ctx context.Context, dep *types.Dependency, actor string, addOpts storage.DependencyAddOptions) error { + t.dirty.MarkDirty("dependencies") + return issueops.AddDependencyInTx(ctx, t.tx, dep, actor, issueops.AddDependencyOpts{ + IsCrossPrefix: types.ExtractPrefix(dep.IssueID) != types.ExtractPrefix(dep.DependsOnID), + SkipCycleCheck: addOpts.SkipCycleCheck, + }) +} + +func (t *embeddedTransaction) RemoveDependency(ctx context.Context, issueID, dependsOnID string, actor string) error { + t.dirty.MarkDirty("dependencies") + return issueops.RemoveDependencyInTx(ctx, t.tx, issueID, dependsOnID) +} + +func (t *embeddedTransaction) GetDependencyRecords(ctx context.Context, issueID string) ([]*types.Dependency, error) { + m, err := issueops.GetDependencyRecordsForIssuesInTx(ctx, t.tx, []string{issueID}) + if err != nil { + return nil, err + } + return m[issueID], nil +} + +func (t *embeddedTransaction) AddLabel(ctx context.Context, issueID, label, actor string) error { + t.dirty.MarkDirty("labels") + return issueops.AddLabelInTx(ctx, t.tx, "", "", issueID, label, actor) +} + +func (t *embeddedTransaction) RemoveLabel(ctx context.Context, issueID, label, actor string) error { + t.dirty.MarkDirty("labels") + return issueops.RemoveLabelInTx(ctx, t.tx, "", "", issueID, label, actor) +} + +func (t *embeddedTransaction) GetLabels(ctx context.Context, issueID string) ([]string, error) { + return issueops.GetLabelsInTx(ctx, t.tx, "", issueID) +} + +func (t *embeddedTransaction) SetConfig(ctx context.Context, key, value string) error { + t.dirty.MarkDirty("config") + if err := issueops.SetConfigInTx(ctx, t.tx, key, value); err != nil { + return err + } + // Sync normalized tables when config keys change + switch key { + case "status.custom": + t.dirty.MarkDirty("custom_statuses") + if err := issueops.SyncCustomStatusesTable(ctx, t.tx, value); err != nil { + return fmt.Errorf("syncing custom_statuses table: %w", err) + } + case "types.custom": + t.dirty.MarkDirty("custom_types") + if err := issueops.SyncCustomTypesTable(ctx, t.tx, value); err != nil { + return fmt.Errorf("syncing custom_types table: %w", err) + } + } + return nil +} + +func (t *embeddedTransaction) GetConfig(ctx context.Context, key string) (string, error) { + return issueops.GetConfigInTx(ctx, t.tx, key) +} + +func (t *embeddedTransaction) SetMetadata(ctx context.Context, key, value string) error { + t.dirty.MarkDirty("metadata") + return issueops.SetMetadataInTx(ctx, t.tx, key, value) +} + +func (t *embeddedTransaction) GetMetadata(ctx context.Context, key string) (string, error) { + return issueops.GetMetadataInTx(ctx, t.tx, key) +} + +func (t *embeddedTransaction) SetLocalMetadata(ctx context.Context, key, value string) error { + return issueops.SetLocalMetadataInTx(ctx, t.tx, key, value) +} + +func (t *embeddedTransaction) GetLocalMetadata(ctx context.Context, key string) (string, error) { + return issueops.GetLocalMetadataInTx(ctx, t.tx, key) +} + +func (t *embeddedTransaction) AddComment(ctx context.Context, issueID, actor, comment string) error { + return fmt.Errorf("embeddedTransaction: AddComment not implemented") +} + +func (t *embeddedTransaction) ImportIssueComment(ctx context.Context, issueID, author, text string, createdAt time.Time) (*types.Comment, error) { + return nil, fmt.Errorf("embeddedTransaction: ImportIssueComment not implemented") +} + +func (t *embeddedTransaction) GetIssueComments(ctx context.Context, issueID string) ([]*types.Comment, error) { + return nil, fmt.Errorf("embeddedTransaction: GetIssueComments not implemented") +} + +func (t *embeddedTransaction) CreateIssueImport(ctx context.Context, issue *types.Issue, actor string, skipPrefixValidation bool) error { + bc, err := issueops.NewBatchContext(ctx, t.tx, storage.BatchCreateOptions{SkipPrefixValidation: skipPrefixValidation}) + if err != nil { + return err + } + t.dirty.MarkDirty("issues") + t.dirty.MarkDirty("events") + return issueops.CreateIssueInTx(ctx, t.tx, bc, issue, actor) +} diff --git a/internal/storage/doltlite/version_control.go b/internal/storage/doltlite/version_control.go new file mode 100644 index 000000000..bf40584a6 --- /dev/null +++ b/internal/storage/doltlite/version_control.go @@ -0,0 +1,426 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/schema" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" +) + +// withDBConn opens a short-lived database connection configured for the +// store's database and branch and passes it to fn. Unlike withConn, no +// transaction is started — this is required for Dolt stored procedures +// (CALL DOLT_BRANCH, CALL DOLT_MERGE, etc.) that cannot run inside +// explicit SQL transactions. +func (s *DoltliteStore) withDBConn(ctx context.Context, fn func(db versioncontrolops.DBConn) error) (err error) { + if s.closed.Load() { + return errClosed + } + + var db *sql.DB + var cleanup func() error + db, cleanup, err = OpenSQL(ctx, s.dataDir, s.database, s.branch) + if err != nil { + return + } + defer func() { + err = errors.Join(err, cleanup()) + // Best-effort cleanup of orphaned tmp_pack_* files left by git + // fetch in the Dolt git-remote-cache. Rate-limited internally. + s.cleanGitRemoteCacheGarbage() + }() + + return fn(db) +} + +func (s *DoltliteStore) Commit(ctx context.Context, message string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_add('-A')"); err != nil { + return fmt.Errorf("dolt add: %w", err) + } + if _, err := db.ExecContext(ctx, "SELECT dolt_commit('-m', ?)", message); err != nil { + return fmt.Errorf("dolt commit: %w", err) + } + return nil + }) +} + +// CommitWithConfig commits all working set changes including config. +// DoltliteStore.Commit already includes config via DOLT_ADD('-A'), +// so this is just an alias to satisfy the VersionControl interface (GH#3216). +func (s *DoltliteStore) CommitWithConfig(ctx context.Context, message string) error { + return s.Commit(ctx, message) +} + +func (s *DoltliteStore) AddRemote(ctx context.Context, name, url string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + _, err := db.ExecContext(ctx, "SELECT dolt_remote('add', ?, ?)", name, url) + return err + }) +} + +func (s *DoltliteStore) HasRemote(ctx context.Context, name string) (bool, error) { + var count int + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + return tx.QueryRowContext(ctx, "SELECT count(*) FROM dolt_remotes WHERE name = ?", name).Scan(&count) + }) + if err != nil { + return false, err + } + return count > 0, nil +} + +// --------------------------------------------------------------------------- +// Branch operations +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) Branch(ctx context.Context, name string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_branch(?)", name); err != nil { + return fmt.Errorf("create branch %s: %w", name, err) + } + return schema.CreateIgnoredTablesSQLite(ctx, db) + }) +} + +func (s *DoltliteStore) Checkout(ctx context.Context, branch string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_checkout(?)", branch); err != nil { + return fmt.Errorf("checkout branch %s: %w", branch, err) + } + return schema.CreateIgnoredTablesSQLite(ctx, db) + }) +} + +func (s *DoltliteStore) CurrentBranch(ctx context.Context) (string, error) { + var branch string + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + branch, err = versioncontrolops.CurrentBranch(ctx, db) + return err + }) + return branch, err +} + +func (s *DoltliteStore) DeleteBranch(ctx context.Context, branch string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_branch('-D', ?)", branch); err != nil { + return fmt.Errorf("delete branch %s: %w", branch, err) + } + return nil + }) +} + +func (s *DoltliteStore) ListBranches(ctx context.Context) ([]string, error) { + var branches []string + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + branches, err = versioncontrolops.ListBranches(ctx, db) + return err + }) + return branches, err +} + +// --------------------------------------------------------------------------- +// Version control operations +// --------------------------------------------------------------------------- + +// commitAuthor returns the author string for merge commits. +const commitAuthor = commitName + " <" + commitEmail + ">" + +func (s *DoltliteStore) CommitExists(ctx context.Context, commitHash string) (bool, error) { + var exists bool + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + exists, err = versioncontrolops.CommitExists(ctx, db, commitHash) + return err + }) + return exists, err +} + +func (s *DoltliteStore) Status(ctx context.Context) (*storage.Status, error) { + var status *storage.Status + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + status, err = versioncontrolops.Status(ctx, db) + return err + }) + return status, err +} + +func (s *DoltliteStore) Log(ctx context.Context, limit int) ([]storage.CommitInfo, error) { + var query string + var args []any + if limit > 0 { + query = "SELECT commit_hash, committer, email, date, message FROM dolt_log ORDER BY date DESC LIMIT ?" + args = []any{limit} + } else { + query = "SELECT commit_hash, committer, email, date, message FROM dolt_log ORDER BY date DESC" + } + var commits []storage.CommitInfo + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + rows, err := db.QueryContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("get log: %w", err) + } + defer rows.Close() + for rows.Next() { + var c storage.CommitInfo + var date string + if err := rows.Scan(&c.Hash, &c.Author, &c.Email, &date, &c.Message); err != nil { + return fmt.Errorf("scan commit: %w", err) + } + c.Date = parseDoltliteTime(date) + commits = append(commits, c) + } + return rows.Err() + }) + return commits, err +} + +func parseDoltliteTime(s string) time.Time { + for _, layout := range []string{ + time.RFC3339Nano, + time.RFC3339, + "2006-01-02 15:04:05.999999999-07:00", + "2006-01-02 15:04:05.999999999Z07:00", + "2006-01-02 15:04:05", + } { + if t, err := time.Parse(layout, s); err == nil { + return t + } + } + return time.Time{} +} + +func (s *DoltliteStore) Merge(ctx context.Context, branch string) ([]storage.Conflict, error) { + var conflicts []storage.Conflict + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + _, err := db.ExecContext(ctx, "SELECT dolt_merge(?)", branch) + if err != nil { + c, conflictErr := versioncontrolops.GetConflicts(ctx, db) + if conflictErr == nil && len(c) > 0 { + conflicts = c + return nil + } + return fmt.Errorf("merge branch %s: %w", branch, err) + } + return nil + }) + return conflicts, err +} + +func (s *DoltliteStore) GetConflicts(ctx context.Context) ([]storage.Conflict, error) { + var conflicts []storage.Conflict + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + conflicts, err = versioncontrolops.GetConflicts(ctx, db) + return err + }) + return conflicts, err +} + +func (s *DoltliteStore) ResolveConflicts(ctx context.Context, table string, strategy string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + switch strategy { + case "ours": + _, err := db.ExecContext(ctx, "SELECT dolt_conflicts_resolve('--ours', ?)", table) + return err + case "theirs": + _, err := db.ExecContext(ctx, "SELECT dolt_conflicts_resolve('--theirs', ?)", table) + return err + default: + return fmt.Errorf("unknown conflict resolution strategy: %s", strategy) + } + }) +} + +// --------------------------------------------------------------------------- +// Remote operations +// --------------------------------------------------------------------------- + +const defaultRemote = "origin" + +func (s *DoltliteStore) RemoveRemote(ctx context.Context, name string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + _, err := db.ExecContext(ctx, "SELECT dolt_remote('remove', ?)", name) + return err + }) +} + +func (s *DoltliteStore) ListRemotes(ctx context.Context) ([]storage.RemoteInfo, error) { + var remotes []storage.RemoteInfo + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + remotes, err = versioncontrolops.ListRemotes(ctx, db) + return err + }) + return remotes, err +} + +func (s *DoltliteStore) Push(ctx context.Context) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + _, err := db.ExecContext(ctx, "SELECT dolt_push(?, ?)", defaultRemote, s.branch) + return err + }) +} + +func (s *DoltliteStore) Pull(ctx context.Context) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + _, err := db.ExecContext(ctx, "SELECT dolt_pull(?, ?)", defaultRemote, s.branch) + return err + }) +} + +func (s *DoltliteStore) ForcePush(ctx context.Context) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + _, err := db.ExecContext(ctx, "SELECT dolt_push(?, ?, '--force')", defaultRemote, s.branch) + return err + }) +} + +func (s *DoltliteStore) PushRemote(ctx context.Context, remote string, force bool) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if force { + _, err := db.ExecContext(ctx, "SELECT dolt_push(?, ?, '--force')", remote, s.branch) + return err + } + _, err := db.ExecContext(ctx, "SELECT dolt_push(?, ?)", remote, s.branch) + return err + }) +} + +func (s *DoltliteStore) PullRemote(ctx context.Context, remote string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + _, err := db.ExecContext(ctx, "SELECT dolt_pull(?, ?)", remote, s.branch) + return err + }) +} + +func (s *DoltliteStore) Fetch(ctx context.Context, peer string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + _, err := db.ExecContext(ctx, "SELECT dolt_fetch(?)", peer) + return err + }) +} + +func (s *DoltliteStore) PushTo(ctx context.Context, peer string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + _, err := db.ExecContext(ctx, "SELECT dolt_push(?, ?)", peer, s.branch) + return err + }) +} + +func (s *DoltliteStore) PullFrom(ctx context.Context, peer string) ([]storage.Conflict, error) { + // Auto-commit pending changes before pull to prevent + // "cannot merge with uncommitted changes" errors. + if _, err := s.CommitPending(ctx, "beads"); err != nil { + return nil, fmt.Errorf("commit pending before pull: %w", err) + } + + var conflicts []storage.Conflict + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + if _, pullErr := db.ExecContext(ctx, "SELECT dolt_pull(?, ?)", peer, s.branch); pullErr != nil { + c, conflictErr := versioncontrolops.GetConflicts(ctx, db) + if conflictErr == nil && len(c) > 0 { + conflicts = c + return nil + } + return fmt.Errorf("pull from %s: %w", peer, pullErr) + } + return nil + }) + return conflicts, err +} + +// --------------------------------------------------------------------------- +// Backup operations +// --------------------------------------------------------------------------- + +func (s *DoltliteStore) BackupAdd(ctx context.Context, name, url string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + return versioncontrolops.BackupAdd(ctx, db, name, url) + }) +} + +func (s *DoltliteStore) BackupSync(ctx context.Context, name string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + return versioncontrolops.BackupSync(ctx, db, name) + }) +} + +func (s *DoltliteStore) BackupRemove(ctx context.Context, name string) error { + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + return versioncontrolops.BackupRemove(ctx, db, name) + }) +} + +// BackupDatabase registers dir as a file:// Dolt backup remote and syncs +// the database to it. The dir must exist locally. This preserves full Dolt +// commit history. +func (s *DoltliteStore) BackupDatabase(ctx context.Context, dir string) error { + info, err := os.Stat(dir) + if err != nil { + return fmt.Errorf("backup destination does not exist: %w", err) + } + if !info.IsDir() { + return fmt.Errorf("backup destination is not a directory: %s", dir) + } + + backupURL, err := versioncontrolops.DirToFileURL(dir) + if err != nil { + return err + } + backupName := "backup_export" + + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + // Register as a backup remote (idempotent — remove first if exists). + _ = versioncontrolops.BackupRemove(ctx, db, backupName) + if err := versioncontrolops.BackupAdd(ctx, db, backupName, backupURL); err != nil { + // Another backup (e.g. "default" registered by `bd backup init`) may + // already point to this URL. In that case, sync using the existing + // remote name rather than failing. + if conflict := versioncontrolops.ExtractAddressConflictName(err); conflict != "" { + if syncErr := versioncontrolops.BackupSync(ctx, db, conflict); syncErr != nil { + return fmt.Errorf("sync to backup: %w", syncErr) + } + return nil + } + return fmt.Errorf("register backup remote: %w", err) + } + if err := versioncontrolops.BackupSync(ctx, db, backupName); err != nil { + return fmt.Errorf("sync to backup: %w", err) + } + return nil + }) +} + +// RestoreDatabase restores the database from a Dolt backup at dir. +// The dir must exist locally and contain a valid Dolt backup. +// When force is true, an existing database is overwritten. +func (s *DoltliteStore) RestoreDatabase(ctx context.Context, dir string, force bool) error { + info, err := os.Stat(dir) + if err != nil { + return fmt.Errorf("backup source does not exist: %w", err) + } + if !info.IsDir() { + return fmt.Errorf("backup source is not a directory: %s", dir) + } + + backupURL, err := versioncontrolops.DirToFileURL(dir) + if err != nil { + return err + } + + return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + return versioncontrolops.BackupRestore(ctx, db, backupURL, s.database, force) + }) +} diff --git a/internal/storage/schema/helpers.go b/internal/storage/schema/helpers.go index 8cddbe5bd..c98957099 100644 --- a/internal/storage/schema/helpers.go +++ b/internal/storage/schema/helpers.go @@ -3,6 +3,7 @@ package schema import ( "context" "fmt" + "strings" ) // EnsureIgnoredTables checks whether the dolt_ignore'd wisp tables exist in @@ -67,7 +68,18 @@ func TableExists(ctx context.Context, db DBConn, table string) (bool, error) { // #nosec G202 -- table names come from internal constants, not user input. rows, err := db.QueryContext(ctx, "SHOW TABLES LIKE '"+table+"'") //nolint:gosec // G202: table name is an internal constant if err != nil { - return false, fmt.Errorf("check table %s: %w", table, err) + if !strings.Contains(strings.ToLower(err.Error()), "syntax") { + return false, fmt.Errorf("check table %s: %w", table, err) + } + var name string + err = db.QueryRowContext(ctx, "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", table).Scan(&name) + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "no rows") { + return false, nil + } + return false, fmt.Errorf("check sqlite table %s: %w", table, err) + } + return true, nil } defer rows.Close() return rows.Next(), nil diff --git a/internal/storage/schema/sqlite.go b/internal/storage/schema/sqlite.go new file mode 100644 index 000000000..a071abb9a --- /dev/null +++ b/internal/storage/schema/sqlite.go @@ -0,0 +1,162 @@ +package schema + +import ( + "context" + "fmt" + "io/fs" + "regexp" + "sort" + "strings" +) + +var ( + createTableRe = regexp.MustCompile(`(?is)^\s*CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\s+` + "`?" + `([A-Za-z0-9_]+)` + "`?") + inlineIndexRe = regexp.MustCompile(`(?i)^\s*(?:UNIQUE\s+)?INDEX\s+` + "`?" + `([A-Za-z0-9_]+)` + "`?" + `\s*(\([^)]+\))\s*,?\s*$`) +) + +// MigrateUpSQLite applies the shared Beads migrations after translating the +// small MySQL/Dolt SQL subset that SQLite does not parse. +func MigrateUpSQLite(ctx context.Context, db DBConn) (int, error) { + if _, err := db.ExecContext(ctx, schemaMigrationsBootstrapSQL); err != nil { + return 0, fmt.Errorf("creating schema_migrations table: %w", err) + } + + var current int + if err := db.QueryRowContext(ctx, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations").Scan(¤t); err != nil { + return 0, fmt.Errorf("reading current migration version: %w", err) + } + if current >= LatestVersion() { + return 0, nil + } + + entries, err := fs.ReadDir(upMigrations, "migrations") + if err != nil { + return 0, fmt.Errorf("reading embedded migrations: %w", err) + } + + var pending []migrationFile + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".up.sql") { + continue + } + v, err := parseVersion(e.Name()) + if err != nil { + return 0, fmt.Errorf("parsing migration filename %q: %w", e.Name(), err) + } + if v > current { + pending = append(pending, migrationFile{version: v, name: e.Name()}) + } + } + sort.Slice(pending, func(i, j int) bool { return pending[i].version < pending[j].version }) + + for _, mf := range pending { + data, err := upMigrations.ReadFile("migrations/" + mf.name) + if err != nil { + return 0, fmt.Errorf("reading migration %s: %w", mf.name, err) + } + for _, stmt := range translateSQLiteStatements(splitStatements(string(data))) { + if strings.TrimSpace(stmt) == "" { + continue + } + if _, err := db.ExecContext(ctx, stmt); err != nil { + if !isConcurrentInitError(err) { + return 0, fmt.Errorf("migration %s: statement failed: %w\nSQL: %s", mf.name, err, stmt) + } + } + } + if _, err := db.ExecContext(ctx, "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)", mf.version); err != nil { + if !isConcurrentInitError(err) { + return 0, fmt.Errorf("recording migration %s: %w", mf.name, err) + } + } + } + return len(pending), nil +} + +// CreateIgnoredTablesSQLite recreates dolt-ignored tables using SQLite syntax. +func CreateIgnoredTablesSQLite(ctx context.Context, db DBConn) error { + if _, err := db.ExecContext(ctx, "CREATE TABLE IF NOT EXISTS dolt_ignore (pattern TEXT NOT NULL PRIMARY KEY, ignored BOOLEAN NOT NULL)"); err != nil { + return fmt.Errorf("create dolt_ignore: %w", err) + } + for _, stmt := range translateSQLiteStatements(IgnoredTableDDL()) { + if strings.TrimSpace(stmt) == "" { + continue + } + if _, err := db.ExecContext(ctx, stmt); err != nil { + if !isConcurrentInitError(err) { + return fmt.Errorf("create ignored table: %w\nSQL: %s", err, stmt) + } + } + } + return nil +} + +func translateSQLiteStatements(stmts []string) []string { + var out []string + for _, stmt := range stmts { + stmt = translateSQLiteBasics(stmt) + if strings.HasPrefix(strings.ToUpper(strings.TrimSpace(stmt)), "CREATE TABLE") { + tableStmt, indexes := splitInlineIndexes(stmt) + out = append(out, tableStmt) + out = append(out, indexes...) + continue + } + out = append(out, stmt) + } + return out +} + +func translateSQLiteBasics(stmt string) string { + repls := []struct{ old, new string }{ + {"INSERT IGNORE INTO", "INSERT OR IGNORE INTO"}, + {"ON UPDATE CURRENT_TIMESTAMP", ""}, + {"JSON DEFAULT (JSON_OBJECT())", "TEXT DEFAULT '{}'"}, + {"JSON DEFAULT (json_object())", "TEXT DEFAULT '{}'"}, + {" JSON ", " TEXT "}, + {"NOW()", "CURRENT_TIMESTAMP"}, + {"now()", "CURRENT_TIMESTAMP"}, + {"CREATE OR REPLACE VIEW", "CREATE VIEW IF NOT EXISTS"}, + {"create or replace view", "CREATE VIEW IF NOT EXISTS"}, + {"ESCAPE '\\\\'", "ESCAPE '\\'"}, + } + for _, r := range repls { + stmt = strings.ReplaceAll(stmt, r.old, r.new) + } + if regexp.MustCompile(`(?i)^CREATE\s+INDEX\s+`).MatchString(stmt) && + !regexp.MustCompile(`(?i)^CREATE\s+INDEX\s+IF\s+NOT\s+EXISTS\s+`).MatchString(stmt) { + stmt = regexp.MustCompile(`(?i)^CREATE\s+INDEX\s+`).ReplaceAllString(stmt, "CREATE INDEX IF NOT EXISTS ") + } + if regexp.MustCompile(`(?i)^CALL\s+DOLT_(ADD|COMMIT)\(`).MatchString(stmt) { + return "" + } + return stmt +} + +func splitInlineIndexes(stmt string) (string, []string) { + m := createTableRe.FindStringSubmatch(stmt) + if len(m) != 2 { + return stmt, nil + } + table := m[1] + lines := strings.Split(stmt, "\n") + var kept []string + var indexes []string + for _, line := range lines { + trimmed := strings.TrimSpace(line) + idx := inlineIndexRe.FindStringSubmatch(trimmed) + if len(idx) == 3 { + indexes = append(indexes, fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s %s", idx[1], table, idx[2])) + continue + } + kept = append(kept, line) + } + for i := len(kept) - 1; i >= 0; i-- { + trimmed := strings.TrimSpace(kept[i]) + if trimmed == "" || trimmed == ")" || trimmed == ");" { + continue + } + kept[i] = strings.TrimRight(kept[i], " \t,") + break + } + return strings.Join(kept, "\n"), indexes +} From 8a23af329a41ad039f3d5968a473439c03acdaf5 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Wed, 29 Apr 2026 12:15:14 +1000 Subject: [PATCH 02/15] Make doltlite an explicit backend --- beads_cgo.go | 24 +++++++++- beads_nocgo.go | 3 ++ cmd/bd/init.go | 76 ++++++++++++++++++++----------- cmd/bd/main.go | 6 ++- cmd/bd/store_factory.go | 22 +++++++-- cmd/bd/store_factory_nocgo.go | 10 ++++ internal/configfile/configfile.go | 31 ++++++++++--- 7 files changed, 133 insertions(+), 39 deletions(-) diff --git a/beads_cgo.go b/beads_cgo.go index e9b3c7683..415893619 100644 --- a/beads_cgo.go +++ b/beads_cgo.go @@ -4,6 +4,7 @@ package beads import ( "context" + "path/filepath" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage/dolt" @@ -19,6 +20,8 @@ import ( // exclusive flock to prevent corruption from concurrent access. This matches // the behavior of the bd CLI. // - Server mode: Connects to an external dolt sql-server via OpenFromConfig. +// - Doltlite mode: Opens a local doltlite database when metadata.json has +// backend="doltlite". // // The returned Storage must be closed when no longer needed. In embedded mode // the caller must also defer the Unlocker returned by the flock; pass nil-safe @@ -40,13 +43,30 @@ func OpenBestAvailable(ctx context.Context, beadsDir string) (Storage, embeddedd return store, embeddeddolt.NoopLock{}, nil } + if err == nil && cfg != nil && cfg.IsDoltliteBackend() { + database := cfg.GetDoltDatabase() + store, err := doltlite.New(ctx, beadsDir, database, "main") + if err != nil { + return nil, nil, err + } + return store, embeddeddolt.NoopLock{}, nil + } + + // Embedded mode: acquire exclusive flock first. + dataDir := filepath.Join(beadsDir, "embeddeddolt") + lock, err := embeddeddolt.TryLock(dataDir) + if err != nil { + return nil, nil, err + } + database := configfile.DefaultDoltDatabase if cfg != nil { database = cfg.GetDoltDatabase() } - store, err := doltlite.New(ctx, beadsDir, database, "main") + store, err := embeddeddolt.New(ctx, beadsDir, database, "main", embeddeddolt.WithLock(lock)) if err != nil { + lock.Unlock() return nil, nil, err } - return store, embeddeddolt.NoopLock{}, nil + return store, lock, nil } diff --git a/beads_nocgo.go b/beads_nocgo.go index 9269f0b9a..80b090cc2 100644 --- a/beads_nocgo.go +++ b/beads_nocgo.go @@ -18,6 +18,9 @@ import ( // beadsDir is the path to the .beads directory. func OpenBestAvailable(ctx context.Context, beadsDir string) (Storage, embeddeddolt.Unlocker, error) { cfg, err := configfile.Load(beadsDir) + if err == nil && cfg != nil && cfg.IsDoltliteBackend() { + return nil, nil, fmt.Errorf("doltlite requires a CGO build") + } if err == nil && cfg != nil && cfg.IsDoltServerMode() { store, err := dolt.NewFromConfig(ctx, beadsDir) if err != nil { diff --git a/cmd/bd/init.go b/cmd/bd/init.go index 269003c1d..5b0bb7a39 100644 --- a/cmd/bd/init.go +++ b/cmd/bd/init.go @@ -103,9 +103,10 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): sharedServer, _ := cmd.Flags().GetBool("shared-server") externalServer, _ := cmd.Flags().GetBool("external") - // Handle --backend flag: "dolt" is the only supported backend. - // "sqlite" is accepted for backward compatibility but prints a - // deprecation notice and exits with an error. + // Handle --backend flag. Dolt remains the default; doltlite is an + // explicit local backend for environments that do not want a Dolt + // sql-server lifecycle. + backend := configfile.BackendDolt if backendFlag == "sqlite" { fmt.Fprintf(os.Stderr, "%s The SQLite backend has been removed.\n\n", ui.RenderWarn("⚠ DEPRECATED:")) fmt.Fprintf(os.Stderr, "Dolt is now the default (and only) storage backend for beads.\n") @@ -115,8 +116,17 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): fmt.Fprintf(os.Stderr, " bd init --from-jsonl\n\n") fmt.Fprintf(os.Stderr, "See: https://github.com/steveyegge/beads/blob/main/docs/DOLT-BACKEND.md\n") os.Exit(1) - } else if backendFlag != "" && backendFlag != "dolt" { - FatalError("unknown backend %q: only \"dolt\" is supported", backendFlag) + } else if backendFlag == configfile.BackendDoltlite { + backend = configfile.BackendDoltlite + } else if backendFlag != "" && backendFlag != configfile.BackendDolt { + FatalError("unknown backend %q: supported backends are \"dolt\" and \"doltlite\"", backendFlag) + } + useDoltlite := backend == configfile.BackendDoltlite + if useDoltlite && (initServerMode || sharedServer || externalServer) { + FatalError("--backend=doltlite is local-only and cannot be combined with --server, --shared-server, or --external") + } + if useDoltlite && initRemoteChanged && initRemote != "" { + FatalError("--backend=doltlite does not support Dolt remote bootstrap") } // Validate --database format early, before any side effects. @@ -148,11 +158,8 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): FatalError("--team requires interactive prompts and cannot be used with --non-interactive") } - // Dolt is the only supported backend - backend := configfile.BackendDolt - // Also treat BEADS_DOLT_SERVER_MODE=1 env var as --server. - if os.Getenv("BEADS_DOLT_SERVER_MODE") == "1" { + if !useDoltlite && os.Getenv("BEADS_DOLT_SERVER_MODE") == "1" { initServerMode = true } @@ -160,7 +167,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): // the server-backed store path during init. Without this, init can // persist shared-server intent in YAML while still creating an embedded // store and recording dolt_mode=embedded in metadata.json (GH#2946). - if sharedServer || strings.EqualFold(os.Getenv("BEADS_DOLT_SHARED_SERVER"), "true") || os.Getenv("BEADS_DOLT_SHARED_SERVER") == "1" { + if !useDoltlite && (sharedServer || strings.EqualFold(os.Getenv("BEADS_DOLT_SHARED_SERVER"), "true") || os.Getenv("BEADS_DOLT_SHARED_SERVER") == "1") { initServerMode = true } @@ -182,7 +189,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): // Reject hyphens in --database for embedded mode. Must run AFTER // serverMode is set above — otherwise isEmbeddedMode() always returns // true and incorrectly rejects server-mode names (GH#3231). - if database != "" && strings.ContainsRune(database, '-') && isEmbeddedMode() { + if database != "" && strings.ContainsRune(database, '-') && (isEmbeddedMode() || useDoltlite) { FatalError("database name %q contains hyphens which are invalid in embedded mode; use underscores instead (e.g. %q)", database, sanitizeDBName(database)) } @@ -321,7 +328,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): earlyRemoteHasDoltData = gitOriginHasDoltDataRef() } } - if earlySyncURL != "" { + if !useDoltlite && earlySyncURL != "" { earlyDecision := CheckRemoteSafety(RemoteSafetyInput{ Force: force, ReinitLocal: reinitLocal, @@ -575,7 +582,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): syncFromRemote := false remoteHasDoltData := false - if syncURL != "" { + if !useDoltlite && syncURL != "" { // sync.remote was explicitly configured. Treat as bootstrap- // from-remote intent; CheckRemoteSafety still enforces that // --force/--reinit-local can't silently fight that intent. @@ -583,7 +590,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): // http:// to git+http:// and break Dolt remotesapi endpoints // configured explicitly by the user (GH#3339). syncFromRemote = true - } else if syncRemoteSource == initSyncRemoteNone && isGitRepo() && !isBareGitRepo() { + } else if !useDoltlite && syncRemoteSource == initSyncRemoteNone && isGitRepo() && !isBareGitRepo() { if originURL, err := gitOriginGetURL(); err == nil && originURL != "" { syncURL = normalizeRemoteURL(originURL) remoteHasDoltData = gitOriginHasDoltDataRef() @@ -635,7 +642,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): } } } - if syncFromRemote { + if !useDoltlite && syncFromRemote { var err error cloneCfg := initTimeCloneConfig(initServerMode, serverHost, serverPort, serverSocket, serverUser, dbName) err = cloneFromRemoteWithMode(ctx, beadsDir, syncURL, dbName, cloneCfg, initRemoteCloneMode(initServerMode, externalServer)) @@ -700,10 +707,13 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): doltCfg.ServerUser = serverUser } - initLock, err := acquireEmbeddedLock(beadsDir, initServerMode) - if err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) + var initLock embeddeddolt.Unlocker = embeddeddolt.NoopLock{} + if !useDoltlite { + initLock, err = acquireEmbeddedLock(beadsDir, initServerMode) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } } defer initLock.Unlock() @@ -755,16 +765,21 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): } } - store, err := newDoltStore(ctx, doltCfg, embeddeddolt.WithLock(initLock)) + var store storage.DoltStorage + if useDoltlite { + store, err = newDoltliteStore(ctx, beadsDir, dbName) + } else { + store, err = newDoltStore(ctx, doltCfg, embeddeddolt.WithLock(initLock)) + } if err != nil { - fmt.Fprintf(os.Stderr, "Error: failed to open Dolt store: %v\n", err) + fmt.Fprintf(os.Stderr, "Error: failed to open %s store: %v\n", backend, err) os.Exit(1) } // Initialize global database schema and config in shared-server mode. // Opens a separate store connection to beads_global with CreateIfMissing // to trigger schema migration, then seeds the issue prefix and project ID. - if sharedServer || doltserver.IsSharedServerMode() { + if !useDoltlite && (sharedServer || doltserver.IsSharedServerMode()) { initGlobalDatabaseConfig(ctx, doltCfg, quiet) } @@ -775,7 +790,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): // git origin URLs for plain source repos must NOT be registered — // they cause every Dolt fetch to fail and leak tmp_pack_* files // that can consume 100+ GB of disk space (GH#3354, GH#3356). - if shouldWireInitRemote(syncURL, syncFromRemote, syncURLFromConfig) { + if !useDoltlite && shouldWireInitRemote(syncURL, syncFromRemote, syncURLFromConfig) { hasRemote, _ := store.HasRemote(ctx, "origin") if !hasRemote { if err := store.AddRemote(ctx, "origin", syncURL); err != nil { @@ -795,7 +810,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): // Set the issue prefix in config (only if not already configured — // avoid clobbering when multiple rigs share the same Dolt database) existing, _ := store.GetConfig(ctx, "issue_prefix") - if existing == "" { + if existing == "" || useDoltlite { // Sanitize dots to underscores so issue IDs (e.g. "GPUPolynomials_jl-1") // remain valid identifiers. Must match DoltDatabase sanitization above. issuePrefix := strings.ReplaceAll(prefix, ".", "_") @@ -884,6 +899,15 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): // Always store backend explicitly in metadata.json cfg.Backend = backend + if backend == configfile.BackendDoltlite { + cfg.Database = "doltlite" + if database != "" { + cfg.DoltDatabase = database + } else if cfg.DoltDatabase == "" && prefix != "" { + cfg.DoltDatabase = strings.ReplaceAll(prefix, "-", "_") + } + cfg.DoltMode = configfile.DoltModeEmbedded + } // Metadata.json.database should point to the Dolt directory (not beads.db). // Backward-compat: older dolt setups left this as "beads.db", which is misleading. if backend == configfile.BackendDolt { @@ -1424,8 +1448,8 @@ func init() { initCmd.Flags().Bool("non-interactive", false, "Skip all interactive prompts (auto-detected in CI or non-TTY environments)") initCmd.Flags().String("role", "", "Set beads role without prompting: \"maintainer\" or \"contributor\"") - // Backend selection (dolt is the only supported backend; sqlite accepted for deprecation notice) - initCmd.Flags().String("backend", "", "Storage backend (default: dolt). --backend=sqlite prints deprecation notice.") + // Backend selection (dolt is the default; sqlite accepted for deprecation notice) + initCmd.Flags().String("backend", "", "Storage backend (dolt|doltlite; default: dolt). --backend=sqlite prints deprecation notice.") // Dolt server connection flags initCmd.Flags().Bool("server", false, "Use external dolt sql-server instead of embedded engine") diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 15ed3bacf..9fb759d06 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -984,7 +984,11 @@ var rootCmd = &cobra.Command{ // Removing them WILL cause unrecoverable data corruption and data loss. // Dolt manages these files itself; external interference is never safe. - store, err = newDoltStore(rootCtx, doltCfg) + if cfg != nil && cfg.IsDoltliteBackend() { + store, err = newDoltliteStore(rootCtx, beadsDir, doltCfg.Database) + } else { + store, err = newDoltStore(rootCtx, doltCfg) + } // Track final read-only state for staleness checks (GH#1089) storeIsReadOnly = doltCfg.ReadOnly diff --git a/cmd/bd/store_factory.go b/cmd/bd/store_factory.go index 91a668437..a73b89649 100644 --- a/cmd/bd/store_factory.go +++ b/cmd/bd/store_factory.go @@ -45,7 +45,14 @@ func newDoltStore(ctx context.Context, cfg *dolt.Config, opts ...embeddeddolt.Op if cfg.ServerMode { return dolt.New(ctx, cfg) } - return doltlite.New(ctx, cfg.BeadsDir, cfg.Database, "main") + return embeddeddolt.New(ctx, cfg.BeadsDir, cfg.Database, "main", opts...) +} + +func newDoltliteStore(ctx context.Context, beadsDir, database string) (storage.DoltStorage, error) { + if database == "" { + database = configfile.DefaultDoltDatabase + } + return doltlite.New(ctx, beadsDir, database, "main") } // acquireEmbeddedLock acquires an exclusive flock on the embeddeddolt data @@ -56,7 +63,8 @@ func acquireEmbeddedLock(beadsDir string, serverMode bool) (embeddeddolt.Unlocke if serverMode { return embeddeddolt.NoopLock{}, nil } - return embeddeddolt.NoopLock{}, nil + dataDir := filepath.Join(beadsDir, "embeddeddolt") + return embeddeddolt.TryLock(dataDir) } // newDoltStoreFromConfig creates a storage backend from the beads directory's @@ -67,6 +75,9 @@ func acquireEmbeddedLock(beadsDir string, serverMode bool) (embeddeddolt.Unlocke // auto-sanitized to underscores and the fix is persisted to metadata.json. func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { cfg, err := configfile.Load(beadsDir) + if err == nil && cfg != nil && cfg.IsDoltliteBackend() { + return newDoltliteStore(ctx, beadsDir, cfg.GetDoltDatabase()) + } if err == nil && cfg != nil && cfg.IsDoltServerMode() { return dolt.NewFromConfig(ctx, beadsDir) } @@ -80,7 +91,7 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltS } database = sanitized } - return doltlite.New(ctx, beadsDir, database, "main") + return embeddeddolt.New(ctx, beadsDir, database, "main") } // migrateHyphenatedDB renames a legacy hyphenated database directory and @@ -131,6 +142,9 @@ func migrateHyphenatedDB(beadsDir string, cfg *configfile.Config, oldName, newNa // hydration from mutating foreign projects (GH#3231). func newReadOnlyStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { cfg, err := configfile.Load(beadsDir) + if err == nil && cfg != nil && cfg.IsDoltliteBackend() { + return newDoltliteStore(ctx, beadsDir, cfg.GetDoltDatabase()) + } if err == nil && cfg != nil && cfg.IsDoltServerMode() { return dolt.NewFromConfigWithOptions(ctx, beadsDir, &dolt.Config{ReadOnly: true}) } @@ -141,5 +155,5 @@ func newReadOnlyStoreFromConfig(ctx context.Context, beadsDir string) (storage.D if sanitized := sanitizeDBName(database); sanitized != database { database = sanitized } - return doltlite.New(ctx, beadsDir, database, "main") + return embeddeddolt.New(ctx, beadsDir, database, "main") } diff --git a/cmd/bd/store_factory_nocgo.go b/cmd/bd/store_factory_nocgo.go index ec1b9b57e..0c5291d49 100644 --- a/cmd/bd/store_factory_nocgo.go +++ b/cmd/bd/store_factory_nocgo.go @@ -27,6 +27,10 @@ func newDoltStore(ctx context.Context, cfg *dolt.Config, _ ...embeddeddolt.Optio return dolt.New(ctx, cfg) } +func newDoltliteStore(_ context.Context, _, _ string) (storage.DoltStorage, error) { + return nil, fmt.Errorf("doltlite requires a CGO build") +} + // acquireEmbeddedLock returns a no-op lock in non-CGO builds. func acquireEmbeddedLock(_ string, _ bool) (embeddeddolt.Unlocker, error) { return embeddeddolt.NoopLock{}, nil @@ -35,6 +39,9 @@ func acquireEmbeddedLock(_ string, _ bool) (embeddeddolt.Unlocker, error) { // newDoltStoreFromConfig creates a server-mode storage backend from config. func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { cfg, err := configfile.Load(beadsDir) + if err == nil && cfg != nil && cfg.IsDoltliteBackend() { + return nil, fmt.Errorf("doltlite requires a CGO build") + } if err == nil && cfg != nil && cfg.IsDoltServerMode() { return dolt.NewFromConfig(ctx, beadsDir) } @@ -44,6 +51,9 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltS // newReadOnlyStoreFromConfig creates a read-only server-mode storage backend. func newReadOnlyStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { cfg, err := configfile.Load(beadsDir) + if err == nil && cfg != nil && cfg.IsDoltliteBackend() { + return nil, fmt.Errorf("doltlite requires a CGO build") + } if err == nil && cfg != nil && cfg.IsDoltServerMode() { return dolt.NewFromConfigWithOptions(ctx, beadsDir, &dolt.Config{ReadOnly: true}) } diff --git a/internal/configfile/configfile.go b/internal/configfile/configfile.go index ef83f8259..0c8d1de29 100644 --- a/internal/configfile/configfile.go +++ b/internal/configfile/configfile.go @@ -175,7 +175,8 @@ func (c *Config) GetStaleClosedIssuesDays() int { // Backend constants const ( - BackendDolt = "dolt" + BackendDolt = "dolt" + BackendDoltlite = "doltlite" ) // BackendCapabilities describes behavioral constraints for a storage backend. @@ -193,10 +194,15 @@ type BackendCapabilities struct { } // CapabilitiesForBackend returns capabilities for a backend string. -// Dolt is the only supported backend. Returns SingleProcessOnly=true by default; -// use Config.GetCapabilities() to properly handle server mode. -func CapabilitiesForBackend(_ string) BackendCapabilities { - return BackendCapabilities{SingleProcessOnly: true} +// Embedded Dolt and doltlite are single-process-only; Dolt server mode is +// handled by Config.GetCapabilities(). +func CapabilitiesForBackend(backend string) BackendCapabilities { + switch strings.ToLower(strings.TrimSpace(backend)) { + case BackendDolt, BackendDoltlite, "": + return BackendCapabilities{SingleProcessOnly: true} + default: + return BackendCapabilities{SingleProcessOnly: true} + } } // GetCapabilities returns the backend capabilities for this config. @@ -211,11 +217,24 @@ func (c *Config) GetCapabilities() BackendCapabilities { return CapabilitiesForBackend(backend) } -// GetBackend returns the backend type. Always returns "dolt". +// GetBackend returns the backend type. Missing/legacy values default to "dolt". func (c *Config) GetBackend() string { + if c != nil { + switch strings.ToLower(strings.TrimSpace(c.Backend)) { + case BackendDoltlite: + return BackendDoltlite + case BackendDolt, "": + return BackendDolt + } + } return BackendDolt } +// IsDoltliteBackend returns true when metadata explicitly selects doltlite. +func (c *Config) IsDoltliteBackend() bool { + return c.GetBackend() == BackendDoltlite +} + // Dolt mode constants const ( DoltModeEmbedded = "embedded" From 1e918b9ee110df2d4d3ae102cd4241acd9a9d5d5 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Thu, 30 Apr 2026 04:42:59 +1000 Subject: [PATCH 03/15] feat(storage): add doltlite backend support --- cmd/bd/main.go | 11 +- internal/storage/doltlite/create_issue.go | 17 +-- internal/storage/doltlite/issues.go | 4 +- internal/storage/doltlite/labels.go | 2 +- internal/storage/doltlite/list_queries.go | 4 +- .../storage/doltlite/multiprocess_test.go | 75 ++++++++++++ internal/storage/doltlite/open.go | 7 +- internal/storage/doltlite/queries.go | 2 +- internal/storage/doltlite/smoke_test.go | 44 +++++++ internal/storage/doltlite/store.go | 112 ++++++++++++------ internal/storage/doltlite/transaction.go | 8 +- internal/storage/doltlite/version_control.go | 20 ++-- internal/storage/issueops/close.go | 6 +- internal/storage/issueops/create.go | 14 ++- internal/storage/issueops/dialect.go | 45 +++++++ internal/storage/issueops/filters.go | 8 +- internal/storage/issueops/filters_test.go | 22 ++++ internal/storage/issueops/helpers.go | 16 +++ internal/storage/issueops/labels.go | 15 ++- internal/storage/issueops/ready_work.go | 38 ++++-- internal/storage/issueops/search.go | 14 ++- internal/storage/issueops/update.go | 22 +++- internal/utils/id_parser.go | 21 ++-- 23 files changed, 407 insertions(+), 120 deletions(-) create mode 100644 internal/storage/doltlite/multiprocess_test.go create mode 100644 internal/storage/issueops/dialect.go diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 9fb759d06..88df956be 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1101,11 +1101,12 @@ var rootCmd = &cobra.Command{ } } - // Auto-backup: export JSONL to .beads/backup/ if enabled and due - maybeAutoBackup(rootCtx) - - // Auto-export: write git-tracked JSONL for portability if enabled and due - maybeAutoExport(rootCtx) + // Auto-backup/export write JSONL and may run git operations. Keep read-only + // commands like list/show/status as pure reads. + if !isReadOnlyCommand(cmd.Name()) { + maybeAutoBackup(rootCtx) + maybeAutoExport(rootCtx) + } // Auto-push: push to Dolt remote if enabled and due. // Skip for read-only commands to avoid unnecessary network operations diff --git a/internal/storage/doltlite/create_issue.go b/internal/storage/doltlite/create_issue.go index 56ab0a83e..5a3656738 100644 --- a/internal/storage/doltlite/create_issue.go +++ b/internal/storage/doltlite/create_issue.go @@ -7,7 +7,6 @@ import ( "database/sql" "fmt" - "github.com/google/uuid" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" @@ -96,11 +95,11 @@ func createIssueSQLite(ctx context.Context, tx *sql.Tx, bc *issueops.BatchContex return err } if existingCount == 0 { - if err := recordEventSQLite(ctx, tx, eventTable, issue.ID, types.EventCreated, actor, ""); err != nil { + if err := issueops.RecordEventInTableWithDialect(ctx, tx, eventTable, issue.ID, types.EventCreated, actor, "", issueops.SQLDialectSQLite); err != nil { return fmt.Errorf("failed to record event for %s: %w", issue.ID, err) } } - if err := issueops.PersistLabels(ctx, tx, issue); err != nil { + if err := issueops.PersistLabelsWithDialect(ctx, tx, issue, issueops.SQLDialectSQLite); err != nil { return err } return issueops.PersistComments(ctx, tx, issue) @@ -145,15 +144,3 @@ func insertIssueSQLite(ctx context.Context, tx *sql.Tx, table string, issue *typ } return nil } - -func recordEventSQLite(ctx context.Context, tx *sql.Tx, table, issueID string, eventType types.EventType, actor, newValue string) error { - id := uuid.Must(uuid.NewV7()).String() - _, err := tx.ExecContext(ctx, fmt.Sprintf(` - INSERT INTO %s (id, issue_id, event_type, actor, old_value, new_value) - VALUES (?, ?, ?, ?, ?, ?) - `, table), id, issueID, eventType, actor, "", newValue) - if err != nil { - return fmt.Errorf("record event in %s: %w", table, err) - } - return nil -} diff --git a/internal/storage/doltlite/issues.go b/internal/storage/doltlite/issues.go index a36ab8a1d..9fcc50b45 100644 --- a/internal/storage/doltlite/issues.go +++ b/internal/storage/doltlite/issues.go @@ -37,7 +37,7 @@ func (s *DoltliteStore) UpdateIssue(ctx context.Context, id string, updates map[ } return s.withConn(ctx, true, func(tx *sql.Tx) error { - _, err := issueops.UpdateIssueInTx(ctx, tx, id, updates, actor) + _, err := issueops.UpdateIssueInTxWithDialect(ctx, tx, id, updates, actor, issueops.SQLDialectSQLite) return err }) } @@ -71,7 +71,7 @@ func (s *DoltliteStore) UpdateIssueType(ctx context.Context, id string, issueTyp // Delegates SQL work to issueops; EmbeddedDolt auto-commits the transaction. func (s *DoltliteStore) CloseIssue(ctx context.Context, id string, reason string, actor string, session string) error { return s.withConn(ctx, true, func(tx *sql.Tx) error { - _, err := issueops.CloseIssueInTx(ctx, tx, id, reason, actor, session) + _, err := issueops.CloseIssueInTxWithDialect(ctx, tx, id, reason, actor, session, issueops.SQLDialectSQLite) return err }) } diff --git a/internal/storage/doltlite/labels.go b/internal/storage/doltlite/labels.go index c9987d5a2..1fa157933 100644 --- a/internal/storage/doltlite/labels.go +++ b/internal/storage/doltlite/labels.go @@ -21,7 +21,7 @@ func (s *DoltliteStore) GetLabels(ctx context.Context, issueID string) ([]string func (s *DoltliteStore) AddLabel(ctx context.Context, issueID, label, actor string) error { return s.withConn(ctx, true, func(tx *sql.Tx) error { - return issueops.AddLabelInTx(ctx, tx, "", "", issueID, label, actor) + return issueops.AddLabelInTxWithDialect(ctx, tx, "", "", issueID, label, actor, issueops.SQLDialectSQLite) }) } diff --git a/internal/storage/doltlite/list_queries.go b/internal/storage/doltlite/list_queries.go index e5c3209eb..7e5022abf 100644 --- a/internal/storage/doltlite/list_queries.go +++ b/internal/storage/doltlite/list_queries.go @@ -14,7 +14,7 @@ func (s *DoltliteStore) SearchIssues(ctx context.Context, query string, filter t var result []*types.Issue err := s.withConn(ctx, false, func(tx *sql.Tx) error { var err error - result, err = issueops.SearchIssuesInTx(ctx, tx, query, filter) + result, err = issueops.SearchIssuesInTxWithDialect(ctx, tx, query, filter, issueops.SQLDialectSQLite) return err }) return result, err @@ -25,7 +25,7 @@ func (s *DoltliteStore) ListWisps(ctx context.Context, filter types.WispFilter) var result []*types.Issue err := s.withConn(ctx, false, func(tx *sql.Tx) error { var err error - result, err = issueops.SearchIssuesInTx(ctx, tx, "", issueFilter) + result, err = issueops.SearchIssuesInTxWithDialect(ctx, tx, "", issueFilter, issueops.SQLDialectSQLite) return err }) return result, err diff --git a/internal/storage/doltlite/multiprocess_test.go b/internal/storage/doltlite/multiprocess_test.go new file mode 100644 index 000000000..38ac736c8 --- /dev/null +++ b/internal/storage/doltlite/multiprocess_test.go @@ -0,0 +1,75 @@ +//go:build cgo + +package doltlite_test + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/doltlite" +) + +func TestConcurrentOpenWhilePeerStoreAlive(t *testing.T) { + beadsDir := filepath.Join(t.TempDir(), ".beads") + readyPath := filepath.Join(t.TempDir(), "ready") + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=TestConcurrentOpenHelper") + cmd.Env = append(os.Environ(), + "BEADS_DOLTLITE_OPEN_HELPER=1", + "BEADS_DOLTLITE_TEST_DIR="+beadsDir, + "BEADS_DOLTLITE_READY="+readyPath, + ) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatalf("start helper: %v", err) + } + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(readyPath); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("helper did not open store") + } + time.Sleep(25 * time.Millisecond) + } + + openCtx, openCancel := context.WithTimeout(t.Context(), time.Second) + defer openCancel() + store, err := doltlite.New(openCtx, beadsDir, "beads", "main") + if err != nil { + t.Fatalf("second open while peer store alive: %v", err) + } + if err := store.Close(); err != nil { + t.Fatalf("close second store: %v", err) + } +} + +func TestConcurrentOpenHelper(t *testing.T) { + if os.Getenv("BEADS_DOLTLITE_OPEN_HELPER") != "1" { + t.Skip("helper only") + } + beadsDir := os.Getenv("BEADS_DOLTLITE_TEST_DIR") + readyPath := os.Getenv("BEADS_DOLTLITE_READY") + store, err := doltlite.New(t.Context(), beadsDir, "beads", "main") + if err != nil { + t.Fatalf("helper open: %v", err) + } + defer func() { _ = store.Close() }() + if err := os.WriteFile(readyPath, []byte("ready\n"), 0o600); err != nil { + t.Fatalf("write ready: %v", err) + } + time.Sleep(2 * time.Second) +} diff --git a/internal/storage/doltlite/open.go b/internal/storage/doltlite/open.go index 25f7d5ccd..a0a24142c 100644 --- a/internal/storage/doltlite/open.go +++ b/internal/storage/doltlite/open.go @@ -20,8 +20,9 @@ import ( var validIdentifier = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) const ( - commitName = "beads" - commitEmail = "beads@local" + commitName = "beads" + commitEmail = "beads@local" + defaultBusyTimeout = 10000 ) // OpenSQL opens an doltlite database at dir. The returned cleanup @@ -67,7 +68,7 @@ func buildDSN(dir, database string) (string, error) { if os.PathSeparator == '\\' { path = strings.ReplaceAll(path, `\`, `/`) } - return path, nil + return fmt.Sprintf("%s?_busy_timeout=%d", path, defaultBusyTimeout), nil } func sqlStringLiteral(s string) string { diff --git a/internal/storage/doltlite/queries.go b/internal/storage/doltlite/queries.go index f74054052..61f1a278c 100644 --- a/internal/storage/doltlite/queries.go +++ b/internal/storage/doltlite/queries.go @@ -16,7 +16,7 @@ func (s *DoltliteStore) GetReadyWork(ctx context.Context, filter types.WorkFilte var result []*types.Issue err := s.withConn(ctx, false, func(tx *sql.Tx) error { var err error - result, err = issueops.GetReadyWorkInTx(ctx, tx, filter, computeBlockedIDsWrapper) + result, err = issueops.GetReadyWorkInTxWithDialect(ctx, tx, filter, computeBlockedIDsWrapper, issueops.SQLDialectSQLite) return err }) return result, err diff --git a/internal/storage/doltlite/smoke_test.go b/internal/storage/doltlite/smoke_test.go index 459684c75..f75b32170 100644 --- a/internal/storage/doltlite/smoke_test.go +++ b/internal/storage/doltlite/smoke_test.go @@ -51,6 +51,50 @@ func TestSmokeCreateGetCommit(t *testing.T) { } } +func TestSmokeLabels(t *testing.T) { + ctx := t.Context() + store, err := doltlite.New(ctx, filepath.Join(t.TempDir(), ".beads"), "beads", "main") + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + + now := time.Now().UTC() + issue := &types.Issue{ + ID: "bd-label", + Title: "doltlite labels", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now, + UpdatedAt: now, + Labels: []string{"gc:session"}, + } + if err := store.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("CreateIssue: %v", err) + } + if err := store.AddLabel(ctx, issue.ID, "agent:worker", "test"); err != nil { + t.Fatalf("AddLabel: %v", err) + } + labels, err := store.GetLabels(ctx, issue.ID) + if err != nil { + t.Fatalf("GetLabels: %v", err) + } + got := map[string]bool{} + for _, label := range labels { + got[label] = true + } + for _, want := range []string{"gc:session", "agent:worker"} { + if !got[want] { + t.Fatalf("labels = %v, missing %q", labels, want) + } + } +} + func TestSmokeVersionControl(t *testing.T) { ctx := t.Context() store, err := doltlite.New(ctx, filepath.Join(t.TempDir(), ".beads"), "beads", "main") diff --git a/internal/storage/doltlite/store.go b/internal/storage/doltlite/store.go index 935f92da1..e87d26a19 100644 --- a/internal/storage/doltlite/store.go +++ b/internal/storage/doltlite/store.go @@ -34,10 +34,9 @@ var _ storage.Compactor = (*DoltliteStore)(nil) // time the embedded engine's write lock is held, reducing contention when // multiple processes access the same database concurrently. // -// The store holds an exclusive flock on the data directory for its entire -// lifetime. This prevents concurrent processes from initializing the embedded -// Dolt engine on the same directory, which causes a nil-pointer panic in -// DoltDB.SetCrashOnFatalError (GH#2571). +// Schema bootstrap is protected by a short exclusive flock. Normal operations +// rely on doltlite's file-level locking and conflict detection so multiple bd +// processes can read concurrently and serialize writes. type DoltliteStore struct { dataDir string beadsDir string @@ -45,8 +44,6 @@ type DoltliteStore struct { branch string credentialKey []byte closed atomic.Bool - lock Unlocker // exclusive flock held for the store's lifetime - ownsLock bool // true when New acquired the lock (false when caller supplied it via WithLock) } // errClosed is returned when a method is called after Close. @@ -59,10 +56,8 @@ type options struct { lock Unlocker // pre-acquired lock; nil means New acquires its own } -// WithLock passes a pre-acquired exclusive lock to New so it does not attempt -// to acquire a second one. The caller retains ownership — Close will NOT -// release a caller-supplied lock. This is used by bd init, which acquires the -// lock earlier to protect pre-initialization steps. +// WithLock passes a pre-acquired exclusive lock to New for schema bootstrap. +// The caller retains ownership. Normal store operations do not hold this lock. func WithLock(lock Unlocker) Option { return func(o *options) { o.lock = lock } } @@ -71,12 +66,8 @@ func WithLock(lock Unlocker) Option { // beadsDir is the .beads/ root; the data directory is derived as /doltlite/. // The database is created automatically if it doesn't exist (initSchema handles this). // -// An exclusive flock is held on the data directory for the store's entire -// lifetime. If another process already holds the lock, New queues with -// exponential backoff until the lock becomes available or the context is -// canceled, instead of panicking during concurrent engine initialization -// (GH#2571). The lock is released when Close is called, unless a pre-acquired -// lock was supplied via WithLock (in which case the caller is responsible for it). +// Schema bootstrap is guarded by a short exclusive flock. After bootstrap, the +// lock is released and normal operations use doltlite's own file-level locks. func New(ctx context.Context, beadsDir, database, branch string, opts ...Option) (*DoltliteStore, error) { if database == "" { return nil, fmt.Errorf("doltlite: database name must not be empty (caller should default to %q)", "beads") @@ -98,32 +89,27 @@ func New(ctx context.Context, beadsDir, database, branch string, opts ...Option) return nil, fmt.Errorf("doltlite: creating data directory: %w", err) } - // Acquire an exclusive flock before initializing the embedded engine. - // Without this, concurrent processes race through NewConnector → - // DoltDB.SetCrashOnFatalError → newDatabase → CollectDBs and one of - // them panics with a nil-pointer dereference (GH#2571). lock := o.lock ownsLock := lock == nil if ownsLock { + var err error lock, err = WaitLock(ctx, dataDir) if err != nil { return nil, err } } + if lock != nil && ownsLock { + defer lock.Unlock() + } s := &DoltliteStore{ dataDir: dataDir, beadsDir: absBeadsDir, database: database, branch: branch, - lock: lock, - ownsLock: ownsLock, } if err := s.initSchema(ctx); err != nil { - if ownsLock { - lock.Unlock() - } return nil, fmt.Errorf("doltlite: init schema: %w", err) } @@ -132,9 +118,6 @@ func New(ctx context.Context, beadsDir, database, branch string, opts ...Option) // dolt_ignore prevents them from being committed. Server mode handles // this in newServerMode(); embedded mode must do it here. (GH#3270) if err := s.ensureIgnoredTables(ctx); err != nil { - if ownsLock { - lock.Unlock() - } return nil, fmt.Errorf("doltlite: ensure ignored tables: %w", err) } @@ -145,6 +128,17 @@ func New(ctx context.Context, beadsDir, database, branch string, opts ...Option) // database or branch, begins an explicit SQL transaction, and passes it to fn. // This is used during initialization when the database may not yet exist. func (s *DoltliteStore) withRootConn(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (err error) { + if commit { + return s.withExclusiveLock(ctx, func() error { + return s.withRetry(ctx, func() error { + return s.withRootConnOnce(ctx, commit, fn) + }) + }) + } + return s.withRootConnOnce(ctx, commit, fn) +} + +func (s *DoltliteStore) withRootConnOnce(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (err error) { if s.closed.Load() { err = errClosed return @@ -190,6 +184,17 @@ func (s *DoltliteStore) withRootConn(ctx context.Context, commit bool, fn func(t // // The database must already exist (created during initSchema). func (s *DoltliteStore) withConn(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (err error) { + if commit { + return s.withExclusiveLock(ctx, func() error { + return s.withRetry(ctx, func() error { + return s.withConnOnce(ctx, commit, fn) + }) + }) + } + return s.withConnOnce(ctx, commit, fn) +} + +func (s *DoltliteStore) withConnOnce(ctx context.Context, commit bool, fn func(tx *sql.Tx) error) (err error) { if s.closed.Load() { err = errClosed return @@ -227,6 +232,45 @@ func (s *DoltliteStore) withConn(ctx context.Context, commit bool, fn func(tx *s return } +func (s *DoltliteStore) withRetry(ctx context.Context, fn func() error) error { + const maxAttempts = 5 + var err error + for attempt := 0; attempt < maxAttempts; attempt++ { + if err = fn(); err == nil { + return nil + } + if !isRetryableConcurrencyError(err) { + return err + } + select { + case <-ctx.Done(): + return errors.Join(err, ctx.Err()) + case <-time.After(time.Duration(50*(1< 0 { @@ -283,7 +287,7 @@ func BuildIssueFilterClauses(query string, filter types.IssueFilter, tables Filt if err := storage.ValidateMetadataKey(k); err != nil { return nil, nil, err } - whereClauses = append(whereClauses, "JSON_UNQUOTE(JSON_EXTRACT(metadata, ?)) = ?") + whereClauses = append(whereClauses, dialect.MetadataEqualsExpr()) args = append(args, storage.JSONMetadataPath(k), filter.MetadataFields[k]) } } diff --git a/internal/storage/issueops/filters_test.go b/internal/storage/issueops/filters_test.go index 1fba866c5..2e4f35ff3 100644 --- a/internal/storage/issueops/filters_test.go +++ b/internal/storage/issueops/filters_test.go @@ -341,6 +341,28 @@ func TestBuildIssueFilterClauses_WispsTables(t *testing.T) { } } +func TestBuildIssueFilterClauses_MetadataDialect(t *testing.T) { + t.Parallel() + + filter := types.IssueFilter{ + MetadataFields: map[string]string{"gc.routed_to": "gastown.boot"}, + } + doltClauses, _, err := BuildIssueFilterClausesWithDialect("", filter, IssuesFilterTables, SQLDialectDolt) + if err != nil { + t.Fatalf("dolt clauses: %v", err) + } + sqliteClauses, _, err := BuildIssueFilterClausesWithDialect("", filter, IssuesFilterTables, SQLDialectSQLite) + if err != nil { + t.Fatalf("sqlite clauses: %v", err) + } + if !strings.Contains(strings.Join(doltClauses, " "), "JSON_UNQUOTE(JSON_EXTRACT") { + t.Fatalf("dolt metadata clauses = %v", doltClauses) + } + if !strings.Contains(strings.Join(sqliteClauses, " "), "json_extract(metadata, ?) = ?") { + t.Fatalf("sqlite metadata clauses = %v", sqliteClauses) + } +} + func TestBuildIssueFilterClauses_CombinedFilters(t *testing.T) { t.Parallel() diff --git a/internal/storage/issueops/helpers.go b/internal/storage/issueops/helpers.go index c1e5f4357..beb58c6a6 100644 --- a/internal/storage/issueops/helpers.go +++ b/internal/storage/issueops/helpers.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" + "github.com/google/uuid" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/idgen" "github.com/steveyegge/beads/internal/storage" @@ -105,6 +106,21 @@ func InsertIssueIntoTable(ctx context.Context, tx *sql.Tx, table string, issue * // //nolint:gosec // G201: table is a hardcoded constant ("events" or "wisp_events") func RecordEventInTable(ctx context.Context, tx *sql.Tx, table, issueID string, eventType types.EventType, actor, newValue string) error { + return RecordEventInTableWithDialect(ctx, tx, table, issueID, eventType, actor, newValue, SQLDialectDolt) +} + +func RecordEventInTableWithDialect(ctx context.Context, tx *sql.Tx, table, issueID string, eventType types.EventType, actor, newValue string, dialect SQLDialect) error { + if dialect == SQLDialectSQLite { + _, err := tx.ExecContext(ctx, fmt.Sprintf(` + INSERT INTO %s (id, issue_id, event_type, actor, old_value, new_value) + VALUES (?, ?, ?, ?, ?, ?) + `, table), uuid.Must(uuid.NewV7()).String(), issueID, eventType, actor, "", newValue) + if err != nil { + return fmt.Errorf("record event in %s: %w", table, err) + } + return nil + } + _, err := tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO %s (issue_id, event_type, actor, old_value, new_value) VALUES (?, ?, ?, ?, ?) diff --git a/internal/storage/issueops/labels.go b/internal/storage/issueops/labels.go index 7d201e7d9..001c86165 100644 --- a/internal/storage/issueops/labels.go +++ b/internal/storage/issueops/labels.go @@ -110,8 +110,11 @@ func GetLabelsForIssuesInTx(ctx context.Context, tx *sql.Tx, issueIDs []string, // AddLabelInTx adds a label to an issue and records an event within an existing // transaction. Automatically routes to wisp tables if the ID is an active wisp. -// Uses INSERT IGNORE for idempotency. func AddLabelInTx(ctx context.Context, tx *sql.Tx, labelTable, eventTable, issueID, label, actor string) error { + return AddLabelInTxWithDialect(ctx, tx, labelTable, eventTable, issueID, label, actor, SQLDialectDolt) +} + +func AddLabelInTxWithDialect(ctx context.Context, tx *sql.Tx, labelTable, eventTable, issueID, label, actor string, dialect SQLDialect) error { if labelTable == "" || eventTable == "" { isWisp := IsActiveWispInTx(ctx, tx, issueID) _, lt, et, _ := WispTableRouting(isWisp) @@ -122,14 +125,16 @@ func AddLabelInTx(ctx context.Context, tx *sql.Tx, labelTable, eventTable, issue eventTable = et } } + insert := "INSERT IGNORE INTO %s (issue_id, label) VALUES (?, ?)" + if dialect == SQLDialectSQLite { + insert = "INSERT OR IGNORE INTO %s (issue_id, label) VALUES (?, ?)" + } //nolint:gosec // G201: labelTable is from WispTableRouting ("labels" or "wisp_labels") - if _, err := tx.ExecContext(ctx, fmt.Sprintf(`INSERT IGNORE INTO %s (issue_id, label) VALUES (?, ?)`, labelTable), issueID, label); err != nil { + if _, err := tx.ExecContext(ctx, fmt.Sprintf(insert, labelTable), issueID, label); err != nil { return fmt.Errorf("add label: %w", err) } comment := "Added label: " + label - //nolint:gosec // G201: eventTable is from WispTableRouting ("events" or "wisp_events") - if _, err := tx.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s (issue_id, event_type, actor, comment) VALUES (?, ?, ?, ?)`, eventTable), - issueID, types.EventLabelAdded, actor, comment); err != nil { + if err := RecordEventInTableWithDialect(ctx, tx, eventTable, issueID, types.EventLabelAdded, actor, comment, dialect); err != nil { return fmt.Errorf("add label: record event: %w", err) } return nil diff --git a/internal/storage/issueops/ready_work.go b/internal/storage/issueops/ready_work.go index fc0f1565c..e3be783d0 100644 --- a/internal/storage/issueops/ready_work.go +++ b/internal/storage/issueops/ready_work.go @@ -21,6 +21,16 @@ func GetReadyWorkInTx( tx *sql.Tx, filter types.WorkFilter, computeBlockedFn func(ctx context.Context, tx *sql.Tx, includeWisps bool) ([]string, error), +) ([]*types.Issue, error) { + return GetReadyWorkInTxWithDialect(ctx, tx, filter, computeBlockedFn, SQLDialectDolt) +} + +func GetReadyWorkInTxWithDialect( + ctx context.Context, + tx *sql.Tx, + filter types.WorkFilter, + computeBlockedFn func(ctx context.Context, tx *sql.Tx, includeWisps bool) ([]string, error), + dialect SQLDialect, ) ([]*types.Issue, error) { // Status filtering: default to open OR in_progress. var statusClause string @@ -79,11 +89,11 @@ func GetReadyWorkInTx( } // Exclude future-deferred issues unless IncludeDeferred is set. if !filter.IncludeDeferred { - whereClauses = append(whereClauses, "(defer_until IS NULL OR defer_until <= UTC_TIMESTAMP())") + whereClauses = append(whereClauses, "(defer_until IS NULL OR defer_until <= "+dialect.CurrentTimestamp()+")") } // Exclude children of future-deferred parents. if !filter.IncludeDeferred { - deferredChildIDs, dcErr := getChildrenOfDeferredParentsInTx(ctx, tx) + deferredChildIDs, dcErr := getChildrenOfDeferredParentsInTx(ctx, tx, dialect) if dcErr == nil && len(deferredChildIDs) > 0 { for start := 0; start < len(deferredChildIDs); start += queryBatchSize { end := start + queryBatchSize @@ -120,7 +130,7 @@ func GetReadyWorkInTx( if descErr != nil { return nil, fmt.Errorf("get parent descendants: %w", descErr) } - parentClauses := []string{"(id LIKE CONCAT(?, '.%') AND id NOT IN (SELECT issue_id FROM dependencies WHERE type = 'parent-child'))"} + parentClauses := []string{"(" + dialect.ChildIDLikeExpr() + " AND id NOT IN (SELECT issue_id FROM dependencies WHERE type = 'parent-child'))"} args = append(args, parentID) for start := 0; start < len(descendantIDs); start += queryBatchSize { end := start + queryBatchSize @@ -136,7 +146,7 @@ func GetReadyWorkInTx( // Molecule filtering: filter to direct children of the specified molecule. if filter.MoleculeID != "" { - whereClauses = append(whereClauses, "(id IN (SELECT issue_id FROM dependencies WHERE type = 'parent-child' AND depends_on_id = ?) OR (id LIKE CONCAT(?, '.%') AND id NOT IN (SELECT issue_id FROM dependencies WHERE type = 'parent-child')))") + whereClauses = append(whereClauses, "(id IN (SELECT issue_id FROM dependencies WHERE type = 'parent-child' AND depends_on_id = ?) OR ("+dialect.ChildIDLikeExpr()+" AND id NOT IN (SELECT issue_id FROM dependencies WHERE type = 'parent-child')))") args = append(args, filter.MoleculeID, filter.MoleculeID) } @@ -145,7 +155,7 @@ func GetReadyWorkInTx( if err := storage.ValidateMetadataKey(filter.HasMetadataKey); err != nil { return nil, err } - whereClauses = append(whereClauses, "JSON_EXTRACT(metadata, ?) IS NOT NULL") + whereClauses = append(whereClauses, dialect.MetadataExistsExpr()) args = append(args, "$."+filter.HasMetadataKey) } @@ -160,7 +170,7 @@ func GetReadyWorkInTx( if err := storage.ValidateMetadataKey(k); err != nil { return nil, err } - whereClauses = append(whereClauses, "JSON_UNQUOTE(JSON_EXTRACT(metadata, ?)) = ?") + whereClauses = append(whereClauses, dialect.MetadataEqualsExpr()) args = append(args, storage.JSONMetadataPath(k), filter.MetadataFields[k]) } } @@ -200,9 +210,10 @@ func GetReadyWorkInTx( case types.SortPolicyPriority: orderBySQL = "ORDER BY priority ASC, created_at DESC, id ASC" case types.SortPolicyHybrid, "": + recentCreatedAt := dialect.RecentCreatedAtExpr() orderBySQL = `ORDER BY - CASE WHEN created_at >= DATE_SUB(NOW(), INTERVAL 48 HOUR) THEN 0 ELSE 1 END ASC, - CASE WHEN created_at >= DATE_SUB(NOW(), INTERVAL 48 HOUR) THEN priority ELSE 999 END ASC, + CASE WHEN created_at >= ` + recentCreatedAt + ` THEN 0 ELSE 1 END ASC, + CASE WHEN created_at >= ` + recentCreatedAt + ` THEN priority ELSE 999 END ASC, created_at ASC, id ASC` default: orderBySQL = "ORDER BY priority ASC, created_at DESC, id ASC" @@ -260,7 +271,7 @@ func GetReadyWorkInTx( s := filter.Status wispFilter.Status = &s } - wisps, wErr := SearchIssuesInTx(ctx, tx, "", wispFilter) + wisps, wErr := SearchIssuesInTxWithDialect(ctx, tx, "", wispFilter, dialect) if wErr == nil { ordered = append(ordered, wisps...) } @@ -271,12 +282,13 @@ func GetReadyWorkInTx( // getChildrenOfDeferredParentsInTx returns IDs of issues whose parent has a // future defer_until. Works within an existing transaction. -func getChildrenOfDeferredParentsInTx(ctx context.Context, tx *sql.Tx) ([]string, error) { +func getChildrenOfDeferredParentsInTx(ctx context.Context, tx *sql.Tx, dialect SQLDialect) ([]string, error) { // Step 1: Get IDs of issues with future defer_until. - deferredRows, err := tx.QueryContext(ctx, ` + query := ` SELECT id FROM issues - WHERE defer_until IS NOT NULL AND defer_until > UTC_TIMESTAMP() - `) + WHERE defer_until IS NOT NULL AND defer_until > ` + dialect.CurrentTimestamp() + ` + ` + deferredRows, err := tx.QueryContext(ctx, query) if err != nil { return nil, fmt.Errorf("deferred parents: get deferred issues: %w", err) } diff --git a/internal/storage/issueops/search.go b/internal/storage/issueops/search.go index b9d297cab..030d93a7d 100644 --- a/internal/storage/issueops/search.go +++ b/internal/storage/issueops/search.go @@ -13,9 +13,13 @@ import ( // It queries the issues table, optionally merges wisps, and returns hydrated issues // with labels populated. func SearchIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter) ([]*types.Issue, error) { + return SearchIssuesInTxWithDialect(ctx, tx, query, filter, SQLDialectDolt) +} + +func SearchIssuesInTxWithDialect(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter, dialect SQLDialect) ([]*types.Issue, error) { // Route ephemeral-only queries to wisps table. if filter.Ephemeral != nil && *filter.Ephemeral { - results, err := searchTableInTx(ctx, tx, query, filter, WispsFilterTables) + results, err := searchTableInTx(ctx, tx, query, filter, WispsFilterTables, dialect) if err != nil && !isTableNotExistError(err) { return nil, fmt.Errorf("search wisps (ephemeral filter): %w", err) } @@ -25,7 +29,7 @@ func SearchIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter type // Fall through: wisps table doesn't exist or returned no results } - results, err := searchTableInTx(ctx, tx, query, filter, IssuesFilterTables) + results, err := searchTableInTx(ctx, tx, query, filter, IssuesFilterTables, dialect) if err != nil { return nil, fmt.Errorf("search issues: %w", err) } @@ -33,7 +37,7 @@ func SearchIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter type // When filter.Ephemeral is nil (search everything), also search the wisps // table and merge results. if filter.Ephemeral == nil { - wispResults, wispErr := searchTableInTx(ctx, tx, query, filter, WispsFilterTables) + wispResults, wispErr := searchTableInTx(ctx, tx, query, filter, WispsFilterTables, dialect) if wispErr != nil && !isTableNotExistError(wispErr) { return nil, fmt.Errorf("search wisps (merge): %w", wispErr) } @@ -54,8 +58,8 @@ func SearchIssuesInTx(ctx context.Context, tx *sql.Tx, query string, filter type } // searchTableInTx runs a filtered search against a specific table set (issues or wisps). -func searchTableInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter, tables FilterTables) ([]*types.Issue, error) { - whereClauses, args, err := BuildIssueFilterClauses(query, filter, tables) +func searchTableInTx(ctx context.Context, tx *sql.Tx, query string, filter types.IssueFilter, tables FilterTables, dialect SQLDialect) ([]*types.Issue, error) { + whereClauses, args, err := BuildIssueFilterClausesWithDialect(query, filter, tables, dialect) if err != nil { return nil, err } diff --git a/internal/storage/issueops/update.go b/internal/storage/issueops/update.go index b489963fe..547a4fcd0 100644 --- a/internal/storage/issueops/update.go +++ b/internal/storage/issueops/update.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/google/uuid" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" ) @@ -126,6 +127,10 @@ type UpdateResult struct { // //nolint:gosec // G201: table names come from WispTableRouting (hardcoded constants) func UpdateIssueInTx(ctx context.Context, tx *sql.Tx, id string, updates map[string]interface{}, actor string) (*UpdateResult, error) { + return UpdateIssueInTxWithDialect(ctx, tx, id, updates, actor, SQLDialectDolt) +} + +func UpdateIssueInTxWithDialect(ctx context.Context, tx *sql.Tx, id string, updates map[string]interface{}, actor string, dialect SQLDialect) (*UpdateResult, error) { // Route to correct table. isWisp := IsActiveWispInTx(ctx, tx, id) issueTable, _, eventTable, _ := WispTableRouting(isWisp) @@ -218,7 +223,7 @@ func UpdateIssueInTx(ctx context.Context, tx *sql.Tx, id string, updates map[str newData, _ := json.Marshal(updates) eventType := DetermineEventType(oldIssue, updates) - if err := RecordFullEventInTable(ctx, tx, eventTable, id, eventType, actor, string(oldData), string(newData)); err != nil { + if err := RecordFullEventInTableWithDialect(ctx, tx, eventTable, id, eventType, actor, string(oldData), string(newData), dialect); err != nil { return nil, fmt.Errorf("failed to record event: %w", err) } @@ -229,6 +234,21 @@ func UpdateIssueInTx(ctx context.Context, tx *sql.Tx, id string, updates map[str // //nolint:gosec // G201: table is from WispTableRouting ("events" or "wisp_events") func RecordFullEventInTable(ctx context.Context, tx *sql.Tx, table, issueID string, eventType types.EventType, actor, oldValue, newValue string) error { + return RecordFullEventInTableWithDialect(ctx, tx, table, issueID, eventType, actor, oldValue, newValue, SQLDialectDolt) +} + +func RecordFullEventInTableWithDialect(ctx context.Context, tx *sql.Tx, table, issueID string, eventType types.EventType, actor, oldValue, newValue string, dialect SQLDialect) error { + if dialect == SQLDialectSQLite { + _, err := tx.ExecContext(ctx, fmt.Sprintf(` + INSERT INTO %s (id, issue_id, event_type, actor, old_value, new_value) + VALUES (?, ?, ?, ?, ?, ?) + `, table), uuid.Must(uuid.NewV7()).String(), issueID, eventType, actor, oldValue, newValue) + if err != nil { + return fmt.Errorf("record event in %s: %w", table, err) + } + return nil + } + _, err := tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO %s (issue_id, event_type, actor, old_value, new_value) VALUES (?, ?, ?, ?, ?) diff --git a/internal/utils/id_parser.go b/internal/utils/id_parser.go index 8ebccf7a2..09896982a 100644 --- a/internal/utils/id_parser.go +++ b/internal/utils/id_parser.go @@ -41,13 +41,11 @@ func ResolvePartialID(ctx context.Context, store storage.Storage, input string) return "", fmt.Errorf("cannot resolve issue ID %q: storage is nil", input) } - // Fast path: Use SearchIssues with exact ID filter (GH#942). - // This uses the same query path as "bd list --id", ensuring consistency. - // Previously we used GetIssue which could fail in cases where SearchIssues - // with filter.IDs succeeded, likely due to subtle query differences. - exactFilter := types.IssueFilter{IDs: []string{input}} - if issues, err := store.SearchIssues(ctx, "", exactFilter); err == nil && len(issues) > 0 { - return issues[0].ID, nil + // Fast path: exact ID lookup should be an indexed primary-key read. This + // keeps exact probes such as "bd show abc-123" from paying the broader + // partial-ID search path when the issue is absent. + if issue, err := store.GetIssue(ctx, input); err == nil && issue != nil { + return issue.ID, nil } // Get the configured prefix @@ -99,10 +97,11 @@ func ResolvePartialID(ctx context.Context, store storage.Storage, input string) normalizedID = prefixWithHyphen + input } - // Try exact match on normalized ID using SearchIssues (GH#942) - normalizedFilter := types.IssueFilter{IDs: []string{normalizedID}} - if issues, err := store.SearchIssues(ctx, "", normalizedFilter); err == nil && len(issues) > 0 { - return issues[0].ID, nil + // Try exact match on normalized ID before falling back to substring search. + if normalizedID != input { + if issue, err := store.GetIssue(ctx, normalizedID); err == nil && issue != nil { + return issue.ID, nil + } } // If exact match failed, try substring search. From f63c8ea610ed1e1a6225c61c88d6a6bdd1049190 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Thu, 30 Apr 2026 06:41:37 +1000 Subject: [PATCH 04/15] fix doltlite claim event dialect --- ...oltlite-backend-verification-2026-04-30.md | 85 ++++++++++++++++++ internal/storage/doltlite/issues.go | 2 +- internal/storage/issueops/claim.go | 10 ++- .../storage/issueops/claim_sqlite_test.go | 87 +++++++++++++++++++ 4 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 docs/dev-notes/doltlite-backend-verification-2026-04-30.md create mode 100644 internal/storage/issueops/claim_sqlite_test.go diff --git a/docs/dev-notes/doltlite-backend-verification-2026-04-30.md b/docs/dev-notes/doltlite-backend-verification-2026-04-30.md new file mode 100644 index 000000000..1c51644d2 --- /dev/null +++ b/docs/dev-notes/doltlite-backend-verification-2026-04-30.md @@ -0,0 +1,85 @@ +# Dolt and Doltlite Backend Verification + +Date: 2026-04-30 + +## Scope + +Focused verification for: + +- `bd where` +- `bd list` +- `bd show` +- `bd create` +- `bd update --claim` +- `bd close` +- `gc doctor` +- `gc status` +- `gc events` + +## Result Summary + +### Dolt backend + +Manual CLI matrix passed: + +- `bd init --backend dolt` +- `bd create` +- `bd where` +- `bd list --json` +- `bd show --json` +- `bd update --claim` +- `bd close --reason done` + +Observed state transitions: + +- after claim: `in_progress`, assignee set +- after close: `closed` + +### Doltlite backend + +Manual CLI matrix is blocked at init: + +```text +Error: failed to open doltlite store: doltlite: init schema: dolt add after migrations: no such function: dolt_add +``` + +This prevents follow-on CLI verification for `create/list/show/claim/close`. + +## Code Fix Applied + +Claim/event recording for SQLite-compatible backends was using the Dolt path and +implicitly relying on `UUID()` in the events table. That fails for SQLite. + +Applied fix: + +- added `ClaimIssueInTxWithDialect(...)` +- routed doltlite `ClaimIssue(...)` through `SQLDialectSQLite` +- claim events now use explicit UUID generation on the SQLite path + +## Regression Coverage + +Added targeted test: + +- `internal/storage/issueops/claim_sqlite_test.go` + +Verified: + +```text +go test ./internal/storage/issueops -run TestClaimIssueInTxWithDialectSQLiteRecordsUUIDEvent -count=1 +``` + +Passes. + +## Additional GC Checks + +From `/data/projects/beads-doltlite`: + +- `gc doctor`: reports missing local `city.toml`, legacy layout warning, invalid inherited Dolt endpoint +- `gc status`: fails to load local `city.toml` +- `gc events --limit 5`: invalid flag; command supports `--since`, `--follow`, `--watch`, `--seq` + +## Remaining Blocker + +Doltlite schema/version-control initialization still assumes `dolt_add` SQL +function availability. That path must be reconciled before full doltlite CLI +matrix coverage is possible. diff --git a/internal/storage/doltlite/issues.go b/internal/storage/doltlite/issues.go index 9fcc50b45..64d65b5a8 100644 --- a/internal/storage/doltlite/issues.go +++ b/internal/storage/doltlite/issues.go @@ -17,7 +17,7 @@ import ( // Delegates SQL work to issueops; EmbeddedDolt auto-commits the transaction. func (s *DoltliteStore) ClaimIssue(ctx context.Context, id string, actor string) error { return s.withConn(ctx, true, func(tx *sql.Tx) error { - _, err := issueops.ClaimIssueInTx(ctx, tx, id, actor) + _, err := issueops.ClaimIssueInTxWithDialect(ctx, tx, id, actor, issueops.SQLDialectSQLite) return err }) } diff --git a/internal/storage/issueops/claim.go b/internal/storage/issueops/claim.go index 0d48ced66..2efaf020a 100644 --- a/internal/storage/issueops/claim.go +++ b/internal/storage/issueops/claim.go @@ -27,6 +27,14 @@ type ClaimResult struct { // //nolint:gosec // G201: table names come from WispTableRouting (hardcoded constants) func ClaimIssueInTx(ctx context.Context, tx *sql.Tx, id string, actor string) (*ClaimResult, error) { + return ClaimIssueInTxWithDialect(ctx, tx, id, actor, SQLDialectDolt) +} + +// ClaimIssueInTxWithDialect atomically claims an issue using compare-and-swap +// semantics while honoring backend-specific SQL differences. +// +//nolint:gosec // G201: table names come from WispTableRouting (hardcoded constants) +func ClaimIssueInTxWithDialect(ctx context.Context, tx *sql.Tx, id string, actor string, dialect SQLDialect) (*ClaimResult, error) { isWisp := IsActiveWispInTx(ctx, tx, id) issueTable, _, eventTable, _ := WispTableRouting(isWisp) @@ -91,7 +99,7 @@ func ClaimIssueInTx(ctx context.Context, tx *sql.Tx, id string, actor string) (* } newData, _ := json.Marshal(newUpdates) - if err := RecordFullEventInTable(ctx, tx, eventTable, id, "claimed", actor, string(oldData), string(newData)); err != nil { + if err := RecordFullEventInTableWithDialect(ctx, tx, eventTable, id, "claimed", actor, string(oldData), string(newData), dialect); err != nil { return nil, fmt.Errorf("failed to record claim event: %w", err) } diff --git a/internal/storage/issueops/claim_sqlite_test.go b/internal/storage/issueops/claim_sqlite_test.go new file mode 100644 index 000000000..ee42ed8ab --- /dev/null +++ b/internal/storage/issueops/claim_sqlite_test.go @@ -0,0 +1,87 @@ +//go:build cgo + +package issueops + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + + "github.com/steveyegge/beads/internal/storage/schema" + "github.com/steveyegge/beads/internal/types" +) + +func TestClaimIssueInTxWithDialectSQLiteRecordsUUIDEvent(t *testing.T) { + ctx := context.Background() + db, err := sql.Open("sqlite3", filepath.Join(t.TempDir(), "beads.db")) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + if err := schema.CreateIgnoredTablesSQLite(ctx, db); err != nil { + t.Fatalf("CreateIgnoredTablesSQLite: %v", err) + } + if _, err := schema.MigrateUpSQLite(ctx, db); err != nil { + t.Fatalf("MigrateUpSQLite: %v", err) + } + + now := time.Now().UTC() + issue := &types.Issue{ + ID: "bd-claim", + Title: "claim sqlite", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now, + UpdatedAt: now, + } + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + defer tx.Rollback() + + if _, err := tx.ExecContext(ctx, ` + INSERT INTO issues ( + id, title, description, design, acceptance_criteria, notes, status, priority, issue_type, + created_at, updated_at, created_by, owner, metadata + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, issue.ID, issue.Title, issue.Description, "", "", "", issue.Status, issue.Priority, issue.IssueType, + issue.CreatedAt, issue.UpdatedAt, "test", "test", "{}"); err != nil { + t.Fatalf("insert issue: %v", err) + } + if _, err := ClaimIssueInTxWithDialect(ctx, tx, issue.ID, "worker", SQLDialectSQLite); err != nil { + t.Fatalf("ClaimIssueInTxWithDialect: %v", err) + } + + got, err := GetIssueInTx(ctx, tx, issue.ID) + if err != nil { + t.Fatalf("GetIssueInTx: %v", err) + } + if got.Assignee != "worker" { + t.Fatalf("assignee = %q, want worker", got.Assignee) + } + if got.Status != types.StatusInProgress { + t.Fatalf("status = %q, want %q", got.Status, types.StatusInProgress) + } + + events, err := GetEventsInTx(ctx, tx, issue.ID, 10) + if err != nil { + t.Fatalf("GetEventsInTx: %v", err) + } + if len(events) == 0 { + t.Fatal("expected claim event") + } + if events[0].ID == "" { + t.Fatal("claim event missing UUID") + } + if events[0].EventType != types.EventType("claimed") { + t.Fatalf("event type = %q, want claimed", events[0].EventType) + } +} From b0b6e68830fdbc96d2f59edff11b76273bd79ca8 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Thu, 30 Apr 2026 12:20:20 +1000 Subject: [PATCH 05/15] feat: add doltlite backend fixes --- internal/storage/doltlite/child_id.go | 2 +- internal/storage/doltlite/dependencies.go | 1 + internal/storage/doltlite/smoke_test.go | 63 ++++++++++++++++++++ internal/storage/doltlite/store.go | 47 +++++++++++++-- internal/storage/doltlite/transaction.go | 1 + internal/storage/doltlite/version_control.go | 7 +-- internal/storage/issueops/child_id.go | 31 ++++++++-- internal/storage/issueops/create.go | 17 +++++- internal/storage/issueops/dependencies.go | 7 ++- 9 files changed, 155 insertions(+), 21 deletions(-) diff --git a/internal/storage/doltlite/child_id.go b/internal/storage/doltlite/child_id.go index 1d77c298a..0acf761a3 100644 --- a/internal/storage/doltlite/child_id.go +++ b/internal/storage/doltlite/child_id.go @@ -13,7 +13,7 @@ func (s *DoltliteStore) GetNextChildID(ctx context.Context, parentID string) (st var childID string err := s.withConn(ctx, true, func(tx *sql.Tx) error { var err error - childID, err = issueops.GetNextChildIDTx(ctx, tx, parentID) + childID, err = issueops.GetNextChildIDTxWithDialect(ctx, tx, parentID, issueops.SQLDialectSQLite) return err }) return childID, err diff --git a/internal/storage/doltlite/dependencies.go b/internal/storage/doltlite/dependencies.go index 6e54699ab..edd6b3264 100644 --- a/internal/storage/doltlite/dependencies.go +++ b/internal/storage/doltlite/dependencies.go @@ -14,6 +14,7 @@ func (s *DoltliteStore) AddDependency(ctx context.Context, dep *types.Dependency return s.withConn(ctx, true, func(tx *sql.Tx) error { return issueops.AddDependencyInTx(ctx, tx, dep, actor, issueops.AddDependencyOpts{ IsCrossPrefix: types.ExtractPrefix(dep.IssueID) != types.ExtractPrefix(dep.DependsOnID), + Dialect: issueops.SQLDialectSQLite, }) }) } diff --git a/internal/storage/doltlite/smoke_test.go b/internal/storage/doltlite/smoke_test.go index f75b32170..0e121bac8 100644 --- a/internal/storage/doltlite/smoke_test.go +++ b/internal/storage/doltlite/smoke_test.go @@ -95,6 +95,69 @@ func TestSmokeLabels(t *testing.T) { } } +func TestSmokeChildIDAndDependencyUseSQLiteDialect(t *testing.T) { + ctx := t.Context() + store, err := doltlite.New(ctx, filepath.Join(t.TempDir(), ".beads"), "beads", "main") + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + + now := time.Now().UTC() + parent := &types.Issue{ + ID: "bd-parent", + Title: "parent", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now, + UpdatedAt: now, + } + child := &types.Issue{ + ID: "bd-parent.1", + Title: "child", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + CreatedAt: now, + UpdatedAt: now, + } + if err := store.CreateIssue(ctx, parent, "test"); err != nil { + t.Fatalf("CreateIssue parent: %v", err) + } + if err := store.CreateIssue(ctx, child, "test"); err != nil { + t.Fatalf("CreateIssue child: %v", err) + } + + next, err := store.GetNextChildID(ctx, parent.ID) + if err != nil { + t.Fatalf("GetNextChildID: %v", err) + } + if next != "bd-parent.2" { + t.Fatalf("next child ID = %q, want bd-parent.2", next) + } + + dep := &types.Dependency{ + IssueID: child.ID, + DependsOnID: parent.ID, + Type: types.DepParentChild, + } + if err := store.AddDependency(ctx, dep, "test"); err != nil { + t.Fatalf("AddDependency: %v", err) + } + deps, err := store.GetDependencyRecords(ctx, child.ID) + if err != nil { + t.Fatalf("GetDependencyRecords: %v", err) + } + if len(deps) != 1 || deps[0].DependsOnID != parent.ID || deps[0].Type != types.DepParentChild { + t.Fatalf("deps = %#v, want parent-child to %s", deps, parent.ID) + } +} + func TestSmokeVersionControl(t *testing.T) { ctx := t.Context() store, err := doltlite.New(ctx, filepath.Join(t.TempDir(), ".beads"), "beads", "main") diff --git a/internal/storage/doltlite/store.go b/internal/storage/doltlite/store.go index e87d26a19..41b08d633 100644 --- a/internal/storage/doltlite/store.go +++ b/internal/storage/doltlite/store.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "strings" + "sync" "sync/atomic" "time" @@ -43,6 +44,9 @@ type DoltliteStore struct { database string branch string credentialKey []byte + dbMu sync.Mutex + db *sql.DB + dbCleanup func() error closed atomic.Bool } @@ -112,6 +116,9 @@ func New(ctx context.Context, beadsDir, database, branch string, opts ...Option) if err := s.initSchema(ctx); err != nil { return nil, fmt.Errorf("doltlite: init schema: %w", err) } + if err := s.openPersistentDB(ctx); err != nil { + return nil, fmt.Errorf("doltlite: open database: %w", err) + } // Ensure dolt_ignore'd wisp tables exist in the working set. // After a clone or branch switch, these tables are absent because @@ -124,6 +131,31 @@ func New(ctx context.Context, beadsDir, database, branch string, opts ...Option) return s, nil } +func (s *DoltliteStore) openPersistentDB(ctx context.Context) error { + s.dbMu.Lock() + defer s.dbMu.Unlock() + if s.db != nil { + return nil + } + db, cleanup, err := OpenSQL(ctx, s.dataDir, s.database, s.branch) + if err != nil { + return err + } + s.db = db + s.dbCleanup = cleanup + return nil +} + +func (s *DoltliteStore) activeDB(ctx context.Context) (*sql.DB, func() error, error) { + s.dbMu.Lock() + db := s.db + s.dbMu.Unlock() + if db != nil { + return db, func() error { return nil }, nil + } + return OpenSQL(ctx, s.dataDir, s.database, s.branch) +} + // withRootConn opens a short-lived database connection without selecting any // database or branch, begins an explicit SQL transaction, and passes it to fn. // This is used during initialization when the database may not yet exist. @@ -202,7 +234,7 @@ func (s *DoltliteStore) withConnOnce(ctx context.Context, commit bool, fn func(t var db *sql.DB var cleanup func() error - db, cleanup, err = OpenSQL(ctx, s.dataDir, s.database, s.branch) + db, cleanup, err = s.activeDB(ctx) if err != nil { return } @@ -301,10 +333,7 @@ func (s *DoltliteStore) initSchema(ctx context.Context) error { return err } if applied > 0 { - if _, err := db.ExecContext(ctx, "SELECT dolt_add('-A')"); err != nil { - return fmt.Errorf("dolt add after migrations: %w", err) - } - if _, err := db.ExecContext(ctx, "SELECT dolt_commit('-m', 'schema: apply migrations')"); err != nil { + if _, err := db.ExecContext(ctx, "SELECT dolt_commit('-A', '-m', 'schema: apply migrations')"); err != nil { if !strings.Contains(err.Error(), "nothing to commit") { return fmt.Errorf("dolt commit after migrations: %w", err) } @@ -474,6 +503,14 @@ func (s *DoltliteStore) GetAllEventsSince(ctx context.Context, since time.Time) // garbage. Subsequent method calls will return errClosed. func (s *DoltliteStore) Close() error { if s.closed.CompareAndSwap(false, true) { + s.dbMu.Lock() + cleanup := s.dbCleanup + s.db = nil + s.dbCleanup = nil + s.dbMu.Unlock() + if cleanup != nil { + _ = cleanup() + } s.cleanGitRemoteCacheGarbage() } return nil diff --git a/internal/storage/doltlite/transaction.go b/internal/storage/doltlite/transaction.go index abacb266c..abe77ba57 100644 --- a/internal/storage/doltlite/transaction.go +++ b/internal/storage/doltlite/transaction.go @@ -99,6 +99,7 @@ func (t *embeddedTransaction) AddDependencyWithOptions(ctx context.Context, dep return issueops.AddDependencyInTx(ctx, t.tx, dep, actor, issueops.AddDependencyOpts{ IsCrossPrefix: types.ExtractPrefix(dep.IssueID) != types.ExtractPrefix(dep.DependsOnID), SkipCycleCheck: addOpts.SkipCycleCheck, + Dialect: issueops.SQLDialectSQLite, }) } diff --git a/internal/storage/doltlite/version_control.go b/internal/storage/doltlite/version_control.go index 61ba68ab1..86ecfd0f3 100644 --- a/internal/storage/doltlite/version_control.go +++ b/internal/storage/doltlite/version_control.go @@ -27,7 +27,7 @@ func (s *DoltliteStore) withDBConn(ctx context.Context, fn func(db versioncontro var db *sql.DB var cleanup func() error - db, cleanup, err = OpenSQL(ctx, s.dataDir, s.database, s.branch) + db, cleanup, err = s.activeDB(ctx) if err != nil { return } @@ -45,10 +45,7 @@ func (s *DoltliteStore) Commit(ctx context.Context, message string) error { return s.withExclusiveLock(ctx, func() error { return s.withRetry(ctx, func() error { return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - if _, err := db.ExecContext(ctx, "SELECT dolt_add('-A')"); err != nil { - return fmt.Errorf("dolt add: %w", err) - } - if _, err := db.ExecContext(ctx, "SELECT dolt_commit('-m', ?)", message); err != nil { + if _, err := db.ExecContext(ctx, "SELECT dolt_commit('-A', '-m', ?)", message); err != nil { return fmt.Errorf("dolt commit: %w", err) } return nil diff --git a/internal/storage/issueops/child_id.go b/internal/storage/issueops/child_id.go index 44b1ef94c..c4d9c9ea2 100644 --- a/internal/storage/issueops/child_id.go +++ b/internal/storage/issueops/child_id.go @@ -13,6 +13,10 @@ import ( // // Returns the full child ID string (e.g., "parent-id.3"). func GetNextChildIDTx(ctx context.Context, tx *sql.Tx, parentID string) (string, error) { + return GetNextChildIDTxWithDialect(ctx, tx, parentID, SQLDialectDolt) +} + +func GetNextChildIDTxWithDialect(ctx context.Context, tx *sql.Tx, parentID string, dialect SQLDialect) (string, error) { var lastChild int err := tx.QueryRowContext(ctx, "SELECT last_child FROM child_counters WHERE parent_id = ?", parentID).Scan(&lastChild) if err == sql.ErrNoRows { @@ -27,11 +31,17 @@ func GetNextChildIDTx(ctx context.Context, tx *sql.Tx, parentID string) (string, // We fetch direct child IDs and parse the numeric suffix in Go rather than // using SQL CAST(SUBSTRING_INDEX(...) AS UNSIGNED), which silently returns 0 // for non-numeric ID suffixes (see GH#2721). - rows, err := tx.QueryContext(ctx, ` + childLikeExpr := "id LIKE CONCAT(?, '.%')" + grandchildLikeExpr := "id NOT LIKE CONCAT(?, '.%.%')" + if dialect == SQLDialectSQLite { + childLikeExpr = "id LIKE (? || '.%')" + grandchildLikeExpr = "id NOT LIKE (? || '.%.%')" + } + rows, err := tx.QueryContext(ctx, fmt.Sprintf(` SELECT id FROM issues - WHERE id LIKE CONCAT(?, '.%') - AND id NOT LIKE CONCAT(?, '.%.%') - `, parentID, parentID) + WHERE %s + AND %s + `, childLikeExpr, grandchildLikeExpr), parentID, parentID) if err != nil { return "", fmt.Errorf("get next child ID: query existing children: %w", err) } @@ -53,10 +63,19 @@ func GetNextChildIDTx(ctx context.Context, tx *sql.Tx, parentID string) (string, nextChild := lastChild + 1 - if _, err := tx.ExecContext(ctx, ` + upsert := ` INSERT INTO child_counters (parent_id, last_child) VALUES (?, ?) ON DUPLICATE KEY UPDATE last_child = ? - `, parentID, nextChild, nextChild); err != nil { + ` + args := []any{parentID, nextChild, nextChild} + if dialect == SQLDialectSQLite { + upsert = ` + INSERT INTO child_counters (parent_id, last_child) VALUES (?, ?) + ON CONFLICT(parent_id) DO UPDATE SET last_child = excluded.last_child + ` + args = []any{parentID, nextChild} + } + if _, err := tx.ExecContext(ctx, upsert, args...); err != nil { return "", fmt.Errorf("get next child ID: update counter: %w", err) } diff --git a/internal/storage/issueops/create.go b/internal/storage/issueops/create.go index 57ab6627d..6523f6618 100644 --- a/internal/storage/issueops/create.go +++ b/internal/storage/issueops/create.go @@ -372,6 +372,10 @@ func PersistDependencies(ctx context.Context, tx *sql.Tx, issues []*types.Issue, // ReconcileChildCounters updates child_counters so that subsequent // bd create --parent doesn't collide with imported hierarchical IDs. func ReconcileChildCounters(ctx context.Context, tx *sql.Tx, issues []*types.Issue) error { + return ReconcileChildCountersWithDialect(ctx, tx, issues, SQLDialectDolt) +} + +func ReconcileChildCountersWithDialect(ctx context.Context, tx *sql.Tx, issues []*types.Issue, dialect SQLDialect) error { childMaxMap := make(map[string]int) for _, issue := range issues { if parentID, childNum, ok := ParseHierarchicalID(issue.ID); ok { @@ -385,10 +389,19 @@ func ReconcileChildCounters(ctx context.Context, tx *sql.Tx, issues []*types.Iss if err := tx.QueryRowContext(ctx, "SELECT 1 FROM issues WHERE id = ?", parentID).Scan(&parentExists); err != nil { continue // parent not in issues table — skip counter } - _, err := tx.ExecContext(ctx, ` + upsert := ` INSERT INTO child_counters (parent_id, last_child) VALUES (?, ?) ON DUPLICATE KEY UPDATE last_child = GREATEST(last_child, ?) - `, parentID, maxChild, maxChild) + ` + args := []any{parentID, maxChild, maxChild} + if dialect == SQLDialectSQLite { + upsert = ` + INSERT INTO child_counters (parent_id, last_child) VALUES (?, ?) + ON CONFLICT(parent_id) DO UPDATE SET last_child = MAX(child_counters.last_child, excluded.last_child) + ` + args = []any{parentID, maxChild} + } + _, err := tx.ExecContext(ctx, upsert, args...) if err != nil { return fmt.Errorf("failed to reconcile child counter for %s: %w", parentID, err) } diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index 502c650c9..963ebea39 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -34,6 +34,8 @@ type AddDependencyOpts struct { // SkipCycleCheck skips the recursive pre-insert cycle check for callers // that intentionally trade validation cost for bulk graph wiring speed. SkipCycleCheck bool + // Dialect selects SQL syntax for timestamp expressions. + Dialect SQLDialect } // AddDependencyInTx validates and inserts a dependency within an existing @@ -81,6 +83,7 @@ func AddDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, a if metadata == "" { metadata = "{}" } + dialect := opts.Dialect // Validate source issue exists and get its type. var sourceType string @@ -176,8 +179,8 @@ func AddDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, a //nolint:gosec // G201: writeTable is from WispTableRouting if _, err := tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO %s (issue_id, depends_on_id, type, created_at, created_by, metadata, thread_id) - VALUES (?, ?, ?, NOW(), ?, ?, ?) - `, writeTable), dep.IssueID, dep.DependsOnID, dep.Type, actor, metadata, dep.ThreadID); err != nil { + VALUES (?, ?, ?, %s, ?, ?, ?) + `, writeTable, dialect.CurrentTimestamp()), dep.IssueID, dep.DependsOnID, dep.Type, actor, metadata, dep.ThreadID); err != nil { return fmt.Errorf("failed to add dependency: %w", err) } return nil From 274193c03f98e4dc7c4f930787f6ed6cdbdc793c Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Thu, 30 Apr 2026 20:52:38 +1000 Subject: [PATCH 06/15] fix: localize doltlite version control state --- internal/storage/doltlite/open.go | 13 +- internal/storage/doltlite/store.go | 141 +++++++++++- internal/storage/doltlite/version_control.go | 218 +++++++++++-------- 3 files changed, 276 insertions(+), 96 deletions(-) diff --git a/internal/storage/doltlite/open.go b/internal/storage/doltlite/open.go index a0a24142c..248fded70 100644 --- a/internal/storage/doltlite/open.go +++ b/internal/storage/doltlite/open.go @@ -6,6 +6,7 @@ import ( "context" "database/sql" "fmt" + "net/url" "os" "path/filepath" "regexp" @@ -28,7 +29,7 @@ const ( // OpenSQL opens an doltlite database at dir. The returned cleanup // function closes the *sql.DB. func OpenSQL(ctx context.Context, dir, database, branch string) (*sql.DB, func() error, error) { - dbPath, err := buildDSN(dir, database) + dbPath, err := buildBranchDSN(dir, database, branch) if err != nil { return nil, nil, err } @@ -57,6 +58,10 @@ func OpenSQL(ctx context.Context, dir, database, branch string) (*sql.DB, func() } func buildDSN(dir, database string) (string, error) { + return buildBranchDSN(dir, database, "") +} + +func buildBranchDSN(dir, database, branch string) (string, error) { if strings.TrimSpace(database) != "" { if !validIdentifier.MatchString(database) { return "", fmt.Errorf("doltlite: invalid database name: %q", database) @@ -64,7 +69,11 @@ func buildDSN(dir, database string) (string, error) { } else { database = "beads" } - path := filepath.Join(dir, database+".db") + filename := database + ".db" + if strings.TrimSpace(branch) != "" { + filename = fmt.Sprintf("%s__%s.db", database, url.QueryEscape(branch)) + } + path := filepath.Join(dir, filename) if os.PathSeparator == '\\' { path = strings.ReplaceAll(path, `\`, `/`) } diff --git a/internal/storage/doltlite/store.go b/internal/storage/doltlite/store.go index 41b08d633..029fa6731 100644 --- a/internal/storage/doltlite/store.go +++ b/internal/storage/doltlite/store.go @@ -4,9 +4,13 @@ package doltlite import ( "context" + "crypto/sha256" "database/sql" + "encoding/hex" "errors" "fmt" + "io" + "net/url" "os" "path/filepath" "strings" @@ -146,6 +150,25 @@ func (s *DoltliteStore) openPersistentDB(ctx context.Context) error { return nil } +func (s *DoltliteStore) closePersistentDB() error { + s.dbMu.Lock() + cleanup := s.dbCleanup + s.db = nil + s.dbCleanup = nil + s.dbMu.Unlock() + if cleanup != nil { + return cleanup() + } + return nil +} + +func (s *DoltliteStore) resetPersistentDB(ctx context.Context) error { + if err := s.closePersistentDB(); err != nil { + return err + } + return s.openPersistentDB(ctx) +} + func (s *DoltliteStore) activeDB(ctx context.Context) (*sql.DB, func() error, error) { s.dbMu.Lock() db := s.db @@ -332,17 +355,120 @@ func (s *DoltliteStore) initSchema(ctx context.Context) error { if err != nil { return err } + if err := s.ensureVersionControlTables(ctx, db); err != nil { + return err + } if applied > 0 { - if _, err := db.ExecContext(ctx, "SELECT dolt_commit('-A', '-m', 'schema: apply migrations')"); err != nil { - if !strings.Contains(err.Error(), "nothing to commit") { - return fmt.Errorf("dolt commit after migrations: %w", err) - } + if err := s.recordSyntheticCommit(ctx, db, "schema: apply migrations"); err != nil { + return fmt.Errorf("record migration commit: %w", err) } } return nil } +func (s *DoltliteStore) ensureVersionControlTables(ctx context.Context, db *sql.DB) error { + stmts := []string{ + `CREATE TABLE IF NOT EXISTS doltlite_commits ( + hash TEXT PRIMARY KEY, + branch TEXT NOT NULL, + committer TEXT NOT NULL, + email TEXT NOT NULL, + date TEXT NOT NULL, + message TEXT NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS idx_doltlite_commits_branch_date + ON doltlite_commits(branch, date DESC)`, + `CREATE TABLE IF NOT EXISTS doltlite_refs ( + branch TEXT PRIMARY KEY, + head_hash TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS dolt_remotes ( + name TEXT PRIMARY KEY, + url TEXT NOT NULL + )`, + } + for _, stmt := range stmts { + if _, err := db.ExecContext(ctx, stmt); err != nil { + return fmt.Errorf("ensure version-control metadata: %w", err) + } + } + return nil +} + +func (s *DoltliteStore) recordSyntheticCommit(ctx context.Context, db interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) + QueryRowContext(context.Context, string, ...any) *sql.Row +}, message string) error { + if message == "" { + message = "doltlite: snapshot" + } + now := time.Now().UTC().Format(time.RFC3339Nano) + sum := sha256.Sum256([]byte(strings.Join([]string{s.branch, now, message}, "\x00"))) + hash := hex.EncodeToString(sum[:]) + + var current string + err := db.QueryRowContext(ctx, "SELECT head_hash FROM doltlite_refs WHERE branch = ?", s.branch).Scan(¤t) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + if current == hash { + return nil + } + + if _, err := db.ExecContext(ctx, ` + INSERT INTO doltlite_commits (hash, branch, committer, email, date, message) + VALUES (?, ?, ?, ?, ?, ?) + `, hash, s.branch, commitName, commitEmail, now, message); err != nil { + return err + } + _, err = db.ExecContext(ctx, ` + INSERT INTO doltlite_refs (branch, head_hash) VALUES (?, ?) + ON CONFLICT(branch) DO UPDATE SET head_hash = excluded.head_hash + `, s.branch, hash) + return err +} + +func (s *DoltliteStore) branchDBPath(branch string) (string, error) { + dsn, err := buildBranchDSN(s.dataDir, s.database, branch) + if err != nil { + return "", err + } + path := strings.SplitN(dsn, "?", 2)[0] + return path, nil +} + +func branchFromDBFilename(database, name string) (string, bool) { + prefix := database + "__" + suffix := ".db" + if !strings.HasPrefix(name, prefix) || !strings.HasSuffix(name, suffix) { + return "", false + } + encoded := strings.TrimSuffix(strings.TrimPrefix(name, prefix), suffix) + branch, err := url.QueryUnescape(encoded) + if err != nil || branch == "" { + return "", false + } + return branch, true +} + +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + defer func() { _ = out.Close() }() + if _, err := io.Copy(out, in); err != nil { + return err + } + return out.Sync() +} + // ensureIgnoredTables creates dolt_ignore'd wisp tables if they don't exist. // Uses withConn (not withRootConn) because the database is already created. func (s *DoltliteStore) ensureIgnoredTables(ctx context.Context) error { @@ -613,9 +739,12 @@ func (s *DoltliteStore) CommitPending(ctx context.Context, actor string) (bool, func (s *DoltliteStore) GetCurrentCommit(ctx context.Context) (string, error) { var hash string - err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - return db.QueryRowContext(ctx, "SELECT dolt_hashof('HEAD')").Scan(&hash) + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + return tx.QueryRowContext(ctx, "SELECT head_hash FROM doltlite_refs WHERE branch = ?", s.branch).Scan(&hash) }) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } return hash, err } diff --git a/internal/storage/doltlite/version_control.go b/internal/storage/doltlite/version_control.go index 86ecfd0f3..ba6de4463 100644 --- a/internal/storage/doltlite/version_control.go +++ b/internal/storage/doltlite/version_control.go @@ -8,10 +8,10 @@ import ( "errors" "fmt" "os" + "slices" "time" "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/storage/schema" "github.com/steveyegge/beads/internal/storage/versioncontrolops" ) @@ -42,15 +42,8 @@ func (s *DoltliteStore) withDBConn(ctx context.Context, fn func(db versioncontro } func (s *DoltliteStore) Commit(ctx context.Context, message string) error { - return s.withExclusiveLock(ctx, func() error { - return s.withRetry(ctx, func() error { - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - if _, err := db.ExecContext(ctx, "SELECT dolt_commit('-A', '-m', ?)", message); err != nil { - return fmt.Errorf("dolt commit: %w", err) - } - return nil - }) - }) + return s.withConn(ctx, true, func(tx *sql.Tx) error { + return s.recordSyntheticCommit(ctx, tx, message) }) } @@ -62,8 +55,11 @@ func (s *DoltliteStore) CommitWithConfig(ctx context.Context, message string) er } func (s *DoltliteStore) AddRemote(ctx context.Context, name, url string) error { - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - _, err := db.ExecContext(ctx, "SELECT dolt_remote('add', ?, ?)", name, url) + return s.withConn(ctx, true, func(tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, ` + INSERT INTO dolt_remotes (name, url) VALUES (?, ?) + ON CONFLICT(name) DO UPDATE SET url = excluded.url + `, name, url) return err }) } @@ -84,36 +80,91 @@ func (s *DoltliteStore) HasRemote(ctx context.Context, name string) (bool, error // --------------------------------------------------------------------------- func (s *DoltliteStore) Branch(ctx context.Context, name string) error { - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - if _, err := db.ExecContext(ctx, "SELECT dolt_branch(?)", name); err != nil { + return s.withExclusiveLock(ctx, func() error { + srcPath, err := s.branchDBPath(s.branch) + if err != nil { + return err + } + dstPath, err := s.branchDBPath(name) + if err != nil { + return err + } + if srcPath == dstPath { + return fmt.Errorf("create branch %s: branch already exists", name) + } + if err := s.closePersistentDB(); err != nil { + return err + } + defer func() { _ = s.openPersistentDB(ctx) }() + if _, err := os.Stat(dstPath); err == nil { + return fmt.Errorf("create branch %s: branch already exists", name) + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if err := copyFile(srcPath, dstPath); err != nil { return fmt.Errorf("create branch %s: %w", name, err) } - return schema.CreateIgnoredTablesSQLite(ctx, db) + db, cleanup, err := OpenSQL(ctx, s.dataDir, s.database, name) + if err != nil { + return err + } + defer func() { _ = cleanup() }() + var head string + _ = db.QueryRowContext(ctx, "SELECT head_hash FROM doltlite_refs WHERE branch = ?", s.branch).Scan(&head) + if _, err := db.ExecContext(ctx, ` + INSERT INTO doltlite_refs (branch, head_hash) VALUES (?, ?) + ON CONFLICT(branch) DO UPDATE SET head_hash = excluded.head_hash + `, name, head); err != nil { + return err + } + return nil }) } func (s *DoltliteStore) Checkout(ctx context.Context, branch string) error { - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - if _, err := db.ExecContext(ctx, "SELECT dolt_checkout(?)", branch); err != nil { - return fmt.Errorf("checkout branch %s: %w", branch, err) + _, err := os.Stat(func() string { + path, _ := s.branchDBPath(branch) + return path + }()) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("checkout branch %s: branch not found", branch) } - return schema.CreateIgnoredTablesSQLite(ctx, db) + return err + } + s.branch = branch + if err := s.resetPersistentDB(ctx); err != nil { + return err + } + return s.withConn(ctx, false, func(tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO doltlite_refs (branch, head_hash) + SELECT ?, COALESCE((SELECT head_hash FROM doltlite_refs WHERE branch = ?), '') + WHERE NOT EXISTS (SELECT 1 FROM doltlite_refs WHERE branch = ?) + `, branch, branch, branch); err != nil { + return err + } + return nil }) } func (s *DoltliteStore) CurrentBranch(ctx context.Context) (string, error) { - var branch string - err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - var err error - branch, err = versioncontrolops.CurrentBranch(ctx, db) - return err - }) - return branch, err + return s.branch, nil } func (s *DoltliteStore) DeleteBranch(ctx context.Context, branch string) error { - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - if _, err := db.ExecContext(ctx, "SELECT dolt_branch('-D', ?)", branch); err != nil { + if branch == s.branch { + return fmt.Errorf("delete branch %s: cannot delete current branch", branch) + } + return s.withExclusiveLock(ctx, func() error { + path, err := s.branchDBPath(branch) + if err != nil { + return err + } + if err := os.Remove(path); err != nil { + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("delete branch %s: branch not found", branch) + } return fmt.Errorf("delete branch %s: %w", branch, err) } return nil @@ -121,13 +172,24 @@ func (s *DoltliteStore) DeleteBranch(ctx context.Context, branch string) error { } func (s *DoltliteStore) ListBranches(ctx context.Context) ([]string, error) { - var branches []string - err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - var err error - branches, err = versioncontrolops.ListBranches(ctx, db) - return err - }) - return branches, err + entries, err := os.ReadDir(s.dataDir) + if err != nil { + return nil, err + } + branches := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + if branch, ok := branchFromDBFilename(s.database, entry.Name()); ok { + branches = append(branches, branch) + } + } + if !slices.Contains(branches, s.branch) { + branches = append(branches, s.branch) + } + slices.Sort(branches) + return branches, nil } // --------------------------------------------------------------------------- @@ -138,37 +200,47 @@ func (s *DoltliteStore) ListBranches(ctx context.Context) ([]string, error) { const commitAuthor = commitName + " <" + commitEmail + ">" func (s *DoltliteStore) CommitExists(ctx context.Context, commitHash string) (bool, error) { - var exists bool - err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - var err error - exists, err = versioncontrolops.CommitExists(ctx, db, commitHash) - return err + var count int + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + return tx.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM doltlite_commits + WHERE branch = ? AND (hash = ? OR hash LIKE ?) + `, s.branch, commitHash, commitHash+"%").Scan(&count) }) - return exists, err + return count > 0, err } func (s *DoltliteStore) Status(ctx context.Context) (*storage.Status, error) { - var status *storage.Status - err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - var err error - status, err = versioncontrolops.Status(ctx, db) - return err - }) - return status, err + return &storage.Status{ + Staged: []storage.StatusEntry{}, + Unstaged: []storage.StatusEntry{}, + }, nil } func (s *DoltliteStore) Log(ctx context.Context, limit int) ([]storage.CommitInfo, error) { var query string var args []any if limit > 0 { - query = "SELECT commit_hash, committer, email, date, message FROM dolt_log ORDER BY date DESC LIMIT ?" - args = []any{limit} + query = ` + SELECT hash, committer, email, date, message + FROM doltlite_commits + WHERE branch = ? + ORDER BY date DESC + LIMIT ? + ` + args = []any{s.branch, limit} } else { - query = "SELECT commit_hash, committer, email, date, message FROM dolt_log ORDER BY date DESC" + query = ` + SELECT hash, committer, email, date, message + FROM doltlite_commits + WHERE branch = ? + ORDER BY date DESC + ` + args = []any{s.branch} } var commits []storage.CommitInfo - err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - rows, err := db.QueryContext(ctx, query, args...) + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + rows, err := tx.QueryContext(ctx, query, args...) if err != nil { return fmt.Errorf("get log: %w", err) } @@ -203,45 +275,15 @@ func parseDoltliteTime(s string) time.Time { } func (s *DoltliteStore) Merge(ctx context.Context, branch string) ([]storage.Conflict, error) { - var conflicts []storage.Conflict - err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - _, err := db.ExecContext(ctx, "SELECT dolt_merge(?)", branch) - if err != nil { - c, conflictErr := versioncontrolops.GetConflicts(ctx, db) - if conflictErr == nil && len(c) > 0 { - conflicts = c - return nil - } - return fmt.Errorf("merge branch %s: %w", branch, err) - } - return nil - }) - return conflicts, err + return nil, fmt.Errorf("doltlite merge unsupported for sqlite backend") } func (s *DoltliteStore) GetConflicts(ctx context.Context) ([]storage.Conflict, error) { - var conflicts []storage.Conflict - err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - var err error - conflicts, err = versioncontrolops.GetConflicts(ctx, db) - return err - }) - return conflicts, err + return nil, nil } func (s *DoltliteStore) ResolveConflicts(ctx context.Context, table string, strategy string) error { - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - switch strategy { - case "ours": - _, err := db.ExecContext(ctx, "SELECT dolt_conflicts_resolve('--ours', ?)", table) - return err - case "theirs": - _, err := db.ExecContext(ctx, "SELECT dolt_conflicts_resolve('--theirs', ?)", table) - return err - default: - return fmt.Errorf("unknown conflict resolution strategy: %s", strategy) - } - }) + return fmt.Errorf("doltlite conflict resolution unsupported for sqlite backend") } // --------------------------------------------------------------------------- @@ -251,8 +293,8 @@ func (s *DoltliteStore) ResolveConflicts(ctx context.Context, table string, stra const defaultRemote = "origin" func (s *DoltliteStore) RemoveRemote(ctx context.Context, name string) error { - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - _, err := db.ExecContext(ctx, "SELECT dolt_remote('remove', ?)", name) + return s.withConn(ctx, true, func(tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, "DELETE FROM dolt_remotes WHERE name = ?", name) return err }) } From 764a62ab253b3d3c7059ff9c08c09ee366760d07 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Fri, 1 May 2026 15:57:58 +1000 Subject: [PATCH 07/15] fix(doltlite): use native versioning --- internal/storage/doltlite/commit_pending.go | 81 ++++ internal/storage/doltlite/federation.go | 40 +- internal/storage/doltlite/open.go | 24 +- internal/storage/doltlite/store.go | 279 ++++++-------- internal/storage/doltlite/time_travel.go | 164 ++++++++ internal/storage/doltlite/version_control.go | 384 +++++++++---------- 6 files changed, 565 insertions(+), 407 deletions(-) create mode 100644 internal/storage/doltlite/commit_pending.go create mode 100644 internal/storage/doltlite/time_travel.go diff --git a/internal/storage/doltlite/commit_pending.go b/internal/storage/doltlite/commit_pending.go new file mode 100644 index 000000000..78b5db1a6 --- /dev/null +++ b/internal/storage/doltlite/commit_pending.go @@ -0,0 +1,81 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "fmt" + "strings" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +func buildDoltliteBatchCommitMessage(ctx context.Context, db issueops.SQLQuerier, actor string) string { + if actor == "" { + actor = "bd" + } + + var added, modified, removed int + rows, err := db.QueryContext(ctx, ` + SELECT diff_type, COUNT(*) as cnt + FROM dolt_diff_issues('HEAD', 'WORKING') + GROUP BY diff_type + `) + if err == nil { + defer rows.Close() + for rows.Next() { + var diffType string + var count int + if scanErr := rows.Scan(&diffType, &count); scanErr == nil { + switch diffType { + case "added": + added = count + case "modified": + modified = count + case "removed": + removed = count + } + } + } + _ = rows.Err() + } + + var otherTables []string + statusRows, statusErr := db.QueryContext(ctx, ` + SELECT table_name FROM dolt_status s + WHERE table_name != 'issues' + AND NOT EXISTS ( + SELECT 1 FROM dolt_ignore di + WHERE di.ignored = 1 + AND s.table_name LIKE di.pattern + )`) + if statusErr == nil { + defer statusRows.Close() + for statusRows.Next() { + var table string + if scanErr := statusRows.Scan(&table); scanErr == nil { + otherTables = append(otherTables, table) + } + } + _ = statusRows.Err() + } + + msg := fmt.Sprintf("bd: batch commit by %s", actor) + var parts []string + if added > 0 { + parts = append(parts, fmt.Sprintf("%d created", added)) + } + if modified > 0 { + parts = append(parts, fmt.Sprintf("%d updated", modified)) + } + if removed > 0 { + parts = append(parts, fmt.Sprintf("%d deleted", removed)) + } + if len(parts) > 0 { + msg += " - " + strings.Join(parts, ", ") + } + if len(otherTables) > 0 { + msg += fmt.Sprintf(" (+ %s)", strings.Join(otherTables, ", ")) + } + return msg +} diff --git a/internal/storage/doltlite/federation.go b/internal/storage/doltlite/federation.go index 4a0ec7edc..bf92aeaca 100644 --- a/internal/storage/doltlite/federation.go +++ b/internal/storage/doltlite/federation.go @@ -114,14 +114,15 @@ func (s *DoltliteStore) AddFederationPeer(ctx context.Context, peer *storage.Fed } if err := s.withConn(ctx, true, func(tx *sql.Tx) error { - if err := issueops.AddFederationPeerInTx(ctx, tx, peer, encryptedPwd); err != nil { - return err - } - // Also add the Dolt remote. - return issueops.AddRemoteIfNotExists(ctx, tx, peer.Name, peer.RemoteURL) + return issueops.AddFederationPeerInTx(ctx, tx, peer, encryptedPwd) }); err != nil { return err } + if peer.RemoteURL != "" { + if err := s.AddRemote(ctx, peer.Name, peer.RemoteURL); err != nil { + return err + } + } return nil } @@ -268,23 +269,28 @@ func (s *DoltliteStore) SyncStatus(ctx context.Context, peer string) (*storage.S Peer: peer, } - // Get ahead/behind counts by comparing refs. - // Dolt's AS OF requires a literal ref, not a parameterized expression. + // Doltlite does not expose a historical dolt_log slice. Report exact zeroes + // only when refs match; otherwise keep counts unknown. remoteRef := peer + "/" + s.branch if err := issueops.ValidateRef(remoteRef); err != nil { status.LocalAhead = -1 status.LocalBehind = -1 } else if err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - query := fmt.Sprintf(` - SELECT - (SELECT COUNT(*) FROM dolt_log WHERE commit_hash NOT IN - (SELECT commit_hash FROM dolt_log AS OF '%s')) as ahead, - (SELECT COUNT(*) FROM dolt_log AS OF '%s' WHERE commit_hash NOT IN - (SELECT commit_hash FROM dolt_log)) as behind - `, remoteRef, remoteRef) - if err := db.QueryRowContext(ctx, query). - Scan(&status.LocalAhead, &status.LocalBehind); err != nil { - // Remote branch may not exist locally yet. + var localHash, remoteHash string + if err := db.QueryRowContext(ctx, "SELECT dolt_hashof('HEAD')").Scan(&localHash); err != nil { + status.LocalAhead = -1 + status.LocalBehind = -1 + return nil + } + if err := db.QueryRowContext(ctx, "SELECT dolt_hashof(?)", remoteRef).Scan(&remoteHash); err != nil { + status.LocalAhead = -1 + status.LocalBehind = -1 + return nil + } + if localHash == remoteHash { + status.LocalAhead = 0 + status.LocalBehind = 0 + } else { status.LocalAhead = -1 status.LocalBehind = -1 } diff --git a/internal/storage/doltlite/open.go b/internal/storage/doltlite/open.go index 248fded70..09b7c7f8a 100644 --- a/internal/storage/doltlite/open.go +++ b/internal/storage/doltlite/open.go @@ -6,7 +6,6 @@ import ( "context" "database/sql" "fmt" - "net/url" "os" "path/filepath" "regexp" @@ -29,7 +28,7 @@ const ( // OpenSQL opens an doltlite database at dir. The returned cleanup // function closes the *sql.DB. func OpenSQL(ctx context.Context, dir, database, branch string) (*sql.DB, func() error, error) { - dbPath, err := buildBranchDSN(dir, database, branch) + dbPath, err := buildDSN(dir, database) if err != nil { return nil, nil, err } @@ -54,14 +53,20 @@ func OpenSQL(ctx context.Context, dir, database, branch string) (*sql.DB, func() return nil, nil, err } + if branch = strings.TrimSpace(branch); branch != "" { + if _, err := db.ExecContext(ctx, "SELECT dolt_checkout(?)", branch); err != nil { + closeErr := cleanup() + if closeErr != nil { + return nil, nil, fmt.Errorf("%w; close: %v", err, closeErr) + } + return nil, nil, fmt.Errorf("doltlite: checkout branch %s: %w", branch, err) + } + } + return db, cleanup, nil } func buildDSN(dir, database string) (string, error) { - return buildBranchDSN(dir, database, "") -} - -func buildBranchDSN(dir, database, branch string) (string, error) { if strings.TrimSpace(database) != "" { if !validIdentifier.MatchString(database) { return "", fmt.Errorf("doltlite: invalid database name: %q", database) @@ -70,16 +75,9 @@ func buildBranchDSN(dir, database, branch string) (string, error) { database = "beads" } filename := database + ".db" - if strings.TrimSpace(branch) != "" { - filename = fmt.Sprintf("%s__%s.db", database, url.QueryEscape(branch)) - } path := filepath.Join(dir, filename) if os.PathSeparator == '\\' { path = strings.ReplaceAll(path, `\`, `/`) } return fmt.Sprintf("%s?_busy_timeout=%d", path, defaultBusyTimeout), nil } - -func sqlStringLiteral(s string) string { - return "'" + strings.ReplaceAll(strings.TrimSpace(s), "'", "''") + "'" -} diff --git a/internal/storage/doltlite/store.go b/internal/storage/doltlite/store.go index 029fa6731..647de7b7c 100644 --- a/internal/storage/doltlite/store.go +++ b/internal/storage/doltlite/store.go @@ -4,13 +4,9 @@ package doltlite import ( "context" - "crypto/sha256" "database/sql" - "encoding/hex" "errors" "fmt" - "io" - "net/url" "os" "path/filepath" "strings" @@ -123,6 +119,13 @@ func New(ctx context.Context, beadsDir, database, branch string, opts ...Option) if err := s.openPersistentDB(ctx); err != nil { return nil, fmt.Errorf("doltlite: open database: %w", err) } + if s.branch == "" { + branch, err := s.CurrentBranch(ctx) + if err != nil { + return nil, fmt.Errorf("doltlite: get current branch: %w", err) + } + s.branch = branch + } // Ensure dolt_ignore'd wisp tables exist in the working set. // After a clone or branch switch, these tables are absent because @@ -150,25 +153,6 @@ func (s *DoltliteStore) openPersistentDB(ctx context.Context) error { return nil } -func (s *DoltliteStore) closePersistentDB() error { - s.dbMu.Lock() - cleanup := s.dbCleanup - s.db = nil - s.dbCleanup = nil - s.dbMu.Unlock() - if cleanup != nil { - return cleanup() - } - return nil -} - -func (s *DoltliteStore) resetPersistentDB(ctx context.Context) error { - if err := s.closePersistentDB(); err != nil { - return err - } - return s.openPersistentDB(ctx) -} - func (s *DoltliteStore) activeDB(ctx context.Context) (*sql.DB, func() error, error) { s.dbMu.Lock() db := s.db @@ -355,120 +339,15 @@ func (s *DoltliteStore) initSchema(ctx context.Context) error { if err != nil { return err } - if err := s.ensureVersionControlTables(ctx, db); err != nil { - return err - } if applied > 0 { - if err := s.recordSyntheticCommit(ctx, db, "schema: apply migrations"); err != nil { - return fmt.Errorf("record migration commit: %w", err) + if err := commitAllNative(ctx, db, "schema: apply migrations"); err != nil { + return fmt.Errorf("commit migration: %w", err) } } return nil } -func (s *DoltliteStore) ensureVersionControlTables(ctx context.Context, db *sql.DB) error { - stmts := []string{ - `CREATE TABLE IF NOT EXISTS doltlite_commits ( - hash TEXT PRIMARY KEY, - branch TEXT NOT NULL, - committer TEXT NOT NULL, - email TEXT NOT NULL, - date TEXT NOT NULL, - message TEXT NOT NULL - )`, - `CREATE INDEX IF NOT EXISTS idx_doltlite_commits_branch_date - ON doltlite_commits(branch, date DESC)`, - `CREATE TABLE IF NOT EXISTS doltlite_refs ( - branch TEXT PRIMARY KEY, - head_hash TEXT NOT NULL - )`, - `CREATE TABLE IF NOT EXISTS dolt_remotes ( - name TEXT PRIMARY KEY, - url TEXT NOT NULL - )`, - } - for _, stmt := range stmts { - if _, err := db.ExecContext(ctx, stmt); err != nil { - return fmt.Errorf("ensure version-control metadata: %w", err) - } - } - return nil -} - -func (s *DoltliteStore) recordSyntheticCommit(ctx context.Context, db interface { - ExecContext(context.Context, string, ...any) (sql.Result, error) - QueryRowContext(context.Context, string, ...any) *sql.Row -}, message string) error { - if message == "" { - message = "doltlite: snapshot" - } - now := time.Now().UTC().Format(time.RFC3339Nano) - sum := sha256.Sum256([]byte(strings.Join([]string{s.branch, now, message}, "\x00"))) - hash := hex.EncodeToString(sum[:]) - - var current string - err := db.QueryRowContext(ctx, "SELECT head_hash FROM doltlite_refs WHERE branch = ?", s.branch).Scan(¤t) - if err != nil && !errors.Is(err, sql.ErrNoRows) { - return err - } - if current == hash { - return nil - } - - if _, err := db.ExecContext(ctx, ` - INSERT INTO doltlite_commits (hash, branch, committer, email, date, message) - VALUES (?, ?, ?, ?, ?, ?) - `, hash, s.branch, commitName, commitEmail, now, message); err != nil { - return err - } - _, err = db.ExecContext(ctx, ` - INSERT INTO doltlite_refs (branch, head_hash) VALUES (?, ?) - ON CONFLICT(branch) DO UPDATE SET head_hash = excluded.head_hash - `, s.branch, hash) - return err -} - -func (s *DoltliteStore) branchDBPath(branch string) (string, error) { - dsn, err := buildBranchDSN(s.dataDir, s.database, branch) - if err != nil { - return "", err - } - path := strings.SplitN(dsn, "?", 2)[0] - return path, nil -} - -func branchFromDBFilename(database, name string) (string, bool) { - prefix := database + "__" - suffix := ".db" - if !strings.HasPrefix(name, prefix) || !strings.HasSuffix(name, suffix) { - return "", false - } - encoded := strings.TrimSuffix(strings.TrimPrefix(name, prefix), suffix) - branch, err := url.QueryUnescape(encoded) - if err != nil || branch == "" { - return "", false - } - return branch, true -} - -func copyFile(src, dst string) error { - in, err := os.Open(src) - if err != nil { - return err - } - defer in.Close() - out, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) - if err != nil { - return err - } - defer func() { _ = out.Close() }() - if _, err := io.Copy(out, in); err != nil { - return err - } - return out.Sync() -} - // ensureIgnoredTables creates dolt_ignore'd wisp tables if they don't exist. // Uses withConn (not withRootConn) because the database is already created. func (s *DoltliteStore) ensureIgnoredTables(ctx context.Context) error { @@ -645,42 +524,110 @@ func (s *DoltliteStore) Close() error { // DoltGC runs Dolt garbage collection to reclaim disk space. func (s *DoltliteStore) DoltGC(ctx context.Context) error { return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - _, err := db.ExecContext(ctx, "SELECT dolt_gc()") - return err + if _, err := db.ExecContext(ctx, "SELECT dolt_gc()"); err != nil { + return fmt.Errorf("doltlite gc: %w", err) + } + return nil }) } -// Flatten squashes all Dolt commit history into a single commit. -// Pins a single *sql.Conn for session-scoped stored procedures. +// Flatten squashes all doltlite commit history into a single commit. func (s *DoltliteStore) Flatten(ctx context.Context) error { - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - if pooled, ok := db.(*sql.DB); ok { - conn, err := pooled.Conn(ctx) - if err != nil { - return err + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + var initialHash string + if err := db.QueryRowContext(ctx, + "SELECT commit_hash FROM dolt_log ORDER BY date ASC LIMIT 1", + ).Scan(&initialHash); err != nil { + return fmt.Errorf("find initial commit: %w", err) + } + + var commitCount int + if err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM dolt_log", + ).Scan(&commitCount); err != nil { + return fmt.Errorf("count commits: %w", err) + } + if commitCount <= 1 { + return nil + } + + steps := []struct { + name string + query string + args []any + }{ + {"create temp branch", "SELECT dolt_branch('flatten-tmp')", nil}, + {"checkout temp branch", "SELECT dolt_checkout('flatten-tmp')", nil}, + {"soft reset to initial", "SELECT dolt_reset('--soft', ?)", []any{initialHash}}, + {"commit flattened snapshot", "SELECT dolt_commit('-A', '-m', 'flatten: squash all history into single commit')", nil}, + {"checkout main", "SELECT dolt_checkout('main')", nil}, + {"reset main to flattened", "SELECT dolt_reset('--hard', 'flatten-tmp')", nil}, + {"delete temp branch", "SELECT dolt_branch('-D', 'flatten-tmp')", nil}, + } + for _, step := range steps { + if _, err := db.ExecContext(ctx, step.query, step.args...); err != nil { + return fmt.Errorf("flatten step %q: %w", step.name, err) } - defer conn.Close() - return versioncontrolops.Flatten(ctx, conn) } - return versioncontrolops.Flatten(ctx, db) + return nil }) } -// Compact squashes old Dolt commits while preserving recent ones. -// Pins a single *sql.Conn for session-scoped stored procedures. +// Compact squashes old doltlite commits while preserving recent ones. func (s *DoltliteStore) Compact(ctx context.Context, initialHash, boundaryHash string, oldCommits int, recentHashes []string) error { - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - // withDBConn returns *sql.DB; pin a single connection for - // session-scoped operations (checkout, reset, cherry-pick). - if pooled, ok := db.(*sql.DB); ok { - conn, err := pooled.Conn(ctx) - if err != nil { + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) (retErr error) { + branchCreated := false + defer func() { + if retErr != nil && branchCreated { + _, _ = db.ExecContext(ctx, "SELECT dolt_checkout('main')") + _, _ = db.ExecContext(ctx, "SELECT dolt_branch('-D', 'compact-tmp')") + } + }() + + execSQL := func(name, query string, args ...any) error { + if _, err := db.ExecContext(ctx, query, args...); err != nil { + return fmt.Errorf("compact step %q: %w", name, err) + } + return nil + } + + if err := execSQL("create temp branch", "SELECT dolt_branch('compact-tmp', ?)", boundaryHash); err != nil { + return err + } + branchCreated = true + + if err := execSQL("checkout temp", "SELECT dolt_checkout('compact-tmp')"); err != nil { + return err + } + if err := execSQL("soft reset to initial", "SELECT dolt_reset('--soft', ?)", initialHash); err != nil { + return err + } + msg := fmt.Sprintf("compact: squash %d commits into base snapshot", oldCommits) + if err := execSQL("commit squashed base", "SELECT dolt_commit('-A', '-m', ?)", msg); err != nil { + return err + } + + for _, hash := range recentHashes { + label := hash + if len(label) > 8 { + label = label[:8] + } + if err := execSQL("cherry-pick "+label, "SELECT dolt_cherry_pick(?)", hash); err != nil { return err } - defer conn.Close() - return versioncontrolops.Compact(ctx, conn, initialHash, boundaryHash, oldCommits, recentHashes) } - return versioncontrolops.Compact(ctx, db, initialHash, boundaryHash, oldCommits, recentHashes) + + if err := execSQL("checkout main", "SELECT dolt_checkout('main')"); err != nil { + return err + } + if err := execSQL("reset main to compacted", "SELECT dolt_reset('--hard', 'compact-tmp')"); err != nil { + return err + } + if err := execSQL("delete temp branch", "SELECT dolt_branch('-D', 'compact-tmp')"); err != nil { + return err + } + + return nil }) } @@ -695,7 +642,11 @@ func (s *DoltliteStore) CLIDir() string { if s.dataDir == "" { return "" } - return filepath.Join(s.dataDir, s.database) + dsn, err := buildDSN(s.dataDir, s.database) + if err != nil { + return "" + } + return strings.SplitN(dsn, "?", 2)[0] } // --------------------------------------------------------------------------- @@ -703,7 +654,7 @@ func (s *DoltliteStore) CLIDir() string { // --------------------------------------------------------------------------- // Branch, Checkout, CurrentBranch, DeleteBranch, ListBranches are -// implemented in version_control.go via versioncontrolops. +// implemented in version_control.go. func (s *DoltliteStore) CommitPending(ctx context.Context, actor string) (bool, error) { var hasPending bool @@ -715,7 +666,7 @@ func (s *DoltliteStore) CommitPending(ctx context.Context, actor string) (bool, return err } if hasPending { - msg = issueops.BuildBatchCommitMessage(ctx, tx, actor) + msg = buildDoltliteBatchCommitMessage(ctx, tx, actor) } return nil }) @@ -735,12 +686,12 @@ func (s *DoltliteStore) CommitPending(ctx context.Context, actor string) (bool, return true, nil } -// CommitExists is implemented in version_control.go via versioncontrolops. +// CommitExists is implemented in version_control.go. func (s *DoltliteStore) GetCurrentCommit(ctx context.Context) (string, error) { var hash string - err := s.withConn(ctx, false, func(tx *sql.Tx) error { - return tx.QueryRowContext(ctx, "SELECT head_hash FROM doltlite_refs WHERE branch = ?", s.branch).Scan(&hash) + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + return db.QueryRowContext(ctx, "SELECT dolt_hashof('HEAD')").Scan(&hash) }) if errors.Is(err, sql.ErrNoRows) { return "", nil @@ -749,7 +700,7 @@ func (s *DoltliteStore) GetCurrentCommit(ctx context.Context) (string, error) { } // Status, Log, Merge, GetConflicts, ResolveConflicts are implemented in -// version_control.go via versioncontrolops. +// version_control.go. // --------------------------------------------------------------------------- // storage.HistoryViewer @@ -759,7 +710,7 @@ func (s *DoltliteStore) History(ctx context.Context, issueID string) ([]*storage var result []*storage.HistoryEntry err := s.withConn(ctx, false, func(tx *sql.Tx) error { var err error - result, err = issueops.HistoryInTx(ctx, tx, issueID) + result, err = doltliteHistoryInTx(ctx, tx, issueID) return err }) return result, err @@ -769,7 +720,7 @@ func (s *DoltliteStore) AsOf(ctx context.Context, issueID string, ref string) (* var result *types.Issue err := s.withConn(ctx, false, func(tx *sql.Tx) error { var err error - result, err = issueops.AsOfInTx(ctx, tx, issueID, ref) + result, err = doltliteAsOfInTx(ctx, tx, issueID, ref) return err }) return result, err @@ -779,7 +730,7 @@ func (s *DoltliteStore) Diff(ctx context.Context, fromRef, toRef string) ([]*sto var result []*storage.DiffEntry err := s.withConn(ctx, false, func(tx *sql.Tx) error { var err error - result, err = issueops.DiffInTx(ctx, tx, fromRef, toRef) + result, err = doltliteDiffInTx(ctx, tx, fromRef, toRef) return err }) return result, err @@ -790,7 +741,7 @@ func (s *DoltliteStore) Diff(ctx context.Context, fromRef, toRef string) ([]*sto // --------------------------------------------------------------------------- // RemoveRemote, ListRemotes, Push, Pull, ForcePush, Fetch, PushTo, PullFrom -// are implemented in version_control.go via versioncontrolops. +// are implemented in version_control.go. // --------------------------------------------------------------------------- // storage.SyncStore diff --git a/internal/storage/doltlite/time_travel.go b/internal/storage/doltlite/time_travel.go new file mode 100644 index 000000000..1b7e78558 --- /dev/null +++ b/internal/storage/doltlite/time_travel.go @@ -0,0 +1,164 @@ +//go:build cgo + +package doltlite + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +func doltliteAsOfInTx(ctx context.Context, tx *sql.Tx, issueID string, ref string) (*types.Issue, error) { + if err := issueops.ValidateRef(ref); err != nil { + return nil, fmt.Errorf("invalid ref: %w", err) + } + + query := fmt.Sprintf(` + SELECT %s + FROM dolt_at_issues(?) + WHERE id = ? + `, issueops.IssueSelectColumns) + issue, err := issueops.ScanIssueFrom(tx.QueryRowContext(ctx, query, ref, issueID)) + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: issue %s as of %s", storage.ErrNotFound, issueID, ref) + } + if err != nil { + return nil, fmt.Errorf("get issue as of %s: %w", ref, err) + } + return issue, nil +} + +func doltliteDiffInTx(ctx context.Context, tx *sql.Tx, fromRef, toRef string) ([]*storage.DiffEntry, error) { + if err := issueops.ValidateRef(fromRef); err != nil { + return nil, fmt.Errorf("invalid fromRef: %w", err) + } + if err := issueops.ValidateRef(toRef); err != nil { + return nil, fmt.Errorf("invalid toRef: %w", err) + } + + rows, err := tx.QueryContext(ctx, ` + SELECT + COALESCE(from_id, '') as from_id, + COALESCE(to_id, '') as to_id, + diff_type, + from_title, to_title, + from_description, to_description, + from_status, to_status, + from_priority, to_priority + FROM dolt_diff_issues(?, ?) + `, fromRef, toRef) + if err != nil { + return nil, fmt.Errorf("failed to get diff: %w", err) + } + defer rows.Close() + + var entries []*storage.DiffEntry + for rows.Next() { + var fromID, toID, diffType string + var fromTitle, toTitle, fromDesc, toDesc, fromStatus, toStatus *string + var fromPriority, toPriority *int + + if err := rows.Scan(&fromID, &toID, &diffType, + &fromTitle, &toTitle, + &fromDesc, &toDesc, + &fromStatus, &toStatus, + &fromPriority, &toPriority); err != nil { + return nil, fmt.Errorf("failed to scan diff: %w", err) + } + + entry := &storage.DiffEntry{DiffType: diffType} + if toID != "" { + entry.IssueID = toID + } else { + entry.IssueID = fromID + } + if diffType != "added" && fromID != "" { + entry.OldValue = &types.Issue{ID: fromID} + if fromTitle != nil { + entry.OldValue.Title = *fromTitle + } + if fromDesc != nil { + entry.OldValue.Description = *fromDesc + } + if fromStatus != nil { + entry.OldValue.Status = types.Status(*fromStatus) + } + if fromPriority != nil { + entry.OldValue.Priority = *fromPriority + } + } + if diffType != "removed" && toID != "" { + entry.NewValue = &types.Issue{ID: toID} + if toTitle != nil { + entry.NewValue.Title = *toTitle + } + if toDesc != nil { + entry.NewValue.Description = *toDesc + } + if toStatus != nil { + entry.NewValue.Status = types.Status(*toStatus) + } + if toPriority != nil { + entry.NewValue.Priority = *toPriority + } + } + + entries = append(entries, entry) + } + + return entries, rows.Err() +} + +func doltliteHistoryInTx(ctx context.Context, tx *sql.Tx, issueID string) ([]*storage.HistoryEntry, error) { + rows, err := tx.QueryContext(ctx, ` + SELECT commit_hash, committer, commit_date + FROM dolt_history_issues + WHERE id = ? + ORDER BY commit_date DESC + `, issueID) + if err != nil { + return nil, fmt.Errorf("failed to get issue history: %w", err) + } + + type historyMeta struct { + hash string + committer string + date any + } + var metas []historyMeta + for rows.Next() { + var meta historyMeta + if err := rows.Scan(&meta.hash, &meta.committer, &meta.date); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("failed to scan history: %w", err) + } + metas = append(metas, meta) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return nil, err + } + if err := rows.Close(); err != nil { + return nil, err + } + + entries := make([]*storage.HistoryEntry, 0, len(metas)) + for _, meta := range metas { + issue, err := doltliteAsOfInTx(ctx, tx, issueID, meta.hash) + if err != nil { + return nil, err + } + entries = append(entries, &storage.HistoryEntry{ + CommitHash: meta.hash, + Committer: meta.committer, + CommitDate: parseDoltliteTimeValue(meta.date), + Issue: issue, + }) + } + return entries, nil +} diff --git a/internal/storage/doltlite/version_control.go b/internal/storage/doltlite/version_control.go index ba6de4463..a9b0ec816 100644 --- a/internal/storage/doltlite/version_control.go +++ b/internal/storage/doltlite/version_control.go @@ -7,19 +7,16 @@ import ( "database/sql" "errors" "fmt" - "os" - "slices" "time" "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/storage/versioncontrolops" ) -// withDBConn opens a short-lived database connection configured for the -// store's database and branch and passes it to fn. Unlike withConn, no -// transaction is started — this is required for Dolt stored procedures -// (CALL DOLT_BRANCH, CALL DOLT_MERGE, etc.) that cannot run inside -// explicit SQL transactions. +// withDBConn opens a database connection configured for the store's native +// doltlite branch and passes it to fn without starting an explicit SQL +// transaction. Version-control functions manage their own transaction boundary. func (s *DoltliteStore) withDBConn(ctx context.Context, fn func(db versioncontrolops.DBConn) error) (err error) { if s.closed.Load() { return errClosed @@ -33,41 +30,72 @@ func (s *DoltliteStore) withDBConn(ctx context.Context, fn func(db versioncontro } defer func() { err = errors.Join(err, cleanup()) - // Best-effort cleanup of orphaned tmp_pack_* files left by git - // fetch in the Dolt git-remote-cache. Rate-limited internally. s.cleanGitRemoteCacheGarbage() }() return fn(db) } +func (s *DoltliteStore) withDBWrite(ctx context.Context, fn func(db versioncontrolops.DBConn) error) error { + return s.withExclusiveLock(ctx, func() error { + return s.withRetry(ctx, func() error { + return s.withDBConn(ctx, fn) + }) + }) +} + +// commitAuthor returns the author string for native doltlite commits. +const commitAuthor = commitName + " <" + commitEmail + ">" + +func commitAllNative(ctx context.Context, db versioncontrolops.DBConn, message string) error { + if message == "" { + message = "doltlite: snapshot" + } + _, err := db.ExecContext(ctx, "SELECT dolt_commit('-A', '-m', ?, '--author', ?)", message, commitAuthor) + if err != nil && !issueops.IsNothingToCommitError(err) { + return fmt.Errorf("doltlite commit: %w", err) + } + return nil +} + func (s *DoltliteStore) Commit(ctx context.Context, message string) error { - return s.withConn(ctx, true, func(tx *sql.Tx) error { - return s.recordSyntheticCommit(ctx, tx, message) + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + return commitAllNative(ctx, db, message) }) } // CommitWithConfig commits all working set changes including config. -// DoltliteStore.Commit already includes config via DOLT_ADD('-A'), -// so this is just an alias to satisfy the VersionControl interface (GH#3216). func (s *DoltliteStore) CommitWithConfig(ctx context.Context, message string) error { return s.Commit(ctx, message) } func (s *DoltliteStore) AddRemote(ctx context.Context, name, url string) error { - return s.withConn(ctx, true, func(tx *sql.Tx) error { - _, err := tx.ExecContext(ctx, ` - INSERT INTO dolt_remotes (name, url) VALUES (?, ?) - ON CONFLICT(name) DO UPDATE SET url = excluded.url - `, name, url) - return err + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + var existing string + err := db.QueryRowContext(ctx, "SELECT url FROM dolt_remotes WHERE name = ?", name).Scan(&existing) + switch { + case err == nil && existing == url: + return nil + case err == nil: + if _, rmErr := db.ExecContext(ctx, "SELECT dolt_remote('remove', ?)", name); rmErr != nil { + return fmt.Errorf("remove existing remote %s: %w", name, rmErr) + } + case errors.Is(err, sql.ErrNoRows): + default: + return fmt.Errorf("lookup remote %s: %w", name, err) + } + + if _, err := db.ExecContext(ctx, "SELECT dolt_remote('add', ?, ?)", name, url); err != nil { + return fmt.Errorf("add remote %s: %w", name, err) + } + return nil }) } func (s *DoltliteStore) HasRemote(ctx context.Context, name string) (bool, error) { var count int - err := s.withConn(ctx, false, func(tx *sql.Tx) error { - return tx.QueryRowContext(ctx, "SELECT count(*) FROM dolt_remotes WHERE name = ?", name).Scan(&count) + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + return db.QueryRowContext(ctx, "SELECT count(*) FROM dolt_remotes WHERE name = ?", name).Scan(&count) }) if err != nil { return false, err @@ -80,91 +108,53 @@ func (s *DoltliteStore) HasRemote(ctx context.Context, name string) (bool, error // --------------------------------------------------------------------------- func (s *DoltliteStore) Branch(ctx context.Context, name string) error { - return s.withExclusiveLock(ctx, func() error { - srcPath, err := s.branchDBPath(s.branch) - if err != nil { - return err - } - dstPath, err := s.branchDBPath(name) - if err != nil { - return err - } - if srcPath == dstPath { - return fmt.Errorf("create branch %s: branch already exists", name) - } - if err := s.closePersistentDB(); err != nil { - return err - } - defer func() { _ = s.openPersistentDB(ctx) }() - if _, err := os.Stat(dstPath); err == nil { - return fmt.Errorf("create branch %s: branch already exists", name) - } else if !errors.Is(err, os.ErrNotExist) { - return err - } - if err := copyFile(srcPath, dstPath); err != nil { + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_branch(?)", name); err != nil { return fmt.Errorf("create branch %s: %w", name, err) } - db, cleanup, err := OpenSQL(ctx, s.dataDir, s.database, name) - if err != nil { - return err - } - defer func() { _ = cleanup() }() - var head string - _ = db.QueryRowContext(ctx, "SELECT head_hash FROM doltlite_refs WHERE branch = ?", s.branch).Scan(&head) - if _, err := db.ExecContext(ctx, ` - INSERT INTO doltlite_refs (branch, head_hash) VALUES (?, ?) - ON CONFLICT(branch) DO UPDATE SET head_hash = excluded.head_hash - `, name, head); err != nil { - return err - } return nil }) } func (s *DoltliteStore) Checkout(ctx context.Context, branch string) error { - _, err := os.Stat(func() string { - path, _ := s.branchDBPath(branch) - return path - }()) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("checkout branch %s: branch not found", branch) + if err := s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_checkout(?)", branch); err != nil { + return fmt.Errorf("checkout branch %s: %w", branch, err) } + return nil + }); err != nil { return err } s.branch = branch - if err := s.resetPersistentDB(ctx); err != nil { - return err - } - return s.withConn(ctx, false, func(tx *sql.Tx) error { - if _, err := tx.ExecContext(ctx, ` - INSERT INTO doltlite_refs (branch, head_hash) - SELECT ?, COALESCE((SELECT head_hash FROM doltlite_refs WHERE branch = ?), '') - WHERE NOT EXISTS (SELECT 1 FROM doltlite_refs WHERE branch = ?) - `, branch, branch, branch); err != nil { - return err - } - return nil - }) + return nil } func (s *DoltliteStore) CurrentBranch(ctx context.Context) (string, error) { - return s.branch, nil + var branch string + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + branch, err = versioncontrolops.CurrentBranch(ctx, db) + return err + }) + if err != nil { + return "", err + } + if branch != "" { + s.branch = branch + } + return branch, nil } func (s *DoltliteStore) DeleteBranch(ctx context.Context, branch string) error { - if branch == s.branch { + current, err := s.CurrentBranch(ctx) + if err != nil { + return err + } + if branch == current { return fmt.Errorf("delete branch %s: cannot delete current branch", branch) } - return s.withExclusiveLock(ctx, func() error { - path, err := s.branchDBPath(branch) - if err != nil { - return err - } - if err := os.Remove(path); err != nil { - if errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("delete branch %s: branch not found", branch) - } + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_branch('-D', ?)", branch); err != nil { return fmt.Errorf("delete branch %s: %w", branch, err) } return nil @@ -172,86 +162,62 @@ func (s *DoltliteStore) DeleteBranch(ctx context.Context, branch string) error { } func (s *DoltliteStore) ListBranches(ctx context.Context) ([]string, error) { - entries, err := os.ReadDir(s.dataDir) - if err != nil { - return nil, err - } - branches := make([]string, 0, len(entries)) - for _, entry := range entries { - if entry.IsDir() { - continue - } - if branch, ok := branchFromDBFilename(s.database, entry.Name()); ok { - branches = append(branches, branch) - } - } - if !slices.Contains(branches, s.branch) { - branches = append(branches, s.branch) - } - slices.Sort(branches) - return branches, nil + var branches []string + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + branches, err = versioncontrolops.ListBranches(ctx, db) + return err + }) + return branches, err } // --------------------------------------------------------------------------- // Version control operations // --------------------------------------------------------------------------- -// commitAuthor returns the author string for merge commits. -const commitAuthor = commitName + " <" + commitEmail + ">" - func (s *DoltliteStore) CommitExists(ctx context.Context, commitHash string) (bool, error) { - var count int - err := s.withConn(ctx, false, func(tx *sql.Tx) error { - return tx.QueryRowContext(ctx, ` - SELECT COUNT(*) FROM doltlite_commits - WHERE branch = ? AND (hash = ? OR hash LIKE ?) - `, s.branch, commitHash, commitHash+"%").Scan(&count) + var exists bool + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + exists, err = versioncontrolops.CommitExists(ctx, db, commitHash) + return err }) - return count > 0, err + return exists, err } func (s *DoltliteStore) Status(ctx context.Context) (*storage.Status, error) { - return &storage.Status{ - Staged: []storage.StatusEntry{}, - Unstaged: []storage.StatusEntry{}, - }, nil + var status *storage.Status + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + status, err = versioncontrolops.Status(ctx, db) + return err + }) + return status, err } func (s *DoltliteStore) Log(ctx context.Context, limit int) ([]storage.CommitInfo, error) { - var query string + query := "SELECT commit_hash, committer, email, date, message FROM dolt_log ORDER BY date DESC" var args []any if limit > 0 { - query = ` - SELECT hash, committer, email, date, message - FROM doltlite_commits - WHERE branch = ? - ORDER BY date DESC - LIMIT ? - ` - args = []any{s.branch, limit} - } else { - query = ` - SELECT hash, committer, email, date, message - FROM doltlite_commits - WHERE branch = ? - ORDER BY date DESC - ` - args = []any{s.branch} + query += " LIMIT ?" + args = append(args, limit) } + var commits []storage.CommitInfo - err := s.withConn(ctx, false, func(tx *sql.Tx) error { - rows, err := tx.QueryContext(ctx, query, args...) + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + rows, err := db.QueryContext(ctx, query, args...) if err != nil { return fmt.Errorf("get log: %w", err) } defer rows.Close() + for rows.Next() { var c storage.CommitInfo - var date string + var date any if err := rows.Scan(&c.Hash, &c.Author, &c.Email, &date, &c.Message); err != nil { return fmt.Errorf("scan commit: %w", err) } - c.Date = parseDoltliteTime(date) + c.Date = parseDoltliteTimeValue(date) commits = append(commits, c) } return rows.Err() @@ -259,6 +225,21 @@ func (s *DoltliteStore) Log(ctx context.Context, limit int) ([]storage.CommitInf return commits, err } +func parseDoltliteTimeValue(v any) time.Time { + switch t := v.(type) { + case time.Time: + return t + case string: + return parseDoltliteTime(t) + case []byte: + return parseDoltliteTime(string(t)) + case int64: + return time.Unix(t, 0).UTC() + default: + return time.Time{} + } +} + func parseDoltliteTime(s string) time.Time { for _, layout := range []string{ time.RFC3339Nano, @@ -275,15 +256,50 @@ func parseDoltliteTime(s string) time.Time { } func (s *DoltliteStore) Merge(ctx context.Context, branch string) ([]storage.Conflict, error) { - return nil, fmt.Errorf("doltlite merge unsupported for sqlite backend") + var conflicts []storage.Conflict + err := s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + if _, mergeErr := db.ExecContext(ctx, "SELECT dolt_merge(?)", branch); mergeErr != nil { + c, conflictErr := versioncontrolops.GetConflicts(ctx, db) + if conflictErr == nil && len(c) > 0 { + conflicts = c + return nil + } + return fmt.Errorf("merge branch %s: %w", branch, mergeErr) + } + return nil + }) + return conflicts, err } func (s *DoltliteStore) GetConflicts(ctx context.Context) ([]storage.Conflict, error) { - return nil, nil + var conflicts []storage.Conflict + err := s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { + var err error + conflicts, err = versioncontrolops.GetConflicts(ctx, db) + return err + }) + return conflicts, err } func (s *DoltliteStore) ResolveConflicts(ctx context.Context, table string, strategy string) error { - return fmt.Errorf("doltlite conflict resolution unsupported for sqlite backend") + if table == "" || !validIdentifier.MatchString(table) { + return fmt.Errorf("invalid table name: %s", table) + } + var flag string + switch strategy { + case "ours": + flag = "--ours" + case "theirs": + flag = "--theirs" + default: + return fmt.Errorf("unknown conflict resolution strategy: %s", strategy) + } + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_conflicts_resolve(?, ?)", flag, table); err != nil { + return fmt.Errorf("resolve conflicts: %w", err) + } + return nil + }) } // --------------------------------------------------------------------------- @@ -293,9 +309,11 @@ func (s *DoltliteStore) ResolveConflicts(ctx context.Context, table string, stra const defaultRemote = "origin" func (s *DoltliteStore) RemoveRemote(ctx context.Context, name string) error { - return s.withConn(ctx, true, func(tx *sql.Tx) error { - _, err := tx.ExecContext(ctx, "DELETE FROM dolt_remotes WHERE name = ?", name) - return err + return s.withDBWrite(ctx, func(db versioncontrolops.DBConn) error { + if _, err := db.ExecContext(ctx, "SELECT dolt_remote('remove', ?)", name); err != nil { + return fmt.Errorf("remove remote %s: %w", name, err) + } + return nil }) } @@ -363,8 +381,6 @@ func (s *DoltliteStore) PushTo(ctx context.Context, peer string) error { } func (s *DoltliteStore) PullFrom(ctx context.Context, peer string) ([]storage.Conflict, error) { - // Auto-commit pending changes before pull to prevent - // "cannot merge with uncommitted changes" errors. if _, err := s.CommitPending(ctx, "beads"); err != nil { return nil, fmt.Errorf("commit pending before pull: %w", err) } @@ -388,82 +404,24 @@ func (s *DoltliteStore) PullFrom(ctx context.Context, peer string) ([]storage.Co // Backup operations // --------------------------------------------------------------------------- +var errDoltliteBackupUnsupported = errors.New("doltlite backup operations unsupported") + func (s *DoltliteStore) BackupAdd(ctx context.Context, name, url string) error { - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - return versioncontrolops.BackupAdd(ctx, db, name, url) - }) + return errDoltliteBackupUnsupported } func (s *DoltliteStore) BackupSync(ctx context.Context, name string) error { - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - return versioncontrolops.BackupSync(ctx, db, name) - }) + return errDoltliteBackupUnsupported } func (s *DoltliteStore) BackupRemove(ctx context.Context, name string) error { - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - return versioncontrolops.BackupRemove(ctx, db, name) - }) + return errDoltliteBackupUnsupported } -// BackupDatabase registers dir as a file:// Dolt backup remote and syncs -// the database to it. The dir must exist locally. This preserves full Dolt -// commit history. func (s *DoltliteStore) BackupDatabase(ctx context.Context, dir string) error { - info, err := os.Stat(dir) - if err != nil { - return fmt.Errorf("backup destination does not exist: %w", err) - } - if !info.IsDir() { - return fmt.Errorf("backup destination is not a directory: %s", dir) - } - - backupURL, err := versioncontrolops.DirToFileURL(dir) - if err != nil { - return err - } - backupName := "backup_export" - - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - // Register as a backup remote (idempotent — remove first if exists). - _ = versioncontrolops.BackupRemove(ctx, db, backupName) - if err := versioncontrolops.BackupAdd(ctx, db, backupName, backupURL); err != nil { - // Another backup (e.g. "default" registered by `bd backup init`) may - // already point to this URL. In that case, sync using the existing - // remote name rather than failing. - if conflict := versioncontrolops.ExtractAddressConflictName(err); conflict != "" { - if syncErr := versioncontrolops.BackupSync(ctx, db, conflict); syncErr != nil { - return fmt.Errorf("sync to backup: %w", syncErr) - } - return nil - } - return fmt.Errorf("register backup remote: %w", err) - } - if err := versioncontrolops.BackupSync(ctx, db, backupName); err != nil { - return fmt.Errorf("sync to backup: %w", err) - } - return nil - }) + return errDoltliteBackupUnsupported } -// RestoreDatabase restores the database from a Dolt backup at dir. -// The dir must exist locally and contain a valid Dolt backup. -// When force is true, an existing database is overwritten. func (s *DoltliteStore) RestoreDatabase(ctx context.Context, dir string, force bool) error { - info, err := os.Stat(dir) - if err != nil { - return fmt.Errorf("backup source does not exist: %w", err) - } - if !info.IsDir() { - return fmt.Errorf("backup source is not a directory: %s", dir) - } - - backupURL, err := versioncontrolops.DirToFileURL(dir) - if err != nil { - return err - } - - return s.withDBConn(ctx, func(db versioncontrolops.DBConn) error { - return versioncontrolops.BackupRestore(ctx, db, backupURL, s.database, force) - }) + return errDoltliteBackupUnsupported } From 0fad78cf3af181c69f1436b075ee85893758487c Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Fri, 1 May 2026 15:58:14 +1000 Subject: [PATCH 08/15] docs(doltlite): record migration rules --- docs/dev-notes/doltlite-beads-checklist.md | 84 ++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 docs/dev-notes/doltlite-beads-checklist.md diff --git a/docs/dev-notes/doltlite-beads-checklist.md b/docs/dev-notes/doltlite-beads-checklist.md new file mode 100644 index 000000000..f20bf2260 --- /dev/null +++ b/docs/dev-notes/doltlite-beads-checklist.md @@ -0,0 +1,84 @@ +# Doltlite Beads Checklist + +Purpose: track runtime findings while the Beads backend migration to `doltlite` +is in progress. + +Rule: agents working on `bd`, `gc`, mail, routing, backup, or config behavior +must update this checklist with concrete findings before ending their session. + +## Current Policy + +- Backups must stay off during the doltlite migration. +- Do not enable auto-backup in project, city, or user config. +- Treat any `CALL DOLT_BACKUP(...)`, `gc dolt ...`, or similar server-era + instructions as suspect until explicitly revalidated for doltlite. + +## Operational Fragment Locations + +Keep both operational-awareness doltlite templates updated when changing agent +runtime guidance: + +- Source template embedded into future `gc` pack artifacts: + `/data/projects/t3code/packages/gascity-config/config/packs/gastown/template-fragments/operational-awareness-doltlite.template.md` +- Current city runtime template used by this Gas Town instance: + `/home/ubuntu/.local/state/t3code/gascity/current/city/packs/gastown/template-fragments/operational-awareness-doltlite.template.md` + +The source template is the long-term source of truth. The runtime template is +what current agents see. Update both when changing doltlite operating rules. + +The packaged source city config also selects this fragment: +`/data/projects/t3code/packages/gascity-config/config/city.toml` + +## How To Check Backup Is Off + +Run these commands and record any drift: + +```bash +bd config get backup.enabled +bd config get backup.git-push +sed -n '1,80p' /data/projects/beads-doltlite/.beads/config.yaml +sed -n '1,80p' /home/ubuntu/.local/state/t3code/gascity/current/city/.beads/config.yaml +sed -n '1,80p' /home/ubuntu/.config/bd/config.yaml +``` + +Expected state: + +- `backup.enabled: false` in project config +- `backup.enabled: false` in city config +- `backup.enabled: false` in user config +- `backup.git-push: false` anywhere it is set +- no agent should rely on backup side effects as part of validation + +## Findings + +- 2026-05-01: deacon startup after `gc prime` carried stale Dolt env overrides + (`BEADS_DOLT_PORT=35819`, `BEADS_DOLT_SERVER_PORT=35819`, `GC_DOLT_PORT=35819`) + even though `gc doctor` and `gc dolt health` showed the live doltlite server + on `127.0.0.1:41465`; `bd list` failed until those vars were overridden to + the live port, so agent startup can be blocked by stale runtime port state. +- 2026-05-01: `bd ready --json` worked in refinery workspace, which confirms + core `bd` reads are functional under current doltlite state. +- 2026-05-01: `bd config get mail.delegate` returned `mail.delegate (not set)` + but also emitted `Warning: auto-backup failed: register backup remote: add + backup backup_export: near "CALL": syntax error`, which shows a stale Dolt + backup codepath still runs in doltlite mode. +- 2026-05-01: direct doltlite storage probe succeeded for message creation, + `replies-to` dependency insertion, message search, and ack-like close/update. +- 2026-05-01: `cmd/bd/backup_auto.go` now skips post-run auto-backup entirely + when the opened store is `*doltlite.DoltliteStore`, preventing stale + `CALL DOLT_BACKUP(...)` warnings from read-only commands in doltlite mode. +- 2026-05-01: `cmd/bd/backup_export.go` now rejects backup export early for + doltlite with a migration-specific error instead of falling through to + Dolt-only backup SQL. +- 2026-05-01: doltlite branch, commit, remote, history, diff, and as-of paths + must use native `SELECT dolt_*` functions and per-table TVFs. Do not recreate + branch-per-file databases or synthetic `doltlite_commits` / `doltlite_refs` + tables in the Beads adapter. + +## Next Checks + +- Verify post-command auto-backup is skipped cleanly, not merely failing with a warning. +- Audit `BackupStore` / `versioncontrolops.Backup*` call sites for unconditional + `CALL DOLT_BACKUP(...)` usage in doltlite mode. +- Add first-class doltlite messaging tests instead of relying on Dolt-backed + `newTestStore(...)` helpers. From f27e413e42d5ec156658fee0b10192601bf08bd9 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Fri, 1 May 2026 21:01:06 +1000 Subject: [PATCH 09/15] docs(doltlite): note rig metadata split-brain --- docs/dev-notes/doltlite-beads-checklist.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/dev-notes/doltlite-beads-checklist.md b/docs/dev-notes/doltlite-beads-checklist.md index f20bf2260..5cf7ac008 100644 --- a/docs/dev-notes/doltlite-beads-checklist.md +++ b/docs/dev-notes/doltlite-beads-checklist.md @@ -74,6 +74,13 @@ Expected state: must use native `SELECT dolt_*` functions and per-table TVFs. Do not recreate branch-per-file databases or synthetic `doltlite_commits` / `doltlite_refs` tables in the Beads adapter. +- 2026-05-01: the active Gas Town `beads-doltlite` rig had + `.beads/doltlite/bd.db` on disk but `.beads/metadata.json` still declared + `backend=dolt` and `dolt_mode=server`, causing `bd` to chase stale Dolt port + env vars. Updating the rig-local metadata to `backend=doltlite`, + `database=doltlite`, and `dolt_mode=embedded` made both `./bd ready --json` + and installed `bd ready --json` open doltlite successfully even with stale + `BEADS_DOLT_*` env vars present. ## Next Checks From 7f42f78f2cf879b5632ca31cc554afcaff3fb9ef Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Sat, 2 May 2026 02:27:38 +1000 Subject: [PATCH 10/15] fix: honor GC scope root in beads discovery --- docs/dev-notes/doltlite-beads-checklist.md | 5 ++ internal/beads/beads.go | 34 ++++++++ internal/beads/beads_test.go | 96 ++++++++++++++++++++++ 3 files changed, 135 insertions(+) diff --git a/docs/dev-notes/doltlite-beads-checklist.md b/docs/dev-notes/doltlite-beads-checklist.md index 5cf7ac008..dad40e067 100644 --- a/docs/dev-notes/doltlite-beads-checklist.md +++ b/docs/dev-notes/doltlite-beads-checklist.md @@ -51,6 +51,11 @@ Expected state: ## Findings +- 2026-05-02: `internal/beads` discovery now honors `GC_BEADS_SCOPE_ROOT` + before cwd/worktree auto-discovery, so polecat sessions launched from + scaffolding worktrees resolve the rig's authoritative `.beads/` instead of + the packaged city `.beads/`. This prevents `bd` from opening the wrong + doltlite store or hanging on unrelated lock files when `BEADS_DIR` is unset. - 2026-05-01: deacon startup after `gc prime` carried stale Dolt env overrides (`BEADS_DOLT_PORT=35819`, `BEADS_DOLT_SERVER_PORT=35819`, `GC_DOLT_PORT=35819`) even though `gc doctor` and `gc dolt health` showed the live doltlite server diff --git a/internal/beads/beads.go b/internal/beads/beads.go index 78b41a8a3..aba7687d4 100644 --- a/internal/beads/beads.go +++ b/internal/beads/beads.go @@ -501,6 +501,14 @@ func FindDatabasePath() string { // Return empty string and let the caller handle it } + // 1a. Gas City sessions can advertise the authoritative repo root via + // GC_BEADS_SCOPE_ROOT even when cwd is a polecat worktree scaffold. + if beadsDir := getGCScopeRootBeadsDir(); beadsDir != "" { + if dbPath := findDatabaseInBeadsDir(beadsDir, false); dbPath != "" { + return dbPath + } + } + // 2. Check BEADS_DB environment variable (deprecated but still supported) if envDB := os.Getenv("BEADS_DB"); envDB != "" { absDB := utils.CanonicalizePath(envDB) @@ -656,6 +664,24 @@ func hasBeadsDatabase(beadsDir string) bool { return false } +// getGCScopeRootBeadsDir resolves the rig-local .beads directory advertised by +// Gas City sessions. GC_BEADS_SCOPE_ROOT points at the authoritative repo root +// for beads commands even when the process cwd is inside a polecat worktree +// that only contains scaffolding artifacts. +func getGCScopeRootBeadsDir() string { + scopeRoot := strings.TrimSpace(os.Getenv("GC_BEADS_SCOPE_ROOT")) + if scopeRoot == "" { + return "" + } + + beadsDir := canonicalizeBeadsDirPath(filepath.Join(scopeRoot, ".beads")) + if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { + return FollowRedirect(beadsDir) + } + + return "" +} + // FindBeadsDir finds the .beads/ directory in the current directory tree. // Returns empty string if not found. // @@ -689,6 +715,14 @@ func FindBeadsDir() string { } } + // 1a. Gas City sessions can advertise the authoritative repo root via + // GC_BEADS_SCOPE_ROOT even when cwd is a polecat worktree scaffold. + if beadsDir := getGCScopeRootBeadsDir(); beadsDir != "" { + if hasBeadsProjectFiles(beadsDir) { + return beadsDir + } + } + // 2. Walk up from CWD toward the repo root, checking each directory for .beads/. // This replaces the former step 1b (CWD-only check) with a proper ancestor walk, // fixing the case where CWD is a subdirectory within a rig (not the rig root itself). diff --git a/internal/beads/beads_test.go b/internal/beads/beads_test.go index 2dcf9e233..627587d0e 100644 --- a/internal/beads/beads_test.go +++ b/internal/beads/beads_test.go @@ -45,6 +45,102 @@ func TestFindDatabasePathEnvVar(t *testing.T) { } } +func TestFindBeadsDirGCScopeRoot(t *testing.T) { + originalBeadsDir := os.Getenv("BEADS_DIR") + originalScopeRoot := os.Getenv("GC_BEADS_SCOPE_ROOT") + t.Cleanup(func() { + if originalBeadsDir != "" { + os.Setenv("BEADS_DIR", originalBeadsDir) + } else { + os.Unsetenv("BEADS_DIR") + } + if originalScopeRoot != "" { + os.Setenv("GC_BEADS_SCOPE_ROOT", originalScopeRoot) + } else { + os.Unsetenv("GC_BEADS_SCOPE_ROOT") + } + }) + os.Unsetenv("BEADS_DIR") + + tmpDir := t.TempDir() + scopeRoot := filepath.Join(tmpDir, "rig") + beadsDir := filepath.Join(scopeRoot, ".beads") + if err := os.MkdirAll(filepath.Join(beadsDir, "doltlite"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"doltlite","database":"doltlite"}`), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Setenv("GC_BEADS_SCOPE_ROOT", scopeRoot); err != nil { + t.Fatal(err) + } + + workspace := filepath.Join(tmpDir, "workspace", "nested") + if err := os.MkdirAll(workspace, 0o755); err != nil { + t.Fatal(err) + } + t.Chdir(workspace) + + result := FindBeadsDir() + resultResolved, _ := filepath.EvalSymlinks(result) + expectedResolved, _ := filepath.EvalSymlinks(beadsDir) + if resultResolved != expectedResolved { + t.Errorf("FindBeadsDir() = %q, want %q from GC_BEADS_SCOPE_ROOT", result, beadsDir) + } +} + +func TestFindDatabasePathGCScopeRoot(t *testing.T) { + originalBeadsDir := os.Getenv("BEADS_DIR") + originalBeadsDB := os.Getenv("BEADS_DB") + originalScopeRoot := os.Getenv("GC_BEADS_SCOPE_ROOT") + t.Cleanup(func() { + if originalBeadsDir != "" { + os.Setenv("BEADS_DIR", originalBeadsDir) + } else { + os.Unsetenv("BEADS_DIR") + } + if originalBeadsDB != "" { + os.Setenv("BEADS_DB", originalBeadsDB) + } else { + os.Unsetenv("BEADS_DB") + } + if originalScopeRoot != "" { + os.Setenv("GC_BEADS_SCOPE_ROOT", originalScopeRoot) + } else { + os.Unsetenv("GC_BEADS_SCOPE_ROOT") + } + }) + os.Unsetenv("BEADS_DIR") + os.Unsetenv("BEADS_DB") + + tmpDir := t.TempDir() + scopeRoot := filepath.Join(tmpDir, "rig") + beadsDir := filepath.Join(scopeRoot, ".beads") + doltliteDir := filepath.Join(beadsDir, "doltlite") + if err := os.MkdirAll(doltliteDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"doltlite","database":"doltlite"}`), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Setenv("GC_BEADS_SCOPE_ROOT", scopeRoot); err != nil { + t.Fatal(err) + } + + workspace := filepath.Join(tmpDir, "workspace", "nested") + if err := os.MkdirAll(workspace, 0o755); err != nil { + t.Fatal(err) + } + t.Chdir(workspace) + + result := FindDatabasePath() + resultResolved, _ := filepath.EvalSymlinks(result) + expectedResolved, _ := filepath.EvalSymlinks(doltliteDir) + if resultResolved != expectedResolved { + t.Errorf("FindDatabasePath() = %q, want %q from GC_BEADS_SCOPE_ROOT", result, doltliteDir) + } +} + func TestFindDatabasePathInTree(t *testing.T) { // Save original env vars originalDB := os.Getenv("BEADS_DB") From 8f72026286d319ac8bda252162f27a39a16d4d71 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Sat, 2 May 2026 06:35:49 +1000 Subject: [PATCH 11/15] add doltlite sqlite uuid function --- internal/storage/doltlite/open.go | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/internal/storage/doltlite/open.go b/internal/storage/doltlite/open.go index 09b7c7f8a..e506c02bf 100644 --- a/internal/storage/doltlite/open.go +++ b/internal/storage/doltlite/open.go @@ -4,6 +4,7 @@ package doltlite import ( "context" + "crypto/rand" "database/sql" "fmt" "os" @@ -11,7 +12,7 @@ import ( "regexp" "strings" - _ "github.com/mattn/go-sqlite3" + "github.com/mattn/go-sqlite3" ) // validIdentifier matches safe SQL identifiers (letters, digits, underscores). @@ -23,8 +24,17 @@ const ( commitName = "beads" commitEmail = "beads@local" defaultBusyTimeout = 10000 + driverName = "sqlite3_doltlite" ) +func init() { + sql.Register(driverName, &sqlite3.SQLiteDriver{ + ConnectHook: func(conn *sqlite3.SQLiteConn) error { + return conn.RegisterFunc("UUID", newUUID, true) + }, + }) +} + // OpenSQL opens an doltlite database at dir. The returned cleanup // function closes the *sql.DB. func OpenSQL(ctx context.Context, dir, database, branch string) (*sql.DB, func() error, error) { @@ -32,7 +42,7 @@ func OpenSQL(ctx context.Context, dir, database, branch string) (*sql.DB, func() if err != nil { return nil, nil, err } - db, err := sql.Open("sqlite3", dbPath) + db, err := sql.Open(driverName, dbPath) if err != nil { return nil, nil, err } @@ -66,6 +76,23 @@ func OpenSQL(ctx context.Context, dir, database, branch string) (*sql.DB, func() return db, cleanup, nil } +func newUUID() (string, error) { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + return fmt.Sprintf( + "%x-%x-%x-%x-%x", + b[0:4], + b[4:6], + b[6:8], + b[8:10], + b[10:16], + ), nil +} + func buildDSN(dir, database string) (string, error) { if strings.TrimSpace(database) != "" { if !validIdentifier.MatchString(database) { From 195fa171388b858f035bb5c4d2afc9e0a43f8c46 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Sun, 3 May 2026 20:36:09 +1000 Subject: [PATCH 12/15] docs: record doltlite audit finding --- docs/dev-notes/doltlite-beads-checklist.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/dev-notes/doltlite-beads-checklist.md b/docs/dev-notes/doltlite-beads-checklist.md index dad40e067..0e07e2172 100644 --- a/docs/dev-notes/doltlite-beads-checklist.md +++ b/docs/dev-notes/doltlite-beads-checklist.md @@ -51,6 +51,14 @@ Expected state: ## Findings +- 2026-05-03: audit bead `bd-d74` confirmed the new `issueops` dialect wrappers + still default Dolt callers to `SQLDialectDolt`, so the storage-selection and + SQL-dialect shims reviewed in this pass do not appear to alter existing Dolt + backend query shapes. The same audit also found a live doltlite regression: + `internal/storage/doltlite/open.go` still runs `SELECT dolt_checkout(?)` on + open and `internal/storage/doltlite/version_control.go` / `commit_pending.go` + still call raw `dolt_*` SQL functions even though the local `sqlite3_doltlite` + driver only registers `UUID()`. Track fix in `bd-31j`. - 2026-05-02: `internal/beads` discovery now honors `GC_BEADS_SCOPE_ROOT` before cwd/worktree auto-discovery, so polecat sessions launched from scaffolding worktrees resolve the rig's authoritative `.beads/` instead of From 5ea85d4daeebd407eaf34162a37bd9afca3a7394 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Tue, 5 May 2026 13:05:25 +1000 Subject: [PATCH 13/15] docs: record standalone rig-store finding --- docs/dev-notes/doltlite-beads-checklist.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/dev-notes/doltlite-beads-checklist.md b/docs/dev-notes/doltlite-beads-checklist.md index 0e07e2172..780b638cd 100644 --- a/docs/dev-notes/doltlite-beads-checklist.md +++ b/docs/dev-notes/doltlite-beads-checklist.md @@ -51,6 +51,13 @@ Expected state: ## Findings +- 2026-05-05: `gc start` one-shot standalone reconciliation was dropping + `rigStores`, `AssignedWorkBeads`, and `workSet` when calling + `buildDesiredStateWithSessionBeads` / `reconcileSessionBeadsAtPath`, so + routed ready work that only exists in rig-local doltlite stores never + surfaced in `ScaleCheckCounts`, wake decisions, or trace output. Wiring the + standalone path through `buildStandaloneRigStores(...)` restores the same + rig-scoped demand visibility the persistent controller runtime already had. - 2026-05-03: audit bead `bd-d74` confirmed the new `issueops` dialect wrappers still default Dolt callers to `SQLDialectDolt`, so the storage-selection and SQL-dialect shims reviewed in this pass do not appear to alter existing Dolt From 061028699a8383d85ab32d4b711d69117e3e4a35 Mon Sep 17 00:00:00 2001 From: "Claude Opus 4.6" Date: Wed, 6 May 2026 07:47:08 +1000 Subject: [PATCH 14/15] docs: narrow doltlite SQL regression (bd-31j) --- docs/dev-notes/doltlite-beads-checklist.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/dev-notes/doltlite-beads-checklist.md b/docs/dev-notes/doltlite-beads-checklist.md index 780b638cd..e98e2b37c 100644 --- a/docs/dev-notes/doltlite-beads-checklist.md +++ b/docs/dev-notes/doltlite-beads-checklist.md @@ -58,6 +58,20 @@ Expected state: surfaced in `ScaleCheckCounts`, wake decisions, or trace output. Wiring the standalone path through `buildStandaloneRigStores(...)` restores the same rig-scoped demand visibility the persistent controller runtime already had. +- 2026-05-06: fresh manual probe with the installed `bd` binary showed the + doltlite SQL surface is present when the binary is linked correctly. Evidence: + `go version -m "$(command -v bd)"` reported `-tags=gms_pure_go,libsqlite3`, + `CGO_ENABLED=1`, `CGO_CFLAGS=-I/data/projects/doltlite/build`, and + `CGO_LDFLAGS=/data/projects/doltlite/libdoltlite.a ...`; then + `bd init --backend doltlite --prefix bd31j --skip-hooks --skip-agents + --non-interactive` succeeded in a fresh temp repo, `bd branch --json` + returned `main`, and `bd create` plus `bd flatten --dry-run` reported 4 + commits instead of the earlier embedded `store.Log()==0` failure shape. + This means the older “`sqlite3_doltlite` only registers `UUID()`” diagnosis + was too broad: native `dolt_*` functions and virtual tables do work in a + correctly linked build. Treat `bd-31j` as a linkage-diagnostics / direct-path + reproducibility question, not proof that Beads must stop using native + doltlite `dolt_*` SQL. - 2026-05-03: audit bead `bd-d74` confirmed the new `issueops` dialect wrappers still default Dolt callers to `SQLDialectDolt`, so the storage-selection and SQL-dialect shims reviewed in this pass do not appear to alter existing Dolt From b8faef07199811fe1cbf6d79a794d45b84364141 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 01:19:25 +0000 Subject: [PATCH 15/15] chore(deps): bump github.com/go-sql-driver/mysql from 1.9.3 to 1.10.0 Bumps [github.com/go-sql-driver/mysql](https://github.com/go-sql-driver/mysql) from 1.9.3 to 1.10.0. - [Release notes](https://github.com/go-sql-driver/mysql/releases) - [Changelog](https://github.com/go-sql-driver/mysql/blob/master/CHANGELOG.md) - [Commits](https://github.com/go-sql-driver/mysql/compare/v1.9.3...v1.10.0) --- updated-dependencies: - dependency-name: github.com/go-sql-driver/mysql dependency-version: 1.10.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index af1224407..66cb4a83f 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/anthropics/anthropic-sdk-go v1.37.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/dolthub/driver v1.86.4 - github.com/go-sql-driver/mysql v1.9.3 + github.com/go-sql-driver/mysql v1.10.0 github.com/mattn/go-sqlite3 v1.14.8 github.com/olebedev/when v1.1.0 github.com/spf13/cobra v1.10.2 @@ -55,7 +55,7 @@ require ( cloud.google.com/go/monitoring v1.24.2 // indirect cloud.google.com/go/storage v1.50.0 // indirect dario.cat/mergo v1.0.2 // indirect - filippo.io/edwards25519 v1.1.1 // indirect + filippo.io/edwards25519 v1.2.0 // indirect github.com/AlekSi/pointer v1.0.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect diff --git a/go.sum b/go.sum index 513290fe7..6973a39ad 100644 --- a/go.sum +++ b/go.sum @@ -103,8 +103,8 @@ contrib.go.opencensus.io/integrations/ocsql v0.1.7/go.mod h1:8DsSdjz3F+APR+0z0Wk dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= -filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= git.sr.ht/~sbinet/gg v0.3.1 h1:LNhjNn8DerC8f9DHLz6lS0YYul/b602DUxDgGkd/Aik= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= @@ -530,8 +530,8 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= -github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= +github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw= +github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=