diff --git a/config/config.go b/config/config.go index 0952e0d3173..c8432dd6d37 100644 --- a/config/config.go +++ b/config/config.go @@ -45,6 +45,8 @@ type Config struct { Import Import Version Version + OnDemandPinning OnDemandPinning + Internal Internal // experimental/unstable options Bitswap Bitswap diff --git a/config/experiments.go b/config/experiments.go index 6c43ac04f07..4cf3af86e76 100644 --- a/config/experiments.go +++ b/config/experiments.go @@ -10,6 +10,7 @@ type Experiments struct { OptimisticProvide bool OptimisticProvideJobsPoolSize int GatewayOverLibp2p bool `json:",omitempty"` + OnDemandPinningEnabled bool GraphsyncEnabled graphsyncEnabled `json:",omitempty"` AcceleratedDHTClient experimentalAcceleratedDHTClient `json:",omitempty"` diff --git a/config/ondemandpin.go b/config/ondemandpin.go new file mode 100644 index 00000000000..2bfa56cce93 --- /dev/null +++ b/config/ondemandpin.go @@ -0,0 +1,74 @@ +package config + +import ( + "fmt" + "time" + + "github.com/libp2p/go-libp2p-kad-dht/amino" +) + +const ( + DefaultOnDemandPinReplicationTargetMin = 5 + DefaultOnDemandPinReplicationTargetMax = 7 + DefaultOnDemandPinCheckInterval = 10 * time.Minute + + // Must exceed amino.DefaultProvideValidity so stale DHT records expire + // before unpin. The extra day covers check-interval skew. + DefaultOnDemandPinUnpinGracePeriod = amino.DefaultProvideValidity + 24*time.Hour + + // Max NextCheckAt delay after repeated check failures (Cap). + DefaultOnDemandPinCheckBackoffMax = DefaultOnDemandPinUnpinGracePeriod +) + +type OnDemandPinning struct { + // Pin when fewer than this many providers are found in the DHT (excluding self). + ReplicationTargetMin OptionalInteger + + // Start the unpin grace period only when more than this many providers are found. + ReplicationTargetMax OptionalInteger + + // How often the checker evaluates all registered CIDs. + CheckInterval OptionalDuration + + // How long replication must stay above max before unpinning; checker adds up to 2*CheckInterval of jitter. + UnpinGracePeriod OptionalDuration + + // When true, the checker logs pin/unpin decisions but does not change the pinset. + DryRun Flag `json:",omitempty"` +} + +// ValidateOnDemandPinningConfig rejects invalid min/max and non-positive durations. +func ValidateOnDemandPinningConfig(cfg *OnDemandPinning) error { + min := cfg.ReplicationTargetMin.WithDefault(DefaultOnDemandPinReplicationTargetMin) + max := cfg.ReplicationTargetMax.WithDefault(DefaultOnDemandPinReplicationTargetMax) + if min < 1 { + return fmt.Errorf("OnDemandPinning.ReplicationTargetMin must be at least 1, got %d", min) + } + if max < min { + return fmt.Errorf("OnDemandPinning.ReplicationTargetMax (%d) must be >= ReplicationTargetMin (%d)", max, min) + } + if interval := cfg.CheckInterval.WithDefault(DefaultOnDemandPinCheckInterval); interval <= 0 { + return fmt.Errorf("OnDemandPinning.CheckInterval must be positive, got %v", interval) + } + if grace := cfg.UnpinGracePeriod.WithDefault(DefaultOnDemandPinUnpinGracePeriod); grace <= 0 { + return fmt.Errorf("OnDemandPinning.UnpinGracePeriod must be positive, got %v", grace) + } + return nil +} + +// ValidateOnDemandPinningRouting rejects Routing.Type=none. +// Otherwise the checker would pin every registered CID on a null router. +func ValidateOnDemandPinningRouting(routingType string) error { + if routingType == "none" { + return fmt.Errorf("on-demand pinning needs provider lookups; Routing.Type=%q is not usable", routingType) + } + return nil +} + +// ValidateOnDemandPinningProvide rejects Provide.Enabled=false (pins would be invisible). +func ValidateOnDemandPinningProvide(provideEnabled bool) error { + if !provideEnabled { + return fmt.Errorf("on-demand pinning needs Provide.Enabled; other nodes cannot discover local pins when providing is off") + } + return nil +} diff --git a/config/ondemandpin_test.go b/config/ondemandpin_test.go new file mode 100644 index 00000000000..7bcf159c6b0 --- /dev/null +++ b/config/ondemandpin_test.go @@ -0,0 +1,54 @@ +package config + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestValidateOnDemandPinningConfig(t *testing.T) { + t.Run("defaults are valid", func(t *testing.T) { + err := ValidateOnDemandPinningConfig(&OnDemandPinning{}) + assert.NoError(t, err) + }) + + t.Run("zero replication min is rejected", func(t *testing.T) { + cfg := &OnDemandPinning{ReplicationTargetMin: *NewOptionalInteger(0)} + err := ValidateOnDemandPinningConfig(cfg) + assert.ErrorContains(t, err, "ReplicationTargetMin") + }) + + t.Run("max below min is rejected", func(t *testing.T) { + cfg := &OnDemandPinning{ + ReplicationTargetMin: *NewOptionalInteger(5), + ReplicationTargetMax: *NewOptionalInteger(4), + } + err := ValidateOnDemandPinningConfig(cfg) + assert.ErrorContains(t, err, "ReplicationTargetMax") + }) + + t.Run("zero check interval is rejected", func(t *testing.T) { + cfg := &OnDemandPinning{CheckInterval: *NewOptionalDuration(0)} + err := ValidateOnDemandPinningConfig(cfg) + assert.ErrorContains(t, err, "CheckInterval") + }) + + t.Run("negative grace period is rejected", func(t *testing.T) { + cfg := &OnDemandPinning{UnpinGracePeriod: *NewOptionalDuration(-time.Hour)} + err := ValidateOnDemandPinningConfig(cfg) + assert.ErrorContains(t, err, "UnpinGracePeriod") + }) +} + +func TestValidateOnDemandPinningRouting(t *testing.T) { + assert.NoError(t, ValidateOnDemandPinningRouting("auto")) + assert.NoError(t, ValidateOnDemandPinningRouting("dht")) + assert.NoError(t, ValidateOnDemandPinningRouting("delegated")) + assert.ErrorContains(t, ValidateOnDemandPinningRouting("none"), "none") +} + +func TestValidateOnDemandPinningProvide(t *testing.T) { + assert.NoError(t, ValidateOnDemandPinningProvide(true)) + assert.ErrorContains(t, ValidateOnDemandPinningProvide(false), "Provide.Enabled") +} diff --git a/core/commands/cmdutils/utils.go b/core/commands/cmdutils/utils.go index 8c93a220c99..6156218f274 100644 --- a/core/commands/cmdutils/utils.go +++ b/core/commands/cmdutils/utils.go @@ -3,6 +3,7 @@ package cmdutils import ( "fmt" "slices" + "strings" cmds "github.com/ipfs/go-ipfs-cmds" @@ -54,14 +55,24 @@ func CheckBlockSize(req *cmds.Request, size uint64) error { return nil } -// ValidatePinName validates that a pin name does not exceed the maximum allowed byte length. -// Returns an error if the name exceeds MaxPinNameBytes (255 bytes). +// ReservedPinNamePrefix is the namespace for pins created by Kubo-internal +// features rather than by users (e.g. "kubo:on-demand" for on-demand pinning). +// ValidatePinName rejects user-supplied names with this prefix, so a pin name +// under this namespace reliably identifies a Kubo-internal pin. +const ReservedPinNamePrefix = "kubo:" + +// ValidatePinName validates that a pin name does not exceed the maximum allowed +// byte length and does not use the Kubo-internal "kubo:" namespace. func ValidatePinName(name string) error { if name == "" { // Empty names are allowed return nil } + if strings.HasPrefix(name, ReservedPinNamePrefix) { + return fmt.Errorf("pin names starting with %q are reserved for kubo-internal pins", ReservedPinNamePrefix) + } + nameBytes := len([]byte(name)) if nameBytes > MaxPinNameBytes { return fmt.Errorf("pin name is %d bytes (max %d bytes)", nameBytes, MaxPinNameBytes) diff --git a/core/commands/cmdutils/utils_test.go b/core/commands/cmdutils/utils_test.go index 4a8551ad07d..8ef109fd561 100644 --- a/core/commands/cmdutils/utils_test.go +++ b/core/commands/cmdutils/utils_test.go @@ -178,4 +178,10 @@ func TestValidatePinName(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "bytes") }) + + t.Run("pin name with reserved prefix is rejected", func(t *testing.T) { + err := ValidatePinName(ReservedPinNamePrefix + "on-demand") + require.Error(t, err) + assert.Contains(t, err.Error(), "reserved") + }) } diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 22df7d5105c..fe7603db5f5 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -168,6 +168,10 @@ func TestCommands(t *testing.T) { "/pin/remote/service/add", "/pin/remote/service/ls", "/pin/remote/service/rm", + "/pin/ondemand", + "/pin/ondemand/add", + "/pin/ondemand/rm", + "/pin/ondemand/ls", "/pin/rm", "/pin/update", "/pin/verify", diff --git a/core/commands/pin/ondemandpin.go b/core/commands/pin/ondemandpin.go new file mode 100644 index 00000000000..6b4f590f168 --- /dev/null +++ b/core/commands/pin/ondemandpin.go @@ -0,0 +1,367 @@ +package pin + +import ( + "context" + "fmt" + "io" + "time" + + cmds "github.com/ipfs/go-ipfs-cmds" + "github.com/ipfs/kubo/config" + cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" + "github.com/ipfs/kubo/core/commands/cmdutils" + "github.com/ipfs/kubo/ondemandpin" + "golang.org/x/sync/errgroup" +) + +const ( + onDemandLiveOptionName = "live" + liveLookupParallelism = 8 +) + +var onDemandPinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Manage on-demand pins.", + ShortDescription: ` +Pins registered CIDs when FindProviders finds few DHT providers; unpins after +the count stays above max for a grace period. Roughly like ipfs-cluster +replication factors +(https://ipfscluster.io/documentation/guides/pinning/#replication-factors), +except each node decides on its own from the DHT. Requires +Experimental.OnDemandPinningEnabled. +`, + }, + Subcommands: map[string]*cmds.Command{ + "add": addOnDemandPinCmd, + "rm": rmOnDemandPinCmd, + "ls": listOnDemandPinCmd, + }, +} + +type OnDemandPinOutput struct { + Cid string `json:"Cid"` +} + +var addOnDemandPinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Register CIDs for on-demand pinning.", + ShortDescription: `Registers CID(s) for on-demand pinning; checker pins when needed.`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("cid", true, true, "CID(s) to register."), + }, + Type: OnDemandPinOutput{}, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + n, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + cfg, err := n.Repo.Config() + if err != nil { + return err + } + if !cfg.Experimental.OnDemandPinningEnabled { + return fmt.Errorf("on-demand pinning is not enabled; set Experimental.OnDemandPinningEnabled = true in config") + } + + store := n.OnDemandPinStore + + api, err := cmdenv.GetApi(env, req) + if err != nil { + return err + } + + for _, arg := range req.Arguments { + p, err := cmdutils.PathOrCidPath(arg) + if err != nil { + return fmt.Errorf("invalid CID or path %q: %w", arg, err) + } + + rp, _, err := api.ResolvePath(req.Context, p) + if err != nil { + return fmt.Errorf("resolving %q: %w", arg, err) + } + c := rp.RootCid() + + if err := store.Add(req.Context, c); err != nil { + return err + } + + if checker := n.OnDemandPinChecker; checker != nil { + checker.Enqueue(c) + } + + if err := res.Emit(&OnDemandPinOutput{ + Cid: c.String(), + }); err != nil { + return err + } + } + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OnDemandPinOutput) error { + fmt.Fprintf(w, "registered %s for on-demand pinning\n", out.Cid) + return nil + }), + }, +} + +var rmOnDemandPinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Remove CIDs from on-demand pinning.", + ShortDescription: ` +Removes CID(s) from the registry. Checker-pinned content is unpinned. + +Works when on-demand pinning is disabled, to clear old registrations. +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("cid", true, true, "CID(s) to remove."), + }, + Type: OnDemandPinOutput{}, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + n, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + store := n.OnDemandPinStore + + api, err := cmdenv.GetApi(env, req) + if err != nil { + return err + } + + for _, arg := range req.Arguments { + p, err := cmdutils.PathOrCidPath(arg) + if err != nil { + return fmt.Errorf("invalid CID or path %q: %w", arg, err) + } + + rp, _, err := api.ResolvePath(req.Context, p) + if err != nil { + return fmt.Errorf("resolving %q: %w", arg, err) + } + c := rp.RootCid() + + // Check the store first so an unregistered CID errors before any pin is touched. + if _, err := store.Get(req.Context, c); err != nil { + return err + } + + // Only remove pins carrying the Kubo-internal kubo:on-demand name; user pins on the same CID are kept. + hasOnDemandPin, err := ondemandpin.PinHasName(req.Context, n.Pinning, c, ondemandpin.OnDemandPinName) + if err != nil { + return fmt.Errorf("checking pin state for %s: %w", c, err) + } + if hasOnDemandPin { + if err := api.Pin().Rm(req.Context, rp); err != nil { + return fmt.Errorf("unpinning %s: %w", c, err) + } + } + + if err := store.Remove(req.Context, c); err != nil { + return err + } + + if err := res.Emit(&OnDemandPinOutput{ + Cid: c.String(), + }); err != nil { + return err + } + } + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OnDemandPinOutput) error { + fmt.Fprintf(w, "removed %s from on-demand pinning\n", out.Cid) + return nil + }), + }, +} + +type OnDemandLsOutput struct { + Cid string `json:"Cid"` + HasOnDemandPin bool `json:"HasOnDemandPin"` + Providers *int `json:"Providers,omitempty"` // live lookup only + ProvidersUnknown bool `json:"ProvidersUnknown,omitempty"` + LastProviderCount *int `json:"LastProviderCount,omitempty"` + LastCheckedAt string `json:"LastCheckedAt,omitempty"` + LastResult string `json:"LastResult,omitempty"` + LastAboveTarget string `json:"LastAboveTarget,omitempty"` + UnpinAt string `json:"UnpinAt,omitempty"` + FailureCount int `json:"FailureCount,omitempty"` + NextCheckAt string `json:"NextCheckAt,omitempty"` + CreatedAt string `json:"CreatedAt"` +} + +var listOnDemandPinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List on-demand pins.", + ShortDescription: ` +Lists registered CIDs with last check result, provider count, and unpin time. +Use --live for a fresh DHT provider count (requires content routing). +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("cid", false, true, "Optional CID(s) to filter."), + }, + Options: []cmds.Option{ + cmds.BoolOption(onDemandLiveOptionName, "l", "Perform live provider lookup."), + }, + Type: OnDemandLsOutput{}, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + n, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + store := n.OnDemandPinStore + + live, _ := req.Options[onDemandLiveOptionName].(bool) + + var replicationMin, replicationMax int + if live { + if n.Routing == nil { + return fmt.Errorf("--live requires content routing; none is available") + } + cfg, err := n.Repo.Config() + if err != nil { + return err + } + replicationMin = int(cfg.OnDemandPinning.ReplicationTargetMin.WithDefault(config.DefaultOnDemandPinReplicationTargetMin)) + replicationMax = int(cfg.OnDemandPinning.ReplicationTargetMax.WithDefault(config.DefaultOnDemandPinReplicationTargetMax)) + } + + var records []ondemandpin.Record + if len(req.Arguments) > 0 { + api, err := cmdenv.GetApi(env, req) + if err != nil { + return err + } + for _, arg := range req.Arguments { + p, err := cmdutils.PathOrCidPath(arg) + if err != nil { + return fmt.Errorf("invalid CID or path %q: %w", arg, err) + } + rp, _, err := api.ResolvePath(req.Context, p) + if err != nil { + return fmt.Errorf("resolving %q: %w", arg, err) + } + rec, err := store.Get(req.Context, rp.RootCid()) + if err != nil { + return err + } + records = append(records, *rec) + } + } else { + records, err = store.List(req.Context) + if err != nil { + return err + } + } + + type liveResult struct { + count int + ok bool + } + liveResults := make([]liveResult, len(records)) + if live && len(records) > 0 { + g, gctx := errgroup.WithContext(req.Context) + g.SetLimit(liveLookupParallelism) + for i := range records { + i := i + c := records[i].Cid + g.Go(func() error { + lookupCtx, cancel := context.WithTimeout(gctx, ondemandpin.CheckTimeout) + defer cancel() + count, ok := ondemandpin.CountProviders(lookupCtx, n.Routing, n.Identity, c, replicationMin, replicationMax) + liveResults[i] = liveResult{count: count, ok: ok} + return nil + }) + } + if err := g.Wait(); err != nil { + return err + } + } + + for i, rec := range records { + hasOnDemandPin, err := ondemandpin.PinHasName(req.Context, n.Pinning, rec.Cid, ondemandpin.OnDemandPinName) + if err != nil { + return fmt.Errorf("checking pin state for %s: %w", rec.Cid, err) + } + out := OnDemandLsOutput{ + Cid: rec.Cid.String(), + HasOnDemandPin: hasOnDemandPin, + LastResult: rec.LastResult, + FailureCount: rec.FailureCount, + CreatedAt: rec.CreatedAt.Format(time.RFC3339), + } + if !rec.LastCheckedAt.IsZero() { + out.LastCheckedAt = rec.LastCheckedAt.Format(time.RFC3339) + } + if rec.LastResult != "" || !rec.LastCheckedAt.IsZero() { + count := rec.LastProviderCount + out.LastProviderCount = &count + } + if !rec.LastAboveTarget.IsZero() { + out.LastAboveTarget = rec.LastAboveTarget.Format(time.RFC3339) + } + if !rec.UnpinAt.IsZero() { + out.UnpinAt = rec.UnpinAt.Format(time.RFC3339) + } + if !rec.NextCheckAt.IsZero() { + out.NextCheckAt = rec.NextCheckAt.Format(time.RFC3339) + } + if live { + if liveResults[i].ok { + count := liveResults[i].count + out.Providers = &count + } else { + out.ProvidersUnknown = true + } + } + + if err := res.Emit(&out); err != nil { + return err + } + } + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OnDemandLsOutput) error { + pinState := "not-pinned" + if out.HasOnDemandPin { + pinState = "pinned" + } + fmt.Fprintf(w, "%s %s", out.Cid, pinState) + if out.Providers != nil { + fmt.Fprintf(w, " providers=%d", *out.Providers) + } else if out.ProvidersUnknown { + fmt.Fprintf(w, " providers=unknown") + } else if out.LastProviderCount != nil { + fmt.Fprintf(w, " last-providers=%d", *out.LastProviderCount) + } + if out.LastResult != "" { + fmt.Fprintf(w, " result=%s", out.LastResult) + } + if out.LastCheckedAt != "" { + fmt.Fprintf(w, " checked=%s", out.LastCheckedAt) + } + if out.UnpinAt != "" { + fmt.Fprintf(w, " unpin-at=%s", out.UnpinAt) + } + if out.NextCheckAt != "" { + fmt.Fprintf(w, " next-check=%s", out.NextCheckAt) + } + if out.FailureCount > 0 { + fmt.Fprintf(w, " failures=%d", out.FailureCount) + } + fmt.Fprintf(w, " created=%s", out.CreatedAt) + fmt.Fprintln(w) + return nil + }), + }, +} diff --git a/core/commands/pin/pin.go b/core/commands/pin/pin.go index 1810f1b5a7b..55e0a1301a1 100644 --- a/core/commands/pin/pin.go +++ b/core/commands/pin/pin.go @@ -39,6 +39,8 @@ var PinCmd = &cmds.Command{ "verify": verifyPinCmd, "update": updatePinCmd, "remote": remotePinCmd, + + "ondemand": onDemandPinCmd, }, } diff --git a/core/core.go b/core/core.go index db8c4d9a66e..c552715c951 100644 --- a/core/core.go +++ b/core/core.go @@ -57,6 +57,7 @@ import ( "github.com/ipfs/kubo/core/node" "github.com/ipfs/kubo/core/node/libp2p" "github.com/ipfs/kubo/fuse/mount" + "github.com/ipfs/kubo/ondemandpin" "github.com/ipfs/kubo/p2p" "github.com/ipfs/kubo/repo" irouting "github.com/ipfs/kubo/routing" @@ -77,6 +78,9 @@ type IpfsNode struct { PrivateKey ic.PrivKey `optional:"true"` // the local node's private Key PNetFingerprint libp2p.PNetFingerprint `optional:"true"` // fingerprint of private network + OnDemandPinStore *ondemandpin.Store + OnDemandPinChecker *ondemandpin.Checker `optional:"true"` + // Services Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances Blockstore bstore.GCBlockstore // the block store (lower level) diff --git a/core/node/groups.go b/core/node/groups.go index 656a58234f1..28db78e9a8d 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -357,6 +357,8 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part // new CIDs still announce via fast-provide-root and 'ipfs provide once'. isProviderEnabled := cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled) + isOnDemandPinEnabled := cfg.Experimental.OnDemandPinningEnabled + return fx.Options( fx.Provide(BitswapOptions(cfg)), fx.Provide(Bitswap(isBitswapServerEnabled, isBitswapLibp2pEnabled, isHTTPRetrievalEnabled)), @@ -373,6 +375,8 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part LibP2P(bcfg, cfg, userResourceOverrides), OnlineProviders(isProviderEnabled, cfg), + + maybeProvide(OnDemandPinChecker(cfg.OnDemandPinning), isOnDemandPinEnabled), ) } @@ -446,6 +450,21 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option { return fx.Error(err) } + // Only validate when enabled; unused sections must not block startup when disabled. + if cfg.Experimental.OnDemandPinningEnabled { + if err := config.ValidateOnDemandPinningConfig(&cfg.OnDemandPinning); err != nil { + return fx.Error(err) + } + routingType := cfg.Routing.Type.WithDefault(config.DefaultRoutingType) + if err := config.ValidateOnDemandPinningRouting(routingType); err != nil { + return fx.Error(err) + } + provideEnabled := cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled) + if err := config.ValidateOnDemandPinningProvide(provideEnabled); err != nil { + return fx.Error(err) + } + } + // Directory sharding settings from Import config. // These globals affect both `ipfs add` and MFS (`ipfs files` API). shardSizeThreshold := cfg.Import.UnixFSHAMTDirectorySizeThreshold.WithDefault(config.DefaultUnixFSHAMTDirectorySizeThreshold) @@ -466,6 +485,7 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option { fx.Provide(BlockService(cfg)), fx.Provide(Pinning(providerStrategy)), fx.Provide(Files(providerStrategy)), + fx.Provide(OnDemandPinStore), Core, ) } diff --git a/core/node/ondemandpin.go b/core/node/ondemandpin.go new file mode 100644 index 00000000000..f02e42a12cb --- /dev/null +++ b/core/node/ondemandpin.go @@ -0,0 +1,170 @@ +package node + +import ( + "context" + "time" + + "github.com/dustin/go-humanize" + blockstore "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/ipld/merkledag" + pin "github.com/ipfs/boxo/pinning/pinner" + "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" + peer "github.com/libp2p/go-libp2p/core/peer" + routing "github.com/libp2p/go-libp2p/core/routing" + "go.uber.org/fx" + + "github.com/ipfs/kubo/config" + "github.com/ipfs/kubo/core/node/helpers" + "github.com/ipfs/kubo/ondemandpin" + "github.com/ipfs/kubo/repo" +) + +// pinIdleTimeout cancels a pin when no new blocks arrive for this long. +const pinIdleTimeout = 2 * time.Minute + +type kuboPinService struct { + pinner pin.Pinner + dag format.DAGService + bs blockstore.GCBlockstore +} + +func (s *kuboPinService) Pin(ctx context.Context, c cid.Cid, name string) error { + defer s.bs.PinLock(ctx).Unlock(ctx) + + node, err := s.fetchRoot(ctx, c) + if err != nil { + return err + } + if err := s.pinDAG(ctx, node, name); err != nil { + return err + } + return s.pinner.Flush(ctx) +} + +func (s *kuboPinService) fetchRoot(ctx context.Context, c cid.Cid) (format.Node, error) { + ctx, cancel := context.WithTimeout(ctx, pinIdleTimeout) + defer cancel() + return s.dag.Get(ctx, c) +} + +// pinDAG recursively pins the DAG rooted at node. +// A background goroutine monitors block-fetching progress and cancels the operation if no new blocks arrive within pinIdleTimeout. +func (s *kuboPinService) pinDAG(ctx context.Context, node format.Node, name string) error { + tracker := new(merkledag.ProgressTracker) + trackerCtx := tracker.DeriveContext(ctx) + pinCtx, cancel := context.WithCancel(trackerCtx) + defer cancel() + + go watchPinProgress(pinCtx, cancel, tracker, pinIdleTimeout) + return s.pinner.Pin(pinCtx, node, true, name) +} + +func watchPinProgress(ctx context.Context, cancel context.CancelFunc, tracker *merkledag.ProgressTracker, timeout time.Duration) { + var last int + lastProgress := time.Now() + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if cur := tracker.Value(); cur != last { + last = cur + lastProgress = time.Now() + } else if time.Since(lastProgress) >= timeout { + cancel() + return + } + } + } +} + +func (s *kuboPinService) Unpin(ctx context.Context, c cid.Cid) error { + defer s.bs.PinLock(ctx).Unlock(ctx) + + if err := s.pinner.Unpin(ctx, c, true); err != nil { + return err + } + return s.pinner.Flush(ctx) +} + +func (s *kuboPinService) IsPinned(ctx context.Context, c cid.Cid) (bool, error) { + _, pinned, err := s.pinner.IsPinned(ctx, c) + return pinned, err +} + +func (s *kuboPinService) HasPinWithName(ctx context.Context, c cid.Cid, name string) (bool, error) { + return ondemandpin.PinHasName(ctx, s.pinner, c, name) +} + +type kuboStorageChecker struct { + repo repo.Repo +} + +func (s *kuboStorageChecker) StorageUsage(ctx context.Context) (uint64, uint64, error) { + cfg, err := s.repo.Config() + if err != nil { + return 0, 0, err + } + used, err := s.repo.GetStorageUsage(ctx) + if err != nil { + return 0, 0, err + } + if cfg.Datastore.StorageMax == "" { + return used, 0, nil + } + max, err := humanize.ParseBytes(cfg.Datastore.StorageMax) + if err != nil { + return 0, 0, err + } + wm := cfg.Datastore.StorageGCWatermark + if wm <= 0 || wm > 100 { + wm = 90 + } + return used, max * uint64(wm) / 100, nil +} + +func OnDemandPinStore(r repo.Repo) *ondemandpin.Store { + return ondemandpin.NewStore(r.Datastore()) +} + +func OnDemandPinChecker(cfg config.OnDemandPinning) func( + mctx helpers.MetricsCtx, + lc fx.Lifecycle, + r repo.Repo, + store *ondemandpin.Store, + pinner pin.Pinner, + cr routing.ContentRouting, + prov DHTProvider, + dag format.DAGService, + bs blockstore.GCBlockstore, + id peer.ID, +) *ondemandpin.Checker { + return func( + mctx helpers.MetricsCtx, + lc fx.Lifecycle, + r repo.Repo, + store *ondemandpin.Store, + pinner pin.Pinner, + cr routing.ContentRouting, + prov DHTProvider, + dag format.DAGService, + bs blockstore.GCBlockstore, + id peer.ID, + ) *ondemandpin.Checker { + pins := &kuboPinService{pinner: pinner, dag: dag, bs: bs} + storage := &kuboStorageChecker{repo: r} + checker := ondemandpin.NewChecker(store, pins, storage, cr, prov, id, cfg) + ctx := helpers.LifecycleCtx(mctx, lc) + + lc.Append(fx.Hook{ + OnStart: func(context.Context) error { + go checker.Run(ctx) + return nil + }, + }) + return checker + } +} diff --git a/docs/changelogs/v0.42.md b/docs/changelogs/v0.42.md index 6e687460335..5270b64988e 100644 --- a/docs/changelogs/v0.42.md +++ b/docs/changelogs/v0.42.md @@ -20,6 +20,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [๐Ÿ“Š OpenTelemetry: scope info now exposed as labels](#-opentelemetry-scope-info-now-exposed-as-labels) - [๐Ÿ”ง Cleaner progress bars](#-cleaner-progress-bars) - [๐Ÿ“ฆ๏ธ Dependency updates](#-dependency-updates) + - [Experimental on-demand pinning](#experimental-on-demand-pinning) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -127,6 +128,50 @@ The Prometheus endpoint no longer emits the `otel_scope_info` metric. Each metri - update `go-car/v2` to [v2.17.0](https://github.com/ipld/go-car/releases/tag/v2.17.0) - update `go.opentelemetry.io` to [v1.44.0](https://github.com/open-telemetry/opentelemetry-go/releases/tag/v1.44.0)(includes [v1.43.0](https://github.com/open-telemetry/opentelemetry-go/releases/tag/v1.43.0)) - update `p2p-forge/client` to [v0.9.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.9.0) (incl. [v0.8.1](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.8.1)) +#### Experimental on-demand pinning + +Automatically pins content when DHT provider counts fall below a configurable +replication target, and unpins once replication has been above target for a +grace period. Helps keeping critical data around, without wasting storage on +overly replicated CIDs. + +The feature is gated behind `Experimental.OnDemandPinningEnabled` and described +in [ipfs/specs#532](https://github.com/ipfs/specs/pull/532). + +```console +$ ipfs config --json Experimental.OnDemandPinningEnabled true +``` + +New CLI commands under `ipfs pin ondemand`: + +- `add` -- register CIDs for on-demand pinning +- `rm` -- deregister and unpin +- `ls` -- list registered CIDs (use `--live` for real-time DHT provider counts) + +Design highlights: + +- **Pin partitioning**: the checker needs to distinguish its pins from user + pins to avoid accidental deletion. This implementation uses boxo's pin name + field ("on-demand"). +- **Storage budget**: skips pinning when repo usage exceeds + `StorageMax * StorageGCWatermark`. +- **Idle timeout**: DAG fetches timeout after 2 minutes without receiving new + blocks, allowing large downloads while skipping dead records. +- **Provide after pin**: the checker publishes a DHT provider record after + pinning so other peers can discover the content on this node. +- **Sybil limitation**: provider counts come from DHT queries, which are + susceptible to Sybil manipulation. Documented as a known limitation. + +Configuration at [`OnDemandPinning`](https://github.com/ipfs/kubo/blob/master/docs/config.md#ondemandpinning): + +| Option | Default | Description | +|---|---|---| +| `OnDemandPinning.ReplicationTarget` | `5` | Minimum providers in DHT (excluding self) | +| `OnDemandPinning.CheckInterval` | `"10m"` | How often the checker runs | +| `OnDemandPinning.UnpinGracePeriod` | `"24h"` | How long above target before unpinning | + +See [experimental features](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#on-demand-pinning) +for full documentation. ### ๐Ÿ“ Changelog diff --git a/docs/changelogs/vFUTURE.md b/docs/changelogs/vFUTURE.md new file mode 100644 index 00000000000..b385ede4392 --- /dev/null +++ b/docs/changelogs/vFUTURE.md @@ -0,0 +1,71 @@ +# Kubo changelog vFUTURE + + + +This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. + +- [vFUTURE](#vfuture) + +## vFUTURE + +- [Overview](#overview) +- [๐Ÿ”ฆ Highlights](#-highlights) + - [Experimental on-demand pinning](#experimental-on-demand-pinning) +- [๐Ÿ“ Changelog](#-changelog) +- [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) + +### Overview + +### ๐Ÿ”ฆ Highlights + +#### Experimental on-demand pinning + +Automatically pins content when DHT provider counts fall below a configurable +min, and unpins once they stay above a max for a grace period. Helps keep +critical data around without storing overly replicated CIDs. Same rough idea as +ipfs-cluster replication factors, except nobody coordinates the pinset; each node reacts to DHT counts locally. + +The feature is gated behind `Experimental.OnDemandPinningEnabled` and described +in [ipfs/specs#532](https://github.com/ipfs/specs/pull/532). + +```console +$ ipfs config --json Experimental.OnDemandPinningEnabled true +``` + +New CLI commands under `ipfs pin ondemand`: + +- `add` register CIDs for on-demand pinning +- `rm` deregister and unpin +- `ls` list registered CIDs (use `--live` for DHT provider counts; errors if routing is unavailable; timed-out lookups show as unknown) + +Design highlights: + +- **Pin partitioning**: the checker marks its pins with the reserved name + `kubo:on-demand` so it never removes a user pin. +- **Storage budget**: skips pinning when repo usage exceeds + `StorageMax * StorageGCWatermark`. +- **Idle timeout**: DAG fetches timeout after 2 minutes without receiving new + blocks, allowing large downloads while skipping dead records. +- **Provide after pin**: after pinning, the checker publishes a DHT provider + record via `DHTProvider.StartProviding` so other peers can discover the + content. Honors `Provide.Enabled` (daemon will not start if providing is off) + and re-announces periodically. +- **Sybil limitation**: provider counts come from DHT queries, which are + susceptible to Sybil manipulation. Documented as a known limitation. + +Configuration at [`OnDemandPinning`](https://github.com/ipfs/kubo/blob/master/docs/config.md#ondemandpinning): + +| Option | Default | Description | +|---|---|---| +| `OnDemandPinning.ReplicationTargetMin` | `5` | Pin when fewer than this many providers (excluding self) | +| `OnDemandPinning.ReplicationTargetMax` | `7` | Start unpin grace only above this many providers | +| `OnDemandPinning.CheckInterval` | `"10m"` | Sweep period; failed CIDs back off up to 72h | +| `OnDemandPinning.UnpinGracePeriod` | `"72h"` | How long above max before unpinning (longer than 48h DHT record validity; plus up to `2 * CheckInterval` jitter) | +| `OnDemandPinning.DryRun` | `false` | Log/record pin/unpin decisions without changing the pinset | + +See [experimental features](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#on-demand-pinning) +for full documentation. + +### ๐Ÿ“ Changelog + +### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/config.md b/docs/config.md index dac39ab1158..77a6fe1edf1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -60,6 +60,7 @@ config file at runtime. - [`Discovery.MDNS.Interval`](#discoverymdnsinterval) - [`Experimental`](#experimental) - [`Experimental.Libp2pStreamMounting`](#experimentallibp2pstreammounting) + - [`Experimental.OnDemandPinningEnabled`](#experimentalondemandpinningenabled) - [`Gateway`](#gateway) - [`Gateway.NoFetch`](#gatewaynofetch) - [`Gateway.NoDNSLink`](#gatewaynodnslink) @@ -125,6 +126,12 @@ config file at runtime. - [`Mounts.FuseAllowOther`](#mountsfuseallowother) - [`Mounts.StoreMtime`](#mountsstoremtime) - [`Mounts.StoreMode`](#mountsstoremode) + - [`OnDemandPinning`](#ondemandpinning) + - [`OnDemandPinning.ReplicationTargetMin`](#ondemandpinningreplicationtargetmin) + - [`OnDemandPinning.ReplicationTargetMax`](#ondemandpinningreplicationtargetmax) + - [`OnDemandPinning.CheckInterval`](#ondemandpinningcheckinterval) + - [`OnDemandPinning.UnpinGracePeriod`](#ondemandpinningunpingraceperiod) + - [`OnDemandPinning.DryRun`](#ondemandpinningdryrun) - [`Pinning`](#pinning) - [`Pinning.RemoteServices`](#pinningremoteservices) - [`Pinning.RemoteServices: API`](#pinningremoteservices-api) @@ -1234,6 +1241,21 @@ in the [new mDNS implementation](https://github.com/libp2p/zeroconf#readme). Toggle and configure experimental features of Kubo. Experimental features are listed [here](./experimental-features.md). +### `Experimental.OnDemandPinningEnabled` + +Enables on-demand pinning. When enabled, the node runs a background checker +that periodically evaluates DHT provider counts for CIDs registered via +`ipfs pin ondemand add`. CIDs with fewer providers than the replication target +are pinned; pins are removed after replication stays above target for a grace +period (default 24h). Requires usable content routing and +[`Provide.Enabled`](#provideenabled). + +See [`OnDemandPinning`](#ondemandpinning) for configuration. + +Default: `false` + +Type: `bool` + ### `Experimental.Libp2pStreamMounting` Enables the `ipfs p2p` commands for tunneling TCP connections through libp2p @@ -2224,6 +2246,67 @@ Default: `"5m"` Type: `duration` +## `OnDemandPinning` + +Configures the on-demand pinning system. Requires +[`Experimental.OnDemandPinningEnabled`](#experimentalondemandpinningenabled). +See [experimental features](./experimental-features.md#on-demand-pinning) for how +this differs from ipfs-cluster replication. + +### `OnDemandPinning.ReplicationTargetMin` + +Pin when fewer than this many providers are found in the DHT (excluding the +local node). + +Default: `5` + +Type: `optionalInteger` + +### `OnDemandPinning.ReplicationTargetMax` + +Start the unpin grace period only when more than this many providers are found +(excluding the local node). Between min and max (inclusive) the checker does +nothing. Must be >= `ReplicationTargetMin`. + +Default: `7` + +Type: `optionalInteger` + +### `OnDemandPinning.CheckInterval` + +How often the checker starts a sweep of registered CIDs. If a sweep overruns +this duration, the next sweep starts when the previous one finishes. After a +failed check, that CID is skipped until `NextCheckAt` +(`CheckInterval * 2^(failures-1)`, max 72h). + +Default: `"10m"` + +Type: `optionalDuration` + +### `OnDemandPinning.UnpinGracePeriod` + +How long the provider count must stay above max before the local pin is removed. +The checker also adds a random delay of up to `2 * CheckInterval` when grace +starts, so nodes that entered grace together do not all unpin at once. + +Should be longer than DHT provider-record validity (48h). A shorter value can +unpin while stale records still make the count look healthy. + +Default: `"72h"` + +Type: `optionalDuration` + +### `OnDemandPinning.DryRun` + +When `true`, the checker still evaluates registered CIDs and records +`LastResult` / `LastProviderCount`, but it does not pin or unpin. Useful for +watching decisions with `ipfs pin ondemand ls` before enabling real pinset +changes. + +Default: `false` + +Type: `flag` + ## `Provide` Configures how your node advertises content to make it discoverable by other diff --git a/docs/experimental-features.md b/docs/experimental-features.md index abd45ff3ca5..960f8d2d873 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -28,6 +28,7 @@ the above issue. - [Noise](#noise) - [Optimistic Provide](#optimistic-provide) - [HTTP Gateway over Libp2p](#http-gateway-over-libp2p) +- [On-Demand Pinning](#on-demand-pinning) --- @@ -609,6 +610,83 @@ ipfs config --json Experimental.GatewayOverLibp2p true - [ ] Needs a mechanism for HTTP handler to signal supported features ([IPIP-425](https://github.com/ipfs/specs/pull/425)) - [ ] Needs an option for Kubo to detect peers that have it enabled and prefer HTTP transport before falling back to bitswap (and use CAR if peer supports dag-scope=entity from [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/)) +## On-Demand Pinning + +### State + +Experimental, disabled by default. + +On-demand pinning lets a node pin content when DHT provider counts fall below +a minimum, and unpin after they stay above a maximum for a grace period +(plus a short random delay). Values between min and max are left alone. +Provider counts come from `FindProviders`, not from the peer routing table. + +[ipfs-cluster replication factors](https://ipfscluster.io/documentation/guides/pinning/#replication-factors) +assign pins among known peers; this feature only watches the DHT and decides +locally. + +The feature consists of: + +- A **registry** of CIDs to monitor, managed via `ipfs pin ondemand add|rm|ls`. +- A **background checker** that periodically queries the DHT for each + registered CID and pins or unpins accordingly. Failed CIDs are retried + with exponential backoff (max 72h). +- **Pin partitioning**: the checker marks its pins with the name + `"kubo:on-demand"`, so the checker can tell its + pins apart from user pins and will never remove a pin it did not create. + +When the checker pins a CID it publishes a DHT provider record (via +`StartProviding`) so other peers can discover the content on this node. +Requires `Provide.Enabled`. + +**Security consideration**: DHT provider counts can be gamed via Sybil +attacks. An attacker could inflate provider counts to trick nodes into +unpinning content that is not actually well-replicated. The grace period +provides partial mitigation by requiring fake provider records to be +sustained for its full duration (default 72 hours, chosen to outlast the +48h DHT provider record validity so records of dead peers expire before +the unpin decision). + +### How to enable + +``` +ipfs config --json Experimental.OnDemandPinningEnabled true +``` + +### Configuring + +See [`OnDemandPinning`](https://github.com/ipfs/kubo/blob/master/docs/config.md#ondemandpinning) +for tunable parameters: `ReplicationTargetMin`, `ReplicationTargetMax`, +`CheckInterval`, `UnpinGracePeriod`, and `DryRun`. + +`ipfs pin ondemand ls` shows the last check time, provider count, result, and +computed unpin time. Set `OnDemandPinning.DryRun` to observe decisions without +changing the pinset. + +### Basic usage + +```bash +# Optional: evaluate without pinning/unpinning +ipfs config --json OnDemandPinning.DryRun true + +# Register a CID for on-demand monitoring +ipfs pin ondemand add QmExample + +# List registered CIDs (last check + unpin time; --live needs content routing) +ipfs pin ondemand ls +ipfs pin ondemand ls --live + +# Remove a CID from on-demand monitoring (also unpins if checker had pinned it) +ipfs pin ondemand rm QmExample +``` + +### Road to being a real feature + +- [ ] Needs more people to use and report on how well it works +- [ ] Needs integration tests in `test/cli/` +- [ ] Needs UI support in ipfs-webui / IPFS Desktop +- [ ] Evaluate Sybil resilience and whether additional mitigations are needed + ## Accelerated DHT Client This feature now lives at [`Routing.AcceleratedDHTClient`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient). diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go new file mode 100644 index 00000000000..5cbbc46f8fa --- /dev/null +++ b/ondemandpin/checker.go @@ -0,0 +1,428 @@ +package ondemandpin + +import ( + "context" + "errors" + "fmt" + "math/rand/v2" + "time" + + pin "github.com/ipfs/boxo/pinning/pinner" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "github.com/ipfs/kubo/config" + "github.com/libp2p/go-libp2p-kad-dht/amino" + peer "github.com/libp2p/go-libp2p/core/peer" + routing "github.com/libp2p/go-libp2p/core/routing" + mh "github.com/multiformats/go-multihash" +) + +var log = logging.Logger("ondemandpin") + +// OnDemandPinName is the pin name the checker uses when creating pins (Kubo specific. Other implementation may divert from this method). +// Pins carrying this name are considered managed by on-demand pinning and may be removed automatically when replication recovers. +// The name is part of the Kubo-internal "kubo:" namespace, which ValidatePinName refuses for user-supplied +// names, so only Kubo-internal code can create pins with this name. +const OnDemandPinName = "kubo:on-demand" + +// CheckTimeout bounds a single provider/pin-state lookup (checker and ls --live). +const CheckTimeout = 5 * time.Minute + +type PinService interface { + Pin(ctx context.Context, c cid.Cid, name string) error + Unpin(ctx context.Context, c cid.Cid) error + IsPinned(ctx context.Context, c cid.Cid) (bool, error) + HasPinWithName(ctx context.Context, c cid.Cid, name string) (bool, error) +} + +type StorageChecker interface { + StorageUsage(ctx context.Context) (used, limit uint64, err error) +} + +// Kubo's DHTProvider; may be a no-op. +type Provider interface { + StartProviding(force bool, keys ...mh.Multihash) error +} + +type Checker struct { + store *Store + pins PinService + storage StorageChecker + routing routing.ContentRouting + provider Provider + selfID peer.ID + + replicationMin int + replicationMax int + checkInterval time.Duration + unpinGracePeriod time.Duration + maxBackoff time.Duration + dryRun bool + + now func() time.Time + graceJitter func() time.Duration + priorityCh chan cid.Cid +} + +func NewChecker( + store *Store, + pins PinService, + storage StorageChecker, + cr routing.ContentRouting, + provider Provider, + selfID peer.ID, + cfg config.OnDemandPinning, +) *Checker { + c := &Checker{ + store: store, + pins: pins, + storage: storage, + routing: cr, + provider: provider, + selfID: selfID, + + replicationMin: int(cfg.ReplicationTargetMin.WithDefault(config.DefaultOnDemandPinReplicationTargetMin)), + replicationMax: int(cfg.ReplicationTargetMax.WithDefault(config.DefaultOnDemandPinReplicationTargetMax)), + checkInterval: cfg.CheckInterval.WithDefault(config.DefaultOnDemandPinCheckInterval), + unpinGracePeriod: cfg.UnpinGracePeriod.WithDefault(config.DefaultOnDemandPinUnpinGracePeriod), + maxBackoff: config.DefaultOnDemandPinCheckBackoffMax, + dryRun: cfg.DryRun.WithDefault(false), + + now: time.Now, + priorityCh: make(chan cid.Cid, 64), + } + c.graceJitter = c.defaultGraceJitter + return c +} + +func (c *Checker) defaultGraceJitter() time.Duration { + maxJitter := 2 * c.checkInterval + if maxJitter <= 0 { + return 0 + } + return time.Duration(rand.Int64N(int64(maxJitter))) +} + +func (c *Checker) Enqueue(ci cid.Cid) { + select { + case c.priorityCh <- ci: + default: + log.Warnw("priority queue full, CID will be checked in next regular cycle", "cid", ci) + } +} + +// Run blocks until ctx is cancelled. +func (c *Checker) Run(ctx context.Context) { + log.Info("on-demand pin checker started") + defer log.Info("on-demand pin checker stopped") + + // Warn when grace period is shorter than record validity (allowed for tests; risky on public DHT). + if c.unpinGracePeriod < amino.DefaultProvideValidity { + log.Warnw("UnpinGracePeriod is shorter than the DHT provider record validity; provider counts may include dead peers and this node may unpin the last live copy", + "gracePeriod", c.unpinGracePeriod, "recordValidity", amino.DefaultProvideValidity) + } + + ticker := time.NewTicker(c.checkInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case ci := <-c.priorityCh: + c.checkOne(ctx, ci) + case <-ticker.C: + c.checkAll(ctx) + } + } +} + +func (c *Checker) checkAll(ctx context.Context) { + records, err := c.store.List(ctx) + if err != nil { + log.Errorw("failed to list on-demand pins", "error", err) + return + } + + log.Infow("starting check cycle", "records", len(records)) + for _, rec := range records { + // Drain priority checks between records so Enqueue'd CIDs don't wait for a full sweep to complete. + select { + case ci := <-c.priorityCh: + c.checkOne(ctx, ci) + default: + } + if ctx.Err() != nil { + return + } + c.checkRecord(ctx, &rec, false) + } +} + +func (c *Checker) checkOne(ctx context.Context, ci cid.Cid) { + rec, err := c.store.Get(ctx, ci) + if err != nil { + log.Debugw("CID not in store, skipping", "cid", ci, "error", err) + return + } + // Ignore NextCheckAt so pin ondemand add is not delayed by a prior failure. + c.checkRecord(ctx, rec, true) +} + +// checkRecord pins below min, starts grace above max, clears grace in the deadband. +// immediate=true clears FailureCount/NextCheckAt before running. +// CheckTimeout covers DHT/pin-state lookup only; Pin uses ctx (daemon lifecycle). +func (c *Checker) checkRecord(ctx context.Context, rec *Record, immediate bool) { + if immediate { + rec.FailureCount = 0 + rec.NextCheckAt = time.Time{} + } else if !rec.NextCheckAt.IsZero() && c.now().Before(rec.NextCheckAt) { + return + } + + lookupCtx, cancel := context.WithTimeout(ctx, CheckTimeout) + defer cancel() + + pinned, err := c.pins.IsPinned(lookupCtx, rec.Cid) + if err != nil { + c.recordFailure(ctx, rec, fmt.Errorf("check pin state: %w", err)) + return + } + hasOnDemandPin, err := c.pins.HasPinWithName(lookupCtx, rec.Cid, OnDemandPinName) + if err != nil { + c.recordFailure(ctx, rec, fmt.Errorf("check pin name: %w", err)) + return + } + if pinned && !hasOnDemandPin { + log.Debugw("skipping: CID has a user-managed pin", "cid", rec.Cid) + rec.LastResult = "user-pin" + c.clearBackoff(ctx, rec) + return + } + + count, ok := CountProviders(lookupCtx, c.routing, c.selfID, rec.Cid, c.replicationMin, c.replicationMax) + if !ok { + rec.LastResult = "lookup-unknown" + c.recordFailure(ctx, rec, fmt.Errorf("provider count unknown")) + return + } + log.Debugw("provider count", "cid", rec.Cid, "count", count, "min", c.replicationMin, "max", c.replicationMax, "hasOnDemandPin", hasOnDemandPin) + + switch { + case count < c.replicationMin: + if err := c.handleUnderReplicated(ctx, lookupCtx, rec, count, hasOnDemandPin); err != nil { + c.recordFailure(ctx, rec, err) + return + } + case count > c.replicationMax: + if err := c.handleWellReplicated(ctx, lookupCtx, rec, count, hasOnDemandPin); err != nil { + c.recordFailure(ctx, rec, err) + return + } + default: + c.clearGrace(ctx, rec) + rec.LastResult = "deadband" + } + rec.LastProviderCount = count + c.clearBackoff(ctx, rec) +} + +// handleUnderReplicated pins the CID if it does not already have OnDemandPinName. +// lookupCtx is for quick pin-state checks; runCtx is for Pin/Provide/store (may outlast CheckTimeout). +func (c *Checker) handleUnderReplicated(runCtx, lookupCtx context.Context, rec *Record, count int, hasOnDemandPin bool) error { + if hasOnDemandPin { + c.clearGrace(runCtx, rec) + rec.LastResult = "holding" + return nil + } + + if !c.hasStorageBudget(runCtx) { + log.Warnw("skipping pin: repo near storage limit", "cid", rec.Cid) + rec.LastResult = "storage-limit" + return nil + } + + // Re-check: a user pin may have appeared during the provider lookup. + pinnedNow, err := c.pins.IsPinned(lookupCtx, rec.Cid) + if err != nil { + return fmt.Errorf("re-check pin state: %w", err) + } + if pinnedNow { + log.Debugw("skipping pin: CID gained a pin during provider lookup", "cid", rec.Cid) + rec.LastResult = "user-pin" + return nil + } + + if c.dryRun { + log.Infow("dry-run: would pin", "cid", rec.Cid, "providers", count, "min", c.replicationMin) + rec.LastResult = "would-pin" + return nil + } + + if err := c.pins.Pin(runCtx, rec.Cid, OnDemandPinName); err != nil { + return fmt.Errorf("pin: %w", err) + } + rec.LastAboveTarget = time.Time{} + rec.UnpinAt = time.Time{} + rec.LastResult = "pinned" + log.Infow("pinned", "cid", rec.Cid, "providers", count, "min", c.replicationMin) + + if err := c.provider.StartProviding(true, rec.Cid.Hash()); err != nil { + log.Warnw("failed to provide after pin", "cid", rec.Cid, "error", err) + } + c.saveRecord(runCtx, rec) + return nil +} + +func (c *Checker) handleWellReplicated(runCtx, lookupCtx context.Context, rec *Record, count int, hasOnDemandPin bool) error { + if !hasOnDemandPin { + rec.LastResult = "above-max" + return nil + } + + if rec.LastAboveTarget.IsZero() { + now := c.now() + jitter := c.graceJitter() + rec.LastAboveTarget = now + rec.UnpinAt = now.Add(c.unpinGracePeriod + jitter) + rec.LastResult = "grace" + c.saveRecord(runCtx, rec) + log.Debugw("grace period started", "cid", rec.Cid, "providers", count, "max", c.replicationMax, "unpinAt", rec.UnpinAt, "jitter", jitter) + return nil + } + + if c.now().Before(rec.UnpinAt) { + rec.LastResult = "grace" + return nil + } + + stillOnDemand, err := c.pins.HasPinWithName(lookupCtx, rec.Cid, OnDemandPinName) + if err != nil { + return fmt.Errorf("check pin name before unpin: %w", err) + } + + if stillOnDemand { + if c.dryRun { + log.Infow("dry-run: would unpin", "cid", rec.Cid, "providers", count, "max", c.replicationMax) + rec.LastResult = "would-unpin" + return nil + } + if err := c.pins.Unpin(runCtx, rec.Cid); err != nil { + return fmt.Errorf("unpin: %w", err) + } + log.Infow("unpinned", "cid", rec.Cid, "providers", count, "max", c.replicationMax) + rec.LastResult = "unpinned" + } else { + log.Infow("relinquishing management: pin name changed externally", "cid", rec.Cid) + rec.LastResult = "released" + } + + rec.LastAboveTarget = time.Time{} + rec.UnpinAt = time.Time{} + c.saveRecord(runCtx, rec) + return nil +} + +func (c *Checker) clearGrace(ctx context.Context, rec *Record) { + if rec.LastAboveTarget.IsZero() && rec.UnpinAt.IsZero() { + return + } + rec.LastAboveTarget = time.Time{} + rec.UnpinAt = time.Time{} + c.saveRecord(ctx, rec) +} + +func (c *Checker) clearBackoff(ctx context.Context, rec *Record) { + rec.LastCheckedAt = c.now() + rec.FailureCount = 0 + rec.NextCheckAt = time.Time{} + c.saveRecord(ctx, rec) +} + +func (c *Checker) recordFailure(ctx context.Context, rec *Record, cause error) { + rec.LastCheckedAt = c.now() + if rec.LastResult == "" { + rec.LastResult = "error" + } + rec.FailureCount++ + delay := c.backoffDelay(rec.FailureCount) + rec.NextCheckAt = c.now().Add(delay) + log.Warnw("check failed", "cid", rec.Cid, "error", cause, "failures", rec.FailureCount, "nextCheckAt", rec.NextCheckAt) + c.saveRecord(ctx, rec) +} + +func (c *Checker) backoffDelay(failures int) time.Duration { + if failures < 1 { + return c.checkInterval + } + d := c.checkInterval + for i := 1; i < failures; i++ { + if d >= c.maxBackoff/2 { + return c.maxBackoff + } + d *= 2 + } + if d > c.maxBackoff { + return c.maxBackoff + } + return d +} + +func (c *Checker) saveRecord(ctx context.Context, rec *Record) { + if err := c.store.Update(ctx, rec); err != nil { + if errors.Is(err, ErrNotRegistered) { + log.Debugw("record gone during check, not recreating", "cid", rec.Cid) + return + } + log.Errorw("failed to update record", "cid", rec.Cid, "error", err) + } +} + +func (c *Checker) hasStorageBudget(ctx context.Context) bool { + if c.storage == nil { + return true + } + used, limit, err := c.storage.StorageUsage(ctx) + if err != nil { + log.Warnw("failed to check storage usage, proceeding with pin", "error", err) + return true + } + if limit == 0 { + return true + } + return used < limit +} + +// CountProviders counts providers excluding self. Asks for max+2 results so +// self can take a slot and we can still see max+1 others. +// ok is false if the lookup was cancelled before reaching min providers. +func CountProviders(ctx context.Context, cr routing.ContentRouting, selfID peer.ID, c cid.Cid, min, max int) (count int, ok bool) { + ch := cr.FindProvidersAsync(ctx, c, max+2) + + seen := make(map[peer.ID]struct{}) + for pi := range ch { + if pi.ID == selfID { + continue + } + seen[pi.ID] = struct{}{} + } + count = len(seen) + if ctx.Err() != nil && count < min { + return count, false + } + return count, true +} + +// PinHasName is used by checker (via PinService.HasPinWithName) and the rm command to identify pins managed by on-demand pinning. +func PinHasName(ctx context.Context, p pin.Pinner, c cid.Cid, name string) (bool, error) { + results, err := p.CheckIfPinnedWithType(ctx, pin.Recursive, true, c) + if err != nil { + return false, err + } + for _, r := range results { + if r.Pinned() && r.Name == name { + return true, nil + } + } + return false, nil +} diff --git a/ondemandpin/checker_test.go b/ondemandpin/checker_test.go new file mode 100644 index 00000000000..0a69bf9dca5 --- /dev/null +++ b/ondemandpin/checker_test.go @@ -0,0 +1,467 @@ +package ondemandpin + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + "github.com/ipfs/kubo/config" + peer "github.com/libp2p/go-libp2p/core/peer" + mh "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeClock struct{ t atomic.Int64 } + +func newFakeClock() *fakeClock { + c := &fakeClock{} + c.t.Store(time.Now().UnixNano()) + return c +} + +func (c *fakeClock) Now() time.Time { return time.Unix(0, c.t.Load()) } +func (c *fakeClock) Advance(d time.Duration) { c.t.Add(int64(d)) } + +type mockRouting struct { + mu sync.Mutex + providers map[cid.Cid][]peer.AddrInfo +} + +func newMockRouting() *mockRouting { + return &mockRouting{providers: make(map[cid.Cid][]peer.AddrInfo)} +} + +func (m *mockRouting) setProviders(c cid.Cid, peerIDs ...peer.ID) { + m.mu.Lock() + defer m.mu.Unlock() + infos := make([]peer.AddrInfo, len(peerIDs)) + for i, pid := range peerIDs { + infos[i] = peer.AddrInfo{ID: pid} + } + m.providers[c] = infos +} + +func (m *mockRouting) FindProvidersAsync(ctx context.Context, c cid.Cid, limit int) <-chan peer.AddrInfo { + ch := make(chan peer.AddrInfo) + go func() { + defer close(ch) + m.mu.Lock() + provs := m.providers[c] + m.mu.Unlock() + for i, pi := range provs { + if i >= limit { + break + } + select { + case ch <- pi: + case <-ctx.Done(): + return + } + } + }() + return ch +} + +func (m *mockRouting) Provide(context.Context, cid.Cid, bool) error { return nil } + +type mockProvider struct { + keys []mh.Multihash +} + +func (m *mockProvider) StartProviding(_ bool, keys ...mh.Multihash) error { + m.keys = append(m.keys, keys...) + return nil +} + +type mockPins struct { + pinned map[cid.Cid]string +} + +func newMockPins() *mockPins { return &mockPins{pinned: make(map[cid.Cid]string)} } + +func (m *mockPins) Pin(_ context.Context, c cid.Cid, name string) error { + m.pinned[c] = name + return nil +} + +func (m *mockPins) Unpin(_ context.Context, c cid.Cid) error { + delete(m.pinned, c) + return nil +} + +func (m *mockPins) IsPinned(_ context.Context, c cid.Cid) (bool, error) { + _, ok := m.pinned[c] + return ok, nil +} + +func (m *mockPins) HasPinWithName(_ context.Context, c cid.Cid, name string) (bool, error) { + n, ok := m.pinned[c] + return ok && n == name, nil +} + +func (m *mockPins) isPinned(c cid.Cid) bool { + _, ok := m.pinned[c] + return ok +} + +func newTestChecker(t *testing.T) (*Checker, *Store, *mockRouting, *mockPins, *fakeClock, *mockProvider) { + t.Helper() + store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) + r := newMockRouting() + p := newMockPins() + prov := &mockProvider{} + clock := newFakeClock() + + checker := NewChecker(store, p, nil, r, prov, peer.ID("self"), config.OnDemandPinning{}) + checker.checkInterval = time.Minute + checker.unpinGracePeriod = 200 * time.Millisecond + checker.now = clock.Now + checker.graceJitter = func() time.Duration { return 0 } + + return checker, store, r, p, clock, prov +} + +func providers(n int) []peer.ID { + out := make([]peer.ID, n) + for i := range out { + out[i] = peer.ID(string(rune('a' + i))) + } + return out +} + +// Under-replicated content gets pinned. +func TestCheckerPinsBelowTarget(t *testing.T) { + ctx := context.Background() + checker, store, r, p, _, prov := newTestChecker(t) + c := testCID(t, "under-replicated") + + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, peer.ID("p1"), peer.ID("p2")) + + checker.checkAll(ctx) + + assert.True(t, p.isPinned(c)) + rec := mustGet(t, store, c) + assert.Equal(t, "pinned", rec.LastResult) + assert.Equal(t, 2, rec.LastProviderCount) + assert.False(t, rec.LastCheckedAt.IsZero()) + require.Equal(t, []mh.Multihash{c.Hash()}, prov.keys) +} + +func TestCheckerDoesNotPinInDeadband(t *testing.T) { + ctx := context.Background() + checker, store, r, p, _, _ := newTestChecker(t) + c := testCID(t, "deadband") + + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, providers(6)...) // min=5, max=7 + + checker.checkAll(ctx) + + assert.False(t, p.isPinned(c)) +} + +func TestCheckerUnpinsAfterGracePeriod(t *testing.T) { + ctx := context.Background() + checker, store, r, p, clock, _ := newTestChecker(t) + c := testCID(t, "recovering") + + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, peer.ID("p1")) + checker.checkAll(ctx) + require.True(t, p.isPinned(c)) + + // Providers recover above max (default 7). + r.setProviders(c, providers(8)...) + checker.checkAll(ctx) + assert.True(t, p.isPinned(c), "not yet past grace period") + + clock.Advance(250 * time.Millisecond) + checker.checkAll(ctx) + assert.False(t, p.isPinned(c), "past grace period") +} + +func TestCheckerGraceIncludesJitter(t *testing.T) { + ctx := context.Background() + checker, store, r, p, clock, _ := newTestChecker(t) + checker.graceJitter = func() time.Duration { return 100 * time.Millisecond } + c := testCID(t, "jitter") + + require.NoError(t, store.Add(ctx, c)) + require.NoError(t, p.Pin(ctx, c, OnDemandPinName)) + r.setProviders(c, providers(8)...) + + checker.checkAll(ctx) + assert.True(t, p.isPinned(c)) + + clock.Advance(250 * time.Millisecond) // grace only; jitter not yet elapsed + checker.checkAll(ctx) + assert.True(t, p.isPinned(c), "still within grace+jitter") + + clock.Advance(100 * time.Millisecond) + checker.checkAll(ctx) + assert.False(t, p.isPinned(c), "past grace+jitter") +} + +type pinDuringLookupRouting struct { + *mockRouting + pins *mockPins + pinName string +} + +func (r *pinDuringLookupRouting) FindProvidersAsync(ctx context.Context, c cid.Cid, limit int) <-chan peer.AddrInfo { + r.pins.pinned[c] = r.pinName + return r.mockRouting.FindProvidersAsync(ctx, c, limit) +} + +// Pin that appeared during the DHT lookup is left alone. +func TestCheckerSkipsPinCreatedDuringLookup(t *testing.T) { + ctx := context.Background() + store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) + r := newMockRouting() + p := newMockPins() + racing := &pinDuringLookupRouting{mockRouting: r, pins: p, pinName: "user-pin"} + checker := NewChecker(store, p, nil, racing, &mockProvider{}, peer.ID("self"), config.OnDemandPinning{}) + checker.now = newFakeClock().Now + + c := testCID(t, "raced") + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, peer.ID("p1")) + + checker.checkAll(ctx) + + assert.Equal(t, "user-pin", p.pinned[c]) +} + +func TestCheckerOwnsPinByNameNotStoreField(t *testing.T) { + ctx := context.Background() + checker, store, r, p, clock, _ := newTestChecker(t) + c := testCID(t, "name-owned") + + require.NoError(t, store.Add(ctx, c)) + require.NoError(t, p.Pin(ctx, c, OnDemandPinName)) + r.setProviders(c, providers(8)...) + + checker.checkAll(ctx) + assert.True(t, p.isPinned(c), "grace period just started") + + clock.Advance(250 * time.Millisecond) + checker.checkAll(ctx) + assert.False(t, p.isPinned(c), "name-owned pin must unpin after grace") +} + +type blockingRouting struct{} + +func (blockingRouting) FindProvidersAsync(ctx context.Context, _ cid.Cid, _ int) <-chan peer.AddrInfo { + ch := make(chan peer.AddrInfo) + go func() { + defer close(ch) + <-ctx.Done() + }() + return ch +} + +func (blockingRouting) Provide(context.Context, cid.Cid, bool) error { return nil } + +func TestCountProvidersUnknownOnCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + count, ok := CountProviders(ctx, blockingRouting{}, peer.ID("self"), testCID(t, "unknown"), 5, 7) + assert.False(t, ok) + assert.Equal(t, 0, count) +} + +type emitThenBlockRouting struct { + providers []peer.ID +} + +func (r emitThenBlockRouting) FindProvidersAsync(ctx context.Context, _ cid.Cid, limit int) <-chan peer.AddrInfo { + ch := make(chan peer.AddrInfo) + go func() { + defer close(ch) + for i, id := range r.providers { + if i >= limit { + break + } + select { + case ch <- peer.AddrInfo{ID: id}: + case <-ctx.Done(): + return + } + } + <-ctx.Done() + }() + return ch +} + +func (emitThenBlockRouting) Provide(context.Context, cid.Cid, bool) error { return nil } + +func TestCountProvidersOkWhenEnoughFoundDespiteCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + r := emitThenBlockRouting{providers: []peer.ID{"p1", "p2", "p3", "p4", "p5"}} + + done := make(chan struct{}) + var count int + var ok bool + go func() { + defer close(done) + count, ok = CountProviders(ctx, r, peer.ID("self"), testCID(t, "enough"), 5, 7) + }() + + time.Sleep(20 * time.Millisecond) // let providers flush + cancel() + <-done + + require.True(t, ok) + assert.Equal(t, 5, count) +} + +func TestCheckerSkipsWhenProviderCountUnknown(t *testing.T) { + ctx := context.Background() + store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) + p := newMockPins() + checker := NewChecker(store, p, nil, blockingRouting{}, &mockProvider{}, peer.ID("self"), config.OnDemandPinning{}) + checker.now = newFakeClock().Now + + c := testCID(t, "hung-lookup") + require.NoError(t, store.Add(ctx, c)) + + checkCtx, cancel := context.WithTimeout(ctx, 20*time.Millisecond) + defer cancel() + checker.checkRecord(checkCtx, mustGet(t, store, c), false) + + assert.False(t, p.isPinned(c)) + rec := mustGet(t, store, c) + assert.Equal(t, 1, rec.FailureCount) + assert.False(t, rec.NextCheckAt.IsZero()) + assert.Equal(t, "lookup-unknown", rec.LastResult) +} + +type failPinOnce struct { + *mockPins + fail bool +} + +func (p *failPinOnce) Pin(ctx context.Context, c cid.Cid, name string) error { + if p.fail { + return errors.New("pin failed") + } + return p.mockPins.Pin(ctx, c, name) +} + +type capturePinCtx struct { + *mockPins + pinCtx context.Context +} + +func (p *capturePinCtx) Pin(ctx context.Context, c cid.Cid, name string) error { + p.pinCtx = ctx + return p.mockPins.Pin(ctx, c, name) +} + +func TestPinContextHasNoCheckTimeout(t *testing.T) { + ctx := context.Background() + store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) + r := newMockRouting() + pins := &capturePinCtx{mockPins: newMockPins()} + checker := NewChecker(store, pins, nil, r, &mockProvider{}, peer.ID("self"), config.OnDemandPinning{}) + checker.now = newFakeClock().Now + checker.graceJitter = func() time.Duration { return 0 } + + c := testCID(t, "pin-ctx") + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, peer.ID("p1")) + + checker.checkAll(ctx) + require.NotNil(t, pins.pinCtx) + _, hasDeadline := pins.pinCtx.Deadline() + assert.False(t, hasDeadline) +} + +func TestCheckerBackoffSkipsUntilDue(t *testing.T) { + ctx := context.Background() + store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) + r := newMockRouting() + pins := &failPinOnce{mockPins: newMockPins(), fail: true} + clock := newFakeClock() + checker := NewChecker(store, pins, nil, r, &mockProvider{}, peer.ID("self"), config.OnDemandPinning{}) + checker.checkInterval = time.Minute + checker.now = clock.Now + checker.graceJitter = func() time.Duration { return 0 } + + c := testCID(t, "backoff") + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, peer.ID("p1")) + + checker.checkAll(ctx) + assert.False(t, pins.isPinned(c)) + rec := mustGet(t, store, c) + require.Equal(t, 1, rec.FailureCount) + require.Equal(t, clock.Now().Add(time.Minute), rec.NextCheckAt) + + pins.fail = false + checker.checkAll(ctx) + assert.False(t, pins.isPinned(c)) + + clock.Advance(time.Minute) + checker.checkAll(ctx) + assert.True(t, pins.isPinned(c)) + rec = mustGet(t, store, c) + assert.Equal(t, 0, rec.FailureCount) + assert.True(t, rec.NextCheckAt.IsZero()) + assert.Equal(t, "pinned", rec.LastResult) +} + +func TestCheckerDryRunDoesNotPinOrUnpin(t *testing.T) { + ctx := context.Background() + store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) + r := newMockRouting() + p := newMockPins() + clock := newFakeClock() + checker := NewChecker(store, p, nil, r, &mockProvider{}, peer.ID("self"), config.OnDemandPinning{ + DryRun: config.True, + }) + checker.checkInterval = time.Minute + checker.unpinGracePeriod = 200 * time.Millisecond + checker.now = clock.Now + checker.graceJitter = func() time.Duration { return 0 } + + c := testCID(t, "dry-run") + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, peer.ID("p1"), peer.ID("p2")) + + checker.checkAll(ctx) + assert.False(t, p.isPinned(c)) + rec := mustGet(t, store, c) + assert.Equal(t, "would-pin", rec.LastResult) + assert.Equal(t, 2, rec.LastProviderCount) + + require.NoError(t, p.Pin(ctx, c, OnDemandPinName)) + r.setProviders(c, providers(8)...) + checker.checkAll(ctx) + assert.True(t, p.isPinned(c)) + rec = mustGet(t, store, c) + assert.Equal(t, "grace", rec.LastResult) + require.False(t, rec.UnpinAt.IsZero()) + + clock.Advance(250 * time.Millisecond) + checker.checkAll(ctx) + assert.True(t, p.isPinned(c), "dry-run must not unpin") + rec = mustGet(t, store, c) + assert.Equal(t, "would-unpin", rec.LastResult) +} + +func mustGet(t *testing.T, store *Store, c cid.Cid) *Record { + t.Helper() + rec, err := store.Get(context.Background(), c) + require.NoError(t, err) + return rec +} diff --git a/ondemandpin/store.go b/ondemandpin/store.go new file mode 100644 index 00000000000..7e015097dbc --- /dev/null +++ b/ondemandpin/store.go @@ -0,0 +1,159 @@ +package ondemandpin + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" +) + +var ( + ErrAlreadyRegistered = errors.New("CID is already registered for on-demand pinning") + ErrNotRegistered = errors.New("CID is not registered for on-demand pinning") +) + +const dsPrefix = "/ondemand-pins/" + +type Record struct { + Cid cid.Cid `json:"Cid"` + LastAboveTarget time.Time `json:"LastAboveTarget,omitempty"` + UnpinAt time.Time `json:"UnpinAt,omitempty"` // grace deadline (includes jitter) + FailureCount int `json:"FailureCount,omitempty"` + NextCheckAt time.Time `json:"NextCheckAt,omitempty"` + LastCheckedAt time.Time `json:"LastCheckedAt,omitempty"` + LastProviderCount int `json:"LastProviderCount,omitempty"` + LastResult string `json:"LastResult,omitempty"` + CreatedAt time.Time `json:"CreatedAt"` +} + +type Store struct { + ds datastore.Batching + mu sync.RWMutex + now func() time.Time +} + +func NewStore(ds datastore.Batching) *Store { + return &Store{ds: ds, now: time.Now} +} + +func dsKey(c cid.Cid) datastore.Key { + return datastore.NewKey(dsPrefix + c.String()) +} + +func (s *Store) Add(ctx context.Context, c cid.Cid) error { + s.mu.Lock() + defer s.mu.Unlock() + + key := dsKey(c) + + if has, err := s.ds.Has(ctx, key); err != nil { + return fmt.Errorf("checking existing record: %w", err) + } else if has { + return fmt.Errorf("%s: %w", c, ErrAlreadyRegistered) + } + + rec := Record{ + Cid: c, + CreatedAt: s.now(), + } + + return s.put(ctx, key, &rec) +} + +func (s *Store) Remove(ctx context.Context, c cid.Cid) error { + s.mu.Lock() + defer s.mu.Unlock() + + key := dsKey(c) + if has, err := s.ds.Has(ctx, key); err != nil { + return fmt.Errorf("checking record: %w", err) + } else if !has { + return fmt.Errorf("%s: %w", c, ErrNotRegistered) + } + + if err := s.ds.Delete(ctx, key); err != nil { + return fmt.Errorf("deleting record: %w", err) + } + return s.ds.Sync(ctx, key) +} + +func (s *Store) Get(ctx context.Context, c cid.Cid) (*Record, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + rec, err := s.get(ctx, dsKey(c)) + if errors.Is(err, datastore.ErrNotFound) { + return nil, fmt.Errorf("%s: %w", c, ErrNotRegistered) + } + return rec, err +} + +func (s *Store) List(ctx context.Context) ([]Record, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + results, err := s.ds.Query(ctx, query.Query{ + Prefix: dsPrefix, + }) + if err != nil { + return nil, fmt.Errorf("querying on-demand pins: %w", err) + } + defer results.Close() + + var records []Record + for result := range results.Next() { + if result.Error != nil { + return nil, fmt.Errorf("iterating on-demand pins: %w", result.Error) + } + var rec Record + if err := json.Unmarshal(result.Value, &rec); err != nil { + return nil, fmt.Errorf("unmarshaling record %s: %w", result.Key, err) + } + records = append(records, rec) + } + return records, nil +} + +func (s *Store) Update(ctx context.Context, rec *Record) error { + s.mu.Lock() + defer s.mu.Unlock() + + key := dsKey(rec.Cid) + has, err := s.ds.Has(ctx, key) + if err != nil { + return fmt.Errorf("checking record: %w", err) + } + if !has { + return fmt.Errorf("%s: %w", rec.Cid, ErrNotRegistered) + } + return s.put(ctx, key, rec) +} + +func (s *Store) get(ctx context.Context, key datastore.Key) (*Record, error) { + val, err := s.ds.Get(ctx, key) + if err != nil { + return nil, fmt.Errorf("getting record: %w", err) + } + var rec Record + if err := json.Unmarshal(val, &rec); err != nil { + return nil, fmt.Errorf("unmarshaling record: %w", err) + } + return &rec, nil +} + +func (s *Store) put(ctx context.Context, key datastore.Key, rec *Record) error { + val, err := json.Marshal(rec) + if err != nil { + return fmt.Errorf("marshaling record: %w", err) + } + if err := s.ds.Put(ctx, key, val); err != nil { + return fmt.Errorf("storing record: %w", err) + } + return s.ds.Sync(ctx, key) +} diff --git a/ondemandpin/store_test.go b/ondemandpin/store_test.go new file mode 100644 index 00000000000..11ff719f9b3 --- /dev/null +++ b/ondemandpin/store_test.go @@ -0,0 +1,82 @@ +package ondemandpin + +import ( + "context" + "testing" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + mh "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testCID(t *testing.T, data string) cid.Cid { + t.Helper() + h, err := mh.Sum([]byte(data), mh.SHA2_256, -1) + require.NoError(t, err) + return cid.NewCidV1(cid.Raw, h) +} + +func newTestStore(t *testing.T) *Store { + t.Helper() + return NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) +} + +// Add, Get, Remove lifecycle works. +func TestStoreRoundTrip(t *testing.T) { + ctx := context.Background() + s := newTestStore(t) + c := testCID(t, "hello") + + require.NoError(t, s.Add(ctx, c)) + + rec, err := s.Get(ctx, c) + require.NoError(t, err) + assert.Equal(t, c, rec.Cid) + + require.NoError(t, s.Remove(ctx, c)) + + _, err = s.Get(ctx, c) + assert.ErrorIs(t, err, ErrNotRegistered) +} + +// Get on a CID that was never registered returns ErrNotRegistered. +func TestStoreGetNotRegistered(t *testing.T) { + ctx := context.Background() + s := newTestStore(t) + + _, err := s.Get(ctx, testCID(t, "never-registered")) + assert.ErrorIs(t, err, ErrNotRegistered) +} + +// List returns all registered records. +func TestStoreList(t *testing.T) { + ctx := context.Background() + s := newTestStore(t) + + require.NoError(t, s.Add(ctx, testCID(t, "a"))) + require.NoError(t, s.Add(ctx, testCID(t, "b"))) + + records, err := s.List(ctx) + require.NoError(t, err) + assert.Len(t, records, 2) +} + +func TestStoreUpdateDoesNotResurrect(t *testing.T) { + ctx := context.Background() + s := newTestStore(t) + c := testCID(t, "gone") + + require.NoError(t, s.Add(ctx, c)) + rec, err := s.Get(ctx, c) + require.NoError(t, err) + + require.NoError(t, s.Remove(ctx, c)) + err = s.Update(ctx, rec) + assert.ErrorIs(t, err, ErrNotRegistered) + + _, err = s.Get(ctx, c) + assert.ErrorIs(t, err, ErrNotRegistered) +}