From 85e21f7a3b66ddac12b400dfef0b0b31ec0f6975 Mon Sep 17 00:00:00 2001 From: ihlec Date: Thu, 26 Mar 2026 11:14:02 +0100 Subject: [PATCH 01/17] feat: add experimental on-demand pinning Add a background checker that automatically pins content when DHT provider counts fall below a configurable replication target and unpins once enough providers exist again after a grace period. Gated behind Experimental.OnDemandPinningEnabled. New CLI commands: ipfs pin ondemand {add,rm,ls} Safety measures: - storage budget check (respects StorageMax/GCWatermark) - idle timeout on recursive DAG fetches (2 min without progress) - pin partitioning via pin name to distinguish on-demand pins from persitent pins. --- config/config.go | 2 + config/experiments.go | 1 + config/ondemandpin.go | 20 +++ core/commands/commands_test.go | 4 + core/commands/pin/ondemandpin.go | 280 ++++++++++++++++++++++++++++++ core/commands/pin/pin.go | 2 + core/core.go | 4 + core/node/groups.go | 5 + core/node/ondemandpin.go | 168 ++++++++++++++++++ docs/config.md | 52 ++++++ docs/experimental-features.md | 62 +++++++ ondemandpin/checker.go | 282 +++++++++++++++++++++++++++++++ ondemandpin/checker_test.go | 165 ++++++++++++++++++ ondemandpin/store.go | 143 ++++++++++++++++ ondemandpin/store_test.go | 57 +++++++ 15 files changed, 1247 insertions(+) create mode 100644 config/ondemandpin.go create mode 100644 core/commands/pin/ondemandpin.go create mode 100644 core/node/ondemandpin.go create mode 100644 ondemandpin/checker.go create mode 100644 ondemandpin/checker_test.go create mode 100644 ondemandpin/store.go create mode 100644 ondemandpin/store_test.go diff --git a/config/config.go b/config/config.go index 0952e0d3173..c8432dd6d37 100644 --- a/config/config.go +++ b/config/config.go @@ -45,6 +45,8 @@ type Config struct { Import Import Version Version + OnDemandPinning OnDemandPinning + Internal Internal // experimental/unstable options Bitswap Bitswap diff --git a/config/experiments.go b/config/experiments.go index 6c43ac04f07..4cf3af86e76 100644 --- a/config/experiments.go +++ b/config/experiments.go @@ -10,6 +10,7 @@ type Experiments struct { OptimisticProvide bool OptimisticProvideJobsPoolSize int GatewayOverLibp2p bool `json:",omitempty"` + OnDemandPinningEnabled bool GraphsyncEnabled graphsyncEnabled `json:",omitempty"` AcceleratedDHTClient experimentalAcceleratedDHTClient `json:",omitempty"` diff --git a/config/ondemandpin.go b/config/ondemandpin.go new file mode 100644 index 00000000000..4b072e94d32 --- /dev/null +++ b/config/ondemandpin.go @@ -0,0 +1,20 @@ +package config + +import "time" + +const ( + DefaultOnDemandPinReplicationTarget = 5 + DefaultOnDemandPinCheckInterval = 10 * time.Minute + DefaultOnDemandPinUnpinGracePeriod = 24 * time.Hour +) + +type OnDemandPinning struct { + // Minimum providers desired in the DHT (excluding self). + ReplicationTarget OptionalInteger + + // How often the checker evaluates all registered CIDs. + CheckInterval OptionalDuration + + // How long replication must stay above target before unpinning. + UnpinGracePeriod OptionalDuration +} diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 22df7d5105c..fe7603db5f5 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -168,6 +168,10 @@ func TestCommands(t *testing.T) { "/pin/remote/service/add", "/pin/remote/service/ls", "/pin/remote/service/rm", + "/pin/ondemand", + "/pin/ondemand/add", + "/pin/ondemand/rm", + "/pin/ondemand/ls", "/pin/rm", "/pin/update", "/pin/verify", diff --git a/core/commands/pin/ondemandpin.go b/core/commands/pin/ondemandpin.go new file mode 100644 index 00000000000..7a5d07ddbcf --- /dev/null +++ b/core/commands/pin/ondemandpin.go @@ -0,0 +1,280 @@ +package pin + +import ( + "fmt" + "io" + "time" + + cmds "github.com/ipfs/go-ipfs-cmds" + "github.com/ipfs/kubo/config" + cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" + "github.com/ipfs/kubo/core/commands/cmdutils" + "github.com/ipfs/kubo/ondemandpin" +) + +const onDemandLiveOptionName = "live" + +var onDemandPinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Manage on-demand pins.", + ShortDescription: ` + On-demand pinning automatically pins content when few target providers exist in the DHT, + and unpins when replication has been above target for a grace period. + Requires Experimental.OnDemandPinningEnabled = true. + `, + }, + Subcommands: map[string]*cmds.Command{ + "add": addOnDemandPinCmd, + "rm": rmOnDemandPinCmd, + "ls": listOnDemandPinCmd, + }, +} + +type OnDemandPinOutput struct { + Cid string `json:"Cid"` +} + +var addOnDemandPinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Register CIDs for on-demand pinning.", + ShortDescription: `Adds the given CID(s) to the on-demand pin registry. The background checker will evaluate replication and pin if needed.`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("cid", true, true, "CID(s) to register."), + }, + Type: OnDemandPinOutput{}, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + n, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + cfg, err := n.Repo.Config() + if err != nil { + return err + } + if !cfg.Experimental.OnDemandPinningEnabled { + return fmt.Errorf("on-demand pinning is not enabled; set Experimental.OnDemandPinningEnabled = true in config") + } + + store := n.OnDemandPinStore + + api, err := cmdenv.GetApi(env, req) + if err != nil { + return err + } + + for _, arg := range req.Arguments { + p, err := cmdutils.PathOrCidPath(arg) + if err != nil { + return fmt.Errorf("invalid CID or path %q: %w", arg, err) + } + + rp, _, err := api.ResolvePath(req.Context, p) + if err != nil { + return fmt.Errorf("resolving %q: %w", arg, err) + } + c := rp.RootCid() + + if err := store.Add(req.Context, c); err != nil { + return err + } + + if checker := n.OnDemandPinChecker; checker != nil { + checker.Enqueue(c) + } + + if err := res.Emit(&OnDemandPinOutput{ + Cid: c.String(), + }); err != nil { + return err + } + } + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OnDemandPinOutput) error { + fmt.Fprintf(w, "registered %s for on-demand pinning\n", out.Cid) + return nil + }), + }, +} + +var rmOnDemandPinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Remove CIDs from on-demand pinning.", + ShortDescription: ` + Removes the given CID(s) from the on-demand pin registry. + If the content was pinned by the checker, it is also unpinned. + Works even when on-demand pinning is disabled, to clean up previously registered CIDs. + `, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("cid", true, true, "CID(s) to remove."), + }, + Type: OnDemandPinOutput{}, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + n, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + store := n.OnDemandPinStore + + api, err := cmdenv.GetApi(env, req) + if err != nil { + return err + } + + for _, arg := range req.Arguments { + p, err := cmdutils.PathOrCidPath(arg) + if err != nil { + return fmt.Errorf("invalid CID or path %q: %w", arg, err) + } + + rp, _, err := api.ResolvePath(req.Context, p) + if err != nil { + return fmt.Errorf("resolving %q: %w", arg, err) + } + c := rp.RootCid() + + isOurs, err := ondemandpin.PinHasName(req.Context, n.Pinning, c, ondemandpin.OnDemandPinName) + if err != nil { + return fmt.Errorf("checking pin state for %s: %w", c, err) + } + if isOurs { + if err := api.Pin().Rm(req.Context, rp); err != nil { + return fmt.Errorf("unpinning %s: %w", c, err) + } + } + + if err := store.Remove(req.Context, c); err != nil { + return err + } + + if err := res.Emit(&OnDemandPinOutput{ + Cid: c.String(), + }); err != nil { + return err + } + } + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OnDemandPinOutput) error { + fmt.Fprintf(w, "removed %s from on-demand pinning\n", out.Cid) + return nil + }), + }, +} + +type OnDemandLsOutput struct { + Cid string `json:"Cid"` + PinnedByUs bool `json:"PinnedByUs"` + Providers *int `json:"Providers,omitempty"` + LastAboveTarget string `json:"LastAboveTarget,omitempty"` + CreatedAt string `json:"CreatedAt"` +} + +var listOnDemandPinCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List on-demand pins.", + ShortDescription: ` + Lists CIDs registered for on-demand pinning with their current state. + Use --live to include real-time provider counts from the DHT. + `, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("cid", false, true, "Optional CID(s) to filter."), + }, + Options: []cmds.Option{ + cmds.BoolOption(onDemandLiveOptionName, "l", "Perform live provider lookup."), + }, + Type: OnDemandLsOutput{}, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + n, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + store := n.OnDemandPinStore + + live, _ := req.Options[onDemandLiveOptionName].(bool) + + var globalTarget int + if live { + cfg, err := n.Repo.Config() + if err != nil { + return err + } + globalTarget = int(cfg.OnDemandPinning.ReplicationTarget.WithDefault(config.DefaultOnDemandPinReplicationTarget)) + } + + var records []ondemandpin.Record + if len(req.Arguments) > 0 { + api, err := cmdenv.GetApi(env, req) + if err != nil { + return err + } + for _, arg := range req.Arguments { + p, err := cmdutils.PathOrCidPath(arg) + if err != nil { + return fmt.Errorf("invalid CID or path %q: %w", arg, err) + } + rp, _, err := api.ResolvePath(req.Context, p) + if err != nil { + return fmt.Errorf("resolving %q: %w", arg, err) + } + rec, err := store.Get(req.Context, rp.RootCid()) + if err != nil { + return err + } + records = append(records, *rec) + } + } else { + records, err = store.List(req.Context) + if err != nil { + return err + } + } + + for _, rec := range records { + out := OnDemandLsOutput{ + Cid: rec.Cid.String(), + PinnedByUs: rec.PinnedByUs, + CreatedAt: rec.CreatedAt.Format(time.RFC3339), + } + if !rec.LastAboveTarget.IsZero() { + out.LastAboveTarget = rec.LastAboveTarget.Format(time.RFC3339) + } + + if live && n.Routing != nil { + count := ondemandpin.CountProviders(req.Context, n.Routing, n.Identity, rec.Cid, globalTarget) + out.Providers = &count + } + + if err := res.Emit(&out); err != nil { + return err + } + } + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OnDemandLsOutput) error { + pinState := "not-pinned" + if out.PinnedByUs { + pinState = "pinned" + } + fmt.Fprintf(w, "%s", out.Cid) + if out.Providers != nil { + fmt.Fprintf(w, " providers=%d", *out.Providers) + } + fmt.Fprintf(w, " %s created=%s", pinState, out.CreatedAt) + if out.LastAboveTarget != "" { + fmt.Fprintf(w, " above-target-since=%s", out.LastAboveTarget) + } + fmt.Fprintln(w) + return nil + }), + }, +} diff --git a/core/commands/pin/pin.go b/core/commands/pin/pin.go index 1810f1b5a7b..55e0a1301a1 100644 --- a/core/commands/pin/pin.go +++ b/core/commands/pin/pin.go @@ -39,6 +39,8 @@ var PinCmd = &cmds.Command{ "verify": verifyPinCmd, "update": updatePinCmd, "remote": remotePinCmd, + + "ondemand": onDemandPinCmd, }, } diff --git a/core/core.go b/core/core.go index db8c4d9a66e..c552715c951 100644 --- a/core/core.go +++ b/core/core.go @@ -57,6 +57,7 @@ import ( "github.com/ipfs/kubo/core/node" "github.com/ipfs/kubo/core/node/libp2p" "github.com/ipfs/kubo/fuse/mount" + "github.com/ipfs/kubo/ondemandpin" "github.com/ipfs/kubo/p2p" "github.com/ipfs/kubo/repo" irouting "github.com/ipfs/kubo/routing" @@ -77,6 +78,9 @@ type IpfsNode struct { PrivateKey ic.PrivKey `optional:"true"` // the local node's private Key PNetFingerprint libp2p.PNetFingerprint `optional:"true"` // fingerprint of private network + OnDemandPinStore *ondemandpin.Store + OnDemandPinChecker *ondemandpin.Checker `optional:"true"` + // Services Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances Blockstore bstore.GCBlockstore // the block store (lower level) diff --git a/core/node/groups.go b/core/node/groups.go index 656a58234f1..a121c7b26a4 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -357,6 +357,8 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part // new CIDs still announce via fast-provide-root and 'ipfs provide once'. isProviderEnabled := cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled) + isOnDemandPinEnabled := cfg.Experimental.OnDemandPinningEnabled + return fx.Options( fx.Provide(BitswapOptions(cfg)), fx.Provide(Bitswap(isBitswapServerEnabled, isBitswapLibp2pEnabled, isHTTPRetrievalEnabled)), @@ -373,6 +375,8 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part LibP2P(bcfg, cfg, userResourceOverrides), OnlineProviders(isProviderEnabled, cfg), + + maybeProvide(OnDemandPinChecker(cfg.OnDemandPinning), isOnDemandPinEnabled), ) } @@ -466,6 +470,7 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option { fx.Provide(BlockService(cfg)), fx.Provide(Pinning(providerStrategy)), fx.Provide(Files(providerStrategy)), + fx.Provide(OnDemandPinStore), Core, ) } diff --git a/core/node/ondemandpin.go b/core/node/ondemandpin.go new file mode 100644 index 00000000000..f542e086a56 --- /dev/null +++ b/core/node/ondemandpin.go @@ -0,0 +1,168 @@ +package node + +import ( + "context" + "time" + + "github.com/dustin/go-humanize" + blockstore "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/ipld/merkledag" + pin "github.com/ipfs/boxo/pinning/pinner" + "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" + peer "github.com/libp2p/go-libp2p/core/peer" + routing "github.com/libp2p/go-libp2p/core/routing" + "go.uber.org/fx" + + "github.com/ipfs/kubo/config" + "github.com/ipfs/kubo/core/node/helpers" + "github.com/ipfs/kubo/ondemandpin" + "github.com/ipfs/kubo/repo" +) + +// pinIdleTimeout cancels a pin when no new blocks arrive for this long. +const pinIdleTimeout = 2 * time.Minute + +type kuboPinService struct { + pinner pin.Pinner + dag format.DAGService + bs blockstore.GCBlockstore +} + +func (s *kuboPinService) Pin(ctx context.Context, c cid.Cid, name string) error { + defer s.bs.PinLock(ctx).Unlock(ctx) + + node, err := s.fetchRoot(ctx, c) + if err != nil { + return err + } + if err := s.pinDAG(ctx, node, name); err != nil { + return err + } + return s.pinner.Flush(ctx) +} + +func (s *kuboPinService) fetchRoot(ctx context.Context, c cid.Cid) (format.Node, error) { + ctx, cancel := context.WithTimeout(ctx, pinIdleTimeout) + defer cancel() + return s.dag.Get(ctx, c) +} + +// pinDAG recursively pins the DAG rooted at node. +// A background goroutine monitors block-fetching progress and cancels the operation if no new blocks arrive within pinIdleTimeout. +func (s *kuboPinService) pinDAG(ctx context.Context, node format.Node, name string) error { + tracker := new(merkledag.ProgressTracker) + trackerCtx := tracker.DeriveContext(ctx) + pinCtx, cancel := context.WithCancel(trackerCtx) + defer cancel() + + go watchPinProgress(pinCtx, cancel, tracker, pinIdleTimeout) + return s.pinner.Pin(pinCtx, node, true, name) +} + +func watchPinProgress(ctx context.Context, cancel context.CancelFunc, tracker *merkledag.ProgressTracker, timeout time.Duration) { + var last int + lastProgress := time.Now() + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if cur := tracker.Value(); cur != last { + last = cur + lastProgress = time.Now() + } else if time.Since(lastProgress) >= timeout { + cancel() + return + } + } + } +} + +func (s *kuboPinService) Unpin(ctx context.Context, c cid.Cid) error { + defer s.bs.PinLock(ctx).Unlock(ctx) + + if err := s.pinner.Unpin(ctx, c, true); err != nil { + return err + } + return s.pinner.Flush(ctx) +} + +func (s *kuboPinService) IsPinned(ctx context.Context, c cid.Cid) (bool, error) { + _, pinned, err := s.pinner.IsPinned(ctx, c) + return pinned, err +} + +func (s *kuboPinService) HasPinWithName(ctx context.Context, c cid.Cid, name string) (bool, error) { + return ondemandpin.PinHasName(ctx, s.pinner, c, name) +} + +type kuboStorageChecker struct { + repo repo.Repo +} + +func (s *kuboStorageChecker) StorageUsage(ctx context.Context) (uint64, uint64, error) { + cfg, err := s.repo.Config() + if err != nil { + return 0, 0, err + } + used, err := s.repo.GetStorageUsage(ctx) + if err != nil { + return 0, 0, err + } + if cfg.Datastore.StorageMax == "" { + return used, 0, nil + } + max, err := humanize.ParseBytes(cfg.Datastore.StorageMax) + if err != nil { + return 0, 0, err + } + wm := cfg.Datastore.StorageGCWatermark + if wm <= 0 || wm > 100 { + wm = 90 + } + return used, max * uint64(wm) / 100, nil +} + +func OnDemandPinStore(r repo.Repo) *ondemandpin.Store { + return ondemandpin.NewStore(r.Datastore()) +} + +func OnDemandPinChecker(cfg config.OnDemandPinning) func( + mctx helpers.MetricsCtx, + lc fx.Lifecycle, + r repo.Repo, + store *ondemandpin.Store, + pinner pin.Pinner, + cr routing.ContentRouting, + dag format.DAGService, + bs blockstore.GCBlockstore, + id peer.ID, +) *ondemandpin.Checker { + return func( + mctx helpers.MetricsCtx, + lc fx.Lifecycle, + r repo.Repo, + store *ondemandpin.Store, + pinner pin.Pinner, + cr routing.ContentRouting, + dag format.DAGService, + bs blockstore.GCBlockstore, + id peer.ID, + ) *ondemandpin.Checker { + pins := &kuboPinService{pinner: pinner, dag: dag, bs: bs} + storage := &kuboStorageChecker{repo: r} + checker := ondemandpin.NewChecker(store, pins, storage, cr, id, cfg) + ctx := helpers.LifecycleCtx(mctx, lc) + + lc.Append(fx.Hook{ + OnStart: func(context.Context) error { + go checker.Run(ctx) + return nil + }, + }) + return checker + } +} diff --git a/docs/config.md b/docs/config.md index dac39ab1158..f9736f95a28 100644 --- a/docs/config.md +++ b/docs/config.md @@ -60,6 +60,7 @@ config file at runtime. - [`Discovery.MDNS.Interval`](#discoverymdnsinterval) - [`Experimental`](#experimental) - [`Experimental.Libp2pStreamMounting`](#experimentallibp2pstreammounting) + - [`Experimental.OnDemandPinningEnabled`](#experimentalondemandpinningenabled) - [`Gateway`](#gateway) - [`Gateway.NoFetch`](#gatewaynofetch) - [`Gateway.NoDNSLink`](#gatewaynodnslink) @@ -125,6 +126,11 @@ config file at runtime. - [`Mounts.FuseAllowOther`](#mountsfuseallowother) - [`Mounts.StoreMtime`](#mountsstoremtime) - [`Mounts.StoreMode`](#mountsstoremode) + - [`OnDemandPinning`](#ondemandpinning) + - [`OnDemandPinning.ReplicationTarget`](#ondemandpinningreplicationtarget) + - [`OnDemandPinning.CheckInterval`](#ondemandpinningcheckinterval) + - [`OnDemandPinning.UnpinGracePeriod`](#ondemandpinningunpingraceperiod) + - [`Pinning`](#pinning) - [`Pinning.RemoteServices`](#pinningremoteservices) - [`Pinning.RemoteServices: API`](#pinningremoteservices-api) @@ -1234,6 +1240,20 @@ in the [new mDNS implementation](https://github.com/libp2p/zeroconf#readme). Toggle and configure experimental features of Kubo. Experimental features are listed [here](./experimental-features.md). +### `Experimental.OnDemandPinningEnabled` + +Enables on-demand pinning. When enabled, the node runs a background checker +that periodically evaluates DHT provider counts for CIDs registered via +`ipfs pin ondemand add`. CIDs with fewer providers than the replication target +are pinned; pins are removed after replication stays above target for a grace +period (default 24h). + +See [`OnDemandPinning`](#ondemandpinning) for configuration. + +Default: `false` + +Type: `bool` + ### `Experimental.Libp2pStreamMounting` Enables the `ipfs p2p` commands for tunneling TCP connections through libp2p @@ -2224,6 +2244,38 @@ Default: `"5m"` Type: `duration` +## `OnDemandPinning` + +Configures the on-demand pinning system. Requires +[`Experimental.OnDemandPinningEnabled`](#experimentalondemandpinningenabled). + +### `OnDemandPinning.ReplicationTarget` + +The minimum number of providers desired in the DHT (excluding the local node). +When fewer providers are found, the node pins the content locally. + +Default: `5` + +Type: `optionalInteger` + +### `OnDemandPinning.CheckInterval` + +How often the background checker evaluates all on-demand pins. + +Default: `"10m"` + +Type: `optionalDuration` + +### `OnDemandPinning.UnpinGracePeriod` + +How long replication must stay above target before the local pin is removed. +This prevents thrashing when provider counts fluctuate near the target +boundary. + +Default: `"24h"` + +Type: `optionalDuration` + ## `Provide` Configures how your node advertises content to make it discoverable by other diff --git a/docs/experimental-features.md b/docs/experimental-features.md index abd45ff3ca5..52377f2dd3c 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -28,6 +28,7 @@ the above issue. - [Noise](#noise) - [Optimistic Provide](#optimistic-provide) - [HTTP Gateway over Libp2p](#http-gateway-over-libp2p) +- [On-Demand Pinning](#on-demand-pinning) --- @@ -609,6 +610,67 @@ ipfs config --json Experimental.GatewayOverLibp2p true - [ ] Needs a mechanism for HTTP handler to signal supported features ([IPIP-425](https://github.com/ipfs/specs/pull/425)) - [ ] Needs an option for Kubo to detect peers that have it enabled and prefer HTTP transport before falling back to bitswap (and use CAR if peer supports dag-scope=entity from [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/)) +## On-Demand Pinning + +### State + +Experimental, disabled by default. + +On-demand pinning lets a node automatically pin content when DHT provider +counts fall below a configurable replication target, and unpin when +replication recovers above target for a grace period. Under-replicated +content gets pinned; storage is freed once enough other providers exist. + +The feature consists of: + +- A **registry** of CIDs to monitor, managed via `ipfs pin ondemand add|rm|ls`. +- A **background checker** that periodically queries the DHT for each + registered CID and pins or unpins accordingly. +- **Pin partitioning**: the checker marks its pins with the name `"on-demand"` + to distinguish them from user-created pins and will never remove a pin it + did not create. + +When the checker pins a CID it also publishes a provider record to the DHT +so other peers can discover the content on this node. + +**Security consideration**: DHT provider counts can be gamed via Sybil +attacks. An attacker could inflate provider counts to trick nodes into +unpinning content that is not actually well-replicated. The grace period +provides partial mitigation by requiring fake provider records to be +sustained for its full duration (default 24 hours). + +### How to enable + +``` +ipfs config --json Experimental.OnDemandPinningEnabled true +``` + +### Configuring + +See [`OnDemandPinning`](https://github.com/ipfs/kubo/blob/master/docs/config.md#ondemandpinning) +for tunable parameters: `ReplicationTarget`, `CheckInterval`, +and `UnpinGracePeriod`. + +### Basic usage + +```bash +# Register a CID for on-demand monitoring +ipfs pin ondemand add QmExample + +# List all registered CIDs (with live provider counts from DHT) +ipfs pin ondemand ls --live + +# Remove a CID from on-demand monitoring (also unpins if checker had pinned it) +ipfs pin ondemand rm QmExample +``` + +### Road to being a real feature + +- [ ] Needs more people to use and report on how well it works +- [ ] Needs integration tests in `test/cli/` +- [ ] Needs UI support in ipfs-webui / IPFS Desktop +- [ ] Evaluate Sybil resilience and whether additional mitigations are needed + ## Accelerated DHT Client This feature now lives at [`Routing.AcceleratedDHTClient`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient). diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go new file mode 100644 index 00000000000..9b121029ea9 --- /dev/null +++ b/ondemandpin/checker.go @@ -0,0 +1,282 @@ +package ondemandpin + +import ( + "context" + "time" + + pin "github.com/ipfs/boxo/pinning/pinner" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "github.com/ipfs/kubo/config" + peer "github.com/libp2p/go-libp2p/core/peer" + routing "github.com/libp2p/go-libp2p/core/routing" +) + +var log = logging.Logger("ondemandpin") + +// OnDemandPinName is the pin name the checker uses when creating pins (Kubo specific. Other implementation may divert from this method). +// Pins carrying this name are considered managed by on-demand pinning and may be removed automatically when replication recovers. +const OnDemandPinName = "on-demand" + +// checkTimeout prevents hung DHT query or pin/unpin operation from blocking the checker indefinitely. +const checkTimeout = 5 * time.Minute + +type PinService interface { + Pin(ctx context.Context, c cid.Cid, name string) error + Unpin(ctx context.Context, c cid.Cid) error + IsPinned(ctx context.Context, c cid.Cid) (bool, error) + HasPinWithName(ctx context.Context, c cid.Cid, name string) (bool, error) +} + +type StorageChecker interface { + StorageUsage(ctx context.Context) (used, limit uint64, err error) +} + +type Checker struct { + store *Store + pins PinService + storage StorageChecker + routing routing.ContentRouting + selfID peer.ID + + replicationTarget int + checkInterval time.Duration + unpinGracePeriod time.Duration + + now func() time.Time + priorityCh chan cid.Cid +} + +func NewChecker( + store *Store, + pins PinService, + storage StorageChecker, + cr routing.ContentRouting, + selfID peer.ID, + cfg config.OnDemandPinning, +) *Checker { + return &Checker{ + store: store, + pins: pins, + storage: storage, + routing: cr, + selfID: selfID, + + replicationTarget: int(cfg.ReplicationTarget.WithDefault(config.DefaultOnDemandPinReplicationTarget)), + checkInterval: cfg.CheckInterval.WithDefault(config.DefaultOnDemandPinCheckInterval), + unpinGracePeriod: cfg.UnpinGracePeriod.WithDefault(config.DefaultOnDemandPinUnpinGracePeriod), + + now: time.Now, + priorityCh: make(chan cid.Cid, 64), + } +} + +func (c *Checker) Enqueue(ci cid.Cid) { + select { + case c.priorityCh <- ci: + default: + log.Warnw("priority queue full, CID will be checked in next regular cycle", "cid", ci) + } +} + +// Run blocks until ctx is cancelled. +func (c *Checker) Run(ctx context.Context) { + log.Info("on-demand pin checker started") + defer log.Info("on-demand pin checker stopped") + + ticker := time.NewTicker(c.checkInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case ci := <-c.priorityCh: + c.checkOne(ctx, ci) + case <-ticker.C: + c.checkAll(ctx) + } + } +} + +func (c *Checker) checkAll(ctx context.Context) { + records, err := c.store.List(ctx) + if err != nil { + log.Errorw("failed to list on-demand pins", "error", err) + return + } + + log.Infow("starting check cycle", "records", len(records)) + for _, rec := range records { + // Drain priority checks between records so Enqueue'd CIDs don't wait for a full sweep to complete. + select { + case ci := <-c.priorityCh: + c.checkOne(ctx, ci) + default: + } + if ctx.Err() != nil { + return + } + c.checkRecord(ctx, &rec) + } +} + +func (c *Checker) checkOne(ctx context.Context, ci cid.Cid) { + rec, err := c.store.Get(ctx, ci) + if err != nil { + log.Debugw("CID not in store, skipping", "cid", ci, "error", err) + return + } + c.checkRecord(ctx, rec) +} + +// checkRecord evaluates a single on-demand pin record in three phases: +// +// 1. Guard: if the CID already has an external pin (not created by us), skip it to avoid interfering with user-managed pins. +// We use PinService.IsPinned (any pin) here โ€” not Record.PinnedByUs (on-demand pins). +// 2. Under-replicated: if provider count is low, pin locally. +// 3. Well-replicated: if provider count is high for a full grace period, unpin (on-demand pins only). +func (c *Checker) checkRecord(ctx context.Context, rec *Record) { + ctx, cancel := context.WithTimeout(ctx, checkTimeout) + defer cancel() + + pinned, err := c.pins.IsPinned(ctx, rec.Cid) + if err != nil { + log.Errorw("failed to check pin state, skipping CID", "cid", rec.Cid, "error", err) + return + } + if !rec.PinnedByUs && pinned { + log.Debugw("skipping: CID has a user-managed pin", "cid", rec.Cid) + return + } + + count := CountProviders(ctx, c.routing, c.selfID, rec.Cid, c.replicationTarget) + log.Debugw("provider count", "cid", rec.Cid, "count", count, "target", c.replicationTarget, "pinnedByUs", rec.PinnedByUs) + + if count < c.replicationTarget { + c.handleUnderReplicated(ctx, rec, count, pinned) + } else { + c.handleWellReplicated(ctx, rec, count) + } +} + +// handleUnderReplicated pins the CID if it isn't already pinned. +func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count int, pinExists bool) { + if rec.PinnedByUs && pinExists { + if !rec.LastAboveTarget.IsZero() { + rec.LastAboveTarget = time.Time{} + c.saveRecord(ctx, rec) + } + return + } + + if rec.PinnedByUs { + log.Warnw("pin was removed externally, will re-pin", "cid", rec.Cid) + rec.PinnedByUs = false + } + + if !c.hasStorageBudget(ctx) { + log.Warnw("skipping pin: repo near storage limit", "cid", rec.Cid) + return + } + + if err := c.pins.Pin(ctx, rec.Cid, OnDemandPinName); err != nil { + log.Errorw("failed to pin", "cid", rec.Cid, "error", err) + return + } + rec.PinnedByUs = true + rec.LastAboveTarget = time.Time{} + log.Infow("pinned", "cid", rec.Cid, "providers", count, "target", c.replicationTarget) + + if err := c.routing.Provide(ctx, rec.Cid, true); err != nil { + log.Warnw("failed to provide after pin", "cid", rec.Cid, "error", err) + } + c.saveRecord(ctx, rec) +} + +// handleWellReplicated manages grace-period-then-unpin. +func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count int) { + if !rec.PinnedByUs { + return + } + + if rec.LastAboveTarget.IsZero() { + rec.LastAboveTarget = c.now() + c.saveRecord(ctx, rec) + log.Debugw("grace period started", "cid", rec.Cid, "providers", count, "target", c.replicationTarget) + return + } + + if c.now().Sub(rec.LastAboveTarget) < c.unpinGracePeriod { + return + } + + hasOurPin, err := c.pins.HasPinWithName(ctx, rec.Cid, OnDemandPinName) + if err != nil { + log.Errorw("failed to check pin name, skipping unpin", "cid", rec.Cid, "error", err) + return + } + + if hasOurPin { + if err := c.pins.Unpin(ctx, rec.Cid); err != nil { + log.Errorw("failed to unpin", "cid", rec.Cid, "error", err) + return + } + log.Infow("unpinned", "cid", rec.Cid, "providers", count, "target", c.replicationTarget) + } else { + log.Infow("relinquishing management: pin name changed externally", "cid", rec.Cid) + } + + rec.PinnedByUs = false + rec.LastAboveTarget = time.Time{} + c.saveRecord(ctx, rec) +} + +func (c *Checker) saveRecord(ctx context.Context, rec *Record) { + if err := c.store.Update(ctx, rec); err != nil { + log.Errorw("failed to update record", "cid", rec.Cid, "error", err) + } +} + +func (c *Checker) hasStorageBudget(ctx context.Context) bool { + if c.storage == nil { + return true + } + used, limit, err := c.storage.StorageUsage(ctx) + if err != nil { + log.Warnw("failed to check storage usage, proceeding with pin", "error", err) + return true + } + if limit == 0 { + return true + } + return used < limit +} + +// CountProviders counts unique providers with target+1 results to ensure that, even if selfID appears, we still discover up to target. +func CountProviders(ctx context.Context, cr routing.ContentRouting, selfID peer.ID, c cid.Cid, target int) int { + ch := cr.FindProvidersAsync(ctx, c, target+1) + + seen := make(map[peer.ID]struct{}) + for pi := range ch { + if pi.ID == selfID { + continue + } + seen[pi.ID] = struct{}{} + } + return len(seen) +} + +// PinHasName is used by checker (via PinService.HasPinWithName) and the rm command to identify pins managed by on-demand pinning. +func PinHasName(ctx context.Context, p pin.Pinner, c cid.Cid, name string) (bool, error) { + results, err := p.CheckIfPinnedWithType(ctx, pin.Recursive, true, c) + if err != nil { + return false, err + } + for _, r := range results { + if r.Pinned() && r.Name == name { + return true, nil + } + } + return false, nil +} diff --git a/ondemandpin/checker_test.go b/ondemandpin/checker_test.go new file mode 100644 index 00000000000..02ced95d2ee --- /dev/null +++ b/ondemandpin/checker_test.go @@ -0,0 +1,165 @@ +package ondemandpin + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + "github.com/ipfs/kubo/config" + peer "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeClock struct{ t atomic.Int64 } + +func newFakeClock() *fakeClock { + c := &fakeClock{} + c.t.Store(time.Now().UnixNano()) + return c +} + +func (c *fakeClock) Now() time.Time { return time.Unix(0, c.t.Load()) } +func (c *fakeClock) Advance(d time.Duration) { c.t.Add(int64(d)) } + +type mockRouting struct { + mu sync.Mutex + providers map[cid.Cid][]peer.AddrInfo +} + +func newMockRouting() *mockRouting { + return &mockRouting{providers: make(map[cid.Cid][]peer.AddrInfo)} +} + +func (m *mockRouting) setProviders(c cid.Cid, peerIDs ...peer.ID) { + m.mu.Lock() + defer m.mu.Unlock() + infos := make([]peer.AddrInfo, len(peerIDs)) + for i, pid := range peerIDs { + infos[i] = peer.AddrInfo{ID: pid} + } + m.providers[c] = infos +} + +func (m *mockRouting) FindProvidersAsync(ctx context.Context, c cid.Cid, limit int) <-chan peer.AddrInfo { + ch := make(chan peer.AddrInfo) + go func() { + defer close(ch) + m.mu.Lock() + provs := m.providers[c] + m.mu.Unlock() + for i, pi := range provs { + if i >= limit { + break + } + select { + case ch <- pi: + case <-ctx.Done(): + return + } + } + }() + return ch +} + +func (m *mockRouting) Provide(context.Context, cid.Cid, bool) error { return nil } + +type mockPins struct { + pinned map[cid.Cid]string +} + +func newMockPins() *mockPins { return &mockPins{pinned: make(map[cid.Cid]string)} } + +func (m *mockPins) Pin(_ context.Context, c cid.Cid, name string) error { + m.pinned[c] = name + return nil +} + +func (m *mockPins) Unpin(_ context.Context, c cid.Cid) error { + delete(m.pinned, c) + return nil +} + +func (m *mockPins) IsPinned(_ context.Context, c cid.Cid) (bool, error) { + _, ok := m.pinned[c] + return ok, nil +} + +func (m *mockPins) HasPinWithName(_ context.Context, c cid.Cid, name string) (bool, error) { + n, ok := m.pinned[c] + return ok && n == name, nil +} + +func (m *mockPins) isPinned(c cid.Cid) bool { + _, ok := m.pinned[c] + return ok +} + +func newTestChecker(t *testing.T) (*Checker, *Store, *mockRouting, *mockPins, *fakeClock) { + t.Helper() + store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) + r := newMockRouting() + p := newMockPins() + clock := newFakeClock() + + checker := NewChecker(store, p, nil, r, peer.ID("self"), config.OnDemandPinning{}) + checker.checkInterval = time.Minute + checker.unpinGracePeriod = 200 * time.Millisecond + checker.now = clock.Now + + return checker, store, r, p, clock +} + +// Under-replicated content gets pinned. +func TestCheckerPinsBelowTarget(t *testing.T) { + ctx := context.Background() + checker, store, r, p, _ := newTestChecker(t) + c := testCID(t, "under-replicated") + + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, peer.ID("p1"), peer.ID("p2")) + + checker.checkAll(ctx) + + assert.True(t, p.isPinned(c)) +} + +// Well-replicated content is left alone. +func TestCheckerDoesNotPinAboveTarget(t *testing.T) { + ctx := context.Background() + checker, store, r, p, _ := newTestChecker(t) + c := testCID(t, "well-replicated") + + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, peer.ID("p1"), peer.ID("p2"), peer.ID("p3"), peer.ID("p4"), peer.ID("p5"), peer.ID("p6")) + + checker.checkAll(ctx) + + assert.False(t, p.isPinned(c)) +} + +// Pinned content is unpinned only after the grace period expires. +func TestCheckerUnpinsAfterGracePeriod(t *testing.T) { + ctx := context.Background() + checker, store, r, p, clock := newTestChecker(t) + c := testCID(t, "recovering") + + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, peer.ID("p1")) + checker.checkAll(ctx) + require.True(t, p.isPinned(c)) + + // Providers recover above target. + r.setProviders(c, peer.ID("p1"), peer.ID("p2"), peer.ID("p3"), peer.ID("p4"), peer.ID("p5"), peer.ID("p6")) + checker.checkAll(ctx) + assert.True(t, p.isPinned(c), "not yet past grace period") + + clock.Advance(250 * time.Millisecond) + checker.checkAll(ctx) + assert.False(t, p.isPinned(c), "past grace period") +} diff --git a/ondemandpin/store.go b/ondemandpin/store.go new file mode 100644 index 00000000000..cd5edc58788 --- /dev/null +++ b/ondemandpin/store.go @@ -0,0 +1,143 @@ +package ondemandpin + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" +) + +var ( + ErrAlreadyRegistered = errors.New("CID is already registered for on-demand pinning") + ErrNotRegistered = errors.New("CID is not registered for on-demand pinning") +) + +const dsPrefix = "/ondemand-pins/" + +type Record struct { + Cid cid.Cid `json:"Cid"` + // PinnedByUs tracks the on-demand pins only (does not inlcude standard pins). + PinnedByUs bool `json:"IsPinned"` + LastAboveTarget time.Time `json:"LastAboveTarget,omitempty"` + CreatedAt time.Time `json:"CreatedAt"` +} + +type Store struct { + ds datastore.Batching + mu sync.RWMutex + now func() time.Time +} + +func NewStore(ds datastore.Batching) *Store { + return &Store{ds: ds, now: time.Now} +} + +func dsKey(c cid.Cid) datastore.Key { + return datastore.NewKey(dsPrefix + c.String()) +} + +func (s *Store) Add(ctx context.Context, c cid.Cid) error { + s.mu.Lock() + defer s.mu.Unlock() + + key := dsKey(c) + + if has, err := s.ds.Has(ctx, key); err != nil { + return fmt.Errorf("checking existing record: %w", err) + } else if has { + return fmt.Errorf("%s: %w", c, ErrAlreadyRegistered) + } + + rec := Record{ + Cid: c, + CreatedAt: s.now(), + } + + return s.put(ctx, key, &rec) +} + +func (s *Store) Remove(ctx context.Context, c cid.Cid) error { + s.mu.Lock() + defer s.mu.Unlock() + + key := dsKey(c) + if has, err := s.ds.Has(ctx, key); err != nil { + return fmt.Errorf("checking record: %w", err) + } else if !has { + return fmt.Errorf("%s: %w", c, ErrNotRegistered) + } + + if err := s.ds.Delete(ctx, key); err != nil { + return fmt.Errorf("deleting record: %w", err) + } + return s.ds.Sync(ctx, key) +} + +func (s *Store) Get(ctx context.Context, c cid.Cid) (*Record, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.get(ctx, dsKey(c)) +} + +func (s *Store) List(ctx context.Context) ([]Record, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + results, err := s.ds.Query(ctx, query.Query{ + Prefix: dsPrefix, + }) + if err != nil { + return nil, fmt.Errorf("querying on-demand pins: %w", err) + } + defer results.Close() + + var records []Record + for result := range results.Next() { + if result.Error != nil { + return nil, fmt.Errorf("iterating on-demand pins: %w", result.Error) + } + var rec Record + if err := json.Unmarshal(result.Value, &rec); err != nil { + return nil, fmt.Errorf("unmarshaling record %s: %w", result.Key, err) + } + records = append(records, rec) + } + return records, nil +} + +func (s *Store) Update(ctx context.Context, rec *Record) error { + s.mu.Lock() + defer s.mu.Unlock() + + return s.put(ctx, dsKey(rec.Cid), rec) +} + +func (s *Store) get(ctx context.Context, key datastore.Key) (*Record, error) { + val, err := s.ds.Get(ctx, key) + if err != nil { + return nil, fmt.Errorf("getting record: %w", err) + } + var rec Record + if err := json.Unmarshal(val, &rec); err != nil { + return nil, fmt.Errorf("unmarshaling record: %w", err) + } + return &rec, nil +} + +func (s *Store) put(ctx context.Context, key datastore.Key, rec *Record) error { + val, err := json.Marshal(rec) + if err != nil { + return fmt.Errorf("marshaling record: %w", err) + } + if err := s.ds.Put(ctx, key, val); err != nil { + return fmt.Errorf("storing record: %w", err) + } + return s.ds.Sync(ctx, key) +} diff --git a/ondemandpin/store_test.go b/ondemandpin/store_test.go new file mode 100644 index 00000000000..7383889f085 --- /dev/null +++ b/ondemandpin/store_test.go @@ -0,0 +1,57 @@ +package ondemandpin + +import ( + "context" + "testing" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + mh "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testCID(t *testing.T, data string) cid.Cid { + t.Helper() + h, err := mh.Sum([]byte(data), mh.SHA2_256, -1) + require.NoError(t, err) + return cid.NewCidV1(cid.Raw, h) +} + +func newTestStore(t *testing.T) *Store { + t.Helper() + return NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) +} + +// Add, Get, Remove lifecycle works. +func TestStoreRoundTrip(t *testing.T) { + ctx := context.Background() + s := newTestStore(t) + c := testCID(t, "hello") + + require.NoError(t, s.Add(ctx, c)) + + rec, err := s.Get(ctx, c) + require.NoError(t, err) + assert.Equal(t, c, rec.Cid) + assert.False(t, rec.PinnedByUs) + + require.NoError(t, s.Remove(ctx, c)) + + _, err = s.Get(ctx, c) + assert.Error(t, err) +} + +// List returns all registered records. +func TestStoreList(t *testing.T) { + ctx := context.Background() + s := newTestStore(t) + + require.NoError(t, s.Add(ctx, testCID(t, "a"))) + require.NoError(t, s.Add(ctx, testCID(t, "b"))) + + records, err := s.List(ctx) + require.NoError(t, err) + assert.Len(t, records, 2) +} From 1015c18eed99d94733462c7c713f0a7223aa593a Mon Sep 17 00:00:00 2001 From: ihlec Date: Wed, 1 Apr 2026 15:05:59 +0200 Subject: [PATCH 02/17] fix: addressed failing checks by cleaning up comments, spelling, and adding v0.42 changelog entry --- core/commands/pin/ondemandpin.go | 24 ++++++++--------- docs/changelogs/v0.42.md | 45 ++++++++++++++++++++++++++++++++ ondemandpin/store.go | 2 +- 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/core/commands/pin/ondemandpin.go b/core/commands/pin/ondemandpin.go index 7a5d07ddbcf..8e1635f93b1 100644 --- a/core/commands/pin/ondemandpin.go +++ b/core/commands/pin/ondemandpin.go @@ -18,10 +18,10 @@ var onDemandPinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Manage on-demand pins.", ShortDescription: ` - On-demand pinning automatically pins content when few target providers exist in the DHT, - and unpins when replication has been above target for a grace period. - Requires Experimental.OnDemandPinningEnabled = true. - `, +On-demand pins when few DHT providers exist in the routing table; unpins after +replication stays above target for a grace period. Requires config +Experimental.OnDemandPinningEnabled. +`, }, Subcommands: map[string]*cmds.Command{ "add": addOnDemandPinCmd, @@ -37,7 +37,7 @@ type OnDemandPinOutput struct { var addOnDemandPinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Register CIDs for on-demand pinning.", - ShortDescription: `Adds the given CID(s) to the on-demand pin registry. The background checker will evaluate replication and pin if needed.`, + ShortDescription: `Registers CID(s) for on-demand pinning; checker pins when needed.`, }, Arguments: []cmds.Argument{ cmds.StringArg("cid", true, true, "CID(s) to register."), @@ -104,10 +104,10 @@ var rmOnDemandPinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Remove CIDs from on-demand pinning.", ShortDescription: ` - Removes the given CID(s) from the on-demand pin registry. - If the content was pinned by the checker, it is also unpinned. - Works even when on-demand pinning is disabled, to clean up previously registered CIDs. - `, +Removes CID(s) from the registry. Checker-pinned content is unpinned. + +Works when on-demand pinning is disabled, to clear old registrations. +`, }, Arguments: []cmds.Argument{ cmds.StringArg("cid", true, true, "CID(s) to remove."), @@ -180,9 +180,9 @@ var listOnDemandPinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List on-demand pins.", ShortDescription: ` - Lists CIDs registered for on-demand pinning with their current state. - Use --live to include real-time provider counts from the DHT. - `, +Lists CIDs registered for on-demand pinning with their current state. +Use --live to include real-time provider counts from the DHT. +`, }, Arguments: []cmds.Argument{ cmds.StringArg("cid", false, true, "Optional CID(s) to filter."), diff --git a/docs/changelogs/v0.42.md b/docs/changelogs/v0.42.md index 6e687460335..5270b64988e 100644 --- a/docs/changelogs/v0.42.md +++ b/docs/changelogs/v0.42.md @@ -20,6 +20,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [๐Ÿ“Š OpenTelemetry: scope info now exposed as labels](#-opentelemetry-scope-info-now-exposed-as-labels) - [๐Ÿ”ง Cleaner progress bars](#-cleaner-progress-bars) - [๐Ÿ“ฆ๏ธ Dependency updates](#-dependency-updates) + - [Experimental on-demand pinning](#experimental-on-demand-pinning) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -127,6 +128,50 @@ The Prometheus endpoint no longer emits the `otel_scope_info` metric. Each metri - update `go-car/v2` to [v2.17.0](https://github.com/ipld/go-car/releases/tag/v2.17.0) - update `go.opentelemetry.io` to [v1.44.0](https://github.com/open-telemetry/opentelemetry-go/releases/tag/v1.44.0)(includes [v1.43.0](https://github.com/open-telemetry/opentelemetry-go/releases/tag/v1.43.0)) - update `p2p-forge/client` to [v0.9.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.9.0) (incl. [v0.8.1](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.8.1)) +#### Experimental on-demand pinning + +Automatically pins content when DHT provider counts fall below a configurable +replication target, and unpins once replication has been above target for a +grace period. Helps keeping critical data around, without wasting storage on +overly replicated CIDs. + +The feature is gated behind `Experimental.OnDemandPinningEnabled` and described +in [ipfs/specs#532](https://github.com/ipfs/specs/pull/532). + +```console +$ ipfs config --json Experimental.OnDemandPinningEnabled true +``` + +New CLI commands under `ipfs pin ondemand`: + +- `add` -- register CIDs for on-demand pinning +- `rm` -- deregister and unpin +- `ls` -- list registered CIDs (use `--live` for real-time DHT provider counts) + +Design highlights: + +- **Pin partitioning**: the checker needs to distinguish its pins from user + pins to avoid accidental deletion. This implementation uses boxo's pin name + field ("on-demand"). +- **Storage budget**: skips pinning when repo usage exceeds + `StorageMax * StorageGCWatermark`. +- **Idle timeout**: DAG fetches timeout after 2 minutes without receiving new + blocks, allowing large downloads while skipping dead records. +- **Provide after pin**: the checker publishes a DHT provider record after + pinning so other peers can discover the content on this node. +- **Sybil limitation**: provider counts come from DHT queries, which are + susceptible to Sybil manipulation. Documented as a known limitation. + +Configuration at [`OnDemandPinning`](https://github.com/ipfs/kubo/blob/master/docs/config.md#ondemandpinning): + +| Option | Default | Description | +|---|---|---| +| `OnDemandPinning.ReplicationTarget` | `5` | Minimum providers in DHT (excluding self) | +| `OnDemandPinning.CheckInterval` | `"10m"` | How often the checker runs | +| `OnDemandPinning.UnpinGracePeriod` | `"24h"` | How long above target before unpinning | + +See [experimental features](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#on-demand-pinning) +for full documentation. ### ๐Ÿ“ Changelog diff --git a/ondemandpin/store.go b/ondemandpin/store.go index cd5edc58788..0c50bb0be45 100644 --- a/ondemandpin/store.go +++ b/ondemandpin/store.go @@ -22,7 +22,7 @@ const dsPrefix = "/ondemand-pins/" type Record struct { Cid cid.Cid `json:"Cid"` - // PinnedByUs tracks the on-demand pins only (does not inlcude standard pins). + // PinnedByUs tracks the on-demand pins only (does not include standard pins). PinnedByUs bool `json:"IsPinned"` LastAboveTarget time.Time `json:"LastAboveTarget,omitempty"` CreatedAt time.Time `json:"CreatedAt"` From a0a150fca37c24a0b1c16b574884cbfe29844faf Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 14 Jul 2026 15:16:17 +0200 Subject: [PATCH 03/17] docs: move changelog highlight to vFUTURE On-demand pinning is not landing in v0.43. Review is open and the design needs changes first, so park the highlight in vFUTURE.md and restore v0.43.md to the upstream skeleton. --- docs/changelogs/vFUTURE.md | 68 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 docs/changelogs/vFUTURE.md diff --git a/docs/changelogs/vFUTURE.md b/docs/changelogs/vFUTURE.md new file mode 100644 index 00000000000..4aced50c67d --- /dev/null +++ b/docs/changelogs/vFUTURE.md @@ -0,0 +1,68 @@ +# Kubo changelog vFUTURE + + + +This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. + +- [vFUTURE](#vfuture) + +## vFUTURE + +- [Overview](#overview) +- [๐Ÿ”ฆ Highlights](#-highlights) + - [Experimental on-demand pinning](#experimental-on-demand-pinning) +- [๐Ÿ“ Changelog](#-changelog) +- [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) + +### Overview + +### ๐Ÿ”ฆ Highlights + +#### Experimental on-demand pinning + +Automatically pins content when DHT provider counts fall below a configurable +replication target, and unpins once replication has been above target for a +grace period. Helps keeping critical data around, without wasting storage on +overly replicated CIDs. + +The feature is gated behind `Experimental.OnDemandPinningEnabled` and described +in [ipfs/specs#532](https://github.com/ipfs/specs/pull/532). + +```console +$ ipfs config --json Experimental.OnDemandPinningEnabled true +``` + +New CLI commands under `ipfs pin ondemand`: + +- `add` register CIDs for on-demand pinning +- `rm` deregister and unpin +- `ls` list registered CIDs (use `--live` for real-time DHT provider counts) + +Design highlights: + +- **Pin partitioning**: the checker needs to distinguish its pins from user + pins to avoid accidental deletion. This implementation uses boxo's pin name + field ("on-demand"). +- **Storage budget**: skips pinning when repo usage exceeds + `StorageMax * StorageGCWatermark`. +- **Idle timeout**: DAG fetches timeout after 2 minutes without receiving new + blocks, allowing large downloads while skipping dead records. +- **Provide after pin**: the checker publishes a DHT provider record after + pinning so other peers can discover the content on this node. +- **Sybil limitation**: provider counts come from DHT queries, which are + susceptible to Sybil manipulation. Documented as a known limitation. + +Configuration at [`OnDemandPinning`](https://github.com/ipfs/kubo/blob/master/docs/config.md#ondemandpinning): + +| Option | Default | Description | +|---|---|---| +| `OnDemandPinning.ReplicationTarget` | `5` | Minimum providers in DHT (excluding self) | +| `OnDemandPinning.CheckInterval` | `"10m"` | How often the checker runs | +| `OnDemandPinning.UnpinGracePeriod` | `"24h"` | How long above target before unpinning | + +See [experimental features](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#on-demand-pinning) +for full documentation. + +### ๐Ÿ“ Changelog + +### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors From 4815e8c66ac610f6cc746844b35e1a39fa332ec2 Mon Sep 17 00:00:00 2001 From: ihlec Date: Fri, 17 Jul 2026 17:53:56 +0200 Subject: [PATCH 04/17] fix(pin): reserve "kubo:on-demand" pin names, make ondemand rm non-destructive --- core/commands/cmdutils/utils.go | 15 +++++++++++++-- core/commands/cmdutils/utils_test.go | 7 ++++++- core/commands/pin/ondemandpin.go | 6 ++++++ docs/experimental-features.md | 6 +++--- ondemandpin/checker.go | 4 +++- ondemandpin/store.go | 6 +++++- ondemandpin/store_test.go | 11 ++++++++++- 7 files changed, 46 insertions(+), 9 deletions(-) diff --git a/core/commands/cmdutils/utils.go b/core/commands/cmdutils/utils.go index 8c93a220c99..6156218f274 100644 --- a/core/commands/cmdutils/utils.go +++ b/core/commands/cmdutils/utils.go @@ -3,6 +3,7 @@ package cmdutils import ( "fmt" "slices" + "strings" cmds "github.com/ipfs/go-ipfs-cmds" @@ -54,14 +55,24 @@ func CheckBlockSize(req *cmds.Request, size uint64) error { return nil } -// ValidatePinName validates that a pin name does not exceed the maximum allowed byte length. -// Returns an error if the name exceeds MaxPinNameBytes (255 bytes). +// ReservedPinNamePrefix is the namespace for pins created by Kubo-internal +// features rather than by users (e.g. "kubo:on-demand" for on-demand pinning). +// ValidatePinName rejects user-supplied names with this prefix, so a pin name +// under this namespace reliably identifies a Kubo-internal pin. +const ReservedPinNamePrefix = "kubo:" + +// ValidatePinName validates that a pin name does not exceed the maximum allowed +// byte length and does not use the Kubo-internal "kubo:" namespace. func ValidatePinName(name string) error { if name == "" { // Empty names are allowed return nil } + if strings.HasPrefix(name, ReservedPinNamePrefix) { + return fmt.Errorf("pin names starting with %q are reserved for kubo-internal pins", ReservedPinNamePrefix) + } + nameBytes := len([]byte(name)) if nameBytes > MaxPinNameBytes { return fmt.Errorf("pin name is %d bytes (max %d bytes)", nameBytes, MaxPinNameBytes) diff --git a/core/commands/cmdutils/utils_test.go b/core/commands/cmdutils/utils_test.go index 4a8551ad07d..4be2ab12cee 100644 --- a/core/commands/cmdutils/utils_test.go +++ b/core/commands/cmdutils/utils_test.go @@ -178,4 +178,9 @@ func TestValidatePinName(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "bytes") }) -} + + t.Run("pin name with reserved prefix is rejected", func(t *testing.T) { + err := ValidatePinName(ReservedPinNamePrefix + "on-demand") + require.Error(t, err) + assert.Contains(t, err.Error(), "reserved") + }) diff --git a/core/commands/pin/ondemandpin.go b/core/commands/pin/ondemandpin.go index 8e1635f93b1..ca9a5cabdfa 100644 --- a/core/commands/pin/ondemandpin.go +++ b/core/commands/pin/ondemandpin.go @@ -138,6 +138,12 @@ Works when on-demand pinning is disabled, to clear old registrations. } c := rp.RootCid() + // Check the store first so an unregistered CID errors before any pin is touched. + if _, err := store.Get(req.Context, c); err != nil { + return err + } + + // Only remove pins carrying the Kubo-internal kubo:on-demand name; user pins on the same CID are kept. isOurs, err := ondemandpin.PinHasName(req.Context, n.Pinning, c, ondemandpin.OnDemandPinName) if err != nil { return fmt.Errorf("checking pin state for %s: %w", c, err) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 52377f2dd3c..e879acb1d0e 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -626,9 +626,9 @@ The feature consists of: - A **registry** of CIDs to monitor, managed via `ipfs pin ondemand add|rm|ls`. - A **background checker** that periodically queries the DHT for each registered CID and pins or unpins accordingly. -- **Pin partitioning**: the checker marks its pins with the name `"on-demand"` - to distinguish them from user-created pins and will never remove a pin it - did not create. +- **Pin partitioning**: the checker marks its pins with the name + `"kubo:on-demand"`, so the checker can tell its + pins apart from user pins and will never remove a pin it did not create. When the checker pins a CID it also publishes a provider record to the DHT so other peers can discover the content on this node. diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go index 9b121029ea9..15d22900f86 100644 --- a/ondemandpin/checker.go +++ b/ondemandpin/checker.go @@ -16,7 +16,9 @@ var log = logging.Logger("ondemandpin") // OnDemandPinName is the pin name the checker uses when creating pins (Kubo specific. Other implementation may divert from this method). // Pins carrying this name are considered managed by on-demand pinning and may be removed automatically when replication recovers. -const OnDemandPinName = "on-demand" +// The name is part of the Kubo-internal "kubo:" namespace, which ValidatePinName refuses for user-supplied +// names, so only Kubo-internal code can create pins with this name. +const OnDemandPinName = "kubo:on-demand" // checkTimeout prevents hung DHT query or pin/unpin operation from blocking the checker indefinitely. const checkTimeout = 5 * time.Minute diff --git a/ondemandpin/store.go b/ondemandpin/store.go index 0c50bb0be45..0ac2b86893d 100644 --- a/ondemandpin/store.go +++ b/ondemandpin/store.go @@ -83,7 +83,11 @@ func (s *Store) Get(ctx context.Context, c cid.Cid) (*Record, error) { s.mu.RLock() defer s.mu.RUnlock() - return s.get(ctx, dsKey(c)) + rec, err := s.get(ctx, dsKey(c)) + if errors.Is(err, datastore.ErrNotFound) { + return nil, fmt.Errorf("%s: %w", c, ErrNotRegistered) + } + return rec, err } func (s *Store) List(ctx context.Context) ([]Record, error) { diff --git a/ondemandpin/store_test.go b/ondemandpin/store_test.go index 7383889f085..d7718cd3086 100644 --- a/ondemandpin/store_test.go +++ b/ondemandpin/store_test.go @@ -40,7 +40,16 @@ func TestStoreRoundTrip(t *testing.T) { require.NoError(t, s.Remove(ctx, c)) _, err = s.Get(ctx, c) - assert.Error(t, err) + assert.ErrorIs(t, err, ErrNotRegistered) +} + +// Get on a CID that was never registered returns ErrNotRegistered. +func TestStoreGetNotRegistered(t *testing.T) { + ctx := context.Background() + s := newTestStore(t) + + _, err := s.Get(ctx, testCID(t, "never-registered")) + assert.ErrorIs(t, err, ErrNotRegistered) } // List returns all registered records. From b65363653159210639f4bbaf23d5d46632a67bae Mon Sep 17 00:00:00 2001 From: ihlec Date: Tue, 21 Jul 2026 09:16:14 +0200 Subject: [PATCH 05/17] fix(ondemandpin): do not overwrite pins created during provider lookup --- ondemandpin/checker.go | 11 +++++++++++ ondemandpin/checker_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go index 15d22900f86..64ae3f8b849 100644 --- a/ondemandpin/checker.go +++ b/ondemandpin/checker.go @@ -182,6 +182,17 @@ func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count return } + // Re-check: a user pin may have appeared during the provider lookup. + pinnedNow, err := c.pins.IsPinned(ctx, rec.Cid) + if err != nil { + log.Errorw("failed to re-check pin state before pinning, skipping CID", "cid", rec.Cid, "error", err) + return + } + if pinnedNow { + log.Debugw("skipping pin: CID gained a pin during provider lookup", "cid", rec.Cid) + return + } + if err := c.pins.Pin(ctx, rec.Cid, OnDemandPinName); err != nil { log.Errorw("failed to pin", "cid", rec.Cid, "error", err) return diff --git a/ondemandpin/checker_test.go b/ondemandpin/checker_test.go index 02ced95d2ee..8fe1af43b9d 100644 --- a/ondemandpin/checker_test.go +++ b/ondemandpin/checker_test.go @@ -163,3 +163,33 @@ func TestCheckerUnpinsAfterGracePeriod(t *testing.T) { checker.checkAll(ctx) assert.False(t, p.isPinned(c), "past grace period") } + +type pinDuringLookupRouting struct { + *mockRouting + pins *mockPins + pinName string +} + +func (r *pinDuringLookupRouting) FindProvidersAsync(ctx context.Context, c cid.Cid, limit int) <-chan peer.AddrInfo { + r.pins.pinned[c] = r.pinName + return r.mockRouting.FindProvidersAsync(ctx, c, limit) +} + +// / Re-check before Pin: a user pin that landed during the DHT lookup must not be overwritten. +func TestCheckerSkipsPinCreatedDuringLookup(t *testing.T) { + ctx := context.Background() + store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) + r := newMockRouting() + p := newMockPins() + racing := &pinDuringLookupRouting{mockRouting: r, pins: p, pinName: "user-pin"} + checker := NewChecker(store, p, nil, racing, peer.ID("self"), config.OnDemandPinning{}) + checker.now = newFakeClock().Now + + c := testCID(t, "raced") + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, peer.ID("p1")) + + checker.checkAll(ctx) + + assert.Equal(t, "user-pin", p.pinned[c]) +} From e66ba68bd75c400e3cdd1f3ede9b60c6849d679a Mon Sep 17 00:00:00 2001 From: ihlec Date: Tue, 21 Jul 2026 10:23:21 +0200 Subject: [PATCH 06/17] fix(ondemandpin): derive unpin grace period from DHT record validity --- config/ondemandpin.go | 11 +++++++++-- docs/changelogs/vFUTURE.md | 2 +- docs/config.md | 5 ++++- docs/experimental-features.md | 4 +++- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/config/ondemandpin.go b/config/ondemandpin.go index 4b072e94d32..c41df9f109e 100644 --- a/config/ondemandpin.go +++ b/config/ondemandpin.go @@ -1,11 +1,18 @@ package config -import "time" +import ( + "time" + + "github.com/libp2p/go-libp2p-kad-dht/amino" +) const ( DefaultOnDemandPinReplicationTarget = 5 DefaultOnDemandPinCheckInterval = 10 * time.Minute - DefaultOnDemandPinUnpinGracePeriod = 24 * time.Hour + + // Must exceed amino.DefaultProvideValidity so stale DHT records expire + // before unpin. The extra day covers check-interval skew. + DefaultOnDemandPinUnpinGracePeriod = amino.DefaultProvideValidity + 24*time.Hour ) type OnDemandPinning struct { diff --git a/docs/changelogs/vFUTURE.md b/docs/changelogs/vFUTURE.md index 4aced50c67d..a2dee550386 100644 --- a/docs/changelogs/vFUTURE.md +++ b/docs/changelogs/vFUTURE.md @@ -58,7 +58,7 @@ Configuration at [`OnDemandPinning`](https://github.com/ipfs/kubo/blob/master/do |---|---|---| | `OnDemandPinning.ReplicationTarget` | `5` | Minimum providers in DHT (excluding self) | | `OnDemandPinning.CheckInterval` | `"10m"` | How often the checker runs | -| `OnDemandPinning.UnpinGracePeriod` | `"24h"` | How long above target before unpinning | +| `OnDemandPinning.UnpinGracePeriod` | `"72h"` | How long above target before unpinning (outlasts the 48h DHT provider record validity) | See [experimental features](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#on-demand-pinning) for full documentation. diff --git a/docs/config.md b/docs/config.md index f9736f95a28..69ca814e5f2 100644 --- a/docs/config.md +++ b/docs/config.md @@ -2272,7 +2272,10 @@ How long replication must stay above target before the local pin is removed. This prevents thrashing when provider counts fluctuate near the target boundary. -Default: `"24h"` +Default must exceed DHT provider-record validity (48h), otherwise a node can +unpin while stale records still inflate the count and it holds the last live copy. + +Default: `"72h"` Type: `optionalDuration` diff --git a/docs/experimental-features.md b/docs/experimental-features.md index e879acb1d0e..ecfad04e74f 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -637,7 +637,9 @@ so other peers can discover the content on this node. attacks. An attacker could inflate provider counts to trick nodes into unpinning content that is not actually well-replicated. The grace period provides partial mitigation by requiring fake provider records to be -sustained for its full duration (default 24 hours). +sustained for its full duration (default 72 hours, chosen to outlast the +48h DHT provider record validity so records of dead peers expire before +the unpin decision). ### How to enable From e022855b0f8571e1824b6aa6fd15c61dde62b8a2 Mon Sep 17 00:00:00 2001 From: ihlec Date: Tue, 21 Jul 2026 11:03:52 +0200 Subject: [PATCH 07/17] fix(ondemandpin): validate OnDemandPinning config at daemon start --- config/ondemandpin.go | 16 ++++++++++++++++ config/ondemandpin_test.go | 33 +++++++++++++++++++++++++++++++++ core/node/groups.go | 7 +++++++ ondemandpin/checker.go | 7 +++++++ 4 files changed, 63 insertions(+) create mode 100644 config/ondemandpin_test.go diff --git a/config/ondemandpin.go b/config/ondemandpin.go index c41df9f109e..dde2a4a83e4 100644 --- a/config/ondemandpin.go +++ b/config/ondemandpin.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "time" "github.com/libp2p/go-libp2p-kad-dht/amino" @@ -25,3 +26,18 @@ type OnDemandPinning struct { // How long replication must stay above target before unpinning. UnpinGracePeriod OptionalDuration } + +// ValidateOnDemandPinningConfig rejects non-positive intervals/grace periods +// and ReplicationTarget < 1 (ticker panic / never pins, then unpins). +func ValidateOnDemandPinningConfig(cfg *OnDemandPinning) error { + if target := cfg.ReplicationTarget.WithDefault(DefaultOnDemandPinReplicationTarget); target < 1 { + return fmt.Errorf("OnDemandPinning.ReplicationTarget must be at least 1, got %d", target) + } + if interval := cfg.CheckInterval.WithDefault(DefaultOnDemandPinCheckInterval); interval <= 0 { + return fmt.Errorf("OnDemandPinning.CheckInterval must be positive, got %v", interval) + } + if grace := cfg.UnpinGracePeriod.WithDefault(DefaultOnDemandPinUnpinGracePeriod); grace <= 0 { + return fmt.Errorf("OnDemandPinning.UnpinGracePeriod must be positive, got %v", grace) + } + return nil +} diff --git a/config/ondemandpin_test.go b/config/ondemandpin_test.go new file mode 100644 index 00000000000..5acf00cc00e --- /dev/null +++ b/config/ondemandpin_test.go @@ -0,0 +1,33 @@ +package config + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestValidateOnDemandPinningConfig(t *testing.T) { + t.Run("defaults are valid", func(t *testing.T) { + err := ValidateOnDemandPinningConfig(&OnDemandPinning{}) + assert.NoError(t, err) + }) + + t.Run("zero replication target is rejected", func(t *testing.T) { + cfg := &OnDemandPinning{ReplicationTarget: *NewOptionalInteger(0)} + err := ValidateOnDemandPinningConfig(cfg) + assert.ErrorContains(t, err, "ReplicationTarget") + }) + + t.Run("zero check interval is rejected", func(t *testing.T) { + cfg := &OnDemandPinning{CheckInterval: *NewOptionalDuration(0)} + err := ValidateOnDemandPinningConfig(cfg) + assert.ErrorContains(t, err, "CheckInterval") + }) + + t.Run("negative grace period is rejected", func(t *testing.T) { + cfg := &OnDemandPinning{UnpinGracePeriod: *NewOptionalDuration(-time.Hour)} + err := ValidateOnDemandPinningConfig(cfg) + assert.ErrorContains(t, err, "UnpinGracePeriod") + }) +} diff --git a/core/node/groups.go b/core/node/groups.go index a121c7b26a4..6c60d2f03e2 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -450,6 +450,13 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option { return fx.Error(err) } + // Only validate when enabled; unused sections must not block startup when disabled. + if cfg.Experimental.OnDemandPinningEnabled { + if err := config.ValidateOnDemandPinningConfig(&cfg.OnDemandPinning); err != nil { + return fx.Error(err) + } + } + // Directory sharding settings from Import config. // These globals affect both `ipfs add` and MFS (`ipfs files` API). shardSizeThreshold := cfg.Import.UnixFSHAMTDirectorySizeThreshold.WithDefault(config.DefaultUnixFSHAMTDirectorySizeThreshold) diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go index 64ae3f8b849..1db9930e970 100644 --- a/ondemandpin/checker.go +++ b/ondemandpin/checker.go @@ -8,6 +8,7 @@ import ( "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "github.com/ipfs/kubo/config" + "github.com/libp2p/go-libp2p-kad-dht/amino" peer "github.com/libp2p/go-libp2p/core/peer" routing "github.com/libp2p/go-libp2p/core/routing" ) @@ -86,6 +87,12 @@ func (c *Checker) Run(ctx context.Context) { log.Info("on-demand pin checker started") defer log.Info("on-demand pin checker stopped") + // Warn when grace period is shorter than record validity (allowed for tests; risky on public DHT). + if c.unpinGracePeriod < amino.DefaultProvideValidity { + log.Warnw("UnpinGracePeriod is shorter than the DHT provider record validity; provider counts may include dead peers and this node may unpin the last live copy", + "gracePeriod", c.unpinGracePeriod, "recordValidity", amino.DefaultProvideValidity) + } + ticker := time.NewTicker(c.checkInterval) defer ticker.Stop() From d8d5a54289ffa0309651ae184427b863152f8282 Mon Sep 17 00:00:00 2001 From: ihlec Date: Tue, 21 Jul 2026 16:57:54 +0200 Subject: [PATCH 08/17] fix(ondemandpin): key ownership on kubo:on-demand pin name only --- core/commands/pin/ondemandpin.go | 10 +++++--- ondemandpin/checker.go | 42 +++++++++++++++----------------- ondemandpin/checker_test.go | 21 +++++++++++++++- ondemandpin/store.go | 4 +-- ondemandpin/store_test.go | 1 - 5 files changed, 48 insertions(+), 30 deletions(-) diff --git a/core/commands/pin/ondemandpin.go b/core/commands/pin/ondemandpin.go index ca9a5cabdfa..8ec6acae52b 100644 --- a/core/commands/pin/ondemandpin.go +++ b/core/commands/pin/ondemandpin.go @@ -144,11 +144,11 @@ Works when on-demand pinning is disabled, to clear old registrations. } // Only remove pins carrying the Kubo-internal kubo:on-demand name; user pins on the same CID are kept. - isOurs, err := ondemandpin.PinHasName(req.Context, n.Pinning, c, ondemandpin.OnDemandPinName) + hasOnDemandPin, err := ondemandpin.PinHasName(req.Context, n.Pinning, c, ondemandpin.OnDemandPinName) if err != nil { return fmt.Errorf("checking pin state for %s: %w", c, err) } - if isOurs { + if hasOnDemandPin { if err := api.Pin().Rm(req.Context, rp); err != nil { return fmt.Errorf("unpinning %s: %w", c, err) } @@ -245,9 +245,13 @@ Use --live to include real-time provider counts from the DHT. } for _, rec := range records { + hasOnDemandPin, err := ondemandpin.PinHasName(req.Context, n.Pinning, rec.Cid, ondemandpin.OnDemandPinName) + if err != nil { + return fmt.Errorf("checking pin state for %s: %w", rec.Cid, err) + } out := OnDemandLsOutput{ Cid: rec.Cid.String(), - PinnedByUs: rec.PinnedByUs, + PinnedByUs: hasOnDemandPin, CreatedAt: rec.CreatedAt.Format(time.RFC3339), } if !rec.LastAboveTarget.IsZero() { diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go index 1db9930e970..0fd1bf0fe17 100644 --- a/ondemandpin/checker.go +++ b/ondemandpin/checker.go @@ -141,10 +141,10 @@ func (c *Checker) checkOne(ctx context.Context, ci cid.Cid) { // checkRecord evaluates a single on-demand pin record in three phases: // -// 1. Guard: if the CID already has an external pin (not created by us), skip it to avoid interfering with user-managed pins. -// We use PinService.IsPinned (any pin) here โ€” not Record.PinnedByUs (on-demand pins). +// 1. Guard: if the CID has a pin without the reserved OnDemandPinName, skip it. // 2. Under-replicated: if provider count is low, pin locally. -// 3. Well-replicated: if provider count is high for a full grace period, unpin (on-demand pins only). +// 3. Well-replicated: if provider count is high for a full grace period, unpin +// only when the pin still has OnDemandPinName. func (c *Checker) checkRecord(ctx context.Context, rec *Record) { ctx, cancel := context.WithTimeout(ctx, checkTimeout) defer cancel() @@ -154,24 +154,29 @@ func (c *Checker) checkRecord(ctx context.Context, rec *Record) { log.Errorw("failed to check pin state, skipping CID", "cid", rec.Cid, "error", err) return } - if !rec.PinnedByUs && pinned { + hasOnDemandPin, err := c.pins.HasPinWithName(ctx, rec.Cid, OnDemandPinName) + if err != nil { + log.Errorw("failed to check pin name, skipping CID", "cid", rec.Cid, "error", err) + return + } + if pinned && !hasOnDemandPin { log.Debugw("skipping: CID has a user-managed pin", "cid", rec.Cid) return } count := CountProviders(ctx, c.routing, c.selfID, rec.Cid, c.replicationTarget) - log.Debugw("provider count", "cid", rec.Cid, "count", count, "target", c.replicationTarget, "pinnedByUs", rec.PinnedByUs) + log.Debugw("provider count", "cid", rec.Cid, "count", count, "target", c.replicationTarget, "hasOnDemandPin", hasOnDemandPin) if count < c.replicationTarget { - c.handleUnderReplicated(ctx, rec, count, pinned) + c.handleUnderReplicated(ctx, rec, count, hasOnDemandPin) } else { - c.handleWellReplicated(ctx, rec, count) + c.handleWellReplicated(ctx, rec, count, hasOnDemandPin) } } -// handleUnderReplicated pins the CID if it isn't already pinned. -func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count int, pinExists bool) { - if rec.PinnedByUs && pinExists { +// handleUnderReplicated pins the CID if it does not already have OnDemandPinName. +func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count int, hasOnDemandPin bool) { + if hasOnDemandPin { if !rec.LastAboveTarget.IsZero() { rec.LastAboveTarget = time.Time{} c.saveRecord(ctx, rec) @@ -179,11 +184,6 @@ func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count return } - if rec.PinnedByUs { - log.Warnw("pin was removed externally, will re-pin", "cid", rec.Cid) - rec.PinnedByUs = false - } - if !c.hasStorageBudget(ctx) { log.Warnw("skipping pin: repo near storage limit", "cid", rec.Cid) return @@ -204,7 +204,6 @@ func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count log.Errorw("failed to pin", "cid", rec.Cid, "error", err) return } - rec.PinnedByUs = true rec.LastAboveTarget = time.Time{} log.Infow("pinned", "cid", rec.Cid, "providers", count, "target", c.replicationTarget) @@ -214,9 +213,9 @@ func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count c.saveRecord(ctx, rec) } -// handleWellReplicated manages grace-period-then-unpin. -func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count int) { - if !rec.PinnedByUs { +// handleWellReplicated manages grace-period-then-unpin for pins with OnDemandPinName. +func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count int, hasOnDemandPin bool) { + if !hasOnDemandPin { return } @@ -231,13 +230,13 @@ func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count i return } - hasOurPin, err := c.pins.HasPinWithName(ctx, rec.Cid, OnDemandPinName) + stillOnDemand, err := c.pins.HasPinWithName(ctx, rec.Cid, OnDemandPinName) if err != nil { log.Errorw("failed to check pin name, skipping unpin", "cid", rec.Cid, "error", err) return } - if hasOurPin { + if stillOnDemand { if err := c.pins.Unpin(ctx, rec.Cid); err != nil { log.Errorw("failed to unpin", "cid", rec.Cid, "error", err) return @@ -247,7 +246,6 @@ func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count i log.Infow("relinquishing management: pin name changed externally", "cid", rec.Cid) } - rec.PinnedByUs = false rec.LastAboveTarget = time.Time{} c.saveRecord(ctx, rec) } diff --git a/ondemandpin/checker_test.go b/ondemandpin/checker_test.go index 8fe1af43b9d..25b3eb620a2 100644 --- a/ondemandpin/checker_test.go +++ b/ondemandpin/checker_test.go @@ -175,7 +175,7 @@ func (r *pinDuringLookupRouting) FindProvidersAsync(ctx context.Context, c cid.C return r.mockRouting.FindProvidersAsync(ctx, c, limit) } -// / Re-check before Pin: a user pin that landed during the DHT lookup must not be overwritten. +// Re-check before Pin: a user pin that landed during the DHT lookup must not be overwritten. func TestCheckerSkipsPinCreatedDuringLookup(t *testing.T) { ctx := context.Background() store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) @@ -193,3 +193,22 @@ func TestCheckerSkipsPinCreatedDuringLookup(t *testing.T) { assert.Equal(t, "user-pin", p.pinned[c]) } + +// A pin with the reserved name is managed even if the store never recorded a +// separate ownership flag (the old crash window between Pin and saveRecord). +func TestCheckerOwnsPinByNameNotStoreField(t *testing.T) { + ctx := context.Background() + checker, store, r, p, clock := newTestChecker(t) + c := testCID(t, "name-owned") + + require.NoError(t, store.Add(ctx, c)) + require.NoError(t, p.Pin(ctx, c, OnDemandPinName)) + r.setProviders(c, peer.ID("p1"), peer.ID("p2"), peer.ID("p3"), peer.ID("p4"), peer.ID("p5"), peer.ID("p6")) + + checker.checkAll(ctx) + assert.True(t, p.isPinned(c), "grace period just started") + + clock.Advance(250 * time.Millisecond) + checker.checkAll(ctx) + assert.False(t, p.isPinned(c), "name-owned pin must unpin after grace") +} diff --git a/ondemandpin/store.go b/ondemandpin/store.go index 0ac2b86893d..6eacf019f0b 100644 --- a/ondemandpin/store.go +++ b/ondemandpin/store.go @@ -21,9 +21,7 @@ var ( const dsPrefix = "/ondemand-pins/" type Record struct { - Cid cid.Cid `json:"Cid"` - // PinnedByUs tracks the on-demand pins only (does not include standard pins). - PinnedByUs bool `json:"IsPinned"` + Cid cid.Cid `json:"Cid"` LastAboveTarget time.Time `json:"LastAboveTarget,omitempty"` CreatedAt time.Time `json:"CreatedAt"` } diff --git a/ondemandpin/store_test.go b/ondemandpin/store_test.go index d7718cd3086..d8bea2003c2 100644 --- a/ondemandpin/store_test.go +++ b/ondemandpin/store_test.go @@ -35,7 +35,6 @@ func TestStoreRoundTrip(t *testing.T) { rec, err := s.Get(ctx, c) require.NoError(t, err) assert.Equal(t, c, rec.Cid) - assert.False(t, rec.PinnedByUs) require.NoError(t, s.Remove(ctx, c)) From 212f15e74b0e96961f99453f9e3290ac78094c3a Mon Sep 17 00:00:00 2001 From: ihlec Date: Tue, 21 Jul 2026 17:39:37 +0200 Subject: [PATCH 09/17] fix(ondemandpin): refuse Null routing and treat failed lookups as unknown replica number --- config/ondemandpin.go | 9 +++ config/ondemandpin_test.go | 7 +++ core/commands/pin/ondemandpin.go | 6 +- core/node/groups.go | 4 ++ ondemandpin/checker.go | 18 ++++-- ondemandpin/checker_test.go | 94 ++++++++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 6 deletions(-) diff --git a/config/ondemandpin.go b/config/ondemandpin.go index dde2a4a83e4..c6e7aadedae 100644 --- a/config/ondemandpin.go +++ b/config/ondemandpin.go @@ -41,3 +41,12 @@ func ValidateOnDemandPinningConfig(cfg *OnDemandPinning) error { } return nil } + +// ValidateOnDemandPinningRouting rejects Routing.Type=none. +// Otherwise the checker would pin every registered CID on a null router. +func ValidateOnDemandPinningRouting(routingType string) error { + if routingType == "none" { + return fmt.Errorf("on-demand pinning requires a routing system that can answer provider queries; Routing.Type=%q cannot", routingType) + } + return nil +} diff --git a/config/ondemandpin_test.go b/config/ondemandpin_test.go index 5acf00cc00e..73b2e91e03d 100644 --- a/config/ondemandpin_test.go +++ b/config/ondemandpin_test.go @@ -31,3 +31,10 @@ func TestValidateOnDemandPinningConfig(t *testing.T) { assert.ErrorContains(t, err, "UnpinGracePeriod") }) } + +func TestValidateOnDemandPinningRouting(t *testing.T) { + assert.NoError(t, ValidateOnDemandPinningRouting("auto")) + assert.NoError(t, ValidateOnDemandPinningRouting("dht")) + assert.NoError(t, ValidateOnDemandPinningRouting("delegated")) + assert.ErrorContains(t, ValidateOnDemandPinningRouting("none"), "Routing.Type") +} diff --git a/core/commands/pin/ondemandpin.go b/core/commands/pin/ondemandpin.go index 8ec6acae52b..43ff0feee3d 100644 --- a/core/commands/pin/ondemandpin.go +++ b/core/commands/pin/ondemandpin.go @@ -259,8 +259,10 @@ Use --live to include real-time provider counts from the DHT. } if live && n.Routing != nil { - count := ondemandpin.CountProviders(req.Context, n.Routing, n.Identity, rec.Cid, globalTarget) - out.Providers = &count + count, ok := ondemandpin.CountProviders(req.Context, n.Routing, n.Identity, rec.Cid, globalTarget) + if ok { + out.Providers = &count + } } if err := res.Emit(&out); err != nil { diff --git a/core/node/groups.go b/core/node/groups.go index 6c60d2f03e2..974b4229b64 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -455,6 +455,10 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option { if err := config.ValidateOnDemandPinningConfig(&cfg.OnDemandPinning); err != nil { return fx.Error(err) } + routingType := cfg.Routing.Type.WithDefault(config.DefaultRoutingType) + if err := config.ValidateOnDemandPinningRouting(routingType); err != nil { + return fx.Error(err) + } } // Directory sharding settings from Import config. diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go index 0fd1bf0fe17..c5eb9eb27ee 100644 --- a/ondemandpin/checker.go +++ b/ondemandpin/checker.go @@ -164,7 +164,12 @@ func (c *Checker) checkRecord(ctx context.Context, rec *Record) { return } - count := CountProviders(ctx, c.routing, c.selfID, rec.Cid, c.replicationTarget) + // Warn if Cancel/timeout and count < target (do not treat these as under-replicated unless DHT walk finished far enough.) + count, ok := CountProviders(ctx, c.routing, c.selfID, rec.Cid, c.replicationTarget) + if !ok { + log.Warnw("provider count unknown (lookup cancelled or timed out), skipping CID", "cid", rec.Cid) + return + } log.Debugw("provider count", "cid", rec.Cid, "count", count, "target", c.replicationTarget, "hasOnDemandPin", hasOnDemandPin) if count < c.replicationTarget { @@ -271,8 +276,9 @@ func (c *Checker) hasStorageBudget(ctx context.Context) bool { return used < limit } -// CountProviders counts unique providers with target+1 results to ensure that, even if selfID appears, we still discover up to target. -func CountProviders(ctx context.Context, cr routing.ContentRouting, selfID peer.ID, c cid.Cid, target int) int { +// CountProviders returns unique providers (asks for target+1 so self can be skipped) +// and ok=false if the lookup was cancelled before count reached target. +func CountProviders(ctx context.Context, cr routing.ContentRouting, selfID peer.ID, c cid.Cid, target int) (count int, ok bool) { ch := cr.FindProvidersAsync(ctx, c, target+1) seen := make(map[peer.ID]struct{}) @@ -282,7 +288,11 @@ func CountProviders(ctx context.Context, cr routing.ContentRouting, selfID peer. } seen[pi.ID] = struct{}{} } - return len(seen) + count = len(seen) + if ctx.Err() != nil && count < target { + return count, false + } + return count, true } // PinHasName is used by checker (via PinService.HasPinWithName) and the rm command to identify pins managed by on-demand pinning. diff --git a/ondemandpin/checker_test.go b/ondemandpin/checker_test.go index 25b3eb620a2..468cd1b29ea 100644 --- a/ondemandpin/checker_test.go +++ b/ondemandpin/checker_test.go @@ -212,3 +212,97 @@ func TestCheckerOwnsPinByNameNotStoreField(t *testing.T) { checker.checkAll(ctx) assert.False(t, p.isPinned(c), "name-owned pin must unpin after grace") } + +// blockingRouting closes only when ctx is cancelled, mimicking a hung lookup. +type blockingRouting struct{} + +func (blockingRouting) FindProvidersAsync(ctx context.Context, _ cid.Cid, _ int) <-chan peer.AddrInfo { + ch := make(chan peer.AddrInfo) + go func() { + defer close(ch) + <-ctx.Done() + }() + return ch +} + +func (blockingRouting) Provide(context.Context, cid.Cid, bool) error { return nil } + +func TestCountProvidersUnknownOnCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + count, ok := CountProviders(ctx, blockingRouting{}, peer.ID("self"), testCID(t, "unknown"), 5) + assert.False(t, ok) + assert.Equal(t, 0, count) +} + +// emitThenBlockRouting emits providers, then blocks until ctx cancel. +type emitThenBlockRouting struct { + providers []peer.ID +} + +func (r emitThenBlockRouting) FindProvidersAsync(ctx context.Context, _ cid.Cid, limit int) <-chan peer.AddrInfo { + ch := make(chan peer.AddrInfo) + go func() { + defer close(ch) + for i, id := range r.providers { + if i >= limit { + break + } + select { + case ch <- peer.AddrInfo{ID: id}: + case <-ctx.Done(): + return + } + } + <-ctx.Done() + }() + return ch +} + +func (emitThenBlockRouting) Provide(context.Context, cid.Cid, bool) error { return nil } + +func TestCountProvidersOkWhenEnoughFoundDespiteCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + r := emitThenBlockRouting{providers: []peer.ID{"p1", "p2", "p3", "p4", "p5"}} + + done := make(chan struct{}) + var count int + var ok bool + go func() { + defer close(done) + count, ok = CountProviders(ctx, r, peer.ID("self"), testCID(t, "enough"), 5) + }() + + time.Sleep(20 * time.Millisecond) // let providers flush + cancel() + <-done + + require.True(t, ok, "count >= target is reliable even if the lookup is then cancelled") + assert.Equal(t, 5, count) +} + +// A cancelled/timed-out provider lookup must not be treated as zero providers. +func TestCheckerSkipsWhenProviderCountUnknown(t *testing.T) { + ctx := context.Background() + store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) + p := newMockPins() + checker := NewChecker(store, p, nil, blockingRouting{}, peer.ID("self"), config.OnDemandPinning{}) + checker.now = newFakeClock().Now + + c := testCID(t, "hung-lookup") + require.NoError(t, store.Add(ctx, c)) + + checkCtx, cancel := context.WithTimeout(ctx, 20*time.Millisecond) + defer cancel() + checker.checkRecord(checkCtx, mustGet(t, store, c)) + + assert.False(t, p.isPinned(c)) +} + +func mustGet(t *testing.T, store *Store, c cid.Cid) *Record { + t.Helper() + rec, err := store.Get(context.Background(), c) + require.NoError(t, err) + return rec +} From bc6fb3f507cc0e12febdf676a36de8f005046a9d Mon Sep 17 00:00:00 2001 From: ihlec Date: Tue, 21 Jul 2026 18:59:56 +0200 Subject: [PATCH 10/17] fix(ondemandpin): add min/max replication deadband and grace jitter --- config/ondemandpin.go | 28 ++++++---- config/ondemandpin_test.go | 17 ++++-- core/commands/pin/ondemandpin.go | 7 +-- docs/changelogs/vFUTURE.md | 5 +- docs/config.md | 29 ++++++---- docs/experimental-features.md | 13 ++--- ondemandpin/checker.go | 92 ++++++++++++++++++++------------ ondemandpin/checker_test.go | 60 +++++++++++++++------ ondemandpin/store.go | 1 + 9 files changed, 169 insertions(+), 83 deletions(-) diff --git a/config/ondemandpin.go b/config/ondemandpin.go index c6e7aadedae..918da01649e 100644 --- a/config/ondemandpin.go +++ b/config/ondemandpin.go @@ -8,8 +8,9 @@ import ( ) const ( - DefaultOnDemandPinReplicationTarget = 5 - DefaultOnDemandPinCheckInterval = 10 * time.Minute + DefaultOnDemandPinReplicationTargetMin = 5 + DefaultOnDemandPinReplicationTargetMax = 7 + DefaultOnDemandPinCheckInterval = 10 * time.Minute // Must exceed amino.DefaultProvideValidity so stale DHT records expire // before unpin. The extra day covers check-interval skew. @@ -17,21 +18,28 @@ const ( ) type OnDemandPinning struct { - // Minimum providers desired in the DHT (excluding self). - ReplicationTarget OptionalInteger + // Pin when fewer than this many providers are found in the DHT (excluding self). + ReplicationTargetMin OptionalInteger + + // Start the unpin grace period only when more than this many providers are found. + ReplicationTargetMax OptionalInteger // How often the checker evaluates all registered CIDs. CheckInterval OptionalDuration - // How long replication must stay above target before unpinning. + // How long replication must stay above max before unpinning; checker adds up to 2*CheckInterval of jitter. UnpinGracePeriod OptionalDuration } -// ValidateOnDemandPinningConfig rejects non-positive intervals/grace periods -// and ReplicationTarget < 1 (ticker panic / never pins, then unpins). +// ValidateOnDemandPinningConfig rejects invalid min/max and non-positive durations. func ValidateOnDemandPinningConfig(cfg *OnDemandPinning) error { - if target := cfg.ReplicationTarget.WithDefault(DefaultOnDemandPinReplicationTarget); target < 1 { - return fmt.Errorf("OnDemandPinning.ReplicationTarget must be at least 1, got %d", target) + min := cfg.ReplicationTargetMin.WithDefault(DefaultOnDemandPinReplicationTargetMin) + max := cfg.ReplicationTargetMax.WithDefault(DefaultOnDemandPinReplicationTargetMax) + if min < 1 { + return fmt.Errorf("OnDemandPinning.ReplicationTargetMin must be at least 1, got %d", min) + } + if max < min { + return fmt.Errorf("OnDemandPinning.ReplicationTargetMax (%d) must be >= ReplicationTargetMin (%d)", max, min) } if interval := cfg.CheckInterval.WithDefault(DefaultOnDemandPinCheckInterval); interval <= 0 { return fmt.Errorf("OnDemandPinning.CheckInterval must be positive, got %v", interval) @@ -46,7 +54,7 @@ func ValidateOnDemandPinningConfig(cfg *OnDemandPinning) error { // Otherwise the checker would pin every registered CID on a null router. func ValidateOnDemandPinningRouting(routingType string) error { if routingType == "none" { - return fmt.Errorf("on-demand pinning requires a routing system that can answer provider queries; Routing.Type=%q cannot", routingType) + return fmt.Errorf("on-demand pinning needs provider lookups; Routing.Type=%q is not usable", routingType) } return nil } diff --git a/config/ondemandpin_test.go b/config/ondemandpin_test.go index 73b2e91e03d..8220febf61e 100644 --- a/config/ondemandpin_test.go +++ b/config/ondemandpin_test.go @@ -13,10 +13,19 @@ func TestValidateOnDemandPinningConfig(t *testing.T) { assert.NoError(t, err) }) - t.Run("zero replication target is rejected", func(t *testing.T) { - cfg := &OnDemandPinning{ReplicationTarget: *NewOptionalInteger(0)} + t.Run("zero replication min is rejected", func(t *testing.T) { + cfg := &OnDemandPinning{ReplicationTargetMin: *NewOptionalInteger(0)} err := ValidateOnDemandPinningConfig(cfg) - assert.ErrorContains(t, err, "ReplicationTarget") + assert.ErrorContains(t, err, "ReplicationTargetMin") + }) + + t.Run("max below min is rejected", func(t *testing.T) { + cfg := &OnDemandPinning{ + ReplicationTargetMin: *NewOptionalInteger(5), + ReplicationTargetMax: *NewOptionalInteger(4), + } + err := ValidateOnDemandPinningConfig(cfg) + assert.ErrorContains(t, err, "ReplicationTargetMax") }) t.Run("zero check interval is rejected", func(t *testing.T) { @@ -36,5 +45,5 @@ func TestValidateOnDemandPinningRouting(t *testing.T) { assert.NoError(t, ValidateOnDemandPinningRouting("auto")) assert.NoError(t, ValidateOnDemandPinningRouting("dht")) assert.NoError(t, ValidateOnDemandPinningRouting("delegated")) - assert.ErrorContains(t, ValidateOnDemandPinningRouting("none"), "Routing.Type") + assert.ErrorContains(t, ValidateOnDemandPinningRouting("none"), "none") } diff --git a/core/commands/pin/ondemandpin.go b/core/commands/pin/ondemandpin.go index 43ff0feee3d..2a68c6b5c4e 100644 --- a/core/commands/pin/ondemandpin.go +++ b/core/commands/pin/ondemandpin.go @@ -207,13 +207,14 @@ Use --live to include real-time provider counts from the DHT. live, _ := req.Options[onDemandLiveOptionName].(bool) - var globalTarget int + var replicationMin, replicationMax int if live { cfg, err := n.Repo.Config() if err != nil { return err } - globalTarget = int(cfg.OnDemandPinning.ReplicationTarget.WithDefault(config.DefaultOnDemandPinReplicationTarget)) + replicationMin = int(cfg.OnDemandPinning.ReplicationTargetMin.WithDefault(config.DefaultOnDemandPinReplicationTargetMin)) + replicationMax = int(cfg.OnDemandPinning.ReplicationTargetMax.WithDefault(config.DefaultOnDemandPinReplicationTargetMax)) } var records []ondemandpin.Record @@ -259,7 +260,7 @@ Use --live to include real-time provider counts from the DHT. } if live && n.Routing != nil { - count, ok := ondemandpin.CountProviders(req.Context, n.Routing, n.Identity, rec.Cid, globalTarget) + count, ok := ondemandpin.CountProviders(req.Context, n.Routing, n.Identity, rec.Cid, replicationMin, replicationMax) if ok { out.Providers = &count } diff --git a/docs/changelogs/vFUTURE.md b/docs/changelogs/vFUTURE.md index a2dee550386..bafbd202f5b 100644 --- a/docs/changelogs/vFUTURE.md +++ b/docs/changelogs/vFUTURE.md @@ -56,9 +56,10 @@ Configuration at [`OnDemandPinning`](https://github.com/ipfs/kubo/blob/master/do | Option | Default | Description | |---|---|---| -| `OnDemandPinning.ReplicationTarget` | `5` | Minimum providers in DHT (excluding self) | +| `OnDemandPinning.ReplicationTargetMin` | `5` | Pin when fewer than this many providers (excluding self) | +| `OnDemandPinning.ReplicationTargetMax` | `7` | Start unpin grace only above this many providers | | `OnDemandPinning.CheckInterval` | `"10m"` | How often the checker runs | -| `OnDemandPinning.UnpinGracePeriod` | `"72h"` | How long above target before unpinning (outlasts the 48h DHT provider record validity) | +| `OnDemandPinning.UnpinGracePeriod` | `"72h"` | How long above max before unpinning (longer than 48h DHT record validity; plus up to `2 * CheckInterval` jitter) | See [experimental features](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#on-demand-pinning) for full documentation. diff --git a/docs/config.md b/docs/config.md index 69ca814e5f2..73db68d2160 100644 --- a/docs/config.md +++ b/docs/config.md @@ -127,7 +127,8 @@ config file at runtime. - [`Mounts.StoreMtime`](#mountsstoremtime) - [`Mounts.StoreMode`](#mountsstoremode) - [`OnDemandPinning`](#ondemandpinning) - - [`OnDemandPinning.ReplicationTarget`](#ondemandpinningreplicationtarget) + - [`OnDemandPinning.ReplicationTargetMin`](#ondemandpinningreplicationtargetmin) + - [`OnDemandPinning.ReplicationTargetMax`](#ondemandpinningreplicationtargetmax) - [`OnDemandPinning.CheckInterval`](#ondemandpinningcheckinterval) - [`OnDemandPinning.UnpinGracePeriod`](#ondemandpinningunpingraceperiod) @@ -2249,15 +2250,25 @@ Type: `duration` Configures the on-demand pinning system. Requires [`Experimental.OnDemandPinningEnabled`](#experimentalondemandpinningenabled). -### `OnDemandPinning.ReplicationTarget` +### `OnDemandPinning.ReplicationTargetMin` -The minimum number of providers desired in the DHT (excluding the local node). -When fewer providers are found, the node pins the content locally. +Pin when fewer than this many providers are found in the DHT (excluding the +local node). Default: `5` Type: `optionalInteger` +### `OnDemandPinning.ReplicationTargetMax` + +Start the unpin grace period only when more than this many providers are found +(excluding the local node). Between min and max (inclusive) the checker does +nothing. Must be >= `ReplicationTargetMin`. + +Default: `7` + +Type: `optionalInteger` + ### `OnDemandPinning.CheckInterval` How often the background checker evaluates all on-demand pins. @@ -2268,12 +2279,12 @@ Type: `optionalDuration` ### `OnDemandPinning.UnpinGracePeriod` -How long replication must stay above target before the local pin is removed. -This prevents thrashing when provider counts fluctuate near the target -boundary. +How long the provider count must stay above max before the local pin is removed. +The checker also adds a random delay of up to `2 * CheckInterval` when grace +starts, so nodes that entered grace together do not all unpin at once. -Default must exceed DHT provider-record validity (48h), otherwise a node can -unpin while stale records still inflate the count and it holds the last live copy. +Should be longer than DHT provider-record validity (48h). A shorter value can +unpin while stale records still make the count look healthy. Default: `"72h"` diff --git a/docs/experimental-features.md b/docs/experimental-features.md index ecfad04e74f..61a0f290b33 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -616,10 +616,11 @@ ipfs config --json Experimental.GatewayOverLibp2p true Experimental, disabled by default. -On-demand pinning lets a node automatically pin content when DHT provider -counts fall below a configurable replication target, and unpin when -replication recovers above target for a grace period. Under-replicated -content gets pinned; storage is freed once enough other providers exist. +On-demand pinning lets a node pin content when DHT provider counts fall below +a minimum, and unpin after they stay above a maximum for a grace period +(plus a short random delay). Values between min and max are left alone. +ipfs-cluster replication factors assign pins among known peers; this feature +only watches the DHT and decides locally. The feature consists of: @@ -650,8 +651,8 @@ ipfs config --json Experimental.OnDemandPinningEnabled true ### Configuring See [`OnDemandPinning`](https://github.com/ipfs/kubo/blob/master/docs/config.md#ondemandpinning) -for tunable parameters: `ReplicationTarget`, `CheckInterval`, -and `UnpinGracePeriod`. +for tunable parameters: `ReplicationTargetMin`, `ReplicationTargetMax`, +`CheckInterval`, and `UnpinGracePeriod`. ### Basic usage diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go index c5eb9eb27ee..f7814b571d9 100644 --- a/ondemandpin/checker.go +++ b/ondemandpin/checker.go @@ -2,6 +2,7 @@ package ondemandpin import ( "context" + "math/rand/v2" "time" pin "github.com/ipfs/boxo/pinning/pinner" @@ -42,12 +43,14 @@ type Checker struct { routing routing.ContentRouting selfID peer.ID - replicationTarget int - checkInterval time.Duration - unpinGracePeriod time.Duration + replicationMin int + replicationMax int + checkInterval time.Duration + unpinGracePeriod time.Duration - now func() time.Time - priorityCh chan cid.Cid + now func() time.Time + graceJitter func() time.Duration + priorityCh chan cid.Cid } func NewChecker( @@ -58,20 +61,31 @@ func NewChecker( selfID peer.ID, cfg config.OnDemandPinning, ) *Checker { - return &Checker{ + c := &Checker{ store: store, pins: pins, storage: storage, routing: cr, selfID: selfID, - replicationTarget: int(cfg.ReplicationTarget.WithDefault(config.DefaultOnDemandPinReplicationTarget)), - checkInterval: cfg.CheckInterval.WithDefault(config.DefaultOnDemandPinCheckInterval), - unpinGracePeriod: cfg.UnpinGracePeriod.WithDefault(config.DefaultOnDemandPinUnpinGracePeriod), + replicationMin: int(cfg.ReplicationTargetMin.WithDefault(config.DefaultOnDemandPinReplicationTargetMin)), + replicationMax: int(cfg.ReplicationTargetMax.WithDefault(config.DefaultOnDemandPinReplicationTargetMax)), + checkInterval: cfg.CheckInterval.WithDefault(config.DefaultOnDemandPinCheckInterval), + unpinGracePeriod: cfg.UnpinGracePeriod.WithDefault(config.DefaultOnDemandPinUnpinGracePeriod), now: time.Now, priorityCh: make(chan cid.Cid, 64), } + c.graceJitter = c.defaultGraceJitter + return c +} + +func (c *Checker) defaultGraceJitter() time.Duration { + maxJitter := 2 * c.checkInterval + if maxJitter <= 0 { + return 0 + } + return time.Duration(rand.Int64N(int64(maxJitter))) } func (c *Checker) Enqueue(ci cid.Cid) { @@ -139,12 +153,8 @@ func (c *Checker) checkOne(ctx context.Context, ci cid.Cid) { c.checkRecord(ctx, rec) } -// checkRecord evaluates a single on-demand pin record in three phases: -// -// 1. Guard: if the CID has a pin without the reserved OnDemandPinName, skip it. -// 2. Under-replicated: if provider count is low, pin locally. -// 3. Well-replicated: if provider count is high for a full grace period, unpin -// only when the pin still has OnDemandPinName. +// checkRecord pins below min, starts a jittered grace period above max, and +// clears grace in the deadband. Skips CIDs with a non-on-demand pin. func (c *Checker) checkRecord(ctx context.Context, rec *Record) { ctx, cancel := context.WithTimeout(ctx, checkTimeout) defer cancel() @@ -164,28 +174,27 @@ func (c *Checker) checkRecord(ctx context.Context, rec *Record) { return } - // Warn if Cancel/timeout and count < target (do not treat these as under-replicated unless DHT walk finished far enough.) - count, ok := CountProviders(ctx, c.routing, c.selfID, rec.Cid, c.replicationTarget) + count, ok := CountProviders(ctx, c.routing, c.selfID, rec.Cid, c.replicationMin, c.replicationMax) if !ok { log.Warnw("provider count unknown (lookup cancelled or timed out), skipping CID", "cid", rec.Cid) return } - log.Debugw("provider count", "cid", rec.Cid, "count", count, "target", c.replicationTarget, "hasOnDemandPin", hasOnDemandPin) + log.Debugw("provider count", "cid", rec.Cid, "count", count, "min", c.replicationMin, "max", c.replicationMax, "hasOnDemandPin", hasOnDemandPin) - if count < c.replicationTarget { + switch { + case count < c.replicationMin: c.handleUnderReplicated(ctx, rec, count, hasOnDemandPin) - } else { + case count > c.replicationMax: c.handleWellReplicated(ctx, rec, count, hasOnDemandPin) + default: + c.clearGrace(ctx, rec) } } // handleUnderReplicated pins the CID if it does not already have OnDemandPinName. func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count int, hasOnDemandPin bool) { if hasOnDemandPin { - if !rec.LastAboveTarget.IsZero() { - rec.LastAboveTarget = time.Time{} - c.saveRecord(ctx, rec) - } + c.clearGrace(ctx, rec) return } @@ -210,7 +219,8 @@ func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count return } rec.LastAboveTarget = time.Time{} - log.Infow("pinned", "cid", rec.Cid, "providers", count, "target", c.replicationTarget) + rec.UnpinAt = time.Time{} + log.Infow("pinned", "cid", rec.Cid, "providers", count, "min", c.replicationMin) if err := c.routing.Provide(ctx, rec.Cid, true); err != nil { log.Warnw("failed to provide after pin", "cid", rec.Cid, "error", err) @@ -225,13 +235,16 @@ func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count i } if rec.LastAboveTarget.IsZero() { - rec.LastAboveTarget = c.now() + now := c.now() + jitter := c.graceJitter() + rec.LastAboveTarget = now + rec.UnpinAt = now.Add(c.unpinGracePeriod + jitter) c.saveRecord(ctx, rec) - log.Debugw("grace period started", "cid", rec.Cid, "providers", count, "target", c.replicationTarget) + log.Debugw("grace period started", "cid", rec.Cid, "providers", count, "max", c.replicationMax, "unpinAt", rec.UnpinAt, "jitter", jitter) return } - if c.now().Sub(rec.LastAboveTarget) < c.unpinGracePeriod { + if c.now().Before(rec.UnpinAt) { return } @@ -246,12 +259,22 @@ func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count i log.Errorw("failed to unpin", "cid", rec.Cid, "error", err) return } - log.Infow("unpinned", "cid", rec.Cid, "providers", count, "target", c.replicationTarget) + log.Infow("unpinned", "cid", rec.Cid, "providers", count, "max", c.replicationMax) } else { log.Infow("relinquishing management: pin name changed externally", "cid", rec.Cid) } rec.LastAboveTarget = time.Time{} + rec.UnpinAt = time.Time{} + c.saveRecord(ctx, rec) +} + +func (c *Checker) clearGrace(ctx context.Context, rec *Record) { + if rec.LastAboveTarget.IsZero() && rec.UnpinAt.IsZero() { + return + } + rec.LastAboveTarget = time.Time{} + rec.UnpinAt = time.Time{} c.saveRecord(ctx, rec) } @@ -276,10 +299,11 @@ func (c *Checker) hasStorageBudget(ctx context.Context) bool { return used < limit } -// CountProviders returns unique providers (asks for target+1 so self can be skipped) -// and ok=false if the lookup was cancelled before count reached target. -func CountProviders(ctx context.Context, cr routing.ContentRouting, selfID peer.ID, c cid.Cid, target int) (count int, ok bool) { - ch := cr.FindProvidersAsync(ctx, c, target+1) +// CountProviders counts providers excluding self. Asks for max+2 results so +// self can take a slot and we can still see max+1 others. +// ok is false if the lookup was cancelled before reaching min providers. +func CountProviders(ctx context.Context, cr routing.ContentRouting, selfID peer.ID, c cid.Cid, min, max int) (count int, ok bool) { + ch := cr.FindProvidersAsync(ctx, c, max+2) seen := make(map[peer.ID]struct{}) for pi := range ch { @@ -289,7 +313,7 @@ func CountProviders(ctx context.Context, cr routing.ContentRouting, selfID peer. seen[pi.ID] = struct{}{} } count = len(seen) - if ctx.Err() != nil && count < target { + if ctx.Err() != nil && count < min { return count, false } return count, true diff --git a/ondemandpin/checker_test.go b/ondemandpin/checker_test.go index 468cd1b29ea..597d0bad941 100644 --- a/ondemandpin/checker_test.go +++ b/ondemandpin/checker_test.go @@ -111,10 +111,19 @@ func newTestChecker(t *testing.T) (*Checker, *Store, *mockRouting, *mockPins, *f checker.checkInterval = time.Minute checker.unpinGracePeriod = 200 * time.Millisecond checker.now = clock.Now + checker.graceJitter = func() time.Duration { return 0 } return checker, store, r, p, clock } +func providers(n int) []peer.ID { + out := make([]peer.ID, n) + for i := range out { + out[i] = peer.ID(string(rune('a' + i))) + } + return out +} + // Under-replicated content gets pinned. func TestCheckerPinsBelowTarget(t *testing.T) { ctx := context.Background() @@ -129,21 +138,21 @@ func TestCheckerPinsBelowTarget(t *testing.T) { assert.True(t, p.isPinned(c)) } -// Well-replicated content is left alone. -func TestCheckerDoesNotPinAboveTarget(t *testing.T) { +// Content at or above min is left alone (including the deadband up to max). +func TestCheckerDoesNotPinInDeadband(t *testing.T) { ctx := context.Background() checker, store, r, p, _ := newTestChecker(t) - c := testCID(t, "well-replicated") + c := testCID(t, "deadband") require.NoError(t, store.Add(ctx, c)) - r.setProviders(c, peer.ID("p1"), peer.ID("p2"), peer.ID("p3"), peer.ID("p4"), peer.ID("p5"), peer.ID("p6")) + r.setProviders(c, providers(6)...) // min=5, max=7 checker.checkAll(ctx) assert.False(t, p.isPinned(c)) } -// Pinned content is unpinned only after the grace period expires. +// Pinned content is unpinned only after the grace period expires above max. func TestCheckerUnpinsAfterGracePeriod(t *testing.T) { ctx := context.Background() checker, store, r, p, clock := newTestChecker(t) @@ -154,8 +163,8 @@ func TestCheckerUnpinsAfterGracePeriod(t *testing.T) { checker.checkAll(ctx) require.True(t, p.isPinned(c)) - // Providers recover above target. - r.setProviders(c, peer.ID("p1"), peer.ID("p2"), peer.ID("p3"), peer.ID("p4"), peer.ID("p5"), peer.ID("p6")) + // Providers recover above max (default 7). + r.setProviders(c, providers(8)...) checker.checkAll(ctx) assert.True(t, p.isPinned(c), "not yet past grace period") @@ -164,6 +173,28 @@ func TestCheckerUnpinsAfterGracePeriod(t *testing.T) { assert.False(t, p.isPinned(c), "past grace period") } +func TestCheckerGraceIncludesJitter(t *testing.T) { + ctx := context.Background() + checker, store, r, p, clock := newTestChecker(t) + checker.graceJitter = func() time.Duration { return 100 * time.Millisecond } + c := testCID(t, "jitter") + + require.NoError(t, store.Add(ctx, c)) + require.NoError(t, p.Pin(ctx, c, OnDemandPinName)) + r.setProviders(c, providers(8)...) + + checker.checkAll(ctx) + assert.True(t, p.isPinned(c)) + + clock.Advance(250 * time.Millisecond) // grace only; jitter not yet elapsed + checker.checkAll(ctx) + assert.True(t, p.isPinned(c), "still within grace+jitter") + + clock.Advance(100 * time.Millisecond) + checker.checkAll(ctx) + assert.False(t, p.isPinned(c), "past grace+jitter") +} + type pinDuringLookupRouting struct { *mockRouting pins *mockPins @@ -175,7 +206,7 @@ func (r *pinDuringLookupRouting) FindProvidersAsync(ctx context.Context, c cid.C return r.mockRouting.FindProvidersAsync(ctx, c, limit) } -// Re-check before Pin: a user pin that landed during the DHT lookup must not be overwritten. +// Pin that appeared during the DHT lookup is left alone. func TestCheckerSkipsPinCreatedDuringLookup(t *testing.T) { ctx := context.Background() store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) @@ -194,8 +225,7 @@ func TestCheckerSkipsPinCreatedDuringLookup(t *testing.T) { assert.Equal(t, "user-pin", p.pinned[c]) } -// A pin with the reserved name is managed even if the store never recorded a -// separate ownership flag (the old crash window between Pin and saveRecord). +// Ownership follows the reserved pin name, not a store flag. func TestCheckerOwnsPinByNameNotStoreField(t *testing.T) { ctx := context.Background() checker, store, r, p, clock := newTestChecker(t) @@ -203,7 +233,7 @@ func TestCheckerOwnsPinByNameNotStoreField(t *testing.T) { require.NoError(t, store.Add(ctx, c)) require.NoError(t, p.Pin(ctx, c, OnDemandPinName)) - r.setProviders(c, peer.ID("p1"), peer.ID("p2"), peer.ID("p3"), peer.ID("p4"), peer.ID("p5"), peer.ID("p6")) + r.setProviders(c, providers(8)...) checker.checkAll(ctx) assert.True(t, p.isPinned(c), "grace period just started") @@ -231,7 +261,7 @@ func TestCountProvidersUnknownOnCancel(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - count, ok := CountProviders(ctx, blockingRouting{}, peer.ID("self"), testCID(t, "unknown"), 5) + count, ok := CountProviders(ctx, blockingRouting{}, peer.ID("self"), testCID(t, "unknown"), 5, 7) assert.False(t, ok) assert.Equal(t, 0, count) } @@ -271,18 +301,18 @@ func TestCountProvidersOkWhenEnoughFoundDespiteCancel(t *testing.T) { var ok bool go func() { defer close(done) - count, ok = CountProviders(ctx, r, peer.ID("self"), testCID(t, "enough"), 5) + count, ok = CountProviders(ctx, r, peer.ID("self"), testCID(t, "enough"), 5, 7) }() time.Sleep(20 * time.Millisecond) // let providers flush cancel() <-done - require.True(t, ok, "count >= target is reliable even if the lookup is then cancelled") + require.True(t, ok) assert.Equal(t, 5, count) } -// A cancelled/timed-out provider lookup must not be treated as zero providers. +// Cancelled provider lookup does not count as zero providers. func TestCheckerSkipsWhenProviderCountUnknown(t *testing.T) { ctx := context.Background() store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) diff --git a/ondemandpin/store.go b/ondemandpin/store.go index 6eacf019f0b..df8979dd91b 100644 --- a/ondemandpin/store.go +++ b/ondemandpin/store.go @@ -23,6 +23,7 @@ const dsPrefix = "/ondemand-pins/" type Record struct { Cid cid.Cid `json:"Cid"` LastAboveTarget time.Time `json:"LastAboveTarget,omitempty"` + UnpinAt time.Time `json:"UnpinAt,omitempty"` // grace deadline (includes jitter) CreatedAt time.Time `json:"CreatedAt"` } From fa1306839be7edd30342f41ec69bc186098508c6 Mon Sep 17 00:00:00 2001 From: ihlec Date: Tue, 21 Jul 2026 19:38:09 +0200 Subject: [PATCH 11/17] fix(ondemandpin): exponential check backoff for failing CIDs --- config/ondemandpin.go | 3 + docs/changelogs/vFUTURE.md | 2 +- docs/config.md | 5 +- docs/experimental-features.md | 3 +- ondemandpin/checker.go | 104 +++++++++++++++++++++++++--------- ondemandpin/checker_test.go | 57 ++++++++++++++++--- ondemandpin/store.go | 2 + 7 files changed, 139 insertions(+), 37 deletions(-) diff --git a/config/ondemandpin.go b/config/ondemandpin.go index 918da01649e..22532e69a01 100644 --- a/config/ondemandpin.go +++ b/config/ondemandpin.go @@ -15,6 +15,9 @@ const ( // Must exceed amino.DefaultProvideValidity so stale DHT records expire // before unpin. The extra day covers check-interval skew. DefaultOnDemandPinUnpinGracePeriod = amino.DefaultProvideValidity + 24*time.Hour + + // Max NextCheckAt delay after repeated check failures (Cap). + DefaultOnDemandPinCheckBackoffMax = DefaultOnDemandPinUnpinGracePeriod ) type OnDemandPinning struct { diff --git a/docs/changelogs/vFUTURE.md b/docs/changelogs/vFUTURE.md index bafbd202f5b..52810a22dca 100644 --- a/docs/changelogs/vFUTURE.md +++ b/docs/changelogs/vFUTURE.md @@ -58,7 +58,7 @@ Configuration at [`OnDemandPinning`](https://github.com/ipfs/kubo/blob/master/do |---|---|---| | `OnDemandPinning.ReplicationTargetMin` | `5` | Pin when fewer than this many providers (excluding self) | | `OnDemandPinning.ReplicationTargetMax` | `7` | Start unpin grace only above this many providers | -| `OnDemandPinning.CheckInterval` | `"10m"` | How often the checker runs | +| `OnDemandPinning.CheckInterval` | `"10m"` | Sweep period; failed CIDs back off up to 72h | | `OnDemandPinning.UnpinGracePeriod` | `"72h"` | How long above max before unpinning (longer than 48h DHT record validity; plus up to `2 * CheckInterval` jitter) | See [experimental features](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#on-demand-pinning) diff --git a/docs/config.md b/docs/config.md index 73db68d2160..bf7bec5f22b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -2271,7 +2271,10 @@ Type: `optionalInteger` ### `OnDemandPinning.CheckInterval` -How often the background checker evaluates all on-demand pins. +How often the checker starts a sweep of registered CIDs. If a sweep overruns +this duration, the next sweep starts when the previous one finishes. After a +failed check, that CID is skipped until `NextCheckAt` +(`CheckInterval * 2^(failures-1)`, max 72h). Default: `"10m"` diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 61a0f290b33..c3f2ae02355 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -626,7 +626,8 @@ The feature consists of: - A **registry** of CIDs to monitor, managed via `ipfs pin ondemand add|rm|ls`. - A **background checker** that periodically queries the DHT for each - registered CID and pins or unpins accordingly. + registered CID and pins or unpins accordingly. Failed CIDs are retried + with exponential backoff (max 72h). - **Pin partitioning**: the checker marks its pins with the name `"kubo:on-demand"`, so the checker can tell its pins apart from user pins and will never remove a pin it did not create. diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go index f7814b571d9..da56ed169b6 100644 --- a/ondemandpin/checker.go +++ b/ondemandpin/checker.go @@ -2,6 +2,7 @@ package ondemandpin import ( "context" + "fmt" "math/rand/v2" "time" @@ -47,6 +48,7 @@ type Checker struct { replicationMax int checkInterval time.Duration unpinGracePeriod time.Duration + maxBackoff time.Duration now func() time.Time graceJitter func() time.Duration @@ -72,6 +74,7 @@ func NewChecker( replicationMax: int(cfg.ReplicationTargetMax.WithDefault(config.DefaultOnDemandPinReplicationTargetMax)), checkInterval: cfg.CheckInterval.WithDefault(config.DefaultOnDemandPinCheckInterval), unpinGracePeriod: cfg.UnpinGracePeriod.WithDefault(config.DefaultOnDemandPinUnpinGracePeriod), + maxBackoff: config.DefaultOnDemandPinCheckBackoffMax, now: time.Now, priorityCh: make(chan cid.Cid, 64), @@ -140,7 +143,7 @@ func (c *Checker) checkAll(ctx context.Context) { if ctx.Err() != nil { return } - c.checkRecord(ctx, &rec) + c.checkRecord(ctx, &rec, false) } } @@ -150,73 +153,87 @@ func (c *Checker) checkOne(ctx context.Context, ci cid.Cid) { log.Debugw("CID not in store, skipping", "cid", ci, "error", err) return } - c.checkRecord(ctx, rec) + // Ignore NextCheckAt so pin ondemand add is not delayed by a prior failure. + c.checkRecord(ctx, rec, true) } -// checkRecord pins below min, starts a jittered grace period above max, and -// clears grace in the deadband. Skips CIDs with a non-on-demand pin. -func (c *Checker) checkRecord(ctx context.Context, rec *Record) { +// checkRecord pins below min, starts grace above max, clears grace in the deadband. +// immediate=true clears FailureCount/NextCheckAt before running. +func (c *Checker) checkRecord(ctx context.Context, rec *Record, immediate bool) { + if immediate { + rec.FailureCount = 0 + rec.NextCheckAt = time.Time{} + } else if !rec.NextCheckAt.IsZero() && c.now().Before(rec.NextCheckAt) { + return + } + ctx, cancel := context.WithTimeout(ctx, checkTimeout) defer cancel() pinned, err := c.pins.IsPinned(ctx, rec.Cid) if err != nil { - log.Errorw("failed to check pin state, skipping CID", "cid", rec.Cid, "error", err) + c.recordFailure(ctx, rec, fmt.Errorf("check pin state: %w", err)) return } hasOnDemandPin, err := c.pins.HasPinWithName(ctx, rec.Cid, OnDemandPinName) if err != nil { - log.Errorw("failed to check pin name, skipping CID", "cid", rec.Cid, "error", err) + c.recordFailure(ctx, rec, fmt.Errorf("check pin name: %w", err)) return } if pinned && !hasOnDemandPin { log.Debugw("skipping: CID has a user-managed pin", "cid", rec.Cid) + c.clearBackoff(ctx, rec) return } count, ok := CountProviders(ctx, c.routing, c.selfID, rec.Cid, c.replicationMin, c.replicationMax) if !ok { - log.Warnw("provider count unknown (lookup cancelled or timed out), skipping CID", "cid", rec.Cid) + c.recordFailure(ctx, rec, fmt.Errorf("provider count unknown")) return } log.Debugw("provider count", "cid", rec.Cid, "count", count, "min", c.replicationMin, "max", c.replicationMax, "hasOnDemandPin", hasOnDemandPin) switch { case count < c.replicationMin: - c.handleUnderReplicated(ctx, rec, count, hasOnDemandPin) + if err := c.handleUnderReplicated(ctx, rec, count, hasOnDemandPin); err != nil { + c.recordFailure(ctx, rec, err) + return + } case count > c.replicationMax: - c.handleWellReplicated(ctx, rec, count, hasOnDemandPin) + if err := c.handleWellReplicated(ctx, rec, count, hasOnDemandPin); err != nil { + c.recordFailure(ctx, rec, err) + return + } default: c.clearGrace(ctx, rec) } + c.clearBackoff(ctx, rec) } // handleUnderReplicated pins the CID if it does not already have OnDemandPinName. -func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count int, hasOnDemandPin bool) { +func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count int, hasOnDemandPin bool) error { if hasOnDemandPin { c.clearGrace(ctx, rec) - return + return nil } if !c.hasStorageBudget(ctx) { log.Warnw("skipping pin: repo near storage limit", "cid", rec.Cid) - return + return nil } // Re-check: a user pin may have appeared during the provider lookup. pinnedNow, err := c.pins.IsPinned(ctx, rec.Cid) if err != nil { - log.Errorw("failed to re-check pin state before pinning, skipping CID", "cid", rec.Cid, "error", err) - return + return fmt.Errorf("re-check pin state: %w", err) } if pinnedNow { log.Debugw("skipping pin: CID gained a pin during provider lookup", "cid", rec.Cid) - return + return nil } if err := c.pins.Pin(ctx, rec.Cid, OnDemandPinName); err != nil { - log.Errorw("failed to pin", "cid", rec.Cid, "error", err) - return + return fmt.Errorf("pin: %w", err) } rec.LastAboveTarget = time.Time{} rec.UnpinAt = time.Time{} @@ -226,12 +243,12 @@ func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count log.Warnw("failed to provide after pin", "cid", rec.Cid, "error", err) } c.saveRecord(ctx, rec) + return nil } -// handleWellReplicated manages grace-period-then-unpin for pins with OnDemandPinName. -func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count int, hasOnDemandPin bool) { +func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count int, hasOnDemandPin bool) error { if !hasOnDemandPin { - return + return nil } if rec.LastAboveTarget.IsZero() { @@ -241,23 +258,21 @@ func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count i rec.UnpinAt = now.Add(c.unpinGracePeriod + jitter) c.saveRecord(ctx, rec) log.Debugw("grace period started", "cid", rec.Cid, "providers", count, "max", c.replicationMax, "unpinAt", rec.UnpinAt, "jitter", jitter) - return + return nil } if c.now().Before(rec.UnpinAt) { - return + return nil } stillOnDemand, err := c.pins.HasPinWithName(ctx, rec.Cid, OnDemandPinName) if err != nil { - log.Errorw("failed to check pin name, skipping unpin", "cid", rec.Cid, "error", err) - return + return fmt.Errorf("check pin name before unpin: %w", err) } if stillOnDemand { if err := c.pins.Unpin(ctx, rec.Cid); err != nil { - log.Errorw("failed to unpin", "cid", rec.Cid, "error", err) - return + return fmt.Errorf("unpin: %w", err) } log.Infow("unpinned", "cid", rec.Cid, "providers", count, "max", c.replicationMax) } else { @@ -267,6 +282,7 @@ func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count i rec.LastAboveTarget = time.Time{} rec.UnpinAt = time.Time{} c.saveRecord(ctx, rec) + return nil } func (c *Checker) clearGrace(ctx context.Context, rec *Record) { @@ -278,6 +294,40 @@ func (c *Checker) clearGrace(ctx context.Context, rec *Record) { c.saveRecord(ctx, rec) } +func (c *Checker) clearBackoff(ctx context.Context, rec *Record) { + if rec.FailureCount == 0 && rec.NextCheckAt.IsZero() { + return + } + rec.FailureCount = 0 + rec.NextCheckAt = time.Time{} + c.saveRecord(ctx, rec) +} + +func (c *Checker) recordFailure(ctx context.Context, rec *Record, cause error) { + rec.FailureCount++ + delay := c.backoffDelay(rec.FailureCount) + rec.NextCheckAt = c.now().Add(delay) + log.Warnw("check failed", "cid", rec.Cid, "error", cause, "failures", rec.FailureCount, "nextCheckAt", rec.NextCheckAt) + c.saveRecord(ctx, rec) +} + +func (c *Checker) backoffDelay(failures int) time.Duration { + if failures < 1 { + return c.checkInterval + } + d := c.checkInterval + for i := 1; i < failures; i++ { + if d >= c.maxBackoff/2 { + return c.maxBackoff + } + d *= 2 + } + if d > c.maxBackoff { + return c.maxBackoff + } + return d +} + func (c *Checker) saveRecord(ctx context.Context, rec *Record) { if err := c.store.Update(ctx, rec); err != nil { log.Errorw("failed to update record", "cid", rec.Cid, "error", err) diff --git a/ondemandpin/checker_test.go b/ondemandpin/checker_test.go index 597d0bad941..f5de6dd231d 100644 --- a/ondemandpin/checker_test.go +++ b/ondemandpin/checker_test.go @@ -2,6 +2,7 @@ package ondemandpin import ( "context" + "errors" "sync" "sync/atomic" "testing" @@ -138,7 +139,6 @@ func TestCheckerPinsBelowTarget(t *testing.T) { assert.True(t, p.isPinned(c)) } -// Content at or above min is left alone (including the deadband up to max). func TestCheckerDoesNotPinInDeadband(t *testing.T) { ctx := context.Background() checker, store, r, p, _ := newTestChecker(t) @@ -152,7 +152,6 @@ func TestCheckerDoesNotPinInDeadband(t *testing.T) { assert.False(t, p.isPinned(c)) } -// Pinned content is unpinned only after the grace period expires above max. func TestCheckerUnpinsAfterGracePeriod(t *testing.T) { ctx := context.Background() checker, store, r, p, clock := newTestChecker(t) @@ -225,7 +224,6 @@ func TestCheckerSkipsPinCreatedDuringLookup(t *testing.T) { assert.Equal(t, "user-pin", p.pinned[c]) } -// Ownership follows the reserved pin name, not a store flag. func TestCheckerOwnsPinByNameNotStoreField(t *testing.T) { ctx := context.Background() checker, store, r, p, clock := newTestChecker(t) @@ -243,7 +241,6 @@ func TestCheckerOwnsPinByNameNotStoreField(t *testing.T) { assert.False(t, p.isPinned(c), "name-owned pin must unpin after grace") } -// blockingRouting closes only when ctx is cancelled, mimicking a hung lookup. type blockingRouting struct{} func (blockingRouting) FindProvidersAsync(ctx context.Context, _ cid.Cid, _ int) <-chan peer.AddrInfo { @@ -266,7 +263,6 @@ func TestCountProvidersUnknownOnCancel(t *testing.T) { assert.Equal(t, 0, count) } -// emitThenBlockRouting emits providers, then blocks until ctx cancel. type emitThenBlockRouting struct { providers []peer.ID } @@ -312,7 +308,6 @@ func TestCountProvidersOkWhenEnoughFoundDespiteCancel(t *testing.T) { assert.Equal(t, 5, count) } -// Cancelled provider lookup does not count as zero providers. func TestCheckerSkipsWhenProviderCountUnknown(t *testing.T) { ctx := context.Background() store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) @@ -325,9 +320,57 @@ func TestCheckerSkipsWhenProviderCountUnknown(t *testing.T) { checkCtx, cancel := context.WithTimeout(ctx, 20*time.Millisecond) defer cancel() - checker.checkRecord(checkCtx, mustGet(t, store, c)) + checker.checkRecord(checkCtx, mustGet(t, store, c), false) assert.False(t, p.isPinned(c)) + rec := mustGet(t, store, c) + assert.Equal(t, 1, rec.FailureCount) + assert.False(t, rec.NextCheckAt.IsZero()) +} + +type failPinOnce struct { + *mockPins + fail bool +} + +func (p *failPinOnce) Pin(ctx context.Context, c cid.Cid, name string) error { + if p.fail { + return errors.New("pin failed") + } + return p.mockPins.Pin(ctx, c, name) +} + +func TestCheckerBackoffSkipsUntilDue(t *testing.T) { + ctx := context.Background() + store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) + r := newMockRouting() + pins := &failPinOnce{mockPins: newMockPins(), fail: true} + clock := newFakeClock() + checker := NewChecker(store, pins, nil, r, peer.ID("self"), config.OnDemandPinning{}) + checker.checkInterval = time.Minute + checker.now = clock.Now + checker.graceJitter = func() time.Duration { return 0 } + + c := testCID(t, "backoff") + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, peer.ID("p1")) + + checker.checkAll(ctx) + assert.False(t, pins.isPinned(c)) + rec := mustGet(t, store, c) + require.Equal(t, 1, rec.FailureCount) + require.Equal(t, clock.Now().Add(time.Minute), rec.NextCheckAt) + + pins.fail = false + checker.checkAll(ctx) + assert.False(t, pins.isPinned(c)) + + clock.Advance(time.Minute) + checker.checkAll(ctx) + assert.True(t, pins.isPinned(c)) + rec = mustGet(t, store, c) + assert.Equal(t, 0, rec.FailureCount) + assert.True(t, rec.NextCheckAt.IsZero()) } func mustGet(t *testing.T, store *Store, c cid.Cid) *Record { diff --git a/ondemandpin/store.go b/ondemandpin/store.go index df8979dd91b..3afd0095d48 100644 --- a/ondemandpin/store.go +++ b/ondemandpin/store.go @@ -24,6 +24,8 @@ type Record struct { Cid cid.Cid `json:"Cid"` LastAboveTarget time.Time `json:"LastAboveTarget,omitempty"` UnpinAt time.Time `json:"UnpinAt,omitempty"` // grace deadline (includes jitter) + FailureCount int `json:"FailureCount,omitempty"` + NextCheckAt time.Time `json:"NextCheckAt,omitempty"` CreatedAt time.Time `json:"CreatedAt"` } From c3b68912c29cb6fa57f53352f5c31fd4d5f21fd6 Mon Sep 17 00:00:00 2001 From: ihlec Date: Tue, 21 Jul 2026 22:56:12 +0200 Subject: [PATCH 12/17] fix(ondemandpin): do not apply checkTimeout to Pin fetches --- ondemandpin/checker.go | 38 +++++++++++++++++++------------------ ondemandpin/checker_test.go | 29 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go index da56ed169b6..2acf4e87e30 100644 --- a/ondemandpin/checker.go +++ b/ondemandpin/checker.go @@ -159,6 +159,7 @@ func (c *Checker) checkOne(ctx context.Context, ci cid.Cid) { // checkRecord pins below min, starts grace above max, clears grace in the deadband. // immediate=true clears FailureCount/NextCheckAt before running. +// checkTimeout covers DHT/pin-state lookup only; Pin uses ctx (daemon lifecycle). func (c *Checker) checkRecord(ctx context.Context, rec *Record, immediate bool) { if immediate { rec.FailureCount = 0 @@ -167,15 +168,15 @@ func (c *Checker) checkRecord(ctx context.Context, rec *Record, immediate bool) return } - ctx, cancel := context.WithTimeout(ctx, checkTimeout) + lookupCtx, cancel := context.WithTimeout(ctx, checkTimeout) defer cancel() - pinned, err := c.pins.IsPinned(ctx, rec.Cid) + pinned, err := c.pins.IsPinned(lookupCtx, rec.Cid) if err != nil { c.recordFailure(ctx, rec, fmt.Errorf("check pin state: %w", err)) return } - hasOnDemandPin, err := c.pins.HasPinWithName(ctx, rec.Cid, OnDemandPinName) + hasOnDemandPin, err := c.pins.HasPinWithName(lookupCtx, rec.Cid, OnDemandPinName) if err != nil { c.recordFailure(ctx, rec, fmt.Errorf("check pin name: %w", err)) return @@ -186,7 +187,7 @@ func (c *Checker) checkRecord(ctx context.Context, rec *Record, immediate bool) return } - count, ok := CountProviders(ctx, c.routing, c.selfID, rec.Cid, c.replicationMin, c.replicationMax) + count, ok := CountProviders(lookupCtx, c.routing, c.selfID, rec.Cid, c.replicationMin, c.replicationMax) if !ok { c.recordFailure(ctx, rec, fmt.Errorf("provider count unknown")) return @@ -195,12 +196,12 @@ func (c *Checker) checkRecord(ctx context.Context, rec *Record, immediate bool) switch { case count < c.replicationMin: - if err := c.handleUnderReplicated(ctx, rec, count, hasOnDemandPin); err != nil { + if err := c.handleUnderReplicated(ctx, lookupCtx, rec, count, hasOnDemandPin); err != nil { c.recordFailure(ctx, rec, err) return } case count > c.replicationMax: - if err := c.handleWellReplicated(ctx, rec, count, hasOnDemandPin); err != nil { + if err := c.handleWellReplicated(ctx, lookupCtx, rec, count, hasOnDemandPin); err != nil { c.recordFailure(ctx, rec, err) return } @@ -211,19 +212,20 @@ func (c *Checker) checkRecord(ctx context.Context, rec *Record, immediate bool) } // handleUnderReplicated pins the CID if it does not already have OnDemandPinName. -func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count int, hasOnDemandPin bool) error { +// lookupCtx is for quick pin-state checks; runCtx is for Pin/Provide/store (may outlast checkTimeout). +func (c *Checker) handleUnderReplicated(runCtx, lookupCtx context.Context, rec *Record, count int, hasOnDemandPin bool) error { if hasOnDemandPin { - c.clearGrace(ctx, rec) + c.clearGrace(runCtx, rec) return nil } - if !c.hasStorageBudget(ctx) { + if !c.hasStorageBudget(runCtx) { log.Warnw("skipping pin: repo near storage limit", "cid", rec.Cid) return nil } // Re-check: a user pin may have appeared during the provider lookup. - pinnedNow, err := c.pins.IsPinned(ctx, rec.Cid) + pinnedNow, err := c.pins.IsPinned(lookupCtx, rec.Cid) if err != nil { return fmt.Errorf("re-check pin state: %w", err) } @@ -232,21 +234,21 @@ func (c *Checker) handleUnderReplicated(ctx context.Context, rec *Record, count return nil } - if err := c.pins.Pin(ctx, rec.Cid, OnDemandPinName); err != nil { + if err := c.pins.Pin(runCtx, rec.Cid, OnDemandPinName); err != nil { return fmt.Errorf("pin: %w", err) } rec.LastAboveTarget = time.Time{} rec.UnpinAt = time.Time{} log.Infow("pinned", "cid", rec.Cid, "providers", count, "min", c.replicationMin) - if err := c.routing.Provide(ctx, rec.Cid, true); err != nil { + if err := c.routing.Provide(runCtx, rec.Cid, true); err != nil { log.Warnw("failed to provide after pin", "cid", rec.Cid, "error", err) } - c.saveRecord(ctx, rec) + c.saveRecord(runCtx, rec) return nil } -func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count int, hasOnDemandPin bool) error { +func (c *Checker) handleWellReplicated(runCtx, lookupCtx context.Context, rec *Record, count int, hasOnDemandPin bool) error { if !hasOnDemandPin { return nil } @@ -256,7 +258,7 @@ func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count i jitter := c.graceJitter() rec.LastAboveTarget = now rec.UnpinAt = now.Add(c.unpinGracePeriod + jitter) - c.saveRecord(ctx, rec) + c.saveRecord(runCtx, rec) log.Debugw("grace period started", "cid", rec.Cid, "providers", count, "max", c.replicationMax, "unpinAt", rec.UnpinAt, "jitter", jitter) return nil } @@ -265,13 +267,13 @@ func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count i return nil } - stillOnDemand, err := c.pins.HasPinWithName(ctx, rec.Cid, OnDemandPinName) + stillOnDemand, err := c.pins.HasPinWithName(lookupCtx, rec.Cid, OnDemandPinName) if err != nil { return fmt.Errorf("check pin name before unpin: %w", err) } if stillOnDemand { - if err := c.pins.Unpin(ctx, rec.Cid); err != nil { + if err := c.pins.Unpin(runCtx, rec.Cid); err != nil { return fmt.Errorf("unpin: %w", err) } log.Infow("unpinned", "cid", rec.Cid, "providers", count, "max", c.replicationMax) @@ -281,7 +283,7 @@ func (c *Checker) handleWellReplicated(ctx context.Context, rec *Record, count i rec.LastAboveTarget = time.Time{} rec.UnpinAt = time.Time{} - c.saveRecord(ctx, rec) + c.saveRecord(runCtx, rec) return nil } diff --git a/ondemandpin/checker_test.go b/ondemandpin/checker_test.go index f5de6dd231d..52e97f8a65b 100644 --- a/ondemandpin/checker_test.go +++ b/ondemandpin/checker_test.go @@ -340,6 +340,35 @@ func (p *failPinOnce) Pin(ctx context.Context, c cid.Cid, name string) error { return p.mockPins.Pin(ctx, c, name) } +type capturePinCtx struct { + *mockPins + pinCtx context.Context +} + +func (p *capturePinCtx) Pin(ctx context.Context, c cid.Cid, name string) error { + p.pinCtx = ctx + return p.mockPins.Pin(ctx, c, name) +} + +func TestPinContextHasNoCheckTimeout(t *testing.T) { + ctx := context.Background() + store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) + r := newMockRouting() + pins := &capturePinCtx{mockPins: newMockPins()} + checker := NewChecker(store, pins, nil, r, peer.ID("self"), config.OnDemandPinning{}) + checker.now = newFakeClock().Now + checker.graceJitter = func() time.Duration { return 0 } + + c := testCID(t, "pin-ctx") + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, peer.ID("p1")) + + checker.checkAll(ctx) + require.NotNil(t, pins.pinCtx) + _, hasDeadline := pins.pinCtx.Deadline() + assert.False(t, hasDeadline) +} + func TestCheckerBackoffSkipsUntilDue(t *testing.T) { ctx := context.Background() store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) From 974103cf35a6d921bbc810081bc6b311776099ad Mon Sep 17 00:00:00 2001 From: ihlec Date: Wed, 22 Jul 2026 01:24:54 +0200 Subject: [PATCH 13/17] fix(ondemandpin): do not resurrect records removed during a check --- ondemandpin/checker.go | 5 +++++ ondemandpin/store.go | 10 +++++++++- ondemandpin/store_test.go | 17 +++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go index 2acf4e87e30..da0600a99fd 100644 --- a/ondemandpin/checker.go +++ b/ondemandpin/checker.go @@ -2,6 +2,7 @@ package ondemandpin import ( "context" + "errors" "fmt" "math/rand/v2" "time" @@ -332,6 +333,10 @@ func (c *Checker) backoffDelay(failures int) time.Duration { func (c *Checker) saveRecord(ctx context.Context, rec *Record) { if err := c.store.Update(ctx, rec); err != nil { + if errors.Is(err, ErrNotRegistered) { + log.Debugw("record gone during check, not recreating", "cid", rec.Cid) + return + } log.Errorw("failed to update record", "cid", rec.Cid, "error", err) } } diff --git a/ondemandpin/store.go b/ondemandpin/store.go index 3afd0095d48..a3f697977e0 100644 --- a/ondemandpin/store.go +++ b/ondemandpin/store.go @@ -121,7 +121,15 @@ func (s *Store) Update(ctx context.Context, rec *Record) error { s.mu.Lock() defer s.mu.Unlock() - return s.put(ctx, dsKey(rec.Cid), rec) + key := dsKey(rec.Cid) + has, err := s.ds.Has(ctx, key) + if err != nil { + return fmt.Errorf("checking record: %w", err) + } + if !has { + return fmt.Errorf("%s: %w", rec.Cid, ErrNotRegistered) + } + return s.put(ctx, key, rec) } func (s *Store) get(ctx context.Context, key datastore.Key) (*Record, error) { diff --git a/ondemandpin/store_test.go b/ondemandpin/store_test.go index d8bea2003c2..11ff719f9b3 100644 --- a/ondemandpin/store_test.go +++ b/ondemandpin/store_test.go @@ -63,3 +63,20 @@ func TestStoreList(t *testing.T) { require.NoError(t, err) assert.Len(t, records, 2) } + +func TestStoreUpdateDoesNotResurrect(t *testing.T) { + ctx := context.Background() + s := newTestStore(t) + c := testCID(t, "gone") + + require.NoError(t, s.Add(ctx, c)) + rec, err := s.Get(ctx, c) + require.NoError(t, err) + + require.NoError(t, s.Remove(ctx, c)) + err = s.Update(ctx, rec) + assert.ErrorIs(t, err, ErrNotRegistered) + + _, err = s.Get(ctx, c) + assert.ErrorIs(t, err, ErrNotRegistered) +} From 34a679a2abccd68c4f3d920c38a074b82c2dc055 Mon Sep 17 00:00:00 2001 From: ihlec Date: Wed, 22 Jul 2026 02:24:34 +0200 Subject: [PATCH 14/17] fix(ondemandpin): more details for ls and DryRun mode --- config/ondemandpin.go | 3 ++ core/commands/pin/ondemandpin.go | 65 ++++++++++++++++++++++++-------- docs/changelogs/vFUTURE.md | 1 + docs/config.md | 13 ++++++- docs/experimental-features.md | 12 +++++- ondemandpin/checker.go | 34 +++++++++++++++-- ondemandpin/checker_test.go | 45 ++++++++++++++++++++++ ondemandpin/store.go | 15 +++++--- 8 files changed, 161 insertions(+), 27 deletions(-) diff --git a/config/ondemandpin.go b/config/ondemandpin.go index 22532e69a01..404bfb3e147 100644 --- a/config/ondemandpin.go +++ b/config/ondemandpin.go @@ -32,6 +32,9 @@ type OnDemandPinning struct { // How long replication must stay above max before unpinning; checker adds up to 2*CheckInterval of jitter. UnpinGracePeriod OptionalDuration + + // When true, the checker logs pin/unpin decisions but does not change the pinset. + DryRun Flag `json:",omitempty"` } // ValidateOnDemandPinningConfig rejects invalid min/max and non-positive durations. diff --git a/core/commands/pin/ondemandpin.go b/core/commands/pin/ondemandpin.go index 2a68c6b5c4e..ec2f932c111 100644 --- a/core/commands/pin/ondemandpin.go +++ b/core/commands/pin/ondemandpin.go @@ -175,19 +175,25 @@ Works when on-demand pinning is disabled, to clear old registrations. } type OnDemandLsOutput struct { - Cid string `json:"Cid"` - PinnedByUs bool `json:"PinnedByUs"` - Providers *int `json:"Providers,omitempty"` - LastAboveTarget string `json:"LastAboveTarget,omitempty"` - CreatedAt string `json:"CreatedAt"` + Cid string `json:"Cid"` + HasOnDemandPin bool `json:"HasOnDemandPin"` + Providers *int `json:"Providers,omitempty"` // live lookup only + LastProviderCount *int `json:"LastProviderCount,omitempty"` + LastCheckedAt string `json:"LastCheckedAt,omitempty"` + LastResult string `json:"LastResult,omitempty"` + LastAboveTarget string `json:"LastAboveTarget,omitempty"` + UnpinAt string `json:"UnpinAt,omitempty"` + FailureCount int `json:"FailureCount,omitempty"` + NextCheckAt string `json:"NextCheckAt,omitempty"` + CreatedAt string `json:"CreatedAt"` } var listOnDemandPinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List on-demand pins.", ShortDescription: ` -Lists CIDs registered for on-demand pinning with their current state. -Use --live to include real-time provider counts from the DHT. +Lists registered CIDs with last check result, provider count, and unpin time. +Use --live for a fresh DHT provider count. `, }, Arguments: []cmds.Argument{ @@ -251,13 +257,28 @@ Use --live to include real-time provider counts from the DHT. return fmt.Errorf("checking pin state for %s: %w", rec.Cid, err) } out := OnDemandLsOutput{ - Cid: rec.Cid.String(), - PinnedByUs: hasOnDemandPin, - CreatedAt: rec.CreatedAt.Format(time.RFC3339), + Cid: rec.Cid.String(), + HasOnDemandPin: hasOnDemandPin, + LastResult: rec.LastResult, + FailureCount: rec.FailureCount, + CreatedAt: rec.CreatedAt.Format(time.RFC3339), + } + if !rec.LastCheckedAt.IsZero() { + out.LastCheckedAt = rec.LastCheckedAt.Format(time.RFC3339) + } + if rec.LastResult != "" || !rec.LastCheckedAt.IsZero() { + count := rec.LastProviderCount + out.LastProviderCount = &count } if !rec.LastAboveTarget.IsZero() { out.LastAboveTarget = rec.LastAboveTarget.Format(time.RFC3339) } + if !rec.UnpinAt.IsZero() { + out.UnpinAt = rec.UnpinAt.Format(time.RFC3339) + } + if !rec.NextCheckAt.IsZero() { + out.NextCheckAt = rec.NextCheckAt.Format(time.RFC3339) + } if live && n.Routing != nil { count, ok := ondemandpin.CountProviders(req.Context, n.Routing, n.Identity, rec.Cid, replicationMin, replicationMax) @@ -275,17 +296,31 @@ Use --live to include real-time provider counts from the DHT. Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OnDemandLsOutput) error { pinState := "not-pinned" - if out.PinnedByUs { + if out.HasOnDemandPin { pinState = "pinned" } - fmt.Fprintf(w, "%s", out.Cid) + fmt.Fprintf(w, "%s %s", out.Cid, pinState) if out.Providers != nil { fmt.Fprintf(w, " providers=%d", *out.Providers) + } else if out.LastProviderCount != nil { + fmt.Fprintf(w, " last-providers=%d", *out.LastProviderCount) + } + if out.LastResult != "" { + fmt.Fprintf(w, " result=%s", out.LastResult) + } + if out.LastCheckedAt != "" { + fmt.Fprintf(w, " checked=%s", out.LastCheckedAt) + } + if out.UnpinAt != "" { + fmt.Fprintf(w, " unpin-at=%s", out.UnpinAt) + } + if out.NextCheckAt != "" { + fmt.Fprintf(w, " next-check=%s", out.NextCheckAt) } - fmt.Fprintf(w, " %s created=%s", pinState, out.CreatedAt) - if out.LastAboveTarget != "" { - fmt.Fprintf(w, " above-target-since=%s", out.LastAboveTarget) + if out.FailureCount > 0 { + fmt.Fprintf(w, " failures=%d", out.FailureCount) } + fmt.Fprintf(w, " created=%s", out.CreatedAt) fmt.Fprintln(w) return nil }), diff --git a/docs/changelogs/vFUTURE.md b/docs/changelogs/vFUTURE.md index 52810a22dca..49fa646638f 100644 --- a/docs/changelogs/vFUTURE.md +++ b/docs/changelogs/vFUTURE.md @@ -60,6 +60,7 @@ Configuration at [`OnDemandPinning`](https://github.com/ipfs/kubo/blob/master/do | `OnDemandPinning.ReplicationTargetMax` | `7` | Start unpin grace only above this many providers | | `OnDemandPinning.CheckInterval` | `"10m"` | Sweep period; failed CIDs back off up to 72h | | `OnDemandPinning.UnpinGracePeriod` | `"72h"` | How long above max before unpinning (longer than 48h DHT record validity; plus up to `2 * CheckInterval` jitter) | +| `OnDemandPinning.DryRun` | `false` | Log/record pin/unpin decisions without changing the pinset | See [experimental features](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#on-demand-pinning) for full documentation. diff --git a/docs/config.md b/docs/config.md index bf7bec5f22b..73bf9ce7941 100644 --- a/docs/config.md +++ b/docs/config.md @@ -131,7 +131,7 @@ config file at runtime. - [`OnDemandPinning.ReplicationTargetMax`](#ondemandpinningreplicationtargetmax) - [`OnDemandPinning.CheckInterval`](#ondemandpinningcheckinterval) - [`OnDemandPinning.UnpinGracePeriod`](#ondemandpinningunpingraceperiod) - + - [`OnDemandPinning.DryRun`](#ondemandpinningdryrun) - [`Pinning`](#pinning) - [`Pinning.RemoteServices`](#pinningremoteservices) - [`Pinning.RemoteServices: API`](#pinningremoteservices-api) @@ -2293,6 +2293,17 @@ Default: `"72h"` Type: `optionalDuration` +### `OnDemandPinning.DryRun` + +When `true`, the checker still evaluates registered CIDs and records +`LastResult` / `LastProviderCount`, but it does not pin or unpin. Useful for +watching decisions with `ipfs pin ondemand ls` before enabling real pinset +changes. + +Default: `false` + +Type: `flag` + ## `Provide` Configures how your node advertises content to make it discoverable by other diff --git a/docs/experimental-features.md b/docs/experimental-features.md index c3f2ae02355..e960382bde5 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -653,15 +653,23 @@ ipfs config --json Experimental.OnDemandPinningEnabled true See [`OnDemandPinning`](https://github.com/ipfs/kubo/blob/master/docs/config.md#ondemandpinning) for tunable parameters: `ReplicationTargetMin`, `ReplicationTargetMax`, -`CheckInterval`, and `UnpinGracePeriod`. +`CheckInterval`, `UnpinGracePeriod`, and `DryRun`. + +`ipfs pin ondemand ls` shows the last check time, provider count, result, and +computed unpin time. Set `OnDemandPinning.DryRun` to observe decisions without +changing the pinset. ### Basic usage ```bash +# Optional: evaluate without pinning/unpinning +ipfs config --json OnDemandPinning.DryRun true + # Register a CID for on-demand monitoring ipfs pin ondemand add QmExample -# List all registered CIDs (with live provider counts from DHT) +# List registered CIDs (last check + unpin time; --live for a fresh DHT count) +ipfs pin ondemand ls ipfs pin ondemand ls --live # Remove a CID from on-demand monitoring (also unpins if checker had pinned it) diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go index da0600a99fd..5316dbfb9b6 100644 --- a/ondemandpin/checker.go +++ b/ondemandpin/checker.go @@ -50,6 +50,7 @@ type Checker struct { checkInterval time.Duration unpinGracePeriod time.Duration maxBackoff time.Duration + dryRun bool now func() time.Time graceJitter func() time.Duration @@ -76,6 +77,7 @@ func NewChecker( checkInterval: cfg.CheckInterval.WithDefault(config.DefaultOnDemandPinCheckInterval), unpinGracePeriod: cfg.UnpinGracePeriod.WithDefault(config.DefaultOnDemandPinUnpinGracePeriod), maxBackoff: config.DefaultOnDemandPinCheckBackoffMax, + dryRun: cfg.DryRun.WithDefault(false), now: time.Now, priorityCh: make(chan cid.Cid, 64), @@ -184,12 +186,14 @@ func (c *Checker) checkRecord(ctx context.Context, rec *Record, immediate bool) } if pinned && !hasOnDemandPin { log.Debugw("skipping: CID has a user-managed pin", "cid", rec.Cid) + rec.LastResult = "user-pin" c.clearBackoff(ctx, rec) return } count, ok := CountProviders(lookupCtx, c.routing, c.selfID, rec.Cid, c.replicationMin, c.replicationMax) if !ok { + rec.LastResult = "lookup-unknown" c.recordFailure(ctx, rec, fmt.Errorf("provider count unknown")) return } @@ -208,7 +212,9 @@ func (c *Checker) checkRecord(ctx context.Context, rec *Record, immediate bool) } default: c.clearGrace(ctx, rec) + rec.LastResult = "deadband" } + rec.LastProviderCount = count c.clearBackoff(ctx, rec) } @@ -217,11 +223,13 @@ func (c *Checker) checkRecord(ctx context.Context, rec *Record, immediate bool) func (c *Checker) handleUnderReplicated(runCtx, lookupCtx context.Context, rec *Record, count int, hasOnDemandPin bool) error { if hasOnDemandPin { c.clearGrace(runCtx, rec) + rec.LastResult = "holding" return nil } if !c.hasStorageBudget(runCtx) { log.Warnw("skipping pin: repo near storage limit", "cid", rec.Cid) + rec.LastResult = "storage-limit" return nil } @@ -232,6 +240,13 @@ func (c *Checker) handleUnderReplicated(runCtx, lookupCtx context.Context, rec * } if pinnedNow { log.Debugw("skipping pin: CID gained a pin during provider lookup", "cid", rec.Cid) + rec.LastResult = "user-pin" + return nil + } + + if c.dryRun { + log.Infow("dry-run: would pin", "cid", rec.Cid, "providers", count, "min", c.replicationMin) + rec.LastResult = "would-pin" return nil } @@ -240,6 +255,7 @@ func (c *Checker) handleUnderReplicated(runCtx, lookupCtx context.Context, rec * } rec.LastAboveTarget = time.Time{} rec.UnpinAt = time.Time{} + rec.LastResult = "pinned" log.Infow("pinned", "cid", rec.Cid, "providers", count, "min", c.replicationMin) if err := c.routing.Provide(runCtx, rec.Cid, true); err != nil { @@ -251,6 +267,7 @@ func (c *Checker) handleUnderReplicated(runCtx, lookupCtx context.Context, rec * func (c *Checker) handleWellReplicated(runCtx, lookupCtx context.Context, rec *Record, count int, hasOnDemandPin bool) error { if !hasOnDemandPin { + rec.LastResult = "above-max" return nil } @@ -259,12 +276,14 @@ func (c *Checker) handleWellReplicated(runCtx, lookupCtx context.Context, rec *R jitter := c.graceJitter() rec.LastAboveTarget = now rec.UnpinAt = now.Add(c.unpinGracePeriod + jitter) + rec.LastResult = "grace" c.saveRecord(runCtx, rec) log.Debugw("grace period started", "cid", rec.Cid, "providers", count, "max", c.replicationMax, "unpinAt", rec.UnpinAt, "jitter", jitter) return nil } if c.now().Before(rec.UnpinAt) { + rec.LastResult = "grace" return nil } @@ -274,12 +293,19 @@ func (c *Checker) handleWellReplicated(runCtx, lookupCtx context.Context, rec *R } if stillOnDemand { + if c.dryRun { + log.Infow("dry-run: would unpin", "cid", rec.Cid, "providers", count, "max", c.replicationMax) + rec.LastResult = "would-unpin" + return nil + } if err := c.pins.Unpin(runCtx, rec.Cid); err != nil { return fmt.Errorf("unpin: %w", err) } log.Infow("unpinned", "cid", rec.Cid, "providers", count, "max", c.replicationMax) + rec.LastResult = "unpinned" } else { log.Infow("relinquishing management: pin name changed externally", "cid", rec.Cid) + rec.LastResult = "released" } rec.LastAboveTarget = time.Time{} @@ -298,15 +324,17 @@ func (c *Checker) clearGrace(ctx context.Context, rec *Record) { } func (c *Checker) clearBackoff(ctx context.Context, rec *Record) { - if rec.FailureCount == 0 && rec.NextCheckAt.IsZero() { - return - } + rec.LastCheckedAt = c.now() rec.FailureCount = 0 rec.NextCheckAt = time.Time{} c.saveRecord(ctx, rec) } func (c *Checker) recordFailure(ctx context.Context, rec *Record, cause error) { + rec.LastCheckedAt = c.now() + if rec.LastResult == "" { + rec.LastResult = "error" + } rec.FailureCount++ delay := c.backoffDelay(rec.FailureCount) rec.NextCheckAt = c.now().Add(delay) diff --git a/ondemandpin/checker_test.go b/ondemandpin/checker_test.go index 52e97f8a65b..638de739efe 100644 --- a/ondemandpin/checker_test.go +++ b/ondemandpin/checker_test.go @@ -137,6 +137,10 @@ func TestCheckerPinsBelowTarget(t *testing.T) { checker.checkAll(ctx) assert.True(t, p.isPinned(c)) + rec := mustGet(t, store, c) + assert.Equal(t, "pinned", rec.LastResult) + assert.Equal(t, 2, rec.LastProviderCount) + assert.False(t, rec.LastCheckedAt.IsZero()) } func TestCheckerDoesNotPinInDeadband(t *testing.T) { @@ -326,6 +330,7 @@ func TestCheckerSkipsWhenProviderCountUnknown(t *testing.T) { rec := mustGet(t, store, c) assert.Equal(t, 1, rec.FailureCount) assert.False(t, rec.NextCheckAt.IsZero()) + assert.Equal(t, "lookup-unknown", rec.LastResult) } type failPinOnce struct { @@ -400,6 +405,46 @@ func TestCheckerBackoffSkipsUntilDue(t *testing.T) { rec = mustGet(t, store, c) assert.Equal(t, 0, rec.FailureCount) assert.True(t, rec.NextCheckAt.IsZero()) + assert.Equal(t, "pinned", rec.LastResult) +} + +func TestCheckerDryRunDoesNotPinOrUnpin(t *testing.T) { + ctx := context.Background() + store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) + r := newMockRouting() + p := newMockPins() + clock := newFakeClock() + checker := NewChecker(store, p, nil, r, peer.ID("self"), config.OnDemandPinning{ + DryRun: config.True, + }) + checker.checkInterval = time.Minute + checker.unpinGracePeriod = 200 * time.Millisecond + checker.now = clock.Now + checker.graceJitter = func() time.Duration { return 0 } + + c := testCID(t, "dry-run") + require.NoError(t, store.Add(ctx, c)) + r.setProviders(c, peer.ID("p1"), peer.ID("p2")) + + checker.checkAll(ctx) + assert.False(t, p.isPinned(c)) + rec := mustGet(t, store, c) + assert.Equal(t, "would-pin", rec.LastResult) + assert.Equal(t, 2, rec.LastProviderCount) + + require.NoError(t, p.Pin(ctx, c, OnDemandPinName)) + r.setProviders(c, providers(8)...) + checker.checkAll(ctx) + assert.True(t, p.isPinned(c)) + rec = mustGet(t, store, c) + assert.Equal(t, "grace", rec.LastResult) + require.False(t, rec.UnpinAt.IsZero()) + + clock.Advance(250 * time.Millisecond) + checker.checkAll(ctx) + assert.True(t, p.isPinned(c), "dry-run must not unpin") + rec = mustGet(t, store, c) + assert.Equal(t, "would-unpin", rec.LastResult) } func mustGet(t *testing.T, store *Store, c cid.Cid) *Record { diff --git a/ondemandpin/store.go b/ondemandpin/store.go index a3f697977e0..7e015097dbc 100644 --- a/ondemandpin/store.go +++ b/ondemandpin/store.go @@ -21,12 +21,15 @@ var ( const dsPrefix = "/ondemand-pins/" type Record struct { - Cid cid.Cid `json:"Cid"` - LastAboveTarget time.Time `json:"LastAboveTarget,omitempty"` - UnpinAt time.Time `json:"UnpinAt,omitempty"` // grace deadline (includes jitter) - FailureCount int `json:"FailureCount,omitempty"` - NextCheckAt time.Time `json:"NextCheckAt,omitempty"` - CreatedAt time.Time `json:"CreatedAt"` + Cid cid.Cid `json:"Cid"` + LastAboveTarget time.Time `json:"LastAboveTarget,omitempty"` + UnpinAt time.Time `json:"UnpinAt,omitempty"` // grace deadline (includes jitter) + FailureCount int `json:"FailureCount,omitempty"` + NextCheckAt time.Time `json:"NextCheckAt,omitempty"` + LastCheckedAt time.Time `json:"LastCheckedAt,omitempty"` + LastProviderCount int `json:"LastProviderCount,omitempty"` + LastResult string `json:"LastResult,omitempty"` + CreatedAt time.Time `json:"CreatedAt"` } type Store struct { From 3066da3a05fb7ef6158f6d68cbfbc6fe0c588c6a Mon Sep 17 00:00:00 2001 From: ihlec Date: Wed, 22 Jul 2026 03:08:01 +0200 Subject: [PATCH 15/17] fix(ondemandpin): make ls --live explicit about routing and lookup failures (also parallel now) --- core/commands/pin/ondemandpin.go | 50 +++++++++++++++++++++++++++----- docs/changelogs/vFUTURE.md | 2 +- docs/experimental-features.md | 2 +- ondemandpin/checker.go | 10 +++---- 4 files changed, 50 insertions(+), 14 deletions(-) diff --git a/core/commands/pin/ondemandpin.go b/core/commands/pin/ondemandpin.go index ec2f932c111..a4c601161e5 100644 --- a/core/commands/pin/ondemandpin.go +++ b/core/commands/pin/ondemandpin.go @@ -1,6 +1,7 @@ package pin import ( + "context" "fmt" "io" "time" @@ -10,9 +11,13 @@ import ( cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" "github.com/ipfs/kubo/core/commands/cmdutils" "github.com/ipfs/kubo/ondemandpin" + "golang.org/x/sync/errgroup" ) -const onDemandLiveOptionName = "live" +const ( + onDemandLiveOptionName = "live" + liveLookupParallelism = 8 +) var onDemandPinCmd = &cmds.Command{ Helptext: cmds.HelpText{ @@ -178,6 +183,7 @@ type OnDemandLsOutput struct { Cid string `json:"Cid"` HasOnDemandPin bool `json:"HasOnDemandPin"` Providers *int `json:"Providers,omitempty"` // live lookup only + ProvidersUnknown bool `json:"ProvidersUnknown,omitempty"` LastProviderCount *int `json:"LastProviderCount,omitempty"` LastCheckedAt string `json:"LastCheckedAt,omitempty"` LastResult string `json:"LastResult,omitempty"` @@ -193,7 +199,7 @@ var listOnDemandPinCmd = &cmds.Command{ Tagline: "List on-demand pins.", ShortDescription: ` Lists registered CIDs with last check result, provider count, and unpin time. -Use --live for a fresh DHT provider count. +Use --live for a fresh DHT provider count (requires content routing). `, }, Arguments: []cmds.Argument{ @@ -215,6 +221,9 @@ Use --live for a fresh DHT provider count. var replicationMin, replicationMax int if live { + if n.Routing == nil { + return fmt.Errorf("--live requires content routing; none is available") + } cfg, err := n.Repo.Config() if err != nil { return err @@ -251,7 +260,31 @@ Use --live for a fresh DHT provider count. } } - for _, rec := range records { + type liveResult struct { + count int + ok bool + } + liveResults := make([]liveResult, len(records)) + if live && len(records) > 0 { + g, gctx := errgroup.WithContext(req.Context) + g.SetLimit(liveLookupParallelism) + for i := range records { + i := i + c := records[i].Cid + g.Go(func() error { + lookupCtx, cancel := context.WithTimeout(gctx, ondemandpin.CheckTimeout) + defer cancel() + count, ok := ondemandpin.CountProviders(lookupCtx, n.Routing, n.Identity, c, replicationMin, replicationMax) + liveResults[i] = liveResult{count: count, ok: ok} + return nil + }) + } + if err := g.Wait(); err != nil { + return err + } + } + + for i, rec := range records { hasOnDemandPin, err := ondemandpin.PinHasName(req.Context, n.Pinning, rec.Cid, ondemandpin.OnDemandPinName) if err != nil { return fmt.Errorf("checking pin state for %s: %w", rec.Cid, err) @@ -279,11 +312,12 @@ Use --live for a fresh DHT provider count. if !rec.NextCheckAt.IsZero() { out.NextCheckAt = rec.NextCheckAt.Format(time.RFC3339) } - - if live && n.Routing != nil { - count, ok := ondemandpin.CountProviders(req.Context, n.Routing, n.Identity, rec.Cid, replicationMin, replicationMax) - if ok { + if live { + if liveResults[i].ok { + count := liveResults[i].count out.Providers = &count + } else { + out.ProvidersUnknown = true } } @@ -302,6 +336,8 @@ Use --live for a fresh DHT provider count. fmt.Fprintf(w, "%s %s", out.Cid, pinState) if out.Providers != nil { fmt.Fprintf(w, " providers=%d", *out.Providers) + } else if out.ProvidersUnknown { + fmt.Fprintf(w, " providers=unknown") } else if out.LastProviderCount != nil { fmt.Fprintf(w, " last-providers=%d", *out.LastProviderCount) } diff --git a/docs/changelogs/vFUTURE.md b/docs/changelogs/vFUTURE.md index 49fa646638f..d47dd913167 100644 --- a/docs/changelogs/vFUTURE.md +++ b/docs/changelogs/vFUTURE.md @@ -36,7 +36,7 @@ New CLI commands under `ipfs pin ondemand`: - `add` register CIDs for on-demand pinning - `rm` deregister and unpin -- `ls` list registered CIDs (use `--live` for real-time DHT provider counts) +- `ls` list registered CIDs (use `--live` for DHT provider counts; errors if routing is unavailable; timed-out lookups show as unknown) Design highlights: diff --git a/docs/experimental-features.md b/docs/experimental-features.md index e960382bde5..03c9671861f 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -668,7 +668,7 @@ ipfs config --json OnDemandPinning.DryRun true # Register a CID for on-demand monitoring ipfs pin ondemand add QmExample -# List registered CIDs (last check + unpin time; --live for a fresh DHT count) +# List registered CIDs (last check + unpin time; --live needs content routing) ipfs pin ondemand ls ipfs pin ondemand ls --live diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go index 5316dbfb9b6..7bd90049249 100644 --- a/ondemandpin/checker.go +++ b/ondemandpin/checker.go @@ -24,8 +24,8 @@ var log = logging.Logger("ondemandpin") // names, so only Kubo-internal code can create pins with this name. const OnDemandPinName = "kubo:on-demand" -// checkTimeout prevents hung DHT query or pin/unpin operation from blocking the checker indefinitely. -const checkTimeout = 5 * time.Minute +// CheckTimeout bounds a single provider/pin-state lookup (checker and ls --live). +const CheckTimeout = 5 * time.Minute type PinService interface { Pin(ctx context.Context, c cid.Cid, name string) error @@ -162,7 +162,7 @@ func (c *Checker) checkOne(ctx context.Context, ci cid.Cid) { // checkRecord pins below min, starts grace above max, clears grace in the deadband. // immediate=true clears FailureCount/NextCheckAt before running. -// checkTimeout covers DHT/pin-state lookup only; Pin uses ctx (daemon lifecycle). +// CheckTimeout covers DHT/pin-state lookup only; Pin uses ctx (daemon lifecycle). func (c *Checker) checkRecord(ctx context.Context, rec *Record, immediate bool) { if immediate { rec.FailureCount = 0 @@ -171,7 +171,7 @@ func (c *Checker) checkRecord(ctx context.Context, rec *Record, immediate bool) return } - lookupCtx, cancel := context.WithTimeout(ctx, checkTimeout) + lookupCtx, cancel := context.WithTimeout(ctx, CheckTimeout) defer cancel() pinned, err := c.pins.IsPinned(lookupCtx, rec.Cid) @@ -219,7 +219,7 @@ func (c *Checker) checkRecord(ctx context.Context, rec *Record, immediate bool) } // handleUnderReplicated pins the CID if it does not already have OnDemandPinName. -// lookupCtx is for quick pin-state checks; runCtx is for Pin/Provide/store (may outlast checkTimeout). +// lookupCtx is for quick pin-state checks; runCtx is for Pin/Provide/store (may outlast CheckTimeout). func (c *Checker) handleUnderReplicated(runCtx, lookupCtx context.Context, rec *Record, count int, hasOnDemandPin bool) error { if hasOnDemandPin { c.clearGrace(runCtx, rec) From 6d89f2427e0769ea64e62dcc850a0b4db9f402a0 Mon Sep 17 00:00:00 2001 From: ihlec Date: Wed, 22 Jul 2026 08:29:32 +0200 Subject: [PATCH 16/17] docs(ondemandpin): clarify naming and ipfs-cluster differentiation --- core/commands/pin/ondemandpin.go | 7 +++++-- docs/changelogs/vFUTURE.md | 11 +++++------ docs/config.md | 2 ++ docs/experimental-features.md | 7 +++++-- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/core/commands/pin/ondemandpin.go b/core/commands/pin/ondemandpin.go index a4c601161e5..6b4f590f168 100644 --- a/core/commands/pin/ondemandpin.go +++ b/core/commands/pin/ondemandpin.go @@ -23,8 +23,11 @@ var onDemandPinCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Manage on-demand pins.", ShortDescription: ` -On-demand pins when few DHT providers exist in the routing table; unpins after -replication stays above target for a grace period. Requires config +Pins registered CIDs when FindProviders finds few DHT providers; unpins after +the count stays above max for a grace period. Roughly like ipfs-cluster +replication factors +(https://ipfscluster.io/documentation/guides/pinning/#replication-factors), +except each node decides on its own from the DHT. Requires Experimental.OnDemandPinningEnabled. `, }, diff --git a/docs/changelogs/vFUTURE.md b/docs/changelogs/vFUTURE.md index d47dd913167..a3a72807a3f 100644 --- a/docs/changelogs/vFUTURE.md +++ b/docs/changelogs/vFUTURE.md @@ -21,9 +21,9 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. #### Experimental on-demand pinning Automatically pins content when DHT provider counts fall below a configurable -replication target, and unpins once replication has been above target for a -grace period. Helps keeping critical data around, without wasting storage on -overly replicated CIDs. +min, and unpins once they stay above a max for a grace period. Helps keep +critical data around without storing overly replicated CIDs. Same rough idea as +ipfs-cluster replication factors, except nobody coordinates the pinset; each node reacts to DHT counts locally. The feature is gated behind `Experimental.OnDemandPinningEnabled` and described in [ipfs/specs#532](https://github.com/ipfs/specs/pull/532). @@ -40,9 +40,8 @@ New CLI commands under `ipfs pin ondemand`: Design highlights: -- **Pin partitioning**: the checker needs to distinguish its pins from user - pins to avoid accidental deletion. This implementation uses boxo's pin name - field ("on-demand"). +- **Pin partitioning**: the checker marks its pins with the reserved name + `kubo:on-demand` so it never removes a user pin. - **Storage budget**: skips pinning when repo usage exceeds `StorageMax * StorageGCWatermark`. - **Idle timeout**: DAG fetches timeout after 2 minutes without receiving new diff --git a/docs/config.md b/docs/config.md index 73bf9ce7941..5631178f065 100644 --- a/docs/config.md +++ b/docs/config.md @@ -2249,6 +2249,8 @@ Type: `duration` Configures the on-demand pinning system. Requires [`Experimental.OnDemandPinningEnabled`](#experimentalondemandpinningenabled). +See [experimental features](./experimental-features.md#on-demand-pinning) for how +this differs from ipfs-cluster replication. ### `OnDemandPinning.ReplicationTargetMin` diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 03c9671861f..2d540bdfe60 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -619,8 +619,11 @@ Experimental, disabled by default. On-demand pinning lets a node pin content when DHT provider counts fall below a minimum, and unpin after they stay above a maximum for a grace period (plus a short random delay). Values between min and max are left alone. -ipfs-cluster replication factors assign pins among known peers; this feature -only watches the DHT and decides locally. +Provider counts come from `FindProviders`, not from the peer routing table. + +[ipfs-cluster replication factors](https://ipfscluster.io/documentation/guides/pinning/#replication-factors) +assign pins among known peers; this feature only watches the DHT and decides +locally. The feature consists of: From 34f707beff0a02bad53195a323446c4a47b24ace Mon Sep 17 00:00:00 2001 From: ihlec Date: Wed, 22 Jul 2026 08:50:19 +0200 Subject: [PATCH 17/17] fix(ondemandpin): announce pins via DHTProvider.StartProviding --- config/ondemandpin.go | 8 ++++++ config/ondemandpin_test.go | 5 ++++ core/commands/cmdutils/utils_test.go | 1 + core/node/groups.go | 4 +++ core/node/ondemandpin.go | 4 ++- docs/changelogs/vFUTURE.md | 6 +++-- docs/config.md | 3 ++- docs/experimental-features.md | 5 ++-- ondemandpin/checker.go | 31 +++++++++++++++-------- ondemandpin/checker_test.go | 38 ++++++++++++++++++---------- 10 files changed, 75 insertions(+), 30 deletions(-) diff --git a/config/ondemandpin.go b/config/ondemandpin.go index 404bfb3e147..2bfa56cce93 100644 --- a/config/ondemandpin.go +++ b/config/ondemandpin.go @@ -64,3 +64,11 @@ func ValidateOnDemandPinningRouting(routingType string) error { } return nil } + +// ValidateOnDemandPinningProvide rejects Provide.Enabled=false (pins would be invisible). +func ValidateOnDemandPinningProvide(provideEnabled bool) error { + if !provideEnabled { + return fmt.Errorf("on-demand pinning needs Provide.Enabled; other nodes cannot discover local pins when providing is off") + } + return nil +} diff --git a/config/ondemandpin_test.go b/config/ondemandpin_test.go index 8220febf61e..7bcf159c6b0 100644 --- a/config/ondemandpin_test.go +++ b/config/ondemandpin_test.go @@ -47,3 +47,8 @@ func TestValidateOnDemandPinningRouting(t *testing.T) { assert.NoError(t, ValidateOnDemandPinningRouting("delegated")) assert.ErrorContains(t, ValidateOnDemandPinningRouting("none"), "none") } + +func TestValidateOnDemandPinningProvide(t *testing.T) { + assert.NoError(t, ValidateOnDemandPinningProvide(true)) + assert.ErrorContains(t, ValidateOnDemandPinningProvide(false), "Provide.Enabled") +} diff --git a/core/commands/cmdutils/utils_test.go b/core/commands/cmdutils/utils_test.go index 4be2ab12cee..8ef109fd561 100644 --- a/core/commands/cmdutils/utils_test.go +++ b/core/commands/cmdutils/utils_test.go @@ -184,3 +184,4 @@ func TestValidatePinName(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "reserved") }) +} diff --git a/core/node/groups.go b/core/node/groups.go index 974b4229b64..28db78e9a8d 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -459,6 +459,10 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option { if err := config.ValidateOnDemandPinningRouting(routingType); err != nil { return fx.Error(err) } + provideEnabled := cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled) + if err := config.ValidateOnDemandPinningProvide(provideEnabled); err != nil { + return fx.Error(err) + } } // Directory sharding settings from Import config. diff --git a/core/node/ondemandpin.go b/core/node/ondemandpin.go index f542e086a56..f02e42a12cb 100644 --- a/core/node/ondemandpin.go +++ b/core/node/ondemandpin.go @@ -137,6 +137,7 @@ func OnDemandPinChecker(cfg config.OnDemandPinning) func( store *ondemandpin.Store, pinner pin.Pinner, cr routing.ContentRouting, + prov DHTProvider, dag format.DAGService, bs blockstore.GCBlockstore, id peer.ID, @@ -148,13 +149,14 @@ func OnDemandPinChecker(cfg config.OnDemandPinning) func( store *ondemandpin.Store, pinner pin.Pinner, cr routing.ContentRouting, + prov DHTProvider, dag format.DAGService, bs blockstore.GCBlockstore, id peer.ID, ) *ondemandpin.Checker { pins := &kuboPinService{pinner: pinner, dag: dag, bs: bs} storage := &kuboStorageChecker{repo: r} - checker := ondemandpin.NewChecker(store, pins, storage, cr, id, cfg) + checker := ondemandpin.NewChecker(store, pins, storage, cr, prov, id, cfg) ctx := helpers.LifecycleCtx(mctx, lc) lc.Append(fx.Hook{ diff --git a/docs/changelogs/vFUTURE.md b/docs/changelogs/vFUTURE.md index a3a72807a3f..b385ede4392 100644 --- a/docs/changelogs/vFUTURE.md +++ b/docs/changelogs/vFUTURE.md @@ -46,8 +46,10 @@ Design highlights: `StorageMax * StorageGCWatermark`. - **Idle timeout**: DAG fetches timeout after 2 minutes without receiving new blocks, allowing large downloads while skipping dead records. -- **Provide after pin**: the checker publishes a DHT provider record after - pinning so other peers can discover the content on this node. +- **Provide after pin**: after pinning, the checker publishes a DHT provider + record via `DHTProvider.StartProviding` so other peers can discover the + content. Honors `Provide.Enabled` (daemon will not start if providing is off) + and re-announces periodically. - **Sybil limitation**: provider counts come from DHT queries, which are susceptible to Sybil manipulation. Documented as a known limitation. diff --git a/docs/config.md b/docs/config.md index 5631178f065..77a6fe1edf1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1247,7 +1247,8 @@ Enables on-demand pinning. When enabled, the node runs a background checker that periodically evaluates DHT provider counts for CIDs registered via `ipfs pin ondemand add`. CIDs with fewer providers than the replication target are pinned; pins are removed after replication stays above target for a grace -period (default 24h). +period (default 24h). Requires usable content routing and +[`Provide.Enabled`](#provideenabled). See [`OnDemandPinning`](#ondemandpinning) for configuration. diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 2d540bdfe60..960f8d2d873 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -635,8 +635,9 @@ The feature consists of: `"kubo:on-demand"`, so the checker can tell its pins apart from user pins and will never remove a pin it did not create. -When the checker pins a CID it also publishes a provider record to the DHT -so other peers can discover the content on this node. +When the checker pins a CID it publishes a DHT provider record (via +`StartProviding`) so other peers can discover the content on this node. +Requires `Provide.Enabled`. **Security consideration**: DHT provider counts can be gamed via Sybil attacks. An attacker could inflate provider counts to trick nodes into diff --git a/ondemandpin/checker.go b/ondemandpin/checker.go index 7bd90049249..5cbbc46f8fa 100644 --- a/ondemandpin/checker.go +++ b/ondemandpin/checker.go @@ -14,6 +14,7 @@ import ( "github.com/libp2p/go-libp2p-kad-dht/amino" peer "github.com/libp2p/go-libp2p/core/peer" routing "github.com/libp2p/go-libp2p/core/routing" + mh "github.com/multiformats/go-multihash" ) var log = logging.Logger("ondemandpin") @@ -38,12 +39,18 @@ type StorageChecker interface { StorageUsage(ctx context.Context) (used, limit uint64, err error) } +// Kubo's DHTProvider; may be a no-op. +type Provider interface { + StartProviding(force bool, keys ...mh.Multihash) error +} + type Checker struct { - store *Store - pins PinService - storage StorageChecker - routing routing.ContentRouting - selfID peer.ID + store *Store + pins PinService + storage StorageChecker + routing routing.ContentRouting + provider Provider + selfID peer.ID replicationMin int replicationMax int @@ -62,15 +69,17 @@ func NewChecker( pins PinService, storage StorageChecker, cr routing.ContentRouting, + provider Provider, selfID peer.ID, cfg config.OnDemandPinning, ) *Checker { c := &Checker{ - store: store, - pins: pins, - storage: storage, - routing: cr, - selfID: selfID, + store: store, + pins: pins, + storage: storage, + routing: cr, + provider: provider, + selfID: selfID, replicationMin: int(cfg.ReplicationTargetMin.WithDefault(config.DefaultOnDemandPinReplicationTargetMin)), replicationMax: int(cfg.ReplicationTargetMax.WithDefault(config.DefaultOnDemandPinReplicationTargetMax)), @@ -258,7 +267,7 @@ func (c *Checker) handleUnderReplicated(runCtx, lookupCtx context.Context, rec * rec.LastResult = "pinned" log.Infow("pinned", "cid", rec.Cid, "providers", count, "min", c.replicationMin) - if err := c.routing.Provide(runCtx, rec.Cid, true); err != nil { + if err := c.provider.StartProviding(true, rec.Cid.Hash()); err != nil { log.Warnw("failed to provide after pin", "cid", rec.Cid, "error", err) } c.saveRecord(runCtx, rec) diff --git a/ondemandpin/checker_test.go b/ondemandpin/checker_test.go index 638de739efe..0a69bf9dca5 100644 --- a/ondemandpin/checker_test.go +++ b/ondemandpin/checker_test.go @@ -13,6 +13,7 @@ import ( dssync "github.com/ipfs/go-datastore/sync" "github.com/ipfs/kubo/config" peer "github.com/libp2p/go-libp2p/core/peer" + mh "github.com/multiformats/go-multihash" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -70,6 +71,15 @@ func (m *mockRouting) FindProvidersAsync(ctx context.Context, c cid.Cid, limit i func (m *mockRouting) Provide(context.Context, cid.Cid, bool) error { return nil } +type mockProvider struct { + keys []mh.Multihash +} + +func (m *mockProvider) StartProviding(_ bool, keys ...mh.Multihash) error { + m.keys = append(m.keys, keys...) + return nil +} + type mockPins struct { pinned map[cid.Cid]string } @@ -101,20 +111,21 @@ func (m *mockPins) isPinned(c cid.Cid) bool { return ok } -func newTestChecker(t *testing.T) (*Checker, *Store, *mockRouting, *mockPins, *fakeClock) { +func newTestChecker(t *testing.T) (*Checker, *Store, *mockRouting, *mockPins, *fakeClock, *mockProvider) { t.Helper() store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) r := newMockRouting() p := newMockPins() + prov := &mockProvider{} clock := newFakeClock() - checker := NewChecker(store, p, nil, r, peer.ID("self"), config.OnDemandPinning{}) + checker := NewChecker(store, p, nil, r, prov, peer.ID("self"), config.OnDemandPinning{}) checker.checkInterval = time.Minute checker.unpinGracePeriod = 200 * time.Millisecond checker.now = clock.Now checker.graceJitter = func() time.Duration { return 0 } - return checker, store, r, p, clock + return checker, store, r, p, clock, prov } func providers(n int) []peer.ID { @@ -128,7 +139,7 @@ func providers(n int) []peer.ID { // Under-replicated content gets pinned. func TestCheckerPinsBelowTarget(t *testing.T) { ctx := context.Background() - checker, store, r, p, _ := newTestChecker(t) + checker, store, r, p, _, prov := newTestChecker(t) c := testCID(t, "under-replicated") require.NoError(t, store.Add(ctx, c)) @@ -141,11 +152,12 @@ func TestCheckerPinsBelowTarget(t *testing.T) { assert.Equal(t, "pinned", rec.LastResult) assert.Equal(t, 2, rec.LastProviderCount) assert.False(t, rec.LastCheckedAt.IsZero()) + require.Equal(t, []mh.Multihash{c.Hash()}, prov.keys) } func TestCheckerDoesNotPinInDeadband(t *testing.T) { ctx := context.Background() - checker, store, r, p, _ := newTestChecker(t) + checker, store, r, p, _, _ := newTestChecker(t) c := testCID(t, "deadband") require.NoError(t, store.Add(ctx, c)) @@ -158,7 +170,7 @@ func TestCheckerDoesNotPinInDeadband(t *testing.T) { func TestCheckerUnpinsAfterGracePeriod(t *testing.T) { ctx := context.Background() - checker, store, r, p, clock := newTestChecker(t) + checker, store, r, p, clock, _ := newTestChecker(t) c := testCID(t, "recovering") require.NoError(t, store.Add(ctx, c)) @@ -178,7 +190,7 @@ func TestCheckerUnpinsAfterGracePeriod(t *testing.T) { func TestCheckerGraceIncludesJitter(t *testing.T) { ctx := context.Background() - checker, store, r, p, clock := newTestChecker(t) + checker, store, r, p, clock, _ := newTestChecker(t) checker.graceJitter = func() time.Duration { return 100 * time.Millisecond } c := testCID(t, "jitter") @@ -216,7 +228,7 @@ func TestCheckerSkipsPinCreatedDuringLookup(t *testing.T) { r := newMockRouting() p := newMockPins() racing := &pinDuringLookupRouting{mockRouting: r, pins: p, pinName: "user-pin"} - checker := NewChecker(store, p, nil, racing, peer.ID("self"), config.OnDemandPinning{}) + checker := NewChecker(store, p, nil, racing, &mockProvider{}, peer.ID("self"), config.OnDemandPinning{}) checker.now = newFakeClock().Now c := testCID(t, "raced") @@ -230,7 +242,7 @@ func TestCheckerSkipsPinCreatedDuringLookup(t *testing.T) { func TestCheckerOwnsPinByNameNotStoreField(t *testing.T) { ctx := context.Background() - checker, store, r, p, clock := newTestChecker(t) + checker, store, r, p, clock, _ := newTestChecker(t) c := testCID(t, "name-owned") require.NoError(t, store.Add(ctx, c)) @@ -316,7 +328,7 @@ func TestCheckerSkipsWhenProviderCountUnknown(t *testing.T) { ctx := context.Background() store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) p := newMockPins() - checker := NewChecker(store, p, nil, blockingRouting{}, peer.ID("self"), config.OnDemandPinning{}) + checker := NewChecker(store, p, nil, blockingRouting{}, &mockProvider{}, peer.ID("self"), config.OnDemandPinning{}) checker.now = newFakeClock().Now c := testCID(t, "hung-lookup") @@ -360,7 +372,7 @@ func TestPinContextHasNoCheckTimeout(t *testing.T) { store := NewStore(dssync.MutexWrap(datastore.NewMapDatastore())) r := newMockRouting() pins := &capturePinCtx{mockPins: newMockPins()} - checker := NewChecker(store, pins, nil, r, peer.ID("self"), config.OnDemandPinning{}) + checker := NewChecker(store, pins, nil, r, &mockProvider{}, peer.ID("self"), config.OnDemandPinning{}) checker.now = newFakeClock().Now checker.graceJitter = func() time.Duration { return 0 } @@ -380,7 +392,7 @@ func TestCheckerBackoffSkipsUntilDue(t *testing.T) { r := newMockRouting() pins := &failPinOnce{mockPins: newMockPins(), fail: true} clock := newFakeClock() - checker := NewChecker(store, pins, nil, r, peer.ID("self"), config.OnDemandPinning{}) + checker := NewChecker(store, pins, nil, r, &mockProvider{}, peer.ID("self"), config.OnDemandPinning{}) checker.checkInterval = time.Minute checker.now = clock.Now checker.graceJitter = func() time.Duration { return 0 } @@ -414,7 +426,7 @@ func TestCheckerDryRunDoesNotPinOrUnpin(t *testing.T) { r := newMockRouting() p := newMockPins() clock := newFakeClock() - checker := NewChecker(store, p, nil, r, peer.ID("self"), config.OnDemandPinning{ + checker := NewChecker(store, p, nil, r, &mockProvider{}, peer.ID("self"), config.OnDemandPinning{ DryRun: config.True, }) checker.checkInterval = time.Minute