From 0d7c1c3d1a367e93d8ecd60347e865c9f76053ad Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 9 Apr 2026 20:38:45 -0700 Subject: [PATCH 1/6] feat: configure replication count --- cmd/root.go | 6 ++++++ cmd/upload/root.go | 1 + docs/content/configuration.md | 5 +++++ pkg/config/config.go | 7 ++++--- pkg/config/preparation.go | 17 +++++++++++++++++ pkg/preparation/preparation.go | 21 ++++++++++++++++++++- pkg/preparation/storacha/storacha.go | 9 ++++++++- pkg/presets/presets.go | 5 ++++- 8 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 pkg/config/preparation.go diff --git a/cmd/root.go b/cmd/root.go index c17d1f92..8da1f5fa 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/viper" "github.com/storacha/guppy/cmd/blob" "github.com/storacha/guppy/cmd/unixfs" + "github.com/storacha/guppy/pkg/presets" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" @@ -89,6 +90,11 @@ func init() { rootCmd.PersistentFlags().String("indexer-url", "", "Indexing service URL (overrides network preset)") cobra.CheckErr(viper.BindPFlag("network.indexer_url", rootCmd.PersistentFlags().Lookup("indexer-url"))) + // Preparation configuration flags + rootCmd.PersistentFlags().Uint("replicas", presets.DefaultReplicas, "Number of replicas to request per shard") + cobra.CheckErr(rootCmd.Flags().MarkHidden("replicas")) + cobra.CheckErr(viper.BindPFlag("preparation.replicas", rootCmd.PersistentFlags().Lookup("replicas"))) + // Add Commands rootCmd.AddCommand( whoamiCmd, diff --git a/cmd/upload/root.go b/cmd/upload/root.go index f43b3f30..7b362367 100644 --- a/cmd/upload/root.go +++ b/cmd/upload/root.go @@ -106,6 +106,7 @@ var Cmd = &cobra.Command{ preparation.WithBlobUploadParallelism(int(rootFlags.parallelism)), preparation.WithAssumeUnchangedSources(rootFlags.assumeUnchangedSources), preparation.WithEventBus(eb), + preparation.WithReplicas(cfg.Preparation.Replicas), ) allUploads, err := api.FindOrCreateUploads(ctx, spaceDID) if err != nil { diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 587a13f3..fbbba508 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -70,6 +70,11 @@ Guppy looks for a TOML config file in these locations, reading only the first fi enabled = false # Public host(s) for the gateway. Required if subdomain mode is enabled. # hosts = ["gateway.example.com"] + +[preparation] + # Number of replicas to request per shard. Cannot be greater than 3. Not + # recommended to set less than 3 except for testing purposes. + replicas = 3 ``` ## Environment Variables diff --git a/pkg/config/config.go b/pkg/config/config.go index 07f1a226..7b8525c5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -7,9 +7,10 @@ import ( ) type Config struct { - Repo RepoConfig `mapstructure:"repo" toml:"repo"` - Gateway GatewayConfig `mapstructure:"gateway" toml:"gateway"` - Network NetworkConfig `mapstructure:"network" toml:"network"` + Repo RepoConfig `mapstructure:"repo" toml:"repo"` + Gateway GatewayConfig `mapstructure:"gateway" toml:"gateway"` + Network NetworkConfig `mapstructure:"network" toml:"network"` + Preparation PreparationConfig `mapstructure:"preparation" toml:"preparation"` } func (c Config) Validate() error { diff --git a/pkg/config/preparation.go b/pkg/config/preparation.go new file mode 100644 index 00000000..0d201e5e --- /dev/null +++ b/pkg/config/preparation.go @@ -0,0 +1,17 @@ +package config + +import "fmt" + +type PreparationConfig struct { + // Number of replicas to request per shard. Cannot be greater than 3. Not + // recommended to set less than 3 except for testing purposes. Should be + // 1 or more if set. + Replicas uint `mapstructure:"replicas" toml:"replicas"` +} + +func (c PreparationConfig) Validate() error { + if c.Replicas == 0 { + return fmt.Errorf("preparation.replicas must be greater than 0") + } + return nil +} diff --git a/pkg/preparation/preparation.go b/pkg/preparation/preparation.go index c32dc5e2..c3a93b74 100644 --- a/pkg/preparation/preparation.go +++ b/pkg/preparation/preparation.go @@ -2,6 +2,7 @@ package preparation import ( "context" + "errors" "fmt" "io/fs" "os" @@ -59,9 +60,13 @@ type config struct { blobUploadParallelism int assumeUnchangedSources bool bus bus.Bus + replicas uint } -const defaultBlobUploadParallelism = 6 +const ( + defaultBlobUploadParallelism = 6 + defaultReplicas = 3 +) func NewAPI(repo Repo, client StorachaClient, options ...Option) API { cfg := &config{ @@ -90,6 +95,7 @@ func NewAPI(repo Repo, client StorachaClient, options ...Option) API { }, maxNodesPerIndex: defaultMaxNodesPerIndex, bus: &bus.NoopBus{}, + replicas: defaultReplicas, } for _, opt := range options { if err := opt(cfg); err != nil { @@ -157,6 +163,7 @@ func NewAPI(repo Repo, client StorachaClient, options ...Option) API { ReaderForIndex: blobsAPI.ReaderForIndex, BlobUploadParallelism: cfg.blobUploadParallelism, Bus: cfg.bus, + Replicas: cfg.replicas, } uploadsAPI = uploads.API{ @@ -228,6 +235,18 @@ func WithBlobUploadParallelism(blobUploadParallelism int) Option { } } +// WithReplicas sets the number of replicas to use when uploading blobs, +// including the original. The default is 3 replicas. +func WithReplicas(replicas uint) Option { + return func(cfg *config) error { + if replicas == 0 { + return errors.New("replica count must be greater than 0") + } + cfg.replicas = replicas + return nil + } +} + func (a API) FindOrCreateSpace(ctx context.Context, spaceDID did.DID, name string, options ...spacesmodel.SpaceOption) (*spacesmodel.Space, error) { return a.Spaces.FindOrCreateSpace(ctx, spaceDID, name, options...) } diff --git a/pkg/preparation/storacha/storacha.go b/pkg/preparation/storacha/storacha.go index 1c1d5313..3fedf426 100644 --- a/pkg/preparation/storacha/storacha.go +++ b/pkg/preparation/storacha/storacha.go @@ -64,6 +64,7 @@ type API struct { ReaderForIndex ReaderForIndexFunc BlobUploadParallelism int Bus bus.Publisher + Replicas uint } var _ uploads.AddShardsForUploadFunc = API{}.AddShardsForUpload @@ -342,6 +343,12 @@ func (a API) spaceBlobReplicate(ctx context.Context, blob model.Blob, spaceDID d ctx, span := tracer.Start(ctx, "space-blob-replicate") defer span.End() + // if the replication count is 1 (or less) then there is nothing to do + replicas := a.Replicas + if replicas <= 1 { + return nil + } + _, _, err := a.Client.SpaceBlobReplicate( ctx, spaceDID, @@ -349,7 +356,7 @@ func (a API) spaceBlobReplicate(ctx context.Context, blob model.Blob, spaceDID d Digest: blob.Digest(), Size: blob.Size(), }, - 3, + replicas, locationCommitment, ) return err diff --git a/pkg/presets/presets.go b/pkg/presets/presets.go index 2ff23e93..d1720824 100644 --- a/pkg/presets/presets.go +++ b/pkg/presets/presets.go @@ -84,7 +84,10 @@ var Networks = []NetworkConfig{ }, } -var DefaultNetwork = Networks[0] +var ( + DefaultNetwork = Networks[0] + DefaultReplicas = uint(3) +) // GetNetworkConfig returns the network config for the passed name or the // STORACHA_NETWORK environment variable if set. If both are empty, the default From 96da40b882ce9d5c2ab1490c826d0d3c14aeab67 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Thu, 9 Apr 2026 20:54:47 -0700 Subject: [PATCH 2/6] fix: tests --- pkg/preparation/storacha/storacha_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/preparation/storacha/storacha_test.go b/pkg/preparation/storacha/storacha_test.go index 9efcc63a..ee5bb431 100644 --- a/pkg/preparation/storacha/storacha_test.go +++ b/pkg/preparation/storacha/storacha_test.go @@ -57,6 +57,7 @@ func TestAddShardsForUpload(t *testing.T) { Client: &client, ReaderForShard: carForShard, BlobUploadParallelism: 1, + Replicas: 3, } blobsApi := blobs.API{ @@ -185,6 +186,7 @@ func TestAddShardsForUpload(t *testing.T) { Client: &client, ReaderForShard: carForShard, BlobUploadParallelism: 1, + Replicas: 3, } blobsApi := blobs.API{ @@ -276,6 +278,7 @@ func TestAddShardsForUpload(t *testing.T) { Client: &client, ReaderForShard: carForShard, BlobUploadParallelism: 1, + Replicas: 3, } blobsApi := blobs.API{ @@ -328,6 +331,7 @@ func TestAddIndexesForUpload(t *testing.T) { Client: &client, ReaderForIndex: carForIndex, BlobUploadParallelism: 1, + Replicas: 3, } blobsApi := blobs.API{ @@ -459,6 +463,7 @@ func TestAddStorachaUploadForUpload(t *testing.T) { Repo: repo, Client: &mclient, BlobUploadParallelism: 1, + Replicas: 3, } upload, _ := testutil.CreateUpload(t, repo, spaceDID, spacesmodel.WithShardSize(1<<16)) From 4a3b76f95571e95e5e4b9c764b0496f2b0c35e08 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 10 Apr 2026 11:56:44 +0100 Subject: [PATCH 3/6] fix: replicas is persistent flag --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 8da1f5fa..03ef5e85 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -92,7 +92,7 @@ func init() { // Preparation configuration flags rootCmd.PersistentFlags().Uint("replicas", presets.DefaultReplicas, "Number of replicas to request per shard") - cobra.CheckErr(rootCmd.Flags().MarkHidden("replicas")) + cobra.CheckErr(rootCmd.PersistentFlags().MarkHidden("replicas")) cobra.CheckErr(viper.BindPFlag("preparation.replicas", rootCmd.PersistentFlags().Lookup("replicas"))) // Add Commands From c79dec20d42b963da88857fb3728aca56fbc2efc Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 14 Apr 2026 11:20:48 +0100 Subject: [PATCH 4/6] refactor: rename config --- cmd/root.go | 2 +- cmd/upload/root.go | 2 +- docs/content/configuration.md | 2 +- pkg/config/config.go | 13 ++++++++----- pkg/config/preparation.go | 17 ----------------- 5 files changed, 11 insertions(+), 25 deletions(-) delete mode 100644 pkg/config/preparation.go diff --git a/cmd/root.go b/cmd/root.go index 03ef5e85..f68d42c6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -93,7 +93,7 @@ func init() { // Preparation configuration flags rootCmd.PersistentFlags().Uint("replicas", presets.DefaultReplicas, "Number of replicas to request per shard") cobra.CheckErr(rootCmd.PersistentFlags().MarkHidden("replicas")) - cobra.CheckErr(viper.BindPFlag("preparation.replicas", rootCmd.PersistentFlags().Lookup("replicas"))) + cobra.CheckErr(viper.BindPFlag("upload.replicas", rootCmd.PersistentFlags().Lookup("replicas"))) // Add Commands rootCmd.AddCommand( diff --git a/cmd/upload/root.go b/cmd/upload/root.go index 7b362367..b7ecba44 100644 --- a/cmd/upload/root.go +++ b/cmd/upload/root.go @@ -106,7 +106,7 @@ var Cmd = &cobra.Command{ preparation.WithBlobUploadParallelism(int(rootFlags.parallelism)), preparation.WithAssumeUnchangedSources(rootFlags.assumeUnchangedSources), preparation.WithEventBus(eb), - preparation.WithReplicas(cfg.Preparation.Replicas), + preparation.WithReplicas(cfg.Upload.Replicas), ) allUploads, err := api.FindOrCreateUploads(ctx, spaceDID) if err != nil { diff --git a/docs/content/configuration.md b/docs/content/configuration.md index fbbba508..e451a00b 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -71,7 +71,7 @@ Guppy looks for a TOML config file in these locations, reading only the first fi # Public host(s) for the gateway. Required if subdomain mode is enabled. # hosts = ["gateway.example.com"] -[preparation] +[upload] # Number of replicas to request per shard. Cannot be greater than 3. Not # recommended to set less than 3 except for testing purposes. replicas = 3 diff --git a/pkg/config/config.go b/pkg/config/config.go index 7b8525c5..50978ce6 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -7,10 +7,10 @@ import ( ) type Config struct { - Repo RepoConfig `mapstructure:"repo" toml:"repo"` - Gateway GatewayConfig `mapstructure:"gateway" toml:"gateway"` - Network NetworkConfig `mapstructure:"network" toml:"network"` - Preparation PreparationConfig `mapstructure:"preparation" toml:"preparation"` + Repo RepoConfig `mapstructure:"repo" toml:"repo"` + Gateway GatewayConfig `mapstructure:"gateway" toml:"gateway"` + Network NetworkConfig `mapstructure:"network" toml:"network"` + Upload UploadConfig `mapstructure:"upload" toml:"upload"` } func (c Config) Validate() error { @@ -21,7 +21,10 @@ func (c Config) Validate() error { if err := c.Gateway.Validate(); err != nil { return err } - return c.Network.Validate() + if err := c.Network.Validate(); err != nil { + return err + } + return c.Upload.Validate() } func Load[T Validatable]() (T, error) { diff --git a/pkg/config/preparation.go b/pkg/config/preparation.go deleted file mode 100644 index 0d201e5e..00000000 --- a/pkg/config/preparation.go +++ /dev/null @@ -1,17 +0,0 @@ -package config - -import "fmt" - -type PreparationConfig struct { - // Number of replicas to request per shard. Cannot be greater than 3. Not - // recommended to set less than 3 except for testing purposes. Should be - // 1 or more if set. - Replicas uint `mapstructure:"replicas" toml:"replicas"` -} - -func (c PreparationConfig) Validate() error { - if c.Replicas == 0 { - return fmt.Errorf("preparation.replicas must be greater than 0") - } - return nil -} From 0ccf758fba595a4ecd0a2a0027f148eb34d765a7 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 15 Apr 2026 10:46:25 +0100 Subject: [PATCH 5/6] fix: add missing file --- pkg/config/upload.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 pkg/config/upload.go diff --git a/pkg/config/upload.go b/pkg/config/upload.go new file mode 100644 index 00000000..4d02ed30 --- /dev/null +++ b/pkg/config/upload.go @@ -0,0 +1,17 @@ +package config + +import "fmt" + +type UploadConfig struct { + // Number of replicas to request per shard. Cannot be greater than 3. Not + // recommended to set less than 3 except for testing purposes. Should be + // 1 or more if set. + Replicas uint `mapstructure:"replicas" toml:"replicas"` +} + +func (c UploadConfig) Validate() error { + if c.Replicas == 0 { + return fmt.Errorf("upload.replicas must be greater than 0") + } + return nil +} From 3aae4bc0df1c4dcf69ca5939af138bacf5b513c5 Mon Sep 17 00:00:00 2001 From: ash Date: Wed, 15 Apr 2026 11:39:56 +0100 Subject: [PATCH 6/6] feat: configure insecure DID resolution (#401) --- cmd/gateway/serve.go | 7 ++++++- cmd/retrieve.go | 7 ++++++- cmd/root.go | 4 ++++ internal/cmdutil/cmdutil.go | 4 ++-- pkg/config/network.go | 8 ++++++++ pkg/presets/presets.go | 3 +++ 6 files changed, 29 insertions(+), 4 deletions(-) diff --git a/cmd/gateway/serve.go b/cmd/gateway/serve.go index f20b0bfe..d5cdda53 100644 --- a/cmd/gateway/serve.go +++ b/cmd/gateway/serve.go @@ -26,6 +26,7 @@ import ( "github.com/spf13/viper" arc "github.com/storacha/go-ds-arc" contentcap "github.com/storacha/go-libstoracha/capabilities/space/content" + "github.com/storacha/go-libstoracha/principalresolver" "github.com/storacha/go-ucanto/core/delegation" "github.com/storacha/go-ucanto/did" "github.com/storacha/go-ucanto/ucan" @@ -156,7 +157,11 @@ var serveCmd = &cobra.Command{ indexer, indexerPrincipal := cmdutil.MustGetIndexClient(cfg.Network) network := cmdutil.MustGetNetworkConfig(cfg.Network, "") - uploadServiceVerifier, err := cmdutil.ResolveDIDWebAndWrap(ctx, network.UploadID) + var resolverOpts []principalresolver.Option + if network.InsecureDIDResolution { + resolverOpts = append(resolverOpts, principalresolver.InsecureResolution()) + } + uploadServiceVerifier, err := cmdutil.ResolveDIDWebAndWrap(ctx, network.UploadID, resolverOpts...) if err != nil { return err } diff --git a/cmd/retrieve.go b/cmd/retrieve.go index b62501f2..ea2ffe8e 100644 --- a/cmd/retrieve.go +++ b/cmd/retrieve.go @@ -10,6 +10,7 @@ import ( "github.com/mitchellh/go-wordwrap" "github.com/spf13/cobra" contentcap "github.com/storacha/go-libstoracha/capabilities/space/content" + "github.com/storacha/go-libstoracha/principalresolver" "github.com/storacha/go-ucanto/core/delegation" "github.com/storacha/go-ucanto/did" "github.com/storacha/go-ucanto/ucan" @@ -81,7 +82,11 @@ var retrieveCmd = &cobra.Command{ }() network := cmdutil.MustGetNetworkConfig(cfg.Network, "") - uploadServiceVerifier, err := cmdutil.ResolveDIDWebAndWrap(ctx, network.UploadID) + var resolverOpts []principalresolver.Option + if network.InsecureDIDResolution { + resolverOpts = append(resolverOpts, principalresolver.InsecureResolution()) + } + uploadServiceVerifier, err := cmdutil.ResolveDIDWebAndWrap(ctx, network.UploadID, resolverOpts...) if err != nil { return err } diff --git a/cmd/root.go b/cmd/root.go index f68d42c6..8e04866b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -90,6 +90,10 @@ func init() { rootCmd.PersistentFlags().String("indexer-url", "", "Indexing service URL (overrides network preset)") cobra.CheckErr(viper.BindPFlag("network.indexer_url", rootCmd.PersistentFlags().Lookup("indexer-url"))) + rootCmd.PersistentFlags().Bool("insecure-did-resolution", false, "Enable insecure DID resolution (overrides network preset)") + cobra.CheckErr(rootCmd.PersistentFlags().MarkHidden("insecure-did-resolution")) + cobra.CheckErr(viper.BindPFlag("network.insecure_did_resolution", rootCmd.PersistentFlags().Lookup("insecure-did-resolution"))) + // Preparation configuration flags rootCmd.PersistentFlags().Uint("replicas", presets.DefaultReplicas, "Number of replicas to request per shard") cobra.CheckErr(rootCmd.PersistentFlags().MarkHidden("replicas")) diff --git a/internal/cmdutil/cmdutil.go b/internal/cmdutil/cmdutil.go index 45d0c014..38cd85d9 100644 --- a/internal/cmdutil/cmdutil.go +++ b/internal/cmdutil/cmdutil.go @@ -242,8 +242,8 @@ func ResolveSpace(c *client.Client, identifier string) (did.DID, error) { return space.DID(), nil } -func ResolveDIDWebAndWrap(ctx context.Context, didWeb did.DID) (principal.Verifier, error) { - resolver, err := principalresolver.NewHTTPResolver([]did.DID{didWeb}) +func ResolveDIDWebAndWrap(ctx context.Context, didWeb did.DID, opts ...principalresolver.Option) (principal.Verifier, error) { + resolver, err := principalresolver.NewHTTPResolver([]did.DID{didWeb}, opts...) if err != nil { return nil, fmt.Errorf("creating principal resolver: %w", err) } diff --git a/pkg/config/network.go b/pkg/config/network.go index dea2e695..cb680b9f 100644 --- a/pkg/config/network.go +++ b/pkg/config/network.go @@ -29,6 +29,9 @@ type NetworkConfig struct { // AuthorizedRetrievals indicates whether UCAN authorized retrievals are supported. // Use a pointer to distinguish between unset and false. AuthorizedRetrievals *bool `mapstructure:"authorized_retrievals" toml:"authorized_retrievals"` + // InsecureDIDResolution enables HTTP (instead of HTTPS) for did:web resolution. + // NB: this should only be used for development purposes. + InsecureDIDResolution bool `mapstructure:"insecure_did_resolution" toml:"insecure_did_resolution,omitempty"` } // IsEmpty returns true if no network configuration fields are set. @@ -140,5 +143,10 @@ func (n NetworkConfig) ToPresetConfig(baseName string) (presets.NetworkConfig, e network.Name = "custom" } + if n.InsecureDIDResolution { + network.InsecureDIDResolution = true + network.Name = "custom" + } + return network, nil } diff --git a/pkg/presets/presets.go b/pkg/presets/presets.go index d1720824..8abcc951 100644 --- a/pkg/presets/presets.go +++ b/pkg/presets/presets.go @@ -42,6 +42,9 @@ type NetworkConfig struct { UploadID did.DID UploadURL url.URL AuthorizedRetrievals bool // Support for UCAN authorized retrievals. + // InsecureDIDResolution enables HTTP (instead of HTTPS) for did:web resolution. + // NB: this should only be used for development purposes. + InsecureDIDResolution bool } // Known network configurations.