feat: configure replication count

cmd/gateway/serve.go

@@ -26,6 +26,7 @@ import (
 	"github.com/spf13/viper"
 	arc "github.com/storacha/go-ds-arc"
 	contentcap "github.com/storacha/go-libstoracha/capabilities/space/content"
+	"github.com/storacha/go-libstoracha/principalresolver"
 	"github.com/storacha/go-ucanto/core/delegation"
 	"github.com/storacha/go-ucanto/did"
 	"github.com/storacha/go-ucanto/ucan"
@@ -156,7 +157,11 @@ var serveCmd = &cobra.Command{
 		indexer, indexerPrincipal := cmdutil.MustGetIndexClient(cfg.Network)
 
 		network := cmdutil.MustGetNetworkConfig(cfg.Network, "")
-		uploadServiceVerifier, err := cmdutil.ResolveDIDWebAndWrap(ctx, network.UploadID)
+		var resolverOpts []principalresolver.Option
+		if network.InsecureDIDResolution {
+			resolverOpts = append(resolverOpts, principalresolver.InsecureResolution())
+		}
+		uploadServiceVerifier, err := cmdutil.ResolveDIDWebAndWrap(ctx, network.UploadID, resolverOpts...)
 		if err != nil {
 			return err
 		}

cmd/retrieve.go

@@ -10,6 +10,7 @@ import (
 	"github.com/mitchellh/go-wordwrap"
 	"github.com/spf13/cobra"
 	contentcap "github.com/storacha/go-libstoracha/capabilities/space/content"
+	"github.com/storacha/go-libstoracha/principalresolver"
 	"github.com/storacha/go-ucanto/core/delegation"
 	"github.com/storacha/go-ucanto/did"
 	"github.com/storacha/go-ucanto/ucan"
@@ -81,7 +82,11 @@ var retrieveCmd = &cobra.Command{
 		}()
 
 		network := cmdutil.MustGetNetworkConfig(cfg.Network, "")
-		uploadServiceVerifier, err := cmdutil.ResolveDIDWebAndWrap(ctx, network.UploadID)
+		var resolverOpts []principalresolver.Option
+		if network.InsecureDIDResolution {
+			resolverOpts = append(resolverOpts, principalresolver.InsecureResolution())
+		}
+		uploadServiceVerifier, err := cmdutil.ResolveDIDWebAndWrap(ctx, network.UploadID, resolverOpts...)
 		if err != nil {
 			return err
 		}

cmd/root.go

@@ -13,6 +13,7 @@ import (
 	"github.com/spf13/viper"
 	"github.com/storacha/guppy/cmd/blob"
 	"github.com/storacha/guppy/cmd/unixfs"
+	"github.com/storacha/guppy/pkg/presets"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/trace"
 
@@ -89,6 +90,15 @@ func init() {
 	rootCmd.PersistentFlags().String("indexer-url", "", "Indexing service URL (overrides network preset)")
 	cobra.CheckErr(viper.BindPFlag("network.indexer_url", rootCmd.PersistentFlags().Lookup("indexer-url")))
 
+	rootCmd.PersistentFlags().Bool("insecure-did-resolution", false, "Enable insecure DID resolution (overrides network preset)")
+	cobra.CheckErr(rootCmd.PersistentFlags().MarkHidden("insecure-did-resolution"))
+	cobra.CheckErr(viper.BindPFlag("network.insecure_did_resolution", rootCmd.PersistentFlags().Lookup("insecure-did-resolution")))
+
+	// Preparation configuration flags
+	rootCmd.PersistentFlags().Uint("replicas", presets.DefaultReplicas, "Number of replicas to request per shard")
+	cobra.CheckErr(rootCmd.PersistentFlags().MarkHidden("replicas"))
+	cobra.CheckErr(viper.BindPFlag("upload.replicas", rootCmd.PersistentFlags().Lookup("replicas")))
+
 	// Add Commands
 	rootCmd.AddCommand(
 		whoamiCmd,

cmd/upload/root.go

@@ -106,6 +106,7 @@ var Cmd = &cobra.Command{
 			preparation.WithBlobUploadParallelism(int(rootFlags.parallelism)),
 			preparation.WithAssumeUnchangedSources(rootFlags.assumeUnchangedSources),
 			preparation.WithEventBus(eb),
+			preparation.WithReplicas(cfg.Upload.Replicas),
 		)
 		allUploads, err := api.FindOrCreateUploads(ctx, spaceDID)
 		if err != nil {

docs/content/configuration.md

@@ -70,6 +70,11 @@ Guppy looks for a TOML config file in these locations, reading only the first fi
     enabled = false
     # Public host(s) for the gateway. Required if subdomain mode is enabled.
     # hosts = ["gateway.example.com"]
+
+[upload]
+  # Number of replicas to request per shard. Cannot be greater than 3. Not
+  # recommended to set less than 3 except for testing purposes.
+  replicas = 3
 ```
 
 ## Environment Variables

internal/cmdutil/cmdutil.go

@@ -242,8 +242,8 @@ func ResolveSpace(c *client.Client, identifier string) (did.DID, error) {
 	return space.DID(), nil
 }
 
-func ResolveDIDWebAndWrap(ctx context.Context, didWeb did.DID) (principal.Verifier, error) {
-	resolver, err := principalresolver.NewHTTPResolver([]did.DID{didWeb})
+func ResolveDIDWebAndWrap(ctx context.Context, didWeb did.DID, opts ...principalresolver.Option) (principal.Verifier, error) {
+	resolver, err := principalresolver.NewHTTPResolver([]did.DID{didWeb}, opts...)
 	if err != nil {
 		return nil, fmt.Errorf("creating principal resolver: %w", err)
 	}

pkg/config/config.go

@@ -10,6 +10,7 @@ type Config struct {
 	Repo    RepoConfig    `mapstructure:"repo" toml:"repo"`
 	Gateway GatewayConfig `mapstructure:"gateway" toml:"gateway"`
 	Network NetworkConfig `mapstructure:"network" toml:"network"`
+	Upload  UploadConfig  `mapstructure:"upload" toml:"upload"`
 }
 
 func (c Config) Validate() error {
@@ -20,7 +21,10 @@ func (c Config) Validate() error {
 	if err := c.Gateway.Validate(); err != nil {
 		return err
 	}
-	return c.Network.Validate()
+	if err := c.Network.Validate(); err != nil {
+		return err
+	}
+	return c.Upload.Validate()
 }
 
 func Load[T Validatable]() (T, error) {

pkg/config/network.go

@@ -29,6 +29,9 @@ type NetworkConfig struct {
 	// AuthorizedRetrievals indicates whether UCAN authorized retrievals are supported.
 	// Use a pointer to distinguish between unset and false.
 	AuthorizedRetrievals *bool `mapstructure:"authorized_retrievals" toml:"authorized_retrievals"`
+	// InsecureDIDResolution enables HTTP (instead of HTTPS) for did:web resolution.
+	// NB: this should only be used for development purposes.
+	InsecureDIDResolution bool `mapstructure:"insecure_did_resolution" toml:"insecure_did_resolution,omitempty"`
 }
 
 // IsEmpty returns true if no network configuration fields are set.
@@ -140,5 +143,10 @@ func (n NetworkConfig) ToPresetConfig(baseName string) (presets.NetworkConfig, e
 		network.Name = "custom"
 	}
 
+	if n.InsecureDIDResolution {
+		network.InsecureDIDResolution = true
+		network.Name = "custom"
+	}
+
 	return network, nil
 }

pkg/config/upload.go

@@ -0,0 +1,17 @@
+package config
+
+import "fmt"
+
+type UploadConfig struct {
+	// Number of replicas to request per shard. Cannot be greater than 3. Not
+	// recommended to set less than 3 except for testing purposes. Should be
+	// 1 or more if set.
+	Replicas uint `mapstructure:"replicas" toml:"replicas"`
+}
+
+func (c UploadConfig) Validate() error {
+	if c.Replicas == 0 {
+		return fmt.Errorf("upload.replicas must be greater than 0")
+	}
+	return nil
+}

pkg/preparation/preparation.go

@@ -2,6 +2,7 @@ package preparation
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/fs"
 	"os"
@@ -59,9 +60,13 @@ type config struct {
 	blobUploadParallelism  int
 	assumeUnchangedSources bool
 	bus                    bus.Bus
+	replicas               uint
 }
 
-const defaultBlobUploadParallelism = 6
+const (
+	defaultBlobUploadParallelism = 6
+	defaultReplicas              = 3
+)
 
 func NewAPI(repo Repo, client StorachaClient, options ...Option) API {
 	cfg := &config{
@@ -90,6 +95,7 @@ func NewAPI(repo Repo, client StorachaClient, options ...Option) API {
 		},
 		maxNodesPerIndex: defaultMaxNodesPerIndex,
 		bus:              &bus.NoopBus{},
+		replicas:         defaultReplicas,
 	}
 	for _, opt := range options {
 		if err := opt(cfg); err != nil {
@@ -157,6 +163,7 @@ func NewAPI(repo Repo, client StorachaClient, options ...Option) API {
 		ReaderForIndex:        blobsAPI.ReaderForIndex,
 		BlobUploadParallelism: cfg.blobUploadParallelism,
 		Bus:                   cfg.bus,
+		Replicas:              cfg.replicas,
 	}
 
 	uploadsAPI = uploads.API{
@@ -228,6 +235,18 @@ func WithBlobUploadParallelism(blobUploadParallelism int) Option {
 	}
 }
 
+// WithReplicas sets the number of replicas to use when uploading blobs,
+// including the original. The default is 3 replicas.
+func WithReplicas(replicas uint) Option {
+	return func(cfg *config) error {
+		if replicas == 0 {
+			return errors.New("replica count must be greater than 0")
+		}
+		cfg.replicas = replicas
+		return nil
+	}
+}
+
 func (a API) FindOrCreateSpace(ctx context.Context, spaceDID did.DID, name string, options ...spacesmodel.SpaceOption) (*spacesmodel.Space, error) {
 	return a.Spaces.FindOrCreateSpace(ctx, spaceDID, name, options...)
 }

pkg/preparation/storacha/storacha.go

@@ -64,6 +64,7 @@ type API struct {
 	ReaderForIndex        ReaderForIndexFunc
 	BlobUploadParallelism int
 	Bus                   bus.Publisher
+	Replicas              uint
 }
 
 var _ uploads.AddShardsForUploadFunc = API{}.AddShardsForUpload
@@ -342,14 +343,20 @@ func (a API) spaceBlobReplicate(ctx context.Context, blob model.Blob, spaceDID d
 	ctx, span := tracer.Start(ctx, "space-blob-replicate")
 	defer span.End()
 
+	// if the replication count is 1 (or less) then there is nothing to do
+	replicas := a.Replicas
+	if replicas <= 1 {
+		return nil
+	}
+
 	_, _, err := a.Client.SpaceBlobReplicate(
 		ctx,
 		spaceDID,
 		types.Blob{
 			Digest: blob.Digest(),
 			Size:   blob.Size(),
 		},
-		3,
+		replicas,
 		locationCommitment,
 	)
 	return err

pkg/preparation/storacha/storacha_test.go

@@ -57,6 +57,7 @@ func TestAddShardsForUpload(t *testing.T) {
 			Client:                &client,
 			ReaderForShard:        carForShard,
 			BlobUploadParallelism: 1,
+			Replicas:              3,
 		}
 
 		blobsApi := blobs.API{
@@ -185,6 +186,7 @@ func TestAddShardsForUpload(t *testing.T) {
 			Client:                &client,
 			ReaderForShard:        carForShard,
 			BlobUploadParallelism: 1,
+			Replicas:              3,
 		}
 
 		blobsApi := blobs.API{
@@ -276,6 +278,7 @@ func TestAddShardsForUpload(t *testing.T) {
 			Client:                &client,
 			ReaderForShard:        carForShard,
 			BlobUploadParallelism: 1,
+			Replicas:              3,
 		}
 
 		blobsApi := blobs.API{
@@ -328,6 +331,7 @@ func TestAddIndexesForUpload(t *testing.T) {
 			Client:                &client,
 			ReaderForIndex:        carForIndex,
 			BlobUploadParallelism: 1,
+			Replicas:              3,
 		}
 
 		blobsApi := blobs.API{
@@ -459,6 +463,7 @@ func TestAddStorachaUploadForUpload(t *testing.T) {
 			Repo:                  repo,
 			Client:                &mclient,
 			BlobUploadParallelism: 1,
+			Replicas:              3,
 		}
 
 		upload, _ := testutil.CreateUpload(t, repo, spaceDID, spacesmodel.WithShardSize(1<<16))

pkg/presets/presets.go

@@ -42,6 +42,9 @@ type NetworkConfig struct {
 	UploadID             did.DID
 	UploadURL            url.URL
 	AuthorizedRetrievals bool // Support for UCAN authorized retrievals.
+	// InsecureDIDResolution enables HTTP (instead of HTTPS) for did:web resolution.
+	// NB: this should only be used for development purposes.
+	InsecureDIDResolution bool
 }
 
 // Known network configurations.
@@ -84,7 +87,10 @@ var Networks = []NetworkConfig{
 	},
 }
 
-var DefaultNetwork = Networks[0]
+var (
+	DefaultNetwork  = Networks[0]
+	DefaultReplicas = uint(3)
+)
 
 // GetNetworkConfig returns the network config for the passed name or the
 // STORACHA_NETWORK environment variable if set. If both are empty, the default