diff --git a/cmd/gateway/serve.go b/cmd/gateway/serve.go index f20b0bfe..d5cdda53 100644 --- a/cmd/gateway/serve.go +++ b/cmd/gateway/serve.go @@ -26,6 +26,7 @@ import ( "github.com/spf13/viper" arc "github.com/storacha/go-ds-arc" contentcap "github.com/storacha/go-libstoracha/capabilities/space/content" + "github.com/storacha/go-libstoracha/principalresolver" "github.com/storacha/go-ucanto/core/delegation" "github.com/storacha/go-ucanto/did" "github.com/storacha/go-ucanto/ucan" @@ -156,7 +157,11 @@ var serveCmd = &cobra.Command{ indexer, indexerPrincipal := cmdutil.MustGetIndexClient(cfg.Network) network := cmdutil.MustGetNetworkConfig(cfg.Network, "") - uploadServiceVerifier, err := cmdutil.ResolveDIDWebAndWrap(ctx, network.UploadID) + var resolverOpts []principalresolver.Option + if network.InsecureDIDResolution { + resolverOpts = append(resolverOpts, principalresolver.InsecureResolution()) + } + uploadServiceVerifier, err := cmdutil.ResolveDIDWebAndWrap(ctx, network.UploadID, resolverOpts...) if err != nil { return err } diff --git a/cmd/retrieve.go b/cmd/retrieve.go index b62501f2..ea2ffe8e 100644 --- a/cmd/retrieve.go +++ b/cmd/retrieve.go @@ -10,6 +10,7 @@ import ( "github.com/mitchellh/go-wordwrap" "github.com/spf13/cobra" contentcap "github.com/storacha/go-libstoracha/capabilities/space/content" + "github.com/storacha/go-libstoracha/principalresolver" "github.com/storacha/go-ucanto/core/delegation" "github.com/storacha/go-ucanto/did" "github.com/storacha/go-ucanto/ucan" @@ -81,7 +82,11 @@ var retrieveCmd = &cobra.Command{ }() network := cmdutil.MustGetNetworkConfig(cfg.Network, "") - uploadServiceVerifier, err := cmdutil.ResolveDIDWebAndWrap(ctx, network.UploadID) + var resolverOpts []principalresolver.Option + if network.InsecureDIDResolution { + resolverOpts = append(resolverOpts, principalresolver.InsecureResolution()) + } + uploadServiceVerifier, err := cmdutil.ResolveDIDWebAndWrap(ctx, network.UploadID, resolverOpts...) if err != nil { return err } diff --git a/cmd/root.go b/cmd/root.go index c17d1f92..8e04866b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/viper" "github.com/storacha/guppy/cmd/blob" "github.com/storacha/guppy/cmd/unixfs" + "github.com/storacha/guppy/pkg/presets" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" @@ -89,6 +90,15 @@ func init() { rootCmd.PersistentFlags().String("indexer-url", "", "Indexing service URL (overrides network preset)") cobra.CheckErr(viper.BindPFlag("network.indexer_url", rootCmd.PersistentFlags().Lookup("indexer-url"))) + rootCmd.PersistentFlags().Bool("insecure-did-resolution", false, "Enable insecure DID resolution (overrides network preset)") + cobra.CheckErr(rootCmd.PersistentFlags().MarkHidden("insecure-did-resolution")) + cobra.CheckErr(viper.BindPFlag("network.insecure_did_resolution", rootCmd.PersistentFlags().Lookup("insecure-did-resolution"))) + + // Preparation configuration flags + rootCmd.PersistentFlags().Uint("replicas", presets.DefaultReplicas, "Number of replicas to request per shard") + cobra.CheckErr(rootCmd.PersistentFlags().MarkHidden("replicas")) + cobra.CheckErr(viper.BindPFlag("upload.replicas", rootCmd.PersistentFlags().Lookup("replicas"))) + // Add Commands rootCmd.AddCommand( whoamiCmd, diff --git a/cmd/upload/root.go b/cmd/upload/root.go index f43b3f30..b7ecba44 100644 --- a/cmd/upload/root.go +++ b/cmd/upload/root.go @@ -106,6 +106,7 @@ var Cmd = &cobra.Command{ preparation.WithBlobUploadParallelism(int(rootFlags.parallelism)), preparation.WithAssumeUnchangedSources(rootFlags.assumeUnchangedSources), preparation.WithEventBus(eb), + preparation.WithReplicas(cfg.Upload.Replicas), ) allUploads, err := api.FindOrCreateUploads(ctx, spaceDID) if err != nil { diff --git a/docs/content/configuration.md b/docs/content/configuration.md index 587a13f3..e451a00b 100644 --- a/docs/content/configuration.md +++ b/docs/content/configuration.md @@ -70,6 +70,11 @@ Guppy looks for a TOML config file in these locations, reading only the first fi enabled = false # Public host(s) for the gateway. Required if subdomain mode is enabled. # hosts = ["gateway.example.com"] + +[upload] + # Number of replicas to request per shard. Cannot be greater than 3. Not + # recommended to set less than 3 except for testing purposes. + replicas = 3 ``` ## Environment Variables diff --git a/internal/cmdutil/cmdutil.go b/internal/cmdutil/cmdutil.go index 45d0c014..38cd85d9 100644 --- a/internal/cmdutil/cmdutil.go +++ b/internal/cmdutil/cmdutil.go @@ -242,8 +242,8 @@ func ResolveSpace(c *client.Client, identifier string) (did.DID, error) { return space.DID(), nil } -func ResolveDIDWebAndWrap(ctx context.Context, didWeb did.DID) (principal.Verifier, error) { - resolver, err := principalresolver.NewHTTPResolver([]did.DID{didWeb}) +func ResolveDIDWebAndWrap(ctx context.Context, didWeb did.DID, opts ...principalresolver.Option) (principal.Verifier, error) { + resolver, err := principalresolver.NewHTTPResolver([]did.DID{didWeb}, opts...) if err != nil { return nil, fmt.Errorf("creating principal resolver: %w", err) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 07f1a226..50978ce6 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -10,6 +10,7 @@ type Config struct { Repo RepoConfig `mapstructure:"repo" toml:"repo"` Gateway GatewayConfig `mapstructure:"gateway" toml:"gateway"` Network NetworkConfig `mapstructure:"network" toml:"network"` + Upload UploadConfig `mapstructure:"upload" toml:"upload"` } func (c Config) Validate() error { @@ -20,7 +21,10 @@ func (c Config) Validate() error { if err := c.Gateway.Validate(); err != nil { return err } - return c.Network.Validate() + if err := c.Network.Validate(); err != nil { + return err + } + return c.Upload.Validate() } func Load[T Validatable]() (T, error) { diff --git a/pkg/config/network.go b/pkg/config/network.go index dea2e695..cb680b9f 100644 --- a/pkg/config/network.go +++ b/pkg/config/network.go @@ -29,6 +29,9 @@ type NetworkConfig struct { // AuthorizedRetrievals indicates whether UCAN authorized retrievals are supported. // Use a pointer to distinguish between unset and false. AuthorizedRetrievals *bool `mapstructure:"authorized_retrievals" toml:"authorized_retrievals"` + // InsecureDIDResolution enables HTTP (instead of HTTPS) for did:web resolution. + // NB: this should only be used for development purposes. + InsecureDIDResolution bool `mapstructure:"insecure_did_resolution" toml:"insecure_did_resolution,omitempty"` } // IsEmpty returns true if no network configuration fields are set. @@ -140,5 +143,10 @@ func (n NetworkConfig) ToPresetConfig(baseName string) (presets.NetworkConfig, e network.Name = "custom" } + if n.InsecureDIDResolution { + network.InsecureDIDResolution = true + network.Name = "custom" + } + return network, nil } diff --git a/pkg/config/upload.go b/pkg/config/upload.go new file mode 100644 index 00000000..4d02ed30 --- /dev/null +++ b/pkg/config/upload.go @@ -0,0 +1,17 @@ +package config + +import "fmt" + +type UploadConfig struct { + // Number of replicas to request per shard. Cannot be greater than 3. Not + // recommended to set less than 3 except for testing purposes. Should be + // 1 or more if set. + Replicas uint `mapstructure:"replicas" toml:"replicas"` +} + +func (c UploadConfig) Validate() error { + if c.Replicas == 0 { + return fmt.Errorf("upload.replicas must be greater than 0") + } + return nil +} diff --git a/pkg/preparation/preparation.go b/pkg/preparation/preparation.go index c32dc5e2..c3a93b74 100644 --- a/pkg/preparation/preparation.go +++ b/pkg/preparation/preparation.go @@ -2,6 +2,7 @@ package preparation import ( "context" + "errors" "fmt" "io/fs" "os" @@ -59,9 +60,13 @@ type config struct { blobUploadParallelism int assumeUnchangedSources bool bus bus.Bus + replicas uint } -const defaultBlobUploadParallelism = 6 +const ( + defaultBlobUploadParallelism = 6 + defaultReplicas = 3 +) func NewAPI(repo Repo, client StorachaClient, options ...Option) API { cfg := &config{ @@ -90,6 +95,7 @@ func NewAPI(repo Repo, client StorachaClient, options ...Option) API { }, maxNodesPerIndex: defaultMaxNodesPerIndex, bus: &bus.NoopBus{}, + replicas: defaultReplicas, } for _, opt := range options { if err := opt(cfg); err != nil { @@ -157,6 +163,7 @@ func NewAPI(repo Repo, client StorachaClient, options ...Option) API { ReaderForIndex: blobsAPI.ReaderForIndex, BlobUploadParallelism: cfg.blobUploadParallelism, Bus: cfg.bus, + Replicas: cfg.replicas, } uploadsAPI = uploads.API{ @@ -228,6 +235,18 @@ func WithBlobUploadParallelism(blobUploadParallelism int) Option { } } +// WithReplicas sets the number of replicas to use when uploading blobs, +// including the original. The default is 3 replicas. +func WithReplicas(replicas uint) Option { + return func(cfg *config) error { + if replicas == 0 { + return errors.New("replica count must be greater than 0") + } + cfg.replicas = replicas + return nil + } +} + func (a API) FindOrCreateSpace(ctx context.Context, spaceDID did.DID, name string, options ...spacesmodel.SpaceOption) (*spacesmodel.Space, error) { return a.Spaces.FindOrCreateSpace(ctx, spaceDID, name, options...) } diff --git a/pkg/preparation/storacha/storacha.go b/pkg/preparation/storacha/storacha.go index 1c1d5313..3fedf426 100644 --- a/pkg/preparation/storacha/storacha.go +++ b/pkg/preparation/storacha/storacha.go @@ -64,6 +64,7 @@ type API struct { ReaderForIndex ReaderForIndexFunc BlobUploadParallelism int Bus bus.Publisher + Replicas uint } var _ uploads.AddShardsForUploadFunc = API{}.AddShardsForUpload @@ -342,6 +343,12 @@ func (a API) spaceBlobReplicate(ctx context.Context, blob model.Blob, spaceDID d ctx, span := tracer.Start(ctx, "space-blob-replicate") defer span.End() + // if the replication count is 1 (or less) then there is nothing to do + replicas := a.Replicas + if replicas <= 1 { + return nil + } + _, _, err := a.Client.SpaceBlobReplicate( ctx, spaceDID, @@ -349,7 +356,7 @@ func (a API) spaceBlobReplicate(ctx context.Context, blob model.Blob, spaceDID d Digest: blob.Digest(), Size: blob.Size(), }, - 3, + replicas, locationCommitment, ) return err diff --git a/pkg/preparation/storacha/storacha_test.go b/pkg/preparation/storacha/storacha_test.go index 9efcc63a..ee5bb431 100644 --- a/pkg/preparation/storacha/storacha_test.go +++ b/pkg/preparation/storacha/storacha_test.go @@ -57,6 +57,7 @@ func TestAddShardsForUpload(t *testing.T) { Client: &client, ReaderForShard: carForShard, BlobUploadParallelism: 1, + Replicas: 3, } blobsApi := blobs.API{ @@ -185,6 +186,7 @@ func TestAddShardsForUpload(t *testing.T) { Client: &client, ReaderForShard: carForShard, BlobUploadParallelism: 1, + Replicas: 3, } blobsApi := blobs.API{ @@ -276,6 +278,7 @@ func TestAddShardsForUpload(t *testing.T) { Client: &client, ReaderForShard: carForShard, BlobUploadParallelism: 1, + Replicas: 3, } blobsApi := blobs.API{ @@ -328,6 +331,7 @@ func TestAddIndexesForUpload(t *testing.T) { Client: &client, ReaderForIndex: carForIndex, BlobUploadParallelism: 1, + Replicas: 3, } blobsApi := blobs.API{ @@ -459,6 +463,7 @@ func TestAddStorachaUploadForUpload(t *testing.T) { Repo: repo, Client: &mclient, BlobUploadParallelism: 1, + Replicas: 3, } upload, _ := testutil.CreateUpload(t, repo, spaceDID, spacesmodel.WithShardSize(1<<16)) diff --git a/pkg/presets/presets.go b/pkg/presets/presets.go index 2ff23e93..8abcc951 100644 --- a/pkg/presets/presets.go +++ b/pkg/presets/presets.go @@ -42,6 +42,9 @@ type NetworkConfig struct { UploadID did.DID UploadURL url.URL AuthorizedRetrievals bool // Support for UCAN authorized retrievals. + // InsecureDIDResolution enables HTTP (instead of HTTPS) for did:web resolution. + // NB: this should only be used for development purposes. + InsecureDIDResolution bool } // Known network configurations. @@ -84,7 +87,10 @@ var Networks = []NetworkConfig{ }, } -var DefaultNetwork = Networks[0] +var ( + DefaultNetwork = Networks[0] + DefaultReplicas = uint(3) +) // GetNetworkConfig returns the network config for the passed name or the // STORACHA_NETWORK environment variable if set. If both are empty, the default