Configurable http client timeout for Azure Blob Storage

Author: jochenehret

azurebs/README.md

@@ -15,10 +15,14 @@ The Azure client requires a JSON configuration file with the following structure
   "account_name":           "<string> (required)",
   "account_key":            "<string> (required)",
   "container_name":         "<string> (required)",
-  "environment":            "<string> (optional, default: 'AzureCloud')"
+  "environment":            "<string> (optional, default: 'AzureCloud')",
+  "put_timeout_in_seconds": "<string> (optional, e.g. '30', default: no timeout)",
+  "http_request_timeout":   "<string> (optional, Go duration e.g. '30s', default: no timeout)"
 }
 ```
 
+`put_timeout_in_seconds` sets a context-level timeout for upload operations. `http_request_timeout` sets a per-request HTTP client timeout that applies to all operations (upload, download, delete, list, etc.).
+
 **Usage examples:**
 ``` bash
 # Upload a blob
@@ -66,7 +70,7 @@ go test $(go list ./azurebs/... | grep -v integration)
   1. Export the following variables into your environment.
 
       ```bash
-      export ACCOUNT_NAME=<your Azure accounnt name>
+      export ACCOUNT_NAME=<your Azure account name>
       export ACCOUNT_KEY=<your Azure account key>
       export CONTAINER_NAME=<the target container name>
       ```

azurebs/client/storage_client.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
+	"net/http"
 	"os"
 	"strconv"
 	"strings"
@@ -104,9 +105,45 @@ func createContext(dsc DefaultStorageClient) (context.Context, context.CancelFun
 }
 
 type DefaultStorageClient struct {
-	credential    *azblob.SharedKeyCredential
-	serviceURL    string
-	storageConfig config.AZStorageConfig
+	credential         *azblob.SharedKeyCredential
+	serviceURL         string
+	storageConfig      config.AZStorageConfig
+	httpRequestTimeout time.Duration
+}
+
+// clientOptions returns azblob.ClientOptions with a timeout-configured http.Client,
+// or nil when no http_request_timeout is set (use SDK defaults).
+func (dsc DefaultStorageClient) blockblobClientOptions() *blockblob.ClientOptions {
+	if dsc.httpRequestTimeout == 0 {
+		return nil
+	}
+	return &blockblob.ClientOptions{
+		ClientOptions: azcore.ClientOptions{
+			Transport: &http.Client{Timeout: dsc.httpRequestTimeout},
+		},
+	}
+}
+
+func (dsc DefaultStorageClient) blobClientOptions() *azBlob.ClientOptions {
+	if dsc.httpRequestTimeout == 0 {
+		return nil
+	}
+	return &azBlob.ClientOptions{
+		ClientOptions: azcore.ClientOptions{
+			Transport: &http.Client{Timeout: dsc.httpRequestTimeout},
+		},
+	}
+}
+
+func (dsc DefaultStorageClient) containerClientOptions() *azContainer.ClientOptions {
+	if dsc.httpRequestTimeout == 0 {
+		return nil
+	}
+	return &azContainer.ClientOptions{
+		ClientOptions: azcore.ClientOptions{
+			Transport: &http.Client{Timeout: dsc.httpRequestTimeout},
+		},
+	}
 }
 
 func NewStorageClient(storageConfig config.AZStorageConfig) (StorageClient, error) {
@@ -115,9 +152,19 @@ func NewStorageClient(storageConfig config.AZStorageConfig) (StorageClient, erro
 		return nil, err
 	}
 
+	httpRequestTimeout, err := storageConfig.HTTPRequestTimeoutValue()
+	if err != nil {
+		return nil, err
+	}
+
 	serviceURL := fmt.Sprintf("https://%s.%s/%s", storageConfig.AccountName, storageConfig.StorageEndpoint(), storageConfig.ContainerName)
 
-	return DefaultStorageClient{credential: credential, serviceURL: serviceURL, storageConfig: storageConfig}, nil
+	return DefaultStorageClient{
+		credential:         credential,
+		serviceURL:         serviceURL,
+		storageConfig:      storageConfig,
+		httpRequestTimeout: httpRequestTimeout,
+	}, nil
 }
 
 func (dsc DefaultStorageClient) Upload(
@@ -138,7 +185,7 @@ func (dsc DefaultStorageClient) Upload(
 	}
 	defer cancel()
 
-	client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil)
+	client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions())
 	if err != nil {
 		return nil, err
 	}
@@ -173,7 +220,7 @@ func (dsc DefaultStorageClient) UploadStream(
 	}
 	defer cancel()
 
-	client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil)
+	client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions())
 	if err != nil {
 		return err
 	}
@@ -196,7 +243,7 @@ func (dsc DefaultStorageClient) Download(
 ) error {
 	blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, source)
 	slog.Info("Downloading blob from container", "container", dsc.storageConfig.ContainerName, "blob", source, "local_file", dest.Name())
-	client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil)
+	client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions())
 	if err != nil {
 		return err
 	}
@@ -226,7 +273,7 @@ func (dsc DefaultStorageClient) Copy(
 	srcURL := fmt.Sprintf("%s/%s", dsc.serviceURL, srcBlob)
 	destURL := fmt.Sprintf("%s/%s", dsc.serviceURL, destBlob)
 
-	destClient, err := blockblob.NewClientWithSharedKeyCredential(destURL, dsc.credential, nil)
+	destClient, err := blockblob.NewClientWithSharedKeyCredential(destURL, dsc.credential, dsc.blockblobClientOptions())
 	if err != nil {
 		return fmt.Errorf("failed to create destination client: %w", err)
 	}
@@ -268,7 +315,7 @@ func (dsc DefaultStorageClient) Delete(
 	blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, dest)
 
 	slog.Info("Deleting blob from container", "container", dsc.storageConfig.ContainerName, "blob", dest, "url", blobURL)
-	client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil)
+	client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions())
 	if err != nil {
 		return err
 	}
@@ -295,7 +342,7 @@ func (dsc DefaultStorageClient) DeleteRecursive(
 		slog.Info("Deleting all blobs in container", "container", dsc.storageConfig.ContainerName)
 	}
 
-	containerClient, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, nil)
+	containerClient, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, dsc.containerClientOptions())
 	if err != nil {
 		return fmt.Errorf("failed to create container client: %w", err)
 	}
@@ -315,7 +362,7 @@ func (dsc DefaultStorageClient) DeleteRecursive(
 
 		for _, blob := range resp.Segment.BlobItems {
 			blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, *blob.Name)
-			blobClient, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil)
+			blobClient, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions())
 			if err != nil {
 				slog.Error("Failed to create blob client", "blob", *blob.Name, "error", err)
 				continue
@@ -338,7 +385,7 @@ func (dsc DefaultStorageClient) Exists(
 	blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, dest)
 
 	slog.Info("Checking if blob exists", "container", dsc.storageConfig.ContainerName, "blob", dest, "url", blobURL)
-	client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil)
+	client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions())
 	if err != nil {
 		return false, err
 	}
@@ -365,7 +412,7 @@ func (dsc DefaultStorageClient) SignedUrl(
 	blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, dest)
 
 	slog.Info("Generating SAS URL for blob", "container", dsc.storageConfig.ContainerName, "blob", dest, "request_type", requestType, "expiration", expiration)
-	client, err := azBlob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil)
+	client, err := azBlob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blobClientOptions())
 	if err != nil {
 		return "", err
 	}
@@ -398,7 +445,7 @@ func (dsc DefaultStorageClient) List(
 		slog.Info("Listing blobs in container", "container", dsc.storageConfig.ContainerName)
 	}
 
-	client, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, nil)
+	client, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, dsc.containerClientOptions())
 	if err != nil {
 		return nil, fmt.Errorf("failed to create container client: %w", err)
 	}
@@ -437,7 +484,7 @@ func (dsc DefaultStorageClient) Properties(
 	blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, dest)
 
 	slog.Info("Getting properties for blob", "container", dsc.storageConfig.ContainerName, "blob", dest, "url", blobURL)
-	client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil)
+	client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, dsc.blockblobClientOptions())
 	if err != nil {
 		return err
 	}
@@ -469,7 +516,7 @@ func (dsc DefaultStorageClient) Properties(
 func (dsc DefaultStorageClient) EnsureContainerExists() error {
 	slog.Info("Ensuring container exists", "container", dsc.storageConfig.ContainerName)
 
-	containerClient, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, nil)
+	containerClient, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, dsc.containerClientOptions())
 	if err != nil {
 		return fmt.Errorf("failed to create container client: %w", err)
 	}

azurebs/config/config.go

@@ -3,7 +3,9 @@ package config
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
+	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
 )
@@ -27,11 +29,34 @@ func init() {
 }
 
 type AZStorageConfig struct {
-	AccountName   string `json:"account_name"`
-	AccountKey    string `json:"account_key"`
-	ContainerName string `json:"container_name"`
-	Environment   string `json:"environment"`
-	Timeout       string `json:"put_timeout_in_seconds"`
+	AccountName        string `json:"account_name"`
+	AccountKey         string `json:"account_key"`
+	ContainerName      string `json:"container_name"`
+	Environment        string `json:"environment"`
+	Timeout            string `json:"put_timeout_in_seconds"`
+	HTTPRequestTimeout string `json:"http_request_timeout"`
+}
+
+// ErrNonPositiveHTTPRequestTimeout is returned when http_request_timeout is <= 0.
+var ErrNonPositiveHTTPRequestTimeout = errors.New("http_request_timeout must be greater than 0")
+
+// HTTPRequestTimeoutValue parses HTTPRequestTimeout as a Go duration string.
+// Returns 0 (no timeout) if the field is empty.
+func (c *AZStorageConfig) HTTPRequestTimeoutValue() (time.Duration, error) {
+	if c.HTTPRequestTimeout == "" {
+		return 0, nil
+	}
+
+	d, err := time.ParseDuration(c.HTTPRequestTimeout)
+	if err != nil {
+		return 0, fmt.Errorf("invalid http_request_timeout: %w", err)
+	}
+
+	if d <= 0 {
+		return 0, ErrNonPositiveHTTPRequestTimeout
+	}
+
+	return d, nil
 }
 
 // NewFromReader returns a new azure-storage-cli configuration struct from the contents of reader.
@@ -53,6 +78,10 @@ func NewFromReader(reader io.Reader) (AZStorageConfig, error) {
 		return AZStorageConfig{}, err
 	}
 
+	if _, err = config.HTTPRequestTimeoutValue(); err != nil {
+		return AZStorageConfig{}, err
+	}
+
 	return config, nil
 }
 

azurebs/config/config_test.go

@@ -3,6 +3,7 @@ package config_test
 import (
 	"bytes"
 	"errors"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -87,6 +88,46 @@ var _ = Describe("Config", func() {
 			})
 		})
 	})
+	Context("http_request_timeout", func() {
+		When("not set", func() {
+			It("returns 0 duration", func() {
+				configJson := []byte(`{"account_name": "a", "account_key": "b", "container_name": "c"}`)
+				cfg, err := config.NewFromReader(bytes.NewReader(configJson))
+				Expect(err).ToNot(HaveOccurred())
+				d, err := cfg.HTTPRequestTimeoutValue()
+				Expect(err).ToNot(HaveOccurred())
+				Expect(d).To(Equal(time.Duration(0)))
+			})
+		})
+
+		When("set to a valid duration", func() {
+			It("returns the parsed duration", func() {
+				configJson := []byte(`{"account_name": "a", "account_key": "b", "container_name": "c", "http_request_timeout": "30s"}`)
+				cfg, err := config.NewFromReader(bytes.NewReader(configJson))
+				Expect(err).ToNot(HaveOccurred())
+				d, err := cfg.HTTPRequestTimeoutValue()
+				Expect(err).ToNot(HaveOccurred())
+				Expect(d).To(Equal(30 * time.Second))
+			})
+		})
+
+		When("set to an invalid duration string", func() {
+			It("returns an error", func() {
+				configJson := []byte(`{"account_name": "a", "account_key": "b", "container_name": "c", "http_request_timeout": "not-a-duration"}`)
+				_, err := config.NewFromReader(bytes.NewReader(configJson))
+				Expect(err).To(HaveOccurred())
+				Expect(err.Error()).To(ContainSubstring("invalid http_request_timeout"))
+			})
+		})
+
+		When("set to a non-positive duration", func() {
+			It("returns an error", func() {
+				configJson := []byte(`{"account_name": "a", "account_key": "b", "container_name": "c", "http_request_timeout": "-5s"}`)
+				_, err := config.NewFromReader(bytes.NewReader(configJson))
+				Expect(err).To(MatchError(config.ErrNonPositiveHTTPRequestTimeout))
+			})
+		})
+	})
 })
 
 type explodingReader struct{}