Allow setting custom upload limits

api-append-object.go

@@ -88,7 +88,7 @@ func (opts *AppendObjectOptions) setChecksumParams(info ObjectInfo) {
 }
 
 func (opts AppendObjectOptions) validate(c *Client) (err error) {
-	if opts.ChunkSize > maxPartSize {
+	if opts.ChunkSize > uint64(c.limits.maxPartSize()) {
 		return errInvalidArgument("Append chunkSize cannot be larger than max part size allowed")
 	}
 	switch {
@@ -212,7 +212,7 @@ func (c *Client) AppendObject(ctx context.Context, bucketName, objectName string
 		if objectSize > 0 {
 			finalObjSize = info.Size + objectSize
 		}
-		totalPartsCount, partSize, lastPartSize, err := OptimalPartInfo(finalObjSize, opts.ChunkSize)
+		totalPartsCount, partSize, lastPartSize, err := c.optimalPartInfo(finalObjSize, opts.ChunkSize)
 		if err != nil {
 			return UploadInfo{}, err
 		}

api-compose-object.go

@@ -83,7 +83,8 @@ type CopyDestOptions struct {
 	// Progress of the entire copy operation will be sent here.
 	Progress io.Reader
 	// PartSize specifies the part size for multipart copy operations.
-	// If not specified, defaults to maxPartSize (5 GiB).
+	// If not specified, defaults to the client's max part size (5 GiB unless
+	// overridden via Options.UploadLimits).
 	PartSize uint64
 
 	// AnnotationDirective controls whether the source object's annotations are
@@ -435,8 +436,21 @@ func (c *Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID st
 // operations. Optionally takes progress reader hook for applications to
 // look at current progress.
 func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ...CopySrcOptions) (UploadInfo, error) {
-	if len(srcs) < 1 || len(srcs) > maxPartsCount {
-		return UploadInfo{}, errInvalidArgument("There must be as least one and up to 10000 source objects.")
+	maxPartsCount := c.limits.maxPartsCount()
+	maxPartSize := c.limits.maxPartSize()
+	maxObjectSize := c.limits.maxObjectSize()
+
+	if len(srcs) < 1 || int64(len(srcs)) > maxPartsCount {
+		return UploadInfo{}, errInvalidArgument(fmt.Sprintf("There must be as least one and up to %d source objects.", maxPartsCount))
+	}
+
+	if dst.PartSize > uint64(maxPartSize) {
+		return UploadInfo{}, errInvalidArgument(fmt.Sprintf(
+			"CopyDestOptions.PartSize %d is larger than the maximum part size of %d", dst.PartSize, maxPartSize))
+	}
+	partSize := int64(dst.PartSize)
+	if partSize == 0 {
+		partSize = maxPartSize
 	}
 
 	for _, src := range srcs {
@@ -475,23 +489,32 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ..
 			srcCopySize = src.End - src.Start + 1
 		}
 
-		// Only the last source may be less than `absMinPartSize`
-		if srcCopySize < absMinPartSize && i < len(srcs)-1 {
+		// Only the last source may be less than the minimum part size
+		if srcCopySize < c.limits.minPartSize() && i < len(srcs)-1 {
 			return UploadInfo{}, errInvalidArgument(
 				fmt.Sprintf("CopySrcOptions %d is too small (%d) and it is not the last part", i, srcCopySize))
 		}
 
 		// Is data to copy too large?
 		totalSize += srcCopySize
 		if totalSize > maxObjectSize {
-			return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> 5GiB * 10000)", totalSize))
+			return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> %d * %d)", totalSize, maxPartSize, maxPartsCount))
 		}
 
 		// record source size
 		srcObjectSizes[i] = srcCopySize
 
 		// calculate parts needed for current source
-		totalParts += partsRequired(srcCopySize, int64(dst.PartSize))
+		reqParts := partsRequired(srcCopySize, partSize)
+		// A part size close to the minimum can make calculateEvenSplits emit
+		// ranges below it, which the remote rejects. Only the very last range
+		// of the last source is allowed to be undersized.
+		if under := undersizedSplits(srcCopySize, reqParts, c.limits.minPartSize()); under > 1 || (under == 1 && i < len(srcs)-1) {
+			return UploadInfo{}, errInvalidArgument(fmt.Sprintf(
+				"CopySrcOptions %d (%d bytes) splits into %d ranges below the minimum part size of %d at a part size of %d",
+				i, srcCopySize, under, c.limits.minPartSize(), partSize))
+		}
+		totalParts += reqParts
 		// Do we need more parts than we are allowed?
 		if totalParts > maxPartsCount {
 			return UploadInfo{}, errInvalidArgument(fmt.Sprintf(
@@ -570,7 +593,7 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ..
 
 		// calculate start/end indices of parts after
 		// splitting.
-		startIdx, endIdx := calculateEvenSplits(srcObjectSizes[i], src, int64(dst.PartSize))
+		startIdx, endIdx := calculateEvenSplits(srcObjectSizes[i], src, partSize)
 		for j, start := range startIdx {
 			end := endIdx[j]
 
@@ -605,10 +628,11 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ..
 }
 
 // partsRequired calculates the number of parts needed for a given size
-// using the specified part size. If partSize is 0, defaults to maxPartSize (5 GiB).
+// using the specified part size. If partSize is 0, defaults to the default
+// max part size (5 GiB); callers with a Client resolve it against its limits.
 func partsRequired(size int64, partSize int64) int64 {
 	if partSize == 0 {
-		partSize = maxPartSize
+		partSize = defaultMaxPartSize
 	}
 	r := size / partSize
 	if size%partSize > 0 {
@@ -617,10 +641,28 @@ func partsRequired(size int64, partSize int64) int64 {
 	return r
 }
 
+// undersizedSplits reports how many of the reqParts ranges calculateEvenSplits
+// generates for size fall below minPartSize. It emits the rem larger ranges
+// first, so the undersized ones are always at the tail.
+func undersizedSplits(size, reqParts, minPartSize int64) int64 {
+	if reqParts <= 0 {
+		// An empty source generates no ranges at all.
+		return 0
+	}
+	quot, rem := size/reqParts, size%reqParts
+	if quot+1 < minPartSize {
+		return reqParts
+	}
+	if quot < minPartSize {
+		return reqParts - rem
+	}
+	return 0
+}
+
 // calculateEvenSplits - computes splits for a source and returns
 // start and end index slices. Splits happen evenly to be sure that no
 // part is less than 5MiB, as that could fail the multipart request if
-// it is not the last part. If partSize is 0, defaults to maxPartSize (5 GiB).
+// it is not the last part. If partSize is 0, partsRequired's default applies.
 func calculateEvenSplits(size int64, src CopySrcOptions, partSize int64) (startIndex, endIndex []int64) {
 	if size == 0 {
 		return startIndex, endIndex

api-compose-object_test.go

@@ -32,7 +32,7 @@ const (
 
 	// oldPartSize is the legacy part size calculation for backward compatibility testing
 	// It was: maxMultipartPutObjectSize / (maxPartsCount - 1)
-	oldPartSize = maxMultipartPutObjectSize / (maxPartsCount - 1)
+	oldPartSize = maxMultipartPutObjectSize / (defaultMaxPartsCount - 1)
 )
 
 func TestPartsRequired(t *testing.T) {
@@ -43,13 +43,13 @@ func TestPartsRequired(t *testing.T) {
 	}{
 		{0, 0, 0},
 		{1, 0, 1},
-		{gb5, 0, 1},               // 5 GiB / 5 GiB = 1 part
-		{gb5p1, 0, 2},             // 5 GiB + 1 byte needs 2 parts
-		{2 * gb5, 0, 2},           // 10 GiB / 5 GiB = 2 parts
-		{gb10p1, 0, 3},            // 10 GiB + 1 byte needs 3 parts
-		{gb10p2, 0, 3},            // 10 GiB + 2 bytes needs 3 parts
-		{gb10p1 + gb10p2, 0, 5},   // 20 GiB + 3 bytes needs 5 parts
-		{maxPartSize * 10, 0, 10}, // exactly 10 parts
+		{gb5, 0, 1},                      // 5 GiB / 5 GiB = 1 part
+		{gb5p1, 0, 2},                    // 5 GiB + 1 byte needs 2 parts
+		{2 * gb5, 0, 2},                  // 10 GiB / 5 GiB = 2 parts
+		{gb10p1, 0, 3},                   // 10 GiB + 1 byte needs 3 parts
+		{gb10p2, 0, 3},                   // 10 GiB + 2 bytes needs 3 parts
+		{gb10p1 + gb10p2, 0, 5},          // 20 GiB + 3 bytes needs 5 parts
+		{defaultMaxPartSize * 10, 0, 10}, // exactly 10 parts
 		// Custom part sizes
 		{gb5, gb1, 5},      // 5 GiB / 1 GiB = 5 parts
 		{gb5p1, gb1, 6},    // 5 GiB + 1 byte / 1 GiB = 6 parts

api-error-response.go

@@ -249,6 +249,33 @@ func errEntityTooLarge(totalSize, maxObjectSize int64, bucketName, objectName st
 	}
 }
 
+// errPartTooLarge - Input part size is larger than the maximum allowed part
+// size, which is configurable via Options.UploadLimits.
+func errPartTooLarge(partSize, maxPartSize int64, bucketName, objectName string) error {
+	msg := fmt.Sprintf("Your proposed part size ‘%d’ exceeds the maximum allowed part size ‘%d’.", partSize, maxPartSize)
+	return ErrorResponse{
+		StatusCode: http.StatusBadRequest,
+		Code:       EntityTooLarge,
+		Message:    msg,
+		BucketName: bucketName,
+		Key:        objectName,
+	}
+}
+
+// errUploadTooLarge - An unknown length reader outlasted the parts the upload
+// was laid out for. The object's real size is not known, so only the number of
+// bytes that fit is reported; it is not an allowed maximum.
+func errUploadTooLarge(uploadedSize, totalPartsCount int64, bucketName, objectName string) error {
+	msg := fmt.Sprintf("Input stream exceeds the ‘%d’ parts this upload allows; ‘%d’ bytes were uploaded before the limit was reached. Set PutObjectOptions.PartSize to upload a larger object.", totalPartsCount, uploadedSize)
+	return ErrorResponse{
+		StatusCode: http.StatusBadRequest,
+		Code:       EntityTooLarge,
+		Message:    msg,
+		BucketName: bucketName,
+		Key:        objectName,
+	}
+}
+
 // errEntityTooSmall - Input size is smaller than supported minimum.
 func errEntityTooSmall(totalSize int64, bucketName, objectName string) error {
 	msg := fmt.Sprintf("Your proposed upload size ‘%d’ is below the minimum allowed object size ‘0B’ for single PUT operation.", totalSize)

api-put-object-common.go

@@ -19,10 +19,12 @@ package minio
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"math"
 	"os"
 
+	"github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 )
 
@@ -60,22 +62,39 @@ func isReadAt(reader io.Reader) (ok bool) {
 }
 
 // OptimalPartInfo - calculate the optimal part info for a given
-// object size.
+// object size, using Amazon S3's upload limits.
 //
 // NOTE: Assumption here is that for any object to be uploaded to any S3 compatible
 // object storage it will have the following parameters as constants.
 //
 //	maxPartsCount - 10000
 //	minPartSize - 16MiB
 //	maxObjectSize - ~48.83TiB (maxPartSize * maxPartsCount)
+//
+// A Client created with Options.UploadLimits uses its own limits instead of
+// these, so its part layout may differ from what this function returns.
 func OptimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize, lastPartSize int64, err error) {
-	// When object size is unknown (-1), default to 5TiB to limit memory usage.
-	// This results in ~537MiB part sizes. For larger objects (up to ~48.83TiB),
-	// callers should set configuredPartSize explicitly to control memory usage.
+	return UploadLimits{}.optimalPartInfo(objectSize, configuredPartSize)
+}
+
+func (c *Client) optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize, lastPartSize int64, err error) {
+	return c.limits.optimalPartInfo(objectSize, configuredPartSize)
+}
+
+// optimalPartInfo - calculate the optimal part info for a given object size
+// within these limits.
+func (l UploadLimits) optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize, lastPartSize int64, err error) {
+	maxPartsCount := l.maxPartsCount()
+	maxObjectSize := l.maxObjectSize()
+
+	// When object size is unknown (-1), default to 5TiB (or the maximum object
+	// size, when lower) to limit memory usage. This results in ~537MiB part
+	// sizes. For larger objects (up to the maximum object size), callers should
+	// set configuredPartSize explicitly to control memory usage.
 	var unknownSize bool
 	if objectSize == -1 {
 		unknownSize = true
-		objectSize = maxMultipartPutObjectSize
+		objectSize = min(maxMultipartPutObjectSize, maxObjectSize)
 	}
 
 	// object size is larger than supported maximum.
@@ -84,42 +103,69 @@ func OptimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCou
 		return totalPartsCount, partSize, lastPartSize, err
 	}
 
+	// An empty object has no parts; the minimum part size below would otherwise
+	// report a part size and a last part size for a layout with no parts in it.
+	if objectSize == 0 && configuredPartSize == 0 {
+		return 0, 0, 0, nil
+	}
+
 	var partSizeFlt float64
 	if configuredPartSize > 0 {
+		// Compared unsigned and up front, so the int64 conversions below cannot
+		// wrap a caller-supplied part size into a negative that slips past them.
+		if configuredPartSize > uint64(l.maxPartSize()) {
+			err = errInvalidArgument(fmt.Sprintf("Input part size is bigger than allowed maximum of %s.", humanize.IBytes(uint64(l.maxPartSize()))))
+			return totalPartsCount, partSize, lastPartSize, err
+		}
+
 		if int64(configuredPartSize) > objectSize {
 			err = errEntityTooLarge(int64(configuredPartSize), objectSize, "", "")
 			return totalPartsCount, partSize, lastPartSize, err
 		}
 
 		if !unknownSize {
 			if objectSize > (int64(configuredPartSize) * maxPartsCount) {
-				err = errInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.")
+				err = errInvalidArgument(fmt.Sprintf("Part size * max_parts(%d) is lesser than input objectSize.", maxPartsCount))
 				return totalPartsCount, partSize, lastPartSize, err
 			}
 		}
 
-		if configuredPartSize < absMinPartSize {
-			err = errInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.")
-			return totalPartsCount, partSize, lastPartSize, err
-		}
-
-		if configuredPartSize > maxPartSize {
-			err = errInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.")
+		if int64(configuredPartSize) < l.minPartSize() {
+			err = errInvalidArgument(fmt.Sprintf("Input part size is smaller than allowed minimum of %s.", humanize.IBytes(uint64(l.minPartSize()))))
 			return totalPartsCount, partSize, lastPartSize, err
 		}
 
 		partSizeFlt = float64(configuredPartSize)
 		if unknownSize {
 			// If input has unknown size and part size is configured
-			// keep it to maximum allowed as per 10000 parts.
+			// keep it to maximum allowed as per the max parts count.
 			objectSize = int64(configuredPartSize) * maxPartsCount
 		}
 	} else {
-		configuredPartSize = minPartSize
+		// Round to a multiple of the internal threshold, but never below a
+		// MinPartSize that was raised above it, or the generated non-final
+		// parts would be rejected by the remote.
+		configuredPartSize = uint64(max(minPartSize, l.minPartSize()))
+		// Round the exact ceiling of objectSize/maxPartsCount, not the truncated
+		// quotient: a truncated quotient that already sits on a
+		// configuredPartSize multiple stays put and needs maxPartsCount+1 parts.
+		smallestPartSize := objectSize / maxPartsCount
+		if objectSize%maxPartsCount != 0 {
+			smallestPartSize++
+		}
 		// Use floats for part size for all calculations to avoid
 		// overflows during float64 to int64 conversions.
-		partSizeFlt = float64(objectSize / maxPartsCount)
-		partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize)
+		partSizeFlt = math.Ceil(float64(smallestPartSize)/float64(configuredPartSize)) * float64(configuredPartSize)
+		// An object smaller than maxPartsCount rounds down to a zero part size,
+		// and a non-final part must never fall below MinPartSize either.
+		if minPS := float64(l.minPartSize()); partSizeFlt < minPS {
+			partSizeFlt = minPS
+		}
+		// Rounding up to a minPartSize multiple can overshoot a MaxPartSize
+		// that was lowered below, or is not a multiple of, minPartSize.
+		if maxPS := float64(l.maxPartSize()); partSizeFlt > maxPS {
+			partSizeFlt = maxPS
+		}
 	}
 
 	// Total parts count.
@@ -131,6 +177,30 @@ func OptimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCou
 	return totalPartsCount, partSize, lastPartSize, nil
 }
 
+// errIfMoreData reports errUploadTooLarge when reader still holds data after
+// the last part allowed by the upload limits was consumed. Unknown length
+// uploads would otherwise silently complete a truncated object.
+//
+// This deliberately fails an upload whose bytes were all transferred: the probe
+// runs after the final part, so a reader that reports anything other than a
+// clean io.EOF — a closed file, a reset connection, a wrapper with its own
+// sentinel — aborts the multipart upload instead of completing it. Silently
+// storing a possibly truncated object is the worse outcome, but callers should
+// expect this error to arrive late and to look like a transport fault.
+func errIfMoreData(reader io.Reader, uploadedSize, totalPartsCount int64, bucketName, objectName string) error {
+	var b [1]byte
+	n, err := readFull(reader, b[:])
+	if n > 0 {
+		return errUploadTooLarge(uploadedSize, totalPartsCount, bucketName, objectName)
+	}
+	// Only a clean EOF proves the reader was drained; anything else has to
+	// surface rather than complete a possibly truncated object.
+	if err != nil && err != io.EOF {
+		return err
+	}
+	return nil
+}
+
 // getUploadID - fetch upload id if already present for an object name
 // or initiate a new request to fetch a new upload id.
 func (c *Client) newUploadID(ctx context.Context, bucketName, objectName string, opts PutObjectOptions) (uploadID string, err error) {