diff --git a/api-append-object.go b/api-append-object.go index 5b34756ee..f40287e50 100644 --- a/api-append-object.go +++ b/api-append-object.go @@ -88,7 +88,7 @@ func (opts *AppendObjectOptions) setChecksumParams(info ObjectInfo) { } func (opts AppendObjectOptions) validate(c *Client) (err error) { - if opts.ChunkSize > maxPartSize { + if opts.ChunkSize > uint64(c.limits.maxPartSize()) { return errInvalidArgument("Append chunkSize cannot be larger than max part size allowed") } switch { @@ -212,7 +212,7 @@ func (c *Client) AppendObject(ctx context.Context, bucketName, objectName string if objectSize > 0 { finalObjSize = info.Size + objectSize } - totalPartsCount, partSize, lastPartSize, err := OptimalPartInfo(finalObjSize, opts.ChunkSize) + totalPartsCount, partSize, lastPartSize, err := c.optimalPartInfo(finalObjSize, opts.ChunkSize) if err != nil { return UploadInfo{}, err } diff --git a/api-compose-object.go b/api-compose-object.go index 2ea833b73..7a9376a65 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -83,7 +83,8 @@ type CopyDestOptions struct { // Progress of the entire copy operation will be sent here. Progress io.Reader // PartSize specifies the part size for multipart copy operations. - // If not specified, defaults to maxPartSize (5 GiB). + // If not specified, defaults to the client's max part size (5 GiB unless + // overridden via Options.UploadLimits). PartSize uint64 // AnnotationDirective controls whether the source object's annotations are @@ -435,8 +436,21 @@ func (c *Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID st // operations. Optionally takes progress reader hook for applications to // look at current progress. func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ...CopySrcOptions) (UploadInfo, error) { - if len(srcs) < 1 || len(srcs) > maxPartsCount { - return UploadInfo{}, errInvalidArgument("There must be as least one and up to 10000 source objects.") + maxPartsCount := c.limits.maxPartsCount() + maxPartSize := c.limits.maxPartSize() + maxObjectSize := c.limits.maxObjectSize() + + if len(srcs) < 1 || int64(len(srcs)) > maxPartsCount { + return UploadInfo{}, errInvalidArgument(fmt.Sprintf("There must be as least one and up to %d source objects.", maxPartsCount)) + } + + if dst.PartSize > uint64(maxPartSize) { + return UploadInfo{}, errInvalidArgument(fmt.Sprintf( + "CopyDestOptions.PartSize %d is larger than the maximum part size of %d", dst.PartSize, maxPartSize)) + } + partSize := int64(dst.PartSize) + if partSize == 0 { + partSize = maxPartSize } for _, src := range srcs { @@ -475,8 +489,8 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs .. srcCopySize = src.End - src.Start + 1 } - // Only the last source may be less than `absMinPartSize` - if srcCopySize < absMinPartSize && i < len(srcs)-1 { + // Only the last source may be less than the minimum part size + if srcCopySize < c.limits.minPartSize() && i < len(srcs)-1 { return UploadInfo{}, errInvalidArgument( fmt.Sprintf("CopySrcOptions %d is too small (%d) and it is not the last part", i, srcCopySize)) } @@ -484,14 +498,23 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs .. // Is data to copy too large? totalSize += srcCopySize if totalSize > maxObjectSize { - return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> 5GiB * 10000)", totalSize)) + return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> %d * %d)", totalSize, maxPartSize, maxPartsCount)) } // record source size srcObjectSizes[i] = srcCopySize // calculate parts needed for current source - totalParts += partsRequired(srcCopySize, int64(dst.PartSize)) + reqParts := partsRequired(srcCopySize, partSize) + // A part size close to the minimum can make calculateEvenSplits emit + // ranges below it, which the remote rejects. Only the very last range + // of the last source is allowed to be undersized. + if under := undersizedSplits(srcCopySize, reqParts, c.limits.minPartSize()); under > 1 || (under == 1 && i < len(srcs)-1) { + return UploadInfo{}, errInvalidArgument(fmt.Sprintf( + "CopySrcOptions %d (%d bytes) splits into %d ranges below the minimum part size of %d at a part size of %d", + i, srcCopySize, under, c.limits.minPartSize(), partSize)) + } + totalParts += reqParts // Do we need more parts than we are allowed? if totalParts > maxPartsCount { return UploadInfo{}, errInvalidArgument(fmt.Sprintf( @@ -570,7 +593,7 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs .. // calculate start/end indices of parts after // splitting. - startIdx, endIdx := calculateEvenSplits(srcObjectSizes[i], src, int64(dst.PartSize)) + startIdx, endIdx := calculateEvenSplits(srcObjectSizes[i], src, partSize) for j, start := range startIdx { end := endIdx[j] @@ -605,10 +628,11 @@ func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs .. } // partsRequired calculates the number of parts needed for a given size -// using the specified part size. If partSize is 0, defaults to maxPartSize (5 GiB). +// using the specified part size. If partSize is 0, defaults to the default +// max part size (5 GiB); callers with a Client resolve it against its limits. func partsRequired(size int64, partSize int64) int64 { if partSize == 0 { - partSize = maxPartSize + partSize = defaultMaxPartSize } r := size / partSize if size%partSize > 0 { @@ -617,10 +641,28 @@ func partsRequired(size int64, partSize int64) int64 { return r } +// undersizedSplits reports how many of the reqParts ranges calculateEvenSplits +// generates for size fall below minPartSize. It emits the rem larger ranges +// first, so the undersized ones are always at the tail. +func undersizedSplits(size, reqParts, minPartSize int64) int64 { + if reqParts <= 0 { + // An empty source generates no ranges at all. + return 0 + } + quot, rem := size/reqParts, size%reqParts + if quot+1 < minPartSize { + return reqParts + } + if quot < minPartSize { + return reqParts - rem + } + return 0 +} + // calculateEvenSplits - computes splits for a source and returns // start and end index slices. Splits happen evenly to be sure that no // part is less than 5MiB, as that could fail the multipart request if -// it is not the last part. If partSize is 0, defaults to maxPartSize (5 GiB). +// it is not the last part. If partSize is 0, partsRequired's default applies. func calculateEvenSplits(size int64, src CopySrcOptions, partSize int64) (startIndex, endIndex []int64) { if size == 0 { return startIndex, endIndex diff --git a/api-compose-object_test.go b/api-compose-object_test.go index 16dbc087b..5556c6b04 100644 --- a/api-compose-object_test.go +++ b/api-compose-object_test.go @@ -32,7 +32,7 @@ const ( // oldPartSize is the legacy part size calculation for backward compatibility testing // It was: maxMultipartPutObjectSize / (maxPartsCount - 1) - oldPartSize = maxMultipartPutObjectSize / (maxPartsCount - 1) + oldPartSize = maxMultipartPutObjectSize / (defaultMaxPartsCount - 1) ) func TestPartsRequired(t *testing.T) { @@ -43,13 +43,13 @@ func TestPartsRequired(t *testing.T) { }{ {0, 0, 0}, {1, 0, 1}, - {gb5, 0, 1}, // 5 GiB / 5 GiB = 1 part - {gb5p1, 0, 2}, // 5 GiB + 1 byte needs 2 parts - {2 * gb5, 0, 2}, // 10 GiB / 5 GiB = 2 parts - {gb10p1, 0, 3}, // 10 GiB + 1 byte needs 3 parts - {gb10p2, 0, 3}, // 10 GiB + 2 bytes needs 3 parts - {gb10p1 + gb10p2, 0, 5}, // 20 GiB + 3 bytes needs 5 parts - {maxPartSize * 10, 0, 10}, // exactly 10 parts + {gb5, 0, 1}, // 5 GiB / 5 GiB = 1 part + {gb5p1, 0, 2}, // 5 GiB + 1 byte needs 2 parts + {2 * gb5, 0, 2}, // 10 GiB / 5 GiB = 2 parts + {gb10p1, 0, 3}, // 10 GiB + 1 byte needs 3 parts + {gb10p2, 0, 3}, // 10 GiB + 2 bytes needs 3 parts + {gb10p1 + gb10p2, 0, 5}, // 20 GiB + 3 bytes needs 5 parts + {defaultMaxPartSize * 10, 0, 10}, // exactly 10 parts // Custom part sizes {gb5, gb1, 5}, // 5 GiB / 1 GiB = 5 parts {gb5p1, gb1, 6}, // 5 GiB + 1 byte / 1 GiB = 6 parts diff --git a/api-error-response.go b/api-error-response.go index 03c7e9435..1a0089f21 100644 --- a/api-error-response.go +++ b/api-error-response.go @@ -249,6 +249,33 @@ func errEntityTooLarge(totalSize, maxObjectSize int64, bucketName, objectName st } } +// errPartTooLarge - Input part size is larger than the maximum allowed part +// size, which is configurable via Options.UploadLimits. +func errPartTooLarge(partSize, maxPartSize int64, bucketName, objectName string) error { + msg := fmt.Sprintf("Your proposed part size ‘%d’ exceeds the maximum allowed part size ‘%d’.", partSize, maxPartSize) + return ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: EntityTooLarge, + Message: msg, + BucketName: bucketName, + Key: objectName, + } +} + +// errUploadTooLarge - An unknown length reader outlasted the parts the upload +// was laid out for. The object's real size is not known, so only the number of +// bytes that fit is reported; it is not an allowed maximum. +func errUploadTooLarge(uploadedSize, totalPartsCount int64, bucketName, objectName string) error { + msg := fmt.Sprintf("Input stream exceeds the ‘%d’ parts this upload allows; ‘%d’ bytes were uploaded before the limit was reached. Set PutObjectOptions.PartSize to upload a larger object.", totalPartsCount, uploadedSize) + return ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: EntityTooLarge, + Message: msg, + BucketName: bucketName, + Key: objectName, + } +} + // errEntityTooSmall - Input size is smaller than supported minimum. func errEntityTooSmall(totalSize int64, bucketName, objectName string) error { msg := fmt.Sprintf("Your proposed upload size ‘%d’ is below the minimum allowed object size ‘0B’ for single PUT operation.", totalSize) diff --git a/api-put-object-common.go b/api-put-object-common.go index a33e6ba55..df7670fac 100644 --- a/api-put-object-common.go +++ b/api-put-object-common.go @@ -19,10 +19,12 @@ package minio import ( "context" + "fmt" "io" "math" "os" + "github.com/dustin/go-humanize" "github.com/minio/minio-go/v7/pkg/s3utils" ) @@ -60,7 +62,7 @@ func isReadAt(reader io.Reader) (ok bool) { } // OptimalPartInfo - calculate the optimal part info for a given -// object size. +// object size, using Amazon S3's upload limits. // // NOTE: Assumption here is that for any object to be uploaded to any S3 compatible // object storage it will have the following parameters as constants. @@ -68,14 +70,31 @@ func isReadAt(reader io.Reader) (ok bool) { // maxPartsCount - 10000 // minPartSize - 16MiB // maxObjectSize - ~48.83TiB (maxPartSize * maxPartsCount) +// +// A Client created with Options.UploadLimits uses its own limits instead of +// these, so its part layout may differ from what this function returns. func OptimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize, lastPartSize int64, err error) { - // When object size is unknown (-1), default to 5TiB to limit memory usage. - // This results in ~537MiB part sizes. For larger objects (up to ~48.83TiB), - // callers should set configuredPartSize explicitly to control memory usage. + return UploadLimits{}.optimalPartInfo(objectSize, configuredPartSize) +} + +func (c *Client) optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize, lastPartSize int64, err error) { + return c.limits.optimalPartInfo(objectSize, configuredPartSize) +} + +// optimalPartInfo - calculate the optimal part info for a given object size +// within these limits. +func (l UploadLimits) optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize, lastPartSize int64, err error) { + maxPartsCount := l.maxPartsCount() + maxObjectSize := l.maxObjectSize() + + // When object size is unknown (-1), default to 5TiB (or the maximum object + // size, when lower) to limit memory usage. This results in ~537MiB part + // sizes. For larger objects (up to the maximum object size), callers should + // set configuredPartSize explicitly to control memory usage. var unknownSize bool if objectSize == -1 { unknownSize = true - objectSize = maxMultipartPutObjectSize + objectSize = min(maxMultipartPutObjectSize, maxObjectSize) } // object size is larger than supported maximum. @@ -84,8 +103,21 @@ func OptimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCou return totalPartsCount, partSize, lastPartSize, err } + // An empty object has no parts; the minimum part size below would otherwise + // report a part size and a last part size for a layout with no parts in it. + if objectSize == 0 && configuredPartSize == 0 { + return 0, 0, 0, nil + } + var partSizeFlt float64 if configuredPartSize > 0 { + // Compared unsigned and up front, so the int64 conversions below cannot + // wrap a caller-supplied part size into a negative that slips past them. + if configuredPartSize > uint64(l.maxPartSize()) { + err = errInvalidArgument(fmt.Sprintf("Input part size is bigger than allowed maximum of %s.", humanize.IBytes(uint64(l.maxPartSize())))) + return totalPartsCount, partSize, lastPartSize, err + } + if int64(configuredPartSize) > objectSize { err = errEntityTooLarge(int64(configuredPartSize), objectSize, "", "") return totalPartsCount, partSize, lastPartSize, err @@ -93,33 +125,47 @@ func OptimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCou if !unknownSize { if objectSize > (int64(configuredPartSize) * maxPartsCount) { - err = errInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") + err = errInvalidArgument(fmt.Sprintf("Part size * max_parts(%d) is lesser than input objectSize.", maxPartsCount)) return totalPartsCount, partSize, lastPartSize, err } } - if configuredPartSize < absMinPartSize { - err = errInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.") - return totalPartsCount, partSize, lastPartSize, err - } - - if configuredPartSize > maxPartSize { - err = errInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.") + if int64(configuredPartSize) < l.minPartSize() { + err = errInvalidArgument(fmt.Sprintf("Input part size is smaller than allowed minimum of %s.", humanize.IBytes(uint64(l.minPartSize())))) return totalPartsCount, partSize, lastPartSize, err } partSizeFlt = float64(configuredPartSize) if unknownSize { // If input has unknown size and part size is configured - // keep it to maximum allowed as per 10000 parts. + // keep it to maximum allowed as per the max parts count. objectSize = int64(configuredPartSize) * maxPartsCount } } else { - configuredPartSize = minPartSize + // Round to a multiple of the internal threshold, but never below a + // MinPartSize that was raised above it, or the generated non-final + // parts would be rejected by the remote. + configuredPartSize = uint64(max(minPartSize, l.minPartSize())) + // Round the exact ceiling of objectSize/maxPartsCount, not the truncated + // quotient: a truncated quotient that already sits on a + // configuredPartSize multiple stays put and needs maxPartsCount+1 parts. + smallestPartSize := objectSize / maxPartsCount + if objectSize%maxPartsCount != 0 { + smallestPartSize++ + } // Use floats for part size for all calculations to avoid // overflows during float64 to int64 conversions. - partSizeFlt = float64(objectSize / maxPartsCount) - partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize) + partSizeFlt = math.Ceil(float64(smallestPartSize)/float64(configuredPartSize)) * float64(configuredPartSize) + // An object smaller than maxPartsCount rounds down to a zero part size, + // and a non-final part must never fall below MinPartSize either. + if minPS := float64(l.minPartSize()); partSizeFlt < minPS { + partSizeFlt = minPS + } + // Rounding up to a minPartSize multiple can overshoot a MaxPartSize + // that was lowered below, or is not a multiple of, minPartSize. + if maxPS := float64(l.maxPartSize()); partSizeFlt > maxPS { + partSizeFlt = maxPS + } } // Total parts count. @@ -131,6 +177,30 @@ func OptimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCou return totalPartsCount, partSize, lastPartSize, nil } +// errIfMoreData reports errUploadTooLarge when reader still holds data after +// the last part allowed by the upload limits was consumed. Unknown length +// uploads would otherwise silently complete a truncated object. +// +// This deliberately fails an upload whose bytes were all transferred: the probe +// runs after the final part, so a reader that reports anything other than a +// clean io.EOF — a closed file, a reset connection, a wrapper with its own +// sentinel — aborts the multipart upload instead of completing it. Silently +// storing a possibly truncated object is the worse outcome, but callers should +// expect this error to arrive late and to look like a transport fault. +func errIfMoreData(reader io.Reader, uploadedSize, totalPartsCount int64, bucketName, objectName string) error { + var b [1]byte + n, err := readFull(reader, b[:]) + if n > 0 { + return errUploadTooLarge(uploadedSize, totalPartsCount, bucketName, objectName) + } + // Only a clean EOF proves the reader was drained; anything else has to + // surface rather than complete a possibly truncated object. + if err != nil && err != io.EOF { + return err + } + return nil +} + // getUploadID - fetch upload id if already present for an object name // or initiate a new request to fetch a new upload id. func (c *Client) newUploadID(ctx context.Context, bucketName, objectName string, opts PutObjectOptions) (uploadID string, err error) { diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index 330c78fd1..66acd02b3 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -45,8 +45,8 @@ func (c *Client) putObjectMultipart(ctx context.Context, bucketName, objectName // Verify if multipart functionality is not available, if not // fall back to single PutObject operation. if errResp.Code == AccessDenied && strings.Contains(errResp.Message, "Access Denied") { - // Verify if size of reader is greater than '5GiB'. - if size > maxSinglePutObjectSize { + // Verify if size of reader is greater than the single PUT limit. + if maxSinglePutObjectSize := c.limits.maxSinglePutObjectSize(); size > maxSinglePutObjectSize { return UploadInfo{}, errEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. @@ -73,7 +73,7 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj var complMultipartUpload completeMultipartUpload // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, _, err := OptimalPartInfo(-1, opts.PartSize) + totalPartsCount, partSize, _, err := c.optimalPartInfo(-1, opts.PartSize) if err != nil { return UploadInfo{}, err } @@ -108,8 +108,10 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj // CRC32C is ~50% faster on AMD64 @ 30GB/s customHeader := make(http.Header) crc := opts.AutoChecksum.Hasher() + var lastErr error for partNumber <= totalPartsCount { length, rErr := readFull(reader, buf) + lastErr = rErr if rErr == io.EOF && partNumber > 1 { break } @@ -175,6 +177,14 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj } } + // A nil read error on the last allowed part means the reader was never + // drained; completing here would store a truncated object. + if lastErr == nil { + if err = errIfMoreData(reader, totalUploadedSize, int64(totalPartsCount), bucketName, objectName); err != nil { + return UploadInfo{}, err + } + } + // Loop over total uploaded parts to save them in // Parts array before completing the multipart request. allParts := make([]ObjectPart, 0, len(partsInfo)) @@ -291,8 +301,8 @@ func (c *Client) uploadPart(ctx context.Context, p uploadPartParams) (ObjectPart if err := s3utils.CheckValidObjectName(p.objectName); err != nil { return ObjectPart{}, err } - if p.size > maxPartSize { - return ObjectPart{}, errEntityTooLarge(p.size, maxPartSize, p.bucketName, p.objectName) + if maxPartSize := c.limits.maxPartSize(); p.size > maxPartSize { + return ObjectPart{}, errPartTooLarge(p.size, maxPartSize, p.bucketName, p.objectName) } if p.size <= -1 { return ObjectPart{}, errEntityTooSmall(p.size, p.bucketName, p.objectName) diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 2eca51c71..79e064e5c 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -57,8 +57,8 @@ func (c *Client) putObjectMultipartStream(ctx context.Context, bucketName, objec // Verify if multipart functionality is not available, if not // fall back to single PutObject operation. if errResp.Code == AccessDenied && strings.Contains(errResp.Message, "Access Denied") { - // Verify if size of reader is greater than '5GiB'. - if size > maxSinglePutObjectSize { + // Verify if size of reader is greater than the single PUT limit. + if maxSinglePutObjectSize := c.limits.maxSinglePutObjectSize(); size > maxSinglePutObjectSize { return UploadInfo{}, errEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. @@ -104,7 +104,7 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN } // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, lastPartSize, err := OptimalPartInfo(size, opts.PartSize) + totalPartsCount, partSize, lastPartSize, err := c.optimalPartInfo(size, opts.PartSize) if err != nil { return UploadInfo{}, err } @@ -302,7 +302,7 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b } // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, lastPartSize, err := OptimalPartInfo(size, opts.PartSize) + totalPartsCount, partSize, lastPartSize, err := c.optimalPartInfo(size, opts.PartSize) if err != nil { return UploadInfo{}, err } @@ -461,7 +461,7 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam defer cancel() // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, _, err := OptimalPartInfo(-1, opts.PartSize) + totalPartsCount, partSize, _, err := c.optimalPartInfo(-1, opts.PartSize) if err != nil { return UploadInfo{}, err } @@ -504,6 +504,7 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam // Part number always starts with '1'. var partNumber int + var lastErr error for partNumber = 1; partNumber <= totalPartsCount; partNumber++ { // Proceed to upload the part. var buf []byte @@ -520,6 +521,7 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam } length, rerr := readFull(reader, buf) + lastErr = rerr if rerr == io.EOF && partNumber > 1 { // Done break @@ -598,6 +600,14 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam default: } + // A nil read error on the last allowed part means the reader was never + // drained; completing here would store a truncated object. + if lastErr == nil { + if err = errIfMoreData(reader, totalUploadedSize, int64(totalPartsCount), bucketName, objectName); err != nil { + return UploadInfo{}, err + } + } + // Complete multipart upload. var complMultipartUpload completeMultipartUpload diff --git a/api-put-object.go b/api-put-object.go index 8fc8313a9..9c8ec2c4b 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -322,6 +322,9 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part // For larger objects (up to ~48.83TiB), set PutObjectOptions.PartSize // to control memory usage and enable uploads beyond 5TiB. // +// The ~48.83TiB ceiling is the product of the client's max part size and +// max parts count, both of which follow Options.UploadLimits. +// // WARNING: Passing down '-1' will use memory and these cannot // be reused for best outcomes for PutObject(), pass the size always. // @@ -350,7 +353,7 @@ func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, r } // Check for largest object size allowed. - if size > int64(maxObjectSize) { + if maxObjectSize := c.limits.maxObjectSize(); size > maxObjectSize { return UploadInfo{}, errEntityTooLarge(size, maxObjectSize, bucketName, objectName) } @@ -374,6 +377,19 @@ func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, r partSize = minPartSize } + // Only an explicitly configured single PUT limit is enforced here. The 5GiB + // default is Amazon's; MinIO/AIStor and others accept far larger single + // PUTs, and PutObjectsSnowball sets DisableMultipart itself, so applying the + // default would refuse uploads that work today. + if maxSinglePut := c.limits.MaxSinglePutObjectSize; maxSinglePut > 0 && size > maxSinglePut { + if opts.DisableMultipart { + return UploadInfo{}, errEntityTooLarge(size, maxSinglePut, bucketName, objectName) + } + if int64(partSize) > maxSinglePut { + partSize = uint64(maxSinglePut) + } + } + if c.overrideSignerType.IsV2() { if size >= 0 && size < int64(partSize) || opts.DisableMultipart { return c.putObject(ctx, bucketName, objectName, reader, size, opts) @@ -415,7 +431,7 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam var complMultipartUpload completeMultipartUpload // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, _, err := OptimalPartInfo(-1, opts.PartSize) + totalPartsCount, partSize, _, err := c.optimalPartInfo(-1, opts.PartSize) if err != nil { return UploadInfo{}, err } @@ -446,8 +462,10 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam customHeader := make(http.Header) crc := opts.AutoChecksum.Hasher() + var lastErr error for partNumber <= totalPartsCount { length, rerr := readFull(reader, buf) + lastErr = rerr if rerr == io.EOF && partNumber > 1 { break } @@ -503,6 +521,14 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam } } + // A nil read error on the last allowed part means the reader was never + // drained; completing here would store a truncated object. + if lastErr == nil { + if err = errIfMoreData(reader, totalUploadedSize, int64(totalPartsCount), bucketName, objectName); err != nil { + return UploadInfo{}, err + } + } + // Loop over total uploaded parts to save them in // Parts array before completing the multipart request. allParts := make([]ObjectPart, 0, len(partsInfo)) diff --git a/api.go b/api.go index 5db074fa9..aa7a7da83 100644 --- a/api.go +++ b/api.go @@ -110,6 +110,10 @@ type Client struct { trailingHeaderSupport bool maxRetries int + // Upload limits enforced before sending a request. Always read through + // its accessors, which resolve zero fields to the S3 defaults. + limits UploadLimits + // RDMA dispatch state. rdmaEnabled mirrors Options.EnableRDMA; // the rest are only touched by rdma.go (built with -tags=rdma) but // have to live on the struct so the stub and the tagged build share @@ -170,6 +174,11 @@ type Options struct { // when the caller supplies PutObjectOptions.RDMABuffer / GetObjectOptions.RDMABuffer. // No-op unless built with -tags=rdma. EnableRDMA bool + + // UploadLimits overrides the upload limits the client enforces before + // sending a request. Leave nil, or leave individual fields zero, to use + // Amazon S3's limits. + UploadLimits *UploadLimits } // Global constants. @@ -340,6 +349,13 @@ func privateNew(endpoint string, opts *Options) (*Client, error) { clnt.maxRetries = opts.MaxRetries } + if opts.UploadLimits != nil { + if err := opts.UploadLimits.validate(); err != nil { + return nil, err + } + clnt.limits = *opts.UploadLimits + } + // Return. return clnt, nil } diff --git a/constants.go b/constants.go index d49efdee5..a56d4a097 100644 --- a/constants.go +++ b/constants.go @@ -18,34 +18,33 @@ package minio // Multipart upload defaults. +// +// The default* values below are Amazon S3's limits. They can be overridden +// per client with Options.UploadLimits; see UploadLimits. -// absMinPartSize - absolute minimum part size (5 MiB) below which +// defaultMinPartSize - absolute minimum part size (5 MiB) below which // a part in a multipart upload may not be uploaded. -const absMinPartSize = 1024 * 1024 * 5 +const defaultMinPartSize = 1024 * 1024 * 5 // minPartSize - minimum part size 16MiB per object after which // putObject behaves internally as multipart. const minPartSize = 1024 * 1024 * 16 -// maxPartsCount - maximum number of parts for a single multipart session. -const maxPartsCount = 10000 +// defaultMaxPartsCount - maximum number of parts for a single multipart session. +const defaultMaxPartsCount = 10000 -// maxPartSize - maximum part size 5GiB for a single multipart upload +// defaultMaxPartSize - maximum part size 5GiB for a single multipart upload // operation. -const maxPartSize = 1024 * 1024 * 1024 * 5 +const defaultMaxPartSize = 1024 * 1024 * 1024 * 5 -// maxSinglePutObjectSize - maximum size 5GiB of object per PUT +// defaultMaxSinglePutObjectSize - maximum size 5GiB of object per PUT // operation. -const maxSinglePutObjectSize = 1024 * 1024 * 1024 * 5 +const defaultMaxSinglePutObjectSize = 1024 * 1024 * 1024 * 5 // maxMultipartPutObjectSize - maximum size 5TiB of object for // Multipart operation. const maxMultipartPutObjectSize = 1024 * 1024 * 1024 * 1024 * 5 -// maxObjectSize - maximum size of an object calculated from -// maxPartSize * maxPartsCount = 5GiB * 10000 = ~48.83TiB -const maxObjectSize = maxPartSize * maxPartsCount - // unsignedPayload - value to be set to X-Amz-Content-Sha256 header when // we don't want to sign the request payload const unsignedPayload = "UNSIGNED-PAYLOAD" diff --git a/upload-limits.go b/upload-limits.go new file mode 100644 index 000000000..a69793391 --- /dev/null +++ b/upload-limits.go @@ -0,0 +1,145 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2025 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import "math" + +// UploadLimits overrides the upload limits the client enforces before sending +// a request. The defaults are the limits Amazon S3 imposes; only raise them +// when the remote endpoint is known to accept the larger values. +// +// A zero field means "use the default", so the zero UploadLimits behaves +// exactly like Amazon S3 — except for MaxSinglePutObjectSize, whose default is +// deliberately not enforced by PutObject. See that field. +// +// New rejects limits it cannot derive a part layout from. No field may be +// negative; the remaining bounds are noted on each field. +type UploadLimits struct { + // MinPartSize is the smallest size allowed for a part that is not the + // last part of a multipart upload. Defaults to 5 MiB. May not exceed + // MaxPartSize. + MinPartSize int64 + + // MaxPartSize is the largest size allowed for a single part. + // Defaults to 5 GiB. MaxPartSize * MaxPartsCount must fit in an int64. + MaxPartSize int64 + + // MaxPartsCount is the maximum number of parts in a single multipart + // upload. Together with MaxPartSize this caps the object size the client + // is willing to upload. Defaults to 10000, and may not exceed 2^53 + // because the part layout is computed in float64. + MaxPartsCount int64 + + // MaxSinglePutObjectSize is the largest object the remote accepts in a + // single PUT. Defaults to 5 GiB. + // + // Unlike the other fields, the 5 GiB default is not enforced by PutObject: + // MinIO and AIStor accept single PUTs far above Amazon's limit, so gating on + // the default would refuse uploads that work today. Setting it explicitly + // does enforce it — PutObject then rejects a larger object outright when + // PutObjectOptions.DisableMultipart is set, and otherwise sends it as a + // multipart upload. + // + // The resolved value, default included, always bounds the single PUT that + // PutObject falls back to when a multipart upload fails with AccessDenied. + // Core.PutObject never checks it. + MaxSinglePutObjectSize int64 +} + +// UploadLimits returns the upload limits this client enforces, with any +// zero field resolved to its Amazon S3 default. +func (c *Client) UploadLimits() UploadLimits { + return UploadLimits{ + MinPartSize: c.limits.minPartSize(), + MaxPartSize: c.limits.maxPartSize(), + MaxPartsCount: c.limits.maxPartsCount(), + MaxSinglePutObjectSize: c.limits.maxSinglePutObjectSize(), + } +} + +// Accessors resolve zero fields to their defaults, so a Client that was not +// built by New still sees the S3 limits. + +func (l UploadLimits) minPartSize() int64 { + if l.MinPartSize > 0 { + return l.MinPartSize + } + return defaultMinPartSize +} + +func (l UploadLimits) maxPartSize() int64 { + if l.MaxPartSize > 0 { + return l.MaxPartSize + } + return defaultMaxPartSize +} + +func (l UploadLimits) maxPartsCount() int64 { + if l.MaxPartsCount > 0 { + return l.MaxPartsCount + } + return defaultMaxPartsCount +} + +func (l UploadLimits) maxSinglePutObjectSize() int64 { + if l.MaxSinglePutObjectSize > 0 { + return l.MaxSinglePutObjectSize + } + return defaultMaxSinglePutObjectSize +} + +// maxObjectSize is the largest object that can be uploaded as a multipart +// upload, ~48.83TiB with the default limits. +func (l UploadLimits) maxObjectSize() int64 { + return l.maxPartSize() * l.maxPartsCount() +} + +func (l UploadLimits) validate() error { + for _, f := range []struct { + name string + value int64 + }{ + {"MinPartSize", l.MinPartSize}, + {"MaxPartSize", l.MaxPartSize}, + {"MaxPartsCount", l.MaxPartsCount}, + {"MaxSinglePutObjectSize", l.MaxSinglePutObjectSize}, + } { + if f.value < 0 { + return errInvalidArgument("UploadLimits." + f.name + " cannot be negative") + } + } + if l.minPartSize() > l.maxPartSize() { + return errInvalidArgument("UploadLimits.MinPartSize cannot be larger than UploadLimits.MaxPartSize") + } + if l.maxPartSize() > math.MaxInt64/l.maxPartsCount() { + return errInvalidArgument("UploadLimits.MaxPartSize multiplied by UploadLimits.MaxPartsCount overflows int64") + } + // The part layout is computed in float64. A MaxPartSize that rounds to or + // above 2^63 does not convert back into int64, and callers allocate buffers + // of the part size the layout reports. + if float64(l.maxPartSize()) >= math.MaxInt64 { + return errInvalidArgument("UploadLimits.MaxPartSize is too large to compute a part layout") + } + // The parts count is bounded by MaxPartsCount and returned as an int, so + // the same rounding has to survive the trip back. Beyond 2^53 float64 no + // longer holds every integer, and at the int64 ceiling it does not convert. + if l.maxPartsCount() > 1<<53 { + return errInvalidArgument("UploadLimits.MaxPartsCount is too large to compute a part layout") + } + return nil +} diff --git a/upload-limits_test.go b/upload-limits_test.go new file mode 100644 index 000000000..2119c6dc7 --- /dev/null +++ b/upload-limits_test.go @@ -0,0 +1,703 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2025 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "errors" + "io" + "math" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// A zero UploadLimits must reproduce Amazon S3's limits. +func TestUploadLimitsDefaults(t *testing.T) { + var l UploadLimits + for _, tc := range []struct { + name string + got int64 + want int64 + }{ + {"minPartSize", l.minPartSize(), 5 * 1024 * 1024}, + {"maxPartSize", l.maxPartSize(), 5 * 1024 * 1024 * 1024}, + {"maxPartsCount", l.maxPartsCount(), 10000}, + {"maxSinglePutObjectSize", l.maxSinglePutObjectSize(), 5 * 1024 * 1024 * 1024}, + {"maxObjectSize", l.maxObjectSize(), 5 * 1024 * 1024 * 1024 * 10000}, + } { + if tc.got != tc.want { + t.Errorf("%s: got %d, want %d", tc.name, tc.got, tc.want) + } + } +} + +// Setting one field must not disturb the others. +func TestUploadLimitsPartialOverride(t *testing.T) { + l := UploadLimits{MaxPartsCount: 100000} + if got := l.maxPartsCount(); got != 100000 { + t.Errorf("maxPartsCount: got %d, want 100000", got) + } + if got := l.maxPartSize(); got != defaultMaxPartSize { + t.Errorf("maxPartSize: got %d, want %d", got, int64(defaultMaxPartSize)) + } + if got := l.minPartSize(); got != defaultMinPartSize { + t.Errorf("minPartSize: got %d, want %d", got, int64(defaultMinPartSize)) + } + if got, want := l.maxObjectSize(), int64(defaultMaxPartSize)*100000; got != want { + t.Errorf("maxObjectSize: got %d, want %d", got, want) + } +} + +// A Client built outside privateNew (as several tests in this package do) must +// keep behaving like the defaults rather than dividing by a zero limit. +func TestUploadLimitsZeroValueClient(t *testing.T) { + c := &Client{} + for _, size := range []int64{-1, 1, 5243928576, defaultMaxPartSize * 10} { + wantParts, wantPart, wantLast, wantErr := OptimalPartInfo(size, 0) + gotParts, gotPart, gotLast, gotErr := c.optimalPartInfo(size, 0) + if gotParts != wantParts || gotPart != wantPart || gotLast != wantLast || (gotErr == nil) != (wantErr == nil) { + t.Errorf("size %d: got (%d, %d, %d, %v), want (%d, %d, %d, %v)", + size, gotParts, gotPart, gotLast, gotErr, wantParts, wantPart, wantLast, wantErr) + } + } + if got := c.limits.maxObjectSize(); got != int64(defaultMaxPartSize)*defaultMaxPartsCount { + t.Errorf("maxObjectSize: got %d, want %d", got, int64(defaultMaxPartSize)*defaultMaxPartsCount) + } +} + +func TestUploadLimitsValidate(t *testing.T) { + testCases := []struct { + name string + limits UploadLimits + wantErr bool + }{ + {"zero value", UploadLimits{}, false}, + {"raised parts count only", UploadLimits{MaxPartsCount: 100000}, false}, + {"raised part size only", UploadLimits{MaxPartSize: 64 * 1024 * 1024 * 1024}, false}, + {"all raised", UploadLimits{ + MinPartSize: 1024, + MaxPartSize: 64 * 1024 * 1024 * 1024, + MaxPartsCount: 100000, + MaxSinglePutObjectSize: 64 * 1024 * 1024 * 1024, + }, false}, + {"negative MinPartSize", UploadLimits{MinPartSize: -1}, true}, + {"negative MaxPartSize", UploadLimits{MaxPartSize: -1}, true}, + {"negative MaxPartsCount", UploadLimits{MaxPartsCount: -1}, true}, + {"negative MaxSinglePutObjectSize", UploadLimits{MaxSinglePutObjectSize: -1}, true}, + // MaxPartSize below the default 5MiB floor. + {"max part size under default min", UploadLimits{MaxPartSize: 1024}, true}, + {"min above max", UploadLimits{MinPartSize: 1024 * 1024 * 1024, MaxPartSize: 1024 * 1024}, true}, + // maxObjectSize() would wrap negative. + {"max object size overflows", UploadLimits{MaxPartSize: math.MaxInt64 / 2, MaxPartsCount: 3}, true}, + {"max object size at the int64 ceiling", UploadLimits{MaxPartSize: math.MaxInt64 / 10000, MaxPartsCount: 10000}, false}, + // totalPartsCount would not survive the float64 round trip. + {"parts count at the int64 ceiling", UploadLimits{MinPartSize: 1, MaxPartSize: 1, MaxPartsCount: math.MaxInt64}, true}, + {"parts count above the float64 exact range", UploadLimits{MinPartSize: 1, MaxPartSize: 1, MaxPartsCount: 1<<53 + 1}, true}, + {"parts count at the float64 exact range", UploadLimits{MinPartSize: 1, MaxPartSize: 1, MaxPartsCount: 1 << 53}, false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if err := tc.limits.validate(); (err != nil) != tc.wantErr { + t.Fatalf("validate() error = %v, wantErr %v", err, tc.wantErr) + } + + limits := tc.limits + c, err := New("play.min.io", &Options{ + Creds: credentials.NewStaticV4("id", "secret", ""), + UploadLimits: &limits, + }) + if (err != nil) != tc.wantErr { + t.Fatalf("New() error = %v, wantErr %v", err, tc.wantErr) + } + if err == nil && c.limits != tc.limits { + t.Fatalf("client limits = %+v, want %+v", c.limits, tc.limits) + } + }) + } +} + +// nil Options.UploadLimits leaves the client on the defaults. +func TestUploadLimitsUnset(t *testing.T) { + c, err := New("play.min.io", &Options{Creds: credentials.NewStaticV4("id", "secret", "")}) + if err != nil { + t.Fatal(err) + } + if (c.limits != UploadLimits{}) { + t.Fatalf("client limits = %+v, want zero value", c.limits) + } +} + +// Raising MaxPartsCount must allow layouts with more than 10000 parts. +func TestOptimalPartInfoRaisedPartsCount(t *testing.T) { + const size = 20 * 1024 * 1024 * 1024 * 1024 // 20TiB + const partSize = 128 * 1024 * 1024 // 128MiB -> 163840 parts + + if _, _, _, err := OptimalPartInfo(size, partSize); err == nil { + t.Fatal("default limits should reject a layout needing more than 10000 parts") + } + + l := UploadLimits{MaxPartsCount: 1000000} + totalParts, gotPartSize, lastPartSize, err := l.optimalPartInfo(size, partSize) + if err != nil { + t.Fatal(err) + } + if totalParts != size/partSize { + t.Errorf("totalParts: got %d, want %d", totalParts, size/partSize) + } + if gotPartSize != partSize { + t.Errorf("partSize: got %d, want %d", gotPartSize, int64(partSize)) + } + if lastPartSize != partSize { + t.Errorf("lastPartSize: got %d, want %d", lastPartSize, int64(partSize)) + } +} + +// Raising MaxPartSize must allow a configured part size above 5GiB. +func TestOptimalPartInfoRaisedPartSize(t *testing.T) { + const partSize = 10 * 1024 * 1024 * 1024 // 10GiB + const size = partSize * 4 + + if _, _, _, err := OptimalPartInfo(size, partSize); err == nil { + t.Fatal("default limits should reject a part size above 5GiB") + } + + l := UploadLimits{MaxPartSize: 64 * 1024 * 1024 * 1024} + totalParts, gotPartSize, _, err := l.optimalPartInfo(size, partSize) + if err != nil { + t.Fatal(err) + } + if totalParts != 4 || gotPartSize != partSize { + t.Errorf("got (%d parts, %d part size), want (4, %d)", totalParts, gotPartSize, int64(partSize)) + } +} + +// Lowering limits must tighten what the client accepts. +func TestOptimalPartInfoLoweredLimits(t *testing.T) { + l := UploadLimits{MaxPartsCount: 20} + + // 20 parts of 5GiB is all this allows. + if got, want := l.maxObjectSize(), int64(defaultMaxPartSize)*20; got != want { + t.Fatalf("maxObjectSize: got %d, want %d", got, want) + } + if _, _, _, err := l.optimalPartInfo(l.maxObjectSize()+1, 0); err == nil { + t.Error("expected an error for an object above the lowered max object size") + } + // A part size that would need more than 20 parts. + if _, _, _, err := l.optimalPartInfo(21*minPartSize, minPartSize); err == nil { + t.Error("expected an error for a layout needing more than 20 parts") + } + + // A lowered MaxPartSize must cap the part size chosen for us, even though + // rounding up to a minPartSize multiple would overshoot it. + small := UploadLimits{MaxPartSize: 20 * 1024 * 1024} + _, partSize, _, err := small.optimalPartInfo(small.maxObjectSize(), 0) + if err != nil { + t.Fatal(err) + } + if partSize > small.maxPartSize() { + t.Errorf("partSize %d exceeds MaxPartSize %d", partSize, small.maxPartSize()) + } +} + +// An unknown size must fall back to the resolved max object size when that is +// below the 5TiB memory cap, instead of failing outright. +func TestOptimalPartInfoUnknownSizeLoweredLimits(t *testing.T) { + l := UploadLimits{MaxPartsCount: 20} // 100GiB + totalParts, partSize, _, err := l.optimalPartInfo(-1, 0) + if err != nil { + t.Fatal(err) + } + if got, want := int64(totalParts)*partSize, l.maxObjectSize(); got != want { + t.Errorf("layout covers %d bytes, want %d", got, want) + } +} + +// The automatically chosen part size must never fall below MinPartSize, or the +// remote rejects every non-final part. +func TestOptimalPartInfoRaisedMinPartSize(t *testing.T) { + for _, tc := range []struct { + name string + limits UploadLimits + objectSize int64 + }{ + // 1GiB/10000 rounds to 16MiB on the internal threshold alone. + {"1GiB at 64MiB minimum", UploadLimits{MinPartSize: 64 * 1024 * 1024}, 1024 * 1024 * 1024}, + {"100 parts at 64MiB minimum", UploadLimits{MinPartSize: 64 * 1024 * 1024}, 100 * 64 * 1024 * 1024}, + // Below maxPartsCount the division rounds down to a zero part size. + {"object smaller than the parts count", UploadLimits{}, 100}, + } { + t.Run(tc.name, func(t *testing.T) { + totalParts, partSize, lastPartSize, err := tc.limits.optimalPartInfo(tc.objectSize, 0) + if err != nil { + t.Fatal(err) + } + if partSize < tc.limits.minPartSize() { + t.Errorf("partSize %d is below MinPartSize %d", partSize, tc.limits.minPartSize()) + } + if totalParts < 0 || int64(totalParts) > tc.limits.maxPartsCount() { + t.Errorf("totalPartsCount = %d, want within [0, %d]", totalParts, tc.limits.maxPartsCount()) + } + if lastPartSize > partSize { + t.Errorf("lastPartSize %d exceeds partSize %d", lastPartSize, partSize) + } + }) + } +} + +// An empty object has no parts, so the layout must be zero throughout rather +// than reporting a minimum-sized part and last part for a zero-part upload. +func TestOptimalPartInfoEmptyObject(t *testing.T) { + totalParts, partSize, lastPartSize, err := OptimalPartInfo(0, 0) + if err != nil { + t.Fatal(err) + } + if totalParts != 0 || partSize != 0 || lastPartSize != 0 { + t.Errorf("got (%d parts, %d part size, %d last part size), want all zero", + totalParts, partSize, lastPartSize) + } +} + +// A MaxPartSize at the int64 ceiling rounds through float64 to a value that +// converts back negative, which would panic the make() in the multipart paths. +func TestUploadLimitsRejectsUnrepresentableMaxPartSize(t *testing.T) { + l := UploadLimits{MaxPartSize: math.MaxInt64, MaxPartsCount: 1} + if err := l.validate(); err == nil { + t.Fatal("validate() accepted a MaxPartSize that cannot round-trip through float64") + } + if _, err := New("play.min.io", &Options{ + Creds: credentials.NewStaticV4("id", "secret", ""), + UploadLimits: &l, + }); err == nil { + t.Fatal("New() accepted a MaxPartSize that cannot round-trip through float64") + } + + // A part size one ulp below the boundary still round-trips, so the layout + // stays positive and make() is safe. + ok := UploadLimits{MaxPartSize: math.MaxInt64 - (1 << 11), MaxPartsCount: 1} + if err := ok.validate(); err != nil { + t.Fatalf("validate() rejected a representable MaxPartSize: %v", err) + } + _, partSize, _, err := ok.optimalPartInfo(ok.maxObjectSize(), 0) + if err != nil { + t.Fatal(err) + } + if partSize <= 0 { + t.Fatalf("partSize = %d, want positive (make would panic)", partSize) + } + + // The same for the parts count: an accepted MaxPartsCount must still yield a + // layout whose part count is a usable int. + parts := UploadLimits{MinPartSize: 1, MaxPartSize: 1, MaxPartsCount: 1 << 53} + if err := parts.validate(); err != nil { + t.Fatalf("validate() rejected a representable MaxPartsCount: %v", err) + } + totalParts, _, _, err := parts.optimalPartInfo(parts.maxObjectSize(), 0) + if err != nil { + t.Fatal(err) + } + if int64(totalParts) != parts.MaxPartsCount { + t.Fatalf("totalPartsCount = %d, want %d", totalParts, parts.MaxPartsCount) + } +} + +// The automatic layout must never need more than maxPartsCount parts. Rounding +// a truncated objectSize/maxPartsCount leaves the part size one byte short +// whenever that quotient already sits on a rounding-unit multiple. +func TestOptimalPartInfoPartsCountCeiling(t *testing.T) { + for _, tc := range []struct { + name string + limits UploadLimits + objectSize int64 + }{ + {"one byte over the 16MiB unit", UploadLimits{}, int64(defaultMaxPartsCount)*minPartSize + 1}, + {"half a unit over", UploadLimits{}, int64(defaultMaxPartsCount)*minPartSize + minPartSize/2}, + {"one under the next unit", UploadLimits{}, int64(defaultMaxPartsCount)*minPartSize*2 - 1}, + {"raised minimum", UploadLimits{MinPartSize: 64 * 1024 * 1024}, int64(defaultMaxPartsCount)*64*1024*1024 + 1}, + {"lowered parts count", UploadLimits{MaxPartsCount: 20}, 20*minPartSize + 1}, + } { + t.Run(tc.name, func(t *testing.T) { + totalParts, partSize, lastPartSize, err := tc.limits.optimalPartInfo(tc.objectSize, 0) + if err != nil { + t.Fatal(err) + } + if int64(totalParts) > tc.limits.maxPartsCount() { + t.Errorf("totalPartsCount = %d, exceeds MaxPartsCount %d (partSize %d)", + totalParts, tc.limits.maxPartsCount(), partSize) + } + if got := int64(totalParts-1)*partSize + lastPartSize; got != tc.objectSize { + t.Errorf("layout covers %d bytes, want %d", got, tc.objectSize) + } + }) + } +} + +// The resolved limits must be readable off a built client without a round trip. +func TestClientUploadLimitsAccessor(t *testing.T) { + // The shape AIStor configures for replication: the two size ceilings raised + // to 5TiB, MinPartSize and MaxPartsCount left at the S3 defaults. + const fiveTiB = int64(5) * 1024 * 1024 * 1024 * 1024 + limits := UploadLimits{MaxPartSize: fiveTiB, MaxSinglePutObjectSize: fiveTiB} + c, err := New("play.min.io", &Options{ + Creds: credentials.NewStaticV4("id", "secret", ""), + UploadLimits: &limits, + }) + if err != nil { + t.Fatal(err) + } + want := UploadLimits{ + MinPartSize: defaultMinPartSize, + MaxPartSize: fiveTiB, + MaxPartsCount: defaultMaxPartsCount, + MaxSinglePutObjectSize: fiveTiB, + } + if got := c.UploadLimits(); got != want { + t.Errorf("UploadLimits() = %+v, want %+v", got, want) + } + + // Core embeds *Client, so it reports the same limits. + core, err := NewCore("play.min.io", &Options{ + Creds: credentials.NewStaticV4("id", "secret", ""), + UploadLimits: &limits, + }) + if err != nil { + t.Fatal(err) + } + if got := core.UploadLimits(); got != want { + t.Errorf("Core.UploadLimits() = %+v, want %+v", got, want) + } + + // Unset limits and a client not built by New both read back as S3. + def := UploadLimits{ + MinPartSize: defaultMinPartSize, + MaxPartSize: defaultMaxPartSize, + MaxPartsCount: defaultMaxPartsCount, + MaxSinglePutObjectSize: defaultMaxSinglePutObjectSize, + } + plain, err := New("play.min.io", &Options{Creds: credentials.NewStaticV4("id", "secret", "")}) + if err != nil { + t.Fatal(err) + } + if got := plain.UploadLimits(); got != def { + t.Errorf("UploadLimits() = %+v, want %+v", got, def) + } + if got := (&Client{}).UploadLimits(); got != def { + t.Errorf("zero-value Client UploadLimits() = %+v, want %+v", got, def) + } +} + +// A part size above MaxInt64 must not wrap negative and slip past the ceiling +// checks that are written in terms of int64. +func TestUploadLimitsUnsignedPartSizeGuards(t *testing.T) { + const huge = uint64(math.MaxInt64) + 1 + + if _, _, _, err := OptimalPartInfo(1024*1024*1024, huge); err == nil { + t.Error("optimalPartInfo accepted a part size above MaxInt64") + } else if msg := ToErrorResponse(err).Message; !strings.Contains(msg, "bigger than allowed maximum") { + t.Errorf("optimalPartInfo error = %q, want it to report the maximum", msg) + } + + c, err := New("play.min.io", &Options{ + Creds: credentials.NewStaticV4("id", "secret", ""), + TrailingHeaders: true, + }) + if err != nil { + t.Fatal(err) + } + if err := (AppendObjectOptions{ChunkSize: huge}).validate(c); err == nil { + t.Error("AppendObjectOptions.validate accepted a chunk size above MaxInt64") + } +} + +// An oversized part must not be reported as a single PUT problem. +func TestPartTooLargeMessage(t *testing.T) { + c, err := New("play.min.io", &Options{Creds: credentials.NewStaticV4("id", "secret", "")}) + if err != nil { + t.Fatal(err) + } + maxPartSize := c.UploadLimits().MaxPartSize + _, err = c.uploadPart(context.Background(), uploadPartParams{ + bucketName: "bucket", objectName: "object", uploadID: "upload-id", + reader: bytes.NewReader(nil), partNumber: 1, size: maxPartSize + 1, + }) + resp := ToErrorResponse(err) + if resp.Code != EntityTooLarge { + t.Fatalf("error code = %q, want %q (err %v)", resp.Code, EntityTooLarge, err) + } + if strings.Contains(resp.Message, "single PUT") { + t.Errorf("part size error mentions a single PUT: %q", resp.Message) + } + if !strings.Contains(resp.Message, "part size") { + t.Errorf("part size error does not mention the part size: %q", resp.Message) + } +} + +// The 5GiB default must not gate PutObject: remotes such as MinIO/AIStor accept +// single PUTs far above Amazon's, and PutObjectsSnowball sets DisableMultipart +// itself. An explicitly configured limit is enforced. +func TestPutObjectSinglePutLimitOnlyWhenConfigured(t *testing.T) { + // Atomic: the aborted 6GiB request below leaves its handler running while + // the test resets the counter. + var singlePuts atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut { + singlePuts.Add(1) + } + io.Copy(io.Discard, r.Body) + w.Header().Set("ETag", `"3858f62230ac3c915f300c664312c11f"`) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + client, err := New(u.Host, &Options{ + Creds: credentials.NewStaticV4("ak", "sk", ""), + Secure: false, + Region: "us-east-1", + }) + if err != nil { + t.Fatal(err) + } + + // The snowball shape at the real default: 6GiB with DisableMultipart. The + // size gate runs before the request, so a canceled context separates the + // two outcomes without putting 6GiB on the wire — EntityTooLarge means the + // default was enforced, a context error means routing let it through. + canceled, cancel := context.WithCancel(context.Background()) + cancel() + _, err = client.PutObject(canceled, "bucket", "snowball.tar", + bytes.NewReader(nil), int64(6)<<30, PutObjectOptions{DisableMultipart: true}) + if code := ToErrorResponse(err).Code; code == EntityTooLarge { + t.Fatalf("PutObject refused a 6GiB single PUT client-side: %v", err) + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("PutObject error = %v, want the canceled context to surface", err) + } + + // A normal-sized object on default limits still goes out as one PUT. + singlePuts.Store(0) + data := bytes.Repeat([]byte("a"), 8192) + if _, err := client.PutObject(context.Background(), "bucket", "object", + bytes.NewReader(data), int64(len(data)), PutObjectOptions{DisableMultipart: true}); err != nil { + t.Fatalf("PutObject: %v", err) + } + if got := singlePuts.Load(); got != 1 { + t.Fatalf("single PUTs = %d, want 1 (upload was refused client-side)", got) + } + + // Setting the limit explicitly opts in to enforcement. + small, err := New(u.Host, &Options{ + Creds: credentials.NewStaticV4("ak", "sk", ""), + Secure: false, + Region: "us-east-1", + UploadLimits: &UploadLimits{MaxSinglePutObjectSize: 4096}, + }) + if err != nil { + t.Fatal(err) + } + singlePuts.Store(0) + _, err = small.PutObject(context.Background(), "bucket", "object", + bytes.NewReader(data), int64(len(data)), PutObjectOptions{DisableMultipart: true}) + if code := ToErrorResponse(err).Code; code != EntityTooLarge { + t.Fatalf("configured limit: error code = %q, want %q (err %v)", code, EntityTooLarge, err) + } + if got := singlePuts.Load(); got != 0 { + t.Fatalf("configured limit: %d PUTs issued, want 0", got) + } +} + +// MaxSinglePutObjectSize is a Client.PutObject routing decision. Core.PutObject +// is the raw S3 call and sends the PUT as given, as its doc comment states. +func TestCorePutObjectIgnoresSinglePutLimit(t *testing.T) { + var puts int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut { + puts++ + } + io.Copy(io.Discard, r.Body) + w.Header().Set("ETag", `"3858f62230ac3c915f300c664312c11f"`) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + limits := UploadLimits{MinPartSize: 1024, MaxSinglePutObjectSize: 4096} + opts := &Options{ + Creds: credentials.NewStaticV4("ak", "sk", ""), + Secure: false, + Region: "us-east-1", + UploadLimits: &limits, + } + core, err := NewCore(u.Host, opts) + if err != nil { + t.Fatal(err) + } + + data := bytes.Repeat([]byte("a"), 8192) + if _, err := core.PutObject(context.Background(), "bucket", "object", + bytes.NewReader(data), int64(len(data)), "", "", PutObjectOptions{}); err != nil { + t.Fatalf("Core.PutObject: %v", err) + } + if puts != 1 { + t.Fatalf("Core.PutObject issued %d PUTs, want 1", puts) + } + + // The limit was set explicitly, so Client.PutObject does refuse the same + // oversized single PUT; only the raw Core call bypasses it. + client, err := New(u.Host, opts) + if err != nil { + t.Fatal(err) + } + _, err = client.PutObject(context.Background(), "bucket", "object", + bytes.NewReader(data), int64(len(data)), PutObjectOptions{DisableMultipart: true}) + if code := ToErrorResponse(err).Code; code != EntityTooLarge { + t.Fatalf("Client.PutObject error code = %q, want %q (err %v)", code, EntityTooLarge, err) + } + if puts != 1 { + t.Fatalf("PUTs issued = %d, want 1 (only the Core call)", puts) + } +} + +// An unknown length stream that outlasts the part budget must fail instead of +// completing a truncated object. +func TestPutObjectUnknownLengthTruncation(t *testing.T) { + var completes int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case r.Method == http.MethodPost && q.Has("uploads"): + w.Header().Set("Content-Type", "application/xml") + io.WriteString(w, ``+ + `bucketobject`+ + `upload-id`) + case r.Method == http.MethodPost && q.Get("uploadId") != "": + completes++ + io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/xml") + io.WriteString(w, ``+ + `bucketobject`+ + `"3858f62230ac3c915f300c664312c11f-2"`) + default: + io.Copy(io.Discard, r.Body) + w.Header().Set("ETag", `"3858f62230ac3c915f300c664312c11f"`) + w.WriteHeader(http.StatusOK) + } + })) + defer srv.Close() + + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + + // A budget of two 1KiB parts against a 4KiB stream. + limits := UploadLimits{MinPartSize: 1024, MaxPartSize: 1024, MaxPartsCount: 2} + + errBroken := errors.New("reader broke") + + for _, tc := range []struct { + name string + // readErr, when set, makes the reader deliver exactly the part budget + // and then fail instead of reaching EOF. + readErr error + creds *credentials.Credentials + opts PutObjectOptions + }{ + {"stream no length", nil, credentials.NewStaticV4("ak", "sk", ""), PutObjectOptions{}}, + {"stream parallel", nil, credentials.NewStaticV4("ak", "sk", ""), PutObjectOptions{ConcurrentStreamParts: true, NumThreads: 2}}, + {"multipart no stream", nil, credentials.NewStaticV2("ak", "sk", ""), PutObjectOptions{}}, + {"failing trailing read", errBroken, credentials.NewStaticV4("ak", "sk", ""), PutObjectOptions{}}, + } { + t.Run(tc.name, func(t *testing.T) { + completes = 0 + l := limits + client, err := New(u.Host, &Options{ + Creds: tc.creds, + Secure: false, + Region: "us-east-1", + UploadLimits: &l, + }) + if err != nil { + t.Fatal(err) + } + + var reader io.Reader = bytes.NewReader(bytes.Repeat([]byte("a"), 4096)) + if tc.readErr != nil { + reader = &failAtEOFReader{ + Reader: bytes.NewReader(bytes.Repeat([]byte("a"), 2048)), + err: tc.readErr, + } + } + _, err = client.PutObject(context.Background(), "bucket", "object", reader, -1, tc.opts) + switch { + case tc.readErr != nil: + if !errors.Is(err, tc.readErr) { + t.Fatalf("PutObject error = %v, want %v", err, tc.readErr) + } + default: + resp := ToErrorResponse(err) + if resp.Code != EntityTooLarge { + t.Fatalf("PutObject error code = %q, want %q (err %v)", resp.Code, EntityTooLarge, err) + } + // The part budget ran out; this is neither a single PUT nor an + // object-size ceiling, and the bytes that fit are not a maximum. + if strings.Contains(resp.Message, "single PUT") { + t.Errorf("truncation error mentions a single PUT: %q", resp.Message) + } + if strings.Contains(resp.Message, "maximum allowed object size") { + t.Errorf("truncation error reports an object-size maximum: %q", resp.Message) + } + // Two 1KiB parts were laid out and uploaded before the reader ran on. + if !strings.Contains(resp.Message, "‘2’ parts") || !strings.Contains(resp.Message, "‘2048’ bytes") { + t.Errorf("truncation error does not report the part budget and bytes uploaded: %q", resp.Message) + } + } + if completes != 0 { + t.Fatalf("completed %d truncated uploads, want 0", completes) + } + }) + } +} + +// failAtEOFReader substitutes err for the io.EOF of the wrapped reader, so the +// trailing zero-byte read reports a failure rather than a drained stream. +type failAtEOFReader struct { + io.Reader + err error +} + +func (r *failAtEOFReader) Read(p []byte) (int, error) { + n, err := r.Reader.Read(p) + if err == io.EOF { + err = r.err + } + return n, err +} diff --git a/validate_uploadpartcopy_checksum_test.go b/validate_uploadpartcopy_checksum_test.go index f0deb69e7..6bee9158f 100644 --- a/validate_uploadpartcopy_checksum_test.go +++ b/validate_uploadpartcopy_checksum_test.go @@ -23,6 +23,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "slices" "strconv" "strings" "testing" @@ -222,17 +223,18 @@ func TestCopyObjectResponseChecksums(t *testing.T) { // TestComposeObjectChecksum5924 validates that ComposeObject sets the requested // checksum algorithm on the multipart upload (so server-side copied parts are // checksummed) and surfaces the composed object's checksum (AIStor #5924). A -// 6 MiB source with a 5 MiB part size forces the two-part multipart-copy path; +// 10 MiB source with a 5 MiB part size forces the two-part multipart-copy path; // a mock endpoint keeps it deterministic in CI without a live server. func TestComposeObjectChecksum5924(t *testing.T) { - const ( - wantCRC32C = "yZRlqg==" - srcSize = 6 * 1024 * 1024 - ) + const wantCRC32C = "yZRlqg==" + srcSize := 10 * 1024 * 1024 var ( gotAlgo string gotMode string gotCompleteBody string + gotRanges []string + initCount int + completeCount int ) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() @@ -245,6 +247,7 @@ func TestComposeObjectChecksum5924(t *testing.T) { w.WriteHeader(http.StatusOK) // Initiate multipart upload (POST ?uploads): record the algorithm header. case r.Method == http.MethodPost && q.Has("uploads"): + initCount++ gotAlgo = r.Header.Get(amzChecksumAlgo) gotMode = r.Header.Get(amzChecksumMode) w.Header().Set("Content-Type", "application/xml") @@ -262,6 +265,7 @@ func TestComposeObjectChecksum5924(t *testing.T) { http.Error(w, "missing x-amz-copy-source", http.StatusBadRequest) return } + gotRanges = append(gotRanges, r.Header.Get("x-amz-copy-source-range")) w.Header().Set("Content-Type", "application/xml") io.WriteString(w, ``+ ``+ @@ -269,9 +273,19 @@ func TestComposeObjectChecksum5924(t *testing.T) { `2026-01-01T00:00:00.000Z`+ ``+wantCRC32C+``+ ``) + // Plain CopyObject (PUT + copy-source, no uploadId): the direct path an + // empty or single small source takes. + case r.Method == http.MethodPut && r.Header.Get("x-amz-copy-source") != "": + w.Header().Set("Content-Type", "application/xml") + io.WriteString(w, ``+ + ``+ + `"3858f62230ac3c915f300c664312c11f"`+ + `2026-01-01T00:00:00.000Z`+ + ``) // CompleteMultipartUpload (POST ?uploadId): capture the part bodies and // echo the object checksum. case r.Method == http.MethodPost && q.Get("uploadId") != "": + completeCount++ body, _ := io.ReadAll(r.Body) gotCompleteBody = string(body) w.Header().Set("Content-Type", "application/xml") @@ -306,7 +320,7 @@ func TestComposeObjectChecksum5924(t *testing.T) { } info, err := client.ComposeObject(context.Background(), - CopyDestOptions{Bucket: "dst-bucket", Object: "dst", ChecksumType: ChecksumCRC32C, PartSize: absMinPartSize}, + CopyDestOptions{Bucket: "dst-bucket", Object: "dst", ChecksumType: ChecksumCRC32C, PartSize: defaultMinPartSize}, CopySrcOptions{Bucket: "src-bucket", Object: "src"}) if err != nil { t.Fatalf("ComposeObject: %v", err) @@ -322,12 +336,18 @@ func TestComposeObjectChecksum5924(t *testing.T) { } // The per-part checksum parsed from CopyPartResult must reach the // CompleteMultipartUpload request body as on every part; - // the 6 MiB source at a 5 MiB part size yields exactly two parts, so a + // the 10 MiB source at a 5 MiB part size yields exactly two parts, so a // dropped second-part checksum would leave only one occurrence. want := "" + wantCRC32C + "" if got := strings.Count(gotCompleteBody, want); got != 2 { t.Fatalf("CompleteMultipartUpload body has %d %q, want 2; body %q", got, want, gotCompleteBody) } + // Every generated copy range but the last must be at least MinPartSize, + // otherwise the remote rejects the part. + wantRanges := []string{"bytes=0-5242879", "bytes=5242880-10485759"} + if !slices.Equal(gotRanges, wantRanges) { + t.Fatalf("copy source ranges = %q, want %q", gotRanges, wantRanges) + } // A composite (non-full-object) algorithm must not set the mode header. if gotMode != "" { t.Fatalf("composite checksum init mode = %q, want empty", gotMode) @@ -336,7 +356,7 @@ func TestComposeObjectChecksum5924(t *testing.T) { // A full-object checksum type additionally sets the mode header on the MPU // init (the dst.ChecksumType.FullObjectRequested() branch). if _, err := client.ComposeObject(context.Background(), - CopyDestOptions{Bucket: "dst-bucket", Object: "dst", ChecksumType: ChecksumFullObjectCRC32C, PartSize: absMinPartSize}, + CopyDestOptions{Bucket: "dst-bucket", Object: "dst", ChecksumType: ChecksumFullObjectCRC32C, PartSize: defaultMinPartSize}, CopySrcOptions{Bucket: "src-bucket", Object: "src"}); err != nil { t.Fatalf("ComposeObject (full object): %v", err) } @@ -346,4 +366,59 @@ func TestComposeObjectChecksum5924(t *testing.T) { if gotMode != "FULL_OBJECT" { t.Fatalf("full-object init checksum mode = %q, want %q", gotMode, "FULL_OBJECT") } + + // A 6 MiB source at a 5 MiB part size splits evenly into two 3 MiB ranges, + // both below MinPartSize, so it must be rejected up front rather than + // rejected by the remote mid-copy. Only the source stat may reach the wire. + srcSize = 6 * 1024 * 1024 + gotRanges, initCount, completeCount = nil, 0, 0 + if _, err := client.ComposeObject(context.Background(), + CopyDestOptions{Bucket: "dst-bucket", Object: "dst", PartSize: defaultMinPartSize}, + CopySrcOptions{Bucket: "src-bucket", Object: "src"}); err == nil { + t.Fatal("ComposeObject: expected a rejection for ranges below the minimum part size") + } + if initCount != 0 || len(gotRanges) != 0 || completeCount != 0 { + t.Fatalf("rejected compose issued %d initiations, %d copies and %d completions, want none", + initCount, len(gotRanges), completeCount) + } + + // A part size above the maximum is rejected too. + if _, err := client.ComposeObject(context.Background(), + CopyDestOptions{Bucket: "dst-bucket", Object: "dst", PartSize: uint64(defaultMaxPartSize) + 1}, + CopySrcOptions{Bucket: "src-bucket", Object: "src"}); err == nil { + t.Fatal("ComposeObject: expected a rejection for a part size above the maximum") + } + + // 2*MinPartSize-1 at a MinPartSize part size splits into a full range plus + // a one-byte-short tail. That tail is the final range of the final source, + // which S3 exempts from the minimum, so it must be accepted. + srcSize = 2*defaultMinPartSize - 1 + gotRanges = nil + if _, err := client.ComposeObject(context.Background(), + CopyDestOptions{Bucket: "dst-bucket", Object: "dst", PartSize: defaultMinPartSize}, + CopySrcOptions{Bucket: "src-bucket", Object: "src"}); err != nil { + t.Fatalf("ComposeObject (short final range): %v", err) + } + wantRanges = []string{"bytes=0-5242879", "bytes=5242880-10485758"} + if !slices.Equal(gotRanges, wantRanges) { + t.Fatalf("copy source ranges = %q, want %q", gotRanges, wantRanges) + } + + // An empty source needs no ranges at all, so the split check must not run + // against a zero part count. ComposeObject copies it directly, without + // opening a multipart upload. + srcSize = 0 + gotRanges, initCount, completeCount = nil, 0, 0 + if _, err := client.ComposeObject(context.Background(), + CopyDestOptions{Bucket: "dst-bucket", Object: "dst", PartSize: defaultMinPartSize}, + CopySrcOptions{Bucket: "src-bucket", Object: "src"}); err != nil { + t.Fatalf("ComposeObject (empty source): %v", err) + } + if len(gotRanges) != 0 { + t.Fatalf("empty source produced copy ranges %q, want none", gotRanges) + } + if initCount != 0 || completeCount != 0 { + t.Fatalf("empty source issued %d initiations and %d completions, want none (direct copy)", + initCount, completeCount) + } }