From 93f1c133be5278f9174548d049fb575551f80058 Mon Sep 17 00:00:00 2001 From: Sergei Chertkov Date: Tue, 9 Jun 2026 11:22:48 +0400 Subject: [PATCH] Fix: stream chunk uploads from disk to avoid OOM on large files Both upload paths read each part fully into memory before PUTting it: Android allocated ByteArray(size) in readChunk, and iOS read the whole chunk into Data before writing the temp file. With large chunk sizes and several parts prepared in parallel, peak memory scaled with chunkSize x parallelism and reliably got OOM-killed on lower-heap devices. Replace both with a fixed 256 KB streaming copy: - Android: a custom RequestBody (streamingFileChunkBody) seeks to the part offset and streams through a reusable 256 KB buffer. OkHttp may invoke writeTo() more than once on retry, so the file is reopened per call. Content-Length is now derived from contentLength(). - iOS: streamChunkToFile copies the part into the temp file through a 256 KB buffer instead of buffering the whole chunk in Data. Both fail fast on a short read so a truncated part can't corrupt the S3 multipart upload. Per-part memory is now the buffer size regardless of chunk size. Fix linter warnings --- .../nitroclouduploader/NitroCloudUploader.kt | 56 ++++++++++++------- ios/NitroCloudUploader.swift | 50 ++++++++++++++--- src/NitroCloudUploader.nitro.ts | 13 ++--- 3 files changed, 83 insertions(+), 36 deletions(-) diff --git a/android/src/main/java/com/margelo/nitro/nitroclouduploader/NitroCloudUploader.kt b/android/src/main/java/com/margelo/nitro/nitroclouduploader/NitroCloudUploader.kt index f93d4dc..134200f 100644 --- a/android/src/main/java/com/margelo/nitro/nitroclouduploader/NitroCloudUploader.kt +++ b/android/src/main/java/com/margelo/nitro/nitroclouduploader/NitroCloudUploader.kt @@ -20,7 +20,6 @@ import androidx.core.app.NotificationCompat import com.margelo.nitro.core.Promise import okhttp3.* import okhttp3.MediaType.Companion.toMediaType -import okhttp3.RequestBody.Companion.toRequestBody import java.io.File import java.io.RandomAccessFile import java.util.concurrent.TimeUnit @@ -601,15 +600,11 @@ class NitroCloudUploader( try { println("📤 Uploading part ${part.partNumber}/${state.totalChunks} (attempt ${retries + 1})") - // ✅ Read chunk - val chunkData = readChunk(filePath, part.offset, part.size) - - // ✅ Upload chunk + // Stream the chunk from disk to avoid allocating the full part in memory. val request = Request.Builder() .url(part.url) - .put(chunkData.toRequestBody("application/octet-stream".toMediaType())) + .put(streamingFileChunkBody(filePath, part.offset, part.size)) .addHeader("Content-Type", "application/octet-stream") - .addHeader("Content-Length", part.size.toString()) .build() val response = httpClient.newCall(request).execute() @@ -722,21 +717,40 @@ class NitroCloudUploader( false } - private fun readChunk(filePath: String, offset: Long, size: Long): ByteArray { - return RandomAccessFile(File(filePath), "r").use { raf -> - if (offset + size > raf.length()) { - throw IllegalArgumentException("Invalid chunk bounds: offset=$offset, size=$size, file=${raf.length()}") - } - - raf.seek(offset) - val buffer = ByteArray(size.toInt()) - val bytesRead = raf.read(buffer) - - if (bytesRead != size.toInt()) { - throw Exception("Read $bytesRead bytes, expected $size") + /** + * Streams a byte range from disk into an OkHttp RequestBody. + * + * Each write uses a fixed-size buffer instead of allocating the full chunk. + * The file is reopened for each write so the body can be replayed if OkHttp retries. + */ + private fun streamingFileChunkBody(filePath: String, offset: Long, size: Long): RequestBody { + val mediaType = "application/octet-stream".toMediaType() + return object : RequestBody() { + override fun contentType() = mediaType + override fun contentLength() = size + override fun writeTo(sink: okio.BufferedSink) { + RandomAccessFile(File(filePath), "r").use { raf -> + if (offset + size > raf.length()) { + throw IllegalArgumentException( + "Invalid chunk bounds: offset=$offset, size=$size, file=${raf.length()}", + ) + } + raf.seek(offset) + val buffer = ByteArray(256 * 1024) // 256 KB + var remaining = size + while (remaining > 0) { + val toRead = minOf(buffer.size.toLong(), remaining).toInt() + val read = raf.read(buffer, 0, toRead) + if (read <= 0) { + throw java.io.IOException( + "Unexpected EOF at offset ${offset + (size - remaining)}", + ) + } + sink.write(buffer, 0, read) + remaining -= read + } + } } - - buffer } } diff --git a/ios/NitroCloudUploader.swift b/ios/NitroCloudUploader.swift index 5b985fb..394c27d 100644 --- a/ios/NitroCloudUploader.swift +++ b/ios/NitroCloudUploader.swift @@ -578,8 +578,7 @@ class UploadSession: NSObject { let tempDir = FileManager.default.temporaryDirectory let tempFileURL = tempDir.appendingPathComponent("chunk_\(uploadId)_\(index).tmp") - let data = try readChunkData(offset: offset, size: size) - try data.write(to: tempFileURL) + try streamChunkToFile(offset: offset, size: size, destination: tempFileURL) guard let url = URL(string: uploadUrls[index]) else { throw NSError(domain: "CloudUploader", code: -1, @@ -599,12 +598,47 @@ class UploadSession: NSObject { } } - private func readChunkData(offset: Int64, size: Int64) throws -> Data { - let handle = try FileHandle(forReadingFrom: fileURL) - defer { try? handle.close() } - - try handle.seek(toOffset: UInt64(offset)) - return handle.readData(ofLength: Int(size)) + /// Copies a byte range from the source file into a temporary chunk file. + /// + /// Uses a fixed-size buffer instead of loading the full chunk into memory. + /// Throws on short reads to avoid uploading incomplete chunk data. + private func streamChunkToFile(offset: Int64, size: Int64, destination: URL) throws { + let bufferSize = 256 * 1024 // 256 KB + let fm = FileManager.default + + let readHandle = try FileHandle(forReadingFrom: fileURL) + defer { try? readHandle.close() } + try readHandle.seek(toOffset: UInt64(offset)) + + // Recreate the temp file so each chunk write starts from a clean file. + if fm.fileExists(atPath: destination.path) { + try fm.removeItem(at: destination) + } + guard fm.createFile(atPath: destination.path, contents: nil, attributes: nil) else { + throw NSError(domain: "CloudUploader", code: -1, + userInfo: [NSLocalizedDescriptionKey: "Failed to create temp file at \(destination.path)"]) + } + + do { + let writeHandle = try FileHandle(forWritingTo: destination) + defer { try? writeHandle.close() } + + var remaining = size + while remaining > 0 { + let toRead = Int(min(Int64(bufferSize), remaining)) + let chunk = readHandle.readData(ofLength: toRead) + if chunk.isEmpty { + throw NSError(domain: "CloudUploader", code: -1, + userInfo: [NSLocalizedDescriptionKey: "Unexpected EOF reading chunk at offset \(offset): wanted \(size) bytes, short by \(remaining)"]) + } + try writeHandle.write(contentsOf: chunk) + remaining -= Int64(chunk.count) + } + } catch { + // Remove any partial file so it cannot be uploaded later. + try? fm.removeItem(at: destination) + throw error + } } private func complete() { diff --git a/src/NitroCloudUploader.nitro.ts b/src/NitroCloudUploader.nitro.ts index f6fa81d..45c4a0a 100644 --- a/src/NitroCloudUploader.nitro.ts +++ b/src/NitroCloudUploader.nitro.ts @@ -4,9 +4,9 @@ import type { HybridObject } from 'react-native-nitro-modules'; * Upload progress event for UI updates */ export interface UploadProgressEvent { - type: string; + type: string; uploadId: string; - progress?: number; + progress?: number; bytesUploaded?: number; totalBytes?: number; chunkIndex?: number; @@ -42,26 +42,25 @@ export interface UploadState { * - Background uploads with optional notifications * - Event system for UI feedback * - Parallel chunk uploads - * + * * Note: Chunk size is calculated automatically: ceil(fileSize / numberOfURLs) */ export interface NitroCloudUploader extends HybridObject<{ ios: 'swift'; android: 'kotlin' }> { - /** * Start uploading a file in chunks * - Supports pause/resume/cancel * - Automatically handles network drops * - Works in background with optional notifications * - Uploads multiple chunks in parallel (default: 3) - * + * * @param uploadId - Unique identifier * @param filePath - Local file path (decoded, no file://) * @param uploadUrls - Presigned URLs for each chunk * @param maxParallel - Number of chunks to upload simultaneously (default: 3) * @param showNotification - Show progress notification (default: true) * @returns Promise that resolves when upload completes - * + * * Note: Chunk size is calculated automatically: ceil(fileSize / uploadUrls.length) */ startUpload( @@ -112,4 +111,4 @@ export interface NitroCloudUploader * Remove event listener */ removeListener(eventType: string): void; -} \ No newline at end of file +}