diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37d94a34..42870c11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ on: pull_request: env: - DEVELOPER_DIR: /Applications/Xcode_16.4.app/Contents/Developer + DEVELOPER_DIR: /Applications/Xcode_26.3.app/Contents/Developer jobs: mac-tests: diff --git a/BitMatch.xcodeproj/project.pbxproj b/BitMatch.xcodeproj/project.pbxproj index deffe4f3..9adeb56b 100644 --- a/BitMatch.xcodeproj/project.pbxproj +++ b/BitMatch.xcodeproj/project.pbxproj @@ -643,7 +643,7 @@ CODE_SIGN_ENTITLEMENTS = BitMatch/BitMatch.entitlements; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 12; + CURRENT_PROJECT_VERSION = 14; DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_HARDENED_RUNTIME = YES; ENABLE_PREVIEWS = YES; @@ -654,7 +654,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MARKETING_VERSION = 0.2.3; + MARKETING_VERSION = 0.2.4; PRODUCT_BUNDLE_IDENTIFIER = BitMatchApp.BitMatch; PRODUCT_NAME = "$(TARGET_NAME)"; REGISTER_APP_GROUPS = YES; @@ -672,7 +672,7 @@ CODE_SIGN_ENTITLEMENTS = BitMatch/BitMatch.entitlements; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 12; + CURRENT_PROJECT_VERSION = 14; DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_HARDENED_RUNTIME = YES; ENABLE_PREVIEWS = YES; @@ -683,7 +683,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MARKETING_VERSION = 0.2.3; + MARKETING_VERSION = 0.2.4; PRODUCT_BUNDLE_IDENTIFIER = BitMatchApp.BitMatch; PRODUCT_NAME = "$(TARGET_NAME)"; REGISTER_APP_GROUPS = YES; @@ -698,11 +698,11 @@ buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 12; + CURRENT_PROJECT_VERSION = 14; DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; GENERATE_INFOPLIST_FILE = YES; MACOSX_DEPLOYMENT_TARGET = 15.5; - MARKETING_VERSION = 0.2.3; + MARKETING_VERSION = 0.2.4; PRODUCT_BUNDLE_IDENTIFIER = BitMatchApp.BitMatchTests; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = NO; @@ -716,11 +716,11 @@ buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 12; + CURRENT_PROJECT_VERSION = 14; DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; GENERATE_INFOPLIST_FILE = YES; MACOSX_DEPLOYMENT_TARGET = 15.5; - MARKETING_VERSION = 0.2.3; + MARKETING_VERSION = 0.2.4; PRODUCT_BUNDLE_IDENTIFIER = BitMatchApp.BitMatchTests; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = NO; @@ -733,10 +733,10 @@ isa = XCBuildConfiguration; buildSettings = { CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 12; + CURRENT_PROJECT_VERSION = 14; DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; GENERATE_INFOPLIST_FILE = YES; - MARKETING_VERSION = 0.2.3; + MARKETING_VERSION = 0.2.4; PRODUCT_BUNDLE_IDENTIFIER = BitMatchApp.BitMatchUITests; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = NO; @@ -749,10 +749,10 @@ isa = XCBuildConfiguration; buildSettings = { CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 12; + CURRENT_PROJECT_VERSION = 14; DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; GENERATE_INFOPLIST_FILE = YES; - MARKETING_VERSION = 0.2.3; + MARKETING_VERSION = 0.2.4; PRODUCT_BUNDLE_IDENTIFIER = BitMatchApp.BitMatchUITests; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = NO; @@ -767,7 +767,7 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 12; + CURRENT_PROJECT_VERSION = 14; DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; @@ -784,7 +784,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 0.2.3; + MARKETING_VERSION = 0.2.4; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = "BitMatchiOS.BitMatch-iPad"; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -802,7 +802,7 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 12; + CURRENT_PROJECT_VERSION = 14; DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; @@ -819,7 +819,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 0.2.3; + MARKETING_VERSION = 0.2.4; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = "BitMatchiOS.BitMatch-iPad"; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -837,11 +837,11 @@ buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 12; + CURRENT_PROJECT_VERSION = 14; DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; GENERATE_INFOPLIST_FILE = YES; IPHONEOS_DEPLOYMENT_TARGET = 18.5; - MARKETING_VERSION = 0.2.3; + MARKETING_VERSION = 0.2.4; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = "BitMatchiOS.BitMatch-iPadTests"; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -858,11 +858,11 @@ buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 12; + CURRENT_PROJECT_VERSION = 14; DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; GENERATE_INFOPLIST_FILE = YES; IPHONEOS_DEPLOYMENT_TARGET = 18.5; - MARKETING_VERSION = 0.2.3; + MARKETING_VERSION = 0.2.4; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = "BitMatchiOS.BitMatch-iPadTests"; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -879,11 +879,11 @@ isa = XCBuildConfiguration; buildSettings = { CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 12; + CURRENT_PROJECT_VERSION = 14; DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; GENERATE_INFOPLIST_FILE = YES; IPHONEOS_DEPLOYMENT_TARGET = 18.5; - MARKETING_VERSION = 0.2.3; + MARKETING_VERSION = 0.2.4; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = "BitMatchiOS.BitMatch-iPadUITests"; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -899,11 +899,11 @@ isa = XCBuildConfiguration; buildSettings = { CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 12; + CURRENT_PROJECT_VERSION = 14; DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; GENERATE_INFOPLIST_FILE = YES; IPHONEOS_DEPLOYMENT_TARGET = 18.5; - MARKETING_VERSION = 0.2.3; + MARKETING_VERSION = 0.2.4; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = "BitMatchiOS.BitMatch-iPadUITests"; PRODUCT_NAME = "$(TARGET_NAME)"; diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cf0fc80..b8945c60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +## [0.2.4] - 2026-10-04 + +A reliability update for large-file verification. + +- Fixed temporary read buffers building up in memory while checking large files. BitMatch now releases them after each chunk. +- Diagnostics now show which numbered file and destination were being checked, how far the read got, and how much memory BitMatch was using. Footage names and paths remain private. +- Verification diagnostics distinguish a matching copy, a mismatch, an error and cancellation, making an interrupted transfer easier to investigate. + +Tested with two 45 GB files verifying together, with memory staying flat, plus a signed-app copy, cancellation, retry, ASC MHL and report walkthrough. Synthetic tests do not reproduce every drive or camera setup; the exact cause of the interruption in issue #10 remains unconfirmed. If it happens again, use **History → Export diagnostics** after reopening BitMatch and attach the file to the issue. + ## [0.2.3] - 2026-10-01 A reliability update for transfers and reports. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 12f0aeef..1239387f 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -6,7 +6,7 @@ Read [docs/THESIS.md](docs/THESIS.md) for what BitMatch promises and the current ## Building -- Xcode 16 or newer; CI uses Xcode 16.4. Open `BitMatch.xcodeproj`. +- Use Xcode 26.3 or newer for the app concurrency settings; CI selects Xcode 26.3. Open `BitMatch.xcodeproj`. - Schemes: - `BitMatch`: the macOS app (macOS 15.5 or newer). - `BitMatch-iPad`: the iPhone and iPad app (iOS/iPadOS 18.5 or newer). Despite the name it targets both device families. @@ -53,7 +53,7 @@ bash test.sh release-builds # macOS and iOS simulator Release builds `ipad-test` refuses to guess a simulator; set `IOS_SIMULATOR_DESTINATION` to one installed on your machine. Set `DERIVED_DATA_ROOT` to reuse a build folder (default `.derived-data/`). -`.github/workflows/ci.yml` defines `mac-test` and `ipad-build` jobs on push and pull request. The README notes that GitHub Actions is currently disabled for the repository, so run both locally before submitting changes. +`.github/workflows/ci.yml` defines Mac tests, the iPad/iPhone build and engine tests on push and pull request. Run the required checks locally before submitting changes; record any host or toolchain failures separately from passing tests. ### Change workflow diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/ChecksumEngine.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/ChecksumEngine.swift index 2f58fa1d..de0fc294 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/ChecksumEngine.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/ChecksumEngine.swift @@ -251,12 +251,14 @@ public final class ChecksumEngine: ChecksumService, Sendable { await Task.yield() try Task.checkCancellation() try await PauseGate.waitIfCurrentIsPaused() - let data = try fileHandle.read(upToCount: chunkSize) ?? Data() - if data.isEmpty { break } - autoreleasepool { - hasher.update(data: data) + // Include the read so temporary Foundation buffers drain per chunk. + let count = try autoreleasepool { () throws -> Int in + let data = try fileHandle.read(upToCount: chunkSize) ?? Data() + if !data.isEmpty { hasher.update(data: data) } + return data.count } - bytesProcessed += Int64(data.count) + if count == 0 { break } + bytesProcessed += Int64(count) let progress = Double(bytesProcessed) / Double(initial.size) progressCallback?(progress, "Computing MD5 (legacy)...") } @@ -289,13 +291,14 @@ public final class ChecksumEngine: ChecksumService, Sendable { await Task.yield() try Task.checkCancellation() try await PauseGate.waitIfCurrentIsPaused() - let data = try fileHandle.read(upToCount: chunkSize) ?? Data() - if data.isEmpty { break } - - autoreleasepool { - hasher.update(data: data) + // Include the read so temporary Foundation buffers drain per chunk. + let count = try autoreleasepool { () throws -> Int in + let data = try fileHandle.read(upToCount: chunkSize) ?? Data() + if !data.isEmpty { hasher.update(data: data) } + return data.count } - bytesProcessed += Int64(data.count) + if count == 0 { break } + bytesProcessed += Int64(count) // Update progress let progress = Double(bytesProcessed) / Double(initial.size) @@ -330,14 +333,14 @@ public final class ChecksumEngine: ChecksumService, Sendable { await Task.yield() try Task.checkCancellation() try await PauseGate.waitIfCurrentIsPaused() - let data = try fileHandle.read(upToCount: chunkSize) ?? Data() - if data.isEmpty { break } - - autoreleasepool { - hasher.update(data: data) + // Include the read so temporary Foundation buffers drain per chunk. + let count = try autoreleasepool { () throws -> Int in + let data = try fileHandle.read(upToCount: chunkSize) ?? Data() + if !data.isEmpty { hasher.update(data: data) } + return data.count } - - bytesProcessed += Int64(data.count) + if count == 0 { break } + bytesProcessed += Int64(count) // Update progress let progress = Double(bytesProcessed) / Double(initial.size) diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DestinationWriter.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DestinationWriter.swift index a1d6810a..e1986661 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DestinationWriter.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DestinationWriter.swift @@ -1244,6 +1244,14 @@ public final class DestinationWriter { let clipIntegrity: ClipIntegrityFinding? if verification.matches, let clipURL, ClipIntegrityCheck.supports(clipURL) { do { + if let ordinal = TransferDiagnostics.verifyOrdinal { + SharedLogger.verifyEvent(.clipInspectionStarted, run: TransferDiagnostics.runID, ordinal: ordinal, destinationIndex: destinationIndex, bytes: verification.fileSize) + } + defer { + if let ordinal = TransferDiagnostics.verifyOrdinal { + SharedLogger.verifyEvent(.clipInspectionFinished, run: TransferDiagnostics.runID, ordinal: ordinal, destinationIndex: destinationIndex, bytes: verification.fileSize) + } + } clipIntegrity = try inspection(destination) } catch { SharedLogger.warning( @@ -1405,7 +1413,13 @@ public final class DestinationWriter { relativePath: relativePath, hooks: hooks ) { hasher.update($0) } - return PinnedDigestRead(digests: hasher.finalize(), identity: identity) + let digests = hasher.finalize() + if let ordinal = TransferDiagnostics.verifyOrdinal { + SharedLogger.verifyEvent(.verifyDigestFinished, run: TransferDiagnostics.runID, + ordinal: ordinal, destinationIndex: destinationIndex, + bytes: identity.size) + } + return PinnedDigestRead(digests: digests, identity: identity) } /// Thorough's independent card pass. SHA-256 and MD5 are accumulated @@ -1431,11 +1445,15 @@ public final class DestinationWriter { try Task.checkCancellation() try await PauseGate.waitIfCurrentIsPaused() let requested = min(4 * 1024 * 1024, Int(Int64(initial.st_size) - bytesRead)) - let data = try handle.read(upToCount: requested) ?? Data() - guard !data.isEmpty else { throw sourceChangedError() } - hasher.update(data) - hooks?.verificationSourceDidRead?(relativePath, data.count) - bytesRead += Int64(data.count) + // Drain each chunk's buffer; see readPinnedDestination. + let count = try autoreleasepool { () throws -> Int in + let data = try handle.read(upToCount: requested) ?? Data() + if !data.isEmpty { hasher.update(data) } + return data.count + } + guard count > 0 else { throw sourceChangedError() } + hooks?.verificationSourceDidRead?(relativePath, count) + bytesRead += Int64(count) } let trailingData = try handle.read(upToCount: 1) ?? Data() var final = stat() @@ -1451,6 +1469,9 @@ public final class DestinationWriter { ) } + /// How often a pipelined verify records how far it has read. + private static let verifyReadBreadcrumbInterval: Int64 = 8 * 1024 * 1024 * 1024 + private static func readPinnedDestination( _ destination: PinnedDestinationFile, destinationIndex: Int, @@ -1463,15 +1484,31 @@ public final class DestinationWriter { let handle = try destination.readingHandle() hooks?.destinationDidOpenForRead?(destinationIndex, relativePath) defer { closeFileHandle(handle, context: "pinned destination") } + let ordinal = TransferDiagnostics.verifyOrdinal + if ordinal != nil { + SharedLogger.verifyEvent(.verifyOpenedDestination, run: TransferDiagnostics.runID, + ordinal: ordinal, destinationIndex: destinationIndex, bytes: 0) + } var bytesRead: Int64 = 0 + var nextBreadcrumb = verifyReadBreadcrumbInterval while true { try Task.checkCancellation() try await PauseGate.waitIfCurrentIsPaused() - let data = try handle.read(upToCount: 4 * 1024 * 1024) ?? Data() - if data.isEmpty { break } - consume(data) - hooks?.destinationDidRead?(destinationIndex, relativePath, data.count) - bytesRead += Int64(data.count) + // Bound temporary Foundation allocations to this chunk, even when + // the pause gate returns without suspending the task. + let count = try autoreleasepool { () throws -> Int in + let data = try handle.read(upToCount: 4 * 1024 * 1024) ?? Data() + if !data.isEmpty { consume(data) } + return data.count + } + if count == 0 { break } + hooks?.destinationDidRead?(destinationIndex, relativePath, count) + bytesRead += Int64(count) + if ordinal != nil, bytesRead >= nextBreadcrumb { + nextBreadcrumb += verifyReadBreadcrumbInterval + SharedLogger.verifyEvent(.verifyRead, run: TransferDiagnostics.runID, + ordinal: ordinal, destinationIndex: destinationIndex, bytes: bytesRead) + } } let final = try destination.snapshot() guard bytesRead == Int64(initial.st_size), pinnedFileRemainedStable(initial, final) else { @@ -1512,15 +1549,19 @@ public final class DestinationWriter { while bytesRead < sourceSize { try Task.checkCancellation() try await PauseGate.waitIfCurrentIsPaused() - let sourceData = try sourceHandle.read(upToCount: 4 * 1024 * 1024) ?? Data() - let destinationData = try destinationHandle.read(upToCount: 4 * 1024 * 1024) ?? Data() - guard !sourceData.isEmpty, !destinationData.isEmpty else { + // Drain each chunk's buffers; see readPinnedDestination. + let (sourceCount, destinationCount, same) = try autoreleasepool { () throws -> (Int, Int, Bool) in + let sourceData = try sourceHandle.read(upToCount: 4 * 1024 * 1024) ?? Data() + let destinationData = try destinationHandle.read(upToCount: 4 * 1024 * 1024) ?? Data() + return (sourceData.count, destinationData.count, sourceData == destinationData) + } + guard sourceCount > 0, destinationCount > 0 else { throw NSError(domain: "DestinationWriter", code: -11, userInfo: [NSLocalizedDescriptionKey: "File changed while comparing bytes"]) } - hooks?.verificationSourceDidRead?(relativePath, sourceData.count) - hooks?.destinationDidRead?(destinationIndex, relativePath, destinationData.count) - if sourceData != destinationData { return false } - bytesRead += Int64(sourceData.count) + hooks?.verificationSourceDidRead?(relativePath, sourceCount) + hooks?.destinationDidRead?(destinationIndex, relativePath, destinationCount) + if !same { return false } + bytesRead += Int64(sourceCount) } let sourceTrailingData = try sourceHandle.read(upToCount: 1) ?? Data() diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift index 5ce504c7..60418c15 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift @@ -44,6 +44,8 @@ extension SharedLogger { case reportStarted, reportFinished, explicitCancel, parentCancelled case phase, cancelOrigin, error, pipelineResult, config, destination, progress, appStarted, appEnded case terminalError, journalCancelled, journalInterrupted, journalFinished, phaseChanged + case verifyStarted, verifyOpenedDestination, verifyRead, verifyDigestFinished, verifyFinished + case verifyConfiguration, clipInspectionStarted, clipInspectionFinished } public static func transferEvent(_ event: TransferEvent, run: UUID?, code: Int = 0, taskCancelled: Bool = false, explicit: Bool = false) { @@ -54,6 +56,22 @@ extension SharedLogger { } } +extension SharedLogger { + /// One verify job's milestone. Numbers only: the job's ordinal, its size + /// or bytes read so far, and the destination index. Never a name or path. + public enum VerifyOutcome: String, Codable, Sendable { case matched, mismatched, failed, cancelled } + public static func verifyConfiguration(run: UUID?, concurrency: Int) { + TransferDiagnosticStore.shared.record(.verifyConfiguration, run: run) { $0.verifyConcurrency = concurrency } + } + public static func verifyEvent(_ event: TransferEvent, run: UUID?, ordinal: Int?, + destinationIndex: Int, bytes: Int64?, totalBytes: Int64? = nil, outcome: VerifyOutcome? = nil) { + TransferDiagnosticStore.shared.record(event, run: run) { + $0.ordinal = ordinal; $0.destinationIndex = destinationIndex; $0.bytesProcessed = bytes; $0.totalBytes = totalBytes; $0.verifyOutcome = outcome + } + logger(for: .transfer).notice("event=\(event.rawValue, privacy: .public) run=\(run?.uuidString ?? "none", privacy: .public) ordinal=\(ordinal ?? -1, privacy: .public) destination=\(destinationIndex, privacy: .public) bytes=\(bytes ?? -1, privacy: .public) outcome=\(outcome?.rawValue ?? "pending", privacy: .public)") + } +} + extension SharedLogger { public static func transferPhase(_ phase: ProgressStage, run: UUID?) { TransferDiagnosticStore.shared.record(.phase, run: run) { $0.phase = phase } @@ -116,6 +134,8 @@ extension SharedLogger { /// tasks receive the same UUID explicitly. Never carries source metadata. public enum TransferDiagnostics { @TaskLocal public static var runID: UUID? + /// The pipelined verify job running in this task, for read breadcrumbs. + @TaskLocal public static var verifyOrdinal: Int? } extension SharedLogger { diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferDiagnosticStore.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferDiagnosticStore.swift index 76b131a6..8ef23180 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferDiagnosticStore.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferDiagnosticStore.swift @@ -1,4 +1,7 @@ import Foundation +#if canImport(Darwin) +import Darwin +#endif /// A bounded, local record of structured events only. Never accepts free-form /// log messages, source metadata, bookmarks, or error descriptions. @@ -33,6 +36,12 @@ public final class TransferDiagnosticStore: @unchecked Sendable { var bytesProcessed: Int64? var totalBytes: Int64? var isASCMHL: Bool? + /// Which verify job in the run (1-based copy order), never a name. + var ordinal: Int? + var verifyOutcome: SharedLogger.VerifyOutcome? + var verifyConcurrency: Int? + /// The process's physical memory footprint, in MiB, when recorded. + var footprintMB: Int? } private let directory: URL private let limit: Int @@ -54,6 +63,7 @@ public final class TransferDiagnosticStore: @unchecked Sendable { appVersion: Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "unknown", build: Bundle.main.object(forInfoDictionaryKey: "CFBundleVersion") as? String ?? "unknown", event: event, run: run) + entry.footprintMB = Self.physicalFootprintMB() configure(&entry) do { let encoder = JSONEncoder() @@ -87,6 +97,24 @@ public final class TransferDiagnosticStore: @unchecked Sendable { } catch { writeFailed = true } // Diagnostics must never fail a transfer. } + /// Physical footprint is a resource-pressure clue, not a diagnosis of + /// why a process ended. Sampling failure must never fail a transfer. + static func physicalFootprintMB() -> Int? { + #if canImport(Darwin) + var info = task_vm_info_data_t() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let result = withUnsafeMutablePointer(to: &info) { + $0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) + } + } + guard result == KERN_SUCCESS else { return nil } + return Int(info.phys_footprint / 1_048_576) + #else + return nil + #endif + } + /// Only our typed records are exported. Never includes unified logs or the /// transfer journal, both of which can hold private production information. public func exportData() throws -> Data { diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift index cf79cd31..5daa455f 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift @@ -162,6 +162,8 @@ private struct VerifyJob: Sendable { let destinationIndex: Int let pinnedRoot: PinnedDestinationDirectory let sourceReadEvidence: DestinationWriter.SourceReadEvidence? + /// Copy order within the run, recorded in diagnostics instead of a name. + let ordinal: Int } private actor SourceReadEvidenceStore { @@ -287,6 +289,7 @@ public final class TransferPipeline: FileOperationsService, Sendable { /// platform managers turn it off with the hidden `DisablePipelinedVerify` /// default; the engine itself reads no settings. private let pipelinedVerification: Bool + private let verificationConcurrency: Int? private let activeOperations = ActiveOperationRegistry() private let pauseGate = PauseGate() @@ -294,12 +297,14 @@ public final class TransferPipeline: FileOperationsService, Sendable { fileSystem: any FileAccess, checksum: any ChecksumService, pipelinedVerification: Bool = true, + verificationConcurrency: Int? = nil, destinationSetupHook: (@Sendable (URL) throws -> Void)? = nil, fanOutHooks: DestinationWriter.FanOutHooks? = nil ) { self.fileSystem = fileSystem self.checksumService = checksum self.pipelinedVerification = pipelinedVerification + self.verificationConcurrency = verificationConcurrency.flatMap { $0 > 0 ? min($0, 16) : nil } self.destinationSetupHook = destinationSetupHook self.fanOutHooks = fanOutHooks } @@ -493,8 +498,10 @@ public final class TransferPipeline: FileOperationsService, Sendable { // Perf 5: pipelined verification on by default for checksum/byte-compare modes; user can disable let shouldPipelineVerify = operation.verificationMode != .quick && pipelinedVerification - // Perf 6: adaptive concurrency based on CPU count - let verifyConcurrency = max(2, ProcessInfo.processInfo.activeProcessorCount / 2) + let verifyConcurrency = verificationConcurrency ?? max(2, ProcessInfo.processInfo.activeProcessorCount / 2) + let diagnosticRun = TransferDiagnostics.runID ?? operation.id + SharedLogger.verifyConfiguration(run: diagnosticRun, concurrency: shouldPipelineVerify ? verifyConcurrency : 1) + let fileOrdinals = Dictionary(uniqueKeysWithValues: sourceManifest.enumerated().map { ($0.element.relativePath, $0.offset + 1) }) // The one way progress is reported. Total bytes: the caller's // estimate, else the average copied file size times the plan. @@ -541,20 +548,32 @@ public final class TransferPipeline: FileOperationsService, Sendable { // One verify, recorded whatever happens except cancellation. let verify: @Sendable (VerifyJob) async throws -> Void = { job in let verificationStarted = Date() + var diagnosticOutcome: SharedLogger.VerifyOutcome = .failed + SharedLogger.verifyEvent(.verifyStarted, run: diagnosticRun, ordinal: job.ordinal, + destinationIndex: job.destinationIndex, bytes: 0, totalBytes: job.fileSize) + defer { + SharedLogger.verifyEvent(.verifyFinished, run: diagnosticRun, ordinal: job.ordinal, + destinationIndex: job.destinationIndex, bytes: (diagnosticOutcome == .matched || diagnosticOutcome == .mismatched) ? job.fileSize : nil, totalBytes: job.fileSize, outcome: diagnosticOutcome) + } do { try Task.checkCancellation() try await self.waitIfPaused() - let checked = try await DestinationWriter.verifyPinnedDestinationFileAndInspectClip( - source: job.source, - pinnedRoot: job.pinnedRoot, - relativePath: job.relativePath, - verificationMode: operation.verificationMode, - checksumService: self.checksumService, - clipURL: job.destination, - sourceReadEvidence: job.sourceReadEvidence, - destinationIndex: job.destinationIndex, - hooks: self.fanOutHooks - ) + let checked = try await TransferDiagnostics.$runID.withValue(diagnosticRun) { + try await TransferDiagnostics.$verifyOrdinal.withValue(job.ordinal) { + try await DestinationWriter.verifyPinnedDestinationFileAndInspectClip( + source: job.source, + pinnedRoot: job.pinnedRoot, + relativePath: job.relativePath, + verificationMode: operation.verificationMode, + checksumService: self.checksumService, + clipURL: job.destination, + sourceReadEvidence: job.sourceReadEvidence, + destinationIndex: job.destinationIndex, + hooks: self.fanOutHooks + ) + } + } + diagnosticOutcome = checked.verification.matches ? .matched : .mismatched let verificationResult = checked.verification let verified = FileOperationResult( sourceURL: job.source, @@ -580,6 +599,7 @@ public final class TransferPipeline: FileOperationsService, Sendable { } await onFileResult?(verified) } catch is CancellationError { + diagnosticOutcome = .cancelled // Propagate interruption even when a verifier throws it without // cancelling its parent task. Never silently drop that outcome. throw CancellationError() @@ -755,7 +775,8 @@ public final class TransferPipeline: FileOperationsService, Sendable { source: srcURL, destination: dstURL, relativePath: relativePath, fileSize: max(0, fileSize), destinationIndex: destIndex, pinnedRoot: pinnedDestination, - sourceReadEvidence: await sourceReadEvidence.evidence(for: relativePath) + sourceReadEvidence: await sourceReadEvidence.evidence(for: relativePath), + ordinal: fileOrdinals[relativePath] ?? 0 )) } @@ -803,6 +824,12 @@ public final class TransferPipeline: FileOperationsService, Sendable { let relativePath = entry.relativePath let destinationFileURL = destFolder.appendingPathComponent(relativePath) let fileStartTime = Date() + let ordinal = fileOrdinals[relativePath] ?? 0 + var diagnosticOutcome: SharedLogger.VerifyOutcome = .failed + SharedLogger.verifyEvent(.verifyStarted, run: diagnosticRun, ordinal: ordinal, destinationIndex: destIndex, bytes: 0, totalBytes: max(0, entry.size)) + defer { + SharedLogger.verifyEvent(.verifyFinished, run: diagnosticRun, ordinal: ordinal, destinationIndex: destIndex, bytes: (diagnosticOutcome == .matched || diagnosticOutcome == .mismatched) ? max(0, entry.size) : nil, totalBytes: max(0, entry.size), outcome: diagnosticOutcome) + } do { let sizeForVerify = max(0, entry.size) @@ -815,17 +842,22 @@ public final class TransferPipeline: FileOperationsService, Sendable { Double(event.snapshot.filesVerified) / Double(max(1, totalFiles)) )) } - let checked = try await DestinationWriter.verifyPinnedDestinationFileAndInspectClip( - source: fileURL, - pinnedRoot: pinnedDestination, - relativePath: relativePath, - verificationMode: operation.verificationMode, - checksumService: self.checksumService, - clipURL: destinationFileURL, - sourceReadEvidence: await sourceReadEvidence.evidence(for: relativePath), - destinationIndex: destIndex, - hooks: fanOutHooks - ) + let checked = try await TransferDiagnostics.$runID.withValue(diagnosticRun) { + try await TransferDiagnostics.$verifyOrdinal.withValue(ordinal) { + try await DestinationWriter.verifyPinnedDestinationFileAndInspectClip( + source: fileURL, + pinnedRoot: pinnedDestination, + relativePath: relativePath, + verificationMode: operation.verificationMode, + checksumService: self.checksumService, + clipURL: destinationFileURL, + sourceReadEvidence: await sourceReadEvidence.evidence(for: relativePath), + destinationIndex: destIndex, + hooks: fanOutHooks + ) + } + } + diagnosticOutcome = checked.verification.matches ? .matched : .mismatched let verificationResult = checked.verification let fileSize = sizeForVerify @@ -847,6 +879,7 @@ public final class TransferPipeline: FileOperationsService, Sendable { await onFileResult?(result) } catch is CancellationError { + diagnosticOutcome = .cancelled throw CancellationError() } catch { let nsErr = error as NSError diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/LargeVerificationTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/LargeVerificationTests.swift new file mode 100644 index 00000000..0e3acca5 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/LargeVerificationTests.swift @@ -0,0 +1,76 @@ +#if os(macOS) +import Foundation +import Darwin +import CryptoKit +import Testing +@testable import BitMatchEngine + +struct LargeVerificationTests { + @Test(.enabled(if: ProcessInfo.processInfo.environment["BITMATCH_LARGE_VERIFY_ROOT"] != nil)) + func eightClipExFATToHFSTransfer() async throws { + let path = try #require(ProcessInfo.processInfo.environment["BITMATCH_LARGE_VERIFY_ROOT"]) + try #require(path.hasPrefix("/tmp/bitmatch-large-verify.")) + let root = URL(fileURLWithPath: path) + let source = root.appendingPathComponent("source/Card") + let destVolume = root.appendingPathComponent("destination") + func fs(_ url: URL) throws -> String { + var facts = statfs(); #expect(statfs(url.path, &facts) == 0) + return withUnsafePointer(to: &facts.f_fstypename) { $0.withMemoryRebound(to: CChar.self, capacity: 16) { String(cString: $0) } } + } + #expect(try fs(root.appendingPathComponent("source")) == "exfat") + #expect(try fs(destVolume) == "hfs") + try FileManager.default.createDirectory(at: source, withIntermediateDirectories: true) + var expected: [String: String] = [:] + for index in 0..<8 { + let name = String(format: "clip-%02d.bin", index) + let file = source.appendingPathComponent(name) + #expect(FileManager.default.createFile(atPath: file.path, contents: nil)) + let handle = try FileHandle(forWritingTo: file) + var sha = SHA256() + // 6 x 256 MiB + 2 x 3 GiB = 7.5 GiB. Deliberately smaller than + // the reporter's card; preserves eight clips and overlapping tails. + let chunks = index < 6 ? 64 : 768 + for _ in 0.. 0) + #expect(measured.peak - measured.initial < 128) + print("pinned memory initialMiB=\(measured.initial) peakMiB=\(measured.peak) chunks=\(measured.chunks)") + } + + /// Full logical size with sparse zero payloads; not a physical drive/media reproduction. + @Test(.enabled(if: ProcessInfo.processInfo.environment["BITMATCH_45GB_TEST"] == "1")) + func two45GBPinnedReadsOverlapWithBoundedMemory() async throws { + let root = FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-45gb-\(UUID())") + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + let size: UInt64 = 45_000_000_000 + let chunk = 4 * 1024 * 1024 + var sha = SHA256() + var remaining = size + while remaining > 0 { + try Task.checkCancellation() + let count = Int(min(UInt64(chunk), remaining)) + autoreleasepool { sha.update(data: Data(repeating: 0, count: count)) } + remaining -= UInt64(count) + } + let expected = sha.finalize().map { String(format: "%02x", $0) }.joined() + var inputs: [(URL, DestinationWriter.SourceReadEvidence)] = [] + for index in 0..<2 { + let file = root.appendingPathComponent("clip\(index).bin") + #expect(FileManager.default.createFile(atPath: file.path, contents: nil)) + let writer = try FileHandle(forWritingTo: file) + try writer.truncate(atOffset: size); try writer.close() + var st = stat(); try #require(lstat(file.path, &st) == 0) + let identity = VerifiedFileIdentity(device: UInt64(st.st_dev), inode: UInt64(st.st_ino), size: Int64(st.st_size), + modificationSeconds: Int64(st.st_mtimespec.tv_sec), modificationNanoseconds: Int64(st.st_mtimespec.tv_nsec), + changeSeconds: Int64(st.st_ctimespec.tv_sec), changeNanoseconds: Int64(st.st_ctimespec.tv_nsec)) + inputs.append((file, .init(digests: VerifiedDigests(sha256: expected), identity: identity))) + } + let initial = try #require(TransferDiagnosticStore.physicalFootprintMB()) + let samples = Mutex((opened: 0, peak: initial, bytes: [UInt64(0), UInt64(0)], chunks: [0, 0], barrierPassed: 0)) + let gate = DispatchSemaphore(value: 0) + let hooks = DestinationWriter.FanOutHooks(destinationDidOpenForRead: { _, _ in + let ready = samples.withLock { state in state.opened += 1; return state.opened == 2 } + if ready { gate.signal(); gate.signal() } + if gate.wait(timeout: .now() + 30) == .success { samples.withLock { $0.barrierPassed += 1 } } + }, destinationDidRead: { index, _, count in + samples.withLock { state in + state.bytes[index] += UInt64(count); state.chunks[index] += 1 + state.peak = max(state.peak, TransferDiagnosticStore.physicalFootprintMB() ?? 0) + } + }) + let pinned = try PinnedDestinationDirectory.open(destination: root, rootComponents: []) + let tasks = inputs.enumerated().map { index, input in + Task.detached { + try await DestinationWriter.verifyPinnedDestinationFileAndInspectClip(source: input.0, pinnedRoot: pinned, + relativePath: input.0.lastPathComponent, verificationMode: .standard, checksumService: ChecksumEngine.shared, + clipURL: input.0, sourceReadEvidence: input.1, destinationIndex: index, hooks: hooks).verification + } + } + defer { tasks.forEach { $0.cancel() } } + for task in tasks { + let result = try await task.value + #expect(result.matches); #expect(result.destinationChecksum == expected) + } + let measured = samples.withLock { $0 } + #expect(measured.barrierPassed == 2) + #expect(measured.bytes == [size, size]) + #expect(measured.peak - initial < 128) + print("45GB pair initialMiB=\(initial) peakMiB=\(measured.peak) bytes=\(measured.bytes) chunks=\(measured.chunks) overlap=\(measured.barrierPassed)") + } + + @Test + func verifyBreadcrumbsHaveConsistentRunAndTerminalOutcomes() async throws { + try await FileOperationsTestLock.shared.run { + for pipelined in [true, false] { + for outcome in ["matched", "failed", "mismatched", "cancelled"] { + let cancelled = outcome == "cancelled" + let root = FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-verify-events-\(UUID())") + let source = root.appendingPathComponent("Card") + let dest = root.appendingPathComponent("Backup") + try FileManager.default.createDirectory(at: source, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: dest, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + try Data(repeating: 0x3a, count: 4 * 1024 * 1024 + 1).write(to: source.appendingPathComponent("private-name.bin")) + let run = UUID() + let pipeline = TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared, + pipelinedVerification: pipelined, verificationConcurrency: 1, + fanOutHooks: .init(afterPublish: { _, outputRoot in + if outcome == "mismatched" { + let handle = try FileHandle(forWritingTo: outputRoot.appendingPathComponent("private-name.bin")) + defer { try? handle.close() } + try handle.write(contentsOf: Data([0])) + } + }, beforeDestinationRead: { _, _ in + if cancelled { throw CancellationError() } + if outcome == "failed" { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EIO)) } + })) + do { + let operation = try await TransferDiagnostics.$runID.withValue(run) { + try await pipeline.performFileOperation(sourceURL: source, destinationURLs: [dest], verificationMode: .standard, + settings: CameraLabelSettings(), progressCallback: { _ in }, onFileResult: nil) + } + #expect(!cancelled) + #expect(operation.results.allSatisfy { $0.success == (outcome == "matched") }) + } catch is CancellationError { #expect(cancelled) } + let object = try #require(JSONSerialization.jsonObject(with: TransferDiagnosticStore.shared.exportData()) as? [String: Any]) + let events = try #require(object["events"] as? [[String: Any]]).filter { $0["run"] as? String == run.uuidString } + #expect(events.first { $0["event"] as? String == "verifyConfiguration" }?["verifyConcurrency"] as? Int == 1) + let verify = events.filter { ($0["event"] as? String)?.hasPrefix("verify") == true && $0["event"] as? String != "verifyConfiguration" } + #expect(verify.first?["event"] as? String == "verifyStarted") + #expect(verify.last?["event"] as? String == "verifyFinished") + #expect(verify.last?["verifyOutcome"] as? String == outcome) + #expect(verify.first?["bytesProcessed"] as? Int == 0) + #expect(verify.first?["totalBytes"] as? Int == 4 * 1024 * 1024 + 1) + if cancelled || outcome == "failed" { #expect(verify.last?["bytesProcessed"] == nil) } + #expect(verify.allSatisfy { $0["ordinal"] as? Int == 1 && $0["destinationIndex"] as? Int == 0 }) + if outcome == "matched" || outcome == "mismatched" { + #expect(verify.contains { $0["event"] as? String == "verifyOpenedDestination" }) + #expect(verify.contains { $0["event"] as? String == "verifyDigestFinished" }) + } + } + } + } + } +} diff --git a/Platforms/iOS/Services/IOSPlatformManager.swift b/Platforms/iOS/Services/IOSPlatformManager.swift index 2b43590a..3f5e3210 100644 --- a/Platforms/iOS/Services/IOSPlatformManager.swift +++ b/Platforms/iOS/Services/IOSPlatformManager.swift @@ -34,7 +34,8 @@ final class IOSPlatformManager: PlatformManager, Sendable { self._fileOperations = TransferPipeline( fileSystem: IOSFileSystemService.shared, checksum: ChecksumEngine.shared, - pipelinedVerification: !UserDefaults.standard.bool(forKey: "DisablePipelinedVerify") + pipelinedVerification: !UserDefaults.standard.bool(forKey: "DisablePipelinedVerify"), + verificationConcurrency: UserDefaults.standard.integer(forKey: "BitMatchVerifyConcurrency") ) self._cameraDetection = SharedCameraDetectionService() } diff --git a/Platforms/macOS/Services/MacOSPlatformManager.swift b/Platforms/macOS/Services/MacOSPlatformManager.swift index 17681351..92f833d5 100644 --- a/Platforms/macOS/Services/MacOSPlatformManager.swift +++ b/Platforms/macOS/Services/MacOSPlatformManager.swift @@ -20,7 +20,8 @@ final class MacOSPlatformManager: PlatformManager, Sendable { private static let _sharedFileOperations = TransferPipeline( fileSystem: MacOSFileSystemService.shared, checksum: ChecksumEngine.shared, - pipelinedVerification: !UserDefaults.standard.bool(forKey: "DisablePipelinedVerify") + pipelinedVerification: !UserDefaults.standard.bool(forKey: "DisablePipelinedVerify"), + verificationConcurrency: UserDefaults.standard.integer(forKey: "BitMatchVerifyConcurrency") ) private static let _sharedCameraDetection = SharedCameraDetectionService() diff --git a/Scripts/test-large-verification.sh b/Scripts/test-large-verification.sh new file mode 100755 index 00000000..4c770085 --- /dev/null +++ b/Scripts/test-large-verification.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Eight synthetic clips, two large tail clips; real exFAT -> HFS+ drivers. +# Virtual disks share the host disk: this is not a USB/dock reproduction. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +PAIR=$(mktemp -d /tmp/bitmatch-large-verify.XXXXXX) +MOUNTS=() +cleanup() { + local busy=0 + for mount in "${MOUNTS[@]}"; do hdiutil detach -quiet "$mount" || busy=1; done + if [[ "$busy" == 0 ]]; then rm -rf "$PAIR"; else echo "Busy disposable images retained: $PAIR" >&2; fi +} +trap cleanup EXIT +mkdir "$PAIR/source" "$PAIR/destination" +for role in source destination; do + fs=ExFAT; size=12g; [[ "$role" == destination ]] && fs=JHFS+ && size=24g + hdiutil create -quiet -size "$size" -type SPARSE -fs "$fs" -volname "BM_VERIFY" -o "$PAIR/$role.sparseimage" + hdiutil attach -quiet -nobrowse -mountpoint "$PAIR/$role" "$PAIR/$role.sparseimage" + MOUNTS+=("$PAIR/$role") +done +SWIFT=(swift) +if [[ "$(uname -m)" == arm64 ]]; then SWIFT=(arch -arm64 swift); fi +BITMATCH_LARGE_VERIFY_ROOT="$PAIR" "${SWIFT[@]}" test --package-path "$ROOT/Packages/BitMatchEngine" --build-system native --filter LargeVerificationTests diff --git a/docs/VERIFICATION_MEMORY.md b/docs/VERIFICATION_MEMORY.md new file mode 100644 index 00000000..6f727e51 --- /dev/null +++ b/docs/VERIFICATION_MEMORY.md @@ -0,0 +1,84 @@ +# Large-file verification investigation + +## What the reporter's diagnostics establish + +The 0.2.3 session completed two transfers through verification, MHL, reports and journal settlement. The later eight-file transfer reached `copyDrained` at 2026-10-04 05:52:33 UTC with no cancellation flag. Its next recorded session begins at 05:56:24 UTC, without a verification-drained or terminal event for that transfer. The journal subsequently records an interruption. + +This is consistent with an unclean interruption during verification. It does not identify a fatal signal, an OS resource termination, force quit, or another cause. The absence of logger errors strengthens the timeline but is not proof that every possible event was recorded. The recorded 0.2.2 run separately has an explicit user cancellation. + +## Changes + +- Include the Foundation read inside each chunk's autorelease pool in pinned destination readback, Thorough source rereading, Paranoid byte comparison, and the MD5/SHA-256/SHA-1 checksum loops. Do not rely on a pause check suspending the task. +- Preserve uncached destination reads, pinned descriptor access, source stability checks, and cancellation propagation. +- Add optional process physical footprint in MiB to structured diagnostics. It is evidence of resource usage, not an OS termination reason. +- Add file-ordinal/destination breadcrumbs for verification start, destination open, reads every 8 GiB, digest finish, and completion. Completion records matched, mismatched, failed or cancelled; it is not a success assertion by itself. +- Add advisory clip-inspection start/end events. These do not change its existing advisory policy. +- Record the effective verification worker count. The default adaptive count is unchanged. + +The app reads an optional diagnostic preference at launch and passes it into the engine. On Mac, quit BitMatch before setting it: + +```sh +defaults write BitMatchApp.BitMatch BitMatchVerifyConcurrency -int 1 +``` + +For two workers use `-int 2`. Restore the normal adaptive behavior with: + +```sh +defaults delete BitMatchApp.BitMatch BitMatchVerifyConcurrency +``` + +Restart the app after either change. Positive overrides are capped at 16; zero/negative/unset use the normal count. The preference limits verification tasks, not copy workers or whole-app memory. + +No names, paths, clip metadata or free-form error messages are added to exported diagnostics. Existing private logs remain private. + +## Reproduction + +An optimized standalone async probe using the old 4 MiB Foundation read/hash loop grew to 1,080,591,632 bytes of physical footprint after reading 1 GiB, at which point its safety cap stopped it. With the read inside the pool, the same probe read 8 GiB and finished near 6.4 MB. This establishes buffer retention on this Mac/toolchain, not the reporter's termination cause. + +Run the isolated actual pinned-read memory regression: + +```sh +BITMATCH_MEMORY_TEST=1 arch -arm64 swift test --package-path Packages/BitMatchEngine --build-system native --filter VerificationMemoryTests +``` + +The 256 MiB sparse-file test checks the real pinned destination loop, all 64 read callbacks, an independently computed SHA-256, and a footprint-growth ceiling of 128 MiB. It is opt-in because process-footprint assertions need an isolated process. Terminal-event coverage also exercises both pipelined and sequential cancellation. + +Run the real-driver eight-clip transfer with one and two verification workers: + +```sh +Scripts/test-large-verification.sh +``` + +It creates disposable exFAT source and HFS+ destination images, writes six 256 MiB clips plus two 3 GiB clips (7.5 GiB total), includes macOS-generated AppleDouble companions, copies/verifies to a separate folder for each worker count, independently rehashes destinations and the source, and detaches/deletes its images without forced unmounts. It needs about 25 GiB free plus build space. It is intentionally smaller than the reporter's roughly 120 GB card. These virtual drives share a physical host disk; it does not reproduce his USB dock, HDD, source clips, or machine memory. + +The investigation remains open pending OS termination evidence or a reproduction on the reporter's workload. FileHandle ownership and the clip parser were reviewed without finding the proposed double-close path; that does not rule out every possible process-level fault. + +## Validation on 2026-10-04 + +- Actual pinned-read regression: green at 18 → 26 MiB for 256 MiB/64 chunks; isolated removal of only the read pool fails at 27 → 267 MiB. The source/destination checksum remains correct in both runs. +- Eight-clip real-driver test: one and two workers both pass, 16 results each (eight media files plus eight AppleDouble companions), all independent destination and unchanged-source SHA-256 checks pass. +- Engine suite: 110 XCTest cases, 15 opt-in skips, zero failures; 126 Swift Testing cases, two opt-in skips, zero failures. Memory and large-transfer tests are separately enabled above. +- Mac build and iPad/iPhone simulator build pass. +- The full local Mac app test suite has not completed: its test host stalls in dyld before entering app code. Default, non-debug-dylib, ad-hoc-signed and explicit ARM64 attempts were sampled or observed and stopped. This is an outstanding validation limit, not a passed suite or proof of a code defect. The owner approved re-enabling GitHub CI on October 4; remote validation is being run on PR #13. + +Detailed logs are kept locally under `dist/validation/verification-memory`. They include failed fixture expectations: Foundation's directory listing hid generated AppleDouble companions, while BitMatch correctly transferred 16 entries. The fixture was corrected with direct `lstat` checks rather than omitting companions. Those failed expectations are not product regressions. + +## Full logical-size concurrent read stress + +```sh +BITMATCH_45GB_TEST=1 arch -arm64 swift test --package-path Packages/BitMatchEngine --build-system native --filter VerificationMemoryTests +``` + +Two sparse 45,000,000,000-byte files exercise 90 GB through the actual uncached pinned verification loop. Both readers must open before either begins reading; the test checks all bytes, both independently computed SHA-256 results, and process-footprint growth below 128 MiB. Sparse zero payloads save physical disk space. This is a full-size read/hash stress test, not a 120 GB physical copy, camera parser, USB dock or HDD reproduction. + +## Additional local validation + +- Two simultaneous 45 GB pinned readers: 90,000,000,000 bytes read, 10,729 chunks per reader, both SHA-256 checks pass, footprint 16 → 20 MiB; 148.699 seconds. Sparse payload limitation applies. +- Developer ID signed, notarized diagnostic app and DMG: 0.2.4 build 13, prepared locally without publishing or changing updater metadata. +- Signed-app walkthrough on disposable exFAT → HFS+ images: Standard copy, explicit cancellation with partial results and unsafe verdict, Retry, MHL, PDF/CSV/JSON reports, and successful journal settlement. Two results (2 GiB media plus AppleDouble). Independent source/destination SHA-256 and both MHL MD5 checks pass. Diagnostics record user cancellation separately from the completed retry. Original setup preferences and queue restored; images detached and removed. +- Address Sanitizer: all eight outcome/ordering combinations in the verification breadcrumb test pass (pipelined/sequential × matched/mismatched/failed/cancelled), 1.059 seconds. Footprint tests are intentionally disabled under sanitizer instrumentation. + +The full Mac test suite remains a separate gate; the signed walkthrough does not substitute for it. CI initially failed with Xcode 16.4 on existing non-Sendable app models crossing async boundaries. CI now selects the runner's Xcode 26.3; local testing remains the requested source of validation. + +- Thread Sanitizer: the same eight verification outcome combinations pass, 1.173 seconds, no race report. +- Full local Mac suite: an ad-hoc-signed Release host with `ENABLE_TESTABILITY=YES` builds, then stalls in dyld dependency loading before app code (224 KiB footprint). It was sampled and stopped. An earlier Release attempt lacked `ENABLE_TESTABILITY` and failed to compile test imports; that configuration error was corrected. Neither attempt is counted as a suite pass.