From 60e5f78668058e61375e19f042897fa069d37168 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 5 Oct 2026 09:59:23 -0500 Subject: [PATCH 1/2] Reproduce Sony Venice issue 8 layout across transfer and comparison modes --- .../SonyVeniceIssue8Tests.swift | 185 ++++++++++++++++++ docs/ISSUE8_SONY_VENICE.md | 31 +++ 2 files changed, 216 insertions(+) create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/SonyVeniceIssue8Tests.swift create mode 100644 docs/ISSUE8_SONY_VENICE.md diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SonyVeniceIssue8Tests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SonyVeniceIssue8Tests.swift new file mode 100644 index 00000000..144e28f7 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SonyVeniceIssue8Tests.swift @@ -0,0 +1,185 @@ +import Foundation +import Testing +@testable import BitMatchEngine + +// Reported names/layout from issue #8; contents are synthetic, not Sony metadata or footage. +struct SonyVeniceIssue8Tests { + private static func diskImage(_ arguments: [String]) -> Int32 { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil") + process.arguments = arguments + do { try process.run(); process.waitUntilExit(); return process.terminationStatus } + catch { return -1 } + } + + static let paths = """ +A001CNXB/.DS_Store +A001CNXB/MEDIAPRO.XML +A001CNXB/General +A001CNXB/General/Sony +A001CNXB/General/Sony/Planning +A001CNXB/Clip +A001CNXB/Clip/A001C002_26091476R01.BIM +A001CNXB/Clip/A001C013_2609140GR01.BIM +A001CNXB/Clip/A001C015_2609147F.MOV +A001CNXB/Clip/A001C008_2609148KM01.XML +A001CNXB/Clip/A001C004_260914AK.MOV +A001CNXB/Clip/A001C010_260914IHR01.BIM +A001CNXB/Clip/A001C007_26091490R01.BIM +A001CNXB/Clip/A001C006_260914XT.MOV +A001CNXB/Clip/A001C011_260914NW.MOV +A001CNXB/Clip/A001C011_260914NWR01.BIM +A001CNXB/Clip/A001C012_260914JVM01.XML +A001CNXB/Clip/A001C014_2609149VM01.XML +A001CNXB/Clip/A001C003_260914IJM01.XML +A001CNXB/Clip/A001C001_260914A8R01.BIM +A001CNXB/Clip/A001C013_2609140G.MOV +A001CNXB/Clip/A001C005_260914K7R01.BIM +A001CNXB/Clip/A001C004_260914AKR01.BIM +A001CNXB/Clip/A001C006_260914XTM01.XML +A001CNXB/Clip/A001C015_2609147FR01.BIM +A001CNXB/Clip/A001C009_260914UQM01.XML +A001CNXB/Clip/A001C012_260914JV.MOV +A001CNXB/Clip/A001C008_2609148K.MOV +A001CNXB/Clip/A001C009_260914UQR01.BIM +A001CNXB/Clip/A001C003_260914IJ.MOV +A001CNXB/Clip/A001C010_260914IH.MOV +A001CNXB/Clip/A001C006_260914XTR01.BIM +A001CNXB/Clip/A001C005_260914K7.MOV +A001CNXB/Clip/A001C015_2609147FM01.XML +A001CNXB/Clip/A001C009_260914UQ.MOV +A001CNXB/Clip/A001C004_260914AKM01.XML +A001CNXB/Clip/A001C005_260914K7M01.XML +A001CNXB/Clip/A001C001_260914A8M01.XML +A001CNXB/Clip/A001C002_26091476.MOV +A001CNXB/Clip/A001C001_260914A8.MOV +A001CNXB/Clip/A001C003_260914IJR01.BIM +A001CNXB/Clip/A001C011_260914NWM01.XML +A001CNXB/Clip/A001C014_2609149V.MOV +A001CNXB/Clip/A001C014_2609149VR01.BIM +A001CNXB/Clip/A001C012_260914JVR01.BIM +A001CNXB/Clip/A001C007_26091490M01.XML +A001CNXB/Clip/A001C007_26091490.MOV +A001CNXB/Clip/A001C008_2609148KR01.BIM +A001CNXB/Clip/A001C010_260914IHM01.XML +A001CNXB/Clip/A001C013_2609140GM01.XML +A001CNXB/Clip/A001C002_26091476M01.XML +A001CNXB/DISCMETA.XML +A001CNXB/Edit +A001CNXB/Sub +A001CNXB/Take +A001CNXB/Thmbnl +A001CNXB/UserData +A001CNXB/CUEUP.XML +""".split(separator: "\n").map(String.init) + + @Test(arguments: [VerificationMode.quick, .standard, .thorough, .paranoid]) + func internalStorageReportedTree(mode: VerificationMode) async throws { + try await checkReportedTree(mode: mode, filesystem: "local") + } + + // hdiutil waits are intentionally opt-in so mounted-image setup does not starve + // short-deadline concurrency tests in the normal parallel Swift Testing run. + @Test(.enabled(if: ProcessInfo.processInfo.environment["BITMATCH_ISSUE8_MOUNTED"] == "1"), + arguments: [VerificationMode.quick, .standard, .thorough, .paranoid], ["ExFAT", "JHFS+"]) + func mountedSourceReportedTree(mode: VerificationMode, filesystem: String) async throws { + try await checkReportedTree(mode: mode, filesystem: filesystem) + } + + private func checkReportedTree(mode: VerificationMode, filesystem: String) async throws { + try await FileOperationsTestLock.shared.run { + let fm = FileManager.default + let root = fm.temporaryDirectory.appendingPathComponent("bitmatch-issue8-\(UUID().uuidString)") + try fm.createDirectory(at: root, withIntermediateDirectories: true) + let mount = root.appendingPathComponent("mnt") + var mounted = false + defer { + // Never remove the mount's contents if detach fails; retain the fixture for cleanup. + let detached = !mounted || Self.diskImage(["detach", "-quiet", mount.path]) == 0 + #expect(detached) + if detached { try? fm.removeItem(at: root) } + } + var sourceBase = root.appendingPathComponent("SOURCE") + if filesystem != "local" { + try fm.createDirectory(at: mount, withIntermediateDirectories: true) + let image = root.appendingPathComponent("source.sparseimage") + try #require(Self.diskImage(["create", "-quiet", "-size", "256m", "-type", "SPARSE", "-fs", filesystem, "-volname", "BMSONY8", "-o", image.path]) == 0) + try #require(Self.diskImage(["attach", "-quiet", "-nobrowse", "-mountpoint", mount.path, image.path]) == 0) + mounted = true + sourceBase = mount + } + let source = sourceBase.appendingPathComponent("A001CNXB") + let backup = root.appendingPathComponent("BACKUP") + let destination = backup.appendingPathComponent("A001CNXB") + try fm.createDirectory(at: source, withIntermediateDirectories: true) + try fm.createDirectory(at: backup, withIntermediateDirectories: true) + let directories: Set = ["General", "General/Sony", "General/Sony/Planning", "Clip", "Edit", "Sub", "Take", "Thmbnl", "UserData"] + var manifest: [String: Data] = [:] + for fullPath in Self.paths { + let relative = String(fullPath.dropFirst("A001CNXB/".count)) + let url = source.appendingPathComponent(relative) + if directories.contains(relative) { + try fm.createDirectory(at: url, withIntermediateDirectories: true) + } else { + try fm.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true) + let data = Data(("synthetic " + relative + "\n").utf8) + try data.write(to: url) + manifest[relative] = data + } + } + #expect(manifest.count == 49) // 15 MOV, 15 BIM, 15 XML, three root XML, .DS_Store + let pipeline = TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared) + let initialEntries = try CardSource.enumerateRegularFiles(base: source) + let initialContents = try Dictionary(uniqueKeysWithValues: initialEntries.map { + ($0.relativePath, try Data(contentsOf: $0.url)) + }) + for attempt in 0..<2 { + let operation = try await pipeline.performFileOperation( + sourceURL: source, destinationURLs: [backup], verificationMode: mode, + settings: CameraLabelSettings(), estimatedTotalBytes: nil, + progressCallback: { _ in }, onFileResult: { _ in }) + #expect(operation.results.count == initialContents.count) + if mode == .quick && attempt == 1 { + // Quick cannot prove existing bytes: refusal is intentional, never overwrite. + #expect(operation.results.allSatisfy { $0.outcome == .failed && $0.error != nil }) + } else { + #expect(operation.results.allSatisfy { mode == .quick ? $0.outcome == .copiedUnverified : $0.outcome == .verified }) + } + let current = try Dictionary(uniqueKeysWithValues: CardSource.enumerateRegularFiles(base: source).map { + ($0.relativePath, try Data(contentsOf: $0.url)) + }) + #expect(current == initialContents) + for (name, bytes) in initialContents { + #expect(try Data(contentsOf: source.appendingPathComponent(name)) == bytes) + #expect(try Data(contentsOf: destination.appendingPathComponent(name)) == bytes) + } + for name in directories { + var isDirectory: ObjCBool = false + #expect(fm.fileExists(atPath: destination.appendingPathComponent(name).path, isDirectory: &isDirectory)) + #expect(isDirectory.boolValue) + } + } + let comparer = FolderComparer(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + func compare() async throws -> CompareStats { + try await comparer.compare(left: source, right: destination, verificationMode: mode, progress: { _ in }) + } + var stats = try await compare() + #expect(stats.onlyInLeftCount == 0 && stats.onlyInRightCount == 0 && stats.mismatchedCount == 0) + // Plant: remove destination-only metadata filtering in FolderComparer; this must fail. + try Data("Finder view state".utf8).write(to: destination.appendingPathComponent("Clip/.DS_Store")) + try Data("Finder companion".utf8).write(to: destination.appendingPathComponent("Clip/._A001C001_260914A8.MOV")) + stats = try await compare() + #expect(stats.onlyInRightCount == 0 && stats.mismatchedCount == 0) + try Data("real extra".utf8).write(to: destination.appendingPathComponent("Clip/EXTRA.BIM")) + stats = try await compare() + #expect(stats.onlyInRightPaths == ["Clip/EXTRA.BIM"]) + try fm.removeItem(at: destination.appendingPathComponent("Clip/EXTRA.BIM")) + let clip = "Clip/A001C001_260914A8.MOV" + var damaged = try #require(manifest[clip]); damaged[0] ^= 1 + try damaged.write(to: destination.appendingPathComponent(clip)) + stats = try await compare() + #expect(stats.mismatchedCount == (mode == .quick ? 0 : 1)) + #expect(try Data(contentsOf: source.appendingPathComponent(clip)) == manifest[clip]) + } + } +} diff --git a/docs/ISSUE8_SONY_VENICE.md b/docs/ISSUE8_SONY_VENICE.md new file mode 100644 index 00000000..6a12e24a --- /dev/null +++ b/docs/ISSUE8_SONY_VENICE.md @@ -0,0 +1,31 @@ +# Sony Venice issue #8 investigation + +2026-10-05, based on main e0753b08 (0.2.4). + +The reporter originally used 0.1.4 and reported “1 only in destination.” Their October 2 comment says both Copy & Verify and comparison failed, but does not identify the retested version or the exact Copy & Verify error. The card was readable without extra Sony software. Their private metadata folder needs approval; a Viewer request was sent and a follow-up posted. The actual Sony XML/BIM contents have not been inspected. + +## Synthetic reproduction + +`SonyVeniceIssue8Tests` uses all 58 reported paths: 49 files (15 MOV, 15 BIM, 18 XML and .DS_Store) and nine directories, including empty directories. Contents are deliberately synthetic; these are not valid camera media or original Sony metadata. + +The matrix covers Quick, Standard, Thorough and Paranoid with local internal-storage, mounted exFAT and mounted HFS+ sources, copying into internal storage. It checks two transfers, unchanged source file names/bytes, identical destination bytes, empty folders and folder comparison. macOS-generated AppleDouble files on exFAT are included in the baseline, rather than assumed absent. Quick refuses the second transfer by design: it cannot prove an existing file matches and must not overwrite it. + +Comparison cases cover a clean copy, extra destination-only Finder .DS_Store/AppleDouble metadata, a real extra BIM file, and same-size corruption of a MOV. Real extras remain differences; Standard/Thorough/Paranoid detect corruption. Quick checks sizes only. + +The first harness mistakenly checked the selected backup root rather than its card subfolder, assumed 49 total files even when macOS created exFAT companions, and assumed Quick could reuse existing files. These were harness errors and were corrected before relying on the matrix. An intermediate enum spelling error also prevented compilation; it is not a product finding. + +## Interpretation + +The 12-case matrix passes on current code. Removing only destination-only Finder metadata filtering makes the tests fail, then restoring it passes. This confirms the tests exercise that protection. The 0.1.4 ComparisonCoordinator used an unfiltered destination-minus-source set, consistent with the original extra-Finder-file report. This does not establish the cause of the later Copy & Verify failure. + +No production behavior changed during this investigation. No release is warranted from these synthetic results alone. Await the original metadata, retested version and exact error. These tests do not exercise a real Sony AXSM/SxS reader, UDF media, permission failures, large actual MOV containers or the signed-app picker/UI. + +Validation logs and access-request receipt are retained locally under `dist/validation/issue8/`. Run the focused matrix with: + +```sh +BITMATCH_ISSUE8_MOUNTED=1 arch -arm64 swift test --package-path Packages/BitMatchEngine --build-system native --filter SonyVeniceIssue8Tests +``` + +Mounted-image cases are opt-in to avoid blocking setup on the cooperative executor during the normal parallel suite. The final focused matrix (two parameterized tests, 12 cases) passed in 6.811 seconds. An earlier full run passed all 110 XCTest cases (15 opt-in skips) and the Sony matrix but missed timing deadlines in `cancellationMidBroadcastPublishesNothingAndReportsNoSuccess` and `testStalePauseDoesNotBlockNextOperation`; those are recorded separately from the Sony outcomes. The final full-suite rerun is recorded below. + +Final full engine rerun: 110 XCTest cases, 15 skipped, zero failures (64.068 seconds); 129 Swift Testing tests in 29 suites, zero failures (8.181 seconds), including the four local Sony cases. Mounted Sony cases were disabled here and passed separately in the explicit 12-case run. No Mac app/UI or physical-reader validation was performed in this batch, which changes tests and documentation only. From 540d5726711a3e8f32a413c27c118c6856a53d03 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 5 Oct 2026 11:19:38 -0500 Subject: [PATCH 2/2] Validate private Sony metadata with opt-in transfer regressions --- .../SonyVeniceIssue8Tests.swift | 35 ++++++++++++++++--- docs/ISSUE8_SONY_VENICE.md | 16 +++++++-- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SonyVeniceIssue8Tests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SonyVeniceIssue8Tests.swift index 144e28f7..eb553c98 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SonyVeniceIssue8Tests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SonyVeniceIssue8Tests.swift @@ -86,7 +86,26 @@ A001CNXB/CUEUP.XML try await checkReportedTree(mode: mode, filesystem: filesystem) } - private func checkReportedTree(mode: VerificationMode, filesystem: String) async throws { + // Private reporter metadata stays outside the repository. Only an explicit local + // path enables these cases; MOV bytes, when requested, remain synthetic. + @Test(.enabled(if: ProcessInfo.processInfo.environment["BITMATCH_ISSUE8_METADATA"] != nil), + arguments: [VerificationMode.quick, .standard, .thorough, .paranoid], [false, true]) + func reporterMetadata(mode: VerificationMode, includeSyntheticMovies: Bool) async throws { + let path = try #require(ProcessInfo.processInfo.environment["BITMATCH_ISSUE8_METADATA"]) + try await checkReportedTree(mode: mode, filesystem: "local", + metadata: URL(fileURLWithPath: path), includeMovies: includeSyntheticMovies) + } + + @Test(.enabled(if: ProcessInfo.processInfo.environment["BITMATCH_ISSUE8_METADATA"] != nil), + arguments: ["ExFAT", "JHFS+"], [false, true]) + func reporterMetadataMounted(filesystem: String, includeSyntheticMovies: Bool) async throws { + let path = try #require(ProcessInfo.processInfo.environment["BITMATCH_ISSUE8_METADATA"]) + try await checkReportedTree(mode: .standard, filesystem: filesystem, + metadata: URL(fileURLWithPath: path), includeMovies: includeSyntheticMovies) + } + + private func checkReportedTree(mode: VerificationMode, filesystem: String, + metadata: URL? = nil, includeMovies: Bool = true) async throws { try await FileOperationsTestLock.shared.run { let fm = FileManager.default let root = fm.temporaryDirectory.appendingPathComponent("bitmatch-issue8-\(UUID().uuidString)") @@ -122,12 +141,20 @@ A001CNXB/CUEUP.XML try fm.createDirectory(at: url, withIntermediateDirectories: true) } else { try fm.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true) - let data = Data(("synthetic " + relative + "\n").utf8) + let data: Data + if let metadata { + let supplied = metadata.appendingPathComponent(relative) + if fm.fileExists(atPath: supplied.path) { + data = try Data(contentsOf: supplied) + } else if includeMovies && url.pathExtension == "MOV" { + data = Data(("synthetic " + relative + "\n").utf8) + } else { continue } + } else { data = Data(("synthetic " + relative + "\n").utf8) } try data.write(to: url) manifest[relative] = data } } - #expect(manifest.count == 49) // 15 MOV, 15 BIM, 15 XML, three root XML, .DS_Store + #expect(manifest.count == (metadata == nil ? 49 : (includeMovies ? 48 : 33))) let pipeline = TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared) let initialEntries = try CardSource.enumerateRegularFiles(base: source) let initialContents = try Dictionary(uniqueKeysWithValues: initialEntries.map { @@ -174,7 +201,7 @@ A001CNXB/CUEUP.XML stats = try await compare() #expect(stats.onlyInRightPaths == ["Clip/EXTRA.BIM"]) try fm.removeItem(at: destination.appendingPathComponent("Clip/EXTRA.BIM")) - let clip = "Clip/A001C001_260914A8.MOV" + let clip = includeMovies ? "Clip/A001C001_260914A8.MOV" : "Clip/A001C001_260914A8R01.BIM" var damaged = try #require(manifest[clip]); damaged[0] ^= 1 try damaged.write(to: destination.appendingPathComponent(clip)) stats = try await compare() diff --git a/docs/ISSUE8_SONY_VENICE.md b/docs/ISSUE8_SONY_VENICE.md index 6a12e24a..d725d20a 100644 --- a/docs/ISSUE8_SONY_VENICE.md +++ b/docs/ISSUE8_SONY_VENICE.md @@ -2,7 +2,7 @@ 2026-10-05, based on main e0753b08 (0.2.4). -The reporter originally used 0.1.4 and reported “1 only in destination.” Their October 2 comment says both Copy & Verify and comparison failed, but does not identify the retested version or the exact Copy & Verify error. The card was readable without extra Sony software. Their private metadata folder needs approval; a Viewer request was sent and a follow-up posted. The actual Sony XML/BIM contents have not been inspected. +The reporter originally used 0.1.4 and reported “1 only in destination.” Their October 2 comment says both Copy & Verify and comparison failed, but does not identify the retested version or the exact Copy & Verify error. The card was readable without extra Sony software. Viewer access was granted after the request and follow-up. The archive contains 18 XML and 15 BIM files (107,664,847 bytes), no MOV footage. All XML files parse successfully; model metadata identifies VENICE (MPC-3610). Original files remain private and outside the committed repository. ## Synthetic reproduction @@ -18,7 +18,7 @@ The first harness mistakenly checked the selected backup root rather than its ca The 12-case matrix passes on current code. Removing only destination-only Finder metadata filtering makes the tests fail, then restoring it passes. This confirms the tests exercise that protection. The 0.1.4 ComparisonCoordinator used an unfiltered destination-minus-source set, consistent with the original extra-Finder-file report. This does not establish the cause of the later Copy & Verify failure. -No production behavior changed during this investigation. No release is warranted from these synthetic results alone. Await the original metadata, retested version and exact error. These tests do not exercise a real Sony AXSM/SxS reader, UDF media, permission failures, large actual MOV containers or the signed-app picker/UI. +No production behavior changed during this investigation. No release is warranted from these synthetic results alone. Await the retested version and exact error; original metadata is now available and checked. These tests do not exercise a real Sony AXSM/SxS reader, UDF media, permission failures, large actual MOV containers or the signed-app picker/UI. Validation logs and access-request receipt are retained locally under `dist/validation/issue8/`. Run the focused matrix with: @@ -29,3 +29,15 @@ BITMATCH_ISSUE8_MOUNTED=1 arch -arm64 swift test --package-path Packages/BitMatc Mounted-image cases are opt-in to avoid blocking setup on the cooperative executor during the normal parallel suite. The final focused matrix (two parameterized tests, 12 cases) passed in 6.811 seconds. An earlier full run passed all 110 XCTest cases (15 opt-in skips) and the Sony matrix but missed timing deadlines in `cancellationMidBroadcastPublishesNothingAndReportsNoSuccess` and `testStalePauseDoesNotBlockNextOperation`; those are recorded separately from the Sony outcomes. The final full-suite rerun is recorded below. Final full engine rerun: 110 XCTest cases, 15 skipped, zero failures (64.068 seconds); 129 Swift Testing tests in 29 suites, zero failures (8.181 seconds), including the four local Sony cases. Mounted Sony cases were disabled here and passed separately in the explicit 12-case run. No Mac app/UI or physical-reader validation was performed in this batch, which changes tests and documentation only. + +## Original metadata follow-up + +The opt-in `BITMATCH_ISSUE8_METADATA` path points to a local private card folder. Tests read the supplied XML/BIM bytes into disposable fixtures; they never edit the original folder or embed it in the repository. Cases cover metadata alone and metadata with 15 synthetic MOV placeholders. All four modes run against internal storage; Standard additionally runs from mounted exFAT and HFS+ sources. They repeat the transfer, check every original source byte remains unchanged, compare folders, ignore destination-only Finder metadata, detect real extra files and detect same-size corruption (BIM when no MOV placeholders are present). Quick remains size-only and refuses repeat copies. + +The first metadata run passed all eight metadata cases and four existing local synthetic cases in 11.502 seconds. The expanded mounted-metadata run is recorded below. There is no evidence here that Sony metadata causes the reported copy/comparison failure. No actual footage bytes, signed-app UI or physical AXSM/SxS/UDF reader are exercised. + +```sh +BITMATCH_ISSUE8_METADATA=/private/local/card-folder arch -arm64 swift test --package-path Packages/BitMatchEngine --build-system native --filter SonyVeniceIssue8Tests +``` + +Expanded original-metadata run: 12 metadata cases (eight mode/content combinations on internal storage, four Standard/content/filesystem combinations from exFAT and HFS+) plus four synthetic internal-storage controls passed in 22.588 seconds, zero failures. Independent SHA-256 checks confirmed all 33 original downloaded files remained unchanged. The earlier full engine suite passed; this follow-up changes only opt-in tests and notes, and does not claim a new full app/UI validation.