From 8ebefb54779e87a2090c3da30a833698528c8f3a Mon Sep 17 00:00:00 2001 From: Leonardo Constanski Date: Tue, 3 Nov 2026 17:33:48 -0300 Subject: [PATCH 1/2] Count blocks shared by APFS clones once st_blocks reports every block a clone shares with its siblings, so a file cloned N times was counted N times in allocated size. On my Mac, cloned files in WhatsApp's container pushed the reported total above the capacity of the 1 TB drive. getattrlistbulk now also requests ATTR_CMNEXT_PRIVATESIZE, CLONEID and CLONE_REFCNT. Each clone keeps its private blocks; the shared blocks are charged to the first member of its clone group seen in the scan. Logical size is unchanged and still reports every copy in full. Co-Authored-By: Claude Opus 5.5 --- .../Scanning/BulkDirectoryReader.swift | 23 +++++++++++++++- Sources/MacDirStat/Scanning/FileScanner.swift | 27 +++++++++++++------ Tests/MacDirStatTests/FileScannerTests.swift | 27 +++++++++++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/Sources/MacDirStat/Scanning/BulkDirectoryReader.swift b/Sources/MacDirStat/Scanning/BulkDirectoryReader.swift index 30c31ce..48654c1 100644 --- a/Sources/MacDirStat/Scanning/BulkDirectoryReader.swift +++ b/Sources/MacDirStat/Scanning/BulkDirectoryReader.swift @@ -8,6 +8,14 @@ enum DirectoryEnumerationMode: Sendable { struct BulkDirectoryEntry { let name: String let metadata: Darwin.stat + var clone: CloneInfo? = nil +} + +/// An APFS clone shares its blocks with other files. st_blocks reports every +/// shared block for each copy, so the scanner counts the shared part once. +struct CloneInfo: Sendable { + let id: UInt64 + let privateBytes: Int64 } enum BulkDirectoryBatch { @@ -32,12 +40,14 @@ struct BulkDirectoryReader { UInt32(ATTR_CMN_FLAGS), UInt32(ATTR_CMN_FILEID)] attributes.commonattr = common.reduce(0, |) attributes.fileattr = UInt32(ATTR_FILE_ALLOCSIZE | ATTR_FILE_DATALENGTH) + // With FSOPT_ATTR_CMN_EXTENDED, forkattr selects extended common attributes. + attributes.forkattr = UInt32(ATTR_CMNEXT_PRIVATESIZE | ATTR_CMNEXT_CLONEID | ATTR_CMNEXT_CLONE_REFCNT) } mutating func next(fd: Int32) throws -> BulkDirectoryBatch { let count = buffer.withUnsafeMutableBytes { getattrlistbulk(fd, &attributes, $0.baseAddress!, $0.count, - UInt64(FSOPT_PACK_INVAL_ATTRS | FSOPT_RETURN_REALDEV)) + UInt64(FSOPT_PACK_INVAL_ATTRS | FSOPT_RETURN_REALDEV | FSOPT_ATTR_CMN_EXTENDED)) } if count < 0 { let code = errno @@ -125,6 +135,17 @@ func decodeBulkDirectoryEntry(_ record: UnsafeRawBufferPointer, fd: Int32) throw metadata.st_flags = flags metadata.st_blocks = allocated / 512 metadata.st_size = logical + // Packed after the file attributes; zero-filled when not returned. + let privateSize = try read(off_t.self) + let cloneID = try read(UInt64.self) + let cloneCount = try read(UInt32.self) + guard start >= offset else { throw BulkMetadataError.malformed } + let cloneFields = UInt32(ATTR_CMNEXT_PRIVATESIZE | ATTR_CMNEXT_CLONEID | ATTR_CMNEXT_CLONE_REFCNT) + if returned.forkattr & cloneFields == cloneFields, cloneCount > 1, + privateSize >= 0, privateSize <= allocated { + return BulkDirectoryEntry(name: String(cString: name), metadata: metadata, + clone: CloneInfo(id: cloneID, privateBytes: privateSize)) + } } else { // Directory attributes describe the underlying mount/firmlink, not its // target. fstatat preserves stat's volume, inode and directory-size rules. diff --git a/Sources/MacDirStat/Scanning/FileScanner.swift b/Sources/MacDirStat/Scanning/FileScanner.swift index 663f27b..cb0f1a2 100644 --- a/Sources/MacDirStat/Scanning/FileScanner.swift +++ b/Sources/MacDirStat/Scanning/FileScanner.swift @@ -54,6 +54,7 @@ private final class ScanState: Sendable { var fileCount = 0 var byteCount: Int64 = 0 var seenInodes = Set() + var seenClones = Set() var lastProgress = ContinuousClock.now } @@ -61,12 +62,21 @@ private final class ScanState: Sendable { self.rootDevice = rootDevice } - func record(_ metadata: Darwin.stat) -> Bool { + /// Returns the metadata to store, or nil for an inode already counted. + /// A clone keeps its private blocks; the blocks it shares with other clones + /// are charged once, to the first member of its clone group seen in the scan. + func record(_ metadata: Darwin.stat, clone: CloneInfo? = nil) -> Darwin.stat? { lock.withLock { state in - guard state.seenInodes.insert(UInt64(metadata.st_ino)).inserted else { return false } - if metadata.st_mode & S_IFMT == S_IFREG { state.fileCount += 1 } - state.byteCount += Int64(metadata.st_blocks) * 512 - return true + guard state.seenInodes.insert(UInt64(metadata.st_ino)).inserted else { return nil } + var counted = metadata + if let clone { + let allocated = Int64(metadata.st_blocks) * 512 + let shared = state.seenClones.insert(clone.id).inserted ? allocated - clone.privateBytes : 0 + counted.st_blocks = blkcnt_t((clone.privateBytes + shared + 511) / 512) + } + if counted.st_mode & S_IFMT == S_IFREG { state.fileCount += 1 } + state.byteCount += Int64(counted.st_blocks) * 512 + return counted } } @@ -186,10 +196,11 @@ private func scanDirectory( return result } - func append(name: String, metadata: Darwin.stat) { + func append(name: String, metadata: Darwin.stat, clone: CloneInfo? = nil) { guard metadata.st_dev == state.rootDevice else { return } let type = metadata.st_mode & S_IFMT - guard type == S_IFDIR || type == S_IFREG, state.record(metadata) else { return } + guard type == S_IFDIR || type == S_IFREG, + let metadata = state.record(metadata, clone: clone) else { return } let child = FileNode(name: name, metadata: metadata) result.children.append(child) if child.isDirectory { @@ -205,7 +216,7 @@ private func scanDirectory( switch try reader.next(fd: fd) { case .entries(let entries, let unavailable): result.unavailableEntries += unavailable - for entry in entries { append(name: entry.name, metadata: entry.metadata) } + for entry in entries { append(name: entry.name, metadata: entry.metadata, clone: entry.clone) } if let progress = state.progress(path: job.path) { continuation.yield(progress) } case .end: return result case .failure(let code): diff --git a/Tests/MacDirStatTests/FileScannerTests.swift b/Tests/MacDirStatTests/FileScannerTests.swift index bd6d8af..6d45ea5 100644 --- a/Tests/MacDirStatTests/FileScannerTests.swift +++ b/Tests/MacDirStatTests/FileScannerTests.swift @@ -323,3 +323,30 @@ func localIntegrationScan() async throws { #expect(bytes == root.totalAllocatedSize) print("Scan summary: files=\(root.fileCount), directories=\(root.directoryCount), logicalBytes=\(root.totalSize), allocatedBytes=\(bytes), cloudOnlyFiles=\(root.cloudOnlyFileCount), unscannedDirectories=\(root.unscannedDirectoryCount), unavailableEntries=\(root.totalUnavailableEntryCount), elapsed=\(ContinuousClock.now - started)") } + +@Test func apfsClonesShareTheirBlocksOnce() async throws { + let fixture = try Fixture() + defer { fixture.remove() } + let original = fixture.root.appendingPathComponent("original.bin") + try Data((0..<1_048_576).map { UInt8(truncatingIfNeeded: $0 &* 31) }).write(to: original) + let copyBytes = Int64(try fixture.metadata(original).st_blocks) * 512 + for folder in ["a", "b", "c"] { + let directory = fixture.root.appendingPathComponent(folder) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + for index in 0..<3 { + let clone = directory.appendingPathComponent("clone-\(index).bin") + guard clonefile(original.path, clone.path, 0) == 0 else { + return // The temporary volume does not support clones (not APFS). + } + } + } + let (root, bytes) = try await completedScan(fixture.root.path) + func fileBytes(_ node: FileNode, _ size: (FileNode) -> Int64) -> Int64 { + node.isDirectory ? node.children.reduce(0) { $0 + fileBytes($1, size) } : size(node) + } + #expect(root.fileCount == 10) + #expect(fileBytes(root, \.allocatedSize) == copyBytes) + // Logical size still reports every copy in full. + #expect(fileBytes(root, \.ownSize) == 10 * 1_048_576) + #expect(bytes == root.totalAllocatedSize) +} From dab86295ec09405e24876e660505ba36b8658d16 Mon Sep 17 00:00:00 2001 From: Leonardo Constanski Date: Fri, 6 Nov 2026 09:50:29 -0300 Subject: [PATCH 2/2] Drop ATTR_CMNEXT_PRIVATESIZE from the clone accounting CLONE_REFCNT counts full clones, so a file with a refcount above one has no private blocks. Charging the whole allocated size to the first member of each clone group gives the same total without the ~0.5 s PRIVATESIZE costs on /Applications. The enumeration benchmark no longer compares allocated size, since the POSIX path can't see clones. Co-Authored-By: Claude Opus 5.5 --- .../Scanning/BulkDirectoryReader.swift | 23 +++++++------------ Sources/MacDirStat/Scanning/FileScanner.swift | 17 +++++--------- Tests/MacDirStatTests/FileScannerTests.swift | 5 ++-- 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/Sources/MacDirStat/Scanning/BulkDirectoryReader.swift b/Sources/MacDirStat/Scanning/BulkDirectoryReader.swift index 48654c1..a70f1b1 100644 --- a/Sources/MacDirStat/Scanning/BulkDirectoryReader.swift +++ b/Sources/MacDirStat/Scanning/BulkDirectoryReader.swift @@ -8,14 +8,9 @@ enum DirectoryEnumerationMode: Sendable { struct BulkDirectoryEntry { let name: String let metadata: Darwin.stat - var clone: CloneInfo? = nil -} - -/// An APFS clone shares its blocks with other files. st_blocks reports every -/// shared block for each copy, so the scanner counts the shared part once. -struct CloneInfo: Sendable { - let id: UInt64 - let privateBytes: Int64 + /// Set for a full APFS clone (refcount above one). Its st_blocks are all + /// shared with the other members of the group, so the scanner counts them once. + var cloneID: UInt64? = nil } enum BulkDirectoryBatch { @@ -41,7 +36,7 @@ struct BulkDirectoryReader { attributes.commonattr = common.reduce(0, |) attributes.fileattr = UInt32(ATTR_FILE_ALLOCSIZE | ATTR_FILE_DATALENGTH) // With FSOPT_ATTR_CMN_EXTENDED, forkattr selects extended common attributes. - attributes.forkattr = UInt32(ATTR_CMNEXT_PRIVATESIZE | ATTR_CMNEXT_CLONEID | ATTR_CMNEXT_CLONE_REFCNT) + attributes.forkattr = UInt32(ATTR_CMNEXT_CLONEID | ATTR_CMNEXT_CLONE_REFCNT) } mutating func next(fd: Int32) throws -> BulkDirectoryBatch { @@ -136,15 +131,13 @@ func decodeBulkDirectoryEntry(_ record: UnsafeRawBufferPointer, fd: Int32) throw metadata.st_blocks = allocated / 512 metadata.st_size = logical // Packed after the file attributes; zero-filled when not returned. - let privateSize = try read(off_t.self) + // CLONE_REFCNT counts full clones, so above one the file has no private blocks. let cloneID = try read(UInt64.self) let cloneCount = try read(UInt32.self) guard start >= offset else { throw BulkMetadataError.malformed } - let cloneFields = UInt32(ATTR_CMNEXT_PRIVATESIZE | ATTR_CMNEXT_CLONEID | ATTR_CMNEXT_CLONE_REFCNT) - if returned.forkattr & cloneFields == cloneFields, cloneCount > 1, - privateSize >= 0, privateSize <= allocated { - return BulkDirectoryEntry(name: String(cString: name), metadata: metadata, - clone: CloneInfo(id: cloneID, privateBytes: privateSize)) + let cloneFields = UInt32(ATTR_CMNEXT_CLONEID | ATTR_CMNEXT_CLONE_REFCNT) + if returned.forkattr & cloneFields == cloneFields, cloneCount > 1 { + return BulkDirectoryEntry(name: String(cString: name), metadata: metadata, cloneID: cloneID) } } else { // Directory attributes describe the underlying mount/firmlink, not its diff --git a/Sources/MacDirStat/Scanning/FileScanner.swift b/Sources/MacDirStat/Scanning/FileScanner.swift index cb0f1a2..d3ac41b 100644 --- a/Sources/MacDirStat/Scanning/FileScanner.swift +++ b/Sources/MacDirStat/Scanning/FileScanner.swift @@ -63,17 +63,12 @@ private final class ScanState: Sendable { } /// Returns the metadata to store, or nil for an inode already counted. - /// A clone keeps its private blocks; the blocks it shares with other clones - /// are charged once, to the first member of its clone group seen in the scan. - func record(_ metadata: Darwin.stat, clone: CloneInfo? = nil) -> Darwin.stat? { + /// A clone group's blocks are charged once, to the first member seen in the scan. + func record(_ metadata: Darwin.stat, cloneID: UInt64? = nil) -> Darwin.stat? { lock.withLock { state in guard state.seenInodes.insert(UInt64(metadata.st_ino)).inserted else { return nil } var counted = metadata - if let clone { - let allocated = Int64(metadata.st_blocks) * 512 - let shared = state.seenClones.insert(clone.id).inserted ? allocated - clone.privateBytes : 0 - counted.st_blocks = blkcnt_t((clone.privateBytes + shared + 511) / 512) - } + if let cloneID, !state.seenClones.insert(cloneID).inserted { counted.st_blocks = 0 } if counted.st_mode & S_IFMT == S_IFREG { state.fileCount += 1 } state.byteCount += Int64(counted.st_blocks) * 512 return counted @@ -196,11 +191,11 @@ private func scanDirectory( return result } - func append(name: String, metadata: Darwin.stat, clone: CloneInfo? = nil) { + func append(name: String, metadata: Darwin.stat, cloneID: UInt64? = nil) { guard metadata.st_dev == state.rootDevice else { return } let type = metadata.st_mode & S_IFMT guard type == S_IFDIR || type == S_IFREG, - let metadata = state.record(metadata, clone: clone) else { return } + let metadata = state.record(metadata, cloneID: cloneID) else { return } let child = FileNode(name: name, metadata: metadata) result.children.append(child) if child.isDirectory { @@ -216,7 +211,7 @@ private func scanDirectory( switch try reader.next(fd: fd) { case .entries(let entries, let unavailable): result.unavailableEntries += unavailable - for entry in entries { append(name: entry.name, metadata: entry.metadata, clone: entry.clone) } + for entry in entries { append(name: entry.name, metadata: entry.metadata, cloneID: entry.cloneID) } if let progress = state.progress(path: job.path) { continuation.yield(progress) } case .end: return result case .failure(let code): diff --git a/Tests/MacDirStatTests/FileScannerTests.swift b/Tests/MacDirStatTests/FileScannerTests.swift index 6d45ea5..1a55b41 100644 --- a/Tests/MacDirStatTests/FileScannerTests.swift +++ b/Tests/MacDirStatTests/FileScannerTests.swift @@ -109,13 +109,14 @@ private func completedScan(_ path: String, limit: Int = 4, @Test(.enabled(if: ProcessInfo.processInfo.environment["MACDIRSTAT_BENCHMARK_PATH"] != nil)) func compareDirectoryEnumerationPerformance() async throws { let path = try #require(ProcessInfo.processInfo.environment["MACDIRSTAT_BENCHMARK_PATH"]) - var reference: (files: Int, directories: Int, logical: Int64, allocated: Int64, cloud: Int, incomplete: Int)? + // Allocated size is left out: only the bulk reader sees APFS clones, so .posix counts them in full. + var reference: (files: Int, directories: Int, logical: Int64, cloud: Int, incomplete: Int)? // Alternate order so both implementations see warm and cold metadata caches. for mode in [DirectoryEnumerationMode.posix, .automatic, .automatic, .posix] { let started = ContinuousClock.now let (root, _) = try await completedScan(path, mode: mode) let elapsed = ContinuousClock.now - started - let totals = (root.fileCount, root.directoryCount, root.totalSize, root.totalAllocatedSize, + let totals = (root.fileCount, root.directoryCount, root.totalSize, root.cloudOnlyFileCount, root.unscannedDirectoryCount) if let reference { #expect(totals == reference)