Count blocks shared by APFS clones once

Sources/MacDirStat/Scanning/BulkDirectoryReader.swift

@@ -8,6 +8,9 @@ enum DirectoryEnumerationMode: Sendable {
 struct BulkDirectoryEntry {
     let name: String
     let metadata: Darwin.stat
+    /// Set for a full APFS clone (refcount above one). Its st_blocks are all
+    /// shared with the other members of the group, so the scanner counts them once.
+    var cloneID: UInt64? = nil
 }
 
 enum BulkDirectoryBatch {
@@ -32,12 +35,14 @@ struct BulkDirectoryReader {
                                 UInt32(ATTR_CMN_FLAGS), UInt32(ATTR_CMN_FILEID)]
         attributes.commonattr = common.reduce(0, |)
         attributes.fileattr = UInt32(ATTR_FILE_ALLOCSIZE | ATTR_FILE_DATALENGTH)
+        // With FSOPT_ATTR_CMN_EXTENDED, forkattr selects extended common attributes.
+        attributes.forkattr = UInt32(ATTR_CMNEXT_CLONEID | ATTR_CMNEXT_CLONE_REFCNT)
     }
 
     mutating func next(fd: Int32) throws -> BulkDirectoryBatch {
         let count = buffer.withUnsafeMutableBytes {
             getattrlistbulk(fd, &attributes, $0.baseAddress!, $0.count,
-                            UInt64(FSOPT_PACK_INVAL_ATTRS | FSOPT_RETURN_REALDEV))
+                            UInt64(FSOPT_PACK_INVAL_ATTRS | FSOPT_RETURN_REALDEV | FSOPT_ATTR_CMN_EXTENDED))
         }
         if count < 0 {
             let code = errno
@@ -125,6 +130,15 @@ func decodeBulkDirectoryEntry(_ record: UnsafeRawBufferPointer, fd: Int32) throw
         metadata.st_flags = flags
         metadata.st_blocks = allocated / 512
         metadata.st_size = logical
+        // Packed after the file attributes; zero-filled when not returned.
+        // CLONE_REFCNT counts full clones, so above one the file has no private blocks.
+        let cloneID = try read(UInt64.self)
+        let cloneCount = try read(UInt32.self)
+        guard start >= offset else { throw BulkMetadataError.malformed }
+        let cloneFields = UInt32(ATTR_CMNEXT_CLONEID | ATTR_CMNEXT_CLONE_REFCNT)
+        if returned.forkattr & cloneFields == cloneFields, cloneCount > 1 {
+            return BulkDirectoryEntry(name: String(cString: name), metadata: metadata, cloneID: cloneID)
+        }
     } else {
         // Directory attributes describe the underlying mount/firmlink, not its
         // target. fstatat preserves stat's volume, inode and directory-size rules.

Sources/MacDirStat/Scanning/FileScanner.swift

@@ -54,19 +54,24 @@ private final class ScanState: Sendable {
         var fileCount = 0
         var byteCount: Int64 = 0
         var seenInodes = Set<UInt64>()
+        var seenClones = Set<UInt64>()
         var lastProgress = ContinuousClock.now
     }
 
     init(rootDevice: dev_t) {
         self.rootDevice = rootDevice
     }
 
-    func record(_ metadata: Darwin.stat) -> Bool {
+    /// Returns the metadata to store, or nil for an inode already counted.
+    /// A clone group's blocks are charged once, to the first member seen in the scan.
+    func record(_ metadata: Darwin.stat, cloneID: UInt64? = nil) -> Darwin.stat? {
         lock.withLock { state in
-            guard state.seenInodes.insert(UInt64(metadata.st_ino)).inserted else { return false }
-            if metadata.st_mode & S_IFMT == S_IFREG { state.fileCount += 1 }
-            state.byteCount += Int64(metadata.st_blocks) * 512
-            return true
+            guard state.seenInodes.insert(UInt64(metadata.st_ino)).inserted else { return nil }
+            var counted = metadata
+            if let cloneID, !state.seenClones.insert(cloneID).inserted { counted.st_blocks = 0 }
+            if counted.st_mode & S_IFMT == S_IFREG { state.fileCount += 1 }
+            state.byteCount += Int64(counted.st_blocks) * 512
+            return counted
         }
     }
 
@@ -186,10 +191,11 @@ private func scanDirectory(
             return result
         }
 
-        func append(name: String, metadata: Darwin.stat) {
+        func append(name: String, metadata: Darwin.stat, cloneID: UInt64? = nil) {
             guard metadata.st_dev == state.rootDevice else { return }
             let type = metadata.st_mode & S_IFMT
-            guard type == S_IFDIR || type == S_IFREG, state.record(metadata) else { return }
+            guard type == S_IFDIR || type == S_IFREG,
+                  let metadata = state.record(metadata, cloneID: cloneID) else { return }
             let child = FileNode(name: name, metadata: metadata)
             result.children.append(child)
             if child.isDirectory {
@@ -205,7 +211,7 @@ private func scanDirectory(
                 switch try reader.next(fd: fd) {
                 case .entries(let entries, let unavailable):
                     result.unavailableEntries += unavailable
-                    for entry in entries { append(name: entry.name, metadata: entry.metadata) }
+                    for entry in entries { append(name: entry.name, metadata: entry.metadata, cloneID: entry.cloneID) }
                     if let progress = state.progress(path: job.path) { continuation.yield(progress) }
                 case .end: return result
                 case .failure(let code):

Tests/MacDirStatTests/FileScannerTests.swift

@@ -109,13 +109,14 @@ private func completedScan(_ path: String, limit: Int = 4,
 @Test(.enabled(if: ProcessInfo.processInfo.environment["MACDIRSTAT_BENCHMARK_PATH"] != nil))
 func compareDirectoryEnumerationPerformance() async throws {
     let path = try #require(ProcessInfo.processInfo.environment["MACDIRSTAT_BENCHMARK_PATH"])
-    var reference: (files: Int, directories: Int, logical: Int64, allocated: Int64, cloud: Int, incomplete: Int)?
+    // Allocated size is left out: only the bulk reader sees APFS clones, so .posix counts them in full.
+    var reference: (files: Int, directories: Int, logical: Int64, cloud: Int, incomplete: Int)?
     // Alternate order so both implementations see warm and cold metadata caches.
     for mode in [DirectoryEnumerationMode.posix, .automatic, .automatic, .posix] {
         let started = ContinuousClock.now
         let (root, _) = try await completedScan(path, mode: mode)
         let elapsed = ContinuousClock.now - started
-        let totals = (root.fileCount, root.directoryCount, root.totalSize, root.totalAllocatedSize,
+        let totals = (root.fileCount, root.directoryCount, root.totalSize,
                       root.cloudOnlyFileCount, root.unscannedDirectoryCount)
         if let reference {
             #expect(totals == reference)
@@ -323,3 +324,30 @@ func localIntegrationScan() async throws {
     #expect(bytes == root.totalAllocatedSize)
     print("Scan summary: files=\(root.fileCount), directories=\(root.directoryCount), logicalBytes=\(root.totalSize), allocatedBytes=\(bytes), cloudOnlyFiles=\(root.cloudOnlyFileCount), unscannedDirectories=\(root.unscannedDirectoryCount), unavailableEntries=\(root.totalUnavailableEntryCount), elapsed=\(ContinuousClock.now - started)")
 }
+
+@Test func apfsClonesShareTheirBlocksOnce() async throws {
+    let fixture = try Fixture()
+    defer { fixture.remove() }
+    let original = fixture.root.appendingPathComponent("original.bin")
+    try Data((0..<1_048_576).map { UInt8(truncatingIfNeeded: $0 &* 31) }).write(to: original)
+    let copyBytes = Int64(try fixture.metadata(original).st_blocks) * 512
+    for folder in ["a", "b", "c"] {
+        let directory = fixture.root.appendingPathComponent(folder)
+        try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
+        for index in 0..<3 {
+            let clone = directory.appendingPathComponent("clone-\(index).bin")
+            guard clonefile(original.path, clone.path, 0) == 0 else {
+                return // The temporary volume does not support clones (not APFS).
+            }
+        }
+    }
+    let (root, bytes) = try await completedScan(fixture.root.path)
+    func fileBytes(_ node: FileNode, _ size: (FileNode) -> Int64) -> Int64 {
+        node.isDirectory ? node.children.reduce(0) { $0 + fileBytes($1, size) } : size(node)
+    }
+    #expect(root.fileCount == 10)
+    #expect(fileBytes(root, \.allocatedSize) == copyBytes)
+    // Logical size still reports every copy in full.
+    #expect(fileBytes(root, \.ownSize) == 10 * 1_048_576)
+    #expect(bytes == root.totalAllocatedSize)
+}