diff --git a/Sources/MacDirStat/Scanning/BulkDirectoryReader.swift b/Sources/MacDirStat/Scanning/BulkDirectoryReader.swift index 30c31ce..a70f1b1 100644 --- a/Sources/MacDirStat/Scanning/BulkDirectoryReader.swift +++ b/Sources/MacDirStat/Scanning/BulkDirectoryReader.swift @@ -8,6 +8,9 @@ enum DirectoryEnumerationMode: Sendable { struct BulkDirectoryEntry { let name: String let metadata: Darwin.stat + /// Set for a full APFS clone (refcount above one). Its st_blocks are all + /// shared with the other members of the group, so the scanner counts them once. + var cloneID: UInt64? = nil } enum BulkDirectoryBatch { @@ -32,12 +35,14 @@ struct BulkDirectoryReader { UInt32(ATTR_CMN_FLAGS), UInt32(ATTR_CMN_FILEID)] attributes.commonattr = common.reduce(0, |) attributes.fileattr = UInt32(ATTR_FILE_ALLOCSIZE | ATTR_FILE_DATALENGTH) + // With FSOPT_ATTR_CMN_EXTENDED, forkattr selects extended common attributes. + attributes.forkattr = UInt32(ATTR_CMNEXT_CLONEID | ATTR_CMNEXT_CLONE_REFCNT) } mutating func next(fd: Int32) throws -> BulkDirectoryBatch { let count = buffer.withUnsafeMutableBytes { getattrlistbulk(fd, &attributes, $0.baseAddress!, $0.count, - UInt64(FSOPT_PACK_INVAL_ATTRS | FSOPT_RETURN_REALDEV)) + UInt64(FSOPT_PACK_INVAL_ATTRS | FSOPT_RETURN_REALDEV | FSOPT_ATTR_CMN_EXTENDED)) } if count < 0 { let code = errno @@ -125,6 +130,15 @@ func decodeBulkDirectoryEntry(_ record: UnsafeRawBufferPointer, fd: Int32) throw metadata.st_flags = flags metadata.st_blocks = allocated / 512 metadata.st_size = logical + // Packed after the file attributes; zero-filled when not returned. + // CLONE_REFCNT counts full clones, so above one the file has no private blocks. + let cloneID = try read(UInt64.self) + let cloneCount = try read(UInt32.self) + guard start >= offset else { throw BulkMetadataError.malformed } + let cloneFields = UInt32(ATTR_CMNEXT_CLONEID | ATTR_CMNEXT_CLONE_REFCNT) + if returned.forkattr & cloneFields == cloneFields, cloneCount > 1 { + return BulkDirectoryEntry(name: String(cString: name), metadata: metadata, cloneID: cloneID) + } } else { // Directory attributes describe the underlying mount/firmlink, not its // target. fstatat preserves stat's volume, inode and directory-size rules. diff --git a/Sources/MacDirStat/Scanning/FileScanner.swift b/Sources/MacDirStat/Scanning/FileScanner.swift index 663f27b..d3ac41b 100644 --- a/Sources/MacDirStat/Scanning/FileScanner.swift +++ b/Sources/MacDirStat/Scanning/FileScanner.swift @@ -54,6 +54,7 @@ private final class ScanState: Sendable { var fileCount = 0 var byteCount: Int64 = 0 var seenInodes = Set() + var seenClones = Set() var lastProgress = ContinuousClock.now } @@ -61,12 +62,16 @@ private final class ScanState: Sendable { self.rootDevice = rootDevice } - func record(_ metadata: Darwin.stat) -> Bool { + /// Returns the metadata to store, or nil for an inode already counted. + /// A clone group's blocks are charged once, to the first member seen in the scan. + func record(_ metadata: Darwin.stat, cloneID: UInt64? = nil) -> Darwin.stat? { lock.withLock { state in - guard state.seenInodes.insert(UInt64(metadata.st_ino)).inserted else { return false } - if metadata.st_mode & S_IFMT == S_IFREG { state.fileCount += 1 } - state.byteCount += Int64(metadata.st_blocks) * 512 - return true + guard state.seenInodes.insert(UInt64(metadata.st_ino)).inserted else { return nil } + var counted = metadata + if let cloneID, !state.seenClones.insert(cloneID).inserted { counted.st_blocks = 0 } + if counted.st_mode & S_IFMT == S_IFREG { state.fileCount += 1 } + state.byteCount += Int64(counted.st_blocks) * 512 + return counted } } @@ -186,10 +191,11 @@ private func scanDirectory( return result } - func append(name: String, metadata: Darwin.stat) { + func append(name: String, metadata: Darwin.stat, cloneID: UInt64? = nil) { guard metadata.st_dev == state.rootDevice else { return } let type = metadata.st_mode & S_IFMT - guard type == S_IFDIR || type == S_IFREG, state.record(metadata) else { return } + guard type == S_IFDIR || type == S_IFREG, + let metadata = state.record(metadata, cloneID: cloneID) else { return } let child = FileNode(name: name, metadata: metadata) result.children.append(child) if child.isDirectory { @@ -205,7 +211,7 @@ private func scanDirectory( switch try reader.next(fd: fd) { case .entries(let entries, let unavailable): result.unavailableEntries += unavailable - for entry in entries { append(name: entry.name, metadata: entry.metadata) } + for entry in entries { append(name: entry.name, metadata: entry.metadata, cloneID: entry.cloneID) } if let progress = state.progress(path: job.path) { continuation.yield(progress) } case .end: return result case .failure(let code): diff --git a/Tests/MacDirStatTests/FileScannerTests.swift b/Tests/MacDirStatTests/FileScannerTests.swift index bd6d8af..1a55b41 100644 --- a/Tests/MacDirStatTests/FileScannerTests.swift +++ b/Tests/MacDirStatTests/FileScannerTests.swift @@ -109,13 +109,14 @@ private func completedScan(_ path: String, limit: Int = 4, @Test(.enabled(if: ProcessInfo.processInfo.environment["MACDIRSTAT_BENCHMARK_PATH"] != nil)) func compareDirectoryEnumerationPerformance() async throws { let path = try #require(ProcessInfo.processInfo.environment["MACDIRSTAT_BENCHMARK_PATH"]) - var reference: (files: Int, directories: Int, logical: Int64, allocated: Int64, cloud: Int, incomplete: Int)? + // Allocated size is left out: only the bulk reader sees APFS clones, so .posix counts them in full. + var reference: (files: Int, directories: Int, logical: Int64, cloud: Int, incomplete: Int)? // Alternate order so both implementations see warm and cold metadata caches. for mode in [DirectoryEnumerationMode.posix, .automatic, .automatic, .posix] { let started = ContinuousClock.now let (root, _) = try await completedScan(path, mode: mode) let elapsed = ContinuousClock.now - started - let totals = (root.fileCount, root.directoryCount, root.totalSize, root.totalAllocatedSize, + let totals = (root.fileCount, root.directoryCount, root.totalSize, root.cloudOnlyFileCount, root.unscannedDirectoryCount) if let reference { #expect(totals == reference) @@ -323,3 +324,30 @@ func localIntegrationScan() async throws { #expect(bytes == root.totalAllocatedSize) print("Scan summary: files=\(root.fileCount), directories=\(root.directoryCount), logicalBytes=\(root.totalSize), allocatedBytes=\(bytes), cloudOnlyFiles=\(root.cloudOnlyFileCount), unscannedDirectories=\(root.unscannedDirectoryCount), unavailableEntries=\(root.totalUnavailableEntryCount), elapsed=\(ContinuousClock.now - started)") } + +@Test func apfsClonesShareTheirBlocksOnce() async throws { + let fixture = try Fixture() + defer { fixture.remove() } + let original = fixture.root.appendingPathComponent("original.bin") + try Data((0..<1_048_576).map { UInt8(truncatingIfNeeded: $0 &* 31) }).write(to: original) + let copyBytes = Int64(try fixture.metadata(original).st_blocks) * 512 + for folder in ["a", "b", "c"] { + let directory = fixture.root.appendingPathComponent(folder) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + for index in 0..<3 { + let clone = directory.appendingPathComponent("clone-\(index).bin") + guard clonefile(original.path, clone.path, 0) == 0 else { + return // The temporary volume does not support clones (not APFS). + } + } + } + let (root, bytes) = try await completedScan(fixture.root.path) + func fileBytes(_ node: FileNode, _ size: (FileNode) -> Int64) -> Int64 { + node.isDirectory ? node.children.reduce(0) { $0 + fileBytes($1, size) } : size(node) + } + #expect(root.fileCount == 10) + #expect(fileBytes(root, \.allocatedSize) == copyBytes) + // Logical size still reports every copy in full. + #expect(fileBytes(root, \.ownSize) == 10 * 1_048_576) + #expect(bytes == root.totalAllocatedSize) +}