Skip to content

fix(sizes): remove Logical mode — apparent size becomes a classified annotation (sparse/compressed) #31

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rducom merged 1 commit into from
Jul 13, 2026
Merged

fix(sizes): remove Logical mode — apparent size becomes a classified annotation (sparse/compressed) #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions Sources/SpaceMatters/App/HeadlessScan.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,19 @@ enum HeadlessScan {

let logical = root.aggLogical.load(ordering: .relaxed)
let physical = root.aggPhysical.load(ordering: .relaxed)
let sparse = root.aggSparseExcess.load(ordering: .relaxed)
let compressed = root.aggCompressedExcess.load(ordering: .relaxed)
let files = root.fileCount.load(ordering: .relaxed)
let dirs = scanner.dirCount.load(ordering: .relaxed)
let errors = scanner.errorCount.load(ordering: .relaxed)

print("roots: \(paths.count)")
print("files: \(files)")
print("dirs: \(dirs)")
print("logical: \(logical) bytes (\(Format.bytes(logical)))")
print("physical: \(physical) bytes (\(Format.bytes(physical)))")
print("on-disk: \(physical) bytes (\(Format.bytes(physical)))")
print("apparent: \(logical) bytes (\(Format.bytes(logical)))")
if sparse > 0 { print("sparse: \(sparse) bytes not allocated (\(Format.bytes(sparse)))") }
if compressed > 0 { print("compressed:\(compressed) bytes saved (\(Format.bytes(compressed)))") }
print("errors: \(errors)")
print(String(format: "elapsed: %.3f s (%.0f files/s)", elapsed, Double(files) / max(elapsed, 0.001)))

Expand All @@ -67,7 +71,7 @@ enum HeadlessScan {
}
}

let top = scanner.snapshotExtensions(metric: .physical, limit: 8)
let top = scanner.snapshotExtensions(limit: 8)
if !top.isEmpty {
print("top types:")
for row in top {
Expand Down Expand Up @@ -119,12 +123,12 @@ enum HeadlessScan {
}
}

print(String(format: "DONE in %.2fs — dirs=%d files=%d on-disk=%@ logical=%@",
print(String(format: "DONE in %.2fs — dirs=%d files=%d on-disk=%@ apparent=%@",
Date().timeIntervalSince(start), scanner.directoryCount,
root.fileCount.load(ordering: .relaxed),
Format.bytes(root.aggPhysical.load(ordering: .relaxed)),
Format.bytes(root.aggLogical.load(ordering: .relaxed))))
for row in scanner.snapshotExtensions(metric: .physical, limit: 6) {
for row in scanner.snapshotExtensions(limit: 6) {
print(" \(row.name): \(Format.bytes(row.physical)) (\(Format.count(row.count)) files)")
}
if let failure = scanner.failure {
Expand Down
113 changes: 93 additions & 20 deletions Sources/SpaceMatters/Model/FSNode.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,27 @@ final class FSNode {
let aggLogical = Atomic<Int64>(0)
let aggPhysical = Atomic<Int64>(0)
let fileCount = Atomic<Int64>(0)
/// Classified apparent-vs-on-disk gap for the subtree: bytes of content
/// length not backed by blocks in sparse files, and same for compressed
/// files. Explains *why* `aggLogical` dwarfs `aggPhysical` where it does —
/// sizes are always displayed on-disk; the gap surfaces as an annotation.
let aggSparseExcess = Atomic<Int64>(0)
let aggCompressedExcess = Atomic<Int64>(0)

/// This directory's own immediate files. Atomic so the UI can read live
/// totals at refresh rate while a worker is still filling them in — a node is
/// published into its parent's `_children` before its own fields are set.
private let _directFilesLogical = Atomic<Int64>(0)
private let _directFilesPhysical = Atomic<Int64>(0)
private let _directFileCount = Atomic<Int64>(0)
private let _directSparseExcess = Atomic<Int64>(0)
private let _directCompressedExcess = Atomic<Int64>(0)

var directFilesLogical: Int64 { _directFilesLogical.load(ordering: .relaxed) }
var directFilesPhysical: Int64 { _directFilesPhysical.load(ordering: .relaxed) }
var directFileCount: Int64 { _directFileCount.load(ordering: .relaxed) }
var directSparseExcess: Int64 { _directSparseExcess.load(ordering: .relaxed) }
var directCompressedExcess: Int64 { _directCompressedExcess.load(ordering: .relaxed) }

/// Extension that accounts for the most bytes among this directory's direct
/// files — used to colour the treemap by file type. Guarded by `gTreeLock`:
Expand Down Expand Up @@ -94,10 +104,13 @@ final class FSNode {
/// Adjust this directory's own-file totals (after a file was removed). Single
/// writer (MainActor post-scan, or the streamed reader thread), so a plain
/// load/store on the atomics is race-free against the UI's atomic reads.
func adjustDirectFiles(logical: Int64, physical: Int64, count: Int64) {
func adjustDirectFiles(logical: Int64, physical: Int64, count: Int64,
sparseExcess: Int64 = 0, compressedExcess: Int64 = 0) {
_directFilesLogical.store(max(0, directFilesLogical + logical), ordering: .relaxed)
_directFilesPhysical.store(max(0, directFilesPhysical + physical), ordering: .relaxed)
_directFileCount.store(max(0, directFileCount + count), ordering: .relaxed)
_directSparseExcess.store(max(0, directSparseExcess + sparseExcess), ordering: .relaxed)
_directCompressedExcess.store(max(0, directCompressedExcess + compressedExcess), ordering: .relaxed)
}

/// Zero this node's subtree aggregates ahead of an in-place re-scan (SPEC-02
Expand All @@ -106,6 +119,8 @@ final class FSNode {
aggLogical.store(0, ordering: .relaxed)
aggPhysical.store(0, ordering: .relaxed)
fileCount.store(0, ordering: .relaxed)
aggSparseExcess.store(0, ordering: .relaxed)
aggCompressedExcess.store(0, ordering: .relaxed)
}

/// Called once by the owning worker after a directory's entries are read.
Expand All @@ -114,11 +129,15 @@ final class FSNode {
filesLogical: Int64,
filesPhysical: Int64,
fileCount: Int64,
dominantExt: ExtKey = .none
dominantExt: ExtKey = .none,
sparseExcess: Int64 = 0,
compressedExcess: Int64 = 0
) {
_directFilesLogical.store(filesLogical, ordering: .relaxed)
_directFilesPhysical.store(filesPhysical, ordering: .relaxed)
_directFileCount.store(fileCount, ordering: .relaxed)
_directSparseExcess.store(sparseExcess, ordering: .relaxed)
_directCompressedExcess.store(compressedExcess, ordering: .relaxed)
gTreeLock.lock()
_dominantExt = dominantExt
_children = children
Expand All @@ -128,20 +147,23 @@ final class FSNode {

// MARK: Convenience

/// The size this app renders everywhere: allocated blocks — what deleting
/// frees, what fills the disk. (The apparent size is annotation-only.)
@inline(__always)
func size(_ metric: SizeMetric) -> Int64 {
switch metric {
case .logical: return aggLogical.load(ordering: .relaxed)
case .physical: return aggPhysical.load(ordering: .relaxed)
}
}
var sizeOnDisk: Int64 { aggPhysical.load(ordering: .relaxed) }

/// Content length (`st_size` sum) — what a copy, upload or non-sparse-aware
/// backup of this subtree would write. Shown only where it notably diverges.
@inline(__always)
func directFilesSize(_ metric: SizeMetric) -> Int64 {
switch metric {
case .logical: return directFilesLogical
case .physical: return directFilesPhysical
}
var sizeApparent: Int64 { aggLogical.load(ordering: .relaxed) }

/// Non-nil when this subtree's apparent size notably exceeds its footprint
/// (sparse or compressed content) — drives the divergence badge.
var divergence: SizeDivergence? {
SizeDivergence.notable(
onDisk: sizeOnDisk, apparent: sizeApparent,
sparseExcess: aggSparseExcess.load(ordering: .relaxed),
compressedExcess: aggCompressedExcess.load(ordering: .relaxed))
}
}

Expand All @@ -154,13 +176,64 @@ extension FSNode: Hashable {
func hash(into hasher: inout Hasher) { hasher.combine(ObjectIdentifier(self)) }
}

/// Which size to report. Physical (on-disk allocated) matches "space used" the
/// way WinDirStats does; logical is the byte length of the content.
enum SizeMetric: String, CaseIterable, Identifiable {
case physical
case logical
var id: String { rawValue }
var label: String { self == .physical ? "On disk" : "Logical" }
/// The gap between an item's *apparent* size (content length — the Finder's
/// "Size", what a copy/upload/backup writes) and its *on-disk* footprint
/// (allocated blocks — what deleting frees).
///
/// The two split when files are sparse (VM/container disk images: huge declared
/// length, few real blocks) or transparently compressed. A 512 GiB sparse image
/// occupying 75 MiB is correct data that *reads* as a bug — so the app renders
/// on-disk sizes everywhere and explains the gap in place, where it's notable,
/// instead of offering a global "Logical" mode that re-weighted the whole map
/// with un-actionable numbers.
struct SizeDivergence: Equatable {
let onDisk: Int64
let apparent: Int64
/// Apparent bytes not backed by blocks, split by cause. May not sum to the
/// whole gap: streamed scans can't classify, and block padding offsets it.
let sparseExcess: Int64
let compressedExcess: Int64

/// Badge threshold: the apparent size must dwarf the footprint enough to
/// mislead (≥ 1.5× and ≥ 8 MiB over). Block-padding noise never qualifies —
/// padding makes on-disk the *larger* of the two.
static let minDelta: Int64 = 8 << 20

static func notable(onDisk: Int64, apparent: Int64,
sparseExcess: Int64, compressedExcess: Int64) -> SizeDivergence? {
guard apparent - onDisk >= minDelta, apparent >= onDisk + onDisk / 2 else { return nil }
return SizeDivergence(onDisk: onDisk, apparent: apparent,
sparseExcess: sparseExcess, compressedExcess: compressedExcess)
}

/// Dominant cause(s) — one is named when it explains ≥ 25% of the gap.
private var causes: (sparse: Bool, compressed: Bool) {
let gap = max(1, apparent - onDisk)
return (sparseExcess * 4 >= gap, compressedExcess * 4 >= gap)
}

var label: String {
switch causes {
case (true, true): return "sparse + compressed"
case (true, false): return "sparse"
case (false, true): return "compressed"
case (false, false): return "sparse or compressed"
}
}

/// Tooltip text: both figures plus what the gap means for the user.
var summary: String {
var s = "\(Format.bytes(onDisk)) on disk · \(Format.bytes(apparent)) apparent."
switch causes {
case (true, false), (true, true):
s += "\nSparse content (disk image, preallocated file): space is allocated on demand. A copy, upload or non-sparse backup can balloon to the apparent size."
case (false, true):
s += "\nCompressed by the filesystem: the content is larger than the space it occupies."
case (false, false):
s += "\nThe content is larger than the space it occupies (sparse or compressed files)."
}
return s
}
}

/// How shared storage is counted. **Attribution** (default) blames every hardlink
Expand Down
15 changes: 11 additions & 4 deletions Sources/SpaceMatters/Scanner/CommandScanner.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,15 +105,15 @@ final class CommandScanner: ScanBackend {
var directoryCount: Int64 { dirCountAtomic.load(ordering: .relaxed) }
var scanErrorCount: Int64 { errAtomic.load(ordering: .relaxed) }

func snapshotExtensions(metric: SizeMetric, limit: Int) -> [ExtRow] {
func snapshotExtensions(limit: Int) -> [ExtRow] {
extLock.lock()
let copy = extStats
extLock.unlock()

var rows = copy.map { key, stat in
ExtRow(key: key, name: key.displayName, logical: stat.logical, physical: stat.physical, count: stat.count)
}
rows.sort { metric == .physical ? $0.physical > $1.physical : $0.logical > $1.logical }
rows.sort { $0.physical > $1.physical }
if rows.count > limit { rows.removeLast(rows.count - limit) }
return rows
}
Expand Down Expand Up @@ -222,6 +222,10 @@ final class CommandScanner: ScanBackend {
if parent !== accParent { flushAccumulator() }
accParent = parent
accLogical += logical; accPhysical += physical; accCount += 1
// `find` reports no BSD flags, but fewer blocks than bytes means
// holes — the Docker.raw / VM disk-image case this stream is full of.
// (ext4 has no transparent compression to mistake it for.)
if physical < logical { accSparseExcess += logical - physical }

let key = ExtKey(fileName: name)
extLock.lock()
Expand All @@ -242,18 +246,21 @@ final class CommandScanner: ScanBackend {
private var accLogical: Int64 = 0
private var accPhysical: Int64 = 0
private var accCount: Int64 = 0
private var accSparseExcess: Int64 = 0

private func flushAccumulator() {
guard let parent = accParent, accCount > 0 else { accParent = nil; return }
parent.adjustDirectFiles(logical: accLogical, physical: accPhysical, count: accCount)
parent.adjustDirectFiles(logical: accLogical, physical: accPhysical, count: accCount,
sparseExcess: accSparseExcess)
var node: FSNode? = parent
while let cur = node {
cur.aggLogical.wrappingAdd(accLogical, ordering: .relaxed)
cur.aggPhysical.wrappingAdd(accPhysical, ordering: .relaxed)
cur.fileCount.wrappingAdd(accCount, ordering: .relaxed)
if accSparseExcess != 0 { cur.aggSparseExcess.wrappingAdd(accSparseExcess, ordering: .relaxed) }
node = cur.parent
}
accParent = nil; accLogical = 0; accPhysical = 0; accCount = 0
accParent = nil; accLogical = 0; accPhysical = 0; accCount = 0; accSparseExcess = 0
}

/// `nil` on Int64 overflow (≥ 20 digits) — the caller drops the record.
Expand Down
28 changes: 19 additions & 9 deletions Sources/SpaceMatters/Scanner/DirectoryScanner.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,18 +171,16 @@ final class DirectoryScanner: ScanBackend {
extLock.unlock()
}

/// Snapshot of the top `limit` extensions by `metric`, materialised as rows.
func snapshotExtensions(metric: SizeMetric, limit: Int) -> [ExtRow] {
/// Snapshot of the top `limit` extensions by on-disk size, materialised as rows.
func snapshotExtensions(limit: Int) -> [ExtRow] {
extLock.lock()
let copy = extStats
extLock.unlock()

var rows = copy.map { key, stat in
ExtRow(key: key, name: key.displayName, logical: stat.logical, physical: stat.physical, count: stat.count)
}
rows.sort { a, b in
metric == .physical ? a.physical > b.physical : a.logical > b.logical
}
rows.sort { $0.physical > $1.physical }
if rows.count > limit { rows.removeLast(rows.count - limit) }
return rows
}
Expand Down Expand Up @@ -298,6 +296,8 @@ final class DirectoryScanner: ScanBackend {
var filesLogical: Int64 = 0
var filesPhysical: Int64 = 0
var fileCount: Int64 = 0
var sparseExcess: Int64 = 0
var compressedExcess: Int64 = 0
var localExt: [ExtKey: ExtStat] = [:]

// Avoid a double slash when the parent is "/" so paths match skip entries.
Expand All @@ -320,17 +320,20 @@ final class DirectoryScanner: ScanBackend {
} else {
var effL = entry.logicalSize
var effP = entry.physicalSize
var effExcess = entry.apparentExcess
// Exact mode: count a multi-linked inode's blocks only on its first
// sighting; later hardlinks contribute 0 bytes (still 1 file entry).
if exact && entry.linkCount > 1 {
linkLock.lock()
let firstSighting = seenInodes
.insert(InodeKey(dev: entry.deviceID, ino: entry.fileID)).inserted
linkLock.unlock()
if !firstSighting { effL = 0; effP = 0 }
if !firstSighting { effL = 0; effP = 0; effExcess = 0 }
}
filesLogical += effL
filesPhysical += effP
if entry.isSparse { sparseExcess += effExcess }
else if entry.isCompressed { compressedExcess += effExcess }
fileCount += 1
let key = ExtKey(name: entry.name, length: entry.nameLength)
localExt[key, default: ExtStat()].add(logical: effL, physical: effP)
Expand All @@ -356,7 +359,9 @@ final class DirectoryScanner: ScanBackend {
filesLogical: filesLogical,
filesPhysical: filesPhysical,
fileCount: fileCount,
dominantExt: dominantExt
dominantExt: dominantExt,
sparseExcess: sparseExcess,
compressedExcess: compressedExcess
)

// Propagate this directory's direct-file totals to itself and every
Expand All @@ -368,6 +373,8 @@ final class DirectoryScanner: ScanBackend {
n.aggLogical.wrappingAdd(filesLogical, ordering: .relaxed)
n.aggPhysical.wrappingAdd(filesPhysical, ordering: .relaxed)
n.fileCount.wrappingAdd(fileCount, ordering: .relaxed)
if sparseExcess != 0 { n.aggSparseExcess.wrappingAdd(sparseExcess, ordering: .relaxed) }
if compressedExcess != 0 { n.aggCompressedExcess.wrappingAdd(compressedExcess, ordering: .relaxed) }
node = n.parent
}
}
Expand All @@ -394,7 +401,10 @@ struct ExtRow: Identifiable {
let count: Int64
var id: ExtKey { key }

func size(_ metric: SizeMetric) -> Int64 {
metric == .physical ? physical : logical
/// Non-nil when this type's apparent bytes dwarf its footprint (the .ext4 /
/// .raw disk-image case) — surfaced as a tooltip, sizes stay on-disk.
/// Cause split isn't tracked per extension; the generic wording covers it.
var divergence: SizeDivergence? {
SizeDivergence.notable(onDisk: physical, apparent: logical, sparseExcess: 0, compressedExcess: 0)
}
}
Loading
Loading