diff --git a/.gitignore b/.gitignore index f1ec77fc9..b8f32ac60 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ docs/* !docs/CROSS_SEEDING.md !docs/architecture.md !docs/linting.md +!docs/remote-backend-design.md .mcp_unused.json config.toml .env diff --git a/docs/architecture.md b/docs/architecture.md index 0f46b3a13..2ec3d016d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -8,6 +8,7 @@ Internal reference for agents and maintainers. Read this before changing cross-m - `internal/api/`: HTTP handlers, middleware, and routing. - `internal/qbittorrent/`: qBittorrent client pool and sync manager. - `internal/services/`: domain services such as cross-seed, Jackett/Torznab, reannounce, and tracker rules. +- `internal/fsops/`: filesystem backend abstraction. Service callsites are migrating from direct `os.*` calls to an `fsops.Backend` (migration in #1915) resolved per instance via `fsops.Pool` — the local backend for instances with local filesystem access, a noop backend (every op returns `ErrNoFilesystemAccess`) otherwise. A future SSH-backed remote backend slots in at the pool (design: `docs/remote-backend-design.md`). - `internal/proxy/`: reverse proxy support for external apps. - `internal/backups/`: scheduled snapshots. - `internal/database/`: SQLite/Postgres migrations and database setup. diff --git a/docs/remote-backend-design.md b/docs/remote-backend-design.md new file mode 100644 index 000000000..830088019 --- /dev/null +++ b/docs/remote-backend-design.md @@ -0,0 +1,308 @@ +# Remote Filesystem Backend Design (SSH/SFTP-native) + +Status: draft for maintainer review. Supersedes the remote-helper design +(PR #1913, closed but kept as reference — it documents the deployable-agent +tier and its NDJSON wire protocol, which remain the fallback if this design +hits a performance wall). + +## Context and Decision + +PR #1914 adds `fsops.Backend`; service callsites migrate from direct +`os.*` calls to it in #1915–#1916. Instances running on a different host than qui +need a remote implementation. + +The prior design deployed a `qui-helper` binary to the remote host and spoke +NDJSON to it over SSH. **Decision: no deployed agent.** The remote backend +uses native SSH primitives on a single connection: + +- the SFTP subsystem (one channel) for everything the protocol can express, +- exec sessions (additional channels on the same connection) for what it + cannot, when the key permits exec, +- a capability probe at connect time that decides which tier the instance + gets. Nothing is persisted; capabilities are re-probed on every connect. + +Rationale: no binaries to build per arch or keep in lockstep with qui; +shared seedboxes essentially always offer SFTP but do not always allow exec; +the key's own restrictions — not qui configuration — pick the trade-off. + +## Capability Tiers + +**SFTP-only** (key restricted to `internal-sftp`): stat, lstat, readdir, +walks, mkdir, and remove always work; free space and hardlink-tree +creation depend on the `statvfs@openssh.com` and `hardlink@openssh.com` +extensions, which the connect probe tracks independently — an operation +whose extension the server does not advertise returns an explicit +unsupported error and appears in the capability report, rather than the +tier being assumed to include it. SFTP v3 attrs carry no inode or +nlink, so file identity is unavailable: `Lstat`/`WalkDir` set `FileIDErr` +when identity is requested. Consumers already degrade on zero FileID — +orphan-scan alias dedup switches off, hardlinked-copy detection reports +no-evidence, dirscan's FileID index skips the files. Reflinks are +unsupported (no SFTP equivalent of FICLONE; `copy-data` is a byte copy, not +CoW). Missing identity also changes hardlink-tree conflict handling: the +local backend proves an existing target is the same link (`os.SameFile`) +and skips it idempotently, but SFTP-only cannot verify that, so an +existing target is always a conflict and fails the create — never a +silent skip (raised by Audionut on #1914). The degraded mode must be +surfaced in the UI, not silent. + +**SFTP+exec**: identity arrives via `find`/`stat` sweeps, reflink support is +probed and used, `SameFilesystem` is exact. "Full functionality" means +every operation has either a supported SFTP extension or an exec +fallback — the probe verifies that per operation, it is not implied by +the tier label. + +`SameFilesystem` with unusable fsids (servers that report zeroes, no exec +fallback): returns an explicit error, never a guess in either direction. +Both migrated callers (season-pack base-dir selection, dirscan link-tree +placement) already treat the error as "don't hardlink" and fall back to +non-link strategies, which is the safe degradation. + +## Operation Mapping + +| fsops.Backend | SFTP-only | with exec | +|---|---|---| +| `Stat` / `Lstat` | `SSH_FXP_STAT` / `LSTAT` (identity → `FileIDErr`) | + `stat` for identity | +| `ReadDir` | `SSH_FXP_READDIR` (attrs included, no per-entry stat) | same | +| `WalkDir` | recursive readdir | `find -printf` streams a tree's paths+identity in one round trip | +| `Statfs` | `statvfs@openssh.com` | same (fallback `df -P`) | +| `SameFilesystem` | fsid compare from statvfs, if the server reports real fsids (probe; some return zero) | `stat -c %d` compare | +| `MkdirAll` | `SSH_FXP_MKDIR` walk-up | same | +| `Remove` | `REMOVE`/`RMDIR`; recursive = readdir-driven bottom-up | recursive = `rm -rf --` | +| `HardlinkTree` | `hardlink@openssh.com` per file | same | +| `ReflinkTree` | unsupported | `cp --reflink=always` | +| `RemoveTree` | `REMOVE` over the recorded handle | same | +| `SupportsReflink` | false | probed once per fs root | + +`StatBatch`/`LstatBatch` were dropped from `fsops.Backend` pending a +consumer; this backend is that consumer. pkg/sftp pipelines concurrent +requests over the one session, and the exec path fills a batch with a single +`xargs -0 stat` round trip — the batch seam is what makes remote hardlink +indexing (hundreds of thousands of lstats) survivable. Re-add them in the +PR that implements this backend. + +## File Identity Over the Wire — DECIDED + +Remote identity is (device, inode) parsed from GNU `find`/`stat` output. But +`hardlink.FileID` is platform-compiled: unix builds carry `Dev`/`Ino`, +Windows builds carry a volume serial plus a 16-byte identifier. A qui host +on Windows cannot represent a Linux seedbox's identity in today's struct. + +**Decision: `hardlink.FileID` becomes an opaque, tagged, comparable +fixed-size form, implemented in the remote-backend PR** (raised by com6056 +on #1914). Opaque is the only viable shape: unix identity is 16 bytes, +Windows identity is up to 24, so no packing into the other platform's +struct is lossless in either direction, and the type is already shared +across develop's consumers, so the ripple is the same size now or later. +Tagged, because untagged bytes let a unix `dev=1,ino=2` collide with a +Windows identity in the same space: `struct { kind uint8; raw [24]byte }`, +kind 0 for no identity, 1 for unix `dev`+`ino` big endian, 2 for a +Windows volume serial plus the 16-byte file id, 3 for a remote-provided +blob. Constructors zero the unused tail of `raw` so `==` stays +well-defined, and `IsZero()` then means "no identity" rather than "a +backend wrote zeros" (load-bearing: the hardlink index trusts any FileID +returned without error, so zeroes-as-identity would collapse torrents +into one delete-safe group). `Bytes()` returns the tagged form: dirscan +keys its FileID index on `string(FileID.Bytes())` and persists it as +`dir_scan_files.file_id` under a partial index on +`(directory_id, file_id)` matched by rename detection, so the encoding +change ships with a migration (or a read path accepting the legacy +widths) whenever it lands. Consumer churn is otherwise unchanged: `==`, +map keys, and `IsZero()` all survive; the churn is per-platform +constructors and test literals. + +Guard regardless of representation: FileID comparisons are only valid +within one backend/host, and `==` still compiles cross-host, so +comparisons go through a helper that takes the backend scope (or the +scope rides in the value). Remote-sourced identity is always kind 3, even +when the remote is unix and the bytes are a `dev`/`ino`: it is parsed +from command output on a host qui does not control, and the tag keeps +peer-asserted identity type-distinct from kernel-attested identity. Peer +identity is advisory. It may suppress work (dedup accounting, +already-seeding skips) but never expands a destructive action: +automations' delete-expansion groups torrents by a FileID signature and +deletes the group with files, so on a remote backend that expansion +degrades to the no-identity behavior. A hostile remote can never reach a +worse outcome than the SFTP-only tier already produces with no identity +at all. + +## Exec Conventions + +- `LC_ALL=C` and NUL separators everywhere; torrent filenames can contain + nearly anything except NUL and `/`. +- Probe GNU vs BSD userland at connect (`find -printf` and `stat -c` are + GNU-only); degrade the affected ops per-tool on BSD remotes. +- Every exec carries a timeout and honors ctx cancellation; output is + size-capped. + +## Path Domains + +Every path handed to a Backend belongs to that backend's filesystem, in +that filesystem's native form — the interface itself is path-domain +agnostic. The local backend speaks host paths, so today's callsites using +host `filepath` are correct by construction. The remote backend speaks +slash-delimited POSIX paths regardless of the qui host's OS, which means +host `filepath` must never touch a remote path: on a Windows host, +`filepath.IsAbs("/data")` is false and `Join` inserts backslashes +(raised by Audionut on #1914). The remote-backend PR introduces a path +dialect for backend-owned path manipulation (Join/Dir/Base/IsAbs/Rel); +the local backend's dialect is the host `filepath`, so existing callsites +keep their exact behavior, and a Windows-hosted qui operating a unix +remote becomes correct by construction rather than by luck. Paths from +qBittorrent's API arrive slash-delimited and stay inside their instance's +backend domain end to end. + +## Path and Command Safety + +- Torrent- and API-derived paths are validated before any SFTP or exec use, + under the same boundary rules the local backends enforce: slash-delimited, + no leading separator, no `..` component, no drive-letter or UNC form where + a relative path is required. Absolute remote paths are permitted only as + instance-level roots (e.g. save paths reported by qBittorrent); torrent + file names always join beneath one. +- Exec command lines never interpolate raw paths into a shell string: + arguments are strictly quoted, and options are terminated with `--` + before any path for `find`, `stat`, `xargs`, `rm`, and `cp`. +- Symlink policy for destructive ops: recursion never follows a link, on + any tier. A symlinked directory is never descended; the link itself is + removed, never the target's contents. `rm -rf --` and the local + backend's `os.RemoveAll` both traverse `openat`-style, so they refuse + links and are immune to a mid-walk swap. SFTP v3 gets only the first + half: no `openat`, no `O_NOFOLLOW`, every path re-resolved server-side, + so a swapped ancestor directory can redirect an unlink between the walk + and the remove. That race is unfixable over v3; bound it by + re-`lstat`ing immediately before each unlink. `RemoveTree` is a path + list, not a walk, and takes the same shape per entry: unlink only, + directories only when empty. +- The never-follow rule covers identity as well as traversal: identity + feeding a destructive decision comes from `lstat`, so a planted symlink + cannot attribute another file's identity to a path inside the tree. +- The remote-backend PR carries tests for traversal payloads, option-like + filenames (`-rf`), shell metacharacters in paths, and symlinked + directories inside a tree marked for recursive removal. + +## Security + +- Dedicated SSH key per instance; never the user's personal key. +- Private key stored AES-GCM encrypted, keyed from `sessionSecret` like + existing credential encryption; the AAD binding (instance id + field) + is new here, today's credential stores pass no additional data. The + pinned host key is stored under the same AEAD with host and port in its + AAD as well: the pin fixes where the key gets used, and encrypting the + key alone says nothing about that. A plaintext-column edit redirecting + the instance then fails as a decryption error, which is unambiguous + tampering, rather than as a host-key mismatch, which is not. The scope + is deliberate: this defeats a DB-write attacker and cross-instance + transplant, not a stolen data directory (`sessionSecret` sits beside + `qui.db` by default) and not rollback to an older row set for the same + instance, which needs state outside the DB. Tests cover + tampered-ciphertext failing closed rather than falling back to TOFU. + The motivating deployment is Postgres, where the database can live on a + different host from the app and `sessionSecret`: there, a DB-write + attacker without app-host access is a realistic position, and the AAD + binding is what stops credential transplant and instance redirection. + On single-host SQLite the binding is cheap belt-and-suspenders. + Mechanically this stays the product's one crypto pattern — the same + AES-GCM/`sessionSecret` helpers the existing credential stores use, + with an AAD argument those stores simply haven't passed before. +- Host key verification is TOFU with explicit confirmation: the first-seen + key is held ephemeral and surfaced as a fingerprint via the ssh-test + flow; it is persisted and enforced only after the user confirms it (or + it matches a preconfigured fingerprint). No connection is trusted for + real operations before that. `InsecureIgnoreHostKey` is forbidden. +- What gets pinned is the marshaled public key and its algorithm, not a + display string; later connects constrain `HostKeyAlgorithms` to the + pinned type, so a key-type change is a mismatch, never a negotiation + accident. Fingerprints render as `SHA256:` for humans only. +- A host-key change after pinning fails closed: no automatic re-pin, and + no fallback to TOFU if the stored pin is missing or unreadable. The + mismatch surfaces both fingerprints and both key types behind a + confirmation deliberately heavier than first contact, one that names + interception as a possible cause and points at out-of-band + verification. A legitimate re-key and an interception look identical to + qui, so the user makes that call, never the code. On a background + reconnect nobody is there to prompt, so a mismatch parks the instance + in a needs-reconfirmation state and fails every fsop until a human + clears it. +- The pin belongs to the host, not the credential: deleting SSH + credentials keeps the pin, `ssh-test` against a pinned instance routes + into the mismatch flow rather than first contact, and editing host or + port drops the pin deliberately and requires fresh confirmation. Tests + cover mismatch-fails-closed, no re-pin via `ssh-test`, and key-type + change. +- Recommended `authorized_keys` template stays the tight one: + `command="internal-sftp",restrict ...` — that key yields the SFTP-only + tier. Granting exec is the user's explicit choice via a less-restricted + key. +- Optional middle tier: a ~20-line POSIX forced-command allowlist script + (technically a deployed artifact, but human-auditable) restores exec's + benefits without an unrestricted key. Not required for any tier to work. +- Never log credentials or key material. + +## Connection Pool + +One pool keyed by instance: lazy dial, reconnect backoff 5s→60s with ±20% +jitter, every operation ctx-cancellable. The sftp client and exec sessions +share the one `x/crypto/ssh` connection. Concurrency comes from sftp +request pipelining plus bounded parallel exec sessions — no helper-process +lifecycle to manage. + +## Schema + +Half of the old design's schema survives: SSH columns on `instances` — +host, port, user, the AEAD-encrypted private key (AAD: instance id + +field), and the pinned host key stored as the marshaled public key plus +its algorithm under the same AEAD (AAD: instance id + field + host + +port). Not a fingerprint column: the `HostKeyAlgorithms` constraint and +the mismatch flow both need the full key, and fingerprints are +display-only (see Security). No helper-deploy columns, no persisted +capabilities. `HasFilesystemAccess` resolves to local | remote | none. +This is the slimmed scope for #1917. + +## API + +- `POST /instances/{id}/ssh-test` — dial with provided credentials, return + host-key fingerprint for TOFU confirmation plus the capability report. +- `DELETE /instances/{id}/ssh-credentials`. +- No deploy/redeploy/helper endpoints. + +## Frontend + +SSH configuration on the instance form; the test flow confirms the host-key +fingerprint and shows the probed tier. Instances on the SFTP-only tier show +a degraded-mode indicator naming what's off (hardlink dedup, reflinks). + +## Windows Remotes + +Not a v1 target, but not special-cased either — the probe handles them for +free. Win32-OpenSSH's sftp-server covers the core ops (stat, readdir, +walks, mkdir, remove), so if the basic-op probe passes, the instance gets +the SFTP-only tier with whatever extensions the server actually advertises +(`statvfs@openssh.com` and `hardlink@openssh.com` support is +version-dependent in Win32-OpenSSH — trust the probe, not assumptions; a +server without the hardlink extension means link-tree cross-seeding is off +for that instance, and degraded mode says so). The exec tier never lights: +exec lands in cmd/PowerShell and the GNU-userland probe fails. A PowerShell +exec dialect (`fsutil file queryfileid`, `fsutil hardlink list`, +`Get-ChildItem` sweeps) is possible later if demand appears; the opaque +FileID form keeps that door open. Remote Windows paths surface in +SFTP's `/C:/...` form and stay slash-delimited at the fsops boundary like +every other remote path. + +## Rollout + +1. Foundation (open): #1914 backend interface, #1915 callsite migration, + #1916 missing-files. +2. #1917 reshaped to the schema above. +3. Remote backend: pool + SFTP implementation + capability probe (re-adds + batch methods), API endpoints, OpenAPI. +4. Frontend. +5. Feature rollout per service, degraded-mode UX. + +Helper/agent tier: explicitly deferred. If SFTP+exec hits a real +performance wall, #1913 has the protocol design ready. + +## Open Questions + +1. BSD/macOS remotes: which exec probes degrade, and is SFTP-only the + supported floor there? diff --git a/internal/fsops/backend.go b/internal/fsops/backend.go new file mode 100644 index 000000000..fc96936ee --- /dev/null +++ b/internal/fsops/backend.go @@ -0,0 +1,95 @@ +// Copyright (c) 2025-2026, s0up and the autobrr contributors. +// SPDX-License-Identifier: GPL-2.0-or-later + +package fsops + +import ( + "context" + "io/fs" + + "github.com/autobrr/qui/pkg/hardlinktree" +) + +// Backend abstracts filesystem operations so services work identically against +// a local filesystem or a future SSH-backed remote. It covers exactly the +// operations qui's services need: syscall-level primitives (stat, walk, +// mkdir, remove) plus the high-level tree operations (HardlinkTree, +// ReflinkTree, RemoveTree) that create-and-rollback as a unit. Path +// manipulation (filepath.Clean, filepath.Rel, etc.) is not part of this +// interface — it stays as direct calls in service code. +// +// Every method accepts a context.Context and must respect cancellation. +// +// Error semantics are portable across implementations: a missing path is +// reported compatibly with errors.Is(err, fs.ErrNotExist) and a denied one +// with errors.Is(err, fs.ErrPermission) — from Stat, Lstat, ReadDir, +// WalkDir (on entries), Remove, and Statfs alike. Remote backends must map +// their transport's errors onto the same sentinels. +type Backend interface { + // --- Read --- + + // Stat returns metadata for a single path, following symlinks — FileID + // and Nlinks describe the target, so symlinked torrent data keeps its + // identity. Returns a non-nil error wrapping fs.ErrNotExist if the path + // does not exist. A failure to resolve identity is reported in + // LstatInfo.FileIDErr, not as a Stat error, so one identity-opaque file + // cannot fail callers that only need metadata. + Stat(ctx context.Context, path string) (*LstatInfo, error) + + // Lstat is like Stat but does not follow symlinks. + Lstat(ctx context.Context, path string) (*LstatInfo, error) + + // ReadDir returns directory entries. + ReadDir(ctx context.Context, path string) ([]DirEntry, error) + + // WalkDir walks a directory tree and streams entries on the returned channel. + // The channel is closed when the walk completes, is cancelled via ctx, or + // hits an unrecoverable error. Callers must drain the channel or cancel + // ctx; abandoning it leaks the walk goroutine. Entries whose metadata + // cannot be read are skipped, not emitted — WalkEntry.Err carries only + // enumeration-level walk failures. + WalkDir(ctx context.Context, root string, opts WalkOptions) (<-chan WalkEntry, error) + + // Statfs returns free/total bytes for the filesystem containing path. + Statfs(ctx context.Context, path string) (*StatfsResult, error) + + // SameFilesystem returns true if both paths reside on the same filesystem + // (same device ID on Unix, same volume serial on Windows). + SameFilesystem(ctx context.Context, p1, p2 string) (bool, error) + + // --- Write (mutating) --- + + // MkdirAll creates a directory and all parents. Equivalent to os.MkdirAll. + // For torrent content and link-tree dirs, pass fsutil.ContentDirMode / + // fsutil.LinkTreeBaseDirMode rather than a hand-typed mode (#1704, #2086). + MkdirAll(ctx context.Context, path string, perm fs.FileMode) error + + // Remove removes a file or directory. If opts.Recursive is true, removes + // the entire tree (like os.RemoveAll). + Remove(ctx context.Context, path string, opts RemoveOptions) error + + // --- High-level (atomic, server-orchestrated) --- + + // HardlinkTree creates a hardlink tree from plan. Rolls back what it + // created on partial failure. The result records the files and dirs this + // call made, for a later RemoveTree. + HardlinkTree(ctx context.Context, plan *hardlinktree.TreePlan) (*TreeCreateResult, error) + + // ReflinkTree creates a reflink (CoW) tree from plan. Rolls back what it + // created on partial failure. The result records the files and dirs this + // call made, for a later RemoveTree. + ReflinkTree(ctx context.Context, plan *hardlinktree.TreePlan) (*TreeCreateResult, error) + + // RemoveTree removes exactly the files and dirs recorded in created — + // never the whole plan, which could delete links shared with sibling + // torrents (discussion #2282). A plan root that already existed before + // the create is therefore NOT removed; callers own pruning it. Safe to + // call with a nil result. + RemoveTree(ctx context.Context, created *TreeCreateResult) error + + // --- Capabilities --- + + // SupportsReflink returns whether the filesystem at path supports CoW + // reflinks. The string return is a human-readable reason when unsupported. + SupportsReflink(ctx context.Context, path string) (bool, string, error) +} diff --git a/internal/fsops/errors.go b/internal/fsops/errors.go new file mode 100644 index 000000000..7cfc25ebb --- /dev/null +++ b/internal/fsops/errors.go @@ -0,0 +1,13 @@ +// Copyright (c) 2025-2026, s0up and the autobrr contributors. +// SPDX-License-Identifier: GPL-2.0-or-later + +package fsops + +import "errors" + +// Sentinel errors returned by Backend implementations. +var ( + // ErrNoFilesystemAccess is returned by the NoopBackend for instances that + // have no filesystem access configured (neither local nor remote). + ErrNoFilesystemAccess = errors.New("filesystem access is not configured for this instance") +) diff --git a/internal/fsops/local/local.go b/internal/fsops/local/local.go new file mode 100644 index 000000000..f3f7a2b7c --- /dev/null +++ b/internal/fsops/local/local.go @@ -0,0 +1,297 @@ +// Copyright (c) 2025-2026, s0up and the autobrr contributors. +// SPDX-License-Identifier: GPL-2.0-or-later + +// Package local implements fsops.Backend by delegating to the local filesystem +// via os.*, pkg/hardlinktree, pkg/reflinktree, pkg/hardlink, and pkg/fsutil. +// This is the default backend used when qui and qBittorrent share a host. +package local + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/autobrr/qui/internal/fsops" + "github.com/autobrr/qui/pkg/fsutil" + "github.com/autobrr/qui/pkg/hardlink" + "github.com/autobrr/qui/pkg/hardlinktree" + "github.com/autobrr/qui/pkg/reflinktree" +) + +// Backend implements fsops.Backend using the local filesystem. +type Backend struct{} + +// NewBackend returns a local filesystem backend. +func NewBackend() *Backend { return &Backend{} } + +// compile-time check +var _ fsops.Backend = (*Backend)(nil) + +func (b *Backend) Stat(ctx context.Context, path string) (*fsops.LstatInfo, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + return osFileInfoToLstat(fi, path), nil +} + +func (b *Backend) Lstat(ctx context.Context, path string) (*fsops.LstatInfo, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + fi, err := os.Lstat(path) + if err != nil { + return nil, err + } + return osFileInfoToLstat(fi, path), nil +} + +func (b *Backend) ReadDir(ctx context.Context, path string) ([]fsops.DirEntry, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + entries, err := os.ReadDir(path) + if err != nil { + return nil, err + } + + result := make([]fsops.DirEntry, 0, len(entries)) + for _, e := range entries { + result = append(result, fsops.DirEntry{ + Name: e.Name(), + IsDir: e.IsDir(), + IsSymlink: e.Type()&os.ModeSymlink != 0, + }) + } + return result, nil +} + +func (b *Backend) WalkDir(ctx context.Context, root string, opts fsops.WalkOptions) (<-chan fsops.WalkEntry, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + // Verify root exists before starting the goroutine. + if _, err := os.Stat(root); err != nil { + return nil, err + } + + ch := make(chan fsops.WalkEntry, 64) + go func() { + defer close(ch) + walkErr := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error { + if ctx.Err() != nil { + return fs.SkipAll + } + + // d is nil when filepath.WalkDir cannot stat the root (TOCTOU + // between our pre-check and the walk's internal stat). + if d == nil { + return walkErr + } + + // Skip hidden files/dirs if requested. + name := d.Name() + if opts.SkipHidden && len(name) > 0 && name[0] == '.' && path != root { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + + // Skip ignored directory names. Case-insensitive: these are OS/NAS + // metadata dirs ($RECYCLE.BIN, @eaDir) whose on-disk case varies. + if d.IsDir() && path != root { + if slices.ContainsFunc(opts.IgnoreDirNames, func(ignored string) bool { + return strings.EqualFold(ignored, name) + }) { + return filepath.SkipDir + } + } + + // Skip ignored paths. + for _, ignored := range opts.IgnorePaths { + if path == ignored { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + } + + entry := fsops.WalkEntry{} + entry.Path = path + + rel, err := filepath.Rel(root, path) + if err == nil { + entry.RelPath = rel + } + + if walkErr != nil { + entry.Err = walkErr + } else { + fi, err := d.Info() + if err != nil { + // An entry that vanished or can't be stat'd is skipped, not + // surfaced: consumers (orphanscan, dirscan) skipped such + // files pre-migration, and emitting it as Err would read as + // a walk failure and abort whole scans. Err stays reserved + // for enumeration-level failures. + return nil + } + entry.Size = fi.Size() + entry.ModTime = fi.ModTime() + entry.Mode = fi.Mode() + entry.IsDir = fi.IsDir() + entry.IsSymlink = fi.Mode()&os.ModeSymlink != 0 + + if opts.WantFileID && fi.Mode().IsRegular() { + fid, nlinks, fidErr := hardlink.GetFileID(fi, path) + if fidErr != nil { + // Identity failure must not read as an unreadable + // entry — that would abort whole scans over one odd + // file. Callers that need identity check FileIDErr. + entry.FileIDErr = fidErr + } else { + entry.FileID = fid + entry.Nlinks = nlinks + } + } + } + + select { + case ch <- entry: + case <-ctx.Done(): + return fs.SkipAll + } + return nil + }) + // Surface a walk abort as a final entry so the caller knows the walk + // did not complete. Per-entry enumeration errors (unreadable + // subdirectory, permission denied on a child) are emitted inline + // above and do NOT abort the walk, so this fires only when the walk + // itself returned an error — in practice the root-stat TOCTOU case, + // where the callback propagates walkErr. Context cancellation is not + // an error — the caller initiated it. + if walkErr != nil && ctx.Err() == nil { + entry := fsops.WalkEntry{Err: walkErr} + entry.Path = root + select { + case ch <- entry: + case <-ctx.Done(): + } + } + }() + return ch, nil +} + +func (b *Backend) SameFilesystem(ctx context.Context, p1, p2 string) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + return fsutil.SameFilesystem(p1, p2) +} + +func (b *Backend) MkdirAll(ctx context.Context, path string, perm fs.FileMode) error { + if err := ctx.Err(); err != nil { + return err + } + return os.MkdirAll(path, perm) +} + +func (b *Backend) Remove(ctx context.Context, path string, opts fsops.RemoveOptions) error { + if err := ctx.Err(); err != nil { + return err + } + if opts.Recursive { + return os.RemoveAll(path) + } + return os.Remove(path) +} + +func (b *Backend) HardlinkTree(ctx context.Context, plan *hardlinktree.TreePlan) (*fsops.TreeCreateResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + created, err := hardlinktree.Create(plan) + if err != nil { + return nil, err + } + return treeCreateResult(created, plan), nil +} + +func (b *Backend) ReflinkTree(ctx context.Context, plan *hardlinktree.TreePlan) (*fsops.TreeCreateResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + created, err := reflinktree.Create(plan) + if err != nil { + return nil, err + } + return treeCreateResult(created, plan), nil +} + +func (b *Backend) RemoveTree(ctx context.Context, created *fsops.TreeCreateResult) error { + if err := ctx.Err(); err != nil { + return err + } + if created == nil { + return nil + } + handle := &hardlinktree.Created{Files: created.Files, Dirs: created.Dirs} + return handle.Rollback() +} + +func treeCreateResult(created *hardlinktree.Created, plan *hardlinktree.TreePlan) *fsops.TreeCreateResult { + return &fsops.TreeCreateResult{ + Created: len(created.Files), + SkippedExists: len(plan.Files) - len(created.Files), + Files: created.Files, + Dirs: created.Dirs, + } +} + +func (b *Backend) SupportsReflink(ctx context.Context, path string) (bool, string, error) { + if err := ctx.Err(); err != nil { + return false, "", err + } + supported, reason := reflinktree.SupportsReflink(path) + return supported, reason, nil +} + +// osFileInfoToFsops converts an os.FileInfo to an fsops.FileInfo. +func osFileInfoToFsops(fi os.FileInfo, path string) *fsops.FileInfo { + return &fsops.FileInfo{ + Path: path, + Size: fi.Size(), + ModTime: fi.ModTime(), + IsDir: fi.IsDir(), + IsSymlink: fi.Mode()&os.ModeSymlink != 0, + Mode: fi.Mode(), + } +} + +// osFileInfoToLstat converts an os.FileInfo from Lstat to an fsops.LstatInfo. +// Identity failure degrades to FileIDErr rather than failing the conversion: +// callers that only want size/mtime keep working, callers that need identity +// check FileIDErr. +func osFileInfoToLstat(fi os.FileInfo, path string) *fsops.LstatInfo { + info := &fsops.LstatInfo{ + FileInfo: *osFileInfoToFsops(fi, path), + } + if fi.Mode().IsRegular() { + fid, nlinks, err := hardlink.GetFileID(fi, path) + if err != nil { + info.FileIDErr = err + } else { + info.FileID = fid + info.Nlinks = nlinks + } + } + return info +} diff --git a/internal/fsops/local/local_test.go b/internal/fsops/local/local_test.go new file mode 100644 index 000000000..fc0585a97 --- /dev/null +++ b/internal/fsops/local/local_test.go @@ -0,0 +1,547 @@ +// Copyright (c) 2025-2026, s0up and the autobrr contributors. +// SPDX-License-Identifier: GPL-2.0-or-later + +package local + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/autobrr/qui/internal/fsops" + "github.com/autobrr/qui/pkg/hardlinktree" +) + +func newBackend() *Backend { return NewBackend() } + +func writeFile(t *testing.T, path, content string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) +} + +func TestStat_ExistingFile(t *testing.T) { + b := newBackend() + dir := t.TempDir() + path := filepath.Join(dir, "file.txt") + writeFile(t, path, "hello") + + fi, err := b.Stat(context.Background(), path) + require.NoError(t, err) + assert.Equal(t, path, fi.Path) + assert.Equal(t, int64(5), fi.Size) + assert.False(t, fi.IsDir) + assert.False(t, fi.IsSymlink) + assert.False(t, fi.ModTime.IsZero()) +} + +func TestPortableNotExistErrors(t *testing.T) { + // The Backend contract promises errors.Is(err, fs.ErrNotExist) for + // missing paths from every read/mutate method, not just Stat — remote + // backends must map onto the same sentinels, so the local backend is + // the reference. + b := newBackend() + ctx := context.Background() + missing := filepath.Join(t.TempDir(), "nope", "missing.mkv") + + _, err := b.Stat(ctx, missing) + require.ErrorIs(t, err, fs.ErrNotExist) + _, err = b.Lstat(ctx, missing) + require.ErrorIs(t, err, fs.ErrNotExist) + _, err = b.ReadDir(ctx, missing) + require.ErrorIs(t, err, fs.ErrNotExist) + _, err = b.WalkDir(ctx, missing, fsops.WalkOptions{}) + require.ErrorIs(t, err, fs.ErrNotExist) + err = b.Remove(ctx, missing, fsops.RemoveOptions{}) + require.ErrorIs(t, err, fs.ErrNotExist) + _, err = b.Statfs(ctx, missing) + require.ErrorIs(t, err, fs.ErrNotExist) +} + +func TestStat_FollowsSymlinkIdentity(t *testing.T) { + b := newBackend() + dir := t.TempDir() + target := filepath.Join(dir, "data.mkv") + writeFile(t, target, "content") + link := filepath.Join(dir, "link.mkv") + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlinks unsupported: %v", err) + } + + viaLink, err := b.Stat(context.Background(), link) + require.NoError(t, err) + direct, err := b.Stat(context.Background(), target) + require.NoError(t, err) + + // Stat follows the link: metadata and identity describe the target, so + // symlinked torrent data keeps already-seeding detection. + assert.False(t, viaLink.IsSymlink) + assert.Equal(t, direct.Size, viaLink.Size) + require.NoError(t, viaLink.FileIDErr) + assert.False(t, viaLink.FileID.IsZero()) + assert.Equal(t, direct.FileID, viaLink.FileID) +} + +func TestStat_Directory(t *testing.T) { + b := newBackend() + dir := t.TempDir() + + fi, err := b.Stat(context.Background(), dir) + require.NoError(t, err) + assert.True(t, fi.IsDir) +} + +func TestStat_NotFound(t *testing.T) { + b := newBackend() + _, err := b.Stat(context.Background(), filepath.Join(t.TempDir(), "nonexistent", "path")) + require.Error(t, err) + assert.True(t, os.IsNotExist(err)) +} + +func TestStat_CancelledContext(t *testing.T) { + b := newBackend() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := b.Stat(ctx, t.TempDir()) + require.ErrorIs(t, err, context.Canceled) +} + +func TestLstat_RegularFile(t *testing.T) { + b := newBackend() + dir := t.TempDir() + path := filepath.Join(dir, "file.txt") + writeFile(t, path, "data") + + info, err := b.Lstat(context.Background(), path) + require.NoError(t, err) + assert.Equal(t, path, info.Path) + assert.False(t, info.IsSymlink) + assert.False(t, info.FileID.IsZero()) + assert.Equal(t, uint64(1), info.Nlinks) +} + +func TestLstat_Symlink(t *testing.T) { + b := newBackend() + dir := t.TempDir() + target := filepath.Join(dir, "target.txt") + writeFile(t, target, "data") + link := filepath.Join(dir, "link.txt") + require.NoError(t, os.Symlink(target, link)) + + info, err := b.Lstat(context.Background(), link) + require.NoError(t, err) + assert.True(t, info.IsSymlink) + // Symlinks are not regular files, so FileID should be zero. + assert.True(t, info.FileID.IsZero()) +} + +func TestLstat_Hardlink(t *testing.T) { + b := newBackend() + dir := t.TempDir() + original := filepath.Join(dir, "original.txt") + writeFile(t, original, "shared") + hardlink := filepath.Join(dir, "hardlink.txt") + require.NoError(t, os.Link(original, hardlink)) + + origInfo, err := b.Lstat(context.Background(), original) + require.NoError(t, err) + linkInfo, err := b.Lstat(context.Background(), hardlink) + require.NoError(t, err) + + assert.Equal(t, origInfo.FileID, linkInfo.FileID) + assert.Equal(t, uint64(2), origInfo.Nlinks) + assert.Equal(t, uint64(2), linkInfo.Nlinks) +} + +func TestReadDir(t *testing.T) { + b := newBackend() + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "a.txt"), "a") + writeFile(t, filepath.Join(dir, "b.txt"), "b") + require.NoError(t, os.Mkdir(filepath.Join(dir, "subdir"), 0o755)) + + entries, err := b.ReadDir(context.Background(), dir) + require.NoError(t, err) + assert.Len(t, entries, 3) + + names := make([]string, len(entries)) + for i, e := range entries { + names[i] = e.Name + } + assert.Contains(t, names, "a.txt") + assert.Contains(t, names, "b.txt") + assert.Contains(t, names, "subdir") +} + +// syslessFileInfo is a regular-file FileInfo whose Sys() carries no +// platform stat data, so hardlink.GetFileID must fail on it. +type syslessFileInfo struct{} + +func (syslessFileInfo) Name() string { return "x.mkv" } +func (syslessFileInfo) Size() int64 { return 42 } +func (syslessFileInfo) Mode() fs.FileMode { return 0 } +func (syslessFileInfo) ModTime() time.Time { return time.Unix(1700000000, 0) } +func (syslessFileInfo) IsDir() bool { return false } +func (syslessFileInfo) Sys() any { return nil } + +func TestLstatConversion_FileIDFailureDegrades(t *testing.T) { + // Identity failure must land in FileIDErr with the rest of the + // metadata intact — not fail the conversion (one identity-opaque + // file must not abort callers that only need size/mtime). + path := filepath.Join(t.TempDir(), "missing", "x.mkv") + info := osFileInfoToLstat(syslessFileInfo{}, path) + + require.Error(t, info.FileIDErr) + assert.True(t, info.FileID.IsZero()) + assert.Equal(t, path, info.Path) + assert.Equal(t, int64(42), info.Size) +} + +func TestWalkDir_Basic(t *testing.T) { + b := newBackend() + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "a.txt"), "aaa") + writeFile(t, filepath.Join(dir, "sub", "b.txt"), "bbb") + + ch, err := b.WalkDir(context.Background(), dir, fsops.WalkOptions{}) + require.NoError(t, err) + + var entries []fsops.WalkEntry + for e := range ch { + entries = append(entries, e) + } + + // Should have: dir itself, a.txt, sub/, sub/b.txt + assert.GreaterOrEqual(t, len(entries), 4) + + paths := make([]string, len(entries)) + for i, e := range entries { + paths[i] = e.RelPath + } + assert.Contains(t, paths, "a.txt") + assert.Contains(t, paths, filepath.Join("sub", "b.txt")) +} + +func TestWalkDir_SkipHidden(t *testing.T) { + b := newBackend() + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "visible.txt"), "v") + writeFile(t, filepath.Join(dir, ".hidden"), "h") + writeFile(t, filepath.Join(dir, ".hiddendir", "inside.txt"), "i") + + ch, err := b.WalkDir(context.Background(), dir, fsops.WalkOptions{SkipHidden: true}) + require.NoError(t, err) + + var relPaths []string + for e := range ch { + relPaths = append(relPaths, e.RelPath) + } + assert.Contains(t, relPaths, "visible.txt") + assert.NotContains(t, relPaths, ".hidden") + assert.NotContains(t, relPaths, filepath.Join(".hiddendir", "inside.txt")) +} + +func TestWalkDir_IgnoreDirNames(t *testing.T) { + b := newBackend() + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "keep.txt"), "k") + writeFile(t, filepath.Join(dir, "node_modules", "pkg.js"), "p") + writeFile(t, filepath.Join(dir, "$recycle.bin", "old.mkv"), "r") + + ch, err := b.WalkDir(context.Background(), dir, fsops.WalkOptions{IgnoreDirNames: []string{"node_modules", "$RECYCLE.BIN"}}) + require.NoError(t, err) + + var relPaths []string + for e := range ch { + relPaths = append(relPaths, e.RelPath) + } + assert.Contains(t, relPaths, "keep.txt") + assert.NotContains(t, relPaths, filepath.Join("node_modules", "pkg.js")) + // Matching is case-insensitive: metadata dir case varies on disk. + assert.NotContains(t, relPaths, filepath.Join("$recycle.bin", "old.mkv")) +} + +func TestWalkDir_ContextCancellation(t *testing.T) { + b := newBackend() + dir := t.TempDir() + // More files than the walk channel buffers (64), so the walk cannot + // complete before the first receive and cancellation must cut it short. + const total = 150 + for i := range total { + writeFile(t, filepath.Join(dir, fmt.Sprintf("f%03d.txt", i)), "x") + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, err := b.WalkDir(ctx, dir, fsops.WalkOptions{}) + require.NoError(t, err) + + // Read a few entries then cancel. + count := 0 + for range ch { + count++ + if count >= 3 { + cancel() + break + } + } + // Drain remaining entries (channel should close promptly). + for range ch { + count++ + } + // After cancel, at most the buffered entries (64) plus one in-flight + // send can still arrive; anything near the full tree means the walk + // ignored cancellation. + assert.Less(t, count, 100) +} + +func TestStatfs_FilePath(t *testing.T) { + // The contract is "the filesystem containing path" — a regular file is a + // valid argument. unix.Statfs accepts files natively; the Windows + // implementation resolves to the containing directory. + b := newBackend() + dir := t.TempDir() + file := filepath.Join(dir, "content.mkv") + writeFile(t, file, "data") + + result, err := b.Statfs(context.Background(), file) + require.NoError(t, err) + assert.Positive(t, result.BytesTotal) +} + +func TestWalkDir_UnreadableSubdirEmitsEntryErrAndContinues(t *testing.T) { + // The per-entry Err path is the load-bearing half of the WalkDir error + // contract: an unreadable subdirectory must surface as an entry with Err + // set while the rest of the walk continues. + if runtime.GOOS == "windows" { + t.Skip("0o000 permissions are not enforced on Windows") + } + if os.Geteuid() == 0 { + t.Skip("root ignores directory permissions") + } + + b := newBackend() + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "readable.txt"), "r") + locked := filepath.Join(dir, "locked") + require.NoError(t, os.Mkdir(locked, 0o700)) + writeFile(t, filepath.Join(locked, "hidden.txt"), "h") + require.NoError(t, os.Chmod(locked, 0o000)) + t.Cleanup(func() { _ = os.Chmod(locked, 0o700) }) + + ch, err := b.WalkDir(context.Background(), dir, fsops.WalkOptions{}) + require.NoError(t, err) + + var errEntries, okPaths []string + for e := range ch { + if e.Err != nil { + errEntries = append(errEntries, e.Path) + } else { + okPaths = append(okPaths, e.RelPath) + } + } + assert.Contains(t, errEntries, locked, "unreadable dir surfaces as an entry with Err") + assert.Contains(t, okPaths, "readable.txt", "walk continues past the unreadable dir") +} + +func TestWalkDir_NonexistentRoot(t *testing.T) { + b := newBackend() + _, err := b.WalkDir(context.Background(), filepath.Join(t.TempDir(), "nonexistent", "root"), fsops.WalkOptions{}) + require.Error(t, err) +} + +func TestWalkDir_WithFileID(t *testing.T) { + b := newBackend() + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "file.txt"), "data") + + ch, err := b.WalkDir(context.Background(), dir, fsops.WalkOptions{WantFileID: true}) + require.NoError(t, err) + + foundFileWithID := false + for e := range ch { + if e.RelPath == "file.txt" && !e.FileID.IsZero() { + foundFileWithID = true + } + } + assert.True(t, foundFileWithID) +} + +func TestStatfs(t *testing.T) { + b := newBackend() + result, err := b.Statfs(context.Background(), t.TempDir()) + require.NoError(t, err) + assert.Positive(t, result.BytesAvailable) + assert.Positive(t, result.BytesTotal) + assert.LessOrEqual(t, result.BytesAvailable, result.BytesTotal) +} + +func TestSameFilesystem_SameDir(t *testing.T) { + b := newBackend() + dir := t.TempDir() + d1 := filepath.Join(dir, "a") + d2 := filepath.Join(dir, "b") + require.NoError(t, os.Mkdir(d1, 0o755)) + require.NoError(t, os.Mkdir(d2, 0o755)) + + same, err := b.SameFilesystem(context.Background(), d1, d2) + require.NoError(t, err) + assert.True(t, same) +} + +func TestMkdirAll(t *testing.T) { + b := newBackend() + dir := t.TempDir() + deep := filepath.Join(dir, "a", "b", "c") + + require.NoError(t, b.MkdirAll(context.Background(), deep, 0o755)) + + fi, err := os.Stat(deep) + require.NoError(t, err) + assert.True(t, fi.IsDir()) +} + +func TestRemove_File(t *testing.T) { + b := newBackend() + dir := t.TempDir() + path := filepath.Join(dir, "file.txt") + writeFile(t, path, "data") + + require.NoError(t, b.Remove(context.Background(), path, fsops.RemoveOptions{})) + _, err := os.Stat(path) + assert.True(t, os.IsNotExist(err)) +} + +func TestRemove_Recursive(t *testing.T) { + b := newBackend() + dir := t.TempDir() + sub := filepath.Join(dir, "sub") + writeFile(t, filepath.Join(sub, "file.txt"), "data") + + require.NoError(t, b.Remove(context.Background(), sub, fsops.RemoveOptions{Recursive: true})) + _, err := os.Stat(sub) + assert.True(t, os.IsNotExist(err)) +} + +func TestRemove_NonRecursive_DirFails(t *testing.T) { + b := newBackend() + dir := t.TempDir() + sub := filepath.Join(dir, "sub") + writeFile(t, filepath.Join(sub, "file.txt"), "data") + + err := b.Remove(context.Background(), sub, fsops.RemoveOptions{}) + require.Error(t, err, "removing non-empty dir without recursive should fail") +} + +func TestHardlinkTree_CreateAndRemove(t *testing.T) { + b := newBackend() + dir := t.TempDir() + + // Create source files. + src1 := filepath.Join(dir, "source", "a.mkv") + src2 := filepath.Join(dir, "source", "b.srt") + writeFile(t, src1, "video data") + writeFile(t, src2, "subtitle data") + + treeRoot := filepath.Join(dir, "tree") + plan := &hardlinktree.TreePlan{ + RootDir: treeRoot, + Files: []hardlinktree.FilePlan{ + {SourcePath: src1, TargetPath: filepath.Join(treeRoot, "a.mkv")}, + {SourcePath: src2, TargetPath: filepath.Join(treeRoot, "b.srt")}, + }, + } + + // Create the tree. + result, err := b.HardlinkTree(context.Background(), plan) + require.NoError(t, err) + assert.Equal(t, 2, result.Created) + + // Verify files exist and are hardlinks. + fi1, err := os.Stat(filepath.Join(treeRoot, "a.mkv")) + require.NoError(t, err) + assert.Equal(t, int64(10), fi1.Size()) + + srcFi, err := os.Stat(src1) + require.NoError(t, err) + assert.True(t, os.SameFile(srcFi, fi1)) + + // Remove the tree using the create result's recorded files/dirs. + require.NoError(t, b.RemoveTree(context.Background(), result)) + _, err = os.Stat(filepath.Join(treeRoot, "a.mkv")) + assert.True(t, os.IsNotExist(err)) + _, err = os.Stat(treeRoot) + assert.True(t, os.IsNotExist(err)) +} + +func TestHardlinkTree_SkippedExists(t *testing.T) { + b := newBackend() + dir := t.TempDir() + src := filepath.Join(dir, "source", "a.mkv") + writeFile(t, src, "video data") + + treeRoot := filepath.Join(dir, "tree") + plan := &hardlinktree.TreePlan{ + RootDir: treeRoot, + Files: []hardlinktree.FilePlan{ + {SourcePath: src, TargetPath: filepath.Join(treeRoot, "a.mkv")}, + }, + } + + first, err := b.HardlinkTree(context.Background(), plan) + require.NoError(t, err) + assert.Equal(t, 1, first.Created) + assert.Equal(t, 0, first.SkippedExists) + + // Re-creating the same plan is idempotent: the existing link is skipped + // and NOT recorded as this call's work. + second, err := b.HardlinkTree(context.Background(), plan) + require.NoError(t, err) + assert.Equal(t, 0, second.Created) + assert.Equal(t, 1, second.SkippedExists) + assert.Empty(t, second.Files) + + // Removing the second (empty) result must leave the first call's link alone. + require.NoError(t, b.RemoveTree(context.Background(), second)) + _, err = os.Stat(filepath.Join(treeRoot, "a.mkv")) + require.NoError(t, err) +} + +func TestSupportsReflink(t *testing.T) { + b := newBackend() + dir := t.TempDir() + + supported, reason, err := b.SupportsReflink(context.Background(), dir) + require.NoError(t, err) + // Result depends on the filesystem, but the call should not error. + _ = supported + _ = reason +} + +func TestWalkDir_IgnorePaths(t *testing.T) { + b := newBackend() + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "keep.txt"), "k") + ignoredFile := filepath.Join(dir, "ignored.txt") + writeFile(t, ignoredFile, "i") + + ch, err := b.WalkDir(context.Background(), dir, fsops.WalkOptions{ + IgnorePaths: []string{ignoredFile}, + }) + require.NoError(t, err) + + var relPaths []string + for e := range ch { + relPaths = append(relPaths, e.RelPath) + } + assert.Contains(t, relPaths, "keep.txt") + assert.NotContains(t, relPaths, "ignored.txt") +} diff --git a/internal/fsops/local/statfs_unix.go b/internal/fsops/local/statfs_unix.go new file mode 100644 index 000000000..5794fddc7 --- /dev/null +++ b/internal/fsops/local/statfs_unix.go @@ -0,0 +1,30 @@ +// Copyright (c) 2025-2026, s0up and the autobrr contributors. +// SPDX-License-Identifier: GPL-2.0-or-later + +//go:build !windows + +package local + +import ( + "context" + "fmt" + + "golang.org/x/sys/unix" + + "github.com/autobrr/qui/internal/fsops" +) + +func (b *Backend) Statfs(ctx context.Context, path string) (*fsops.StatfsResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var stat unix.Statfs_t + if err := unix.Statfs(path, &stat); err != nil { + return nil, fmt.Errorf("statfs %s: %w", path, err) + } + //nolint:gosec,unconvert // int64 cast needed for macOS (uint32) even though Linux is already int64 + return &fsops.StatfsResult{ + BytesAvailable: int64(stat.Bavail) * int64(stat.Bsize), + BytesTotal: int64(stat.Blocks) * int64(stat.Bsize), + }, nil +} diff --git a/internal/fsops/local/statfs_windows.go b/internal/fsops/local/statfs_windows.go new file mode 100644 index 000000000..deace41e5 --- /dev/null +++ b/internal/fsops/local/statfs_windows.go @@ -0,0 +1,53 @@ +// Copyright (c) 2025-2026, s0up and the autobrr contributors. +// SPDX-License-Identifier: GPL-2.0-or-later + +//go:build windows + +package local + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/windows" + + "github.com/autobrr/qui/internal/fsops" +) + +func (b *Backend) Statfs(ctx context.Context, path string) (*fsops.StatfsResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + // GetDiskFreeSpaceEx requires a directory, while the unix reference + // (unix.Statfs) accepts files. Stat first so a missing path keeps its + // portable fs.ErrNotExist, then query the containing directory for + // non-directories. + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + if !fi.IsDir() { + path = filepath.Dir(path) + } + + pathPtr, err := windows.UTF16PtrFromString(path) + if err != nil { + return nil, fmt.Errorf("invalid path %s: %w", path, err) + } + + var freeBytesAvailable, totalBytes, totalFreeBytes uint64 + if err := windows.GetDiskFreeSpaceEx( + pathPtr, &freeBytesAvailable, &totalBytes, &totalFreeBytes, + ); err != nil { + return nil, fmt.Errorf("GetDiskFreeSpaceEx %s: %w", path, err) + } + + //nolint:gosec // uint64 to int64: disk sizes won't exceed int64 max + return &fsops.StatfsResult{ + BytesAvailable: int64(freeBytesAvailable), + BytesTotal: int64(totalBytes), + }, nil +} diff --git a/internal/fsops/noop.go b/internal/fsops/noop.go new file mode 100644 index 000000000..cc671f526 --- /dev/null +++ b/internal/fsops/noop.go @@ -0,0 +1,72 @@ +// Copyright (c) 2025-2026, s0up and the autobrr contributors. +// SPDX-License-Identifier: GPL-2.0-or-later + +package fsops + +import ( + "context" + "io/fs" + + "github.com/autobrr/qui/pkg/hardlinktree" +) + +// noopBackend implements Backend by returning ErrNoFilesystemAccess for every +// operation. Used for instances that have no filesystem access configured. +// Context cancellation takes precedence over ErrNoFilesystemAccess so callers +// get the correct error when a request is cancelled. +type noopBackend struct{} + +var _ Backend = noopBackend{} + +// noopErr returns the context error if cancelled, otherwise ErrNoFilesystemAccess. +func noopErr(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + return ErrNoFilesystemAccess +} + +func (noopBackend) Stat(ctx context.Context, _ string) (*LstatInfo, error) { + return nil, noopErr(ctx) +} +func (noopBackend) Lstat(ctx context.Context, _ string) (*LstatInfo, error) { + return nil, noopErr(ctx) +} +func (noopBackend) ReadDir(ctx context.Context, _ string) ([]DirEntry, error) { + return nil, noopErr(ctx) +} +func (noopBackend) WalkDir(ctx context.Context, _ string, _ WalkOptions) (<-chan WalkEntry, error) { + return nil, noopErr(ctx) +} +func (noopBackend) Statfs(ctx context.Context, _ string) (*StatfsResult, error) { + return nil, noopErr(ctx) +} +func (noopBackend) SameFilesystem(ctx context.Context, _, _ string) (bool, error) { + return false, noopErr(ctx) +} +func (noopBackend) MkdirAll(ctx context.Context, _ string, _ fs.FileMode) error { + return noopErr(ctx) +} +func (noopBackend) Remove(ctx context.Context, _ string, _ RemoveOptions) error { + return noopErr(ctx) +} +func (noopBackend) HardlinkTree(ctx context.Context, _ *hardlinktree.TreePlan) (*TreeCreateResult, error) { + return nil, noopErr(ctx) +} +func (noopBackend) ReflinkTree(ctx context.Context, _ *hardlinktree.TreePlan) (*TreeCreateResult, error) { + return nil, noopErr(ctx) +} +func (noopBackend) RemoveTree(ctx context.Context, created *TreeCreateResult) error { + if err := ctx.Err(); err != nil { + return err + } + // The interface promises a nil handle is safe: there is nothing to + // remove, so a defensive RemoveTree(nil) must not error. + if created == nil { + return nil + } + return ErrNoFilesystemAccess +} +func (noopBackend) SupportsReflink(ctx context.Context, _ string) (bool, string, error) { + return false, "", noopErr(ctx) +} diff --git a/internal/fsops/pool.go b/internal/fsops/pool.go new file mode 100644 index 000000000..2604566e8 --- /dev/null +++ b/internal/fsops/pool.go @@ -0,0 +1,55 @@ +// Copyright (c) 2025-2026, s0up and the autobrr contributors. +// SPDX-License-Identifier: GPL-2.0-or-later + +package fsops + +import ( + "context" + "fmt" + + "github.com/autobrr/qui/internal/models" +) + +// Pool resolves an instance ID to the appropriate Backend. For instances with +// local filesystem access it returns the local backend; for instances without +// access it returns a noop backend that errors on every call. A future remote +// backend slots in here for instances with SSH access configured. +type Pool struct { + instanceStore instanceGetter + local Backend +} + +// instanceGetter is the subset of models.InstanceStore that Pool needs. +// Using an interface keeps the dependency narrow and simplifies testing. +type instanceGetter interface { + Get(ctx context.Context, id int) (*models.Instance, error) +} + +// NewPool creates a Backend pool backed by the given instance store and local backend. +func NewPool(store instanceGetter, local Backend) *Pool { + return &Pool{ + instanceStore: store, + local: local, + } +} + +// GetBackend returns the appropriate Backend for the given instance ID. +// Returns ErrNoFilesystemAccess wrapped in a noop backend for instances +// without filesystem access configured. +func (p *Pool) GetBackend(ctx context.Context, instanceID int) (Backend, error) { + instance, err := p.instanceStore.Get(ctx, instanceID) + if err != nil { + return nil, fmt.Errorf("load instance %d: %w", instanceID, err) + } + if instance == nil { + return nil, fmt.Errorf("instance %d not found", instanceID) + } + + if instance.HasLocalFilesystemAccess { + return p.local, nil + } + + // Future: return a remote backend for instances with SSH access configured. + + return noopBackend{}, nil +} diff --git a/internal/fsops/pool_test.go b/internal/fsops/pool_test.go new file mode 100644 index 000000000..346c07587 --- /dev/null +++ b/internal/fsops/pool_test.go @@ -0,0 +1,138 @@ +// Copyright (c) 2025-2026, s0up and the autobrr contributors. +// SPDX-License-Identifier: GPL-2.0-or-later + +package fsops + +import ( + "context" + "io/fs" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/autobrr/qui/internal/models" + "github.com/autobrr/qui/pkg/hardlinktree" +) + +// fakeInstanceStore implements instanceGetter for tests. +type fakeInstanceStore struct { + instances map[int]*models.Instance +} + +func (s *fakeInstanceStore) Get(_ context.Context, id int) (*models.Instance, error) { + inst, ok := s.instances[id] + if !ok { + return nil, nil + } + return inst, nil +} + +// fakeBackend is a minimal Backend for verifying which backend the pool returns. +type fakeBackend struct{ kind string } + +func (f fakeBackend) Stat(context.Context, string) (*LstatInfo, error) { + return &LstatInfo{}, nil +} +func (f fakeBackend) Lstat(context.Context, string) (*LstatInfo, error) { return nil, nil } +func (f fakeBackend) ReadDir(context.Context, string) ([]DirEntry, error) { + return nil, nil +} +func (f fakeBackend) WalkDir(context.Context, string, WalkOptions) (<-chan WalkEntry, error) { + return nil, nil +} +func (f fakeBackend) Statfs(context.Context, string) (*StatfsResult, error) { return nil, nil } +func (f fakeBackend) SameFilesystem(context.Context, string, string) (bool, error) { + return false, nil +} +func (f fakeBackend) MkdirAll(context.Context, string, fs.FileMode) error { return nil } +func (f fakeBackend) Remove(context.Context, string, RemoveOptions) error { + return nil +} +func (f fakeBackend) HardlinkTree(context.Context, *hardlinktree.TreePlan) (*TreeCreateResult, error) { + return nil, nil +} +func (f fakeBackend) ReflinkTree(context.Context, *hardlinktree.TreePlan) (*TreeCreateResult, error) { + return nil, nil +} +func (f fakeBackend) RemoveTree(context.Context, *TreeCreateResult) error { return nil } +func (f fakeBackend) SupportsReflink(context.Context, string) (bool, string, error) { + return false, "", nil +} + +func TestPool_LocalAccess(t *testing.T) { + store := &fakeInstanceStore{instances: map[int]*models.Instance{ + 1: {ID: 1, HasLocalFilesystemAccess: true}, + }} + local := fakeBackend{kind: "local"} + pool := NewPool(store, local) + + backend, err := pool.GetBackend(context.Background(), 1) + require.NoError(t, err) + assert.Equal(t, Backend(local), backend) +} + +func TestPool_NoAccess(t *testing.T) { + store := &fakeInstanceStore{instances: map[int]*models.Instance{ + 2: {ID: 2, HasLocalFilesystemAccess: false}, + }} + local := fakeBackend{kind: "local"} + pool := NewPool(store, local) + + backend, err := pool.GetBackend(context.Background(), 2) + require.NoError(t, err) + + // All ops should return ErrNoFilesystemAccess. + _, err = backend.Stat(context.Background(), "/any") + require.ErrorIs(t, err, ErrNoFilesystemAccess) + + err = backend.MkdirAll(context.Background(), "/any", 0o755) + require.ErrorIs(t, err, ErrNoFilesystemAccess) +} + +func TestPool_InstanceNotFound(t *testing.T) { + store := &fakeInstanceStore{instances: map[int]*models.Instance{}} + pool := NewPool(store, fakeBackend{}) + + _, err := pool.GetBackend(context.Background(), 999) + require.Error(t, err) + assert.Contains(t, err.Error(), "not found") +} + +func TestNoopBackend_AllMethodsError(t *testing.T) { + b := noopBackend{} + ctx := context.Background() + + _, err := b.Stat(ctx, "/x") + require.ErrorIs(t, err, ErrNoFilesystemAccess) + + _, err = b.Lstat(ctx, "/x") + require.ErrorIs(t, err, ErrNoFilesystemAccess) + + _, err = b.ReadDir(ctx, "/x") + require.ErrorIs(t, err, ErrNoFilesystemAccess) + + _, err = b.WalkDir(ctx, "/x", WalkOptions{}) + require.ErrorIs(t, err, ErrNoFilesystemAccess) + + _, err = b.Statfs(ctx, "/x") + require.ErrorIs(t, err, ErrNoFilesystemAccess) + + _, err = b.SameFilesystem(ctx, "/x", "/y") + require.ErrorIs(t, err, ErrNoFilesystemAccess) + + require.ErrorIs(t, b.MkdirAll(ctx, "/x", 0o755), ErrNoFilesystemAccess) + require.ErrorIs(t, b.Remove(ctx, "/x", RemoveOptions{}), ErrNoFilesystemAccess) + // A nil handle means nothing to remove — safe on every backend. + require.NoError(t, b.RemoveTree(ctx, nil)) + require.ErrorIs(t, b.RemoveTree(ctx, &TreeCreateResult{Files: []string{"/x"}}), ErrNoFilesystemAccess) + + _, err = b.HardlinkTree(ctx, nil) + require.ErrorIs(t, err, ErrNoFilesystemAccess) + + _, err = b.ReflinkTree(ctx, nil) + require.ErrorIs(t, err, ErrNoFilesystemAccess) + + _, _, err = b.SupportsReflink(ctx, "/x") + require.ErrorIs(t, err, ErrNoFilesystemAccess) +} diff --git a/internal/fsops/types.go b/internal/fsops/types.go new file mode 100644 index 000000000..1e846af4e --- /dev/null +++ b/internal/fsops/types.go @@ -0,0 +1,87 @@ +// Copyright (c) 2025-2026, s0up and the autobrr contributors. +// SPDX-License-Identifier: GPL-2.0-or-later + +// Package fsops defines the Backend interface that abstracts filesystem +// operations for qui's services. Current implementations: Local (delegates +// to os.* on the qui host) and Noop (returns ErrNoFilesystemAccess for +// instances without filesystem access). A Remote implementation (SSH-backed) +// is planned. Services still call os.* directly; they adopt this interface +// in a separate callsite migration, after which the transport is transparent +// to them. +package fsops + +import ( + "io/fs" + "time" + + "github.com/autobrr/qui/pkg/hardlink" +) + +// FileInfo holds metadata from a Stat call. +type FileInfo struct { + Path string + Size int64 + ModTime time.Time + IsDir bool + IsSymlink bool + Mode fs.FileMode +} + +// DirEntry holds one entry from a ReadDir call. +type DirEntry struct { + Name string + IsDir bool + IsSymlink bool +} + +// LstatInfo holds metadata from an Lstat call, including hardlink identity. +type LstatInfo struct { + FileInfo + FileID hardlink.FileID + Nlinks uint64 + // FileIDErr is set when FileID/Nlinks could not be resolved (e.g. a + // Windows ACL denial on an otherwise statable file). The rest of the + // struct is still valid; callers that need identity must check this + // instead of treating the whole stat as failed. + FileIDErr error +} + +// WalkEntry is emitted by WalkDir for each filesystem entry encountered. +type WalkEntry struct { + LstatInfo + RelPath string + Err error +} + +// WalkOptions controls the behavior of a WalkDir call. +type WalkOptions struct { + SkipHidden bool + // IgnoreDirNames are directory basenames to skip, matched case-insensitively + // (OS/NAS metadata dirs like $RECYCLE.BIN and @eaDir vary in on-disk case). + IgnoreDirNames []string + IgnorePaths []string + // WantFileID populates FileID and Nlinks on regular-file entries. + WantFileID bool +} + +// StatfsResult holds filesystem space information. +type StatfsResult struct { + BytesAvailable int64 + BytesTotal int64 +} + +// RemoveOptions controls the behavior of a Remove call. +type RemoveOptions struct { + Recursive bool +} + +// TreeCreateResult holds the outcome of a HardlinkTree or ReflinkTree call. +// Files and Dirs record what the call actually created on disk (pre-existing +// paths are excluded), so RemoveTree can undo exactly this call's work without +// touching links shared with sibling torrents (discussion #2282). +type TreeCreateResult struct { + Created int + SkippedExists int + Files []string + Dirs []string +} diff --git a/pkg/sharedextents/sharedextents_windows_test.go b/pkg/sharedextents/sharedextents_windows_test.go index 09262af83..e070c5b3f 100644 --- a/pkg/sharedextents/sharedextents_windows_test.go +++ b/pkg/sharedextents/sharedextents_windows_test.go @@ -142,13 +142,14 @@ func TestFilesShareAllocationReFS(t *testing.T) { require.NoError(t, err) require.NoError(t, os.WriteFile(source, data, 0o600)) //nolint:gosec // Path is under verified test temp directory. require.NoError(t, os.WriteFile(copyPath, data, 0o600)) //nolint:gosec // Path is under verified test temp directory. - require.NoError(t, reflinktree.Create(&hardlinktree.TreePlan{ + _, err = reflinktree.Create(&hardlinktree.TreePlan{ RootDir: dir, Files: []hardlinktree.FilePlan{{ SourcePath: source, TargetPath: clone, }}, - })) + }) + require.NoError(t, err) shared, err := FilesShareAllocation(source, clone) require.NoError(t, err)