diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 74baade6c..667cf6da2 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -291,6 +291,7 @@ jobs: test/sandbox/mount-point-manifests.test.ts \ test/sandbox/mount-point-liveness.test.ts \ test/sandbox/mount-point-passes.test.ts \ + test/sandbox/mount-point-hosts.test.ts \ test/sandbox/stale-mount-point.test.ts \ test/sandbox/linux-bwrap-args-file.test.ts diff --git a/README.md b/README.md index 029ac9c87..0f280fd21 100644 --- a/README.md +++ b/README.md @@ -1130,21 +1130,23 @@ Filesystem restrictions are enforced at the OS level: - No mount point is made where none can be: in a directory that refuses this user, when the refusal is not this user's to lift (another user's directory it may not write, a read-only file system, an immutable directory). bubblewrap could not create the path there, and would stop every command from starting; the sandboxed command cannot create it either, so the deny holds without a mount. The directories above the path are kept from being moved aside as for any other deny. A directory of your own that only its mode closes keeps its mount point, because the command could open it with `chmod`. Should somebody else lift the refusal while a command runs (the owner, an administrator, a remount), that command can create the path; the next one gets its mount point. - On a Windows drive under WSL 2 (`/mnt/c`, a 9p mount) the Windows ACL decides what can be created, and Linux is told nothing of it: every folder looks like yours and writable. The only way to know is to try, so there, and only there, the library makes the mount point itself while it wraps the command, instead of leaving it to bubblewrap when the command runs: the same empty file or directory, at the same path, removed by the same clean-up. It is on the drive from the wrap on, also for a command that is never run, until `cleanupAfterCommand()` or `reset()`. It is made once the manifests have been read, so a wrap that finds them unreadable from the start makes nothing, and before the wrap's own manifest is published: a wrap that comes to nothing in between has it removed by the first clean-up with none of the process's commands outstanding, or at its exit, and a process killed in between leaves it with no manifest to name it. A file deny in a folder that takes subfolders but no files (`C:\`) is covered by an empty directory. If nothing can be made and the folder refuses even to be given the mode it has, no mount point is made, as above. -- **The manifest directories.** Two processes protect each other's sandboxes only when each reads what the other writes, and a mount point whose manifest is lost stays for good. So the places follow from the user id, not from the environment (a login shell and what a service, an IDE, cron or `docker exec` starts differ in `XDG_RUNTIME_DIR`, two shells can differ in `TMPDIR`), and the first of them is local to the host and outlives a logout and a reboot. In the order they are tried for writing: (1) `/var/tmp/srt-mount-points-`; (2) `/run/user//srt-mount-points`, where `/run/user/` is a real directory of the user's, which to be written under also has to have no permission bits for group or others (they do not count for reading: a command that may write that directory can `chmod` it, and must not make other processes read less by that); (3) `/tmp/srt-mount-points-`, whatever `TMPDIR` says; (4) `$XDG_RUNTIME_DIR/srt-mount-points`; (5) `/srt-mount-points-`. The later ones are for where `/var/tmp` is missing or read-only: a minimal image, a `--read-only` container, inside another sandbox. Not the home directory: a network home caches directory listings, and a cleanup relies on the next listing in another process showing a rename; a home is often inside a write root; nothing ages what is left there; and in a home shared between hosts another boot id may be another running host's. A process writes in the first that is a real directory (a symlink at the name is refused), owned by the user, mode `0700` (it is made, and made so), and writable; a candidate under `/dev`, `/proc` or `/sys` is passed over (the sandbox mounts a fresh `/dev` and `/proc` over whatever was bound there). It believes, that is reads for every wrap, every cleanup and `liveMountPoints()`, and collects in: (1) to (3), each for as long as it passes those checks, writable or not; (4) and (5) only when it can write in none of the first three, because only processes with the same environment keep those two out of a sandbox's reach; and the one it has settled on. A manifest in any of them keeps what it names, what cannot be read in any of them puts everything in doubt, and all that is named in one that cannot be written from where the process runs is kept, since nothing can be claimed there. Each manifest records the boot (`/proc/sys/kernel/random/boot_id`), and one of another boot is finished whatever else it says: a reboot ends every record, and the next cleanup removes what was left. That counts only where the manifest's directory is on a kind of file system that this kernel alone reaches (tmpfs, ext2/3/4, xfs, btrfs, overlayfs, zfs, f2fs). On any other (NFS, 9p, virtiofs, FUSE) a manifest may be another running kernel's, and is taken for this kernel's only when its boot id and the reader's can both be read and are equal (two hosts each in their initial PID namespace name that namespace alike); any other counts as one that records no PID namespace, live for seven days, which is also how long what an earlier boot left waits on a local kind that is not on that list. overlayfs says nothing about what its upper layer sits on. systemd ages `/var/tmp` at 30 days (`/tmp` at 10) by access time, and every wrap and cleanup reads every manifest, so only a sandbox that runs that long with no other srt activity is exposed. Every release that writes manifests keeps them in these places, none was released that kept them elsewhere, and a later one has to go on reading them: two versions side by side protect each other on that. When none of them will do (all are read-only from where the process runs, as inside another sandbox), a process that has mount points to record makes a directory of its own under the temp dir, and removes it again at `reset()` or on exit once it is empty: its own sandboxes are covered as before, but no other process sees its manifests or collects what it leaves behind. With no directory at all, the process keeps its mount points in memory and removes them at its own cleanup, once none of its commands is outstanding; no other process knows of them. It does the same when a manifest cannot be written to a directory that is there (a full disk), and its cleanup then keeps every path that a manifest of another process names. Every sandbox that restricts writes has every one of those names bound read-only, as the wrapping process sees them and whichever of them it writes in, and its own directory if that is none of them, whether or not its wrap has mount points. One that a read deny already hides is left hidden: the command cannot reach it, and bound on top of the deny it could read it. The wrap makes the ones that are not there first: the command runs as the same user and may be allowed to write `/tmp`, so a directory it made there itself would pass every check of owner and mode, and be believed. While one of them cannot be looked at (an error of the moment, not "not there"), no command that restricts writes is handed out (`mount_points_unreadable`): left unbound, it would be believed again a moment later. One that can be neither made nor bound is not believed, and the command cannot bring it into being either: where its parent is missing, `/var`, `/run/user` and `/` are root's to write; where its parent is read-only, so is every mount made from it; where another user has the name, in a sticky directory, only that user or root can remove or rename it. The directories between each of them and the write root it lies in, if it lies in one, are pinned like those above a protected path (see "Pinned directories" below), so that none of them can be renamed aside and the manifest directory made again under the old name. So a sandboxed command can neither delete a manifest nor forge one, nor make a manifest directory first, whichever of them the wrapping process writes in; the cases this does not reach are listed under the known limits. A cleanup takes no lock and waits for nothing: any number may run at once, in any processes. +- **The manifest directories.** Two processes protect each other's sandboxes only when each reads what the other writes, and a mount point whose manifest is lost stays for good. So the places follow from the user id, not from the environment (a login shell and what a service, an IDE, cron or `docker exec` starts differ in `XDG_RUNTIME_DIR`, two shells can differ in `TMPDIR`), and the first of them is local to the host and outlives a logout and a reboot. In the order they are tried for writing: (1) `/var/tmp/srt-mount-points-`; (2) `/run/user//srt-mount-points`, where `/run/user/` is a real directory of the user's, which to be written under also has to have no permission bits for group or others (they do not count for reading: a command that may write that directory can `chmod` it, and must not make other processes read less by that); (3) `/tmp/srt-mount-points-`, whatever `TMPDIR` says; (4) `$XDG_RUNTIME_DIR/srt-mount-points`; (5) `/srt-mount-points-`. The later ones are for where `/var/tmp` is missing or read-only: a minimal image, a `--read-only` container, inside another sandbox. A sixth, `/.srt-mount-points`, is written and believed only in a home that other hosts mount too, and only for what they reach: see "Hosts that share a home" below. A process writes in the first that is a real directory (a symlink at the name is refused), owned by the user, mode `0700` (it is made, and made so), and writable; a candidate under `/dev`, `/proc` or `/sys` is passed over (the sandbox mounts a fresh `/dev` and `/proc` over whatever was bound there). It believes, that is reads for every wrap, every cleanup and `liveMountPoints()`, and collects in: (1) to (3), each for as long as it passes those checks, writable or not; (4) and (5) only when it can write in none of the first three, because only processes with the same environment keep those two out of a sandbox's reach; and the one it has settled on. A manifest in any of them keeps what it names, what cannot be read in any of them puts everything in doubt, and all that is named in one that cannot be written from where the process runs is kept, since nothing can be claimed there. Each manifest records the boot (`/proc/sys/kernel/random/boot_id`), and one of another boot is finished whatever else it says: a reboot ends every record, and the next cleanup removes what was left. That counts only where the manifest's directory is on a kind of file system that this kernel alone reaches (tmpfs, ext2/3/4, xfs, btrfs, overlayfs, zfs, f2fs). On any other (NFS, 9p, virtiofs, FUSE) a manifest may be another running kernel's, and is taken for this kernel's only when its boot id and the reader's can both be read and are equal (two hosts each in their initial PID namespace name that namespace alike); any other is live by its age, from the last change to it or to its record: for ten minutes where it says that its writer keeps it young, as every writer does that finds the directory on such a file system, and else for seven days. Ten minutes is therefore also how long what an earlier boot left waits on a local kind that is not on that list, where all its manifest names is on such a kind too; a manifest that names a path on a listed kind as well (a temp dir on tmpfs) keeps the seven days, because no reader can tell it from that of another host which alone reaches that path. overlayfs says nothing about what its upper layer sits on. systemd ages `/var/tmp` at 30 days (`/tmp` at 10) by access time, and every wrap and cleanup reads every manifest, so only a sandbox that runs that long with no other srt activity is exposed. Every release that writes manifests keeps them in these places, none was released that kept them elsewhere, and a later one has to go on reading them: two versions side by side protect each other on that. When none of them will do (all are read-only from where the process runs, as inside another sandbox), a process that has mount points to record makes a directory of its own under the temp dir, and removes it again at `reset()` or on exit once it is empty: its own sandboxes are covered as before, but no other process sees its manifests or collects what it leaves behind. With no directory at all, the process keeps its mount points in memory and removes them at its own cleanup, once none of its commands is outstanding; no other process knows of them. It does the same when a manifest cannot be written to a directory that is there (a full disk), and its cleanup then keeps every path that a manifest of another process names. Every sandbox that restricts writes has every one of those names bound read-only, as the wrapping process sees them and whichever of them it writes in, and its own directory if that is none of them, whether or not its wrap has mount points. One that a read deny already hides is left hidden: the command cannot reach it, and bound on top of the deny it could read it. The wrap makes the ones that are not there first: the command runs as the same user and may be allowed to write `/tmp`, so a directory it made there itself would pass every check of owner and mode, and be believed. While one of them cannot be looked at (an error of the moment, not "not there"), no command that restricts writes is handed out (`mount_points_unreadable`): left unbound, it would be believed again a moment later. One that can be neither made nor bound is not believed, and the command cannot bring it into being either: where its parent is missing, `/var`, `/run/user` and `/` are root's to write; where its parent is read-only, so is every mount made from it; where another user has the name, in a sticky directory, only that user or root can remove or rename it. The directories between each of them and the write root it lies in, if it lies in one, are pinned like those above a protected path (see "Pinned directories" below), so that none of them can be renamed aside and the manifest directory made again under the old name. So a sandboxed command can neither delete a manifest nor forge one, nor make a manifest directory first, whichever of them the wrapping process writes in; the cases this does not reach are listed under the known limits. A cleanup takes no lock and waits for nothing: any number may run at once, in any processes. +- **Hosts that share a home.** A mount point on a file system that several hosts mount (an NFS home on the nodes of a cluster, a project on a shared scratch) is there for all of them, and removing it on one host takes the mount away from under a sandbox on another, once that host's client finds out. A directory on one host's disk tells the others nothing. So where the user's home, as the user database has it and whatever `HOME` says, is on a file system that is not on the list above, a wrap names the mount points that are on such a file system in a second manifest, in `/.srt-mount-points` (a real directory, the user's, mode `0700`, made, bound read-only and pinned by every wrap that restricts writes, like the other places). The manifest where it always was still names all of it, so that a process of the same host that does not see the home (a service kept out of it, a container with another) reads what it did before. The one in the home names nothing but what other hosts reach, so whoever reads it can collect it whole. A command that has both binds both before anything else and puts itself on the record of each, and a claim on either refuses it its start. Every process reads the directory in the home as well as the host's own. It is a place for nothing else: a wrap that names nothing other hosts reach writes nothing there, a cleanup that has nothing to remove that they reach does not go on record there, and it does not stand in for a place of the host's that cannot be written. Where it takes no manifest (a home over its quota, which a network file system says only as the bytes are flushed, so they are), the one in the host's own place is all there is. On a host whose home is its own, is not there, or may not be entered, nothing is read or written there, and the question is asked again every time (the share may be mounted by then; that the user database knows no such user, which is also all it says during an outage, is believed for a minute, and that it gives no answer at all for ten seconds). Such a host keeps the directory out of its sandboxes' reach all the same, because other hosts may believe it: to the clients of a file server, the disk it exports is a network home. So it is bound read-only and pinned wherever it is there, and where it is not there and the command could make it (the home lies in a write root), the wrap makes it first. That empty directory is all a host with a home of its own ever sees of this. While it cannot be told which kind of home it is (the file system gives an error of the moment twice, the user database none at all), or whether the directory is there, no command that restricts writes is handed out (`mount_points_unreadable`) and nothing is removed. No host can ask after another's processes, so what a manifest of another boot says (or of another PID namespace: a container beside this one) goes by its age: it is live until neither it nor its record has been changed for ten minutes, and the process that wrote it sets its time every minute for as long as it holds on to it, by its own clock (on a timer that does not keep the process alive), and says in the manifest that it does. (Its own manifests a process knows, and asks after, wherever they are.) One that does not say so (to its writer the directory may be on a file system of its own) has seven days. What a job that was killed has left there is therefore removed at once by the next cleanup on the host it ran on, and after ten minutes by the next cleanup on any host; what it has left on that host's own disks waits for a cleanup on that host. A cleanup on the host that can ask says what it found, by renaming the claim to `.finished` (see "Another PID namespace" below), so hosts that finish with the same path at the same moment do not each leave it to the other. Of another host's manifest only the paths count that are on such a file system here as well, going by the nearest directory above them that is there, at each reading (`/data/x` there may be another file than `/data/x` here, or lie on a share this host does not mount, or has not mounted yet). One that names more is not collected from here: once it is old nothing is kept for it, and it is left to a host that reaches all it names, for seven days. So is a manifest of another boot in any other directory that both hosts read (a `/var/tmp` that diskless nodes share), where a manifest names all its wrap relies on and all of it counts. A cleanup goes on record in that directory too (the record comes to its name with its time set by the host's own clock, like a manifest), and the record of one on another host counts for a minute, like that of another PID namespace; a wrap that names nothing other hosts reach does not wait for it. What a network file system remembers is dealt with where it matters. A listing is asked of the server whenever the directory is opened (NFS's close-to-open consistency), and each side has just changed the directory itself when it counts: a cleanup goes on record before the reading it decides on, a wrap publishes before it lists the cleanups and reads the manifests again. A look-up is answered from memory for seconds to a minute, so every wrap on a host with such a home opens the directories on such a file system that hold what its profile relies on before it looks at the host for the last time, which has the client compare them with the server's: what another host has removed is then found gone by the wrap, which plans again, and by bubblewrap. That covers a mount point the plan took for the user's own file because this host still remembered it when its manifest had already gone. A cleanup does the same, directory by directory as it goes, before it removes anything there, so that it does not take a file the user has put where another host has removed a mount point for the mount point it remembers, and opens each file it is about to remove, into which another host may have written. And a wrap does it for each path it has found absent before it names the path as its own to remove: what is there after all, and named by no manifest, is a file the user has put there from another host, and the wrap plans again. To spare a server that is short of operations, a manifest there is read once by each process and followed by its name from then on (a manifest is never rewritten), the age of another host's is looked at again only when it could have grown old, the kind of file system is asked once a minute for each directory found to be on such a one, and the times are set one request at a time, without holding up the thread of the program this runs in. The limits are under "Hosts that share a home" in the known limits. - **Commands in parallel.** Call `cleanupAfterCommand()` once for each wrapped command, when that command is over. Every call, with or without an argument, lets go of the commands of this process that have run and ended, whatever else is still running, and collects what they relied on. What a call cannot tell is which command it is for, so with no argument it lets go of nothing else until it has been called once for every wrap handed out: not of the manifest of a command that is wrapped and not yet started (bubblewrap refuses to start without it), nor of the `--args` profiles. Pass the `commandId` the wrap was given, `cleanupAfterCommand({ commandId })`, to let go of that one command's manifest at once: it matters for a command that was wrapped and will never be started, whose mount points otherwise stay named until the last outstanding command has been cleaned up after. The id has to be unique among the wraps in flight. A wrapped string may be run again (a server started again from a kept string) until this process next calls `cleanupAfterCommand()` with none of its runs under way: no other process collects its manifest while this one runs. After that a run is refused. A call too many takes nothing from a running sandbox, but it is counted for some other wrap of this process, and a command that has not started by the time the count is back at zero then refuses to start. `reset()` and the exit handler release everything this process made; a sandbox of this process that is still running then keeps its mount points, through its record, and they go at the next cleanup in any process that reads the manifest directory they are recorded in. - **Removing a path yourself.** `cleanupAfterCommand()` keeps this promise for what the library removes, not for what the embedding program removes of its own accord after a command. Before it removes paths that a `denyWrite` entry of any process may name, it can call `liveMountPoints()`, exported from the package root, which returns a `ReadonlySet`, or `undefined` when it cannot tell. Leave out every path in the set: removing one of those would lift the deny inside a sandbox that is running. On `undefined` (a manifest directory, its listing or a manifest could not be read, or something that is no manifest sits among them) skip every removal of an empty file or an empty directory: a sandbox may be running on paths that could not be listed. It never throws. A path is in the set only when both hold. A manifest names it that belongs to a sandbox that may still be running (or this process could record it nowhere). And what is at the path now still has the shape of a mount point, or cannot be looked at: an empty regular file with a single link, of any mode, or an empty directory, owned by the user. So what a caller spares on the word of the set is never more than an empty placeholder, and a manifest that is out of date or forged cannot make it spare a file with content. Paths are as the wrap recorded them: absolute, with the symbolic links in the directories above the last component resolved and the last component as it stands. The set is a snapshot of the manifest directories, and may be out of date by the time the caller acts (a sandbox may start on a path a moment later): its only safe use is to skip a removal, never to decide that a path is free to write. The call is synchronous, needs no wrap in flight, and makes nothing on the host, a manifest directory included; it reads the manifest directories the calling process believes (see above). Off Linux the set is always empty. - **Known limits.** - - _Another PID namespace._ `/proc/` is local to a PID namespace. A manifest therefore records the PID namespace it was written in, and a process in any other one (a container that shares `/var/tmp` with the host, one srt inside another's sandbox) takes it for live, as every process takes one that records none, for seven days from the last change to the manifest or to its record, whichever is later, and for finished then. Until then it removes nothing that manifest names, and an empty file, or a directory, found at a path it names counts as a mount point for that process's own wraps. A cleanup that can ask says what it found, for those that cannot: once it has claimed a manifest and found no process behind it, it renames the claim once more (to `.finished`), before the reading it decides on, and a process that cannot ask keeps no path for a manifest at that name (one that goes by a manifest's age does not rename it so: whoever can ask goes on asking). Without that, two cleanups in two namespaces that each finish with a manifest naming the same path at the same moment would each keep the path for the other's manifest and drop their own, and the path would stay with nothing left to name it. What such a manifest names alone is still its own namespace's to remove, or anybody's after the seven days. The seven days are for a namespace that has gone: after `docker restart` the container has its files, the host's boot id and a new PID namespace, and nothing else would ever end what the earlier one left. A sandbox that runs longer than that in another PID namespace, sharing both this directory and the project, is no longer protected from processes outside its namespace. The command line has to be run in the PID namespace it was wrapped in: the pid it records means nothing in another. A time namespace is not recorded: from one with another boot-time offset every start time reads differently, so a cleanup there takes a running sandbox for ended. + - _Another PID namespace._ `/proc/` is local to a PID namespace. A manifest therefore records the PID namespace it was written in, and a process in any other one (a container that shares `/var/tmp` with the host, one srt inside another's sandbox) takes it for live, as every process takes one that records none, for seven days from the last change to the manifest or to its record, whichever is later, and for finished then (ten minutes where the manifest says that its writer keeps it young, as it does in a directory that other hosts reach: see "Hosts that share a home"). Until then it removes nothing that manifest names, and an empty file, or a directory, found at a path it names counts as a mount point for that process's own wraps. A cleanup that can ask says what it found, for those that cannot: once it has claimed a manifest and found no process behind it, it renames the claim once more (to `.finished`), before the reading it decides on, and a process that cannot ask keeps no path for a manifest at that name (one that goes by a manifest's age does not rename it so: whoever can ask goes on asking). Without that, two cleanups in two namespaces that each finish with a manifest naming the same path at the same moment would each keep the path for the other's manifest and drop their own, and the path would stay with nothing left to name it. What such a manifest names alone is still its own namespace's to remove, or anybody's after the seven days. The seven days are for a namespace that has gone: after `docker restart` the container has its files, the host's boot id and a new PID namespace, and nothing else would ever end what the earlier one left. A sandbox that runs longer than that in another PID namespace, sharing both this directory and the project, is no longer protected from processes outside its namespace. The command line has to be run in the PID namespace it was wrapped in: the pid it records means nothing in another. A time namespace is not recorded: from one with another boot-time offset every start time reads differently, so a cleanup there takes a running sandbox for ended. - _A sandbox this release did not start, or a directory it does not know of._ The read-only binds of the manifest directories are in the sandboxes this release starts, for the names as the wrapping process sees them. A sandbox started by an older release, or by another tool, that can write `/var/tmp` or `/tmp` can delete and forge manifests there. A process that can write in none of the first three places keeps its manifests where its environment says, which the sandbox of a process with another environment does not have bound, and that sandbox can delete and forge them when it may write there. A cleanup on the host believes what it finds in the directories it goes by. It still removes nothing but an empty, singly linked file of the user's, or an empty directory of the user's. - _A manifest directory whose name a sandbox can still take._ Pinning holds a directory in place; it cannot hold a name that is not a directory, or not there. Where one of the names, or a directory above it, is a symbolic link of the user's that lies inside a write root, a sandboxed command can remove the link, make a directory in its place and a manifest directory in that, and the next wrap or cleanup by a process that reads that name believes what it finds there. Where the parent of a name does not exist, and the user may write the directory it is missing from, and that lies inside a write root (a process that runs as root and has no `/run/user/0`, with `/` or `/run` writable; an `XDG_RUNTIME_DIR` or `TMPDIR` that is not there), the wrap cannot make the manifest directory and a sandboxed command can, parent and all: every process that looks there afterwards believes it. Where `/run/user/` itself is a write root the command can open it to others; every process goes on reading what is under it, but another local user can then move that manifest directory aside. What a cleanup removes on such a manifest's word is bounded as above. - _A `/var/tmp/srt-mount-points-` that another user made._ Another local user can make that name, or the one in `/tmp`, before the user's first wrap. It fails the owner check and is passed over, never repaired, and the user's processes write in the next place. - - _A cleanup that cannot be asked after._ A wrap waits for a cleanup for as long as that one's process runs, which it can ask only of a process in its own PID namespace. The record of any other (a container that shares `/var/tmp` with the host) counts for a minute, and a cleanup removes nothing more two seconds after it set out to go on record, by the wall clock as well as by the one that stands still while the host is suspended. So a cleanup in another PID namespace that is held up for over a minute in the moment between its look at the clock and the removal after it still removes a path from under a sandbox that has started since. Within one PID namespace there is no such moment. + - _Hosts that share a home._ (1) Ages are compared across hosts, so their clocks have to agree, and the file server's with them: a manifest and the record of a cleanup have their time set by the host that writes them, a started record gets the server's as the command line appends to it, so what a job has left counts as young for as long as the server's clock is ahead. A host whose clock is ten minutes ahead takes every other host's sandboxes for ended, and what one whose clock is ahead has left counts as young for that much longer. The minute that another host's cleanup counts for is shorter by what the two clocks differ, and with a minute's difference, either way, a wrap does not wait for it at all. (2) A process keeps its manifests young on a timer. One that does not return to its event loop for ten minutes while its sandbox runs (stopped or suspended with its job, or waiting for the command in a call that blocks), or that has exited and left the sandbox running (started through a supervisor, so that `--die-with-parent` does not end it), is taken for gone by the other hosts, which then remove what it named in the home; on its own host the sandbox is still asked after. The `srt` command and `wrapWithSandbox` with an asynchronous spawn are not affected. So is one whose file server refuses to have the time set for that long. (3) It goes by the home. Hosts that share a project and not the home, or whose home is on one of the listed kinds (a cluster file system that reports itself as one of them, the file server itself, to which the home is a disk of its own), do not see each other's manifests, and a host that exports a project from a disk of its own neither tells the others of its mount points there nor goes by what they say of theirs: see "Mount points recorded where another process does not look". (4) A path has to be spelled alike on every host: a share mounted at `/home` here and at `/mnt/home` there is two sets of paths. (5) The look at the server rests on close-to-open consistency: a mount with `nocto`, or a server whose change attribute is too coarse to show two changes in a row, can hide what another host has just done for as long as the client's attribute cache lasts. (6) A process killed half way through a cleanup, which takes milliseconds, leaves its record. On its own host that is seen at once. On every other, wraps that name what other hosts reach wait five seconds and are refused, until the record is a minute old. (7) Hosts are told apart by their boot id. Virtual machines restored from one memory snapshot have the same, and each takes the other's processes for its own, which are not there. (8) Where the directory cannot be made in a home that lies inside a write root (over its quota), a sandboxed command that can make it later is believed, as under "A manifest directory whose name a sandbox can still take". Where it is there and takes no record of a cleanup, what other hosts reach stays until it does. Where it cannot be written from a host at all (the home is mounted read-only there, or an outer sandbox has bound it so), that host reads what the others keep in it and spares what they rely on, but tells them nothing: they do not spare what its sandboxes rely on, and do not wait for its cleanups. A host that runs a release from before this directory, or that is not Linux, keeps nothing in it and does not keep its sandboxes out of it: a command there that may write the home can write what the others believe. (9) The home is part of every reading. While it gives errors, every wrap that restricts writes is refused, wherever its command works, and nothing is removed; while its server does not answer at all, on a hard mount, every wrap and cleanup waits with it, like everything else that touches the home. A request to set a time that is stuck there keeps the process from ending by itself. (10) Every wrap and cleanup costs the file server some tens of operations, one more read for each manifest it has not seen before, and a look at each manifest of another host every few minutes. + - _A cleanup that cannot be asked after._ A wrap waits for a cleanup for as long as that one's process runs, which it can ask only of a process in its own PID namespace on its own host. The record of any other (a container that shares `/var/tmp` with the host, another host in a shared home) counts for a minute, and a cleanup removes nothing more two seconds after it set out to go on record, by the wall clock as well as by the one that stands still while the host is suspended, and looks at both after all that may have waited for a file server. So a cleanup in another PID namespace that is held up for over a minute in the moment between its look at the clock and the removal after it still removes a path from under a sandbox that has started since. Within one PID namespace there is no such moment. - _A cleanup that is stopped._ While a process is stopped half way through a cleanup, which takes milliseconds, every wrap of that user that names a mount point waits its five seconds and is refused. The error names the record, and that begins with the pid: continue or end that process. - _A directory made again._ A wrap held up twice in one plan, so that a directory it has decided to put a mount point in is removed before it pins that directory and made again before it reads the manifests, names the directory and does not pin it. - _Mount points with no manifest._ Only a manifest makes a path a mount point, so one whose manifest is gone is the user's file from then on and is never removed. That is what a killed process of 0.0.81 or earlier leaves, and what any killed process leaves when the manifest directory is emptied and the project is not: a container made anew with the project on a volume, a cleaner of `/var/tmp`, and, for a process that had to write in a later place, the last logout (`/run/user/` is a tmpfs that logind removes) or a reboot. What stays is, for every `denyWrite` and mandatory deny path that was absent under a writable directory (`.bashrc`, `.bash_profile`, `.zshrc`, `.zprofile`, `.profile`, `.gitconfig`, `.gitmodules`, `.ripgreprc`, `.mcp.json`, `.vscode`, `.idea` in the working directory, and `.claude` with nothing in it), an empty regular file of mode `0444` (`0666` less the umask from bubblewrap before 0.5.0), or an empty directory at a missing intermediate component, owned by the user. An embedding program that knows its own deny list can remove such an empty path itself, after a `liveMountPoints()` that returned a set without it. - _An older release beside this one._ A process of 0.0.77 to 0.0.79 takes every empty `0444` file at a path it denies for a leftover, covers it, and removes it at its own cleanup, from under a sandbox of this release that is still running; one of any other release up to 0.0.81 removes at its cleanup what it made itself, whoever relies on it by then. None of them writes a manifest, reads one, or puts its cleanup on record. The protection holds between processes of this release and later. - _bubblewrap killed._ The process on record is bubblewrap's own, and `--die-with-parent` ends the sandbox with it: killing bubblewrap kills the command. Killed with `SIGKILL`, bubblewrap can be gone, or have exited, a moment before the last process of its sandbox is, and a cleanup in that moment already takes the sandbox for ended. - - _Mount points recorded where another process does not look, or not at all._ Two processes see each other's manifests only when one writes where the other reads. Every process reads the first three places, so what is left is: a process that can write in none of them, which records where its environment says, where only a process in the same position looks, in a private directory (it runs inside another sandbox, where all are read-only), or nowhere; two processes that share a project but none of the three (a container beside the host, a service with a private `/tmp` and `/var/tmp`); and a release that wrote no manifest. Nothing the other process reads then says a sandbox relies on those mount points. A wrap in that other process that denies the same path takes the file for the user's own: it binds it read-only where it is and does not name it, so the file goes when the process that recorded it cleans up, from under the second sandbox if that is still running. The ones that could not be recorded at all are removed by the process that made them, at its own cleanup, whatever other process relies on them, and from under its own running sandbox when that cleanup is `reset()`, the exit handler, or a call made early or once too often. A process that fails to look at its directory once as it records (an I/O error) also moves to the next one, and stays there. + - _Mount points recorded where another process does not look, or not at all._ Two processes see each other's manifests only when one writes where the other reads. Every process reads the first three places, so what is left is: a process that can write in none of them, which records where its environment says, where only a process in the same position looks, in a private directory (it runs inside another sandbox, where all are read-only), or nowhere; two processes that share a project but none of the three (a container beside the host, a service with a private `/tmp` and `/var/tmp`, two hosts that share the project and not the home); and a release that wrote no manifest. Nothing the other process reads then says a sandbox relies on those mount points. A wrap in that other process that denies the same path takes the file for the user's own: it binds it read-only where it is and does not name it, so the file goes when the process that recorded it cleans up, from under the second sandbox if that is still running. The ones that could not be recorded at all are removed by the process that made them, at its own cleanup, whatever other process relies on them, and from under its own running sandbox when that cleanup is `reset()`, the exit handler, or a call made early or once too often. A process that fails to look at its directory once as it records (an I/O error) also moves to the next one, and stays there. - _Exit under a running sandbox._ A process that exits, or calls `reset()`, while a sandbox it started is still running leaves that sandbox's mount points and manifest in place, as it must. They go at that process's own next cleanup if it lives on after `reset()`, and otherwise at the next cleanup by another srt process that reads the manifest directory they are recorded in; with a private directory, no other process does. So a program that exits without first ending its running commands leaves their mount points in the project until the next command of any session has been cleaned up after. **Note (Linux, large profiles):** The wrapped string runs as one argument of `sh -c`, which Linux caps at 32 pages (128 KiB with 4 KiB pages). A profile that would not fit, with 4 KiB to spare for a prefix of the caller's own, has its mounts written to an unnamed file (`O_TMPFILE`) that the wrapping process holds open and bubblewrap reads through `--args`. The string then reads ` -c '…' srt-args /proc//fd/ bwrap … --args 9 …` (with the manifest's record ahead of the profile, and the binds of the root and of the manifest kept on the line ahead of `--args`, for a wrap that named mount points): still a simple command, which opens the profile on fd 9 and runs bubblewrap. The environment and the command stay on the command line; the file holds mount paths only. @@ -1154,7 +1156,7 @@ Filesystem restrictions are enforced at the OS level: - The string must be run while the process that produced it is alive, and before the runtime cleans up after that command (`cleanupAfterCommand()`), which is when the profile and the wrap's mount point manifest are released: with several commands in flight, when the last of them has been cleaned up after (the manifest alone goes earlier: when the command's own `commandId` is passed, and at any call made once a run of it has ended). A string of a wrap that named mount points, run after that, refuses to start rather than running without them, and makes nothing on the host: its manifest's bind is on the command line, whatever the descriptor number of its profile holds by then. This holds for every such wrap, not only an over-long one. Until then it may be run again and again; each run adds its line to the record, and every run on it is asked after. A record past 4 MiB, some fourteen thousand runs, vouches like any that cannot be read: what its manifest names then stays until the next boot. - The profile needs a directory that takes an `O_TMPFILE` file — `os.tmpdir()`, else `/dev/shm` — and a readable `/proc/self/fd`. Without them an over-long profile is refused at wrap time with the reason; there is no fallback to a named file. Profiles that fit the command line do not use any of this. - bubblewrap parses at most 9000 arguments (about 3000 mounts). A profile past that, or a command too long for one argument by itself, fails at wrap time with an error. The [mandatory deny paths](#mandatory-deny-paths-auto-protected-files) count too, and no list in the configuration holds them: a git repository directly inside a writable working directory takes four mounts (its hooks, its config, and the two directories above them), so some 730 repositories reach the limit by themselves. The error says how many of the arguments are theirs. There are fewer when the command runs from a directory with fewer repositories in it, or when `allowWrite` names fewer of them; a lower `mandatoryDenySearchDepth` leaves the deeper ones unprotected. -- Every one of these wrap-time refusals is a `LinuxSandboxProfileError`, exported from the package root, with a `LinuxSandboxProfileErrorCode` on `.code` to tell the cases apart (`too_many_arguments`, `nul_in_path`, `args_file_unavailable`, `command_too_long`, `too_many_mount_points`, `mount_points_unreadable`, `mount_points_changed`); branch on `.code` rather than on the message. They say the configuration expands to a profile this host cannot run, except `command_too_long` and `nul_in_path`, which also fire on what the embedding program passed in, and the two that are about other sandboxes. `mount_points_unreadable`: the manifests could not be read, so the wrap cannot tell a file of the user's from a mount point that a running sandbox relies on, and does not guess; or one of their directories could not be looked at, so it cannot be kept out of the command's reach. It refuses every wrap of that user that restricts writes, and ends by itself, but for a file system that has gone from under `TMPDIR` or `XDG_RUNTIME_DIR` and stays gone: this process is out of descriptors or memory, until it is not; a file system gives an error, until it answers again; something that cannot be read as a manifest sits at a manifest's name, for an hour from its last change (half a second where it is no JSON at all) or for as long as a process on the record beside it runs; a process runs on a record whose manifest is gone, until it ends. The message, and `.cause`, name the reason, and the file where there is one. `mount_points_changed`: what the profile relies on on the host changed under each of three plans, because other sandboxes' mount points were being made and removed there, or their manifests kept moving while the wrap read them, or another process's cleanup is held up half way and what it set out to remove may go at any moment (`.cause` is its record, whose name begins with its pid); wrap again, once. `too_many_mount_points`: the mount points do not fit one manifest (a mebibyte). A wrap that threw has already let go of what it held and is not counted among the commands to clean up after: `cleanupAfterCommand()` need not be called for it, and with other commands in flight should not be, since the call would be taken for one of theirs. +- Every one of these wrap-time refusals is a `LinuxSandboxProfileError`, exported from the package root, with a `LinuxSandboxProfileErrorCode` on `.code` to tell the cases apart (`too_many_arguments`, `nul_in_path`, `args_file_unavailable`, `command_too_long`, `too_many_mount_points`, `mount_points_unreadable`, `mount_points_changed`); branch on `.code` rather than on the message. They say the configuration expands to a profile this host cannot run, except `command_too_long` and `nul_in_path`, which also fire on what the embedding program passed in, and the two that are about other sandboxes. `mount_points_unreadable`: the manifests could not be read, so the wrap cannot tell a file of the user's from a mount point that a running sandbox relies on, and does not guess; or one of their directories could not be looked at, or it cannot be told whether the home is a network home (its file system gives errors, the user database no answer), so they cannot all be kept out of the command's reach. It refuses every wrap of that user that restricts writes, and ends by itself, but for a file system that has gone from under `TMPDIR` or `XDG_RUNTIME_DIR` and stays gone: this process is out of descriptors or memory, until it is not; a file system gives an error, or the user database no answer, until it answers again; something that cannot be read as a manifest sits at a manifest's name, for an hour from its last change (half a second where it is no JSON at all) or for as long as a process on the record beside it runs; a process runs on a record whose manifest is gone, until it ends. The message, and `.cause`, name the reason, and the file where there is one. `mount_points_changed`: what the profile relies on on the host changed under each of three plans, because other sandboxes' mount points were being made and removed there, or their manifests kept moving while the wrap read them, or another process's cleanup is held up half way and what it set out to remove may go at any moment (`.cause` is its record, whose name begins with its pid); wrap again, once. `too_many_mount_points`: the mount points do not fit one manifest (a mebibyte). A wrap that threw has already let go of what it held and is not counted among the commands to clean up after: `cleanupAfterCommand()` need not be called for it, and with other commands in flight should not be, since the call would be taken for one of theirs. ### Mandatory Deny Paths (Auto-Protected Files) diff --git a/src/sandbox/bwrap-mount-manifests.ts b/src/sandbox/bwrap-mount-manifests.ts index 013a89aef..a773aaa66 100644 --- a/src/sandbox/bwrap-mount-manifests.ts +++ b/src/sandbox/bwrap-mount-manifests.ts @@ -33,11 +33,20 @@ * mountPointManifestDirectories}). Not covered: a sandbox this library did not * start; a name, or one above it, that is a link inside a write root; a parent * that is not there, where the command may make it. + * + * A mount point on a file system that other hosts mount too (a network home) is + * there for all of them, and a directory of this host's tells them nothing. So + * a wrap names those in a manifest of their own in the home directory, where + * that is on such a file system (see {@link networkManifestDirectoryName}). No + * host can ask after another's processes, so there a manifest is live by its + * age, and the process that wrote it keeps it young (see {@link + * keepOwnManifestsYoung}). */ +import { spawnSync } from 'node:child_process' import { randomBytes } from 'node:crypto' import * as fs from 'node:fs' -import { tmpdir } from 'node:os' +import { tmpdir, userInfo } from 'node:os' import path from 'node:path' import { z } from 'zod/v3' import { logForDebugging } from '../utils/debug.js' @@ -97,6 +106,23 @@ const UNREADABLE_MANIFEST_MAX_AGE_MS = 60 * 60 * 1000 */ const OTHER_NAMESPACE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000 +/** + * A manifest whose processes cannot be asked after, and whose writer keeps it + * young, counts as live until it and its record are this old. Far more than + * {@link KEPT_YOUNG_EVERY_MS}, for what lies between two hosts: their clocks, a + * minute of cached attributes, a timer that is late. + */ +const OTHER_KERNEL_MAX_AGE_MS = 10 * 60 * 1000 + +/** How often a process sets the time of its manifests that other kernels read. */ +const KEPT_YOUNG_EVERY_MS = 60 * 1000 + +/** + * For how long one listing is good in a pass over a directory that other + * kernels reach, where a listing is a round trip to a server. + */ +const NETWORK_LISTING_GOOD_FOR_MS = 20 + /** * A pass removes nothing more this long after it set out to go on record, and * leaves the rest to the next: an ordinary one takes milliseconds. @@ -114,8 +140,9 @@ const bothClocks = (): readonly [number, number] => [ ] /** - * A pass that cannot be asked after (another PID namespace's) counts as under - * way until its record is this old: well past {@link PASS_GOOD_FOR_MS}. + * A pass that cannot be asked after (another PID namespace's, another host's) + * counts as under way until its record is this old: well past {@link + * PASS_GOOD_FOR_MS}. */ const PASS_MAX_AGE_MS = 60 * 1000 @@ -159,11 +186,17 @@ const ManifestSchema = z.object({ */ ns: z.string().optional(), /** - * The boot it was written in. Absent where that could not be read. As read, - * both are absent where nothing follows from them (see {@link - * ONE_KERNEL_FILE_SYSTEMS}). + * The boot it was written in. Absent where that could not be read. All that + * tells this kernel's from another's where more than one reaches the directory + * (see {@link ONE_KERNEL_FILE_SYSTEMS}). */ boot: z.string().optional(), + /** + * Its writer sets its time every {@link KEPT_YOUNG_EVERY_MS} for as long as it + * holds on to it. Said by the writer, who alone knows: the same directory can + * be on one kind of file system to the writer and on another to a reader. + */ + keptYoung: z.boolean().optional(), /** When the manifest was written, for {@link MANIFEST_GRACE_MS}. */ created: z.number(), /** The mount points bwrap makes on the host for that wrap's deny paths. */ @@ -184,6 +217,16 @@ type Manifest = z.infer & { finished: boolean /** Its started record, `.started`. */ record: string + /** + * In a directory that other kernels reach, and not of this kernel and PID + * namespace for all that can be told: nothing here can ask after it. + */ + elsewhere: boolean + /** + * Another host's, and it names what this host does not reach (which, of one in + * the home, is left out of its lists). + */ + namesMore: boolean /** This process wrote it and the caller says its command is over. */ released: boolean } @@ -196,7 +239,12 @@ type Manifest = z.infer & { */ const ownManifests = new Map< string, - { commandKey: string | undefined; over: boolean } + { + commandKey: string | undefined + over: boolean + /** Other kernels read it, and have its age to go by. */ + byItsAge: boolean + } >() /** @@ -360,12 +408,12 @@ function ownBootId(): string | undefined { * `statfs`'s type. On any other (NFS, 9p, virtiofs, FUSE) a manifest may be * another running kernel's, and only the boot id tells: two hosts each in their * initial PID namespace name that alike. So there a manifest is this kernel's - * only when its boot id and ours can both be read and are equal. Of any other, - * neither its boot nor its namespace says anything: it records none, and is - * live for {@link OTHER_NAMESPACE_MAX_AGE_MS}. + * only when its boot id and ours can both be read and are equal. Any other, and + * one of another PID namespace of this kernel, is live by its age (see {@link + * OTHER_KERNEL_MAX_AGE_MS}). * * INVARIANT: a list of what is let in, not of what is kept out. On a local kind - * that is missing, what an earlier boot left waits those seven days instead of + * that is missing, what an earlier boot left waits those minutes instead of * going at once. A kind wrongly let in, as the next network file system would be * by a list of the known ones, has a live sandbox judged finished and its mount * points removed. @@ -380,24 +428,78 @@ const ONE_KERNEL_FILE_SYSTEMS = new Set([ 0xf2f52010, // f2fs ]) -/** Whether `dir` is on one of those. Not where that cannot be asked. */ -function isReachedByOneKernel(dir: string): boolean { +/** + * The directories found on none of those, and until when that is believed, by + * `performance.now()`: on a network file system the question is a round trip + * each time. For a minute, because a share can be unmounted, and what another + * host names there is then not this host's to judge or collect any more. Never + * the other answer: a share can be mounted, and that has to be seen at once. + */ +const foundWithinOthersReach = new Map() +const FOUND_WITHIN_OTHERS_REACH_FOR_MS = 60 * 1000 + +/** + * Whether `dir` is on one of those, or on a kind that other kernels reach too; + * `absent` where it is not there. What else cannot be asked is thrown with + * `orThrow`, and else counts as within their reach. + */ +function reachOf( + dir: string, + orThrow = false, +): 'one kernel' | 'others too' | 'absent' { + if (performance.now() < (foundWithinOthersReach.get(dir) ?? -Infinity)) { + return 'others too' + } + let type: number try { - return ONE_KERNEL_FILE_SYSTEMS.has(Number(fs.statfsSync(dir).type)) - } catch { - return false + type = Number(fs.statfsSync(dir).type) + } catch (e) { + if (isAbsenceErrno(e)) return 'absent' + if (orThrow) throw e + return 'others too' + } + if (ONE_KERNEL_FILE_SYSTEMS.has(type)) { + return 'one kernel' } + if (foundWithinOthersReach.size >= 1024) foundWithinOthersReach.clear() + foundWithinOthersReach.set( + dir, + performance.now() + FOUND_WITHIN_OTHERS_REACH_FOR_MS, + ) + return 'others too' } +/** Whether `dir` is on one of those. Not where that cannot be asked. */ +const isReachedByOneKernel = (dir: string): boolean => + reachOf(dir) === 'one kernel' + /** - * Whether `file` was last written over `ms` ago. One that cannot be looked at - * was not; one that is not there counts as `absent` says. + * Whether other kernels reach what is in `dir`, going by the nearest directory + * at or above it that is there: on a host that does not have `/scratch`, + * `/scratch/x` is a name on its own root and no share. */ -function olderThan(file: string, ms: number, absent = false): boolean { +function othersReachInto(dir: string): boolean { + for (;;) { + const reach = reachOf(dir) + if (reach !== 'absent' || dir === path.dirname(dir)) { + return reach !== 'one kernel' + } + dir = path.dirname(dir) + } +} + +/** Whether other kernels reach the path `p`. */ +const othersReach = (p: string): boolean => othersReachInto(path.dirname(p)) + +/** + * Whether `file` was last written over `ms` ago. One that cannot be looked at, + * or is not there, was not. + */ +function olderThan(file: string, ms: number): boolean { try { return Date.now() - fs.lstatSync(file).mtimeMs > ms - } catch (e) { - return absent && isAbsenceErrno(e) + } catch { + return false } } @@ -445,6 +547,11 @@ function isOurPrivateDirectory( * planted at the name is refused, not followed. */ function makeOurPrivateDirectory(dir: string): boolean { + // As it mostly is, and all that follows comes to this question in the end: in + // a network home each step of it is a round trip. + if (!isReachedByOneKernel(dir) && isOurPrivateDirectory(dir)) { + return true + } try { fs.mkdirSync(dir, { mode: 0o700 }) } catch { @@ -485,7 +592,8 @@ function isBuriedBySandboxMounts(dir: string): boolean { ) } -// Test seam: what stands in for `/` in the names that follow from the user id. +// Test seam: what stands in for `/` in the names that follow from the user id, +// and in /etc/passwd. let root = '/' /** @@ -498,12 +606,6 @@ let root = '/' * else as well. Not to be read under: where it is a write root the command can * chmod it, and must not make other processes read less by that. Reading more * does no harm, a manifest being only a claim about a path. - * - * Not the home directory. A network home caches listings, and the claim by - * rename, the reading after it and the listing before a removal all need the - * next readdir in another process to show a rename; a home is often inside a - * write root; nothing ages what is left there; and in a shared one another boot - * id may be another live host's. */ function fixedManifestDirectoryNames( orThrow = false, @@ -524,6 +626,324 @@ function fixedManifestDirectoryNames( ] } +/** What the directory in a network home is called. */ +const NETWORK_DIRECTORY = '.srt-mount-points' + +/** For how long "no such user" is taken for an answer. */ +const NO_HOME_GOOD_FOR_MS = 60 * 1000 + +/** + * For how long no answer at all is not asked for again. Each asking holds the + * thread for as long as `getent` is given, and a wrap that is refused asks more + * than once. + */ +const NO_ANSWER_GOOD_FOR_MS = 10 * 1000 + +let home: { dir: string | undefined; asked: number } | undefined +let noAnswer: { error: Error; asked: number } | undefined + +/** + * The user's home as the user database has it, not as the environment does, or + * `undefined`. Asked once: the database may be a directory service. + * + * Node asks the C library. Bun answers `os.userInfo()` with `$HOME`, so there it + * is /etc/passwd and then `getent`, at its usual places and never looked up on + * `PATH`, for a user only a directory service knows. + * + * INVARIANT: no answer but a home is kept for good. A process that took an + * outage for "no home" would neither read what the others keep there nor keep + * its own sandboxes out of it, for as long as it runs. So it throws where it gets + * no answer; and "no such user", which is also all that a directory service that + * is down has the C library say, is asked again after {@link + * NO_HOME_GOOD_FOR_MS}. + */ +function homeDirectory(): string | undefined { + if ( + home !== undefined && + (home.dir !== undefined || + performance.now() - home.asked < NO_HOME_GOOD_FOR_MS) + ) { + return home.dir + } + if ( + noAnswer !== undefined && + performance.now() - noAnswer.asked < NO_ANSWER_GOOD_FOR_MS + ) { + throw noAnswer.error + } + const uid = String(process.getuid?.() ?? 0) + const inPasswdLines = (text: string): string | undefined => + text + .split('\n') + .map(line => line.split(':')) + .find(fields => fields[2] === uid)?.[5] + let dir: string | undefined + if (root === '/' && process.versions['bun'] === undefined) { + try { + dir = userInfo().homedir + } catch { + // No such user, for all it says. + } + } else { + try { + dir = inPasswdLines( + fs.readFileSync(path.join(root, 'etc/passwd'), 'utf8'), + ) + } catch (e) { + if (!isAbsenceErrno(e)) throw e + } + const getent = + dir === undefined + ? ['usr/bin/getent', 'bin/getent', 'run/current-system/sw/bin/getent'] + .map(file => path.join(root, file)) + .find(file => fs.existsSync(file)) + : undefined + if (getent !== undefined) { + const asked = spawnSync(getent, ['passwd', uid], { + encoding: 'utf8', + env: {}, + stdio: ['ignore', 'pipe', 'ignore'], + timeout: 5000, + }) + // 2 is its word for "no such key". + if (asked.status !== 0 && asked.status !== 2) { + noAnswer = { + error: new Error( + `${getent} passwd ${uid} gave no answer (${asked.error?.message ?? `status ${asked.status}`})`, + ), + asked: performance.now(), + } + throw noAnswer.error + } + dir = inPasswdLines(asked.stdout) + } + } + home = { + dir: dir !== undefined && path.isAbsolute(dir) ? dir : undefined, + asked: performance.now(), + } + return home.dir +} + +/** + * What {@link networkManifestDirectoryName} has found, once it has found one. + * For what follows from the directories a wrap or a pass has already worked out: + * it asks nothing, so it does not throw half way through either. + */ +let network: string | undefined + +/** + * Where the manifests go that name what other hosts reach too: in the user's + * home directory, where that is itself on a file system more than one kernel + * reaches, and `undefined` where it is not. Hosts that share a home work it out + * alike, and on a host with a home of its own nothing is read or written there + * (it is kept out of reach there too: see {@link + * mountPointManifestDirectories}). + * + * Read, bound read-only and pinned like the places that follow from the user id, + * and written only for what other hosts reach: it is no place for what they do + * not, and does not stand in for one. + * + * INVARIANT: only on the file system's own word. A home that is not there, or + * that this user may not enter, is no network home: no wrap could make or bind + * the directory in it, and a sandboxed command that then made both would be + * believed. Throws where it cannot tell (an error of the moment, twice, on a + * home that may hold the directory): a wrap that went on would leave it unbound. + * And only that there is one is remembered: a home on this host's own disk may + * be what shows until the share is mounted over it. + * + * What speaks against a home directory, and what is done about it: + * + * - A network file system caches listings, and all of this rests on one process + * seeing what another has put into the directory. A client asks the server + * again for a directory it opens, and for one it has itself just changed. + * Where it matters each side has: a pass goes on record before the reading it + * decides on, and a wrap publishes before it lists the passes and reads the + * manifests again. + * - It caches what is in the directories the mount points are in as well: see + * {@link lookAfreshAt}. + * - Another boot id may be another live host's: see {@link + * ONE_KERNEL_FILE_SYSTEMS}. + * - Nothing ages what is left there: see {@link OTHER_KERNEL_MAX_AGE_MS}. + * - It is often inside a write root: it is bound read-only and what lies above + * it is pinned, like the others. + */ +function networkManifestDirectoryName(): string | undefined { + if (network !== undefined) { + return network + } + const dir = homeDirectory() + if (dir === undefined || isBuriedBySandboxMounts(dir)) { + return undefined + } + for (const last of [false, true]) { + try { + if (reachOf(dir, true) === 'others too') { + network = path.join(dir, NETWORK_DIRECTORY) + } + break + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'EACCES') break + if (last) throw e + } + } + return network +} + +/** + * Whether other hosts that keep their manifests where this one does reach one of + * `paths`: what they do is then this wrap's business, and the other way round. + */ +export function otherHostsReach(paths: readonly string[]): boolean { + // Nothing is asked for a wrap that names nothing: it may restrict no write. + return ( + paths.length > 0 && + networkManifestDirectoryName() !== undefined && + paths.some(othersReach) + ) +} + +/** + * Has a network file system compare what it remembers of `p` with the server's, + * which it does for what is opened (close-to-open consistency). + */ +function openAndClose(p: string, flags: number): void { + try { + fs.closeSync(fs.openSync(p, flags)) + } catch { + // Not there, or not to be opened: the look that follows says so. + } +} + +/** + * Has this host ask again what is in the directories that hold `paths`, those + * of them that other hosts reach. A network file system answers a look-up from + * what it remembers, for seconds to a minute, so a mount point another host has + * just removed would be found there still, by the wrap and by bubblewrap, which + * would mount on what the server no longer has: the mount goes when the client + * finds out. Opening a directory has the client compare it with the server's, + * and look up again what is in one that has changed. + * + * Only where there is a network home: without the manifests in it no host knows + * what another is about, and there is nothing to find out in time. + */ +export function lookAfreshAt(paths: Iterable): void { + if (network === undefined) { + return + } + for (const dir of new Set([...paths].map(p => path.dirname(p)))) { + if (othersReachInto(dir)) { + openAndClose(dir, fs.constants.O_RDONLY | fs.constants.O_DIRECTORY) + } + } +} + +let keptYoung: ReturnType | undefined + +/** Sets the time of `file` to now, by this host's clock, if it can be set. */ +function touch(file: string): void { + const now = new Date() + try { + fs.utimesSync(file, now, now) + } catch { + // It has the server's. + } +} + +/** + * Puts a file that holds `text` at `file`, in a directory other kernels read. + * They go by its age, by this host's clock and not the server's, so it comes to + * its name whole and with its time set. Throws, with nothing left there, where + * it cannot be. + */ +function putWhereOtherKernelsRead(file: string, text: string): void { + const temporary = `${file}${TEMPORARY_SUFFIX}` + try { + const fd = fs.openSync(temporary, 'wx', 0o600) + try { + if (text !== '') { + // A network file system takes the bytes, and says only as they are + // flushed that it has no room for them. Bun passes on neither what + // closing the file says nor what the `flush` option finds. + fs.writeFileSync(fd, text) + fs.fsyncSync(fd) + } + } finally { + fs.closeSync(fd) + } + touch(temporary) + fs.renameSync(temporary, file) + } catch (e) { + drop(temporary) + throw e + } +} + +/** + * Has the time of every manifest of this process that other kernels read set + * every {@link KEPT_YOUNG_EVERY_MS}, for as long as this process knows of one: + * its age is all they have to go by. By this host's clock and not the server's, + * so that what has to agree is the clocks of the hosts. + * + * On a timer, so a process that does not come back to its event loop for {@link + * OTHER_KERNEL_MAX_AGE_MS} (stopped with its sandbox, or waiting for its command + * in a call that blocks) is taken for gone by the other hosts. The timer does + * not keep the process alive, and an error does not end it: the next tick may do. + */ +function keepOwnManifestsYoung(): void { + const some = [...ownManifests.values()].some(own => own.byItsAge) + if (some && keptYoung === undefined) { + keptYoung = setInterval(setTheTimesOfOwnManifests, KEPT_YOUNG_EVERY_MS) + keptYoung.unref() + } else if (!some && keptYoung !== undefined) { + clearInterval(keptYoung) + keptYoung = undefined + } +} + +let settingTimes = false + +/** + * One tick of that. Not with the calls that wait: nobody has asked for anything + * when it runs, and a server that does not answer would hold up the one thread + * of whatever program this is part of. One request at a time, and no tick while + * the last is at it, so that such a server ties up one thread of the pool those + * requests run on and not all of them. + */ +function setTheTimesOfOwnManifests(): void { + // Those a pass has collected since the last tick are forgotten by now. + keepOwnManifestsYoung() + if (settingTimes) { + return + } + settingTimes = true + const left = [...ownManifests].filter(([, own]) => own.byItsAge) + const next = (): void => { + const [file, own] = left.shift() ?? [] + if (file === undefined || own === undefined) { + settingTimes = false + return + } + const now = new Date() + const done = (e: NodeJS.ErrnoException | null): void => { + if (e !== null && !(isAbsenceErrno(e) && own.over)) { + logForDebugging( + `[Sandbox Linux] The time of the manifest ${file} could not be set (${String(e)}) - other hosts take its mount points for left behind once it is old`, + { level: 'warn' }, + ) + } + next() + } + fs.utimes(file, now, now, e => + // A pass of this host has it, and gives it back if a sandbox runs. + e !== null && isAbsenceErrno(e) + ? fs.utimes(`${idOf(file)}${CLAIMED_SUFFIX}`, now, now, done) + : done(e), + ) + } + next() +} + /** * Where processes of this user keep manifests that others can find, in the * order tried: {@link fixedManifestDirectoryNames}, then, for a process that @@ -552,8 +972,9 @@ function sharedManifestDirectoryNames( * claims and collects in those it can write. The names that follow from the * user id; the ones the environment gives only where none of those can be * written, because only processes with that environment keep them out of a - * sandbox's reach; and the one it has settled on. Each only while it is a - * directory of ours alone. Makes nothing. Throws where it cannot tell. + * sandbox's reach; {@link networkManifestDirectoryName}; and the one it has + * settled on. Each only while it is a directory of ours alone. Makes nothing. + * Throws where it cannot tell. * * Each directory once, whatever it is called: where /var/tmp is a link to /tmp, * or a bind mount of it, two of the names are one directory. Every manifest in @@ -565,6 +986,8 @@ function believedManifestDirectories(): string[] { ) ? fixedManifestDirectoryNames(true) : sharedManifestDirectoryNames(true) + const inTheHome = networkManifestDirectoryName() + if (inTheHome !== undefined) names.push(inTheHome) if (manifestDirectory !== undefined) names.push(manifestDirectory) const byIdentity = new Map() for (const dir of names) { @@ -650,7 +1073,8 @@ function ensureManifestDirectory(): string | undefined { /** * Every directory a wrap must bind read-only because some process of this user * keeps manifests in it and a collect believes what it finds there: every - * shared name, whichever this process writes to, plus the one it settled on. + * shared name, whichever this process writes to, the one in a network home, and + * the one it settled on. * * Each is made if it can be, so a sandboxed command cannot make and fill it * first. One that can be neither made nor bound is not believed, and the @@ -662,11 +1086,20 @@ function ensureManifestDirectory(): string | undefined { * - another user has the name, in a sticky directory: only that user or root * can remove or rename it. * + * INVARIANT: the one in the home is kept out of reach on a host that does not + * believe it as well. A home that is this host's own disk may be a network home + * to others (it is to the clients of the server that exports it), who believe + * what they find there. So it is bound wherever it is there, and where it is + * not, but the command could make it (`commandMayMake`), it is made first. + * * With `recording`, the directory this wrap's manifest will go to is settled * first, so the wrap can pin what lies above it before it publishes. Throws * where it cannot tell whether one of them is there. */ -export function mountPointManifestDirectories(recording = false): string[] { +export function mountPointManifestDirectories( + recording = false, + commandMayMake: (dir: string) => boolean = () => false, +): string[] { if (!onLinux()) { return [] } @@ -682,6 +1115,21 @@ export function mountPointManifestDirectories(recording = false): string[] { dirs.add(candidate) } } + const believed = networkManifestDirectoryName() + const home = homeDirectory() + const inTheHome = + believed ?? + (home === undefined || isBuriedBySandboxMounts(home) + ? undefined + : path.join(home, NETWORK_DIRECTORY)) + if (inTheHome !== undefined) { + if (believed !== undefined || commandMayMake(inTheHome)) { + makeOurPrivateDirectory(inTheHome) + } + if (isToBeBound(inTheHome)) { + dirs.add(inTheHome) + } + } if (recording) { ensureManifestDirectory() } @@ -769,33 +1217,94 @@ function readOwnFile(file: string, max: number): string | undefined { /** `/` of a manifest, its claim or its started record. */ const idOf = (file: string): string => file.slice(0, file.lastIndexOf('.')) +/** + * What the manifests hold that have been read in a directory other kernels + * reach, by `/`, for as long as they are listed. There each read + * is several round trips to a server, every wrap reads every manifest, and a + * manifest is never rewritten: its name is all that changes. + */ +const remembered = new Map< + string, + { + fields: z.infer + /** + * Until when, by `performance.now()`, it is young whatever happens: its time + * is only ever set forward. A look at it is a round trip as well, and every + * pass judges every manifest of every host. + */ + youngUntil: number + } +>() + /** * The manifest at `file`, which is its own name or a claim's, or `undefined` * where nothing is there. Throws like {@link readOwnFile}, and {@link NotOurs} - * for what does not hold a manifest's fields. + * for what does not hold a manifest's fields. One that is {@link remembered} is + * taken to be where the caller has just found it listed. `othersReadIt` says + * whether its directory is one that other kernels reach. */ -function readManifest(file: string): Manifest | undefined { - const text = readOwnFile(file, MANIFEST_MAX_BYTES) - if (text === undefined) { - return undefined - } - let json: unknown - try { - json = JSON.parse(text) - } catch { - throw new NotJson(`${file} is not JSON`) - } - const parsed = ManifestSchema.safeParse(json) - if (!parsed.success) { - throw new NotOurs(`${file} is not a manifest this version can read`) - } +function readManifest( + file: string, + othersReadIt: boolean, + reached: (p: string) => boolean = othersReach, +): Manifest | undefined { const id = idOf(file) + let fields = othersReadIt ? remembered.get(id)?.fields : undefined + if (fields === undefined) { + let text: string | undefined + try { + text = readOwnFile(file, MANIFEST_MAX_BYTES) + } catch (e) { + // A network file system's word for a file another host has removed since + // this one looked it up or opened it. Of a manifest, whose reader then + // starts over; a record that cannot be read vouches, whatever the error. + if ((e as NodeJS.ErrnoException).code !== 'ESTALE') throw e + } + if (text === undefined) { + return undefined + } + let json: unknown + try { + json = JSON.parse(text) + } catch { + throw new NotJson(`${file} is not JSON`) + } + const parsed = ManifestSchema.safeParse(json) + if (!parsed.success) { + throw new NotOurs(`${file} is not a manifest this version can read`) + } + fields = parsed.data + if (othersReadIt) remembered.set(id, { fields, youngUntil: -Infinity }) + } + const ofThisBoot = fields.boot !== undefined && fields.boot === ownBootId() + const elsewhere = + othersReadIt && + // Its own it knows, where it cannot read what would say so. + !ownManifests.has(`${id}${MANIFEST_SUFFIX}`) && + !(ofThisBoot && fields.ns !== undefined && fields.ns === ownPidNamespace()) + // INVARIANT: of what another host has put into the home only what that host + // and this one both reach counts. `/data/x` there may be another file than + // here, and here it may be the user's own. Asked at every reading: a share may + // have been mounted since the last. + const inTheHome = path.dirname(file) === network + const counts = elsewhere && inTheHome ? reached : (): boolean => true return { - ...parsed.data, + ...fields, + paths: fields.paths.filter(counts), + sources: fields.sources.filter(counts), file, claimed: !file.endsWith(MANIFEST_SUFFIX), finished: file.endsWith(FINISHED_SUFFIX), record: `${id}${STARTED_SUFFIX}`, + elsewhere, + // In any other directory that both hosts read (a /var/tmp that diskless + // nodes share) a manifest names all its wrap relies on, and all of it + // counts, as before there was one in the home. It is as little for this + // host to collect. + namesMore: + elsewhere && + (inTheHome || !ofThisBoot) && + ![...fields.paths, ...fields.sources].every(reached), released: ownManifests.get(`${id}${MANIFEST_SUFFIX}`)?.over === true, } } @@ -848,39 +1357,82 @@ const isOfAnotherBoot = (manifest: Manifest): boolean => manifest.boot !== (ownBootId() ?? manifest.boot) /** - * Whether this process can tell if the processes behind a manifest run: none of - * another boot does, and of this one it can ask after those of its own PID - * namespace. What {@link isLive} says of any other goes by its age alone. + * Whether this process can tell if the processes behind a manifest run: of this + * kernel's, none of another boot does, and of this one it can ask after those of + * its own PID namespace. What {@link isLive} says of any other goes by its age + * alone. */ function canBeAskedAfter(manifest: Manifest): boolean { const here = ownPidNamespace() return ( - isOfAnotherBoot(manifest) || (here !== undefined && manifest.ns === here) + !manifest.elsewhere && + (isOfAnotherBoot(manifest) || (here !== undefined && manifest.ns === here)) ) } +/** + * A manifest is no longer at the name a listing showed it at: renamed, or + * dropped. What was made of that listing is out of date. + */ +class Moved extends Error {} + +/** + * {@link isLive} for a manifest whose processes cannot be asked after: until it + * and its record are old, by what its writer says it does about that. What + * cannot be looked at is live. + * + * Throws {@link Moved} for one that is gone from its name. Not "live": whoever + * dropped it may have kept a path for a manifest of the caller's, and a caller + * that kept the path for this one in turn would leave it to nobody. And not + * "finished": it may only have been claimed, and be given back. + */ +function isLiveByItsAge(manifest: Manifest): boolean { + const maxAge = manifest.keptYoung + ? OTHER_KERNEL_MAX_AGE_MS + : OTHER_NAMESPACE_MAX_AGE_MS + const known = remembered.get(idOf(manifest.file)) + if (known !== undefined && performance.now() < known.youngUntil) { + return true + } + let age: number + try { + age = Date.now() - fs.lstatSync(manifest.file).mtimeMs + } catch (e) { + if (isAbsenceErrno(e)) throw new Moved(manifest.file) + return true + } + try { + age = Math.min(age, Date.now() - fs.lstatSync(manifest.record).mtimeMs) + } catch (e) { + if (!isAbsenceErrno(e)) return true + } + if (known !== undefined) { + known.youngUntil = performance.now() + (maxAge - age) + } + return age <= maxAge +} + /** * Whether a sandbox that named this manifest may be running, or may yet start. - * None of another boot is, whatever else the manifest says. Only a process in - * the PID namespace that wrote it can tell; to any other it is live until {@link - * OTHER_NAMESPACE_MAX_AGE_MS}. A process on its started record that runs keeps + * Where other kernels reach its directory, one that is not of this kernel and + * PID namespace goes by its age, which its writer keeps low for as long as it + * holds on to it. Where only this kernel does, none of another boot is, whatever + * else the manifest says, and only a process in the PID namespace that wrote it + * can tell; to any other it is live until {@link OTHER_NAMESPACE_MAX_AGE_MS}. A process on its started record that runs keeps * it live, whatever else. With none, no sandbox is past a claim, and one at its * own name that the caller has not released is live while its writer runs (its * command line may be run yet, for the first time or again), or while it is * young and has no record. */ function isLive(manifest: Manifest): boolean { + if (manifest.elsewhere) { + return isLiveByItsAge(manifest) + } if (isOfAnotherBoot(manifest)) { return false } if (!canBeAskedAfter(manifest)) { - return ( - ownPidNamespace() === undefined || - !( - olderThan(manifest.file, OTHER_NAMESPACE_MAX_AGE_MS) && - olderThan(manifest.record, OTHER_NAMESPACE_MAX_AGE_MS, true) - ) - ) + return ownPidNamespace() === undefined || isLiveByItsAge(manifest) } const vouched = recordVouches(manifest.record) if (vouched === true) { @@ -895,6 +1447,18 @@ function isLive(manifest: Manifest): boolean { ) } +/** + * Whether `manifest` names what this host does not reach, and is not so old that + * no host that does will come for it any more. Collected from here, the rest of + * what it names would have nothing left to name it; and held on to from here, + * with what this host reaches removed, it would go on naming that. + */ +const isForAnotherHostToCollect = (manifest: Manifest): boolean => + manifest.namesMore && + (performance.now() < + (remembered.get(idOf(manifest.file))?.youngUntil ?? -Infinity) || + !olderThan(manifest.file, OTHER_NAMESPACE_MAX_AGE_MS)) + /** * What became of a path a pass set out to remove: `left` when it is gone or no * longer what bubblewrap or this library made, `occupied` when it is a directory @@ -1023,87 +1587,142 @@ export type MountPointManifest = { /** * Record the mount points this wrap relies on, and the empty directories they - * bind from, before the sandbox can start. + * bind from, before the sandbox can start. Returns what was written. * - * `undefined` when there is nothing to record or no manifest could be written. - * What could not be recorded is kept in memory and removed at this process's + * All of it goes into a manifest in a place of this host's, as on a host with a + * home of its own: a process of this host that does not see the home (a service + * kept out of it, a container with another) reads it there. What other hosts + * reach too goes into a second one as well, where they read it (see {@link + * networkManifestDirectoryName}), and nothing else does: whoever reads a manifest + * can then collect it whole. Where that one cannot be written (a home over its + * quota) the first is all there is. + * + * What could be recorded nowhere is kept in memory and removed at this process's * own clean-up, since no other process will know of it. `too-large`, with - * nothing written or added to that, for one its own reader would refuse: every pass of - * every process would turn back at it. + * nothing written or added to that, for more than one manifest holds: its own + * reader would refuse it, and every pass of every process would turn back at it. */ export function publishMountPointManifest( mountPoints: readonly string[], sources: readonly string[], commandKey?: string, -): MountPointManifest | 'too-large' | undefined { +): MountPointManifest[] | 'too-large' { if (!onLinux() || (mountPoints.length === 0 && sources.length === 0)) { - return undefined + return [] + } + type Named = { paths: string[]; sources: string[] } + const writer = { + version: MANIFEST_VERSION, + ...ownProcess(), + ns: ownPidNamespace(), + boot: ownBootId(), + created: Date.now(), + } + const textOf = (named: Named, keptYoung: boolean): string => { + const body: z.infer = { + ...writer, + ...(keptYoung ? { keptYoung } : {}), + ...named, + } + return JSON.stringify(body) } - const dir = ensureManifestDirectory() - if (dir !== undefined) { + const all = { + paths: [...new Set(mountPoints)], + sources: [...new Set(sources)], + } + if (Buffer.byteLength(textOf(all, true)) > MANIFEST_MAX_BYTES) { + return 'too-large' + } + const written = ( + dir: string | undefined, + named: Named, + ): MountPointManifest | undefined => { + if (dir === undefined) { + return undefined + } const id = path.join( dir, `${process.pid}-${randomBytes(8).toString('hex')}`, ) const file = `${id}${MANIFEST_SUFFIX}` const temporary = `${file}${TEMPORARY_SUFFIX}` - const body: z.infer = { - version: MANIFEST_VERSION, - ...ownProcess(), - ns: ownPidNamespace(), - boot: ownBootId(), - created: Date.now(), - paths: [...new Set(mountPoints)], - sources: [...new Set(sources)], - } - const text = JSON.stringify(body) - if (Buffer.byteLength(text) > MANIFEST_MAX_BYTES) { - return 'too-large' - } - // Written beside the manifest and renamed onto it, so that it is read whole - // or not at all. + const byItsAge = !isReachedByOneKernel(dir) try { - fs.writeFileSync(temporary, text, { mode: 0o600 }) - fs.renameSync(temporary, file) - ownManifests.set(file, { commandKey, over: false }) - // Not what an earlier wrap could record nowhere and this one names too: - // that wrap's sandbox may outlast this manifest. - for (const named of [...mountPoints, ...sources]) { - if (unrecorded.madeByAPlan.delete(named)) { - unrecorded.paths.delete(named) - unrecorded.sources.delete(named) - } + if (byItsAge) { + putWhereOtherKernelsRead(file, textOf(named, true)) + } else { + // Written beside the manifest and renamed onto it, so that it is read + // whole or not at all. + fs.writeFileSync(temporary, textOf(named, false), { mode: 0o600 }) + fs.renameSync(temporary, file) } + ownManifests.set(file, { commandKey, over: false, byItsAge }) + keepOwnManifestsYoung() return { dir, file, started: `${id}${STARTED_SUFFIX}` } } catch (e) { - try { - fs.unlinkSync(temporary) - } catch { - // Never made. - } + if (!byItsAge) drop(temporary) logForDebugging( - `[Sandbox Linux] Could not record the mount points this command relies on (${String(e)}) - they are kept in memory and removed at this process's own clean-up`, + `[Sandbox Linux] Could not record the mount points this command relies on in ${dir} (${String(e)})`, { level: 'warn' }, ) + return undefined } } - for (const mountPoint of mountPoints) unrecorded.paths.add(mountPoint) - for (const source of sources) unrecorded.sources.add(source) - for (const named of [...mountPoints, ...sources]) { - unrecorded.madeByAPlan.delete(named) + let inTheHome: string | undefined + try { + inTheHome = networkManifestDirectoryName() + } catch { + // The wrap has asked before this, and is refused by what asks after it. + } + const theirsToo = + inTheHome === undefined + ? { paths: [], sources: [] } + : { + paths: all.paths.filter(othersReach), + sources: all.sources.filter(othersReach), + } + const inTheirSight = + inTheHome !== undefined && + theirsToo.paths.length + theirsToo.sources.length > 0 && + makeOurPrivateDirectory(inTheHome) + ? written(inTheHome, theirsToo) + : undefined + const here = written(ensureManifestDirectory(), all) + if (here === undefined) { + // Not what the other hosts' sight of it covers: that is for them. + logForDebugging( + "[Sandbox Linux] Mount points that could not be recorded on this host are kept in memory and removed at this process's own clean-up", + { level: 'warn' }, + ) + for (const mountPoint of all.paths) unrecorded.paths.add(mountPoint) + for (const source of all.sources) unrecorded.sources.add(source) + for (const named of [...all.paths, ...all.sources]) { + unrecorded.madeByAPlan.delete(named) + } + } else { + // Not what an earlier wrap could record nowhere and this one names too: + // that wrap's sandbox may outlast this manifest. + for (const named of [...all.paths, ...all.sources]) { + if (unrecorded.madeByAPlan.delete(named)) { + unrecorded.paths.delete(named) + unrecorded.sources.delete(named) + } + } } - return undefined + return [inTheirSight, here].filter(manifest => manifest !== undefined) } /** - * Let go of the manifest of a wrap that produced no command: released, and + * Let go of the manifests of a wrap that produced no command: released, and * collected by a pass like any other. Not unlinked: a pass elsewhere may have * spared a path on its word and dropped the manifest that named the path till * then, and what nothing names any more stays on the host for good. */ -export function discardMountPointManifest(file: string): void { - const own = ownManifests.get(file) - if (own !== undefined) own.over = true +export function discardMountPointManifests(files: readonly string[]): void { + for (const file of files) { + const own = ownManifests.get(file) + if (own !== undefined) own.over = true + } collectMountPoints('none') } @@ -1143,17 +1762,18 @@ type Reading = { * one directory is doubt. * * A pass renames a manifest to claim it, to give it back and to say that it is - * finished, which can hide it - * from a listing under way and from the open that follows one, and one published - * while the directory is listed can be missed with its record shown. A record is - * never renamed. So the reading starts over for a listed manifest that is gone - * when opened, and for a record with no manifest listed beside it whose manifest - * is there when asked for by name. + * finished, which can hide it from a listing under way, or show it twice, and + * hide it from the open that follows one; and one published while the directory + * is listed can be missed with its record shown. A record is never renamed. So + * the reading starts over for a manifest that is listed at two names, for one + * that is gone when opened, and for a record with no manifest listed beside it + * whose manifest is there when asked for by name. One that is {@link remembered} + * is not opened, and is where the listing showed it, as of the listing. * - * A record with no manifest at any of its names puts the reading in doubt only while - * a process on it runs (see {@link recordVouches}) and is spent otherwise, one - * with no whole line past {@link MANIFEST_GRACE_MS} (its shell may be about to - * write to it). + * A record with no manifest at any of its names puts the reading in doubt only + * while a process on it runs (see {@link recordVouches}) and is spent otherwise, + * one with no whole line past {@link MANIFEST_GRACE_MS} (its shell may be about + * to write to it). * * Whatever is at a manifest's name and cannot be read puts the reading in doubt * too: always where that is an error of the moment (a sandboxed command can use @@ -1164,6 +1784,17 @@ type Reading = { */ function readManifests(dirs: readonly string[]): Reading { const oneKernel = dirs.filter(isReachedByOneKernel) + // Each directory is asked after once in a reading, whatever the answer. + const reachedInto = new Map() + const reached = (p: string): boolean => { + const dir = path.dirname(p) + let answer = reachedInto.get(dir) + if (answer === undefined) { + answer = othersReachInto(dir) + reachedInto.set(dir, answer) + } + return answer + } // What cannot be asked after counts as there. const isThere = (file: string): boolean => { try { @@ -1183,24 +1814,26 @@ function readManifests(dirs: readonly string[]): Reading { const reading: Reading = { files, manifests: [], spent: [] } const listed = new Set() let moved = false + let twice = false for (const file of files) { if (!MANIFEST_SUFFIXES.some(suffix => file.endsWith(suffix))) { continue } + // Listed at two of its names: renamed while the directory was listed. + if (listed.has(idOf(file))) { + twice = true + } listed.add(idOf(file)) try { - const manifest = readManifest(file) + const manifest = readManifest( + file, + !oneKernel.includes(path.dirname(file)), + reached, + ) if (manifest === undefined) { moved = true continue } - if ( - !oneKernel.includes(path.dirname(file)) && - (manifest.boot === undefined || manifest.boot !== ownBootId()) - ) { - delete manifest.boot - delete manifest.ns - } reading.manifests.push(manifest) } catch (e) { if ( @@ -1220,6 +1853,9 @@ function readManifests(dirs: readonly string[]): Reading { } } } + for (const id of remembered.keys()) { + if (!listed.has(id)) remembered.delete(id) + } for (const file of files) { if (file.endsWith(TEMPORARY_SUFFIX)) { // Left by a process killed between writing and moving into place. @@ -1255,7 +1891,9 @@ function readManifests(dirs: readonly string[]): Reading { } } } - if (moved && attempt < LISTING_ATTEMPTS) { + // One that is at two names for good (a copy, a restored snapshot) is read + // at both in the end, and judged at each like any other. + if ((moved || twice) && attempt < LISTING_ATTEMPTS) { continue } if (moved) { @@ -1285,11 +1923,11 @@ export type NamedMountPoints = { * * `earlier` is a reading this call may answer with. A manifest is never * rewritten, so it still holds when the directories list nothing it did not, - * `own` (a manifest the caller has published since) apart. + * `own` (the manifests the caller has published since) apart. */ export function namedMountPoints( earlier?: NamedMountPoints, - own?: string, + own: readonly string[] = [], ): NamedMountPoints | { inDoubt: string; keptChanging?: boolean } { try { const dirs = onLinux() ? believedManifestDirectories() : [] @@ -1297,7 +1935,9 @@ export function namedMountPoints( earlier !== undefined && // What a plan has made itself since then is named from that moment. [...unrecorded.paths].every(kept => earlier.paths.has(kept)) && - filesIn(dirs).every(file => earlier.listed.has(file) || file === own) + filesIn(dirs).every( + file => earlier.listed.has(file) || own.includes(file), + ) ) { return earlier } @@ -1353,18 +1993,27 @@ export function liveMountPoints(): ReadonlySet | undefined { if (!onLinux()) { return spared } - const named = new Set(unrecorded.paths) + const named = new Set() try { - const reading = readManifests(believedManifestDirectories()) - if (reading.inDoubt !== undefined) { - throw new Error(reading.inDoubt) - } - for (const manifest of reading.manifests) { - if ( - isLive(manifest) && - !(manifest.finished && !canBeAskedAfter(manifest)) - ) { - for (const mountPoint of manifest.paths) named.add(mountPoint) + for (let attempt = 1; ; attempt++) { + const reading = readManifests(believedManifestDirectories()) + if (reading.inDoubt !== undefined) { + throw new Error(reading.inDoubt) + } + named.clear() + for (const mountPoint of unrecorded.paths) named.add(mountPoint) + try { + for (const manifest of reading.manifests) { + if ( + !(manifest.finished && !canBeAskedAfter(manifest)) && + isLive(manifest) + ) { + for (const mountPoint of manifest.paths) named.add(mountPoint) + } + } + break + } catch (e) { + if (!(e instanceof Moved) || attempt === LISTING_ATTEMPTS) throw e } } } catch (e) { @@ -1396,30 +2045,43 @@ const inAPassName = { /** * Puts a pass of this process on record in each of `dirs`, until {@link - * takePassOffRecord}. `false`, with nothing left there, where it could not be - * in one of them: nothing may then be removed on a manifest's word. + * takePassOffRecord}, and returns those it is on record in. None, with nothing + * left there, where it could not be in one of this host's own: nothing may then + * be removed on a manifest's word. All but the one in the home where that takes + * no record (a home over its quota), which need only stop what other hosts reach + * from being removed. * * The name says whose it is: the pid, its start time, its PID namespace and the * boot, each left empty where it cannot be read. In the name, so that a pass is * on record whole or not at all and a listing is all it takes to judge one. */ -function putPassOnRecord(dirs: readonly string[]): boolean { +function putPassOnRecord(dirs: readonly string[]): string[] { const { pid, start } = ownProcess() const name = `${pid}-${start}-${inAPassName.ns()}-${inAPassName.boot()}-${ownPassMark}${randomBytes(4).toString('hex')}${PASS_SUFFIX}` ownPassOnRecord = dirs.map(dir => path.join(dir, name)) - try { - for (const file of ownPassOnRecord) { - fs.closeSync(fs.openSync(file, 'wx', 0o600)) + const onRecordIn: string[] = [] + for (const dir of dirs) { + const file = path.join(dir, name) + try { + if (isReachedByOneKernel(dir)) { + fs.closeSync(fs.openSync(file, 'wx', 0o600)) + } else { + putWhereOtherKernelsRead(file, '') + } + onRecordIn.push(dir) + } catch (e) { + const inTheHome = dir === network + logForDebugging( + `[Sandbox Linux] The clean-up of mount points could not be put on record in ${dir} (${String(e)}) - leaving what ${inTheHome ? 'other hosts reach' : 'the manifests name'} where it is`, + { level: 'warn' }, + ) + if (!inTheHome) { + takePassOffRecord() + return [] + } } - return true - } catch (e) { - takePassOffRecord() - logForDebugging( - `[Sandbox Linux] The clean-up of mount points could not be put on record (${String(e)}) - leaving what the manifests name where it is`, - { level: 'warn' }, - ) - return false } + return onRecordIn } /** One that stays behind is dropped by the next pass, of any process. */ @@ -1447,7 +2109,9 @@ function stateOfPass( ): 'under way' | 'over' | 'overdue' { const byItsAge = (): 'under way' | 'over' | 'overdue' => { try { - return Date.now() - fs.lstatSync(file).mtimeMs > PASS_MAX_AGE_MS + // Either way: one stamped by a clock that is hours ahead would else hold + // up every wrap for those hours. + return Math.abs(Date.now() - fs.lstatSync(file).mtimeMs) > PASS_MAX_AGE_MS ? 'overdue' : 'under way' } catch (e) { @@ -1477,12 +2141,18 @@ function stateOfPass( /** * The passes on record that may be under way. Throws where the directories * cannot be told or listed. + * + * Without `ofOtherHosts`, for a wrap that names nothing they reach, not those in + * the home: a pass of another host removes nothing else, and one of this host + * that may is on record in a place of this host's as well. */ -export function passesUnderWay(): string[] { +export function passesUnderWay(ofOtherHosts = true): string[] { if (!onLinux()) { return [] } - const dirs = believedManifestDirectories() + const dirs = believedManifestDirectories().filter( + dir => ofOtherHosts || dir !== network, + ) const oneKernel = dirs.filter(isReachedByOneKernel) return filesIn(dirs).filter( file => @@ -1506,10 +2176,14 @@ export async function passesToEnd( passes: readonly string[], signal?: AbortSignal, waitMs = WAITS_FOR_PASSES_MS, + ofOtherHosts = true, ): Promise { const deadline = performance.now() + waitMs let left = passes - for (let pause = 1; ; pause = Math.min(pause * 2, 50)) { + // Each look at the home is some round trips, from every host that waits. + const longest = (): number => + left.some(pass => path.dirname(pass) === network) ? 250 : 50 + for (let pause = 1; ; pause = Math.min(pause * 2, longest())) { if (left.length === 0 || performance.now() >= deadline) { return left[0] } @@ -1517,7 +2191,9 @@ export async function passesToEnd( signal?.throwIfAborted() // By its own name, which no other has: a directory that has two may be // listed by the other one this time. - const underWay = new Set(passesUnderWay().map(pass => path.basename(pass))) + const underWay = new Set( + passesUnderWay(ofOtherHosts).map(pass => path.basename(pass)), + ) left = left.filter(pass => underWay.has(path.basename(pass))) } } @@ -1554,6 +2230,20 @@ export function collectMountPoints(release: OwnManifestRelease): string[] { ensureManifestDirectory() let dirs: string[] try { + // The one in the home, where it is there, is set right as a wrap sets it. A + // `chmod -R g+rX ~` opens it up, and passed over for that, what the other + // hosts keep in it would go unread while what they reach is removed. + const inTheHome = networkManifestDirectoryName() + if ( + inTheHome !== undefined && + isOurPrivateDirectory(inTheHome, { + orThrow: true, + toWrite: false, + alone: false, + }) + ) { + makeOurPrivateDirectory(inTheHome) + } dirs = believedManifestDirectories() } catch (e) { logForDebugging( @@ -1607,17 +2297,19 @@ const childrenFirst = (paths: Iterable): string[] => * behind, before that reading; and a pass that cannot ask keeps nothing on the * word of a manifest at that name. A path that is kept is dropped from the * manifests of this pass and left to the one it is kept for. To a pass that - * cannot ask (one in another PID namespace) a claim is live like any other, so - * without the name each of two such passes would leave the path to the other's - * manifest. With it, whichever reads second finds the other's finished, and of - * all the manifests that name a path the pass over the last to be finished - * keeps it for none. What such a manifest names alone is still its own - * namespace's to remove, or anybody's once it is old. A pass that goes by the - * age of a manifest does not give it the name: whoever can ask goes on asking. + * cannot ask (one in another PID namespace, or on another host) a claim is live + * like any other, so without the name each of two such passes would leave the + * path to the other's manifest. With it, whichever reads second finds the + * other's finished, and of all the manifests that name a path the pass over the + * last to be finished keeps it for none. What such a manifest names alone is + * still for those to remove who can ask, or for anybody once it is old. A pass + * that goes by the age of a manifest does not give it the name: whoever can ask + * goes on asking. * * INVARIANT: the pass is on record in every directory it can write from before * that reading until after its last removal, and removes nothing where it could - * not be put there. A wrap lists the passes once it has published, and hands no + * not be put there (in the home, only a pass that may remove what other hosts + * reach, and that alone stays where the home takes no record). A wrap lists the passes once it has published, and hands no * command out while one is under way (see {@link passesToEnd}). So the wrap of * a manifest that the reading missed finds the pass on record, and no sandbox * starts under that manifest until the pass is over, for however long the pass @@ -1625,8 +2317,9 @@ const childrenFirst = (paths: Iterable): string[] => * * INVARIANT: the directories, worked out afresh, are listed again before every * removal (in a pass of over {@link LISTS_BEFORE_EACH_REMOVAL_UP_TO} - * candidates, whenever the listing is {@link LISTING_GOOD_FOR_MS} old), and - * what has come to its name keeps what it names. The record above does not make + * candidates, whenever the listing is {@link LISTING_GOOD_FOR_MS} old; those + * that other kernels reach, whenever it is {@link NETWORK_LISTING_GOOD_FOR_MS} + * old), and what has come to its name keeps what it names. The record above does not make * this redundant. A wrap sees no record of a pass that could go on none, or only * in a directory the wrap does not list (one inside another sandbox, which reads * the shared places and cannot write them); and a pass is on no record in a @@ -1643,6 +2336,11 @@ function collect(dirs: readonly string[], unrecordedToo: boolean): string[] { before.spent.forEach(drop) const own = new Set() let claims = false + // Whether other hosts reach any of what this pass may come to remove. + let othersReachSome = + unrecordedToo && + network !== undefined && + [...unrecorded.paths, ...unrecorded.sources].some(othersReach) if (before.inDoubt === undefined) { // Forget own manifests that something else has removed. const there = new Set(before.files.map(idOf)) @@ -1651,12 +2349,22 @@ function collect(dirs: readonly string[], unrecordedToo: boolean): string[] { ownManifests.delete(file) } } + // Another pass is at one that has moved: it is that one's. + const isLiveOrHasMoved = (manifest: Manifest): boolean => { + try { + return isLive(manifest) + } catch (e) { + if (e instanceof Moved) return true + throw e + } + } for (const manifest of before.manifests) { + if (isForAnotherHostToCollect(manifest)) continue const claim = manifest.claimed ? manifest.file : `${idOf(manifest.file)}${CLAIMED_SUFFIX}` if (!manifest.claimed) { - if (isLive(manifest)) continue + if (isLiveOrHasMoved(manifest)) continue try { fs.renameSync(manifest.file, claim) own.add(claim) @@ -1666,8 +2374,18 @@ function collect(dirs: readonly string[], unrecordedToo: boolean): string[] { } } // Asked again, now that no sandbox can start under it. - const over = !isLive({ ...manifest, file: claim, claimed: true }) + const over = !isLiveOrHasMoved({ + ...manifest, + file: claim, + claimed: true, + }) claims ||= over || own.has(claim) + // What is in the home is collected on record there, whatever it names. + othersReachSome ||= + over && + network !== undefined && + (path.dirname(claim) === network || + [...manifest.paths, ...manifest.sources].some(othersReach)) // Not of one that is only old: those who can ask go on asking. if (over && !manifest.finished && canBeAskedAfter(manifest)) { try { @@ -1695,18 +2413,31 @@ function collect(dirs: readonly string[], unrecordedToo: boolean): string[] { return [] } const writable = dirs.filter(dir => isOurPrivateDirectory(dir)) - // From before its record is made, whose age others go by. + // From before its record has its time: to whoever goes by the age of that, + // what a server takes to put it at its name is time the pass has had. const since = bothClocks() try { // Off the record it removes nothing on a manifest's word. What is left to // remove is what this process could record nowhere, which no wrap elsewhere // knows for a mount point or waits for. + // + // Not on record in the home for what no other host reaches: every wrap on + // every host that names what they do reach would wait for it. + const onRecordIn = putPassOnRecord( + writable.filter(dir => dir !== network || othersReachSome), + ) return removeWhatIsFinished( dirs, - putPassOnRecord(writable) ? writable : [], + onRecordIn, own, unrecordedToo, since, + // Where the home cannot be written from here at all (mounted read-only, + // bound so by an outer sandbox) no pass of this host is ever seen there. + // It goes by the manifests it reads, as it would with no home to read. + network !== undefined && + writable.includes(network) && + !onRecordIn.includes(network), ) } finally { takePassOffRecord() @@ -1716,7 +2447,9 @@ function collect(dirs: readonly string[], unrecordedToo: boolean): string[] { /** * The rest of {@link collect}, for as long as the pass is on record, which it * has been `since`: `own` are the claims it has made, `writable` the directories - * it can claim in. + * it can claim in and is on record in. With `unseenByOtherHosts` it is not on + * record where they look, though it could be, and nothing that they reach may + * go. */ function removeWhatIsFinished( dirs: readonly string[], @@ -1724,16 +2457,8 @@ function removeWhatIsFinished( own: ReadonlySet, unrecordedToo: boolean, since: readonly [number, number], + unseenByOtherHosts: boolean, ): string[] { - const reading = readManifests(dirs) - if (reading.inDoubt !== undefined) { - logForDebugging( - `[Sandbox Linux] A sandbox may be running on mount points that cannot be listed - leaving every mount point where it is (${reading.inDoubt})`, - { level: 'warn' }, - ) - return [] - } - const kept = new Set() const keep = ( manifest: Record<'paths' | 'sources', Iterable>, @@ -1742,32 +2467,64 @@ function removeWhatIsFinished( kept.add(named) } } - // A wrap of this process is outstanding, and may be the one that relies on it. - if (!unrecordedToo) keep(unrecorded) const finished: Manifest[] = [] - for (const manifest of reading.manifests) { - const asked = canBeAskedAfter(manifest) - if ( - manifest.claimed && - !isLive(manifest) && - // Else its last process has ended, or the renaming has failed, since the - // claims: it is the next pass's, which says so before it reads. - (manifest.finished || !asked) && - writable.includes(path.dirname(manifest.file)) - ) { - finished.push(manifest) - continue - } - // Not for this pass to remove, and nothing to keep anything for. - if (manifest.finished && !asked) { - continue + let reading: Reading + for (let attempt = 1; ; attempt++) { + reading = readManifests(dirs) + if (reading.inDoubt !== undefined) { + logForDebugging( + `[Sandbox Linux] A sandbox may be running on mount points that cannot be listed - leaving every mount point where it is (${reading.inDoubt})`, + { level: 'warn' }, + ) + return [] } - keep(manifest) - if (own.has(manifest.file)) { - try { - fs.renameSync(manifest.file, `${idOf(manifest.file)}${MANIFEST_SUFFIX}`) - } catch { - // Stays claimed, and kept while a process on its record runs. + kept.clear() + finished.length = 0 + // A wrap of this process is outstanding, and may be the one that relies on it. + if (!unrecordedToo) keep(unrecorded) + try { + for (const manifest of reading.manifests) { + if (isForAnotherHostToCollect(manifest)) { + if (!manifest.finished && isLive(manifest)) keep(manifest) + continue + } + const asked = canBeAskedAfter(manifest) + if ( + manifest.claimed && + !isLive(manifest) && + // Else its last process has ended, or the renaming has failed, since + // the claims: it is the next pass's, which says so before it reads. + (manifest.finished || !asked) && + writable.includes(path.dirname(manifest.file)) + ) { + finished.push(manifest) + continue + } + // Not for this pass to remove, and nothing to keep anything for. + if (manifest.finished && !asked) { + continue + } + keep(manifest) + if (own.has(manifest.file)) { + try { + fs.renameSync( + manifest.file, + `${idOf(manifest.file)}${MANIFEST_SUFFIX}`, + ) + } catch { + // Stays claimed, and kept while a process on its record runs. + } + } + } + break + } catch (e) { + if (!(e instanceof Moved)) throw e + if (attempt === LISTING_ATTEMPTS) { + logForDebugging( + `[Sandbox Linux] The mount point manifests kept changing (${e.message}) - leaving every mount point where it is`, + { level: 'warn' }, + ) + return [] } } } @@ -1785,18 +2542,40 @@ function removeWhatIsFinished( } } const known = new Set(reading.files) - const goodFor = - candidates.size <= LISTS_BEFORE_EACH_REMOVAL_UP_TO ? 0 : LISTING_GOOD_FOR_MS - let listedAt = -Infinity + // This host's own directories as ever. Those that other kernels reach, where + // a listing is a round trip, by a time of their own. + const listed = [ + { + oneKernel: true, + at: -Infinity, + goodFor: + candidates.size <= LISTS_BEFORE_EACH_REMOVAL_UP_TO + ? 0 + : LISTING_GOOD_FOR_MS, + }, + { oneKernel: false, at: -Infinity, goodFor: NETWORK_LISTING_GOOD_FOR_MS }, + ] const newcomersKeepWhatTheyName = (): boolean => { + const due = listed.filter( + ({ at, goodFor }) => performance.now() - at >= goodFor, + ) + if (due.length === 0) return true try { // The first place last: most is published there, so that listing is the // one to be a few microseconds old. - for (const file of filesIn(believedManifestDirectories().reverse())) { + const again = believedManifestDirectories() + .reverse() + .filter(dir => + due.some(({ oneKernel }) => oneKernel === isReachedByOneKernel(dir)), + ) + for (const file of filesIn(again)) { if (known.has(file) || !file.endsWith(MANIFEST_SUFFIX)) continue // Published, or given back, since the reading. One that is gone again // is looked for the next time: it may be given back once more. - const arrived = readManifest(file) + const arrived = readManifest( + file, + !isReachedByOneKernel(path.dirname(file)), + ) if (arrived === undefined) continue keep(arrived) known.add(file) @@ -1808,24 +2587,42 @@ function removeWhatIsFinished( ) return false } - listedAt = performance.now() + for (const listing of due) listing.at = performance.now() return true } const removed: string[] = [] const notRemoved = new Set() + const lookedAfreshAt = new Set() for (const candidate of childrenFirst(candidates.keys())) { - if ( - performance.now() - listedAt >= goodFor && - !newcomersKeepWhatTheyName() - ) { + if (!newcomersKeepWhatTheyName()) { return removed } if (kept.has(candidate)) { continue } + if (unseenByOtherHosts && othersReach(candidate)) { + notRemoved.add(candidate) + continue + } + // What is removed is looked at first, and removed by its name: another host + // may have put a file of the user's there since this one last asked. + if (!lookedAfreshAt.has(path.dirname(candidate))) { + lookedAfreshAt.add(path.dirname(candidate)) + lookAfreshAt([candidate]) + } + // Or have written into the very file (old bubblewrap makes it 0644), which + // leaves the directory as it was. + if (network !== undefined && othersReach(candidate)) { + openAndClose( + candidate, + fs.constants.O_RDONLY | + fs.constants.O_NOFOLLOW | + fs.constants.O_NONBLOCK, + ) + } // To a process that cannot ask after this one the pass is over by its age. - // The last thing before a removal, after all that may have taken time. + // Asked last, after all that may have waited for a server. if (bothClocks().some((now, i) => now - since[i]! > PASS_GOOD_FOR_MS)) { logForDebugging( '[Sandbox Linux] The clean-up of mount points has taken too long - leaving the rest to the next', @@ -1869,9 +2666,9 @@ function removeWhatIsFinished( } /** - * Look for the places that follow from the user id under `under` instead of - * `/`. Test seam, and nothing else may call it: no environment keeps a test out - * of the ones the user's own processes keep. + * Look for the places that follow from the user id, for /etc/passwd and for + * `getent` under `under` instead of `/`. Test seam, and nothing else may call it: no + * environment keeps a test out of the ones the user's own processes keep. */ export function setMountPointManifestPlacesForTesting(under: string): void { root = under @@ -1887,7 +2684,17 @@ export function setMountPointManifestPlacesForTesting(under: string): void { export function forgetMountPointManifestDirectory(): void { pidNamespace = null bootId = undefined + home = undefined + noAnswer = undefined + network = undefined + foundWithinOthersReach.clear() + remembered.clear() manifestDirectory = undefined manifestDirectoryIsPrivate = false manifestDirectoryUnavailable = false + if (keptYoung !== undefined) { + clearInterval(keptYoung) + keptYoung = undefined + } + settingTimes = false } diff --git a/src/sandbox/linux-sandbox-utils.ts b/src/sandbox/linux-sandbox-utils.ts index cf3b1efce..9c24369a2 100644 --- a/src/sandbox/linux-sandbox-utils.ts +++ b/src/sandbox/linux-sandbox-utils.ts @@ -11,11 +11,13 @@ import path, { join } from 'node:path' import { ripGrep, RipgrepError, type RipgrepConfig } from '../utils/ripgrep.js' import { collectMountPoints, - discardMountPointManifest, + discardMountPointManifests, holdsMountPointManifest, keepTrackOfMountPoint, keepTrackOfMountSource, kindOfMountPoint, + lookAfreshAt, + otherHostsReach, mountPointManifestDirectories, namedMountPoints, passesToEnd, @@ -999,8 +1001,9 @@ export type LinuxSandboxProfileErrorCode = /** * The mount point manifests cannot be read, so a path cannot be told from a * mount point a running sandbox relies on; or one of their directories cannot - * be looked at, so it cannot be kept out of the command's reach. It ends by - * itself: when this process has descriptors again, when the file system + * be looked at, or whether the home is a network home cannot be told, so they + * cannot all be kept out of the command's reach. It ends by itself: when this + * process has descriptors again, when the file system or the user database * answers again, when what sits among them and is no manifest is an hour old, * when the process on a record with no manifest has ended. `.cause` is the * reason, with the file where there is one, which the message holds too. @@ -1114,8 +1117,8 @@ export class WorkingDirectoryChanged extends Error { * command among them, stays on the line. The result stays a simple command, * so a prefix (`exec`, `timeout 30`) or a suffix (`&& next`) still composes. * - * With `started`, the record of a wrap that named mount points, the string is - * a shell that appends its own /proc/self/stat line to that record and then + * With `started`, the records of a wrap that named mount points, the string is + * a shell that appends its own /proc/self/stat line to each of them and then * execs bwrap in its place: the pid on record is bwrap's, written before bwrap * has made a mount. Read by the shell itself, a redirection on a builtin: a * child's line would be that of a process gone at once. Not /proc/$$: under an @@ -1131,19 +1134,14 @@ function renderBwrapInvocation( bwrapBinary: string, bwrapArgs: string[], mounts: { start: number; end: number; mandatoryDenyArgs: number }, - started?: string, + started: readonly string[] = [], ): string { /** What the string's own shell does, to the file it is given as "$1". */ type Step = { run: string; file: string } - const record: Step[] = - started === undefined - ? [] - : [ - { - run: 'srt_record() { local s && read -r s >"$1"; } && srt_record "$1"', - file: started, - }, - ] + const record: Step[] = started.map(file => ({ + run: 'srt_record() { local s && read -r s >"$1"; } && srt_record "$1"', + file, + })) /** `words`, run by a shell named `name` once it has taken `steps`. */ const afterSteps = (steps: Step[], name: string, words: string[]): string => steps.length === 0 @@ -2243,9 +2241,9 @@ async function generateFilesystemArgs( args: string[] /** How many of them, from the first, have to stay on the command line. */ onTheLine: number - manifest: MountPointManifest | undefined - /** Where the manifest's started record is, as the binds spell its directory. */ - started: string | undefined + manifests: MountPointManifest[] + /** Where their started records are, as the binds spell their directories. */ + started: string[] /** How many of them are the mandatory denies'. */ mandatoryDenyArgs: number } @@ -2262,7 +2260,9 @@ async function generateFilesystemArgs( let manifestDirs: string[] = [] const makeManifestDirs = (recording: boolean): void => { try { - manifestDirs = mountPointManifestDirectories(recording) + manifestDirs = mountPointManifestDirectories(recording, dir => + isWithinAnyAllowedWritePath(nameLocationOf(dir)), + ) } catch (err) { throw new LinuxSandboxProfileError( 'mount_points_unreadable', @@ -2273,8 +2273,10 @@ async function generateFilesystemArgs( } // What the manifests name, as of the last time it was asked. let named: NamedMountPoints | undefined - const whatIsNamedNow = (ownManifest?: string): ReadonlySet => { - const now = namedMountPoints(named, ownManifest) + const whatIsNamedNow = ( + ownManifests: readonly string[] = [], + ): ReadonlySet => { + const now = namedMountPoints(named, ownManifests) if ('inDoubt' in now) { if (now.keptChanging) { throw new MountPlanFailed(now.inDoubt) @@ -3732,9 +3734,11 @@ async function generateFilesystemArgs( // another sandbox's mount points, which go when no live manifest names them. // A file among them was made again after the deny loop took it for the // caller's own. - const mountPointsReliedOn = (ownManifest?: string): string[] => { + const mountPointsReliedOn = ( + ownManifests: readonly string[] = [], + ): string[] => { const found = new Set() - const namedNow = boundFrom.size > 0 ? whatIsNamedNow(ownManifest) : found + const namedNow = boundFrom.size > 0 ? whatIsNamedNow(ownManifests) : found for (const relied of namedNow.size > 0 ? [...madeIn, ...boundFrom] : []) { for (let dir = relied; dir !== '/'; dir = path.dirname(dir)) { if (namedNow.has(dir)) found.add(dir) @@ -3751,13 +3755,47 @@ async function generateFilesystemArgs( // a reading taken after the last look at the host above. A directory it left // out (the one a placeholder is nested in, a pin) goes with its maker, and the // command is then refused its start or, once the pin is mounted, loses it. + // + // INVARIANT, as in the deny loop: only a path found absent is named as this + // wrap's own to remove. Of a directory that other hosts reach, absent may be + // what this host remembers, and the user may have put an empty file there + // from another host since. So the server is asked, before anything of this + // wrap's names the path: what is there now and no manifest names is the + // caller's own, and the next plan finds it. (What the plan has made itself, on + // a 9p drive, counts as named from the moment it is made.) + let othersMayHaveMadeOne: boolean + try { + othersMayHaveMadeOne = otherHostsReach(mountPoints) + } catch (err) { + throw new LinuxSandboxProfileError( + 'mount_points_unreadable', + `Whether other hosts reach the mount points cannot be told (${errorText(err)})`, + errorText(err), + ) + } + if (othersMayHaveMadeOne) { + lookAfreshAt(mountPoints) + const thereNow = mountPoints.filter(made => { + try { + fs.lstatSync(made) + return true + } catch { + // As the plan found it, or not to be looked at: bubblewrap says which. + return false + } + }) + if (thereNow.length > 0) { + const namedNow = whatIsNamedNow() + if (thereNow.some(made => !namedNow.has(made))) return undefined + } + } const names = [...mountPoints, ...mountPointsReliedOn()] - const manifest = publishMountPointManifest( + const manifests = publishMountPointManifest( names, emptySource === undefined ? [] : [emptySource], commandKey, ) - if (manifest === 'too-large') { + if (manifests === 'too-large') { throw new LinuxSandboxProfileError( 'too_many_mount_points', `Sandbox profile relies on ${names.length} mount points, more than one manifest holds; reduce the write denies on paths that do not exist`, @@ -3784,13 +3822,17 @@ async function generateFilesystemArgs( // is waited for, and the look at the host comes after. Asked without giving // the thread back first, which is all there is to it but for a moment now // and then. - if (manifest !== undefined) { + if (manifests.length > 0) { let heldUp: string | undefined try { - const underWay = passesUnderWay() + const ofOtherHosts = otherHostsReach([ + ...names, + ...(emptySource === undefined ? [] : [emptySource]), + ]) + const underWay = passesUnderWay(ofOtherHosts) heldUp = underWay.length > 0 - ? await passesToEnd(underWay, abortSignal) + ? await passesToEnd(underWay, abortSignal, undefined, ofOtherHosts) : undefined } catch (err) { if (abortSignal?.aborted) throw err @@ -3810,18 +3852,25 @@ async function generateFilesystemArgs( // One plan, one working directory, on both sides of this await too. if (workingDirectory() !== startedIn) throw new WorkingDirectoryChanged() } + // What another host has removed is to be found gone, by the look below and + // by bubblewrap: meanwhile, or before the plan was made, which may have + // taken for the caller's own a mount point this host only remembers. Its + // manifest is dropped after it, so no reading names it any more. + lookAfreshAt([...names, ...madeIn, ...boundFrom]) holds = // Else let go of during the wait, by a forced clean-up: no command starts // under it any more. - (manifest === undefined || holdsMountPointManifest(manifest.file)) && + manifests.every(({ file }) => holdsMountPointManifest(file)) && [...boundFrom].every(relied => isThere(relied, false)) && [...madeIn].every(relied => isThere(relied, true)) && - mountPointsReliedOn(manifest?.file).every(dir => names.includes(dir)) && + mountPointsReliedOn(manifests.map(({ file }) => file)).every(dir => + names.includes(dir), + ) && // Else settled on since the pins were made: nothing is pinned above it. - (manifest === undefined || manifestDirs.includes(manifest.dir)) + manifests.every(({ dir }) => manifestDirs.includes(dir)) } finally { - if (!holds && manifest !== undefined) { - discardMountPointManifest(manifest.file) + if (!holds && manifests.length > 0) { + discardMountPointManifests(manifests.map(({ file }) => file)) } } if (!holds) { @@ -3838,12 +3887,12 @@ async function generateFilesystemArgs( // their sandboxes. const isInReach = (dir: string): boolean => !isHiddenByTmpfs(dir, 'writes and reads') - let started: string | undefined + const started: string[] = [] let onTheLine = 0 - if (manifest !== undefined) { + for (const manifest of manifests) { const dir = canonicalForm(manifest.dir) const file = path.join(dir, path.basename(manifest.file)) - started = path.join(dir, path.basename(manifest.started)) + started.push(path.join(dir, path.basename(manifest.started))) // INVARIANT: the manifest is the first thing bound after the root, and its // bind is on the command line, not in an --args file. bubblewrap looks up // every bind source before it makes anything, so a command started once a @@ -3853,10 +3902,11 @@ async function generateFilesystemArgs( // and bound any later it would by then have made mount points on the host // that nothing names or removes. On the line, because the --args file is // opened by a descriptor number that another wrap's file may have by then. - args.splice(afterRootBind, 0, '--ro-bind', file, file) - onTheLine = afterRootBind + 3 + args.splice(afterRootBind + onTheLine, 0, '--ro-bind', file, file) + onTheLine += 3 if (isInReach(dir)) args.push('--ro-bind', dir, dir) } + if (onTheLine > 0) onTheLine += afterRootBind if (names.length > 0 || emptySource !== undefined) { registerExitCleanupHandler() } @@ -3866,7 +3916,7 @@ async function generateFilesystemArgs( // replaced - is not an error. // Each where it really is, once: two of the names may lead to one place. const others = new Set(manifestDirs.map(canonicalForm)) - if (manifest !== undefined) others.delete(canonicalForm(manifest.dir)) + for (const { dir } of manifests) others.delete(canonicalForm(dir)) for (const dir of others) { if (isInReach(dir)) args.push('--ro-bind-try', dir, dir) } @@ -3899,7 +3949,7 @@ async function generateFilesystemArgs( args.push('--ro-bind', dir, dir) } - return { args, onTheLine, manifest, started, mandatoryDenyArgs } + return { args, onTheLine, manifests, started, mandatoryDenyArgs } } /** @@ -4026,9 +4076,9 @@ export async function wrapCommandWithSandboxLinux( const bwrapArgs: string[] = ['--new-session', '--die-with-parent'] let applySeccompPrefix: string | undefined - // The manifest naming the mount points this wrap relies on, dropped again if + // The manifests naming the mount points this wrap relies on, dropped again if // no command comes of it. - let manifest: MountPointManifest | undefined + let manifests: MountPointManifest[] = [] try { // ========== SECCOMP FILTER (Unix Socket Blocking) ========== @@ -4225,7 +4275,7 @@ export async function wrapCommandWithSandboxLinux( } // Held for the catch below: a wrap that produces no command has to let go // of the mount points it named. - manifest = filesystem.manifest + manifests = filesystem.manifests const mounts = { start: bwrapArgs.length + filesystem.onTheLine, end: bwrapArgs.length + filesystem.args.length, @@ -4363,9 +4413,9 @@ export async function wrapCommandWithSandboxLinux( if (forcedCleanups !== countedAfter) activeSandboxCount++ return wrappedCommand } catch (error) { - // No command came of this wrap, so no sandbox starts under its manifest. - if (manifest !== undefined) { - discardMountPointManifest(manifest.file) + // No command came of this wrap, so no sandbox starts under its manifests. + if (manifests.length > 0) { + discardMountPointManifests(manifests.map(({ file }) => file)) } // Its place in the count goes back through the clean-up: a command that // ended meanwhile left to this wrap what goes when none is outstanding. diff --git a/test/helpers/isolated/host.ts b/test/helpers/isolated/host.ts new file mode 100644 index 000000000..f7951ee34 --- /dev/null +++ b/test/helpers/isolated/host.ts @@ -0,0 +1,49 @@ +import { spyOn } from 'bun:test' +import * as fs from 'node:fs' +import { setMountPointManifestPlacesForTesting } from '../../../src/sandbox/bwrap-mount-manifests.js' + +// Whatever loads this is a process on a host of its own, as far as the mount +// point manifests can tell: the places that follow from the user id, and the +// /etc/passwd that says where the user's home is, are under $SRT_TEST_HOST (see +// makePlaces), the boot is $SRT_TEST_BOOT, and everything under +// $SRT_TEST_NETWORK is on NFS. Two processes that differ in the first two and +// have one home under the third are two hosts that mount one home. What it does +// not stand in for is what a network file system caches. +const { SRT_TEST_HOST, SRT_TEST_BOOT, SRT_TEST_NETWORK } = process.env +if ( + SRT_TEST_HOST === undefined || + SRT_TEST_BOOT === undefined || + SRT_TEST_NETWORK === undefined +) { + throw new Error('a host needs its three variables') +} + +const NFS_SUPER_MAGIC = 0x6969 +const statfs = fs.statfsSync +spyOn(fs, 'statfsSync').mockImplementation((( + p: unknown, + ...rest: unknown[] +) => { + const real = (statfs as (...args: unknown[]) => { type: number })(p, ...rest) + return `${String(p)}/`.startsWith(`${SRT_TEST_NETWORK}/`) + ? { ...real, type: NFS_SUPER_MAGIC } + : real +}) as never) + +const read = fs.readFileSync +spyOn(fs, 'readFileSync').mockImplementation((( + file: unknown, + ...rest: unknown[] +) => + file === '/proc/sys/kernel/random/boot_id' + ? `${SRT_TEST_BOOT}\n` + : (read as (...args: unknown[]) => unknown)(file, ...rest)) as never) + +setMountPointManifestPlacesForTesting(SRT_TEST_HOST) + +export * from '../../../src/sandbox/linux-sandbox-utils.js' +export { + collectMountPoints, + liveMountPoints, + publishMountPointManifest, +} from '../../../src/sandbox/bwrap-mount-manifests.js' diff --git a/test/helpers/private-manifest-directory.ts b/test/helpers/private-manifest-directory.ts index 8ee398342..bd381fe83 100644 --- a/test/helpers/private-manifest-directory.ts +++ b/test/helpers/private-manifest-directory.ts @@ -9,6 +9,7 @@ import { readdirSync, realpathSync, rmSync, + writeFileSync, } from 'node:fs' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' @@ -57,12 +58,21 @@ function endSandboxesOnRecord(dir: string): void { /** * Makes `/var/tmp`, `/run/user/UID` and `/tmp` under `root`, as a system has - * them. Returns the places the manifests are kept in there, in the order they - * are tried, none of them made: for `setMountPointManifestPlacesForTesting`, or - * {@link inPrivateNamespace}. + * them, and an `/etc/passwd` that gives the user `home` (one under `root`, not + * made, unless given). Returns the places the manifests are kept in there, in + * the order they are tried, none of them made: for + * `setMountPointManifestPlacesForTesting`, or {@link inPrivateNamespace}. */ -export function makePlaces(root: string): [string, string, string] { +export function makePlaces( + root: string, + home = join(root, 'home/user'), +): [string, string, string] { const uid = process.getuid?.() ?? 0 + mkdirSync(join(root, 'etc'), { recursive: true }) + writeFileSync( + join(root, 'etc/passwd'), + `user:x:${uid}:${uid}::${home}:/bin/sh\n`, + ) const places = [ join(root, 'var/tmp', `srt-mount-points-${uid}`), join(root, 'run/user', String(uid), 'srt-mount-points'), @@ -79,9 +89,10 @@ export function makePlaces(root: string): [string, string, string] { * The start of a command line that runs what follows it in a mount namespace of * its own, where what {@link makePlaces} made under `root` stands at its real * name: for a child process that is to work the places out for itself, with - * nothing put in its way, and not reach the user's own. `keep`, this repository - * and the runtime are bound back where they are: they may lie under one of the - * three. No PID namespace. Needs `bwrapCanNamespace`. + * nothing put in its way, and not reach the user's own, the one in a network + * home included. `keep`, this repository and the runtime are bound back where + * they are: they may lie under one of the three. No PID namespace. Needs + * `bwrapCanNamespace`. */ export function inPrivateNamespace(root: string, keep: string[]): string[] { return [ @@ -94,6 +105,9 @@ export function inPrivateNamespace(root: string, keep: string[]): string[] { ...['/var/tmp', '/run/user', '/tmp'] .filter(dir => existsSync(dir)) .flatMap(dir => ['--bind', join(root, dir), dir]), + '--ro-bind', + join(root, 'etc/passwd'), + '/etc/passwd', ...[ join(import.meta.dir, '../..'), dirname(process.execPath), diff --git a/test/sandbox/mount-point-hosts.test.ts b/test/sandbox/mount-point-hosts.test.ts new file mode 100644 index 000000000..9f885e67f --- /dev/null +++ b/test/sandbox/mount-point-hosts.test.ts @@ -0,0 +1,2568 @@ +import { describe, it, expect, beforeEach, afterEach, spyOn } from 'bun:test' +import { type ChildProcess, spawn, spawnSync } from 'node:child_process' +import * as fs from 'node:fs' +import { + chmodSync, + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + readlinkSync, + realpathSync, + rmSync, + utimesSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { basename, dirname, join } from 'node:path' +import { + collectMountPoints, + forgetMountPointManifestDirectory, + liveMountPoints, + lookAfreshAt, + mountPointManifestDirectories, + namedMountPoints, + otherHostsReach, + passesToEnd, + passesUnderWay, + publishMountPointManifest, + setMountPointManifestPlacesForTesting, +} from '../../src/sandbox/bwrap-mount-manifests.js' +import { + cleanupBwrapMountPoints, + wrapCommandWithSandboxLinux, +} from '../../src/sandbox/linux-sandbox-utils.js' +import { manifestOf } from '../helpers/bwrap-argv.js' +import { bwrapCanNamespace } from '../helpers/bwrap-namespace.js' +import { + leftover, + sleep, + until, + writeScript, +} from '../helpers/mount-point-fixtures.js' +import { isLinux } from '../helpers/platform.js' +import { makePlaces } from '../helpers/private-manifest-directory.js' + +const NFS = 0x6969 +const EXT4 = 0xef53 +const MINUTE_MS = 60 * 1000 + +/** + * A mount point on a file system that several hosts mount is there for all of + * them, and none can ask after another's processes or read a directory of + * another's. So a wrap that names one keeps its manifest in the user's home, + * where that is on such a file system, and there a manifest of another host is + * live by its age, which its writer keeps low. + */ +describe.if(isLinux)('Mount points that other hosts reach', () => { + const BWRAP_CAN_NAMESPACE = bwrapCanNamespace() + const NOT_ROOT = process.getuid?.() !== 0 + const HOST = JSON.stringify( + join(import.meta.dir, '../helpers/isolated/host.ts'), + ) + const OWN_NAMESPACE = isLinux ? readlinkSync('/proc/self/ns/pid') : '' + const OWN_BOOT = isLinux + ? readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim() + : '' + const ANOTHER_BOOT = '11111111-2222-3333-4444-555555555555' + /** Echoed first by every command that runs for real. */ + const BOOTED = 'BOOTED' + + let BASE: string + let HERE: string // what stands in for `/` on this host + let PLACES: string[] // this host's own places + let NETWORK: string // what every host mounts + let HOME: string // the user's home, on it + let SHARED: string // where the manifests go that other hosts read + let PROJECT: string // a project on it + let LOCK: string // a denyWrite path in that, absent to begin with + let LOCAL: string // a directory only this host has + let statfs: { mockRestore(): void } + let others: ChildProcess[] + + beforeEach(() => { + BASE = realpathSync(mkdtempSync(join(tmpdir(), 'mount-point-hosts-'))) + HERE = join(BASE, 'here') + NETWORK = join(BASE, 'network') + HOME = join(NETWORK, 'home') + SHARED = join(HOME, '.srt-mount-points') + PROJECT = join(NETWORK, 'project') + LOCK = join(PROJECT, 'config.lock') + LOCAL = join(BASE, 'local') + for (const dir of [HERE, HOME, PROJECT, LOCAL]) { + mkdirSync(dir, { recursive: true }) + } + PLACES = makePlaces(HERE, HOME) + statfs = networkIs(NFS) + setMountPointManifestPlacesForTesting(HERE) + cleanupBwrapMountPoints({ force: true }) + others = [] + }) + + afterEach(() => { + for (const other of others) { + for (const signal of ['SIGCONT', 'SIGKILL'] as const) { + try { + if (other.pid !== undefined) process.kill(other.pid, signal) + } catch { + // Gone already. + } + } + } + cleanupBwrapMountPoints({ force: true }) + statfs.mockRestore() + setMountPointManifestPlacesForTesting('/nonexistent') + rmSync(BASE, { recursive: true, force: true }) + }) + + /** Has what is under NETWORK be on a file system of that kind. */ + function networkIs(type: number): { mockRestore(): void } { + return spyOn(fs, 'statfsSync').mockImplementation(((p: unknown) => { + if (!existsSync(String(p))) { + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }) + } + // A plan asks through a descriptor it holds. + const at = String(p).startsWith('/proc/self/fd/') + ? readlinkSync(String(p)) + : String(p) + return { type: `${at}/`.startsWith(`${NETWORK}/`) ? type : EXT4 } + }) as never) + } + + function aged(file: string, ms: number): string { + const then = new Date(Date.now() - ms) + utimesSync(file, then, then) + return file + } + + /** + * As `aged`, of a file this process may have looked at. A manifest's time is + * only ever set forward, and the library counts on that. + */ + function setBack(file: string, ms: number): string { + forgetMountPointManifestDirectory() + return aged(file, ms) + } + + /** A manifest another host wrote into the shared directory, just now. */ + function manifestOfAnotherHost( + paths: string[], + sources: string[] = [], + ): string { + mkdirSync(SHARED, { recursive: true, mode: 0o700 }) + chmodSync(SHARED, 0o700) + // A pid that is this process's here, and there somebody else's. + const file = join( + SHARED, + `${process.pid}-${Math.random().toString(16).slice(2)}.json`, + ) + writeFileSync( + file, + JSON.stringify({ + version: 1, + pid: process.pid, + start: '1', + ns: OWN_NAMESPACE, + boot: ANOTHER_BOOT, + keptYoung: true, + created: Date.now(), + paths, + sources, + }), + { mode: 0o600 }, + ) + return file + } + + /** Publishes for a wrap that names those. Returns the manifests written. */ + function publish(paths: string[], sources: string[] = []): string[] { + const manifests = publishMountPointManifest(paths, sources) + if (manifests === 'too-large') throw new Error(manifests) + return manifests.map(({ file }) => file) + } + + function named(): string[] | undefined { + const reading = namedMountPoints() + return 'inDoubt' in reading ? undefined : [...reading.paths].sort() + } + + const live = (): string[] => [...(liveMountPoints() ?? ['in doubt'])] + + // ---- where a manifest goes --------------------------------------------------- + + /** What the manifest at `file` names. */ + function namedIn(file: string): { paths: string[]; sources: string[] } { + const { paths, sources } = JSON.parse(readFileSync(file, 'utf8')) as { + paths: string[] + sources: string[] + } + return { paths, sources } + } + + it("keeps a manifest of a wrap that names one in the home directory, beside the one in this host's own first place", () => { + expect(publish([LOCK]).map(dirname)).toEqual([SHARED, PLACES[0]!]) + expect(lstatSync(SHARED).mode & 0o777).toBe(0o700) + }) + + it("names only what other hosts reach there, and all of it in this host's own first place", () => { + // Whoever reads a manifest can then remove all it names: what a job left on + // its own host's disk is not forgotten when another host sweeps the home. + // And a process of this host that does not see the home (a service kept + // out of it, a container with another) reads what it always did. + const [a, b] = [join(LOCAL, 'a'), join(LOCAL, 'b')] + const source = join(LOCAL, 'empty') + const [theirsToo, here] = publish([a, LOCK, b], [source]) + expect(dirname(theirsToo!)).toBe(SHARED) + expect(namedIn(theirsToo!)).toEqual({ paths: [LOCK], sources: [] }) + expect(dirname(here!)).toBe(PLACES[0]!) + expect(namedIn(here!)).toEqual({ paths: [a, LOCK, b], sources: [source] }) + }) + + it("keeps the manifest of a wrap that names none in this host's own first place", () => { + expect(publish([join(LOCAL, 'config.lock')]).map(dirname)).toEqual([ + PLACES[0]!, + ]) + expect(existsSync(SHARED)).toBe(false) + }) + + it.if(NOT_ROOT)( + 'has that one alone where the home takes no directory', + () => { + chmodSync(HOME, 0o500) + try { + const local = join(LOCAL, 'config.lock') + const [here, ...more] = publish([LOCK, local]) + expect(more).toEqual([]) + expect(dirname(here!)).toBe(PLACES[0]!) + expect(namedIn(here!).paths).toEqual([LOCK, local]) + } finally { + chmodSync(HOME, 0o700) + } + }, + ) + + it('does so too where the directory in the home takes no manifest, as in a home over its quota', () => { + // A network file system takes the bytes and says so only as they are + // flushed. Published all the same, an empty file would put every reading on + // every host in doubt, and then be dropped with nothing naming the path. + const spy = spyOn(fs, 'fsyncSync').mockImplementation((() => { + throw Object.assign(new Error('EDQUOT'), { code: 'EDQUOT' }) + }) as never) + try { + const [here, ...more] = publish([LOCK]) + expect(more).toEqual([]) + expect(dirname(here!)).toBe(PLACES[0]!) + expect(namedIn(here!).paths).toEqual([LOCK]) + expect(readdirSync(SHARED)).toEqual([]) + } finally { + spy.mockRestore() + } + }) + + it('writes neither of the two when both together are more than one manifest holds', () => { + const many = Array.from({ length: 12000 }, (_, i) => + join(i % 2 ? LOCAL : PROJECT, `${'x'.repeat(80)}${i}`), + ) + expect(publishMountPointManifest(many, [])).toBe('too-large') + expect(existsSync(SHARED) ? readdirSync(SHARED) : []).toEqual([]) + expect(existsSync(PLACES[0]!) ? readdirSync(PLACES[0]!) : []).toEqual([]) + }) + + it('reads and writes nothing in a home that only this host reaches, and makes nothing there that no command could', () => { + statfs.mockRestore() + statfs = networkIs(EXT4) + forgetMountPointManifestDirectory() + expect(mountPointManifestDirectories(true)).not.toContain(SHARED) + expect(publish([LOCK]).map(dirname)).toEqual([PLACES[0]!]) + expect(existsSync(SHARED)).toBe(false) + // One that is there all the same is nobody's word. + manifestOfAnotherHost([LOCK]) + leftover(LOCK) + collectMountPoints('all') + expect(named()).toEqual([]) + }) + + const wrapThatMayWrite = (...allowOnly: string[]): Promise => + wrapCommandWithSandboxLinux({ + command: `echo ${BOOTED}; echo x > ${join(SHARED, 'forged.json')}; echo forged=$?`, + needsNetworkRestriction: false, + readConfig: { denyOnly: [] }, + writeConfig: { allowOnly, denyWithinAllow: [] }, + }) + + it('keeps the directory out of reach there all the same: to the clients of a server, the disk it exports is a network home', async () => { + statfs.mockRestore() + statfs = networkIs(EXT4) + forgetMountPointManifestDirectory() + // Not there, and not the command's to make. + expect(await wrapThatMayWrite(LOCAL)).not.toContain(`-try ${SHARED} `) + expect(existsSync(SHARED)).toBe(false) + // The command's to make, so it is made first. + const command = await wrapThatMayWrite(NETWORK) + expect(lstatSync(SHARED).mode & 0o777).toBe(0o700) + expect(command).toContain(`--ro-bind-try ${SHARED} ${SHARED}`) + expect(command).toContain(`--ro-bind ${HOME} ${HOME}`) + // There, so it is bound, like any other place. + expect(await wrapThatMayWrite(LOCAL)).toContain( + `--ro-bind-try ${SHARED} ${SHARED}`, + ) + // And is nobody's word here for that. + manifestOfAnotherHost([LOCK]) + expect(named()).toEqual([]) + expect(publish([LOCK]).map(dirname)).toEqual([PLACES[0]!]) + }) + + it('asks whether the command could make it of where the home really is, however the user database spells it', async () => { + statfs.mockRestore() + statfs = networkIs(EXT4) + const spelled = join(BASE, 'home-of-user') + fs.symlinkSync(HOME, spelled) + makePlaces(HERE, spelled) + forgetMountPointManifestDirectory() + const command = await wrapThatMayWrite(HOME) + expect(lstatSync(SHARED).isDirectory()).toBe(true) + expect(command).toContain(`--ro-bind-try ${SHARED} ${SHARED}`) + }) + + it.if(BWRAP_CAN_NAMESPACE)( + 'lets no command of such a host write what the others would believe', + async () => { + statfs.mockRestore() + statfs = networkIs(EXT4) + forgetMountPointManifestDirectory() + const { said } = run(await wrapThatMayWrite(HOME)) + expect(said).toContain(BOOTED) + expect(said).toMatch(/forged=[1-9]/) + expect(readdirSync(SHARED)).toEqual([]) + }, + ) + + it.each([NFS, EXT4])( + 'hands out no command while the directory there cannot be looked at: it may be there, and would go unbound (%d)', + async type => { + mountPointManifestDirectories() + statfs.mockRestore() + statfs = networkIs(type) + forgetMountPointManifestDirectory() + const lstat = fs.lstatSync + const spy = spyOn(fs, 'lstatSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (file === SHARED) { + throw Object.assign(new Error('EIO'), { code: 'EIO' }) + } + return (lstat as (...args: unknown[]) => unknown)(file, ...rest) + }) as never) + try { + expect(() => mountPointManifestDirectories()).toThrow('EIO') + const refused = await wrapThatMayWrite(LOCAL).then( + () => undefined, + (error: unknown) => error, + ) + expect((refused as { code?: string }).code).toBe( + 'mount_points_unreadable', + ) + } finally { + spy.mockRestore() + } + }, + ) + + it("hands out none while one of this host's own places cannot be looked at either", async () => { + mountPointManifestDirectories() + const lstat = fs.lstatSync + const spy = spyOn(fs, 'lstatSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (file === PLACES[2]) { + throw Object.assign(new Error('EIO'), { code: 'EIO' }) + } + return (lstat as (...args: unknown[]) => unknown)(file, ...rest) + }) as never) + try { + expect(() => mountPointManifestDirectories()).toThrow('EIO') + const refused = await wrapThatMayWrite(LOCAL).then( + () => undefined, + (error: unknown) => error, + ) + expect((refused as { code?: string }).code).toBe( + 'mount_points_unreadable', + ) + } finally { + spy.mockRestore() + } + }) + + /** Has the file system the home is on not say what kind it is. */ + function homeSays(code: string): void { + statfs.mockRestore() + statfs = spyOn(fs, 'statfsSync').mockImplementation(((p: unknown) => { + if (p === HOME) throw Object.assign(new Error(code), { code }) + return { type: EXT4 } + }) as never) + forgetMountPointManifestDirectory() + } + + it.each(['ENOENT', 'EACCES'])( + 'takes a home that is not there, or not to be entered, for no network home, and asks again (%s)', + code => { + // No wrap could make a directory in it, so one that a sandboxed command + // made there later, home and all, would be believed. + homeSays(code) + manifestOfAnotherHost([LOCK]) + expect(named()).toEqual([]) + statfs.mockRestore() + statfs = networkIs(NFS) + expect(mountPointManifestDirectories()).toContain(SHARED) + expect(named()).toEqual([LOCK]) + }, + ) + + it('hands out no command, and removes nothing, while it cannot tell whether the home is one', async () => { + // The directory may be there, with what other hosts rely on in it, and the + // command would find it unbound. + leftover(LOCK) + aged(manifestOfAnotherHost([LOCK]), 11 * MINUTE_MS) + homeSays('EIO') + expect(named()).toBeUndefined() + expect(liveMountPoints()).toBeUndefined() + expect(collectMountPoints('all')).toEqual([]) + const refused = await wrapCommandWithSandboxLinux({ + command: 'true', + needsNetworkRestriction: false, + readConfig: { denyOnly: [] }, + writeConfig: { allowOnly: [LOCAL], denyWithinAllow: [] }, + }).then( + () => undefined, + (error: unknown) => error, + ) + expect((refused as { code?: string }).code).toBe('mount_points_unreadable') + statfs.mockRestore() + statfs = networkIs(NFS) + expect(collectMountPoints('all')).toEqual([LOCK]) + }) + + it('asks nothing of the home for a wrap that restricts no write', async () => { + homeSays('EIO') + expect( + await wrapCommandWithSandboxLinux({ + command: 'true', + needsNetworkRestriction: false, + readConfig: { denyOnly: [LOCAL] }, + }), + ).toContain(`--tmpfs ${LOCAL}`) + }) + + it('remembers that there is a network home, and never that there is none', () => { + // What shows until the share is mounted over it is on this host's own disk. + statfs.mockRestore() + statfs = networkIs(EXT4) + forgetMountPointManifestDirectory() + expect(mountPointManifestDirectories()).not.toContain(SHARED) + statfs.mockRestore() + statfs = networkIs(NFS) + expect(mountPointManifestDirectories()).toContain(SHARED) + // And a share does not turn into a disk of this host's. + statfs.mockRestore() + statfs = networkIs(EXT4) + expect(mountPointManifestDirectories()).toContain(SHARED) + }) + + it('asks a second time before it gives up on a home that gives an error of the moment', () => { + let failures = 0 + statfs.mockRestore() + statfs = spyOn(fs, 'statfsSync').mockImplementation(((p: unknown) => { + if (p === HOME && failures++ === 0) { + throw Object.assign(new Error('ESTALE'), { code: 'ESTALE' }) + } + return { type: `${String(p)}/`.startsWith(`${NETWORK}/`) ? NFS : EXT4 } + }) as never) + forgetMountPointManifestDirectory() + expect(mountPointManifestDirectories()).toContain(SHARED) + expect(failures).toBeGreaterThan(1) + }) + + it('asks the user database again after a minute when it knew no such user: that is all an outage has it say', () => { + const passwd = join(HERE, 'etc/passwd') + const known = readFileSync(passwd, 'utf8') + writeFileSync(passwd, '') + forgetMountPointManifestDirectory() + expect(mountPointManifestDirectories()).not.toContain(SHARED) + writeFileSync(passwd, known) + expect(mountPointManifestDirectories()).not.toContain(SHARED) + const now = performance.now.bind(performance) + const later = spyOn(performance, 'now').mockImplementation( + () => now() + MINUTE_MS + 1000, + ) + try { + expect(mountPointManifestDirectories()).toContain(SHARED) + } finally { + later.mockRestore() + } + }) + + it('does not take a user database that cannot be read for one that knows no such user', () => { + const read = fs.readFileSync + const spy = spyOn(fs, 'readFileSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (file === join(HERE, 'etc/passwd')) { + throw Object.assign(new Error('EMFILE'), { code: 'EMFILE' }) + } + return (read as (...args: unknown[]) => unknown)(file, ...rest) + }) as never) + try { + forgetMountPointManifestDirectory() + expect(() => mountPointManifestDirectories()).toThrow('EMFILE') + expect(named()).toBeUndefined() + expect(collectMountPoints('all')).toEqual([]) + } finally { + spy.mockRestore() + } + }) + + /** Puts a `getent` where the library looks for it, and forgets the user. */ + function getentIs(script: string): void { + mkdirSync(join(HERE, 'usr/bin'), { recursive: true }) + writeFileSync(join(HERE, 'usr/bin/getent'), `#!/bin/sh\n${script}\n`, { + mode: 0o755, + }) + writeFileSync(join(HERE, 'etc/passwd'), '') + forgetMountPointManifestDirectory() + } + + it('asks getent for a user that /etc/passwd does not have', () => { + const uid = process.getuid?.() ?? 0 + getentIs( + `[ "$*" = "passwd ${uid}" ] && echo "user:x:${uid}:${uid}::${HOME}:/bin/sh"`, + ) + expect(mountPointManifestDirectories()).toContain(SHARED) + }) + + it('takes its word that there is no such user', () => { + getentIs('exit 2') + expect(mountPointManifestDirectories()).not.toContain(SHARED) + }) + + it('does not ask it again at once when it gave no answer: each asking holds up the thread', () => { + const asked = join(BASE, 'asked') + getentIs(`echo x >> ${asked}; exit 1`) + const times = (): number => readFileSync(asked, 'utf8').length / 2 + expect(() => mountPointManifestDirectories()).toThrow('gave no answer') + expect(() => mountPointManifestDirectories()).toThrow('gave no answer') + expect(named()).toBeUndefined() + expect(times()).toBe(1) + const now = performance.now.bind(performance) + const later = spyOn(performance, 'now').mockImplementation( + () => now() + 11_000, + ) + try { + expect(() => mountPointManifestDirectories()).toThrow('gave no answer') + expect(times()).toBe(2) + } finally { + later.mockRestore() + } + }) + + it('goes by the home the user database gives, whatever the environment says', () => { + const saved = process.env['HOME'] + process.env['HOME'] = join(NETWORK, 'elsewhere') + mkdirSync(process.env['HOME']) + try { + forgetMountPointManifestDirectory() + expect(publish([LOCK]).map(dirname)).toContain(SHARED) + } finally { + if (saved === undefined) delete process.env['HOME'] + else process.env['HOME'] = saved + } + }) + + it('keeps none there for a user the database does not know', () => { + writeFileSync( + join(HERE, 'etc/passwd'), + 'root:x:4294967294:0::/root:/bin/sh\n', + ) + forgetMountPointManifestDirectory() + expect(publish([LOCK]).map(dirname)).toEqual([PLACES[0]!]) + }) + + it.if(NOT_ROOT)( + "is no place for what other hosts do not reach, and stands in for none of this host's", + () => { + // A read-only image: none of the three can be written, the home can. + const runtime = join(BASE, 'runtime') + mkdirSync(runtime, { mode: 0o700 }) + const saved = process.env['XDG_RUNTIME_DIR'] + process.env['XDG_RUNTIME_DIR'] = runtime + for (const place of PLACES) { + rmSync(place, { recursive: true, force: true }) + chmodSync(dirname(place), 0o500) + } + try { + forgetMountPointManifestDirectory() + mountPointManifestDirectories() + const local = join(LOCAL, 'config.lock') + expect(publish([local]).map(dirname)).toEqual([ + join(runtime, 'srt-mount-points'), + ]) + // And what the environment names is still believed. + expect(named()).toEqual([local]) + } finally { + for (const [i, place] of PLACES.entries()) { + chmodSync(dirname(place), i === 1 ? 0o700 : 0o1777) + } + collectMountPoints('all') + if (saved === undefined) delete process.env['XDG_RUNTIME_DIR'] + else process.env['XDG_RUNTIME_DIR'] = saved + } + }, + ) + + it('has every wrap that restricts writes make the directory, bind it read-only and pin what lies above it', async () => { + expect(mountPointManifestDirectories()).toContain(SHARED) + const command = await wrapCommandWithSandboxLinux({ + command: 'true', + needsNetworkRestriction: false, + readConfig: { denyOnly: [] }, + writeConfig: { allowOnly: [NETWORK], denyWithinAllow: [] }, + }) + expect(command).toContain(`--ro-bind-try ${SHARED} ${SHARED}`) + expect(command).toContain(`--ro-bind ${HOME} ${HOME}`) + }) + + it('leaves the directory hidden where a read deny covers the home', async () => { + const command = await wrapCommandWithSandboxLinux({ + command: 'true', + needsNetworkRestriction: false, + readConfig: { denyOnly: [HOME] }, + writeConfig: { allowOnly: [PROJECT], denyWithinAllow: [LOCK] }, + }) + expect(manifestsOf(command).map(dirname)).toContain(SHARED) + expect(command).toContain(`--tmpfs ${HOME}`) + expect(command).not.toContain(` ${SHARED} ${SHARED}`) + }) + + /** The manifests of a wrapped command, by the records its shell is given. */ + const manifestsOf = (command: string): string[] => + [...command.matchAll(/ (\S+)\.started(?= )/g)].map(([, id]) => `${id}.json`) + + const wrapBoth = (command: string): Promise => + wrapCommandWithSandboxLinux({ + command, + needsNetworkRestriction: false, + readConfig: { denyOnly: [] }, + writeConfig: { + allowOnly: [PROJECT, LOCAL], + denyWithinAllow: [LOCK, join(LOCAL, 'config.lock')], + }, + }) + + const run = (command: string): { status: number | null; said: string } => { + const r = spawnSync(command, { shell: true, encoding: 'utf8' }) + return { status: r.status, said: `${r.stdout}${r.stderr}` } + } + + it('has a command that names both kinds bind both manifests before anything else, and put itself on both records', async () => { + const command = await wrapBoth('true') + const [theirsToo, here, ...more] = manifestsOf(command) + expect(more).toEqual([]) + expect([theirsToo, here].map(file => dirname(file!))).toEqual([ + SHARED, + PLACES[0]!, + ]) + expect(command).toContain( + ` --ro-bind / / --ro-bind ${theirsToo} ${theirsToo} --ro-bind ${here} ${here} `, + ) + for (const dir of [SHARED, PLACES[0]!]) { + expect(command).toContain(` --ro-bind ${dir} ${dir}`) + } + }) + + it.if(BWRAP_CAN_NAMESPACE)( + 'runs it with both denied, leaves a line on each record, and removes both mount points afterwards', + async () => { + const local = join(LOCAL, 'config.lock') + const command = await wrapBoth( + `echo ${BOOTED}; echo x > ${LOCK}; echo a=$?; echo x > ${local}; echo b=$?`, + ) + const { said } = run(command) + expect(said).toContain(BOOTED) + expect(said).toMatch(/a=[1-9]/) + expect(said).toMatch(/b=[1-9]/) + for (const manifest of manifestsOf(command)) { + expect( + readFileSync(manifest.replace(/json$/, 'started'), 'utf8'), + ).toMatch(/^\d+ \(.*\n$/) + } + expect([LOCK, local].map(p => existsSync(p))).toEqual([true, true]) + cleanupBwrapMountPoints() + expect([LOCK, local].map(p => existsSync(p))).toEqual([false, false]) + expect(readdirSync(SHARED)).toEqual([]) + expect(readdirSync(PLACES[0]!)).toEqual([]) + }, + ) + + it.if(BWRAP_CAN_NAMESPACE).each([0, 1])( + 'refuses it its start once either manifest is claimed (%d)', + async which => { + const command = await wrapBoth(`echo ${BOOTED}`) + const manifest = manifestsOf(command)[which]! + fs.renameSync(manifest, manifest.replace(/json$/, 'claimed')) + const { status, said } = run(command) + expect(status).not.toBe(0) + expect(said).not.toContain(BOOTED) + expect(existsSync(LOCK)).toBe(false) + expect(existsSync(join(LOCAL, 'config.lock'))).toBe(false) + }, + ) + + // ---- what another host's manifest says --------------------------------------- + + it("takes an empty file that another host's manifest names for a mount point, covers it and names it", async () => { + leftover(LOCK) + manifestOfAnotherHost([LOCK]) + expect(named()).toEqual([LOCK]) + const command = await wrapCommandWithSandboxLinux({ + command: 'true', + needsNetworkRestriction: false, + readConfig: { denyOnly: [] }, + writeConfig: { allowOnly: [PROJECT], denyWithinAllow: [LOCK] }, + }) + expect(command).toContain(`--ro-bind /dev/null ${LOCK}`) + const own = manifestOf(command)! + expect(dirname(own)).toBe(SHARED) + expect(readFileSync(own, 'utf8')).toContain(LOCK) + }) + + it('keeps what it names, though the pid in it is nobody here, or somebody else', () => { + leftover(LOCK) + const theirs = manifestOfAnotherHost([LOCK]) + expect(live()).toEqual([LOCK]) + expect(collectMountPoints('all')).toEqual([]) + expect(existsSync(LOCK)).toBe(true) + expect(existsSync(theirs)).toBe(true) + }) + + it('takes it for left behind once neither it nor its record has been touched for ten minutes', () => { + leftover(LOCK) + const theirs = manifestOfAnotherHost([LOCK]) + const record = theirs.replace(/\.json$/, '.started') + writeFileSync(record, `${process.pid} (bwrap) S\n`, { mode: 0o600 }) + + setBack(theirs, 9 * MINUTE_MS) + setBack(record, 60 * MINUTE_MS) + expect(collectMountPoints('all')).toEqual([]) + setBack(theirs, 60 * MINUTE_MS) + setBack(record, 9 * MINUTE_MS) + expect(collectMountPoints('all')).toEqual([]) + + setBack(record, 11 * MINUTE_MS) + expect(live()).toEqual([]) + expect(collectMountPoints('all')).toEqual([LOCK]) + expect(readdirSync(SHARED)).toEqual([]) + }) + it('goes by it only for what both hosts reach: the same name on this host is another file', () => { + const mine = join(LOCAL, 'config.lock') + const source = join(LOCAL, 'claude-empty-abc') + leftover(mine) + leftover(LOCK) + mkdirSync(source, { mode: 0o700 }) + manifestOfAnotherHost([mine, LOCK], [source]) + expect(named()).toEqual([LOCK]) + expect(live()).toEqual([LOCK]) + }) + + for (const [what, elsewhere] of [ + ['is another file on this host', (): string => join(LOCAL, 'config.lock')], + [ + 'lies in a directory this host does not have', + (): string => join(BASE, 'scratch/project/config.lock'), + ], + ] as const) { + it(`leaves one that has grown old to a host that reaches all it names, when some of it ${what}`, () => { + // Collected from here, the rest would have nothing left to name it. + leftover(LOCK) + const theirs = manifestOfAnotherHost([elsewhere(), LOCK]) + aged(theirs, 11 * MINUTE_MS) + const open = spyOn(fs, 'openSync') + try { + expect(collectMountPoints('all')).toEqual([]) + // Nor is it worth a record, which wraps on every host would wait for. + expect( + open.mock.calls.filter(([file]) => + /\.pass(\.tmp)?$/.test(String(file)), + ), + ).toEqual([]) + } finally { + open.mockRestore() + } + expect(existsSync(LOCK)).toBe(true) + expect(readdirSync(SHARED)).toEqual([basename(theirs)]) + expect(live()).toEqual([]) + + // Nothing is kept for it: it is old. + publish([LOCK]) + expect(collectMountPoints('all')).toEqual([LOCK]) + expect(readdirSync(SHARED)).toEqual([basename(theirs)]) + + // No such host has come for seven days. + leftover(LOCK) + setBack(theirs, 7 * 24 * 60 * MINUTE_MS + 1000) + expect(collectMountPoints('all')).toEqual([LOCK]) + expect(readdirSync(SHARED)).toEqual([]) + }) + } + + it.each([ + ['names nothing', (): string[] => [], 11 * MINUTE_MS], + [ + 'names nothing that counts here, once no host has come for it', + (): string[] => [join(LOCAL, 'config.lock')], + 7 * 24 * 60 * MINUTE_MS + 1000, + ], + ] as const)( + 'is rid of one there that %s, and does not claim it and give it back for ever', + (_, paths, age) => { + aged(manifestOfAnotherHost(paths()), age) + expect(collectMountPoints('all')).toEqual([]) + expect(readdirSync(SHARED)).toEqual([]) + }, + ) + + it('counts what it names by what this host mounts now, not when it first read it', () => { + // Read while the share the project is on was not mounted yet. + statfs.mockRestore() + statfs = spyOn(fs, 'statfsSync').mockImplementation(((p: unknown) => ({ + type: `${String(p)}/`.startsWith(`${HOME}/`) ? NFS : EXT4, + })) as never) + forgetMountPointManifestDirectory() + leftover(LOCK) + manifestOfAnotherHost([LOCK]) + expect(named()).toEqual([]) + statfs.mockRestore() + statfs = networkIs(NFS) + expect(named()).toEqual([LOCK]) + expect(live()).toEqual([LOCK]) + }) + + it('leaves one in any other directory that both hosts read to its own host as well, and goes by all it names till then', () => { + // A /var/tmp that diskless nodes share. The manifest there names all its + // wrap relies on, what is on its own host's disk included. + statfs.mockRestore() + statfs = spyOn(fs, 'statfsSync').mockImplementation(((p: unknown) => ({ + type: [NETWORK, PLACES[0]!].some(far => + `${String(p)}/`.startsWith(`${far}/`), + ) + ? NFS + : EXT4, + })) as never) + forgetMountPointManifestDirectory() + mountPointManifestDirectories() + const mine = join(LOCAL, 'config.lock') + leftover(mine) + leftover(LOCK) + const written = manifestOfAnotherHost([mine, LOCK]) + const theirs = join(PLACES[0]!, basename(written)) + fs.renameSync(written, theirs) + expect(named()).toEqual([mine, LOCK].sort()) + aged(theirs, 11 * MINUTE_MS) + expect(collectMountPoints('all')).toEqual([]) + expect(readdirSync(PLACES[0]!)).toEqual([basename(theirs)]) + setBack(theirs, 7 * 24 * 60 * MINUTE_MS + 1000) + expect(collectMountPoints('all').sort()).toEqual([mine, LOCK].sort()) + expect(readdirSync(PLACES[0]!)).toEqual([]) + }) + + it('knows a manifest there for its own where it cannot read which boot this is', () => { + // By its age, which this process itself keeps low, it would stay for as long + // as the process runs. + const read = fs.readFileSync + const spy = spyOn(fs, 'readFileSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (file === '/proc/sys/kernel/random/boot_id') { + throw Object.assign(new Error('EACCES'), { code: 'EACCES' }) + } + return (read as (...args: unknown[]) => unknown)(file, ...rest) + }) as never) + try { + forgetMountPointManifestDirectory() + leftover(LOCK) + expect(publish([LOCK]).map(dirname)).toContain(SHARED) + expect(live()).toEqual([LOCK]) + expect(collectMountPoints('all')).toEqual([LOCK]) + expect(readdirSync(SHARED)).toEqual([]) + } finally { + spy.mockRestore() + forgetMountPointManifestDirectory() + } + }) + + it('takes what lies in a directory that has gone from a share for reached all the same', () => { + // A mount point nested in another, which went first: nothing is left of it + // to remove, and nothing for another host to come for. + aged( + manifestOfAnotherHost([join(PROJECT, 'deep/er/config.lock')]), + 11 * MINUTE_MS, + ) + expect(collectMountPoints('all')).toEqual([]) + expect(readdirSync(SHARED)).toEqual([]) + }) + + it('goes by the age of one of this boot and another PID namespace as well: a container beside this one', () => { + leftover(LOCK) + const theirs = manifestOfAnotherHost([LOCK]) + writeFileSync( + theirs, + readFileSync(theirs, 'utf8') + .replace(ANOTHER_BOOT, OWN_BOOT) + .replace(OWN_NAMESPACE, 'pid:[1]'), + ) + setBack(theirs, 9 * MINUTE_MS) + expect(collectMountPoints('all')).toEqual([]) + setBack(theirs, 11 * MINUTE_MS) + expect(collectMountPoints('all')).toEqual([LOCK]) + }) + + it('goes by seven days for one whose writer does not say that it keeps it young', () => { + // To that writer the directory may be on a file system of its own. + leftover(LOCK) + const theirs = manifestOfAnotherHost([LOCK]) + writeFileSync( + theirs, + readFileSync(theirs, 'utf8').replace('"keptYoung":true,', ''), + ) + aged(theirs, 6 * 24 * 60 * MINUTE_MS) + expect(collectMountPoints('all')).toEqual([]) + setBack(theirs, 7 * 24 * 60 * MINUTE_MS + 1000) + expect(collectMountPoints('all')).toEqual([LOCK]) + }) + + it('goes by ten minutes for one that says so wherever it finds it: to this reader the directory may be on a disk of its own', () => { + const mine = join(LOCAL, 'config.lock') + leftover(mine) + mkdirSync(PLACES[0]!, { recursive: true, mode: 0o700 }) + const theirs = join(PLACES[0]!, basename(manifestOfAnotherHost([mine]))) + fs.renameSync(join(SHARED, basename(theirs)), theirs) + writeFileSync( + theirs, + readFileSync(theirs, 'utf8') + .replace(ANOTHER_BOOT, OWN_BOOT) + .replace(OWN_NAMESPACE, 'pid:[1]'), + ) + aged(theirs, 9 * MINUTE_MS) + expect(collectMountPoints('all')).toEqual([]) + aged(theirs, 11 * MINUTE_MS) + expect(collectMountPoints('all')).toEqual([mine]) + }) + + it('says in a manifest that other kernels read that it keeps it young, and in no other', () => { + const [theirsToo, here] = publish([LOCK, join(LOCAL, 'config.lock')]) + expect(JSON.parse(readFileSync(theirsToo!, 'utf8'))).toMatchObject({ + keptYoung: true, + }) + expect(JSON.parse(readFileSync(here!, 'utf8'))).not.toHaveProperty( + 'keptYoung', + ) + }) + + it('goes by it so too when it arrives while a pass is under way', () => { + // Kept for another host's file of the same name, this host's own leftover + // would be dropped from its manifest and stay for good. + const mine = join(LOCAL, 'config.lock') + leftover(mine) + mkdirSync(SHARED, { recursive: true, mode: 0o700 }) + mkdirSync(PLACES[0]!, { recursive: true, mode: 0o700 }) + const left = join(PLACES[0]!, '4194000-0123456789abcdef.json') + writeFileSync( + left, + JSON.stringify({ + version: 1, + pid: 4194000, + start: '1', + ns: OWN_NAMESPACE, + boot: OWN_BOOT, + created: 1, + paths: [mine], + sources: [], + }), + { mode: 0o600 }, + ) + const open = fs.openSync + let arrived = false + // The reading a pass decides on has listed every directory by the time it + // opens what it has found finished. + const spy = spyOn(fs, 'openSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (String(file).endsWith('.finished') && !arrived) { + arrived = true + manifestOfAnotherHost([mine]) + } + return (open as (...args: unknown[]) => number)(file, ...rest) + }) as never) + try { + expect(collectMountPoints('all')).toEqual([mine]) + } finally { + spy.mockRestore() + } + expect(arrived).toBe(true) + }) + + it('asks after a process as ever when the manifest there is of this boot', () => { + leftover(LOCK) + const theirs = manifestOfAnotherHost([LOCK]) + writeFileSync( + theirs, + readFileSync(theirs, 'utf8') + .replace(ANOTHER_BOOT, OWN_BOOT) + .replace(/"created":\d+/, '"created":1'), + ) + // Its writer has this process's pid and another start time: gone. + expect(collectMountPoints('all')).toEqual([LOCK]) + }) + + it('keeps nothing for one that its own host has found finished, and leaves that one to its host', () => { + const other = join(PROJECT, 'other.lock') + leftover(LOCK) + leftover(other) + const theirs = manifestOfAnotherHost([LOCK, other]) + const finished = theirs.replace(/json$/, 'finished') + fs.renameSync(theirs, finished) + publish([LOCK]) + expect(named()).toEqual([LOCK, other].sort()) + expect(live()).toEqual([LOCK]) + expect(collectMountPoints('all')).toEqual([LOCK]) + // That host is at it. + expect(existsSync(other)).toBe(true) + expect(readdirSync(SHARED)).toEqual([basename(finished)]) + setBack(finished, 11 * MINUTE_MS) + expect(collectMountPoints('all')).toEqual([other]) + expect(readdirSync(SHARED)).toEqual([]) + }) + + it('keeps what one names that its own host has only claimed: a sandbox may have got past the claim', () => { + leftover(LOCK) + const theirs = manifestOfAnotherHost([LOCK]) + const claim = theirs.replace(/json$/, 'claimed') + fs.renameSync(theirs, claim) + publish([LOCK]) + expect(collectMountPoints('all')).toEqual([]) + expect(existsSync(LOCK)).toBe(true) + expect(readdirSync(SHARED)).toEqual([basename(claim)]) + }) + + // ---- round trips ----------------------------------------------------------------- + + /** Which files under SHARED are opened while `what` runs. */ + function opened(what: () => void): string[] { + const open = spyOn(fs, 'openSync') + try { + what() + return open.mock.calls + .map(([file]) => String(file)) + .filter(file => file === SHARED || dirname(file) === SHARED) + } finally { + open.mockRestore() + } + } + + it('reads a manifest there once, and follows it by its name from then on', () => { + leftover(LOCK) + const theirs = manifestOfAnotherHost([LOCK]) + expect(opened(() => expect(named()).toEqual([LOCK]))).toEqual([theirs]) + expect(opened(() => expect(named()).toEqual([LOCK]))).toEqual([]) + expect(opened(() => expect(live()).toEqual([LOCK]))).toEqual([]) + const finished = theirs.replace(/json$/, 'finished') + fs.renameSync(theirs, finished) + expect(opened(() => expect(live()).toEqual([]))).toEqual([]) + expect(opened(() => expect(named()).toEqual([LOCK]))).toEqual([]) + // Forgotten with its name: what comes to that name later is read. + rmSync(finished) + expect(named()).toEqual([]) + writeFileSync(theirs, 'not a manifest', { mode: 0o600 }) + expect(named()).toBeUndefined() + }) + + it('asks once what file system a directory that other hosts reach is on, and every time of one of its own', () => { + const asked = (): string[] => + (statfs as unknown as { mock: { calls: unknown[][] } }).mock.calls.map( + ([dir]) => String(dir), + ) + mountPointManifestDirectories() + forgetMountPointManifestDirectory() + ;(statfs as unknown as { mockClear(): void }).mockClear() + publish([LOCK, join(LOCAL, 'config.lock')]) + named() + named() + expect(asked().filter(dir => dir === PROJECT)).toHaveLength(1) + expect(asked().filter(dir => dir === SHARED)).toHaveLength(1) + expect(asked().filter(dir => dir === PLACES[0]).length).toBeGreaterThan(1) + }) + + it('asks again after a minute: a share can be unmounted, and what is named there is then not for this host to collect', () => { + leftover(LOCK) + const theirs = aged(manifestOfAnotherHost([LOCK]), 11 * MINUTE_MS) + expect(named()).toEqual([LOCK]) + // What shows in its place is this host's own disk. + statfs.mockRestore() + statfs = spyOn(fs, 'statfsSync').mockImplementation(((p: unknown) => ({ + type: `${String(p)}/`.startsWith(`${HOME}/`) ? NFS : EXT4, + })) as never) + const now = performance.now.bind(performance) + const later = spyOn(performance, 'now').mockImplementation( + () => now() + MINUTE_MS + 1000, + ) + try { + expect(named()).toEqual([]) + expect(collectMountPoints('all')).toEqual([]) + } finally { + later.mockRestore() + } + expect(existsSync(LOCK)).toBe(true) + expect(readdirSync(SHARED)).toEqual([basename(theirs)]) + }) + + it('asks once in a reading what file system a directory of its own is on, however many paths in it are named', () => { + manifestOfAnotherHost( + Array.from({ length: 20 }, (_, i) => join(LOCAL, `config${i}.lock`)), + ) + named() + const asked = statfs as unknown as { + mockClear(): void + mock: { calls: unknown[][] } + } + asked.mockClear() + expect(named()).toEqual([]) + expect(asked.mock.calls.filter(([dir]) => dir === LOCAL).length).toBe(1) + }) + + it('reads again when one that it remembers is listed at two of its names: it was renamed meanwhile', () => { + // Taken at both, the name it has left would keep what it names and the one + // it has come to would be dropped for that, with nothing removed. + leftover(LOCK) + const [mine] = publish([LOCK]) + expect(named()).toEqual([LOCK]) + const readdir = fs.readdirSync + let listings = 0 + const spy = spyOn(fs, 'readdirSync').mockImplementation((( + dir: unknown, + options: unknown, + ) => { + const names = (readdir as (d: unknown, o: unknown) => string[])( + dir, + options, + ) + return dir === SHARED && listings++ === 0 + ? [...names, basename(mine!).replace(/json$/, 'finished')] + : names + }) as never) + try { + expect(named()).toEqual([LOCK]) + expect(listings).toBeGreaterThan(1) + } finally { + spy.mockRestore() + } + }) + + it('looks at a manifest of another host again only when it could have grown old', () => { + // Every pass judges every manifest of every host, and a look is a round + // trip. Its time is only ever set forward. + leftover(LOCK) + const theirs = aged(manifestOfAnotherHost([LOCK]), 2 * MINUTE_MS) + const looks = (what: () => void): number => { + const lstat = spyOn(fs, 'lstatSync') + try { + what() + return lstat.mock.calls.filter(([file]) => file === theirs).length + } finally { + lstat.mockRestore() + } + } + expect(looks(() => expect(live()).toEqual([LOCK]))).toBe(1) + expect(looks(() => expect(live()).toEqual([LOCK]))).toBe(0) + expect(looks(() => collectMountPoints('all'))).toBe(0) + const now = performance.now.bind(performance) + const date = Date.now.bind(Date) + const later = [ + spyOn(performance, 'now').mockImplementation( + () => now() + 8 * MINUTE_MS + 1000, + ), + spyOn(Date, 'now').mockImplementation( + () => date() + 8 * MINUTE_MS + 1000, + ), + ] + try { + expect(looks(() => expect(live()).toEqual([]))).toBe(1) + } finally { + later.forEach(spy => spy.mockRestore()) + } + }) + + it('reads one that is at two of its names for good at both in the end, and is rid of it', () => { + // A copy, a restored snapshot. In doubt for ever, it would refuse every wrap + // on every host. + leftover(LOCK) + const theirs = aged(manifestOfAnotherHost([LOCK]), 11 * MINUTE_MS) + const claim = theirs.replace(/json$/, 'claimed') + fs.copyFileSync(theirs, claim) + aged(claim, 11 * MINUTE_MS) + expect(named()).toEqual([LOCK]) + expect(collectMountPoints('all')).toEqual([LOCK]) + expect(readdirSync(SHARED)).toEqual([]) + }) + + it("reads one in this host's own places every time", () => { + const [mine] = publish([join(LOCAL, 'config.lock')]) + const open = spyOn(fs, 'openSync') + try { + named() + named() + expect(open.mock.calls.filter(([file]) => file === mine).length).toBe(2) + } finally { + open.mockRestore() + } + }) + + it.each(['openSync', 'fstatSync', 'readFileSync'] as const)( + 'takes a manifest that another host removes while it is read for gone, not for unreadable (%s)', + call => { + leftover(LOCK) + const theirs = manifestOfAnotherHost([LOCK]) + const open = fs.openSync as (...args: unknown[]) => number + const real = fs[call] as (...args: unknown[]) => unknown + let fd: number | undefined + // What a network file system says of a file the server no longer has. + const stale = (): never => { + rmSync(theirs) + throw Object.assign(new Error('ESTALE'), { code: 'ESTALE' }) + } + const spies: { mockRestore(): void }[] = [ + spyOn(fs, 'openSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (file !== theirs) return open(file, ...rest) + return call === 'openSync' ? stale() : (fd = open(file, ...rest)) + }) as never), + ] + if (call !== 'openSync') { + spies.push( + spyOn(fs, call).mockImplementation((( + what: unknown, + ...rest: unknown[] + ) => (what === fd ? stale() : real(what, ...rest))) as never), + ) + } + try { + expect(named()).toEqual([]) + } finally { + spies.forEach(spy => spy.mockRestore()) + } + }, + ) + + it('takes a record that cannot be read for that reason to vouch, like any that cannot be read', () => { + // For a manifest "gone" only starts the reading over. For a record it would + // lift the deny from under a running sandbox. + leftover(LOCK) + const [mine] = publish([LOCK]) + const record = mine!.replace(/json$/, 'started') + writeFileSync(record, readFileSync('/proc/self/stat', 'utf8'), { + mode: 0o600, + }) + const open = fs.openSync + const spy = spyOn(fs, 'openSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (file === record) { + throw Object.assign(new Error('ESTALE'), { code: 'ESTALE' }) + } + return (open as (...args: unknown[]) => number)(file, ...rest) + }) as never) + try { + expect(collectMountPoints('all')).toEqual([]) + } finally { + spy.mockRestore() + } + expect(existsSync(LOCK)).toBe(true) + expect(existsSync(mine!)).toBe(true) + rmSync(record) + expect(collectMountPoints('all')).toEqual([LOCK]) + }) + + it('judges afresh when a manifest it remembers has gone from its name by the time it looks at it', () => { + // A third host is collecting what a dead one left. It has kept LOCK for this + // host's manifest, which was live then, and drops the one it claimed. Kept + // for that one in turn, LOCK would be left to nobody. + leftover(LOCK) + const left = manifestOfAnotherHost([LOCK]) + const theirs = left.replace(/json$/, 'claimed') + fs.renameSync(left, theirs) + aged(theirs, 11 * MINUTE_MS) + expect(named()).toEqual([LOCK]) + publish([LOCK]) + const lstat = fs.lstatSync + const spy = spyOn(fs, 'lstatSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + // On record by then: the look of the reading it decides on. + if ( + file === theirs && + readdirSync(SHARED).some(name => name.endsWith('.pass')) + ) { + rmSync(theirs, { force: true }) + } + return (lstat as (...args: unknown[]) => unknown)(file, ...rest) + }) as never) + try { + expect(collectMountPoints('all')).toEqual([LOCK]) + } finally { + spy.mockRestore() + } + expect(readdirSync(SHARED)).toEqual([]) + }) + + it.each([ + ['in the home', (): string => SHARED], + ["in this host's own place, of another PID namespace", () => PLACES[0]!], + ] as const)( + 'reads again, when asked what is live, if one %s has gone from its name by the time it looks at it', + (_, dir) => { + // Its host has claimed it, and will give it back: a sandbox is on its + // record. In doubt, whoever asked would have to take every path for live. + leftover(LOCK) + const written = manifestOfAnotherHost([LOCK]) + const theirs = join(dir(), basename(written)) + mkdirSync(dir(), { recursive: true, mode: 0o700 }) + writeFileSync( + theirs, + readFileSync(written, 'utf8') + .replace(ANOTHER_BOOT, OWN_BOOT) + .replace(OWN_NAMESPACE, 'pid:[1]'), + { mode: 0o600 }, + ) + if (theirs !== written) rmSync(written) + let read = false + let moved = false + const open = fs.openSync + const lstat = fs.lstatSync + const spies = [ + spyOn(fs, 'openSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + read ||= file === theirs + return (open as (...args: unknown[]) => number)(file, ...rest) + }) as never), + spyOn(fs, 'lstatSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (file === theirs && read && !moved) { + moved = true + fs.renameSync(theirs, theirs.replace(/json$/, 'claimed')) + } + return (lstat as (...args: unknown[]) => unknown)(file, ...rest) + }) as never), + ] + try { + expect(live()).toEqual([LOCK]) + } finally { + spies.forEach(spy => spy.mockRestore()) + } + expect(moved).toBe(true) + }, + ) + + it('asks no more of the directory there, once it is as it should be, than whether it is', () => { + mountPointManifestDirectories() + const mkdir = spyOn(fs, 'mkdirSync') + try { + expect(opened(() => mountPointManifestDirectories())).toEqual([]) + expect(mkdir.mock.calls.map(([dir]) => dir)).not.toContain(SHARED) + } finally { + mkdir.mockRestore() + } + // One that is not is still put right. + chmodSync(SHARED, 0o755) + mountPointManifestDirectories() + expect(lstatSync(SHARED).mode & 0o777).toBe(0o700) + }) + + // ---- kept young ---------------------------------------------------------------- + + /** Captures the timer the library sets, to run it by hand. */ + function timers(): { + ticks: (() => void)[] + unrefs: number + cleared: number + restore(): void + } { + const state = { ticks: [] as (() => void)[], unrefs: 0, cleared: 0 } + const spies = [ + spyOn(globalThis, 'setInterval').mockImplementation((( + tick: () => void, + ) => { + state.ticks.push(tick) + return { unref: () => state.unrefs++ } + }) as never), + spyOn(globalThis, 'clearInterval').mockImplementation((() => { + state.cleared++ + }) as never), + ] + return Object.assign(state, { + restore: () => spies.forEach(spy => spy.mockRestore()), + }) + } + + it("has its time set by this host's clock before it is published, whatever the server's says", () => { + const close = fs.closeSync + const rename = fs.renameSync + let ageWhenPublished: number | undefined + const spies = [ + spyOn(fs, 'closeSync').mockImplementation(((fd: number) => { + close(fd) + // A server an hour behind. + for (const name of readdirSync(SHARED)) { + if (name.endsWith('.tmp')) aged(join(SHARED, name), 60 * MINUTE_MS) + } + }) as never), + spyOn(fs, 'renameSync').mockImplementation((( + from: string, + to: string, + ) => { + if (dirname(to) === SHARED) { + ageWhenPublished = Date.now() - lstatSync(from).mtimeMs + } + rename(from, to) + }) as never), + ] + try { + publish([LOCK]) + } finally { + spies.forEach(spy => spy.mockRestore()) + } + expect(ageWhenPublished).toBeLessThan(MINUTE_MS) + }) + + const isYoung = (file: string): boolean => + Date.now() - lstatSync(file).mtimeMs < MINUTE_MS + + it('has its time set again every minute for as long as its process knows of it, on a timer that keeps no process alive', async () => { + const timer = timers() + try { + const [manifest] = publish([LOCK]) + expect(timer.ticks).toHaveLength(1) + expect(timer.unrefs).toBe(1) + publish([join(PROJECT, 'second.lock')]) + expect(timer.ticks).toHaveLength(1) + + aged(manifest!, 5 * MINUTE_MS) + timer.ticks[0]!() + await until('its time to be set', () => isYoung(manifest!)) + expect(timer.cleared).toBe(0) + + collectMountPoints('all') + expect(readdirSync(SHARED)).toEqual([]) + timer.ticks[0]!() + expect(timer.cleared).toBe(1) + } finally { + timer.restore() + } + }) + + it('does not wait for the server on that timer: nobody has asked for anything when it runs', () => { + const timer = timers() + const waits = spyOn(fs, 'utimesSync') + try { + publish([LOCK]) + waits.mockClear() + timer.ticks[0]!() + expect(waits).not.toHaveBeenCalled() + } finally { + waits.mockRestore() + timer.restore() + } + }) + + it('goes on after a tick that failed: the next may do', async () => { + const timer = timers() + const utimes = fs.utimes + const spy = spyOn(fs, 'utimes').mockImplementation((( + ...args: unknown[] + ) => { + ;(args.at(-1) as (e: Error) => void)( + Object.assign(new Error('EIO'), { code: 'EIO' }), + ) + }) as never) + try { + const [manifest] = publish([LOCK]) + aged(manifest!, 5 * MINUTE_MS) + timer.ticks[0]!() + expect(timer.cleared).toBe(0) + spy.mockImplementation(utimes as never) + timer.ticks[0]!() + await until('its time to be set', () => isYoung(manifest!)) + } finally { + spy.mockRestore() + timer.restore() + } + }) + + it('has one request under way at a time, and starts no tick while the last is at it', () => { + // A server that does not answer then ties up one thread of the pool these + // requests run on, and not all of them. + const timer = timers() + const pending: (() => void)[] = [] + const spy = spyOn(fs, 'utimes').mockImplementation((( + ...args: unknown[] + ) => { + pending.push(() => (args.at(-1) as (e: null) => void)(null)) + }) as never) + try { + publish([LOCK]) + publish([join(PROJECT, 'second.lock')]) + timer.ticks[0]!() + expect(pending).toHaveLength(1) + timer.ticks[0]!() + expect(pending).toHaveLength(1) + pending.shift()!() + expect(pending).toHaveLength(1) + pending.shift()!() + expect(pending).toHaveLength(0) + timer.ticks[0]!() + expect(pending).toHaveLength(1) + } finally { + spy.mockRestore() + timer.restore() + } + }) + + it('keeps it young at the name of a claim too: a pass that finds a sandbox gives it back', async () => { + const timer = timers() + try { + const [manifest] = publish([LOCK]) + const claim = manifest!.replace(/json$/, 'claimed') + fs.renameSync(manifest!, claim) + aged(claim, 5 * MINUTE_MS) + timer.ticks[0]!() + await until('its time to be set', () => isYoung(claim)) + expect(timer.cleared).toBe(0) + } finally { + timer.restore() + } + }) + + it('sets no timer for a manifest that only this host reads', () => { + const timer = timers() + try { + publish([join(LOCAL, 'config.lock')]) + expect(timer.ticks).toEqual([]) + } finally { + timer.restore() + } + }) + + // ---- clean-ups on record --------------------------------------------------------- + + /** Puts a pass of another host on record. */ + function passOfAnotherHost(): string { + mkdirSync(SHARED, { recursive: true, mode: 0o700 }) + const file = join( + SHARED, + `${process.pid}-1-${/\d+/.exec(OWN_NAMESPACE)![0]}-${ANOTHER_BOOT.replace(/-/g, '')}-0123456789abcdef.pass`, + ) + writeFileSync(file, '', { mode: 0o600 }) + return file + } + + it("goes by the age of another host's, whose process cannot be asked after", () => { + const pass = passOfAnotherHost() + expect(passesUnderWay()).toEqual([pass]) + aged(pass, 40_000) + expect(passesUnderWay()).toEqual([pass]) + aged(pass, 80_000) + expect(passesUnderWay()).toEqual([]) + // And leaves its record to its own host. + collectMountPoints('all') + expect(existsSync(pass)).toBe(true) + }) + + it('is of no concern to a wrap that names nothing other hosts reach', async () => { + const pass = passOfAnotherHost() + expect(passesUnderWay(false)).toEqual([]) + const wrap = (deny: string, allow: string): Promise => + wrapCommandWithSandboxLinux({ + command: 'true', + needsNetworkRestriction: false, + readConfig: { denyOnly: [] }, + writeConfig: { allowOnly: [allow], denyWithinAllow: [deny] }, + }) + // Held up by it, it would be refused. + await wrap(join(LOCAL, 'config.lock'), LOCAL) + // As one that does is. + const refused = await wrap(LOCK, PROJECT).then( + () => undefined, + (error: unknown) => error, + ) + expect((refused as { code?: string }).code).toBe('mount_points_changed') + expect((refused as { cause?: unknown }).cause).toBe(pass) + }, 15000) + + it("puts its own on record in the home directory as well, with its time set by this host's clock", () => { + leftover(LOCK) + aged(manifestOfAnotherHost([LOCK]), 11 * MINUTE_MS) + const unlink = fs.unlinkSync + const utimes = spyOn(fs, 'utimesSync') + let onRecord: string[] = [] + const spy = spyOn(fs, 'unlinkSync').mockImplementation(((file: string) => { + if (file === LOCK) { + onRecord = [SHARED, ...PLACES].flatMap(dir => + existsSync(dir) + ? readdirSync(dir) + .filter(name => name.endsWith('.pass')) + .map(() => dir) + : [], + ) + } + unlink(file) + }) as never) + try { + expect(collectMountPoints('all')).toEqual([LOCK]) + expect(onRecord).toContain(SHARED) + expect( + utimes.mock.calls + .map(([file]) => String(file)) + .filter(file => file.endsWith('.pass.tmp')) + .map(dirname), + ).toEqual([SHARED]) + } finally { + spy.mockRestore() + utimes.mockRestore() + } + }) + + it("has it come to its name there with its time set already, whatever the server's clock says", () => { + // Listed with the time of a server two minutes behind, it would be taken + // for overdue by a wrap on another host, which would not wait for it. + leftover(LOCK) + aged(manifestOfAnotherHost([LOCK]), 11 * MINUTE_MS) + const close = fs.closeSync + let stamped = false + let agesAtItsName: number[] = [] + const look = (): void => { + agesAtItsName = [ + ...agesAtItsName, + ...readdirSync(SHARED) + .filter(name => name.endsWith('.pass')) + .map(name => Date.now() - lstatSync(join(SHARED, name)).mtimeMs), + ] + } + const rename = fs.renameSync + const spies = [ + spyOn(fs, 'closeSync').mockImplementation(((fd: number) => { + close(fd) + for (const name of readdirSync(SHARED)) { + if (/\.pass(\.tmp)?$/.test(name) && !stamped) { + stamped = true + aged(join(SHARED, name), 2 * MINUTE_MS) + } + } + look() + }) as never), + spyOn(fs, 'renameSync').mockImplementation((( + from: string, + to: string, + ) => { + rename(from, to) + look() + }) as never), + ] + try { + expect(collectMountPoints('all')).toEqual([LOCK]) + } finally { + spies.forEach(spy => spy.mockRestore()) + } + expect(agesAtItsName.length).toBeGreaterThan(0) + expect(Math.max(...agesAtItsName)).toBeLessThan(MINUTE_MS) + }) + + /** Has the clock go on by `ms` whenever `held` is called. */ + function aClockThatIsHeldUp(): { + heldUpFor(ms: number): void + mockRestore(): void + } { + const now = performance.now.bind(performance) + let lost = 0 + const spy = spyOn(performance, 'now').mockImplementation(() => now() + lost) + return { + heldUpFor: ms => { + lost += ms + }, + mockRestore: () => spy.mockRestore(), + } + } + + it('counts what the server takes to put its record at its name as time it has had: others go by the age of that', () => { + leftover(LOCK) + aged(manifestOfAnotherHost([LOCK]), 11 * MINUTE_MS) + const clock = aClockThatIsHeldUp() + const rename = fs.renameSync + const spy = spyOn(fs, 'renameSync').mockImplementation((( + from: string, + to: string, + ) => { + rename(from, to) + if (to.endsWith('.pass') && dirname(to) === SHARED) clock.heldUpFor(3000) + }) as never) + try { + expect(collectMountPoints('all')).toEqual([]) + } finally { + spy.mockRestore() + clock.mockRestore() + } + expect(existsSync(LOCK)).toBe(true) + expect(collectMountPoints('all')).toEqual([LOCK]) + }) + + it('looks at the clock after its last listing, not before: that waits for the server too', () => { + leftover(LOCK) + aged(manifestOfAnotherHost([LOCK]), 11 * MINUTE_MS) + const clock = aClockThatIsHeldUp() + const readdir = fs.readdirSync + let onRecord = 0 + const spy = spyOn(fs, 'readdirSync').mockImplementation((( + dir: unknown, + ...rest: unknown[] + ) => { + const names = (readdir as (...args: unknown[]) => string[])(dir, ...rest) + // The reading it decides on, and then the one before the removal. + if ( + dir === SHARED && + names.some(name => name.endsWith('.pass')) && + ++onRecord === 2 + ) { + clock.heldUpFor(3000) + } + return names + }) as never) + try { + expect(collectMountPoints('all')).toEqual([]) + } finally { + spy.mockRestore() + clock.mockRestore() + } + expect(onRecord).toBe(2) + expect(existsSync(LOCK)).toBe(true) + }) + + it('takes its fresh look directory by directory as it goes, and removes what it has the time for', () => { + // All of them first, a slow server would have every pass use up its time on + // that and none remove anything. + const second = join(NETWORK, 'second') + mkdirSync(second) + const locks = [LOCK, join(second, 'config.lock')] + locks.forEach(leftover) + aged(manifestOfAnotherHost(locks), 11 * MINUTE_MS) + const clock = aClockThatIsHeldUp() + const open = fs.openSync + const spy = spyOn(fs, 'openSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (file === PROJECT || file === second) clock.heldUpFor(1500) + return (open as (...args: unknown[]) => number)(file, ...rest) + }) as never) + try { + expect(collectMountPoints('all').length).toBe(1) + } finally { + spy.mockRestore() + clock.mockRestore() + } + expect(collectMountPoints('all').length).toBe(1) + expect(locks.map(lock => existsSync(lock))).toEqual([false, false]) + expect(readdirSync(SHARED)).toEqual([]) + }) + + it('has a wrap that waits for one of another host look a few times a second, not twenty', async () => { + const pass = passOfAnotherHost() + const readdir = spyOn(fs, 'readdirSync') + try { + expect(await passesToEnd([pass], undefined, 1000)).toBe(pass) + expect( + readdir.mock.calls.filter(([dir]) => dir === SHARED).length, + ).toBeLessThan(16) + } finally { + readdir.mockRestore() + } + }) + + it("lists this host's own places before every removal all the same, and the home by a time of its own", () => { + // A pass that no wrap can see has nothing else to keep what is published + // after its reading. + const locks = ['a', 'b', 'c'].map(name => join(PROJECT, `${name}.lock`)) + locks.forEach(leftover) + aged(manifestOfAnotherHost(locks), 11 * MINUTE_MS) + mountPointManifestDirectories(true) + const unlink = fs.unlinkSync + const readdir = spyOn(fs, 'readdirSync') + // No time passes, so that none of this depends on how fast it runs. + const at = performance.now() + const clock = spyOn(performance, 'now').mockImplementation(() => at) + const listedBefore: Record[] = [] + const spy = spyOn(fs, 'unlinkSync').mockImplementation(((file: string) => { + if (locks.includes(file)) { + const of = (dir: string): number => + readdir.mock.calls.filter(([listed]) => listed === dir).length + listedBefore.push({ own: of(PLACES[0]!), home: of(SHARED) }) + } + unlink(file) + }) as never) + try { + expect(collectMountPoints('all').sort()).toEqual(locks) + } finally { + spy.mockRestore() + readdir.mockRestore() + clock.mockRestore() + } + const [first, , last] = listedBefore + expect(last!['own']! - first!['own']!).toBe(2) + expect(last!['home']! - first!['home']!).toBe(0) + }) + + it('sets the directory in the home right before it reads, as a wrap does: opened up, it would be passed over', () => { + // `chmod -R g+rX ~`. What another host relies on would go unread. + leftover(LOCK) + manifestOfAnotherHost([LOCK]) + publish([LOCK]) + chmodSync(SHARED, 0o755) + expect(collectMountPoints('all')).toEqual([]) + expect(existsSync(LOCK)).toBe(true) + expect(lstatSync(SHARED).mode & 0o777).toBe(0o700) + }) + + it("still removes what is this host's alone where the home takes no record, and nothing that other hosts reach", () => { + // A home over its quota. Placeholders would else pile up in every project + // on every host that shares it. + const mine = join(LOCAL, 'config.lock') + leftover(mine) + leftover(LOCK) + publish([mine, LOCK]) + const open = fs.openSync + const spy = spyOn(fs, 'openSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if ( + /\.pass(\.tmp)?$/.test(String(file)) && + dirname(String(file)) === SHARED + ) { + throw Object.assign(new Error('EDQUOT'), { code: 'EDQUOT' }) + } + return (open as (...args: unknown[]) => number)(file, ...rest) + }) as never) + try { + expect(collectMountPoints('all')).toEqual([mine]) + } finally { + spy.mockRestore() + } + expect(existsSync(LOCK)).toBe(true) + expect(named()).toEqual([LOCK]) + expect(readdirSync(PLACES[0]!)).toEqual([]) + expect(collectMountPoints('all')).toEqual([LOCK]) + }) + + it("removes nothing that other hosts reach off the record in the home, though a manifest of this host's own names it", () => { + // Recorded there because the home took no manifest either. + leftover(LOCK) + const full = (file: unknown): void => { + if (dirname(String(file)) === SHARED) { + throw Object.assign(new Error('EDQUOT'), { code: 'EDQUOT' }) + } + } + const open = fs.openSync + const write = fs.writeFileSync + const spies = [ + spyOn(fs, 'openSync').mockImplementation((( + file: unknown, + flags: unknown, + ...rest: unknown[] + ) => { + if (flags === 'wx') full(file) + return (open as (...args: unknown[]) => number)(file, flags, ...rest) + }) as never), + spyOn(fs, 'writeFileSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + full(file) + ;(write as (...args: unknown[]) => void)(file, ...rest) + }) as never), + ] + try { + expect(publish([LOCK]).map(dirname)).toEqual([PLACES[0]!]) + expect(collectMountPoints('all')).toEqual([]) + expect(existsSync(LOCK)).toBe(true) + expect(named()).toEqual([LOCK]) + } finally { + spies.forEach(spy => spy.mockRestore()) + } + expect(collectMountPoints('all')).toEqual([LOCK]) + }) + + it('goes on no record in the home for what no other host reaches', () => { + // Every wrap on every host that names what they do reach would wait for it. + const mine = join(LOCAL, 'config.lock') + leftover(mine) + mountPointManifestDirectories() + publish([mine]) + expect( + opened(() => expect(collectMountPoints('all')).toEqual([mine])), + ).toEqual([]) + }) + + it('goes by the manifests it reads where the home cannot be written from here at all', () => { + // Mounted read-only on this host, or bound so by an outer sandbox. No pass + // of this host is ever on record there: waiting for that, nothing on a + // share would ever be removed. + const other = join(PROJECT, 'other.lock') + leftover(LOCK) + leftover(other) + manifestOfAnotherHost([other]) + const access = fs.accessSync + const spy = spyOn(fs, 'accessSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (file === SHARED) { + throw Object.assign(new Error('EROFS'), { code: 'EROFS' }) + } + ;(access as (...args: unknown[]) => void)(file, ...rest) + }) as never) + try { + expect(publish([LOCK, other]).map(dirname)).toEqual([PLACES[0]!]) + expect(collectMountPoints('all')).toEqual([LOCK]) + expect(existsSync(other)).toBe(true) + expect(readdirSync(PLACES[0]!)).toEqual([]) + } finally { + spy.mockRestore() + } + }) + + it("takes another host's that is stamped far ahead for overdue as well: its clock is wrong", () => { + const pass = passOfAnotherHost() + const ahead = new Date(Date.now() + 8 * 60 * MINUTE_MS) + utimesSync(pass, ahead, ahead) + expect(passesUnderWay()).toEqual([]) + }) + + // ---- what this host remembers of a directory --------------------------------------- + + it('opens the directories that hold what other hosts reach, each once, and no other', () => { + const open = spyOn(fs, 'openSync') + try { + lookAfreshAt([ + LOCK, + join(PROJECT, 'second.lock'), + join(HOME, '.bashrc'), + join(LOCAL, 'config.lock'), + join(NETWORK, 'not-there', 'x'), + ]) + expect(open.mock.calls.map(([dir]) => dir)).toEqual([ + PROJECT, + HOME, + join(NETWORK, 'not-there'), + ]) + for (const [, flags] of open.mock.calls) { + expect(Number(flags) & fs.constants.O_DIRECTORY).not.toBe(0) + } + } finally { + open.mockRestore() + } + }) + + it('opens none, and asks nothing, on a host with no network home', () => { + statfs.mockRestore() + statfs = networkIs(EXT4) + forgetMountPointManifestDirectory() + mountPointManifestDirectories() + ;(statfs as unknown as { mockClear(): void }).mockClear() + const open = spyOn(fs, 'openSync') + try { + lookAfreshAt([LOCK, join(HOME, '.bashrc')]) + expect(open).not.toHaveBeenCalled() + expect(statfs).not.toHaveBeenCalled() + } finally { + open.mockRestore() + } + }) + + it('has a wrap do so once its manifest is published, before it looks at the host again', async () => { + const open = fs.openSync + const seen: string[] = [] + const spy = spyOn(fs, 'openSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (file === PROJECT && Number(rest[0]) & fs.constants.O_DIRECTORY) { + seen.push( + readdirSync(SHARED).some(name => name.endsWith('.json')) + ? 'published' + : 'not published', + ) + } + return (open as (...args: unknown[]) => number)(file, ...rest) + }) as never) + try { + await wrapCommandWithSandboxLinux({ + command: 'true', + needsNetworkRestriction: false, + readConfig: { denyOnly: [] }, + writeConfig: { allowOnly: [PROJECT], denyWithinAllow: [LOCK] }, + }) + } finally { + spy.mockRestore() + } + expect(seen.at(-1)).toBe('published') + }) + + it("plans again when what it took for the caller's own turns out to be a mount point another host has removed", async () => { + // That host removed it and then dropped its manifest. This one still + // remembers the file, reads that nothing names it, and would bind it onto + // itself: a mount on what the server no longer has, gone when the client + // finds out. It finds out when the directory is opened, which stands in + // for that here. + leftover(LOCK) + const open = fs.openSync + const spy = spyOn(fs, 'openSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (file === PROJECT) rmSync(LOCK, { force: true }) + return (open as (...args: unknown[]) => number)(file, ...rest) + }) as never) + let command: string + try { + command = await wrapCommandWithSandboxLinux({ + command: 'true', + needsNetworkRestriction: false, + readConfig: { denyOnly: [] }, + writeConfig: { allowOnly: [PROJECT], denyWithinAllow: [LOCK] }, + }) + } finally { + spy.mockRestore() + } + expect(command).toContain(`--ro-bind /dev/null ${LOCK}`) + expect(command).not.toContain(`--ro-bind ${LOCK} ${LOCK}`) + expect(namedIn(manifestOf(command)!).paths).toEqual([LOCK]) + }) + + /** Not the plan's own open of a directory, by O_PATH, which asks no server. */ + const isToRead = (flags: unknown): boolean => + (Number(flags) & 0o10000000) === 0 + + it('plans again when what it found absent turns out to be a file the user has put there from another host', async () => { + // Named as this wrap's own, it would go with the next clean-up. This host + // finds out when the directory is opened, which stands in for that here. + let there = false + const open = fs.openSync + const spy = spyOn(fs, 'openSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (file === PROJECT && isToRead(rest[0]) && !there) { + there = true + writeFileSync(LOCK, '') + } + return (open as (...args: unknown[]) => number)(file, ...rest) + }) as never) + let command: string + try { + command = await wrapCommandWithSandboxLinux({ + command: 'true', + needsNetworkRestriction: false, + readConfig: { denyOnly: [] }, + writeConfig: { allowOnly: [PROJECT], denyWithinAllow: [LOCK] }, + }) + } finally { + spy.mockRestore() + } + expect(command).toContain(`--ro-bind ${LOCK} ${LOCK}`) + expect(manifestsOf(command)).toEqual([]) + cleanupBwrapMountPoints({ force: true }) + expect(existsSync(LOCK)).toBe(true) + }) + + it('does not when a manifest names what has come to be there: another sandbox has made it', async () => { + let there = false + const open = fs.openSync + const spy = spyOn(fs, 'openSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + if (file === PROJECT && isToRead(rest[0]) && !there) { + there = true + manifestOfAnotherHost([LOCK]) + leftover(LOCK) + } + return (open as (...args: unknown[]) => number)(file, ...rest) + }) as never) + let command: string + try { + command = await wrapCommandWithSandboxLinux({ + command: 'true', + needsNetworkRestriction: false, + readConfig: { denyOnly: [] }, + writeConfig: { allowOnly: [PROJECT], denyWithinAllow: [LOCK] }, + }) + } finally { + spy.mockRestore() + } + expect(command).toContain(`--ro-bind /dev/null ${LOCK}`) + expect(namedIn(manifestOf(command)!).paths).toEqual([LOCK]) + }) + + it("does not take what its own plan has made, on a 9p drive, for a file of the user's", async () => { + // There the plan makes the mount point itself, so it is there at that look. + statfs.mockRestore() + statfs = networkIs(0x01021997) + forgetMountPointManifestDirectory() + const deep = join(PROJECT, 'a/b/c') + const own = join(PROJECT, 'own.empty') + writeFileSync(own, '') + // The caller's empty file has the manifests read before anything is made. + const command = await wrapCommandWithSandboxLinux({ + command: 'true', + needsNetworkRestriction: false, + readConfig: { denyOnly: [] }, + writeConfig: { + allowOnly: [PROJECT], + denyWithinAllow: [own, LOCK, deep], + }, + }) + expect(command).toContain(`--ro-bind /dev/null ${LOCK}`) + expect(namedIn(manifestOf(command)!).paths.sort()).toEqual( + [LOCK, join(PROJECT, 'a')].sort(), + ) + cleanupBwrapMountPoints() + expect(readdirSync(PROJECT)).toEqual(['own.empty']) + }) + + it('has a clean-up do so before it removes anything: what this host remembers of a path may be a mount point that has gone', () => { + // Another host has removed it, and the user has put a file of theirs there. + leftover(LOCK) + aged(manifestOfAnotherHost([LOCK]), 11 * MINUTE_MS) + const remembered = lstatSync(LOCK) + rmSync(LOCK) + writeFileSync(LOCK, 'theirs') + let asked = false + const open = fs.openSync + const lstat = fs.lstatSync + const spies = [ + spyOn(fs, 'openSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + asked ||= file === PROJECT + return (open as (...args: unknown[]) => number)(file, ...rest) + }) as never), + spyOn(fs, 'lstatSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => + file === LOCK && !asked + ? remembered + : (lstat as (...args: unknown[]) => unknown)( + file, + ...rest, + )) as never), + ] + try { + expect(collectMountPoints('all')).toEqual([]) + } finally { + spies.forEach(spy => spy.mockRestore()) + } + expect(readFileSync(LOCK, 'utf8')).toBe('theirs') + expect(readdirSync(SHARED)).toEqual([]) + }) + + it('asks after the file itself as well: another host may have written into it, which leaves the directory as it was', () => { + // Old bubblewrap makes it 0644. What this host remembers of it is empty. + leftover(LOCK) + aged(manifestOfAnotherHost([LOCK]), 11 * MINUTE_MS) + const remembered = lstatSync(LOCK) + chmodSync(LOCK, 0o644) + writeFileSync(LOCK, 'theirs') + let asked = false + const open = fs.openSync + const lstat = fs.lstatSync + const spies = [ + spyOn(fs, 'openSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => { + asked ||= file === LOCK + return (open as (...args: unknown[]) => number)(file, ...rest) + }) as never), + spyOn(fs, 'lstatSync').mockImplementation((( + file: unknown, + ...rest: unknown[] + ) => + file === LOCK && !asked + ? remembered + : (lstat as (...args: unknown[]) => unknown)( + file, + ...rest, + )) as never), + ] + try { + expect(collectMountPoints('all')).toEqual([]) + } finally { + spies.forEach(spy => spy.mockRestore()) + } + expect(readFileSync(LOCK, 'utf8')).toBe('theirs') + }) + + it('counts the directory that mount points bind from among what other hosts reach', () => { + expect(otherHostsReach([join(LOCAL, 'x')])).toBe(false) + expect(otherHostsReach([join(LOCAL, 'x'), join(NETWORK, 'tmp/e')])).toBe( + true, + ) + }) + + // ---- two hosts, for real but for the network ------------------------------------------ + + /** The environment of a process on the host called `name`. */ + function onHost(name: string): Record { + const root = join(BASE, name) + if (!existsSync(root)) { + mkdirSync(root) + makePlaces(root, HOME) + } + return { + ...(process.env as Record), + SRT_TEST_HOST: root, + SRT_TEST_BOOT: `${name.padEnd(8, '0')}-0000-0000-0000-000000000000`, + SRT_TEST_NETWORK: NETWORK, + } + } + + const script = (name: string, lines: string[]): string => + writeScript(BASE, name, lines) + + /** Wraps `command` with LOCK denied, runs it, and then does `after`. */ + const wrapAndRun = (command: string, out: string, after: string[]) => [ + `import { spawn } from 'node:child_process'`, + `import { writeFileSync } from 'node:fs'`, + `const m = await import(${HOST})`, + `const wrapped = await m.wrapCommandWithSandboxLinux({`, + ` command: ${JSON.stringify(command)},`, + ` needsNetworkRestriction: false,`, + ` readConfig: { denyOnly: [] },`, + ` writeConfig: { allowOnly: [${JSON.stringify(PROJECT)}], denyWithinAllow: [${JSON.stringify(LOCK)}] },`, + `})`, + `const child = spawn(wrapped, { shell: true })`, + `let said = ''`, + `child.stdout.on('data', d => (said += d))`, + `child.stderr.on('data', d => (said += d))`, + `await new Promise(resolve => child.on('exit', resolve))`, + `writeFileSync(${JSON.stringify(out)}, said)`, + ...after, + ] + + function runOn(host: string, file: string): number | null { + return spawnSync(process.execPath, [file], { + env: onHost(host), + encoding: 'utf8', + timeout: 60000, + }).status + } + + /** Starts `file` on `host`, in a process group of its own. */ + function startOn(host: string, file: string): ChildProcess { + const child = spawn(process.execPath, [file], { + env: onHost(host), + stdio: 'ignore', + detached: true, + }) + others.push(child) + return child + } + + const exitOf = (child: ChildProcess): Promise => + new Promise(resolve => child.on('exit', resolve)) + + /** A command that says it is up, waits to be told, and tries the denied write. */ + const triesWhenTold = (up: string, go: string): string => + `echo ${BOOTED}; touch ${up}; while [ ! -e ${go} ]; do sleep 0.05; done; rm -f ${LOCK}; echo pwned > ${LOCK}; echo rc=$?` + + it.if(BWRAP_CAN_NAMESPACE)( + 'keep a mount point for the sandbox on one while the other wraps, runs and cleans up over the same path', + async () => { + const up = join(PROJECT, 'up') + const go = join(PROJECT, 'go') + const out = join(BASE, 'first.out') + const first = startOn( + 'aaaa', + script( + 'first.ts', + wrapAndRun(triesWhenTold(up, go), out, [ + 'm.cleanupBwrapMountPoints()', + ]), + ), + ) + const ended = exitOf(first) + await until('the first sandbox to be up', () => existsSync(up)) + expect(existsSync(LOCK)).toBe(true) + + // The whole life of a command on the other host, over the same path. + const second = join(BASE, 'second.out') + expect( + runOn( + 'bbbb', + script( + 'second.ts', + wrapAndRun( + `echo ${BOOTED}; rm -f ${LOCK}; echo pwned > ${LOCK}; echo rc=$?`, + second, + [ + 'm.cleanupBwrapMountPoints()', + 'm.cleanupBwrapMountPoints({ force: true })', + ], + ), + ), + ), + ).toBe(0) + expect(readFileSync(second, 'utf8')).toContain(BOOTED) + expect(readFileSync(second, 'utf8')).toMatch(/rc=[1-9]/) + expect(lstatSync(LOCK).size).toBe(0) + + writeFileSync(go, '') + await ended + expect(readFileSync(out, 'utf8')).toContain(BOOTED) + expect(readFileSync(out, 'utf8')).toMatch(/rc=[1-9]/) + expect(existsSync(LOCK)).toBe(false) + expect(readdirSync(SHARED)).toEqual([]) + }, + 90000, + ) + + it.if(BWRAP_CAN_NAMESPACE)( + 'keep a mount point for the sandbox on one when the other, whose sandbox made it, has finished and cleaned up', + async () => { + const at = (name: string): string => join(PROJECT, name) + const outs = [join(BASE, 'maker.out'), join(BASE, 'later.out')] + const maker = startOn( + 'aaaa', + script( + 'maker.ts', + wrapAndRun(triesWhenTold(at('up-1'), at('go-1')), outs[0]!, [ + 'm.cleanupBwrapMountPoints({ force: true })', + ]), + ), + ) + const makerEnded = exitOf(maker) + await until('the sandbox that makes it to be up', () => + existsSync(at('up-1')), + ) + const later = startOn( + 'bbbb', + script( + 'later.ts', + wrapAndRun(triesWhenTold(at('up-2'), at('go-2')), outs[1]!, [ + 'm.cleanupBwrapMountPoints({ force: true })', + ]), + ), + ) + const laterEnded = exitOf(later) + await until('the later sandbox to be up', () => existsSync(at('up-2'))) + + writeFileSync(at('go-1'), '') + await makerEnded + expect(readFileSync(outs[0]!, 'utf8')).toMatch(/rc=[1-9]/) + expect(lstatSync(LOCK).size).toBe(0) + + writeFileSync(at('go-2'), '') + await laterEnded + expect(readFileSync(outs[1]!, 'utf8')).toContain(BOOTED) + expect(readFileSync(outs[1]!, 'utf8')).toMatch(/rc=[1-9]/) + expect(existsSync(LOCK)).toBe(false) + expect(readdirSync(SHARED)).toEqual([]) + }, + 90000, + ) + + /** A job on the host `aaaa`, killed with its sandbox while that is up. */ + async function aKilledJob(): Promise { + const up = join(PROJECT, 'up') + const job = startOn( + 'aaaa', + script( + 'job.ts', + wrapAndRun( + triesWhenTold(up, join(PROJECT, 'never')), + join(BASE, 'job.out'), + [], + ), + ), + ) + const ended = exitOf(job) + await until('the job to be up', () => existsSync(up)) + // The process, its shell and bubblewrap, which takes the sandbox with it. + process.kill(-job.pid!, 'SIGKILL') + await ended + expect(existsSync(LOCK)).toBe(true) + expect(readdirSync(SHARED)).not.toEqual([]) + } + + const sweep = (): string => + script('sweep.ts', [ + `const m = await import(${HOST})`, + `m.collectMountPoints('all')`, + ]) + + it.if(BWRAP_CAN_NAMESPACE)( + 'have what a job killed on one of them left swept from the other, once it has not been heard of for ten minutes', + async () => { + await aKilledJob() + expect(runOn('bbbb', sweep())).toBe(0) + expect(existsSync(LOCK)).toBe(true) + + const left = readdirSync(SHARED) + for (const name of left) aged(join(SHARED, name), 11 * MINUTE_MS) + expect(runOn('bbbb', sweep())).toBe(0) + expect(existsSync(LOCK)).toBe(false) + expect(readdirSync(SHARED)).toEqual([]) + }, + 90000, + ) + + it.if(BWRAP_CAN_NAMESPACE)( + 'have it swept at once by the next clean-up on the host it ran on', + async () => { + await aKilledJob() + // bubblewrap may take a moment to go. + await until('the clean-up to find the sandbox gone', () => { + expect(runOn('aaaa', sweep())).toBe(0) + return !existsSync(LOCK) + }) + expect(readdirSync(SHARED)).toEqual([]) + }, + 90000, + ) + + it.if(BWRAP_CAN_NAMESPACE)( + 'start no sandbox on one while a clean-up on the other is stopped before its removal', + async () => { + leftover(LOCK) + aged(manifestOfAnotherHost([LOCK]), 11 * MINUTE_MS) + const about = join(BASE, 'about-to-remove') + const cleaner = spawn( + process.execPath, + [ + script('stopped-cleaner.ts', [ + `import { spyOn } from 'bun:test'`, + `import * as fs from 'node:fs'`, + `const m = await import(${HOST})`, + `const unlink = fs.unlinkSync`, + `spyOn(fs, 'unlinkSync').mockImplementation((file) => {`, + ` if (file === ${JSON.stringify(LOCK)}) {`, + ` fs.writeFileSync(${JSON.stringify(about)}, '')`, + ` process.kill(process.pid, 'SIGSTOP')`, + ` }`, + ` unlink(file)`, + `})`, + `m.collectMountPoints('all')`, + ]), + ], + { env: onHost('aaaa'), stdio: 'ignore' }, + ) + others.push(cleaner) + const cleaned = new Promise(resolve => cleaner.on('exit', resolve)) + await until('the clean-up to reach the removal', () => existsSync(about)) + + const out = join(BASE, 'second.out') + const second = spawn( + process.execPath, + [ + script( + 'second.ts', + wrapAndRun( + `echo ${BOOTED}; rm -f ${LOCK}; echo pwned > ${LOCK}; echo rc=$?`, + out, + ['m.cleanupBwrapMountPoints()'], + ), + ), + ], + { env: onHost('bbbb'), stdio: 'ignore' }, + ) + others.push(second) + const ended = new Promise(resolve => second.on('exit', resolve)) + await sleep(1500) + expect(existsSync(out)).toBe(false) + + process.kill(cleaner.pid!, 'SIGCONT') + await cleaned + await ended + expect(readFileSync(out, 'utf8')).toContain(BOOTED) + expect(readFileSync(out, 'utf8')).toMatch(/rc=[1-9]/) + expect(existsSync(LOCK)).toBe(false) + }, + 90000, + ) + + it('leave a mount point to neither when both finish with it at once, each of which would keep it for the other', async () => { + // Each meets the other once it has made its claims, and again once it has + // read and is about to remove or drop. + leftover(LOCK) + const side = (me: string, other: string): string => { + const at = (who: string, where: string): string => + JSON.stringify(join(BASE, `${who}.${where}`)) + const meet = (where: string): string => + `fs.writeFileSync(${at(me, where)}, ''); while (!fs.existsSync(${at(other, where)})) Bun.sleepSync(1)` + return script(`${me}.ts`, [ + `import { spyOn } from 'bun:test'`, + `import * as fs from 'node:fs'`, + `const m = await import(${HOST})`, + `m.publishMountPointManifest([${JSON.stringify(LOCK)}], [])`, + meet('published'), + `const open = fs.openSync`, + `const unlink = fs.unlinkSync`, + `let met = 0`, + `spyOn(fs, 'openSync').mockImplementation((p, ...rest) => {`, + ` if (met === 0 && String(p).endsWith('.pass')) { met = 1; ${meet('claimed')} }`, + ` return open(p, ...rest)`, + `})`, + `spyOn(fs, 'unlinkSync').mockImplementation(p => {`, + ` if (met === 1) { met = 2; ${meet('read')} }`, + ` return unlink(p)`, + `})`, + `m.collectMountPoints('all')`, + ]) + } + const ended = [ + ['aaaa', 'bbbb'], + ['bbbb', 'aaaa'], + ].map(([me, other]) => { + const host = spawn(process.execPath, [side(me!, other!)], { + env: onHost(me!), + stdio: 'inherit', + }) + others.push(host) + return new Promise(resolve => host.on('exit', resolve)) + }) + expect(await Promise.all(ended)).toEqual([0, 0]) + expect(existsSync(LOCK)).toBe(false) + expect(readdirSync(SHARED)).toEqual([]) + }, 30000) +}) diff --git a/test/sandbox/mount-point-manifests.test.ts b/test/sandbox/mount-point-manifests.test.ts index 225fe9b91..b6f6c7a48 100644 --- a/test/sandbox/mount-point-manifests.test.ts +++ b/test/sandbox/mount-point-manifests.test.ts @@ -23,7 +23,7 @@ import { tmpdir } from 'node:os' import { basename, dirname, join } from 'node:path' import { collectMountPoints, - discardMountPointManifest, + discardMountPointManifests, forgetMountPointManifestDirectory, keepTrackOfMountPoint, keepTrackOfMountSource, @@ -114,8 +114,8 @@ describe.if(isLinux)('The mount point manifests', () => { /** The manifest this process publishes for a wrap that names `paths`. */ function publish(paths: string[], sources: string[] = []): string { - const manifest = publishMountPointManifest(paths, sources) - if (manifest === undefined || manifest === 'too-large') { + const [manifest] = publishMountPointManifest(paths, sources) + if (manifest === undefined || typeof manifest === 'string') { throw new Error(`not published: ${manifest}`) } return manifest.file @@ -191,7 +191,7 @@ describe.if(isLinux)('The mount point manifests', () => { } const COLLECT = `m.collectMountPoints('all')` - const PUBLISH = `console.log(JSON.stringify(m.publishMountPointManifest(['/nonexistent/x'], []) ?? null))` + const PUBLISH = `console.log(JSON.stringify(m.publishMountPointManifest(['/nonexistent/x'], [])[0] ?? null))` it.if(BWRAP_CAN_NAMESPACE && NOT_ROOT)( 'collects nothing on the word of a directory that is read-only from here', @@ -259,7 +259,7 @@ describe.if(isLinux)('The mount point manifests', () => { let published: { status: number | null; stdout: string } | undefined const pass = duringTheNextPass(() => { published = inAChild( - `console.log(JSON.stringify(m.publishMountPointManifest([${JSON.stringify(X)}], []) ?? null))`, + `console.log(JSON.stringify(m.publishMountPointManifest([${JSON.stringify(X)}], [])[0] ?? null))`, ) }) let removed: string[] @@ -292,7 +292,7 @@ describe.if(isLinux)('The mount point manifests', () => { expect(readdirSync(DIR)).toEqual([basename(newcomer!)]) const live = started(manifestOfADeadProcess([Y]), thisProcess()) - discardMountPointManifest(newcomer!) + discardMountPointManifests([newcomer!]) expect(existsSync(X)).toBe(false) // What another manifest names as well is that one's to keep. expect(existsSync(Y)).toBe(true) @@ -1162,7 +1162,7 @@ describe.if(isLinux)('The mount point manifests', () => { return (write as (...args: unknown[]) => void)(file, ...rest) }) as never) try { - expect(publishMountPointManifest(paths, sources)).toBeUndefined() + expect(publishMountPointManifest(paths, sources)).toEqual([]) } finally { spy.mockRestore() } @@ -1696,7 +1696,13 @@ describe.if(isLinux)('The mount point manifests', () => { if (type instanceof Error) throw type return { type } }) as never) - return { restore: () => spy.mockRestore() } + // With what the library remembers of the answers. + return { + restore: () => { + spy.mockRestore() + forgetMountPointManifestDirectory() + }, + } } it('is finished when written in another boot, whatever runs on its record and whichever PID namespace wrote it', () => { @@ -1773,6 +1779,8 @@ describe.if(isLinux)('The mount point manifests', () => { const bothAged = (manifest: string, minutes: number): void => { aged(manifest, minutes) aged(recordOf(manifest), minutes) + // A time is only ever set forward, and the library counts on that. + forgetMountPointManifestDirectory() } forgetMountPointManifestDirectory() const spies = [onAFileSystemOf(0x6969)] @@ -1860,7 +1868,7 @@ describe.if(isLinux)('The mount point manifests', () => { // at with a /dev/shm of the child's own, not the one every process has. const SHM = '/dev/shm' const launcher = ['bwrap', '--dev-bind', '/', '/', '--tmpfs', SHM, '--'] - const publishAndList = `const { readdirSync } = await import('node:fs')\nconsole.log(JSON.stringify({ manifest: m.publishMountPointManifest(['/nonexistent/x'], []) ?? null, made: readdirSync(${JSON.stringify(SHM)}) }))` + const publishAndList = `const { readdirSync } = await import('node:fs')\nconsole.log(JSON.stringify({ manifest: m.publishMountPointManifest(['/nonexistent/x'], [])[0] ?? null, made: readdirSync(${JSON.stringify(SHM)}) }))` const fellBack = inAChild(publishAndList, { launcher, env: { XDG_RUNTIME_DIR: SHM }, @@ -2537,7 +2545,7 @@ describe.if(isLinux)('The mount point manifests', () => { `m.setMountPointManifestPlacesForTesting(${JSON.stringify(join(BASE, 'root'))})`, `const named = [...m.namedMountPoints().paths].sort()`, `const removed = m.collectMountPoints('all')`, - `console.log(JSON.stringify({ named, removed, file: m.publishMountPointManifest(['/nonexistent/x'], []).file }))`, + `console.log(JSON.stringify({ named, removed, file: m.publishMountPointManifest(['/nonexistent/x'], [])[0].file }))`, ].join('\n'), { launcher: [ @@ -2578,7 +2586,7 @@ describe.if(isLinux)('The mount point manifests', () => { `const at = ${JSON.stringify({ X, directory, source })}`, `const { existsSync } = await import('node:fs')`, `const there = () => [at.X, at.directory, at.source].map(p => existsSync(p))`, - `const published = m.publishMountPointManifest([at.X, at.directory], [at.source]) ?? null`, + `const published = m.publishMountPointManifest([at.X, at.directory], [at.source])[0] ?? null`, `const named = [...m.namedMountPoints().paths].sort()`, `const held = [...m.liveMountPoints()].sort()`, `const some = { removed: m.collectMountPoints('none'), there: there() }`, @@ -3007,12 +3015,12 @@ describe.if(isLinux)('The mount point manifests', () => { } // The caller knows what it has published itself. const own = publish([Y]) - expect(namedMountPoints(first, own)).toBe(first) + expect(namedMountPoints(first, [own])).toBe(first) expect([...reads(namedMountPoints(first)).paths].sort()).toEqual( [X, Y].sort(), ) manifestOfADeadProcess([Z]) - expect([...reads(namedMountPoints(first, own)).paths].sort()).toEqual( + expect([...reads(namedMountPoints(first, [own])).paths].sort()).toEqual( [X, Y, Z].sort(), ) }) @@ -3110,7 +3118,7 @@ describe.if(isLinux)('The mount point manifests', () => { const elsewhere = inAChild( [ `const u = await import(${LIBRARY})`, - `const published = m.publishMountPointManifest([${JSON.stringify(X)}], []) ?? null`, + `const published = m.publishMountPointManifest([${JSON.stringify(X)}], [])[0] ?? null`, `const live = [...m.liveMountPoints()]`, `const directories = m.mountPointManifestDirectories()`, `const collected = m.collectMountPoints('all')`, diff --git a/test/sandbox/mount-point-passes.test.ts b/test/sandbox/mount-point-passes.test.ts index 4124a423b..a16404112 100644 --- a/test/sandbox/mount-point-passes.test.ts +++ b/test/sandbox/mount-point-passes.test.ts @@ -273,7 +273,7 @@ describe.if(isLinux)('A clean-up of mount points, on record', () => { }) as never), ] try { - expect(publishMountPointManifest([LOCK], [])).toBeUndefined() + expect(publishMountPointManifest([LOCK], [])).toEqual([]) leftover(LOCK) expect(collectMountPoints('all')).toEqual([LOCK]) } finally { @@ -854,8 +854,8 @@ describe.if(isLinux)('A clean-up of mount points, on record', () => { /** The manifest this process publishes for a wrap that names `paths`. */ function publish(paths: string[], commandKey?: string): string { - const manifest = publishMountPointManifest(paths, [], commandKey) - if (manifest === undefined || manifest === 'too-large') { + const [manifest] = publishMountPointManifest(paths, [], commandKey) + if (manifest === undefined || typeof manifest === 'string') { throw new Error(`not published: ${manifest}`) } return manifest.file