From aad97a8e20899279913acd761f0d6b23fc61041d Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 00:00:26 +0200 Subject: [PATCH 01/37] fix(daemon): bound Docker API calls Add DockerCallTimeout next to the backoff constants and apply it to every request/response Docker call: a per-container timeout in processOne, a per-call timeout inside the processor (Processor.CallTimeout, set from DockerCallTimeout) for ContainerInspect and ServiceInspect so every entry point is bounded, every ContainerList (startup and systemd re-apply enumeration) and the startup Info call. Event stream establishment is bounded in the caller: moby's Events blocks until dockerd sends the response headers, so it runs in a goroutine on a cancelable stream context and is raced against a timer after a bounded Ping pre-check. On timeout the request is canceled and the goroutine awaited, and the loop re-enters the backoff; on success the stream context lives until reconnect or shutdown. A failed initial subscription no longer blocks startup: listenEvents resubscribes from the original since. --- cmd/swarm-device-access/main.go | 53 ++-- cmd/swarm-device-access/main_test.go | 37 ++- internal/daemon/daemon.go | 34 ++- internal/daemon/daemon_test.go | 114 +++++++-- internal/daemon/events.go | 142 +++++++++-- internal/daemon/events_test.go | 59 ++++- internal/daemon/timeout_test.go | 355 +++++++++++++++++++++++++++ internal/processor/processor.go | 29 ++- 8 files changed, 759 insertions(+), 64 deletions(-) create mode 100644 internal/daemon/timeout_test.go diff --git a/cmd/swarm-device-access/main.go b/cmd/swarm-device-access/main.go index a546a29..a99e14c 100644 --- a/cmd/swarm-device-access/main.go +++ b/cmd/swarm-device-access/main.go @@ -28,6 +28,7 @@ import ( "os" "os/signal" "syscall" + "time" "github.com/moby/moby/client" @@ -109,22 +110,7 @@ func run() int { recorder := observability.NewRecorder() - isSwarmManager := false - - infoResult, infoErr := cli.Info(rootCtx, client.InfoOptions{}) - if infoErr != nil { - log.Warn( - "could not query docker info; assuming worker node (service-label inspection disabled)", - "err", - infoErr, - ) - } else { - isSwarmManager = infoResult.Info.Swarm.ControlAvailable - log.Info("swarm role detected", - "manager", isSwarmManager, - "local_node_state", infoResult.Info.Swarm.LocalNodeState, - ) - } + isSwarmManager := detectSwarmManager(rootCtx, cli, daemon.DockerCallTimeout) // Start optional observability servers before the main loop so they are // reachable during startup enumeration. @@ -143,6 +129,7 @@ func run() int { HostRoot: hostRootPath, ProcRoot: "/", IsSwarmManager: isSwarmManager, + CallTimeout: daemon.DockerCallTimeout, } runErr := daemon.Run(rootCtx, daemon.Options{ @@ -160,3 +147,37 @@ func run() int { return 0 } + +// infoClient is the subset of *client.Client used for role detection. +type infoClient interface { + Info(ctx context.Context, options client.InfoOptions) (client.SystemInfoResult, error) +} + +// detectSwarmManager reports whether this node is a Swarm manager. The Info +// call is bounded: an unresponsive dockerd must not stall startup, and a +// failure or timeout degrades to the worker behavior. +func detectSwarmManager(ctx context.Context, cli infoClient, timeout time.Duration) bool { + log := logger.L() + + infoCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + infoResult, infoErr := cli.Info(infoCtx, client.InfoOptions{}) + if infoErr != nil { + log.Warn( + "could not query docker info; assuming worker node (service-label inspection disabled)", + "err", + infoErr, + ) + + return false + } + + isSwarmManager := infoResult.Info.Swarm.ControlAvailable + log.Info("swarm role detected", + "manager", isSwarmManager, + "local_node_state", infoResult.Info.Swarm.LocalNodeState, + ) + + return isSwarmManager +} diff --git a/cmd/swarm-device-access/main_test.go b/cmd/swarm-device-access/main_test.go index 7832cc5..57160f5 100644 --- a/cmd/swarm-device-access/main_test.go +++ b/cmd/swarm-device-access/main_test.go @@ -16,6 +16,39 @@ * limitations under the License. */ -// Tests have moved to internal/daemon and internal/processor. - package main + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/moby/moby/client" +) + +// hungInfo blocks Info until its context is done, like a wedged dockerd. +type hungInfo struct{} + +func (hungInfo) Info(ctx context.Context, _ client.InfoOptions) (client.SystemInfoResult, error) { + <-ctx.Done() + + return client.SystemInfoResult{}, fmt.Errorf("hung info: %w", ctx.Err()) +} + +func TestDetectSwarmManager_HungInfoIsBounded(t *testing.T) { + const timeout = 50 * time.Millisecond + + done := make(chan bool, 1) + + go func() { done <- detectSwarmManager(context.Background(), hungInfo{}, timeout) }() + + select { + case manager := <-done: + if manager { + t.Error("a timed-out Info must degrade to worker, got manager") + } + case <-time.After(20 * timeout): + t.Fatal("detectSwarmManager did not return: Info is not bounded") + } +} diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index fa2c130..9aacefa 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -39,6 +39,7 @@ type dockerAPI interface { options client.ContainerListOptions, ) (client.ContainerListResult, error) Events(ctx context.Context, options client.EventsListOptions) client.EventsResult + Ping(ctx context.Context, options client.PingOptions) (client.PingResult, error) } // Options bundles the dependencies required by Run. @@ -46,6 +47,18 @@ type Options struct { Docker dockerAPI Proc *processor.Processor Metrics *observability.Recorder + + // callTimeout overrides DockerCallTimeout; zero means the default. Only + // tests set it, to exercise hung calls without waiting the full timeout. + callTimeout time.Duration +} + +func (o *Options) timeout() time.Duration { + if o.callTimeout > 0 { + return o.callTimeout + } + + return DockerCallTimeout } // Run subscribes to the Docker event stream, enumerates existing containers, @@ -66,8 +79,13 @@ func run(ctx context.Context, opts Options, startWatcher func(context.Context, O // The client delivers messages on an unbuffered channel, so events that // arrive during the enumeration below wait (with backpressure on the - // socket) until listenEvents starts consuming them. - stream := opts.Docker.Events(ctx, eventListOptions(formatSince(since))) + // socket) until listenEvents starts consuming them. A failed subscription + // leaves stream empty; listenEvents then resubscribes from since, so the + // events of the enumeration window are still replayed. + stream, cancelStream, subErr := subscribe(ctx, opts.Docker, formatSince(since), opts.timeout()) + if subErr != nil { + log.Warn("could not subscribe to docker events; will retry", "err", subErr) + } processed := make(map[string]time.Time) @@ -77,6 +95,7 @@ func run(ctx context.Context, opts Options, startWatcher func(context.Context, O processed, opts.Metrics, processorApply(opts.Proc), + opts.timeout(), ) if processErr != nil { log.Warn("could not enumerate existing containers", "err", processErr) @@ -84,7 +103,7 @@ func run(ctx context.Context, opts Options, startWatcher func(context.Context, O startWatcher(ctx, opts) - listenEvents(ctx, opts, processed, since, stream.Messages, stream.Err) + listenEvents(ctx, opts, processed, since, stream, cancelStream) return nil } @@ -99,10 +118,15 @@ func processExistingContainers( processed map[string]time.Time, metrics *observability.Recorder, apply applyFn, + timeout time.Duration, ) error { log := logger.L() - list, err := cli.ContainerList(ctx, client.ContainerListOptions{}) + listCtx, cancelList := context.WithTimeout(ctx, timeout) + list, err := cli.ContainerList(listCtx, client.ContainerListOptions{}) + + cancelList() + if err != nil { return fmt.Errorf("list containers: %w", err) } @@ -119,6 +143,7 @@ func processExistingContainers( containers[idx].ID, metrics, apply, + timeout, "could not process running container", ) if processErr != nil { @@ -171,6 +196,7 @@ func startReloadWatcher(ctx context.Context, opts Options) { fresh, opts.Metrics, processorApply(opts.Proc), + opts.timeout(), ) if processErr != nil { log.Warn("could not re-apply rules after systemd reload", diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index b964ad9..979e5b7 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -22,6 +22,7 @@ import ( "bytes" "context" "errors" + "fmt" "log/slog" "slices" "strings" @@ -43,19 +44,25 @@ import ( // fakeDocker is a dockerAPI test double. The first Events call returns // firstMsgs/firstErrs; later calls return streams that stay open until ctx is // done. onList, when set, runs inside ContainerList (i.e. during enumeration). +// hangList and hangEvents make the corresponding call block until its context +// is done, like a wedged dockerd; hangEvents is keyed by the 1-based call number. type fakeDocker struct { - mu sync.Mutex - calls []string - sinces []string - listTime time.Time - containers []container.Summary - firstMsgs chan events.Message - firstErrs chan error - onList func() + mu sync.Mutex + calls []string + sinces []string + listTime time.Time + containers []container.Summary + firstMsgs chan events.Message + firstErrs chan error + onList func() + hangList bool + hangEvents map[int]bool + eventsReturned int + eventsCtx context.Context //nolint:containedctx // records the stream context so tests can assert its lifetime } func (f *fakeDocker) ContainerList( - _ context.Context, + ctx context.Context, _ client.ContainerListOptions, ) (client.ContainerListResult, error) { f.mu.Lock() @@ -63,6 +70,12 @@ func (f *fakeDocker) ContainerList( f.listTime = time.Now() f.mu.Unlock() + if f.hangList { + <-ctx.Done() + + return client.ContainerListResult{}, fmt.Errorf("hung list: %w", ctx.Err()) + } + if f.onList != nil { f.onList() } @@ -71,22 +84,57 @@ func (f *fakeDocker) ContainerList( } func (f *fakeDocker) Events( - _ context.Context, + ctx context.Context, options client.EventsListOptions, ) client.EventsResult { f.mu.Lock() - defer f.mu.Unlock() - f.calls = append(f.calls, "events") f.sinces = append(f.sinces, options.Since) + callNum := len(f.sinces) + hang := f.hangEvents[callNum] + f.eventsCtx = ctx + f.mu.Unlock() + + defer func() { + f.mu.Lock() + f.eventsReturned++ + f.mu.Unlock() + }() - if len(f.sinces) == 1 && f.firstMsgs != nil { + if hang { + <-ctx.Done() + + errs := make(chan error, 1) + errs <- ctx.Err() + + return client.EventsResult{Messages: make(chan events.Message), Err: errs} + } + + if callNum == 1 && f.firstMsgs != nil { return client.EventsResult{Messages: f.firstMsgs, Err: f.firstErrs} } return client.EventsResult{Messages: make(chan events.Message), Err: make(chan error)} } +func (*fakeDocker) Ping(context.Context, client.PingOptions) (client.PingResult, error) { + return client.PingResult{}, nil +} + +func (f *fakeDocker) lastEventsContext() context.Context { + f.mu.Lock() + defer f.mu.Unlock() + + return f.eventsCtx +} + +func (f *fakeDocker) eventsCalls() (started, returned int) { + f.mu.Lock() + defer f.mu.Unlock() + + return len(f.sinces), f.eventsReturned +} + func (f *fakeDocker) snapshot() (calls, sinces []string, listTime time.Time) { f.mu.Lock() defer f.mu.Unlock() @@ -134,7 +182,13 @@ func (r *recordingInspector) inspected() []string { func waitFor(t *testing.T, cond func() bool) { t.Helper() - deadline := time.Now().Add(2 * time.Second) + waitForWithin(t, 2*time.Second, cond) +} + +func waitForWithin(t *testing.T, limit time.Duration, cond func() bool) { + t.Helper() + + deadline := time.Now().Add(limit) for time.Now().Before(deadline) { if cond() { return @@ -232,8 +286,8 @@ func TestListenEvents_ReconnectSinceAfterLastEvent(t *testing.T) { opts, map[string]time.Time{}, time.Now().Add(-time.Hour), - msgs, - make(chan error), + client.EventsResult{Messages: msgs, Err: make(chan error)}, + func() {}, ) }() @@ -278,7 +332,14 @@ func TestListenEvents_ReconnectBeforeAnyEventUsesInitialSince(t *testing.T) { go func() { defer close(done) - listenEvents(ctx, opts, map[string]time.Time{}, initial, msgs, make(chan error)) + listenEvents( + ctx, + opts, + map[string]time.Time{}, + initial, + client.EventsResult{Messages: msgs, Err: make(chan error)}, + func() {}, + ) }() waitFor(t, func() bool { @@ -428,7 +489,14 @@ func TestProcessOne_StartupFailureMatchesEventPath(t *testing.T) { docker := &fakeDocker{containers: []container.Summary{{ID: "bad"}}} processed := map[string]time.Time{} - err := processExistingContainers(context.Background(), docker, processed, metrics, failingApply) + err := processExistingContainers( + context.Background(), + docker, + processed, + metrics, + failingApply, + DockerCallTimeout, + ) if err != nil { t.Fatalf("processExistingContainers: %v", err) } @@ -472,6 +540,7 @@ func TestProcessOne_StartupFailureMatchesEventPath(t *testing.T) { new(int64), metrics, failingApply, + DockerCallTimeout, ) assertFailureReported(t, before, readMetrics(t), buf.String(), "could not process container") @@ -486,7 +555,14 @@ func TestProcessOne_StartupSuccessRecordsMetrics(t *testing.T) { docker := &fakeDocker{containers: []container.Summary{{ID: "good"}}} processed := map[string]time.Time{} - err := processExistingContainers(context.Background(), docker, processed, metrics, noopApply) + err := processExistingContainers( + context.Background(), + docker, + processed, + metrics, + noopApply, + DockerCallTimeout, + ) if err != nil { t.Fatalf("processExistingContainers: %v", err) } diff --git a/internal/daemon/events.go b/internal/daemon/events.go index 18c65d6..f97698b 100644 --- a/internal/daemon/events.go +++ b/internal/daemon/events.go @@ -21,6 +21,7 @@ package daemon import ( "context" "errors" + "fmt" "time" "github.com/moby/moby/api/types/events" @@ -33,8 +34,16 @@ import ( const ( minBackoff = 1 * time.Second maxBackoff = 30 * time.Second + + // DockerCallTimeout bounds every request/response Docker API call and the + // establishment of the event stream. Without it a wedged dockerd blocks the + // caller forever, and nothing behind it (retries, reconnects, shutdown) + // ever runs. + DockerCallTimeout = 10 * time.Second ) +var errSubscribeTimeout = errors.New("docker event stream not established before timeout") + // applyFn is the per-container rule-application callback injected into consumeEvents. // In production this wraps Processor.ProcessContainer; in tests it is replaced by a fake. type applyFn func(ctx context.Context, id string) error @@ -69,10 +78,11 @@ func resubscribeSince(initial time.Time, lastEventNano int64) string { // listenEvents consumes Docker container events and applies device rules. // "start" covers fresh starts and restart's second phase; "unpause" covers -// resume from a paused state if the cgroup state was cleared. msgs and errs -// are the stream already opened by Run before the startup enumeration. On -// stream error it reconnects with exponential backoff (capped) rather than -// terminating the daemon — replaces the upstream log.Fatal(err) pattern. +// resume from a paused state if the cgroup state was cleared. stream is the +// stream already opened by Run before the startup enumeration (zero value when +// that subscription failed) and cancelStream releases it. On stream error it +// reconnects with exponential backoff (capped) rather than terminating the +// daemon — replaces the upstream log.Fatal(err) pattern. // // /readyz reflects live Docker event-stream health via observability.SetReady. func listenEvents( @@ -80,11 +90,12 @@ func listenEvents( opts Options, processed map[string]time.Time, since time.Time, - msgs <-chan events.Message, - errs <-chan error, + stream client.EventsResult, + cancelStream context.CancelFunc, ) { log := logger.L() backoff := minBackoff + timeout := opts.timeout() var lastEventNano int64 @@ -96,34 +107,119 @@ func listenEvents( for { if ctx.Err() != nil { + cancelStream() + return } - if msgs == nil { - stream := opts.Docker.Events( + if stream.Messages == nil { + var subErr error + + stream, cancelStream, subErr = subscribe( ctx, - eventListOptions(resubscribeSince(since, lastEventNano)), + opts.Docker, + resubscribeSince(since, lastEventNano), + timeout, ) - msgs, errs = stream.Messages, stream.Err + if subErr != nil { + if ctx.Err() != nil { + return + } + + log.Error("could not subscribe to docker events, retrying", + "err", subErr, "backoff", backoff) + opts.Metrics.IncDockerReconnect() + observability.SetReady(false) + sleepCtx(ctx, backoff) + backoff = nextBackoff(backoff) + + continue + } } observability.SetReady(true) log.Debug("subscribed to docker events") - disconnected := consumeEvents(ctx, msgs, errs, processed, &backoff, clearProcessed, + disconnected := consumeEvents( + ctx, + stream.Messages, + stream.Err, + processed, + &backoff, + clearProcessed, &lastEventNano, opts.Metrics, - processorApply(opts.Proc)) + processorApply(opts.Proc), + timeout, + ) + + // The stream context lives exactly as long as this subscription: a + // reconnect opens a new one and shutdown must release the request. + cancelStream() + if !disconnected { return } - msgs, errs = nil, nil + stream = client.EventsResult{} observability.SetReady(false) } } +// subscribe opens the Docker event stream with a bounded establishment. +// +// The client's Events call does not return until dockerd has sent the +// response headers, so a hung dockerd would block the caller with nothing +// able to act as a watchdog, and a deadline on the stream context would also +// kill the stream once established. Events therefore runs in a goroutine on a +// cancelable context and is raced against a timer here; on timeout the +// request is canceled and the goroutine is waited for, so it never leaks. On +// success the returned cancel func owns the stream context for its lifetime. +func subscribe( + ctx context.Context, + docker dockerAPI, + since string, + timeout time.Duration, +) (client.EventsResult, context.CancelFunc, error) { + pingCtx, cancelPing := context.WithTimeout(ctx, timeout) + _, pingErr := docker.Ping(pingCtx, client.PingOptions{}) + + cancelPing() + + if pingErr != nil { + return client.EventsResult{}, func() {}, fmt.Errorf("ping docker: %w", pingErr) + } + + streamCtx, cancel := context.WithCancel(ctx) + established := make(chan client.EventsResult, 1) + + go func() { + established <- docker.Events(streamCtx, eventListOptions(since)) + }() + + timer := time.NewTimer(timeout) + defer timer.Stop() + + select { + case stream := <-established: + return stream, cancel, nil + case <-timer.C: + cancel() + <-established + + return client.EventsResult{}, func() {}, errSubscribeTimeout + case <-ctx.Done(): + cancel() + <-established + + return client.EventsResult{}, func() {}, fmt.Errorf( + "subscribe to docker events: %w", + ctx.Err(), + ) + } +} + // consumeEvents drains one events.Subscribe lifecycle. Returns true if the // caller should reconnect, false on context cancellation. lastEventNano is // updated with the timestamp of every received event. @@ -143,6 +239,7 @@ func consumeEvents( lastEventNano *int64, metrics *observability.Recorder, apply applyFn, + timeout time.Duration, ) bool { log := logger.L() @@ -216,7 +313,14 @@ func consumeEvents( } } - _ = processOne(ctx, msg.Actor.ID, metrics, apply, "could not process container") + _ = processOne( + ctx, + msg.Actor.ID, + metrics, + apply, + timeout, + "could not process container", + ) } } } @@ -224,17 +328,23 @@ func consumeEvents( // processOne applies device rules to one container and records the outcome: // rules-applied result, apply duration, last-event timestamp on success, and // a WARN with logMsg on failure. It is shared by the startup enumeration and -// the event stream so both paths report identically. +// the event stream so both paths report identically. The apply runs under a +// per-container timeout so a hung Docker call made while processing (inspect) +// cannot stall the enumeration or the event loop behind it. func processOne( ctx context.Context, containerID string, metrics *observability.Recorder, apply applyFn, + timeout time.Duration, logMsg string, ) error { start := time.Now() - applyErr := apply(ctx, containerID) + applyCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + applyErr := apply(applyCtx, containerID) if applyErr != nil { logger.L().Warn(logMsg, "id", containerID, "err", applyErr) metrics.RecordRuleApplied(false) diff --git a/internal/daemon/events_test.go b/internal/daemon/events_test.go index 85ffbf4..de84806 100644 --- a/internal/daemon/events_test.go +++ b/internal/daemon/events_test.go @@ -104,6 +104,7 @@ func TestConsumeEvents_ContextCancelledReturnsNoReconnect(t *testing.T) { new(int64), nil, noopApply, + DockerCallTimeout, ) if got { t.Error( @@ -129,6 +130,7 @@ func TestConsumeEvents_StreamErrorReturnsReconnect(t *testing.T) { new(int64), nil, noopApply, + DockerCallTimeout, ) if !got { t.Error("consumeEvents should return true (reconnect) on stream error") @@ -158,6 +160,7 @@ func TestConsumeEvents_ContextErrFromStreamErrorNoReconnect(t *testing.T) { new(int64), nil, noopApply, + DockerCallTimeout, ) if got { t.Error("consumeEvents should return false when stream error is context.Canceled") @@ -198,6 +201,7 @@ func TestConsumeEvents_ArbitraryStreamErrorAfterCancelNoReconnect(t *testing.T) new(int64), nil, noopApply, + DockerCallTimeout, ) if got { t.Error("consumeEvents should return false when the context is already canceled") @@ -250,6 +254,7 @@ func TestConsumeEvents_ChannelCloseReturnsReconnect(t *testing.T) { new(int64), nil, noopApply, + DockerCallTimeout, ) if !got { t.Error("consumeEvents should return true (reconnect) on channel close") @@ -281,7 +286,18 @@ func TestConsumeEvents_EventCallsApply(t *testing.T) { cancel() }() - consumeEvents(ctx, msgs, errs, map[string]time.Time{}, &backoff, nil, new(int64), nil, apply) + consumeEvents( + ctx, + msgs, + errs, + map[string]time.Time{}, + &backoff, + nil, + new(int64), + nil, + apply, + DockerCallTimeout, + ) if called.Load() != 2 { t.Errorf("apply called %d times, want 2", called.Load()) @@ -313,7 +329,18 @@ func TestConsumeEvents_DeduplicatesProcessedIDs(t *testing.T) { cancel() }() - consumeEvents(ctx, msgs, errs, processed, &backoff, nil, new(int64), nil, apply) + consumeEvents( + ctx, + msgs, + errs, + processed, + &backoff, + nil, + new(int64), + nil, + apply, + DockerCallTimeout, + ) if called.Load() != 0 { t.Errorf("apply called %d times for deduplicated ID, want 0", called.Load()) @@ -344,7 +371,18 @@ func TestConsumeEvents_ClearProcessedOnTTL(t *testing.T) { cancel() }() - consumeEvents(ctx, msgs, errs, processed, &backoff, clearCh, new(int64), nil, noopApply) + consumeEvents( + ctx, + msgs, + errs, + processed, + &backoff, + clearCh, + new(int64), + nil, + noopApply, + DockerCallTimeout, + ) if len(processed) != 0 { t.Errorf("processed map has %d entries after TTL, want 0", len(processed)) @@ -375,6 +413,7 @@ func TestConsumeEvents_BackoffResetsOnSuccessfulEvent(t *testing.T) { new(int64), nil, noopApply, + DockerCallTimeout, ) if backoff != minBackoff { @@ -422,7 +461,18 @@ func TestConsumeEvents_RestartWithinWindow(t *testing.T) { cancel() }() - consumeEvents(ctx, msgs, errs, processed, &backoff, nil, new(int64), nil, apply) + consumeEvents( + ctx, + msgs, + errs, + processed, + &backoff, + nil, + new(int64), + nil, + apply, + DockerCallTimeout, + ) if called.Load() != tc.wantApply { t.Errorf("apply called %d times, want %d", called.Load(), tc.wantApply) @@ -463,6 +513,7 @@ func TestConsumeEvents_TracksLastEventNano(t *testing.T) { &lastEventNano, nil, noopApply, + DockerCallTimeout, ) if lastEventNano != 300 { diff --git a/internal/daemon/timeout_test.go b/internal/daemon/timeout_test.go new file mode 100644 index 0000000..f0f35ee --- /dev/null +++ b/internal/daemon/timeout_test.go @@ -0,0 +1,355 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package daemon + +import ( + "context" + "errors" + "fmt" + "slices" + "sync" + "testing" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/client" + + "github.com/leinardi/swarm-device-access/internal/config" + "github.com/leinardi/swarm-device-access/internal/processor" +) + +// testCallTimeout stands in for DockerCallTimeout so hung calls resolve fast. +const testCallTimeout = 50 * time.Millisecond + +// hangingInspector blocks ContainerInspect (when hangInspect) or +// ServiceInspect until the call's context is done, and records the context +// error each hung call observed. +type hangingInspector struct { + hangInspect bool + + mu sync.Mutex + ctxErrs []error +} + +func (h *hangingInspector) ContainerInspect( + ctx context.Context, + _ string, + _ client.ContainerInspectOptions, +) (client.ContainerInspectResult, error) { + if h.hangInspect { + return client.ContainerInspectResult{}, h.hang(ctx) + } + + return client.ContainerInspectResult{Container: container.InspectResponse{ + State: &container.State{Pid: 4242}, + Config: &container.Config{Labels: map[string]string{"com.docker.swarm.service.id": "svc"}}, + }}, nil +} + +func (h *hangingInspector) ServiceInspect( + ctx context.Context, + _ string, + _ client.ServiceInspectOptions, +) (client.ServiceInspectResult, error) { + return client.ServiceInspectResult{}, h.hang(ctx) +} + +func (h *hangingInspector) hang(ctx context.Context) error { + <-ctx.Done() + + h.mu.Lock() + h.ctxErrs = append(h.ctxErrs, ctx.Err()) + h.mu.Unlock() + + return fmt.Errorf("hung call: %w", ctx.Err()) +} + +func (h *hangingInspector) observed() []error { + h.mu.Lock() + defer h.mu.Unlock() + + return slices.Clone(h.ctxErrs) +} + +// assertBounded fails when fn does not return within a generous multiple of +// testCallTimeout, i.e. when the hung call was not bounded. +func assertBounded(t *testing.T, name string, fn func()) { + t.Helper() + + done := make(chan struct{}) + + go func() { + defer close(done) + + fn() + }() + + select { + case <-done: + case <-time.After(20 * testCallTimeout): + t.Fatalf("%s did not return within %v: hung call is not bounded", name, 20*testCallTimeout) + } +} + +func TestProcessExistingContainers_HungContainerListIsBounded(t *testing.T) { + docker := &fakeDocker{hangList: true} + + var err error + + assertBounded(t, "processExistingContainers", func() { + err = processExistingContainers( + context.Background(), + docker, + map[string]time.Time{}, + nil, + noopApply, + testCallTimeout, + ) + }) + + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v, want context.DeadlineExceeded", err) + } +} + +func TestProcessOne_HungContainerInspectIsBounded(t *testing.T) { + insp := &hangingInspector{hangInspect: true} + proc := &processor.Processor{Inspector: insp, Cfg: config.NewStore()} + + var err error + + assertBounded(t, "processOne", func() { + err = processOne( + context.Background(), + "c1", + nil, + processorApply(proc), + testCallTimeout, + "fail", + ) + }) + + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v, want context.DeadlineExceeded", err) + } +} + +// TestProcessContainer_HungInspectsAreBoundedPerCall calls the processor +// directly with an unbounded context: its own per-call timeout must bound +// both ContainerInspect and ServiceInspect. +func TestProcessContainer_HungInspectsAreBoundedPerCall(t *testing.T) { + for _, hangInspect := range []bool{true, false} { + insp := &hangingInspector{hangInspect: hangInspect} + proc := &processor.Processor{ + Inspector: insp, + Cfg: config.NewStore(), + IsSwarmManager: true, + ProcRoot: t.TempDir(), + CallTimeout: testCallTimeout, + } + + assertBounded(t, "ProcessContainer", func() { + _ = proc.ProcessContainer(context.Background(), "c1") + }) + + got := insp.observed() + if len(got) != 1 || !errors.Is(got[0], context.DeadlineExceeded) { + t.Fatalf( + "hangInspect=%v: hung call context errors = %v, want one DeadlineExceeded", + hangInspect, + got, + ) + } + } +} + +func TestProcessOne_HungServiceInspectIsBounded(t *testing.T) { + insp := &hangingInspector{} + proc := &processor.Processor{ + Inspector: insp, + Cfg: config.NewStore(), + IsSwarmManager: true, + ProcRoot: t.TempDir(), + } + + assertBounded(t, "processOne", func() { + _ = processOne( + context.Background(), + "c1", + nil, + processorApply(proc), + testCallTimeout, + "fail", + ) + }) + + got := insp.observed() + if len(got) != 1 || !errors.Is(got[0], context.DeadlineExceeded) { + t.Fatalf("ServiceInspect context errors = %v, want one DeadlineExceeded", got) + } +} + +func TestSubscribe_HungEventsTimesOutWithoutLeak(t *testing.T) { + docker := &fakeDocker{hangEvents: map[int]bool{1: true}} + + var err error + + assertBounded(t, "subscribe", func() { + _, _, err = subscribe(context.Background(), docker, "", testCallTimeout) + }) + + if !errors.Is(err, errSubscribeTimeout) { + t.Fatalf("err = %v, want errSubscribeTimeout", err) + } + + // subscribe waits for the canceled Events goroutine before returning, so + // the call has already returned here rather than eventually. + started, returned := docker.eventsCalls() + if started != 1 || returned != 1 { + t.Fatalf("Events started=%d returned=%d, want 1/1 (goroutine leaked)", started, returned) + } +} + +func TestSubscribe_SuccessKeepsStreamOpenUntilCancel(t *testing.T) { + docker := &fakeDocker{} + + stream, cancelStream, err := subscribe(context.Background(), docker, "", testCallTimeout) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + if stream.Messages == nil { + t.Fatal("stream has no message channel") + } + + // Real elapsed window: well past the establishment timeout, the stream + // context must still be live because it must not inherit that deadline. + time.Sleep(2 * testCallTimeout) + + streamCtx := docker.lastEventsContext() + if streamCtx.Err() != nil { + t.Fatalf("stream context ended after establishment: %v", streamCtx.Err()) + } + + cancelStream() + + if streamCtx.Err() == nil { + t.Fatal("cancelStream did not end the stream context") + } +} + +// TestRun_InitialEventsHangStillEnumeratesAndResubscribes checks that a hung +// initial subscription neither blocks the startup enumeration nor the event +// loop, which resubscribes from the original since. +func TestRun_InitialEventsHangStillEnumeratesAndResubscribes(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + docker := &fakeDocker{ + containers: []container.Summary{{ID: "listed"}}, + hangEvents: map[int]bool{1: true}, + } + insp := &recordingInspector{} + opts := Options{ + Docker: docker, + Proc: &processor.Processor{Inspector: insp, Cfg: config.NewStore()}, + callTimeout: testCallTimeout, + } + + done := make(chan struct{}) + + go func() { + defer close(done) + + _ = run(ctx, opts, noopWatcher) + }() + + waitFor(t, func() bool { + started, _ := docker.eventsCalls() + + return started >= 2 && slices.Contains(insp.inspected(), "listed") + }) + cancel() + <-done + + _, sinces, _ := docker.snapshot() + if sinces[1] != sinces[0] { + t.Errorf("resubscribe Since = %q, want the initial %q", sinces[1], sinces[0]) + } + + started, returned := docker.eventsCalls() + if started != returned { + t.Errorf( + "Events started=%d returned=%d after shutdown (goroutine leaked)", + started, + returned, + ) + } +} + +// TestListenEvents_ReconnectEventsHangRetries checks that a hung reconnect is +// abandoned after the timeout and retried through the backoff loop. +func TestListenEvents_ReconnectEventsHangRetries(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + docker := &fakeDocker{hangEvents: map[int]bool{1: true}} + opts := Options{ + Docker: docker, + Proc: &processor.Processor{Inspector: &recordingInspector{}, Cfg: config.NewStore()}, + callTimeout: testCallTimeout, + } + + done := make(chan struct{}) + + go func() { + defer close(done) + + // A zero stream makes listenEvents subscribe immediately, as after a + // dropped connection. + listenEvents( + ctx, + opts, + map[string]time.Time{}, + time.Now(), + client.EventsResult{}, + func() {}, + ) + }() + + // First resubscribe hangs and times out, then minBackoff passes before the + // second attempt: poll until it happens, allowing for minBackoff on a slow + // runner. + waitForWithin(t, 5*minBackoff, func() bool { + started, _ := docker.eventsCalls() + + return started >= 2 + }) + cancel() + <-done + + started, returned := docker.eventsCalls() + if started != returned { + t.Errorf( + "Events started=%d returned=%d after shutdown (goroutine leaked)", + started, + returned, + ) + } +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 70eabb9..001c2a3 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -22,6 +22,7 @@ import ( "context" "fmt" "path/filepath" + "time" "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/swarm" @@ -64,7 +65,11 @@ type deviceRuleKey struct { // bind-mount /dev/... paths. Inspector and Cfg are required; Metrics may be // nil (calls become no-ops). HostRoot is the container-internal path to the // host root (typically "/host"). ProcRoot is used for /proc lookups ("/" in -// production, temp dir in tests). +// production, temp dir in tests). CallTimeout bounds each Docker call the +// processor makes, independent of the caller's context, so every entry point +// (not only the daemon's per-container wrapper) has bounded Docker I/O; +// production sets it to daemon.DockerCallTimeout, and zero leaves the calls +// bounded by the caller's context only. type Processor struct { Inspector DockerInspector Cfg *config.Store @@ -72,6 +77,7 @@ type Processor struct { HostRoot string ProcRoot string IsSwarmManager bool + CallTimeout time.Duration } // ProcessContainer inspects a container and applies cgroup BPF device-allow @@ -81,11 +87,15 @@ type Processor struct { func (p *Processor) ProcessContainer(ctx context.Context, containerID string) error { log := logger.L() + inspectCtx, cancelInspect := p.callContext(ctx) inspected, inspectErr := p.Inspector.ContainerInspect( - ctx, + inspectCtx, containerID, client.ContainerInspectOptions{}, ) + + cancelInspect() + if inspectErr != nil { return fmt.Errorf("inspect container %q: %w", containerID, inspectErr) } @@ -202,6 +212,15 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er return nil } +// callContext derives the context for one Docker call. +func (p *Processor) callContext(ctx context.Context) (context.Context, context.CancelFunc) { + if p.CallTimeout <= 0 { + return context.WithCancel(ctx) + } + + return context.WithTimeout(ctx, p.CallTimeout) +} + // containerRules aggregates the per-mount results for one container. type containerRules struct { granted []cgroup.DeviceRule // deduplicated across mounts @@ -274,11 +293,15 @@ func (p *Processor) resolveServiceLabels( return svc, nil } + svcCtx, cancelSvc := p.callContext(ctx) inspected, svcErr := p.Inspector.ServiceInspect( - ctx, + svcCtx, serviceID, client.ServiceInspectOptions{}, ) + + cancelSvc() + if svcErr != nil { log.Warn("could not inspect parent service; using container labels only", "id", containerID, From 1a4c174ca62c18a875834fea59034d05a1cf08a3 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 00:05:27 +0200 Subject: [PATCH 02/37] fix(policy)!: drop deploy-level (service) label support Policy now comes from the global config and the container's own labels only, so the decision for a container is identical on every node and across daemon restarts. Service-level labels were visible only on managers, unenforceable on workers and silently dropped when the service inspect failed, which turned a lost narrowing label into wider access. Removed: resolveServiceLabels, policy.MergeLabels, policy.KnownLabels, Processor.IsSwarmManager, the Info-based Swarm role detection at startup and the opt-in-granted-via-service-level-label log. ServiceInspect leaves the DockerInspector interface; the processor makes no service or node inspection. Unknown swarm-device-access.* keys are now reported on the container labels, the only label input left. BREAKING CHANGE: labels under deploy.labels are ignored; put swarm-device-access.* labels under the service's top-level labels: so they reach the container. A service whose only opt-in is a deploy-level label is now skipped. --- .agents/skills/trust-boundary/SKILL.md | 29 +- README.md | 35 ++- SECURITY.md | 7 + cmd/swarm-device-access/main.go | 50 +--- cmd/swarm-device-access/main_test.go | 37 +-- docs/architecture.md | 9 +- docs/testing.md | 2 +- .../audio-passthrough/docker-compose.yaml | 2 +- examples/gpu-passthrough/docker-compose.yaml | 2 +- .../smartctl-exporter/docker-compose.yaml | 2 +- examples/usb-passthrough/docker-compose.yaml | 2 +- examples/v4l2-passthrough/docker-compose.yaml | 2 +- internal/daemon/daemon_test.go | 8 - internal/daemon/timeout_test.go | 78 +---- internal/policy/policy.go | 32 --- internal/policy/policy_test.go | 56 ---- internal/processor/processor.go | 112 +------- internal/processor/processor_test.go | 266 +++++------------- 18 files changed, 153 insertions(+), 578 deletions(-) diff --git a/.agents/skills/trust-boundary/SKILL.md b/.agents/skills/trust-boundary/SKILL.md index 6f58843..db4d7b7 100644 --- a/.agents/skills/trust-boundary/SKILL.md +++ b/.agents/skills/trust-boundary/SKILL.md @@ -66,9 +66,12 @@ flags, and exits `1` with `invalid config: …` on failure; an unreadable or unp ## 3. Labels are untrusted input -Anyone who can `docker service create` or `docker run` sets `swarm-device-access.*` labels, and on -manager nodes service labels are merged in too (`policy.MergeLabels`, container labels winning, -called from `Processor.ProcessContainer` in `internal/processor/processor.go`). Treat every label +Anyone who can `docker service create` or `docker run` sets `swarm-device-access.*` labels. The +container's own labels are the **only** label input: `Processor.ProcessContainer` +(`internal/processor/processor.go`) parses `Config.Labels` from the container inspect and nothing +else. Swarm service labels (`deploy.labels:`) are ignored, and the processor never calls +`ServiceInspect` or `Info` (neither is part of `DockerInspector`), so a decision cannot depend on +which node runs the daemon or on whether a service inspect happened to succeed. Treat every label as attacker-controlled. - `policy.ParseContainer` rejects a non-boolean `enable` and any malformed glob in `device-allow` @@ -81,21 +84,17 @@ as attacker-controlled. - An empty or all-whitespace `device-allow` label means "inherit the global allow set", not "allow nothing" — that is documented in the README label table; keep it that way or change both. -Covered by `TestParseContainer`, `TestDeviceAllowed`, `TestExplicitlyAllowed`, `TestMergeLabels` -and `TestProcessContainer_SwarmServiceLabels`. +Covered by `TestParseContainer`, `TestDeviceAllowed`, `TestExplicitlyAllowed`, +`TestProcessContainer_IgnoresServiceLevelLabels` and +`TestProcessContainer_WarnsOnUnknownContainerLabel`. - [ ] New label parsing returns an error on malformed input, and the caller skips the container. - [ ] A new label can only narrow; a test proves deny still beats allow with it set. -- [ ] Unknown `swarm-device-access.*` keys keep being reported (`policy.UnknownLabels`), not - silently accepted. -- [ ] **Known gap:** on a manager node, when the parent-service inspect fails, - `Processor.resolveServiceLabels` logs `could not inspect parent service; using container - labels only` and returns no service labels — so a service-level `device-deny` (set under - `deploy.labels:`, which the README already tells users not to do) is silently dropped and - the container gets wider access than the deployer asked for. It is not an escalation — the - same deployer sets both label sets — but it is the "lost narrowing label means wider access" - pattern this item forbids. Do not extend it; the fail-closed fix is to skip the container - when a service it belongs to cannot be inspected. +- [ ] Unknown `swarm-device-access.*` keys on the container keep being reported + (`policy.UnknownLabels`, warned by `ProcessContainer`), not silently accepted. +- [ ] No second label source (service, node, stack) is added back: a label that only some nodes + can read, or that disappears when an API call fails, turns a lost narrowing label into wider + access. ## 4. Only `/dev` mount sources become rules diff --git a/README.md b/README.md index ff4f46e..312c145 100644 --- a/README.md +++ b/README.md @@ -98,9 +98,8 @@ services: - /dev/nvidia0:/dev/nvidia0 - /dev/nvidiactl:/dev/nvidiactl - /dev/nvidia-uvm:/dev/nvidia-uvm - # Use top-level labels: so the daemon can read them on both manager and - # worker nodes. deploy.labels: are service-level metadata only visible to - # the Docker API on manager nodes. + # Use top-level labels: so they reach the task container, where the daemon + # reads them. Labels under deploy.labels: are ignored. labels: swarm-device-access.enable: "true" swarm-device-access.device-allow: "/dev/nvidia*" @@ -161,22 +160,20 @@ Declare these labels under top-level `labels:` in your service definition. Docker copies top-level labels into each task container, so the daemon can read them on every node — including worker nodes. -> **Note:** Do not use `deploy.labels:` for these labels. `deploy.labels:` are -> Swarm service metadata that only the Docker Swarm API (manager nodes) can -> read. The daemon cannot see them on worker nodes and will warn you if it -> detects them on a manager. - -### Worker and Manager Nodes - -The daemon auto-detects its role at startup by querying the local Docker API. -On **worker nodes** it skips the Swarm service inspect entirely (no spurious -warnings). On **manager nodes** it additionally reads service-level metadata -for backward compatibility — but emits a warning if it finds -`swarm-device-access.*` labels under `deploy.labels:`, advising you to move -them to top-level `labels:`. - -If the node role changes (promote/demote), restart the daemon so it re-detects -its role. +Policy comes from the daemon's own configuration and the container's labels +only, so every node makes the same decision for the same container, before and +after a daemon restart. Labels under `deploy.labels:` (Swarm service metadata) +are **ignored** on every node. + +> **Migrating from earlier versions:** older releases also read +> `swarm-device-access.*` labels from `deploy.labels:` on manager nodes and +> warned about them there. That support is removed: service-level labels are +> never read, and the daemon no longer queries the node's Swarm role or +> inspects the parent service. A service whose only opt-in (or narrowing +> `device-allow`/`device-deny`) sits under `deploy.labels:` is now processed as +> if those labels were absent, which in the default `opt-in` mode means it is +> skipped. Move the labels to the service's top-level `labels:` so they reach +> the container. If you bind-mount a directory (for example `source: /dev/dri`), the `device-allow` glob is evaluated **per child node** inside that directory — diff --git a/SECURITY.md b/SECURITY.md index b72d827..a3b6c89 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -18,6 +18,13 @@ The security boundary is the host Docker socket: any process that can create containers with `/dev/...` bind mounts can influence which device-allow rules this daemon applies. Do not expose the daemon's host socket to untrusted callers. +**Policy inputs.** Which devices a container may open is decided only by the +daemon's own configuration (flags and config file) and the container's own +`swarm-device-access.*` labels. Swarm service-level labels (`deploy.labels:`) +are ignored, and the daemon does not inspect services or the node's Swarm role, +so a decision cannot differ between manager and worker nodes or change after a +daemon restart. + **Opt-in default.** The daemon defaults to `-policy-mode=opt-in`, processing only containers that carry `swarm-device-access.enable: "true"`. This minimises blast radius — containers that happen to bind-mount `/dev/...` paths are not silently diff --git a/cmd/swarm-device-access/main.go b/cmd/swarm-device-access/main.go index a99e14c..37e3e5c 100644 --- a/cmd/swarm-device-access/main.go +++ b/cmd/swarm-device-access/main.go @@ -28,7 +28,6 @@ import ( "os" "os/signal" "syscall" - "time" "github.com/moby/moby/client" @@ -110,8 +109,6 @@ func run() int { recorder := observability.NewRecorder() - isSwarmManager := detectSwarmManager(rootCtx, cli, daemon.DockerCallTimeout) - // Start optional observability servers before the main loop so they are // reachable during startup enumeration. if *metricsAddr != "" { @@ -123,13 +120,12 @@ func run() int { } proc := &processor.Processor{ - Inspector: cli, - Cfg: store, - Metrics: recorder, - HostRoot: hostRootPath, - ProcRoot: "/", - IsSwarmManager: isSwarmManager, - CallTimeout: daemon.DockerCallTimeout, + Inspector: cli, + Cfg: store, + Metrics: recorder, + HostRoot: hostRootPath, + ProcRoot: "/", + CallTimeout: daemon.DockerCallTimeout, } runErr := daemon.Run(rootCtx, daemon.Options{ @@ -147,37 +143,3 @@ func run() int { return 0 } - -// infoClient is the subset of *client.Client used for role detection. -type infoClient interface { - Info(ctx context.Context, options client.InfoOptions) (client.SystemInfoResult, error) -} - -// detectSwarmManager reports whether this node is a Swarm manager. The Info -// call is bounded: an unresponsive dockerd must not stall startup, and a -// failure or timeout degrades to the worker behavior. -func detectSwarmManager(ctx context.Context, cli infoClient, timeout time.Duration) bool { - log := logger.L() - - infoCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - infoResult, infoErr := cli.Info(infoCtx, client.InfoOptions{}) - if infoErr != nil { - log.Warn( - "could not query docker info; assuming worker node (service-label inspection disabled)", - "err", - infoErr, - ) - - return false - } - - isSwarmManager := infoResult.Info.Swarm.ControlAvailable - log.Info("swarm role detected", - "manager", isSwarmManager, - "local_node_state", infoResult.Info.Swarm.LocalNodeState, - ) - - return isSwarmManager -} diff --git a/cmd/swarm-device-access/main_test.go b/cmd/swarm-device-access/main_test.go index 57160f5..7832cc5 100644 --- a/cmd/swarm-device-access/main_test.go +++ b/cmd/swarm-device-access/main_test.go @@ -16,39 +16,6 @@ * limitations under the License. */ -package main - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/moby/moby/client" -) - -// hungInfo blocks Info until its context is done, like a wedged dockerd. -type hungInfo struct{} - -func (hungInfo) Info(ctx context.Context, _ client.InfoOptions) (client.SystemInfoResult, error) { - <-ctx.Done() +// Tests have moved to internal/daemon and internal/processor. - return client.SystemInfoResult{}, fmt.Errorf("hung info: %w", ctx.Err()) -} - -func TestDetectSwarmManager_HungInfoIsBounded(t *testing.T) { - const timeout = 50 * time.Millisecond - - done := make(chan bool, 1) - - go func() { done <- detectSwarmManager(context.Background(), hungInfo{}, timeout) }() - - select { - case manager := <-done: - if manager { - t.Error("a timed-out Info must degrade to worker, got manager") - } - case <-time.After(20 * timeout): - t.Fatal("detectSwarmManager did not return: Info is not bounded") - } -} +package main diff --git a/docs/architecture.md b/docs/architecture.md index f6d58fb..30000be 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -167,10 +167,11 @@ The daemon uses a two-level policy: **Per-container policy** (Docker labels): Declare these labels under top-level `labels:` in your Swarm stack file. -Docker copies top-level labels into each task container so the daemon can read -them on both manager and worker nodes. Do **not** use `deploy.labels:` — those -are Swarm service metadata only accessible via the manager API; the daemon -cannot see them on worker nodes and will warn if it finds them on a manager. +Docker copies top-level labels into each task container, which is the only +place the daemon reads them. Labels under `deploy.labels:` are Swarm service +metadata and are ignored: the daemon never inspects the parent service or the +node's Swarm role, so the policy for a container is the same on every node and +across daemon restarts. | Label | Description | | --- | --- | diff --git a/docs/testing.md b/docs/testing.md index 4b65d93..cfe5a12 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -98,7 +98,7 @@ or deployment behavior: service's top-level `labels:`, which Docker copies into every task container; `docker service create --container-label` writes to the same location. Do not use `deploy.labels:` or `docker service create --label`: those are service - metadata that workers cannot read, see the README.) + metadata, which the daemon ignores; see the README.) 4. If the host uses cgroup v2, confirm a `BPF_CGROUP_DEVICE` program is attached to the consumer cgroup with `bpftool`. diff --git a/examples/audio-passthrough/docker-compose.yaml b/examples/audio-passthrough/docker-compose.yaml index b962b8f..6a987c8 100644 --- a/examples/audio-passthrough/docker-compose.yaml +++ b/examples/audio-passthrough/docker-compose.yaml @@ -15,7 +15,7 @@ services: # /dev/snd is a directory; the daemon walks it and applies a rule per device. - /dev/snd:/dev/snd command: [ "aplay", "-l" ] - # Use top-level labels: — deploy.labels: are only visible on manager nodes. + # Use top-level labels: so they reach the container; deploy.labels: are ignored. labels: swarm-device-access.enable: "true" swarm-device-access.device-allow: "/dev/snd/*" diff --git a/examples/gpu-passthrough/docker-compose.yaml b/examples/gpu-passthrough/docker-compose.yaml index eec9d49..d184388 100644 --- a/examples/gpu-passthrough/docker-compose.yaml +++ b/examples/gpu-passthrough/docker-compose.yaml @@ -19,7 +19,7 @@ services: - /dev/nvidiactl:/dev/nvidiactl - /dev/nvidia-uvm:/dev/nvidia-uvm command: [ "nvidia-smi" ] - # Use top-level labels: — deploy.labels: are only visible on manager nodes. + # Use top-level labels: so they reach the container; deploy.labels: are ignored. # Required when daemon runs in opt-in mode (the default). labels: swarm-device-access.enable: "true" diff --git a/examples/smartctl-exporter/docker-compose.yaml b/examples/smartctl-exporter/docker-compose.yaml index 83d7be9..0e89569 100644 --- a/examples/smartctl-exporter/docker-compose.yaml +++ b/examples/smartctl-exporter/docker-compose.yaml @@ -31,7 +31,7 @@ services: - SYS_ADMIN security_opt: - no-new-privileges=true - # Use top-level labels: — deploy.labels: are only visible on manager nodes. + # Use top-level labels: so they reach the container; deploy.labels: are ignored. labels: swarm-device-access.enable: "true" swarm-device-access.device-allow: "/dev/sd*,/dev/nvme*,/dev/sg*,/dev/disk/*" diff --git a/examples/usb-passthrough/docker-compose.yaml b/examples/usb-passthrough/docker-compose.yaml index 332bf32..6a1129e 100644 --- a/examples/usb-passthrough/docker-compose.yaml +++ b/examples/usb-passthrough/docker-compose.yaml @@ -44,7 +44,7 @@ services: # Bind-mount the entire USB bus; the daemon will apply rules to each device. - /dev/bus/usb:/dev/bus/usb command: [ "lsusb" ] - # Use top-level labels: — deploy.labels: are only visible on manager nodes. + # Use top-level labels: so they reach the container; deploy.labels: are ignored. labels: swarm-device-access.enable: "true" swarm-device-access.device-allow: "/dev/bus/usb/*" diff --git a/examples/v4l2-passthrough/docker-compose.yaml b/examples/v4l2-passthrough/docker-compose.yaml index ada0964..b0c7268 100644 --- a/examples/v4l2-passthrough/docker-compose.yaml +++ b/examples/v4l2-passthrough/docker-compose.yaml @@ -52,7 +52,7 @@ services: - -t - "5" - /output/capture.mp4 - # Use top-level labels: — deploy.labels: are only visible on manager nodes. + # Use top-level labels: so they reach the container; deploy.labels: are ignored. labels: swarm-device-access.enable: "true" swarm-device-access.device-allow: "/dev/video*" diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index 979e5b7..1fd45eb 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -164,14 +164,6 @@ func (r *recordingInspector) ContainerInspect( }, nil } -func (*recordingInspector) ServiceInspect( - _ context.Context, - _ string, - _ client.ServiceInspectOptions, -) (client.ServiceInspectResult, error) { - return client.ServiceInspectResult{}, nil -} - func (r *recordingInspector) inspected() []string { r.mu.Lock() defer r.mu.Unlock() diff --git a/internal/daemon/timeout_test.go b/internal/daemon/timeout_test.go index f0f35ee..c7fe794 100644 --- a/internal/daemon/timeout_test.go +++ b/internal/daemon/timeout_test.go @@ -37,12 +37,9 @@ import ( // testCallTimeout stands in for DockerCallTimeout so hung calls resolve fast. const testCallTimeout = 50 * time.Millisecond -// hangingInspector blocks ContainerInspect (when hangInspect) or -// ServiceInspect until the call's context is done, and records the context -// error each hung call observed. +// hangingInspector blocks ContainerInspect until the call's context is done, +// and records the context error each hung call observed. type hangingInspector struct { - hangInspect bool - mu sync.Mutex ctxErrs []error } @@ -52,22 +49,7 @@ func (h *hangingInspector) ContainerInspect( _ string, _ client.ContainerInspectOptions, ) (client.ContainerInspectResult, error) { - if h.hangInspect { - return client.ContainerInspectResult{}, h.hang(ctx) - } - - return client.ContainerInspectResult{Container: container.InspectResponse{ - State: &container.State{Pid: 4242}, - Config: &container.Config{Labels: map[string]string{"com.docker.swarm.service.id": "svc"}}, - }}, nil -} - -func (h *hangingInspector) ServiceInspect( - ctx context.Context, - _ string, - _ client.ServiceInspectOptions, -) (client.ServiceInspectResult, error) { - return client.ServiceInspectResult{}, h.hang(ctx) + return client.ContainerInspectResult{}, h.hang(ctx) } func (h *hangingInspector) hang(ctx context.Context) error { @@ -129,7 +111,7 @@ func TestProcessExistingContainers_HungContainerListIsBounded(t *testing.T) { } func TestProcessOne_HungContainerInspectIsBounded(t *testing.T) { - insp := &hangingInspector{hangInspect: true} + insp := &hangingInspector{} proc := &processor.Processor{Inspector: insp, Cfg: config.NewStore()} var err error @@ -150,58 +132,24 @@ func TestProcessOne_HungContainerInspectIsBounded(t *testing.T) { } } -// TestProcessContainer_HungInspectsAreBoundedPerCall calls the processor +// TestProcessContainer_HungInspectIsBoundedPerCall calls the processor // directly with an unbounded context: its own per-call timeout must bound -// both ContainerInspect and ServiceInspect. -func TestProcessContainer_HungInspectsAreBoundedPerCall(t *testing.T) { - for _, hangInspect := range []bool{true, false} { - insp := &hangingInspector{hangInspect: hangInspect} - proc := &processor.Processor{ - Inspector: insp, - Cfg: config.NewStore(), - IsSwarmManager: true, - ProcRoot: t.TempDir(), - CallTimeout: testCallTimeout, - } - - assertBounded(t, "ProcessContainer", func() { - _ = proc.ProcessContainer(context.Background(), "c1") - }) - - got := insp.observed() - if len(got) != 1 || !errors.Is(got[0], context.DeadlineExceeded) { - t.Fatalf( - "hangInspect=%v: hung call context errors = %v, want one DeadlineExceeded", - hangInspect, - got, - ) - } - } -} - -func TestProcessOne_HungServiceInspectIsBounded(t *testing.T) { +// ContainerInspect. +func TestProcessContainer_HungInspectIsBoundedPerCall(t *testing.T) { insp := &hangingInspector{} proc := &processor.Processor{ - Inspector: insp, - Cfg: config.NewStore(), - IsSwarmManager: true, - ProcRoot: t.TempDir(), + Inspector: insp, + Cfg: config.NewStore(), + CallTimeout: testCallTimeout, } - assertBounded(t, "processOne", func() { - _ = processOne( - context.Background(), - "c1", - nil, - processorApply(proc), - testCallTimeout, - "fail", - ) + assertBounded(t, "ProcessContainer", func() { + _ = proc.ProcessContainer(context.Background(), "c1") }) got := insp.observed() if len(got) != 1 || !errors.Is(got[0], context.DeadlineExceeded) { - t.Fatalf("ServiceInspect context errors = %v, want one DeadlineExceeded", got) + t.Fatalf("hung call context errors = %v, want one DeadlineExceeded", got) } } diff --git a/internal/policy/policy.go b/internal/policy/policy.go index e100b6c..03c9752 100644 --- a/internal/policy/policy.go +++ b/internal/policy/policy.go @@ -20,7 +20,6 @@ package policy import ( "fmt" - "maps" "path/filepath" "sort" "strconv" @@ -248,17 +247,6 @@ var knownLabels = map[string]struct{}{ LabelDeviceDeny: {}, } -// MergeLabels returns a merged label map: service labels as base, container -// labels win on conflict. Nil inputs are treated as empty maps. -func MergeLabels(service, container map[string]string) map[string]string { - merged := make(map[string]string, len(service)+len(container)) - - maps.Copy(merged, service) - maps.Copy(merged, container) - - return merged -} - // UnknownLabels returns a sorted slice of keys in labels that start with // LabelPrefix but are not in the known label set. Returns nil when none. func UnknownLabels(labels map[string]string) []string { @@ -280,23 +268,3 @@ func UnknownLabels(labels map[string]string) []string { return unknown } - -// KnownLabels returns a sorted list of swarm-device-access.* keys from labels -// that are recognized by this package (i.e. the inverse of UnknownLabels). -func KnownLabels(labels map[string]string) []string { - var known []string - - for k := range labels { - if _, ok := knownLabels[k]; ok { - known = append(known, k) - } - } - - if len(known) == 0 { - return nil - } - - sort.Strings(known) - - return known -} diff --git a/internal/policy/policy_test.go b/internal/policy/policy_test.go index 9c740d5..27cd99f 100644 --- a/internal/policy/policy_test.go +++ b/internal/policy/policy_test.go @@ -393,62 +393,6 @@ func sliceEqual(a, b []string) bool { return true } -func TestMergeLabels(t *testing.T) { - t.Parallel() - - cases := []struct { - name string - service map[string]string - container map[string]string - want map[string]string - }{ - { - name: "both nil", - want: map[string]string{}, - }, - { - name: "nil service", - container: map[string]string{"a": "1"}, - want: map[string]string{"a": "1"}, - }, - { - name: "nil container", - service: map[string]string{"a": "1"}, - want: map[string]string{"a": "1"}, - }, - { - name: "disjoint union", - service: map[string]string{"a": "1"}, - container: map[string]string{"b": "2"}, - want: map[string]string{"a": "1", "b": "2"}, - }, - { - name: "container wins on conflict", - service: map[string]string{"a": "service"}, - container: map[string]string{"a": "container"}, - want: map[string]string{"a": "container"}, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - got := policy.MergeLabels(tc.service, tc.container) - - if len(got) != len(tc.want) { - t.Fatalf("MergeLabels() len=%d, want %d; got=%v", len(got), len(tc.want), got) - } - - for k, wantV := range tc.want { - if got[k] != wantV { - t.Errorf("key %q: got %q, want %q", k, got[k], wantV) - } - } - }) - } -} - func TestUnknownLabels(t *testing.T) { t.Parallel() diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 001c2a3..b8a7fa4 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -25,7 +25,6 @@ import ( "time" "github.com/moby/moby/api/types/container" - "github.com/moby/moby/api/types/swarm" "github.com/moby/moby/client" "github.com/leinardi/swarm-device-access/internal/cgroup" @@ -35,22 +34,16 @@ import ( "github.com/leinardi/swarm-device-access/internal/policy" ) -const swarmServiceIDLabel = "com.docker.swarm.service.id" - // DockerInspector is the subset of *client.Client used by Processor. // It exists solely to allow unit tests to inject a fake without standing up a -// real Docker daemon. +// real Docker daemon. Policy comes from the global config and the container's +// own labels only, so no service or node inspection is part of it. type DockerInspector interface { ContainerInspect( ctx context.Context, containerID string, options client.ContainerInspectOptions, ) (client.ContainerInspectResult, error) - ServiceInspect( - ctx context.Context, - serviceID string, - options client.ServiceInspectOptions, - ) (client.ServiceInspectResult, error) } // deviceRuleKey is the deduplication key for cgroup device rules collected @@ -71,13 +64,12 @@ type deviceRuleKey struct { // production sets it to daemon.DockerCallTimeout, and zero leaves the calls // bounded by the caller's context only. type Processor struct { - Inspector DockerInspector - Cfg *config.Store - Metrics *observability.Recorder - HostRoot string - ProcRoot string - IsSwarmManager bool - CallTimeout time.Duration + Inspector DockerInspector + Cfg *config.Store + Metrics *observability.Recorder + HostRoot string + ProcRoot string + CallTimeout time.Duration } // ProcessContainer inspects a container and applies cgroup BPF device-allow @@ -114,13 +106,16 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er containerLabels = info.Config.Labels } - svc, serviceLabels := p.resolveServiceLabels(ctx, containerID, containerLabels) - - effectiveLabels := policy.MergeLabels(serviceLabels, containerLabels) + // The container's labels are the only label input, so this is the one + // place a misspelled key can be reported instead of silently ignored. + for _, unknownKey := range policy.UnknownLabels(containerLabels) { + log.Warn("unrecognized swarm-device-access label on container", + "id", containerID, "label", unknownKey) + } cfg := p.Cfg.Load() - cpol, parseErr := policy.ParseContainer(effectiveLabels) + cpol, parseErr := policy.ParseContainer(containerLabels) if parseErr != nil { log.Warn("container skipped: invalid policy labels", "id", containerID, "err", parseErr) @@ -139,17 +134,6 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er return nil } - if svc.Spec.Name != "" { - cpolContainer, _ := policy.ParseContainer(containerLabels) - if !cfg.Policy.Enabled(cpolContainer) { - log.Info("opt-in granted via service-level label", - "id", containerID, - "service_id", containerLabels[swarmServiceIDLabel], - "service_name", svc.Spec.Name, - ) - } - } - p.Metrics.RecordContainerScanned() pid := info.State.Pid @@ -274,72 +258,6 @@ func collectContainerRules( return result } -// resolveServiceLabels fetches parent service labels on manager nodes. On -// worker nodes (IsSwarmManager=false) it returns immediately with zero values -// so ProcessContainer can continue using container-level labels only. -func (p *Processor) resolveServiceLabels( - ctx context.Context, - containerID string, - containerLabels map[string]string, -) (svc swarm.Service, serviceLabels map[string]string) { - log := logger.L() - - if !p.IsSwarmManager { - return svc, nil - } - - serviceID := containerLabels[swarmServiceIDLabel] - if serviceID == "" { - return svc, nil - } - - svcCtx, cancelSvc := p.callContext(ctx) - inspected, svcErr := p.Inspector.ServiceInspect( - svcCtx, - serviceID, - client.ServiceInspectOptions{}, - ) - - cancelSvc() - - if svcErr != nil { - log.Warn("could not inspect parent service; using container labels only", - "id", containerID, - "service_id", serviceID, - "err", svcErr, - ) - - return swarm.Service{}, nil - } - - svc = inspected.Service - serviceLabels = svc.Spec.Labels - - for _, unknownKey := range policy.UnknownLabels(serviceLabels) { - log.Warn("unrecognized swarm-device-access label on parent service", - "id", containerID, - "service_id", serviceID, - "label", unknownKey, - ) - } - - for _, knownKey := range policy.KnownLabels(serviceLabels) { - log.Warn( - "swarm-device-access label set via deploy.labels on parent service; move to top-level labels: so worker nodes can read it", - "id", - containerID, - "service_id", - serviceID, - "service", - svc.Spec.Name, - "label", - knownKey, - ) - } - - return svc, serviceLabels -} - // applyRulesToCgroup logs and (unless dryRun) attaches the collected device // rules to the cgroup at cgroupPath via a single AddDeviceRules call. func (p *Processor) applyRulesToCgroup( diff --git a/internal/processor/processor_test.go b/internal/processor/processor_test.go index 3ef5ddd..ec41152 100644 --- a/internal/processor/processor_test.go +++ b/internal/processor/processor_test.go @@ -23,7 +23,6 @@ import ( "context" "errors" "log/slog" - "maps" "os" "path/filepath" "strconv" @@ -46,7 +45,6 @@ type fakeInspector struct { result container.InspectResponse err error serviceResult swarm.Service - serviceErr error serviceCalls int } @@ -58,6 +56,9 @@ func (f *fakeInspector) ContainerInspect( return client.ContainerInspectResult{Container: f.result}, f.err } +// ServiceInspect is not part of DockerInspector. It stays on the fake, with +// a call counter, so a test can prove the processor never reaches for the +// parent service even if the interface were widened again. func (f *fakeInspector) ServiceInspect( _ context.Context, _ string, @@ -65,9 +66,16 @@ func (f *fakeInspector) ServiceInspect( ) (client.ServiceInspectResult, error) { f.serviceCalls++ - return client.ServiceInspectResult{Service: f.serviceResult}, f.serviceErr + return client.ServiceInspectResult{Service: f.serviceResult}, nil } +// testCgroupContent and testMountinfoContent describe a cgroup v2 container, +// the only layout the processor tests need. +const ( + testCgroupContent = "0::/docker/testcontainer\n" + testMountinfoContent = "35 22 0:29 / /sys/fs/cgroup rw,nosuid,nodev shared:11 - cgroup2 cgroup2 rw\n" //nolint:dupword // cgroup2 appears twice: fs type and superblock type in mountinfo format +) + // buildProcRoot creates a minimal /proc//{cgroup,mountinfo} structure // under a temp dir so ProcessContainer can resolve the cgroup path without a // real /proc filesystem. @@ -76,7 +84,6 @@ func (f *fakeInspector) ServiceInspect( func buildProcRoot( t *testing.T, pid int, - cgroupContent, mountinfoContent string, ) string { t.Helper() @@ -90,7 +97,7 @@ func buildProcRoot( err = os.WriteFile( filepath.Join(procDir, "cgroup"), - []byte(cgroupContent), + []byte(testCgroupContent), 0o600, ) if err != nil { @@ -99,7 +106,7 @@ func buildProcRoot( err = os.WriteFile( filepath.Join(procDir, "mountinfo"), - []byte(mountinfoContent), + []byte(testMountinfoContent), 0o600, ) if err != nil { @@ -246,10 +253,7 @@ func TestProcessContainer_ZeroPid(t *testing.T) { func TestProcessContainer_NoDevMounts(t *testing.T) { const pid = 42 - cgroupContent := "0::/docker/testcontainer\n" - mountinfoContent := "35 22 0:29 / /sys/fs/cgroup rw,nosuid,nodev shared:11 - cgroup2 cgroup2 rw\n" //nolint:dupword // cgroup2 appears twice: fs type and superblock type in mountinfo format - - root := buildProcRoot(t, pid, cgroupContent, mountinfoContent) + root := buildProcRoot(t, pid) state := &container.State{Pid: pid} insp := &fakeInspector{result: container.InspectResponse{ @@ -281,10 +285,7 @@ func TestProcessContainer_NoDevMounts(t *testing.T) { func TestProcessContainer_DevMountFilterApplied(t *testing.T) { const pid = 43 - cgroupContent := "0::/docker/testcontainer\n" - mountinfoContent := "35 22 0:29 / /sys/fs/cgroup rw,nosuid,nodev shared:11 - cgroup2 cgroup2 rw\n" //nolint:dupword // cgroup2 appears twice: fs type and superblock type in mountinfo format - - root := buildProcRoot(t, pid, cgroupContent, mountinfoContent) + root := buildProcRoot(t, pid) state := &container.State{Pid: pid} insp := &fakeInspector{result: container.InspectResponse{ @@ -312,10 +313,7 @@ func TestProcessContainer_DevMountFilterApplied(t *testing.T) { func TestProcessContainer_DevMount_DryRunNoError(t *testing.T) { const pid = 44 - cgroupContent := "0::/docker/testcontainer\n" - mountinfoContent := "35 22 0:29 / /sys/fs/cgroup rw,nosuid,nodev shared:11 - cgroup2 cgroup2 rw\n" //nolint:dupword // cgroup2 appears twice: fs type and superblock type in mountinfo format - - root := buildProcRoot(t, pid, cgroupContent, mountinfoContent) + root := buildProcRoot(t, pid) state := &container.State{Pid: pid} insp := &fakeInspector{result: container.InspectResponse{ @@ -340,10 +338,7 @@ func TestProcessContainer_DevMount_DryRunNoError(t *testing.T) { func TestProcessContainer_DeduplicatesDuplicateMounts(t *testing.T) { const pid = 45 - cgroupContent := "0::/docker/testcontainer\n" - mountinfoContent := "35 22 0:29 / /sys/fs/cgroup rw,nosuid,nodev shared:11 - cgroup2 cgroup2 rw\n" //nolint:dupword // cgroup2 appears twice: fs type and superblock type in mountinfo format - - root := buildProcRoot(t, pid, cgroupContent, mountinfoContent) + root := buildProcRoot(t, pid) state := &container.State{Pid: pid} insp := &fakeInspector{result: container.InspectResponse{ @@ -369,10 +364,7 @@ func TestProcessContainer_DeduplicatesDuplicateMounts(t *testing.T) { func TestProcessContainer_OptInSkipsUnlabelled(t *testing.T) { const pid = 50 - cgroupContent := "0::/docker/testcontainer\n" - mountinfoContent := "35 22 0:29 / /sys/fs/cgroup rw,nosuid,nodev shared:11 - cgroup2 cgroup2 rw\n" //nolint:dupword // cgroup2 appears twice: fs type and superblock type in mountinfo format - - root := buildProcRoot(t, pid, cgroupContent, mountinfoContent) + root := buildProcRoot(t, pid) state := &container.State{Pid: pid} insp := &fakeInspector{result: container.InspectResponse{ @@ -397,10 +389,7 @@ func TestProcessContainer_OptInSkipsUnlabelled(t *testing.T) { func TestProcessContainer_OptInProcessesEnabled(t *testing.T) { const pid = 51 - cgroupContent := "0::/docker/testcontainer\n" - mountinfoContent := "35 22 0:29 / /sys/fs/cgroup rw,nosuid,nodev shared:11 - cgroup2 cgroup2 rw\n" //nolint:dupword // cgroup2 appears twice: fs type and superblock type in mountinfo format - - root := buildProcRoot(t, pid, cgroupContent, mountinfoContent) + root := buildProcRoot(t, pid) state := &container.State{Pid: pid} insp := &fakeInspector{result: container.InspectResponse{ @@ -791,10 +780,7 @@ func TestCollectMountRules_SingleFileNonDevice(t *testing.T) { func TestProcessContainer_DevDirectorySummary(t *testing.T) { const pid = 46 - cgroupContent := "0::/docker/testcontainer\n" - mountinfoContent := "35 22 0:29 / /sys/fs/cgroup rw,nosuid,nodev shared:11 - cgroup2 cgroup2 rw\n" //nolint:dupword // cgroup2 appears twice: fs type and superblock type in mountinfo format - - root := buildProcRoot(t, pid, cgroupContent, mountinfoContent) + root := buildProcRoot(t, pid) state := &container.State{Pid: pid} insp := &fakeInspector{result: container.InspectResponse{ @@ -864,181 +850,67 @@ func captureLogger(t *testing.T) *bytes.Buffer { return &buf } -//nolint:tparallel // subtests share the global logger via captureLogger; parallel would cause log interleaving -func TestProcessContainer_SwarmServiceLabels(t *testing.T) { - t.Parallel() - - const ( - cid = "abc123" - serviceID = "svc456" - pid = 51 - ) - - makeSwarmContainer := func(extraLabels map[string]string) container.InspectResponse { - labels := map[string]string{swarmServiceIDLabel: serviceID} - maps.Copy(labels, extraLabels) - - return container.InspectResponse{ - State: &container.State{Pid: pid}, - Config: &container.Config{Labels: labels}, - } - } +// TestProcessContainer_IgnoresServiceLevelLabels checks that policy comes +// from the container's own labels only: the parent service is never +// inspected, and a container whose only opt-in is a service-level (deploy.labels) +// label is skipped. +func TestProcessContainer_IgnoresServiceLevelLabels(t *testing.T) { + buf := captureLogger(t) - cgroupContent := "0::/docker/testcontainer\n" - mountinfoContent := "35 22 0:29 / /sys/fs/cgroup rw,nosuid,nodev shared:11 - cgroup2 cgroup2 rw\n" //nolint:dupword // cgroup2 appears twice: fs type and superblock type in mountinfo format + const pid = 51 - cases := []struct { - name string - containerInfo container.InspectResponse - svcResult swarm.Service - svcErr error - store *config.Store - isSwarmManager bool - wantServiceCall bool - wantLogMsg string - wantSkip bool - }{ - { - name: "deploy.labels-only grants opt-in on manager", - containerInfo: makeSwarmContainer(nil), - svcResult: swarm.Service{ - Spec: swarm.ServiceSpec{ - Annotations: swarm.Annotations{ - Name: "my-service", - Labels: map[string]string{policy.LabelEnable: "true"}, - }, - }, - }, - store: newStore(policy.ModeOptIn, true), - isSwarmManager: true, - wantServiceCall: true, - wantLogMsg: "opt-in granted via service-level label", - }, - { - name: "container labels override service on manager", - containerInfo: makeSwarmContainer(map[string]string{ - policy.LabelEnable: "false", - }), - svcResult: swarm.Service{ - Spec: swarm.ServiceSpec{ - Annotations: swarm.Annotations{ - Name: "my-service", - Labels: map[string]string{policy.LabelEnable: "true"}, - }, - }, - }, - store: newStore(policy.ModeOptIn, true), - isSwarmManager: true, - wantServiceCall: true, - wantSkip: true, - }, - { - name: "non-Swarm passthrough", - containerInfo: container.InspectResponse{ - State: &container.State{Pid: pid}, - Config: &container.Config{Labels: map[string]string{ - policy.LabelEnable: "true", - }}, - }, - store: newStore(policy.ModeOptIn, true), - isSwarmManager: true, - wantServiceCall: false, - }, - { - name: "service inspect error is non-fatal on manager", - containerInfo: makeSwarmContainer(map[string]string{ - policy.LabelEnable: "true", - }), - svcErr: errDaemonUnavail, - store: newStore(policy.ModeOptIn, true), - isSwarmManager: true, - wantServiceCall: true, - wantLogMsg: "could not inspect parent service", - }, - { - name: "typo WARN on service label", - containerInfo: makeSwarmContainer(map[string]string{ - policy.LabelEnable: "true", - }), - svcResult: swarm.Service{ - Spec: swarm.ServiceSpec{ - Annotations: swarm.Annotations{ - Name: "my-service", - Labels: map[string]string{policy.LabelPrefix + "enabled": "true"}, - }, - }, - }, - store: newStore(policy.ModeOptIn, true), - isSwarmManager: true, - wantServiceCall: true, - wantLogMsg: "unrecognized swarm-device-access label on parent service", - }, - { - name: "worker node skips service inspect", - containerInfo: makeSwarmContainer(map[string]string{ - policy.LabelEnable: "true", - }), - store: newStore(policy.ModeOptIn, true), - isSwarmManager: false, - wantServiceCall: false, - }, - { - name: "manager warns when known label set via deploy.labels", - containerInfo: makeSwarmContainer(nil), - svcResult: swarm.Service{ - Spec: swarm.ServiceSpec{ - Annotations: swarm.Annotations{ - Name: "my-service", - Labels: map[string]string{policy.LabelEnable: "true"}, - }, - }, - }, - store: newStore(policy.ModeOptIn, true), - isSwarmManager: true, - wantServiceCall: true, - wantLogMsg: "swarm-device-access label set via deploy.labels on parent service", + inspector := &fakeInspector{ + result: container.InspectResponse{ + State: &container.State{Pid: pid}, + Config: &container.Config{Labels: map[string]string{ + "com.docker.swarm.service.id": "svc456", + }}, }, + serviceResult: swarm.Service{Spec: swarm.ServiceSpec{Annotations: swarm.Annotations{ + Name: "my-service", + Labels: map[string]string{policy.LabelEnable: "true"}, + }}}, } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - buf := captureLogger(t) - - procRoot := buildProcRoot(t, pid, cgroupContent, mountinfoContent) + proc := &Processor{ + Inspector: inspector, + Cfg: newStore(policy.ModeOptIn, true), + } - inspector := &fakeInspector{ - result: tc.containerInfo, - serviceResult: tc.svcResult, - serviceErr: tc.svcErr, - } + err := proc.ProcessContainer(context.Background(), "abc123") + if err != nil { + t.Fatalf("ProcessContainer: %v", err) + } - proc := &Processor{ - Inspector: inspector, - Cfg: tc.store, - HostRoot: t.TempDir(), - ProcRoot: procRoot, - IsSwarmManager: tc.isSwarmManager, - } + if inspector.serviceCalls != 0 { + t.Errorf("ServiceInspect called %d times, want never", inspector.serviceCalls) + } - _ = proc.ProcessContainer(context.Background(), cid) + if !strings.Contains(buf.String(), "skipped by policy") { + t.Errorf("service-level opt-in must not enable the container; log:\n%s", buf.String()) + } +} - logOutput := buf.String() +func TestProcessContainer_WarnsOnUnknownContainerLabel(t *testing.T) { + buf := captureLogger(t) - if tc.wantServiceCall && inspector.serviceCalls == 0 { - t.Error("expected ServiceInspect to be called, was not") - } + inspector := &fakeInspector{ + result: container.InspectResponse{ + State: &container.State{Pid: 51}, + Config: &container.Config{Labels: map[string]string{ + policy.LabelPrefix + "enabled": "true", + }}, + }, + } - if !tc.wantServiceCall && inspector.serviceCalls > 0 { - t.Errorf("expected no ServiceInspect call, got %d", inspector.serviceCalls) - } + proc := &Processor{Inspector: inspector, Cfg: newStore(policy.ModeOptIn, true)} - if tc.wantLogMsg != "" && !strings.Contains(logOutput, tc.wantLogMsg) { - t.Errorf("expected log to contain %q, got:\n%s", tc.wantLogMsg, logOutput) - } + err := proc.ProcessContainer(context.Background(), "abc123") + if err != nil { + t.Fatalf("ProcessContainer: %v", err) + } - if tc.wantSkip && !strings.Contains(logOutput, "skipped by policy") { - t.Errorf("expected skip-by-policy log, got:\n%s", logOutput) - } - }) + if !strings.Contains(buf.String(), "unrecognized swarm-device-access label on container") { + t.Errorf("expected unknown-label warning, got:\n%s", buf.String()) } } From 26fb530b1d3d9631f5a9dec301c60c122ba94018 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 00:08:16 +0200 Subject: [PATCH 03/37] fix(processor): serialize policy evaluation and cgroup mutations Add one Processor-wide mutex held from the config load through the cgroup mutation, not around the mutation alone: the event loop, the startup enumeration and the systemd re-apply can process the same cgroup concurrently, and a worker that computed rules under an old config must not apply them after a newer config was loaded and applied by another worker. Docker calls made under the lock stay bounded by CallTimeout. A test holds one worker between compute and apply, starts a second worker for the same container, publishes a narrower config, and checks the second neither computes while the first holds the lock nor sees the old config afterwards. --- internal/processor/processor.go | 24 +++++ internal/processor/serialize_test.go | 129 +++++++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 internal/processor/serialize_test.go diff --git a/internal/processor/processor.go b/internal/processor/processor.go index b8a7fa4..789ef6b 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -22,6 +22,7 @@ import ( "context" "fmt" "path/filepath" + "sync" "time" "github.com/moby/moby/api/types/container" @@ -63,6 +64,8 @@ type deviceRuleKey struct { // (not only the daemon's per-container wrapper) has bounded Docker I/O; // production sets it to daemon.DockerCallTimeout, and zero leaves the calls // bounded by the caller's context only. +// +// A Processor must not be copied after first use. type Processor struct { Inspector DockerInspector Cfg *config.Store @@ -70,6 +73,20 @@ type Processor struct { HostRoot string ProcRoot string CallTimeout time.Duration + + // mu serializes policy evaluation and cgroup mutation across every + // caller (event loop, startup enumeration, systemd re-apply, reload). It + // is held from the config load through the mutation, not around the + // mutation alone: otherwise a worker that computed rules under an old + // config could apply them after a newer config was published and a + // newer computation had already been applied. One global lock is enough + // at this scale; any Docker I/O made while holding it is bounded by + // CallTimeout. + mu sync.Mutex + + // afterCompute, when set, runs under mu after the rules are computed + // and before they are applied. Tests use it to hold a worker mid-flight. + afterCompute func() } // ProcessContainer inspects a container and applies cgroup BPF device-allow @@ -113,6 +130,9 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er "id", containerID, "label", unknownKey) } + p.mu.Lock() + defer p.mu.Unlock() + cfg := p.Cfg.Load() cpol, parseErr := policy.ParseContainer(containerLabels) @@ -175,6 +195,10 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er p.Metrics.AddRuleFailures(len(collected.deviceErrs)) } + if p.afterCompute != nil { + p.afterCompute() + } + if len(collected.granted) > 0 { applyErr := p.applyRulesToCgroup(api, collected.granted, cgroupPath, pid, cfg.DryRun) if applyErr != nil { diff --git a/internal/processor/serialize_test.go b/internal/processor/serialize_test.go new file mode 100644 index 0000000..2d529f0 --- /dev/null +++ b/internal/processor/serialize_test.go @@ -0,0 +1,129 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processor + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/client" + + "github.com/leinardi/swarm-device-access/internal/config" + "github.com/leinardi/swarm-device-access/internal/policy" +) + +// countingInspector is a concurrency-safe DockerInspector that counts calls. +type countingInspector struct { + result container.InspectResponse + calls atomic.Int32 +} + +func (c *countingInspector) ContainerInspect( + _ context.Context, + _ string, + _ client.ContainerInspectOptions, +) (client.ContainerInspectResult, error) { + c.calls.Add(1) + + return client.ContainerInspectResult{Container: c.result}, nil +} + +// TestProcessContainer_SerializesConfigLoadThroughApply holds one worker +// between computing its rules and applying them, then starts a second worker +// for the same container and publishes a new config. The second worker must +// not load the config or compute anything until the first has finished, so +// no computation can be overtaken by an older one. +func TestProcessContainer_SerializesConfigLoadThroughApply(t *testing.T) { + const pid = 60 + + insp := &countingInspector{result: container.InspectResponse{State: &container.State{Pid: pid}}} + store := newStore(policy.ModeAll, true) + proc := &Processor{ + Inspector: insp, + Cfg: store, + HostRoot: "/host", + ProcRoot: buildProcRoot(t, pid), + } + + var ( + computed atomic.Int32 + modes []policy.Mode + modesMu sync.Mutex + firstInside = make(chan struct{}) + release = make(chan struct{}) + ) + + proc.afterCompute = func() { + modesMu.Lock() + + modes = append(modes, store.Load().Policy.Mode) + modesMu.Unlock() + + if computed.Add(1) == 1 { + close(firstInside) + <-release + } + } + + var workers sync.WaitGroup + + workers.Go(func() { _ = proc.ProcessContainer(context.Background(), "c1") }) + + <-firstInside + + workers.Go(func() { _ = proc.ProcessContainer(context.Background(), "c1") }) + + // Positive signal first: the second worker has finished its inspect, so it + // is at (or past) the lock. + deadline := time.Now().Add(2 * time.Second) + for insp.calls.Load() < 2 { + if time.Now().After(deadline) { + t.Fatal("second worker never inspected the container") + } + + time.Sleep(time.Millisecond) + } + + store.Set(config.Runtime{Policy: policy.Global{Mode: policy.ModeOptIn}, DryRun: true}) + + // Negative assertion: nothing can be polled for "did not happen", so give + // the second worker a bounded window to (wrongly) get past the lock. + time.Sleep(50 * time.Millisecond) + + if got := computed.Load(); got != 1 { + t.Fatalf("second worker computed while the first held the lock (computed=%d)", got) + } + + close(release) + workers.Wait() + + if got := computed.Load(); got != 1 { + // The second worker loads the new opt-in config and skips the + // unlabeled container before computing anything. + t.Fatalf("computed = %d after release, want 1 (second run must see the new config)", got) + } + + if len(modes) != 1 || modes[0] != policy.ModeAll { + t.Fatalf("modes seen by computing workers = %v, want [all]", modes) + } +} From 787ac02120236cb2606d774de4a5f37f11063c3b Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 00:17:12 +0200 Subject: [PATCH 04/37] fix(cgroup)!: replace cgroup v2 device filters without an unfiltered window AddDeviceRules detached every attached program before attaching the replacements, as NVIDIA's code does because it runs before the container starts. This daemon mutates live containers, so that left an unfiltered window, and a failed attach or daemon death left the container unfiltered for good. Programs are now replaced pairwise: atomically with BPF_F_REPLACE (via cilium's ReplaceProgram anchor) when the kernel supports it, probed on first use and cached, otherwise by attaching the new program before detaching the original. A detach failure rolls that pair back; earlier pairs stay replaced, which under BPF_F_ALLOW_MULTI is narrower, never wider. FindAttachedCgroupDeviceFilters now returns the kernel's total, the inaccessible count and the attach flags, and closes opened handles on every error path. A filter attached without BPF_F_ALLOW_MULTI skips the container (unsupported_attach_mode); any inaccessible filter is a retryable ErrFiltersInaccessible with nothing mutated; no attached filter is ErrFilterMissing instead of the Return()-only fallback the verifier always rejected. Also: RLIMIT_MEMLOCK is removed once at startup with rlimit.RemoveMemlock, runtime.KeepAlive covers the query buffer right after the syscall, logging goes through logger.L, and the kernel calls sit behind an ops seam with order tests. BREAKING CHANGE: containers whose runtime attaches the device filter without BPF_F_ALLOW_MULTI, and hosts where an LSM policy hides a device filter from the daemon, are now refused (skipped or retried without changes) instead of handled best-effort. See Host Requirements in the README. --- README.md | 26 ++ cmd/swarm-device-access/main.go | 14 + internal/cgroup/api.go | 7 +- internal/cgroup/ebpf.go | 71 +++-- internal/cgroup/ebpf_test.go | 5 +- internal/cgroup/v2.go | 165 ++++++----- internal/cgroup/v2ops.go | 244 ++++++++++++++++ internal/cgroup/v2swap_test.go | 430 ++++++++++++++++++++++++++++ internal/processor/applyerr_test.go | 109 +++++++ internal/processor/processor.go | 32 ++- 10 files changed, 1003 insertions(+), 100 deletions(-) create mode 100644 internal/cgroup/v2ops.go create mode 100644 internal/cgroup/v2swap_test.go create mode 100644 internal/processor/applyerr_test.go diff --git a/README.md b/README.md index 312c145..1970bf3 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,32 @@ Swarm does not allow those runtime options directly on a service. The common workaround is to deploy a small wrapper service that runs the Docker CLI and uses the host Docker socket to launch the real privileged daemon container. +### Host Requirements + +On cgroup v2 hosts the daemon adds its grants to the device filter the +container runtime already attached, replacing that program in place rather +than removing it first, so the container is never left unfiltered. That needs: + +- The runtime's device filter attached with `BPF_F_ALLOW_MULTI`, as runc 1.0+, + crun and systemd do. A filter attached without it (exclusive or + `BPF_F_ALLOW_OVERRIDE`) cannot be replaced without a window in which the + container has no filter at all, so the container is skipped: the daemon logs + an error naming the attach mode and counts it under + `sda_containers_skipped_total{reason="unsupported_attach_mode"}`. +- Every device filter attached to the container's cgroup readable by the + daemon. If any of them is hidden from it (for example an SELinux policy that + labels systemd's device filter so the daemon cannot open it), nothing is + changed and the container is retried: an unreadable program cannot be + checked or safely replaced. This is stricter than runc, which ignores such + programs, and such policies are unsupported. +- A device filter attached at all. A container whose cgroup has none (for + example after `systemctl daemon-reload` wiped it) is not granted anything. + +Atomic replacement (`BPF_F_REPLACE`) is used when the kernel supports it +(Linux 5.5+, detected on first use); otherwise each new program is attached +before the original is detached, which is narrower, never wider, if it is +interrupted. + ### Docker Compose for Swarm ```yaml diff --git a/cmd/swarm-device-access/main.go b/cmd/swarm-device-access/main.go index 37e3e5c..56ef8b3 100644 --- a/cmd/swarm-device-access/main.go +++ b/cmd/swarm-device-access/main.go @@ -29,6 +29,7 @@ import ( "os/signal" "syscall" + "github.com/cilium/ebpf/rlimit" "github.com/moby/moby/client" "github.com/leinardi/swarm-device-access/internal/config" @@ -88,6 +89,19 @@ func run() int { "device_deny", cfg.Policy.DeviceDeny, ) + // Lift RLIMIT_MEMLOCK once for the process so BPF_PROG_LOAD does not fail + // on kernels that still charge BPF memory to it (no-op from Linux 5.11, + // where memcg accounting replaced it). The limit is not inherited by + // containers. + memlockErr := rlimit.RemoveMemlock() + if memlockErr != nil { + log.Warn( + "could not remove RLIMIT_MEMLOCK; loading BPF device filters may fail", + "err", + memlockErr, + ) + } + rootCtx, cancelRoot := signal.NotifyContext( context.Background(), syscall.SIGINT, diff --git a/internal/cgroup/api.go b/internal/cgroup/api.go index a52f7c9..48e8706 100644 --- a/internal/cgroup/api.go +++ b/internal/cgroup/api.go @@ -52,7 +52,7 @@ func New(version int) (Interface, error) { case 1: return &cgroupv1{}, nil case 2: - return &cgroupv2{}, nil + return &cgroupv2{ops: kernelOps{}, replace: processReplaceProbe}, nil default: return nil, fmt.Errorf( //nolint:err113 // dynamic content "invalid cgroup version %d", @@ -65,7 +65,10 @@ var errNoDeviceOrUnifiedCgroup = errors.New("no devices or unified cgroup entrie type ( cgroupv1 struct{} - cgroupv2 struct{} + cgroupv2 struct { + ops v2ops + replace *replaceProbe + } ) var ( diff --git a/internal/cgroup/ebpf.go b/internal/cgroup/ebpf.go index c83f0f3..512b704 100644 --- a/internal/cgroup/ebpf.go +++ b/internal/cgroup/ebpf.go @@ -32,7 +32,6 @@ import ( "encoding/hex" "errors" "fmt" - "log/slog" "math" "os" "runtime" @@ -43,11 +42,14 @@ import ( "github.com/cilium/ebpf/asm" "github.com/cilium/ebpf/link" "golang.org/x/sys/unix" + + "github.com/leinardi/swarm-device-access/internal/logger" ) var ( errProgramFinalized = errors.New("the program is finalized") errIncompleteProgList = errors.New("could not get complete list of CGROUP_DEVICE programs") + errNoOriginalProgram = errors.New("no original device filter program to wrap") ) type program struct { @@ -235,7 +237,17 @@ func (p *program) finalize( } // FindAttachedCgroupDeviceFilters finds all ebpf prgrams associated with 'dirFd' that control device access. -func FindAttachedCgroupDeviceFilters(dirFd int) ([]*ebpf.Program, error) { +// +// It returns the handles it could open, the total number of programs the +// kernel reports as attached (ProgCnt), how many of those could not be opened +// (EACCES/EPERM, typically an LSM label) and the cgroup's attach flags for +// BPF_CGROUP_DEVICE. total is the only reliable "nothing attached" signal: an +// empty handle slice can also mean every attached program is inaccessible. +// On error no handle is returned and every handle opened so far is closed; +// on success the caller owns (and must close) the returned handles. +func FindAttachedCgroupDeviceFilters( + dirFd int, +) (progs []*ebpf.Program, total, inaccessible int, attachFlags uint32, err error) { type bpfAttrQuery struct { TargetFd uint32 AttachType uint32 @@ -263,9 +275,14 @@ func FindAttachedCgroupDeviceFilters(dirFd int) ([]*ebpf.Program, error) { uintptr(unix.BPF_PROG_QUERY), uintptr(unsafe.Pointer(&query)), unsafe.Sizeof(query)) - size = int(query.ProgCnt) + // The kernel writes into progIds through a pointer the GC cannot see, + // on success and on ENOSPC alike, so the buffer must stay alive until + // the syscall has returned, not merely until the ids are read below. + runtime.KeepAlive(progIds) runtime.KeepAlive(query) + size = int(query.ProgCnt) + if errno != 0 { // On ENOSPC we get the correct number of programs. if errno == unix.ENOSPC { @@ -273,7 +290,7 @@ func FindAttachedCgroupDeviceFilters(dirFd int) ([]*ebpf.Program, error) { continue } - return nil, fmt.Errorf("bpf_prog_query(BPF_CGROUP_DEVICE) failed: %w", errno) + return nil, 0, 0, 0, fmt.Errorf("bpf_prog_query(BPF_CGROUP_DEVICE) failed: %w", errno) } // Convert the ids to program handles. @@ -281,41 +298,42 @@ func FindAttachedCgroupDeviceFilters(dirFd int) ([]*ebpf.Program, error) { programs := make([]*ebpf.Program, 0, len(progIds)) for _, progId := range progIds { - program, err := ebpf.NewProgramFromID(ebpf.ProgramID(progId)) - if err != nil { - // We skip over programs that give us -EACCES or -EPERM. This - // is necessary because there may be BPF programs that have - // been attached (such as with --systemd-cgroup) which have an - // LSM label that blocks us from interacting with the program. - // - // Because additional BPF_CGROUP_DEVICE programs only can add - // restrictions, there's no real issue with just ignoring these - // programs (and stops runc from breaking on distributions with - // very strict SELinux policies). - if errors.Is(err, os.ErrPermission) { - slog.Debug( - "ignoring existing CGROUP_DEVICE program (cannot be accessed -- likely LSM policy)", + program, openErr := ebpf.NewProgramFromID(ebpf.ProgramID(progId)) + if openErr != nil { + // Programs that give us -EACCES or -EPERM (for example an LSM + // label on systemd's device filter) are counted, not skipped: + // runc ignores them because extra filters only restrict, but + // such a program could also be one of our own wrappers holding + // a grant, and without opening it neither its ownership nor + // the result of a replacement can be established. + if errors.Is(openErr, os.ErrPermission) { + logger.L().Debug( + "existing CGROUP_DEVICE program cannot be accessed (likely LSM policy)", "prog_id", progId, "err", - err, + openErr, ) + inaccessible++ + continue } - return nil, fmt.Errorf("cannot fetch program from id: %w", err) + for _, opened := range programs { + opened.Close() + } + + return nil, 0, 0, 0, fmt.Errorf("cannot fetch program from id: %w", openErr) } programs = append(programs, program) } - runtime.KeepAlive(progIds) - - return programs, nil + return programs, size, inaccessible, query.AttachFlags, nil } - return nil, errIncompleteProgList + return nil, 0, 0, 0, errIncompleteProgList } // PrependDeviceFilter prepends a set of instructions for further device filtering to an existing device filtering ebpf program. @@ -323,8 +341,11 @@ func PrependDeviceFilter( devices []DeviceRule, origInsts asm.Instructions, ) (asm.Instructions, error) { + // A bare Return() tail leaves R0 unset on fall-through, which the + // verifier rejects, and there is no safe default verdict to invent here: + // the original program is what denies everything not granted. if len(origInsts) == 0 { - origInsts = asm.Instructions{asm.Return()} + return nil, errNoOriginalProgram } labelPrefix, err := randomLabelPrefix() diff --git a/internal/cgroup/ebpf_test.go b/internal/cgroup/ebpf_test.go index 42aef4d..5fb227e 100644 --- a/internal/cgroup/ebpf_test.go +++ b/internal/cgroup/ebpf_test.go @@ -278,7 +278,10 @@ func TestPrependDeviceFilter_RoundTrip(t *testing.T) { {Allow: true, Type: "b", Major: ptr64(8), Minor: ptr64(0), Access: "rwm"}, } - insts, err := PrependDeviceFilter(rules, nil) + insts, err := PrependDeviceFilter( + rules, + asm.Instructions{asm.Mov.Imm32(asm.R0, 0), asm.Return()}, + ) if err != nil { t.Fatalf("PrependDeviceFilter: %v", err) } diff --git a/internal/cgroup/v2.go b/internal/cgroup/v2.go index 6281fda..85a9bf8 100644 --- a/internal/cgroup/v2.go +++ b/internal/cgroup/v2.go @@ -23,7 +23,6 @@ import ( "errors" "fmt" "io" - "log/slog" "os" "path/filepath" "strconv" @@ -152,118 +151,141 @@ func scanProcCgroupV2(r io.Reader, path string, prefix string) (string, error) { } // AddDeviceRules adds a set of device rules for the device cgroup at cgroupPath. +// +// Every attached program is replaced by a copy with the rules prepended. +// Unlike NVIDIA's upstream code, which detaches every program before +// attaching the replacements because it runs strictly before the container +// starts, this daemon mutates live containers: detaching first would leave +// the container unfiltered in between, and permanently so if the attach +// failed or the daemon died. Programs are therefore replaced pairwise, each +// new program attached (or atomically swapped in) before its original goes. func (c *cgroupv2) AddDeviceRules(cgroupPath string, rules []DeviceRule) error { // Open the cgroup path. - dirFD, err := unix.Open(cgroupPath, unix.O_DIRECTORY|unix.O_RDONLY, 0) + dirFD, err := unix.Open(cgroupPath, unix.O_DIRECTORY|unix.O_RDONLY|unix.O_CLOEXEC, 0) if err != nil { return fmt.Errorf("unable to open the cgroup path: %w", err) } defer unix.Close(dirFD) + return c.addDeviceRules(dirFD, rules) +} + +func (c *cgroupv2) addDeviceRules(dirFD int, rules []DeviceRule) error { // Find any existing eBPF device filter programs attached to this cgroup. - oldProgs, err := FindAttachedCgroupDeviceFilters(dirFD) + oldProgs, total, inaccessible, attachFlags, err := c.ops.query(dirFD) if err != nil { return fmt.Errorf( "unable to find any existing device filters attached to the cgroup: %w", err, ) } - - // Generate a new set of eBPF programs by prepending instructions for the - // new devices to the instructions of each existing program. - // If no existing programs found, create a new program with just our device filter. - var newProgs []*ebpf.Program - defer func() { - for _, p := range newProgs { - p.Close() - } - }() - - if len(oldProgs) == 0 { - oldInsts := asm.Instructions{asm.Return()} - - var newProg *ebpf.Program - - newProg, err = generateNewProgram(rules, oldInsts) - if err != nil { - return fmt.Errorf( - "unable to generate new device filter program with no existing programs: %w", - err, - ) - } - - newProgs = append(newProgs, newProg) + defer closeAll(oldProgs) + + switch { + case inaccessible > 0: + return fmt.Errorf("%w: %d of %d programs", ErrFiltersInaccessible, inaccessible, total) + case total == 0: + return ErrFilterMissing + case attachFlags&unix.BPF_F_ALLOW_MULTI == 0: + return fmt.Errorf( + "%w: attach mode %s", + ErrUnsupportedAttachMode, + describeAttachFlags(attachFlags), + ) } + // Build every replacement before touching the cgroup, so a generation or + // load failure mutates nothing. + newProgs := make([]progHandle, 0, len(oldProgs)) + defer func() { closeAll(newProgs) }() + for _, oldProg := range oldProgs { var ( - oldInfo *ebpf.ProgramInfo oldInsts asm.Instructions - newProg *ebpf.Program + newProg progHandle ) - oldInfo, err = oldProg.Info() + oldInsts, err = c.ops.instructions(oldProg) if err != nil { - return fmt.Errorf( - "unable to get Info() of the original device filters program: %w", - err, - ) + return err } - oldInsts, err = oldInfo.Instructions() + newProg, err = c.generateNewProgram(rules, oldInsts) if err != nil { return fmt.Errorf( - "unable to get the instructions of the original device filters program: %w", + "unable to generate new device filter program from existing programs: %w", err, ) } - newProg, err = generateNewProgram(rules, oldInsts) + newProgs = append(newProgs, newProg) + } + + // Pairs are independent: under BPF_F_ALLOW_MULTI the verdict is the AND of + // every program, so a half-replaced set is narrower, never wider (an + // original not yet replaced masks the new grant). A failure therefore + // stops here without undoing earlier pairs; a retry completes the rest. + for idx := range oldProgs { + err = c.swap(dirFD, oldProgs[idx], newProgs[idx]) if err != nil { return fmt.Errorf( - "unable to generate new device filter program from existing programs: %w", + "replace device filter program %d of %d: %w", + idx+1, + len(oldProgs), err, ) } - - newProgs = append(newProgs, newProg) } - // Increase `ulimit -l` limit to avoid BPF_PROG_LOAD error below. - // This limit is not inherited into the container. - memlockLimit := &unix.Rlimit{ - Cur: unix.RLIM_INFINITY, - Max: unix.RLIM_INFINITY, + return nil +} + +// swap replaces oldProg with newProg on dirFD: atomically when the kernel +// supports BPF_F_REPLACE, otherwise by attaching newProg before detaching +// oldProg. If that detach fails, newProg is detached again so the pair is +// left as it was (both attached would make a later pass stack another copy). +func (c *cgroupv2) swap(dirFD int, oldProg, newProg progHandle) error { + if c.replace.usable() { + err := c.ops.attach(newProg, dirFD, unix.BPF_F_ALLOW_MULTI, oldProg) + c.replace.record(err) + + if err == nil { + return nil + } + + if c.replace.usable() { + return fmt.Errorf("unable to replace device filters program: %w", err) + } } - rlimitErr := unix.Setrlimit(unix.RLIMIT_MEMLOCK, memlockLimit) - if rlimitErr != nil { - slog.Default().Warn("setrlimit RLIMIT_MEMLOCK failed; BPF_PROG_LOAD may fail", - "err", rlimitErr) + err := c.ops.attach(newProg, dirFD, unix.BPF_F_ALLOW_MULTI, nil) + if err != nil { + return fmt.Errorf("unable to attach new device filters program: %w", err) } - // Replace the set of existing eBPF programs with the new ones. - // We don't have to worry about atomically replacing each program (i.e. by - // using BPF_F_REPLACE) because we know that the code here is always run - // strictly *before* a container begins executing. - for _, oldProg := range oldProgs { - err = DetachCgroupDeviceFilter(oldProg, dirFD) - if err != nil { - return fmt.Errorf("unable to detach original device filters program: %w", err) - } + detachErr := c.ops.detach(oldProg, dirFD) + if detachErr == nil { + return nil } - for _, newProg := range newProgs { - err = AttachCgroupDeviceFilter(newProg, dirFD) - if err != nil { - return fmt.Errorf("unable to attach new device filters program: %w", err) - } + detachErr = fmt.Errorf("unable to detach original device filters program: %w", detachErr) + + rollbackErr := c.ops.detach(newProg, dirFD) + if rollbackErr != nil { + return errors.Join( + detachErr, + fmt.Errorf("roll back new device filters program: %w", rollbackErr), + ) } - return nil + return detachErr } -func generateNewProgram(rules []DeviceRule, oldInsts asm.Instructions) (*ebpf.Program, error) { +//nolint:ireturn // progHandle hides the kernel handle so the swap logic can be tested with fakes +func (c *cgroupv2) generateNewProgram( + rules []DeviceRule, + oldInsts asm.Instructions, +) (progHandle, error) { // Prepend instructions for the new devices to the original set of instructions. newInsts, err := PrependDeviceFilter(rules, oldInsts) if err != nil { @@ -280,10 +302,11 @@ func generateNewProgram(rules []DeviceRule, oldInsts asm.Instructions) (*ebpf.Pr License: bpfProgramLicense, } - newProg, err := ebpf.NewProgram(spec) - if err != nil { - return nil, fmt.Errorf("unable to create new device filters program: %w", err) - } + return c.ops.load(spec) +} - return newProg, nil +func closeAll(progs []progHandle) { + for _, prog := range progs { + prog.Close() + } } diff --git a/internal/cgroup/v2ops.go b/internal/cgroup/v2ops.go new file mode 100644 index 0000000..603a659 --- /dev/null +++ b/internal/cgroup/v2ops.go @@ -0,0 +1,244 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cgroup + +import ( + "errors" + "fmt" + "strings" + "sync/atomic" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/asm" + "github.com/cilium/ebpf/link" + "golang.org/x/sys/unix" +) + +var ( + // ErrFiltersInaccessible reports that at least one device filter attached + // to the cgroup could not be opened (typically hidden by an LSM policy). + // Nothing is attached or detached: such a program cannot be inspected, so + // neither its ownership nor the outcome of a replacement can be known. + // The condition can clear (the program may be detached), so it is + // retryable. + ErrFiltersInaccessible = errors.New("device filters attached to the cgroup cannot be accessed") + + // ErrUnsupportedAttachMode reports that the cgroup's device filter is + // attached without BPF_F_ALLOW_MULTI. Adding a program would then fail + // with EBUSY, and replacing it would require detaching the runtime's + // filter first, leaving the container unfiltered; the container is skipped. + ErrUnsupportedAttachMode = errors.New("device filter attached without BPF_F_ALLOW_MULTI") + + // ErrFilterMissing reports that no device filter is attached to the + // cgroup, so there is no runtime program to add grants to. + ErrFilterMissing = errors.New("no device filter attached to the cgroup") +) + +// progHandle is an open BPF program. The real implementation wraps an +// *ebpf.Program; tests use fakes, since a real handle needs the kernel. +type progHandle interface { + Close() error +} + +// v2ops is the kernel surface cgroupv2 uses. It exists so the ordering of +// attach, detach and close can be tested without privileges. +type v2ops interface { + // query returns the programs attached to dirFD for BPF_CGROUP_DEVICE: the + // handles that could be opened, the kernel's total count, the number that + // could not be opened and the attach flags. + query(dirFD int) (progs []progHandle, total, inaccessible int, attachFlags uint32, err error) + load(spec *ebpf.ProgramSpec) (progHandle, error) + // attach attaches prog with flags; a non-nil replace makes it an atomic + // replacement of that program. + attach(prog progHandle, dirFD int, flags uint32, replace progHandle) error + detach(prog progHandle, dirFD int) error + instructions(prog progHandle) (asm.Instructions, error) +} + +// kernelProg adapts *ebpf.Program to progHandle. +type kernelProg struct { + *ebpf.Program +} + +var errForeignHandle = errors.New("program handle was not opened by the kernel ops") + +// kernelProgram unwraps a handle produced by kernelOps. +func kernelProgram(handle progHandle) (*ebpf.Program, error) { + prog, ok := handle.(kernelProg) + if !ok { + return nil, fmt.Errorf("%w: %T", errForeignHandle, handle) + } + + return prog.Program, nil +} + +// kernelOps is the production v2ops. +type kernelOps struct{} + +var _ v2ops = kernelOps{} + +func (kernelOps) query(dirFD int) ([]progHandle, int, int, uint32, error) { + progs, total, inaccessible, flags, err := FindAttachedCgroupDeviceFilters(dirFD) + if err != nil { + return nil, 0, 0, 0, err + } + + handles := make([]progHandle, 0, len(progs)) + for _, prog := range progs { + handles = append(handles, kernelProg{prog}) + } + + return handles, total, inaccessible, flags, nil +} + +//nolint:ireturn // progHandle hides the kernel handle so the swap logic can be tested with fakes +func (kernelOps) load(spec *ebpf.ProgramSpec) (progHandle, error) { + prog, err := ebpf.NewProgram(spec) + if err != nil { + return nil, fmt.Errorf("unable to create new device filters program: %w", err) + } + + return kernelProg{prog}, nil +} + +func (kernelOps) attach(prog progHandle, dirFD int, flags uint32, replace progHandle) error { + newProg, err := kernelProgram(prog) + if err != nil { + return err + } + + opts := link.RawAttachProgramOptions{ + Target: dirFD, + Program: newProg, + Attach: ebpf.AttachCGroupDevice, + // Never BPF_F_REPLACE here: cilium rejects anchor flags in Flags and + // sets it itself from the ReplaceProgram anchor. + Flags: flags, + } + + if replace != nil { + var oldProg *ebpf.Program + + oldProg, err = kernelProgram(replace) + if err != nil { + return err + } + + opts.Anchor = link.ReplaceProgram(oldProg) + } + + err = link.RawAttachProgram(opts) + if err != nil { + return fmt.Errorf( + "failed to call BPF_PROG_ATTACH (BPF_CGROUP_DEVICE, flags %s): %w", + describeAttachFlags(flags), + err, + ) + } + + return nil +} + +func (kernelOps) detach(prog progHandle, dirFD int) error { + kprog, err := kernelProgram(prog) + if err != nil { + return err + } + + return DetachCgroupDeviceFilter(kprog, dirFD) +} + +func (kernelOps) instructions(prog progHandle) (asm.Instructions, error) { + kprog, err := kernelProgram(prog) + if err != nil { + return nil, err + } + + info, err := kprog.Info() + if err != nil { + return nil, fmt.Errorf( + "unable to get Info() of the original device filters program: %w", + err, + ) + } + + insts, err := info.Instructions() + if err != nil { + return nil, fmt.Errorf( + "unable to get the instructions of the original device filters program: %w", + err, + ) + } + + return insts, nil +} + +// Replace-support states for replaceProbe. +const ( + replaceUnknown int32 = iota + replaceSupported + replaceUnsupported +) + +// replaceProbe caches whether the kernel supports atomic BPF_F_REPLACE for +// cgroup programs (Linux 5.5+). It is probed on first use rather than +// hard-coded from a kernel version: the first replace attempt either succeeds +// or fails with EINVAL (unknown flag on older kernels), and the answer is +// kept for the life of the process. Degrading to attach-then-detach on a +// kernel that could have replaced is safe, only less tidy, so a stray EINVAL +// cannot make anything wider. +type replaceProbe struct { + state atomic.Int32 +} + +// processReplaceProbe is shared by every cgroupv2 instance: cgroup.New +// builds a fresh instance per call, and the kernel's answer does not change. +var processReplaceProbe = &replaceProbe{} + +func (r *replaceProbe) usable() bool { + return r.state.Load() != replaceUnsupported +} + +func (r *replaceProbe) record(err error) { + switch { + case err == nil: + r.state.CompareAndSwap(replaceUnknown, replaceSupported) + case errors.Is(err, unix.EINVAL): + r.state.CompareAndSwap(replaceUnknown, replaceUnsupported) + } +} + +// describeAttachFlags names cgroup attach flags for logs and errors. +func describeAttachFlags(flags uint32) string { + var names []string + + if flags&unix.BPF_F_ALLOW_OVERRIDE != 0 { + names = append(names, "BPF_F_ALLOW_OVERRIDE") + } + + if flags&unix.BPF_F_ALLOW_MULTI != 0 { + names = append(names, "BPF_F_ALLOW_MULTI") + } + + if len(names) == 0 { + return "none (exclusive)" + } + + return strings.Join(names, "|") +} diff --git a/internal/cgroup/v2swap_test.go b/internal/cgroup/v2swap_test.go new file mode 100644 index 0000000..e9f3f48 --- /dev/null +++ b/internal/cgroup/v2swap_test.go @@ -0,0 +1,430 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cgroup + +import ( + "errors" + "fmt" + "slices" + "testing" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/asm" + "golang.org/x/sys/unix" +) + +var ( + errInjected = errors.New("injected failure") + errRollback = errors.New("injected rollback failure") + errLoadFails = errors.New("injected load failure") + errBadFlags = errors.New("unexpected attach flags") +) + +// fakeProg is a progHandle naming the program it refers to. Several handles +// may refer to the same program (every query opens fresh ones). +type fakeProg struct { + name string + closed bool +} + +func (p *fakeProg) Close() error { + p.closed = true + + return nil +} + +// fakeOps models one cgroup's BPF_CGROUP_DEVICE attachment list. errs maps a +// call description ("attach N1", "replace O1->N1", "detach O1", "load N2") to +// the error that call returns; a failing call changes nothing. +type fakeOps struct { + attached []string + flags uint32 + inaccessible int + errs map[string]error + + calls []string + opened []*fakeProg + loads int +} + +func newFakeOps(attached ...string) *fakeOps { + return &fakeOps{attached: attached, flags: unix.BPF_F_ALLOW_MULTI, errs: map[string]error{}} +} + +func (f *fakeOps) open(name string) *fakeProg { + prog := &fakeProg{name: name} + f.opened = append(f.opened, prog) + + return prog +} + +func (f *fakeOps) query(int) ([]progHandle, int, int, uint32, error) { + handles := make([]progHandle, 0, len(f.attached)) + for _, name := range f.attached { + handles = append(handles, f.open(name)) + } + + return handles, len(f.attached) + f.inaccessible, f.inaccessible, f.flags, nil +} + +func nameOf(handle progHandle) string { + prog, ok := handle.(*fakeProg) + if !ok { + return fmt.Sprintf("foreign %T", handle) + } + + return prog.name +} + +//nolint:ireturn // implements v2ops.load +func (f *fakeOps) load(*ebpf.ProgramSpec) (progHandle, error) { + f.loads++ + name := fmt.Sprintf("N%d", f.loads) + + call := "load " + name + f.calls = append(f.calls, call) + + err := f.errs[call] + if err != nil { + return nil, err + } + + return f.open(name), nil +} + +func (f *fakeOps) attach(prog progHandle, _ int, flags uint32, replace progHandle) error { + if flags != unix.BPF_F_ALLOW_MULTI { + return fmt.Errorf("%w: attach flags %#x, want BPF_F_ALLOW_MULTI only", errBadFlags, flags) + } + + name := nameOf(prog) + + call := "attach " + name + if replace != nil { + call = "replace " + nameOf(replace) + "->" + name + } + + f.calls = append(f.calls, call) + + err := f.errs[call] + if err != nil { + return err + } + + if replace == nil { + f.attached = append(f.attached, name) + + return nil + } + + idx := slices.Index(f.attached, nameOf(replace)) + if idx < 0 { + return unix.ENOENT + } + + f.attached[idx] = name + + return nil +} + +func (f *fakeOps) detach(prog progHandle, _ int) error { + name := nameOf(prog) + + call := "detach " + name + f.calls = append(f.calls, call) + + err := f.errs[call] + if err != nil { + return err + } + + f.attached = slices.DeleteFunc(f.attached, func(n string) bool { return n == name }) + + return nil +} + +func (f *fakeOps) instructions(progHandle) (asm.Instructions, error) { + return asm.Instructions{asm.Mov.Imm32(asm.R0, 0), asm.Return()}, nil +} + +func (f *fakeOps) assertAllClosed(t *testing.T) { + t.Helper() + + for _, prog := range f.opened { + if !prog.closed { + t.Errorf("handle %s left open", prog.name) + } + } +} + +func testRules() []DeviceRule { + major, minor := int64(10), int64(200) + + return []DeviceRule{{Allow: true, Type: "c", Major: &major, Minor: &minor, Access: "rwm"}} +} + +func probeIn(state int32) *replaceProbe { + probe := &replaceProbe{} + probe.state.Store(state) + + return probe +} + +func run(ops *fakeOps, probe *replaceProbe) error { + c := &cgroupv2{ops: ops, replace: probe} + + return c.addDeviceRules(3, testRules()) +} + +func assertCalls(t *testing.T, ops *fakeOps, want ...string) { + t.Helper() + + if !slices.Equal(ops.calls, want) { + t.Errorf("calls = %q, want %q", ops.calls, want) + } +} + +func assertAttached(t *testing.T, ops *fakeOps, want ...string) { + t.Helper() + + if !slices.Equal(ops.attached, want) { + t.Errorf("attached = %q, want %q", ops.attached, want) + } +} + +func TestSwap_UsesAtomicReplaceWhenSupported(t *testing.T) { + for _, state := range []int32{replaceUnknown, replaceSupported} { + ops := newFakeOps("O1", "O2") + probe := probeIn(state) + + err := run(ops, probe) + if err != nil { + t.Fatalf("state %d: %v", state, err) + } + + assertCalls(t, ops, "load N1", "load N2", "replace O1->N1", "replace O2->N2") + assertAttached(t, ops, "N1", "N2") + ops.assertAllClosed(t) + + if probe.state.Load() != replaceSupported { + t.Errorf("probe state = %d, want supported", probe.state.Load()) + } + } +} + +func TestSwap_AttachBeforeDetachWhenReplaceUnsupported(t *testing.T) { + ops := newFakeOps("O1", "O2") + + err := run(ops, probeIn(replaceUnsupported)) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops, "load N1", "load N2", "attach N1", "detach O1", "attach N2", "detach O2") + assertAttached(t, ops, "N1", "N2") + ops.assertAllClosed(t) +} + +// TestSwap_EINVALOnFirstReplaceIsCachedAsUnsupported covers the probe: an +// EINVAL on the first replace switches to attach-then-detach for this and +// every later pair, without retrying the replace. +func TestSwap_EINVALOnFirstReplaceIsCachedAsUnsupported(t *testing.T) { + ops := newFakeOps("O1", "O2") + ops.errs["replace O1->N1"] = unix.EINVAL + probe := probeIn(replaceUnknown) + + err := run(ops, probe) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops, + "load N1", "load N2", "replace O1->N1", "attach N1", "detach O1", "attach N2", "detach O2") + assertAttached(t, ops, "N1", "N2") + ops.assertAllClosed(t) + + if probe.state.Load() != replaceUnsupported { + t.Errorf("probe state = %d, want unsupported", probe.state.Load()) + } +} + +// TestSwap_ReplaceFailureOnSupportingKernelIsReturned checks that a replace +// failure other than the probe's EINVAL is an error, not a fallback. +func TestSwap_ReplaceFailureOnSupportingKernelIsReturned(t *testing.T) { + ops := newFakeOps("O1") + ops.errs["replace O1->N1"] = errInjected + + err := run(ops, probeIn(replaceSupported)) + if !errors.Is(err, errInjected) { + t.Fatalf("err = %v, want injected", err) + } + + assertCalls(t, ops, "load N1", "replace O1->N1") + assertAttached(t, ops, "O1") + ops.assertAllClosed(t) +} + +func TestSwap_FirstAttachFailureDetachesNothing(t *testing.T) { + ops := newFakeOps("O1", "O2") + ops.errs["attach N1"] = errInjected + + err := run(ops, probeIn(replaceUnsupported)) + if !errors.Is(err, errInjected) { + t.Fatalf("err = %v, want injected", err) + } + + assertCalls(t, ops, "load N1", "load N2", "attach N1") + assertAttached(t, ops, "O1", "O2") + ops.assertAllClosed(t) +} + +func TestSwap_LaterAttachFailureLeavesEarlierPairReplaced(t *testing.T) { + ops := newFakeOps("O1", "O2") + ops.errs["attach N2"] = errInjected + + err := run(ops, probeIn(replaceUnsupported)) + if !errors.Is(err, errInjected) { + t.Fatalf("err = %v, want injected", err) + } + + assertCalls(t, ops, "load N1", "load N2", "attach N1", "detach O1", "attach N2") + assertAttached(t, ops, "O2", "N1") + ops.assertAllClosed(t) +} + +func TestSwap_DetachFailureRollsBackThatPair(t *testing.T) { + ops := newFakeOps("O1") + ops.errs["detach O1"] = errInjected + + err := run(ops, probeIn(replaceUnsupported)) + if !errors.Is(err, errInjected) { + t.Fatalf("err = %v, want injected", err) + } + + assertCalls(t, ops, "load N1", "attach N1", "detach O1", "detach N1") + assertAttached(t, ops, "O1") + ops.assertAllClosed(t) +} + +func TestSwap_RollbackFailureIsJoined(t *testing.T) { + ops := newFakeOps("O1") + ops.errs["detach O1"] = errInjected + ops.errs["detach N1"] = errRollback + + err := run(ops, probeIn(replaceUnsupported)) + if !errors.Is(err, errInjected) || !errors.Is(err, errRollback) { + t.Fatalf("err = %v, want both the detach and the rollback error", err) + } + + assertCalls(t, ops, "load N1", "attach N1", "detach O1", "detach N1") + assertAttached(t, ops, "O1", "N1") + ops.assertAllClosed(t) +} + +func TestSwap_SecondDetachFailureLeavesPairOneReplacedPairTwoRolledBack(t *testing.T) { + ops := newFakeOps("O1", "O2") + ops.errs["detach O2"] = errInjected + + err := run(ops, probeIn(replaceUnsupported)) + if !errors.Is(err, errInjected) { + t.Fatalf("err = %v, want injected", err) + } + + assertCalls(t, ops, + "load N1", "load N2", "attach N1", "detach O1", "attach N2", "detach O2", "detach N2") + assertAttached(t, ops, "O2", "N1") + ops.assertAllClosed(t) +} + +func TestSwap_NonMultiAttachModeIsRefused(t *testing.T) { + for _, flags := range []uint32{0, unix.BPF_F_ALLOW_OVERRIDE} { + ops := newFakeOps("O1") + ops.flags = flags + + err := run(ops, probeIn(replaceUnknown)) + if !errors.Is(err, ErrUnsupportedAttachMode) { + t.Fatalf("flags %#x: err = %v, want ErrUnsupportedAttachMode", flags, err) + } + + assertCalls(t, ops) + assertAttached(t, ops, "O1") + ops.assertAllClosed(t) + } +} + +func TestSwap_AnyInaccessibleProgramRefusesEverything(t *testing.T) { + ops := newFakeOps("O1") + ops.inaccessible = 1 + + err := run(ops, probeIn(replaceUnknown)) + if !errors.Is(err, ErrFiltersInaccessible) { + t.Fatalf("err = %v, want ErrFiltersInaccessible", err) + } + + assertCalls(t, ops) + assertAttached(t, ops, "O1") + ops.assertAllClosed(t) +} + +// TestSwap_OnlyInaccessibleProgramsIsNotMissing checks that "no handles" +// is not read as "nothing attached" when the kernel counted programs. +func TestSwap_OnlyInaccessibleProgramsIsNotMissing(t *testing.T) { + ops := newFakeOps() + ops.inaccessible = 2 + + err := run(ops, probeIn(replaceUnknown)) + if !errors.Is(err, ErrFiltersInaccessible) || errors.Is(err, ErrFilterMissing) { + t.Fatalf("err = %v, want ErrFiltersInaccessible only", err) + } + + assertCalls(t, ops) +} + +func TestSwap_NoProgramAttachedFailsClosed(t *testing.T) { + ops := newFakeOps() + + err := run(ops, probeIn(replaceUnknown)) + if !errors.Is(err, ErrFilterMissing) { + t.Fatalf("err = %v, want ErrFilterMissing", err) + } + + assertCalls(t, ops) + assertAttached(t, ops) +} + +func TestSwap_LoadFailureMutatesNothing(t *testing.T) { + ops := newFakeOps("O1", "O2") + ops.errs["load N2"] = errLoadFails + + err := run(ops, probeIn(replaceUnknown)) + if !errors.Is(err, errLoadFails) { + t.Fatalf("err = %v, want load failure", err) + } + + assertCalls(t, ops, "load N1", "load N2") + assertAttached(t, ops, "O1", "O2") + ops.assertAllClosed(t) +} + +func TestPrependDeviceFilter_RefusesMissingOriginal(t *testing.T) { + _, err := PrependDeviceFilter(testRules(), nil) + if !errors.Is(err, errNoOriginalProgram) { + t.Fatalf("err = %v, want errNoOriginalProgram", err) + } +} diff --git a/internal/processor/applyerr_test.go b/internal/processor/applyerr_test.go new file mode 100644 index 0000000..12da6d4 --- /dev/null +++ b/internal/processor/applyerr_test.go @@ -0,0 +1,109 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processor + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/mount" + + "github.com/leinardi/swarm-device-access/internal/cgroup" + "github.com/leinardi/swarm-device-access/internal/policy" +) + +// failingCgroup is a cgroup.Interface whose mutation returns err. The path +// lookups delegate to the real v2 implementation, which reads the fake /proc. +type failingCgroup struct { + cgroup.Interface + + err error + calls int +} + +func (f *failingCgroup) AddDeviceRules(string, []cgroup.DeviceRule) error { + f.calls++ + + return f.err +} + +func devNullProcessor(t *testing.T, fake *failingCgroup) *Processor { + t.Helper() + + const pid = 70 + + real2, err := cgroup.New(2) + if err != nil { + t.Fatal(err) + } + + fake.Interface = real2 + + return &Processor{ + Inspector: &fakeInspector{result: container.InspectResponse{ + State: &container.State{Pid: pid}, + Mounts: []container.MountPoint{ + {Source: "/dev/null", Destination: "/dev/null", Type: mount.TypeBind}, + }, + }}, + Cfg: newStore(policy.ModeAll, false), + HostRoot: "/host", + ProcRoot: buildProcRoot(t, pid), + newCgroup: func(int) (cgroup.Interface, error) { return fake, nil }, + } +} + +func TestProcessContainer_UnsupportedAttachModeSkipsContainer(t *testing.T) { + buf := captureLogger(t) + fake := &failingCgroup{ + err: fmt.Errorf("%w: attach mode none (exclusive)", cgroup.ErrUnsupportedAttachMode), + } + + err := devNullProcessor(t, fake).ProcessContainer(context.Background(), "abc") + if err != nil { + t.Fatalf("unsupported attach mode must skip, not fail: %v", err) + } + + if fake.calls != 1 { + t.Fatalf("AddDeviceRules calls = %d, want 1", fake.calls) + } + + out := buf.String() + if !strings.Contains(out, "level=ERROR") || + !strings.Contains(out, "attach mode none (exclusive)") { + t.Errorf("expected ERROR naming the attach mode, got:\n%s", out) + } +} + +func TestProcessContainer_InaccessibleFiltersIsRetryableError(t *testing.T) { + fake := &failingCgroup{err: fmt.Errorf("%w: 1 of 2 programs", cgroup.ErrFiltersInaccessible)} + + err := devNullProcessor(t, fake).ProcessContainer(context.Background(), "abc") + if !errors.Is(err, cgroup.ErrFiltersInaccessible) { + t.Fatalf("err = %v, want ErrFiltersInaccessible", err) + } + + if !strings.Contains(err.Error(), "filters_inaccessible") { + t.Errorf("error %q does not carry the reason", err) + } +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 789ef6b..2929511 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -20,6 +20,7 @@ package processor import ( "context" + "errors" "fmt" "path/filepath" "sync" @@ -84,6 +85,10 @@ type Processor struct { // CallTimeout. mu sync.Mutex + // newCgroup builds the cgroup API for a version; nil means cgroup.New. + // Tests replace it to observe or fail the mutation without a kernel. + newCgroup func(version int) (cgroup.Interface, error) + // afterCompute, when set, runs under mu after the rules are computed // and before they are applied. Tests use it to hold a worker mid-flight. afterCompute func() @@ -165,7 +170,12 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er log.Debug("cgroup version detected", "pid", pid, "version", cgroupVersion) - api, apiErr := cgroup.New(cgroupVersion) + newCgroup := p.newCgroup + if newCgroup == nil { + newCgroup = cgroup.New + } + + api, apiErr := newCgroup(cgroupVersion) if apiErr != nil { return fmt.Errorf("init cgroup api (version=%d): %w", cgroupVersion, apiErr) } @@ -201,6 +211,26 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er if len(collected.granted) > 0 { applyErr := p.applyRulesToCgroup(api, collected.granted, cgroupPath, pid, cfg.DryRun) + if errors.Is(applyErr, cgroup.ErrUnsupportedAttachMode) { + // Permanent for this container: the runtime chose the attach mode. + // Nothing was attached or detached, so it keeps only the runtime's + // own device filter. + log.Error("container skipped: device filter attach mode not supported; "+ + "the runtime must attach device filters with BPF_F_ALLOW_MULTI", + "id", containerID, "cgroup", cgroupPath, "err", applyErr) + p.Metrics.RecordContainerSkipped("unsupported_attach_mode") + + return nil + } + + if errors.Is(applyErr, cgroup.ErrFiltersInaccessible) { + return fmt.Errorf( + "container %q (reason filters_inaccessible, retryable): %w", + containerID, + applyErr, + ) + } + if applyErr != nil { return applyErr } From a96fadd7952ac9e50e71e3f87ee08602a66504f3 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 09:39:03 +0200 Subject: [PATCH 05/37] feat(cgroup): set device rules through a cgroup handle with a cgroup v1 ledger AddDeviceRules(path, rules) becomes SetDeviceRules(handle, rules): make the cgroup's daemon-owned grants equal exactly rules. The processor opens the cgroup directory once (O_DIRECTORY|O_RDONLY|O_CLOEXEC) and records its path and inode from Fstat on that descriptor; v2 queries and attaches against it and v1 reaches devices.list, devices.allow and devices.deny with openat, so a cgroup recreated at the same path cannot receive the old container's rules. cgroup v1 gets a baseline-aware ledger, owned by the Processor and keyed by path plus inode. Set computes the runtime baseline as devices.list minus the owned bits, owns only desired bits outside it, revokes before it grants (stopping on a failed deny, so A to B never ends as A plus B), updates the ledger after every successful write, restores owned bits removed externally, and re-reads devices.list to report drift as a retryable error. Baseline bits are never denied and redundant grants never recorded. cgroup v2 keeps its grant-only wrapping behind the new signature for now. README and SECURITY.md document the v1-only gap: grants made by a previous daemon instance look like baseline after a restart and are not revoked. --- README.md | 7 + SECURITY.md | 9 +- internal/cgroup/api.go | 22 +- internal/cgroup/api_test.go | 32 +- internal/cgroup/handle.go | 78 ++++ internal/cgroup/ledger.go | 74 ++++ internal/cgroup/v1.go | 47 --- internal/cgroup/v1set.go | 444 ++++++++++++++++++++ internal/cgroup/v1set_test.go | 601 +++++++++++++++++++++++++++ internal/cgroup/v2.go | 16 +- internal/processor/applyerr_test.go | 63 ++- internal/processor/processor.go | 30 +- internal/processor/processor_test.go | 2 +- test/integration/daemon_test.go | 2 +- 14 files changed, 1340 insertions(+), 87 deletions(-) create mode 100644 internal/cgroup/handle.go create mode 100644 internal/cgroup/ledger.go create mode 100644 internal/cgroup/v1set.go create mode 100644 internal/cgroup/v1set_test.go diff --git a/README.md b/README.md index 1970bf3..ad71bfb 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,13 @@ Atomic replacement (`BPF_F_REPLACE`) is used when the kernel supports it before the original is detached, which is narrower, never wider, if it is interrupted. +On cgroup v1 hosts the daemon writes `devices.allow` and, to take a grant back, +`devices.deny`. It remembers in memory which exceptions it added itself, so it +never revokes one the runtime made. A restarted daemon has lost that memory: +exceptions granted by the previous instance are then indistinguishable from the +runtime's and are not revoked when policy narrows, until the container is +restarted. cgroup v2 hosts do not have this gap. + ### Docker Compose for Swarm ```yaml diff --git a/SECURITY.md b/SECURITY.md index a3b6c89..9a605d7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,7 +6,7 @@ - Connects to the host Docker socket (`/var/run/docker.sock`) and subscribes to container events. - Reads `/proc//{cgroup,mountinfo}` for every starting container. -- Writes eBPF `BPF_CGROUP_DEVICE` programs into the host cgroup v2 hierarchy, or writes to `devices.allow` on cgroup v1 hosts. +- Writes eBPF `BPF_CGROUP_DEVICE` programs into the host cgroup v2 hierarchy, or writes to `devices.allow` and `devices.deny` on cgroup v1 hosts. - Requires `privileged: true`, `cgroupns: host`, `pid: host`, `userns: host`, and bind mounts of the host `/sys` and `/dev`. The combination of `privileged: true`, host `/dev` bind mount, and host Docker @@ -51,4 +51,11 @@ Expected response time: acknowledgment within 7 days, patch or mitigation plan w - The daemon must run as root. Dropping to a minimal capability set (`CAP_BPF`, `CAP_PERFMON`, `CAP_SYS_ADMIN`, `CAP_SYS_RESOURCE`) is theoretically possible but has not been tested across kernel versions and is not supported at this time. - The daemon uses `privileged: true` in the reference Swarm deployment. This is required because Swarm rejects `--cap-add` on service definitions. In non-Swarm deployments, explicit capabilities can be used instead. +- **cgroup v1: grants made by a previous daemon instance are not revoked.** On cgroup v1 the daemon + can only tell its own `devices.allow` exceptions from the runtime's through an in-memory ledger. A + restarted daemon starts with an empty ledger, so exceptions its predecessor added look like the + runtime's baseline and are never revoked by a later narrowing; they go away when the container is + restarted. Within one daemon run, narrowing revokes exactly what was granted and never touches the + runtime's own exceptions. cgroup v1 is deprecated by Docker and systemd; cgroup v2 hosts are not + affected by this gap. - The cgroup BPF code in `internal/cgroup/` is derived from [NVIDIA's container toolkit](https://github.com/NVIDIA/libnvidia-container) (Apache 2.0). Security issues in that code should also be reported to NVIDIA. diff --git a/internal/cgroup/api.go b/internal/cgroup/api.go index 48e8706..60b0906 100644 --- a/internal/cgroup/api.go +++ b/internal/cgroup/api.go @@ -43,14 +43,25 @@ type DeviceRule struct { type Interface interface { GetDeviceCGroupMountPath(procRootPath string, pid int) (string, string, error) GetDeviceCGroupRootPath(procRootPath string, prefix string, pid int) (string, error) - AddDeviceRules(cgroupPath string, devices []DeviceRule) error + // SetDeviceRules makes the daemon-owned device grants of the cgroup + // behind handle equal exactly devices. Every read and write goes through + // the handle's descriptor, never the path. + SetDeviceRules(handle *CgroupHandle, devices []DeviceRule) error } +// New returns the cgroup API for version. ledger records the cgroup v1 +// grants this daemon made; it must outlive the call (one per Processor) +// and is required for version 1. +// //nolint:ireturn // intentional: callers use the interface -func New(version int) (Interface, error) { +func New(version int, ledger *Ledger) (Interface, error) { switch version { case 1: - return &cgroupv1{}, nil + if ledger == nil { + return nil, errMissingLedger + } + + return &cgroupv1{ledger: ledger, files: openatFiles{}}, nil case 2: return &cgroupv2{ops: kernelOps{}, replace: processReplaceProbe}, nil default: @@ -64,7 +75,10 @@ func New(version int) (Interface, error) { var errNoDeviceOrUnifiedCgroup = errors.New("no devices or unified cgroup entries found") type ( - cgroupv1 struct{} + cgroupv1 struct { + ledger *Ledger + files v1files + } cgroupv2 struct { ops v2ops replace *replaceProbe diff --git a/internal/cgroup/api_test.go b/internal/cgroup/api_test.go index 88a957a..068b888 100644 --- a/internal/cgroup/api_test.go +++ b/internal/cgroup/api_test.go @@ -20,6 +20,7 @@ package cgroup import ( "encoding/json" + "errors" "os" "strings" "testing" @@ -93,41 +94,48 @@ func TestDeviceRule_JSON_RoundTrip(t *testing.T) { } func TestNew_InvalidVersion(t *testing.T) { - _, err := New(-1) + _, err := New(-1, &Ledger{}) if err == nil { - t.Error("New(-1) must return an error") + t.Error("New(-1, &Ledger{}) must return an error") } - _, err = New(99) + _, err = New(99, &Ledger{}) if err == nil { - t.Error("New(99) must return an error") + t.Error("New(99, &Ledger{}) must return an error") } } func TestNew_ValidVersions(t *testing.T) { - v1, err := New(1) + v1, err := New(1, &Ledger{}) if err != nil { - t.Errorf("New(1) returned err: %v", err) + t.Errorf("New(1, &Ledger{}) returned err: %v", err) } if v1 == nil { - t.Error("New(1) returned nil Interface") + t.Error("New(1, &Ledger{}) returned nil Interface") } - v2, err := New(2) + v2, err := New(2, nil) if err != nil { - t.Errorf("New(2) returned err: %v", err) + t.Errorf("New(2, nil) returned err: %v", err) } if v2 == nil { - t.Error("New(2) returned nil Interface") + t.Error("New(2, nil) returned nil Interface") + } +} + +func TestNew_V1RequiresLedger(t *testing.T) { + _, err := New(1, nil) + if !errors.Is(err, errMissingLedger) { + t.Fatalf("New(1, nil) err = %v, want errMissingLedger", err) } } func TestNew_InvalidVersionIncludesValue(t *testing.T) { - _, err := New(42) + _, err := New(42, &Ledger{}) if err == nil { - t.Fatal("New(42) must return an error") + t.Fatal("New(42, &Ledger{}) must return an error") } if !strings.Contains(err.Error(), "42") { diff --git a/internal/cgroup/handle.go b/internal/cgroup/handle.go new file mode 100644 index 0000000..58821c2 --- /dev/null +++ b/internal/cgroup/handle.go @@ -0,0 +1,78 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cgroup + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +// Identity names one cgroup directory: its path and the inode of the +// directory. A cgroup removed and recreated at the same path gets a new +// inode, so state keyed by Identity never carries over to the new cgroup. +type Identity struct { + Path string + Inode uint64 +} + +// CgroupHandle is an open cgroup directory. The directory is resolved by +// path exactly once, in OpenCgroup; every later read and write goes through +// the descriptor, so a container that exits and a cgroup recreated at the +// same path between resolution and mutation cannot receive the old +// container's rules (the descriptor still refers to the removed directory). +type CgroupHandle struct { + fd int + identity Identity +} + +// OpenCgroup opens the cgroup directory at path and records its identity +// from Fstat on the same descriptor. The caller must Close the handle. +func OpenCgroup(path string) (*CgroupHandle, error) { + fd, err := unix.Open(path, unix.O_DIRECTORY|unix.O_RDONLY|unix.O_CLOEXEC, 0) + if err != nil { + return nil, fmt.Errorf("open cgroup %q: %w", path, err) + } + + var stat unix.Stat_t + + err = unix.Fstat(fd, &stat) + if err != nil { + unix.Close(fd) + + return nil, fmt.Errorf("stat cgroup %q: %w", path, err) + } + + return &CgroupHandle{fd: fd, identity: Identity{Path: path, Inode: stat.Ino}}, nil +} + +// Identity returns the path and inode the handle was opened with. +func (h *CgroupHandle) Identity() Identity { + return h.identity +} + +// Close releases the descriptor. +func (h *CgroupHandle) Close() error { + err := unix.Close(h.fd) + if err != nil { + return fmt.Errorf("close cgroup %q: %w", h.identity.Path, err) + } + + return nil +} diff --git a/internal/cgroup/ledger.go b/internal/cgroup/ledger.go new file mode 100644 index 0000000..94c85f2 --- /dev/null +++ b/internal/cgroup/ledger.go @@ -0,0 +1,74 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cgroup + +import "sync" + +// Ledger records, per cgroup v1 directory, the device permission bits this +// daemon itself added to devices.allow. devices.list cannot tell a runtime +// exception from one of ours, so without the ledger a later narrowing could +// neither find what to revoke nor avoid revoking the runtime's own baseline. +// +// One Ledger lives as long as the Processor that owns it and is passed to +// every cgroupv1 instance (cgroup.New builds a fresh one per call). Entries +// are keyed by Identity, so a cgroup recreated at the same path starts with +// nothing owned. The zero value is ready to use. +type Ledger struct { + mu sync.Mutex + entries map[Identity]map[devBit]struct{} +} + +// owned returns a copy of the bits recorded for id. +func (l *Ledger) owned(id Identity) map[devBit]struct{} { + l.mu.Lock() + defer l.mu.Unlock() + + out := make(map[devBit]struct{}, len(l.entries[id])) + for bit := range l.entries[id] { + out[bit] = struct{}{} + } + + return out +} + +func (l *Ledger) add(id Identity, bit devBit) { + l.mu.Lock() + defer l.mu.Unlock() + + if l.entries == nil { + l.entries = make(map[Identity]map[devBit]struct{}) + } + + if l.entries[id] == nil { + l.entries[id] = make(map[devBit]struct{}) + } + + l.entries[id][bit] = struct{}{} +} + +func (l *Ledger) remove(id Identity, bit devBit) { + l.mu.Lock() + defer l.mu.Unlock() + + delete(l.entries[id], bit) + + if len(l.entries[id]) == 0 { + delete(l.entries, id) + } +} diff --git a/internal/cgroup/v1.go b/internal/cgroup/v1.go index 1514893..3a4b6a5 100644 --- a/internal/cgroup/v1.go +++ b/internal/cgroup/v1.go @@ -149,50 +149,3 @@ func scanProcCgroupV1(r io.Reader, path string, prefix string) (string, error) { return "", errNoDevicesCgroup } - -// AddDeviceRules adds a set of device rules for the device cgroup at cgroupPath. -func (c *cgroupv1) AddDeviceRules(cgroupPath string, rules []DeviceRule) error { - // Loop through all rules in the set of device rules and add that rule to the device. - for _, rule := range rules { - err := c.addDeviceRule(cgroupPath, &rule) - if err != nil { - return err - } - } - - return nil -} - -func (c *cgroupv1) addDeviceRule(cgroupPath string, rule *DeviceRule) error { - // Check the major/minor numbers of the device in the device rule. - if rule.Major == nil { - return errNoMajor - } - - if rule.Minor == nil { - return errNoMinor - } - - // Open the appropriate allow/deny file. - var path string - if rule.Allow { - path = filepath.Join(cgroupPath, "devices.allow") - } else { - path = filepath.Join(cgroupPath, "devices.deny") - } - - file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) - if err != nil { - return fmt.Errorf("open %q: %w", path, err) - } - defer file.Close() - - // Write the device rule into the file. - _, err = fmt.Fprintf(file, - "%s %d:%d %s", rule.Type, *rule.Major, *rule.Minor, rule.Access) - if err != nil { - return fmt.Errorf("write device rule to %q: %w", path, err) - } - - return nil -} diff --git a/internal/cgroup/v1set.go b/internal/cgroup/v1set.go new file mode 100644 index 0000000..68f1fd3 --- /dev/null +++ b/internal/cgroup/v1set.go @@ -0,0 +1,444 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cgroup + +import ( + "bufio" + "bytes" + "cmp" + "errors" + "fmt" + "io" + "os" + "slices" + "strconv" + "strings" + + "golang.org/x/sys/unix" +) + +// Access bits of a cgroup v1 device exception. +const ( + accRead byte = 1 << iota // r + accWrite // w + accMknod // m +) + +const ( + devicesList = "devices.list" + devicesAllow = "devices.allow" + devicesDeny = "devices.deny" + + // wildcardNum stands for "*" in a major or minor number. + wildcardNum = -1 +) + +var ( + // ErrDeviceRulesDrift reports that devices.list, re-read after a Set, + // does not show exactly the daemon-owned bits the Set established (an + // external writer, or a write the kernel accepted without effect). The + // ledger already reflects every successful write, so a retry recomputes + // from the file and converges. + ErrDeviceRulesDrift = errors.New("cgroup v1 device exceptions differ from the daemon-owned set") + + errMissingLedger = errors.New("cgroup v1 requires a ledger") + errDenyRule = errors.New("cgroup v1 set accepts allow rules only") + errConcreteRule = errors.New("cgroup v1 set needs a c or b rule with concrete major and minor") + errListEntry = errors.New("malformed devices.list entry") + errUnknownAccess = errors.New("unknown device access") +) + +// devBit is one access bit of one device: the unit the v1 ledger owns. +type devBit struct { + typ byte // 'c' or 'b' + major int64 + minor int64 + access byte // exactly one of accRead, accWrite, accMknod +} + +func (b devBit) line() string { + return fmt.Sprintf("%c %d:%d %s", b.typ, b.major, b.minor, accessString(b.access)) +} + +func compareBits(left, right devBit) int { + return cmp.Or( + cmp.Compare(left.typ, right.typ), + cmp.Compare(left.major, right.major), + cmp.Compare(left.minor, right.minor), + cmp.Compare(left.access, right.access), + ) +} + +// listEntry is one devices.list line; typ may be 'a' (all) and major or +// minor may be wildcardNum. access is a mask. +type listEntry struct { + typ byte + major int64 + minor int64 + access byte +} + +func (e listEntry) covers(bit devBit) bool { + return (e.typ == 'a' || e.typ == bit.typ) && + (e.major == wildcardNum || e.major == bit.major) && + (e.minor == wildcardNum || e.minor == bit.minor) && + e.access&bit.access != 0 +} + +func covered(entries []listEntry, bit devBit) bool { + return slices.ContainsFunc(entries, func(e listEntry) bool { return e.covers(bit) }) +} + +func accessString(mask byte) string { + var out strings.Builder + + if mask&accRead != 0 { + out.WriteByte('r') + } + + if mask&accWrite != 0 { + out.WriteByte('w') + } + + if mask&accMknod != 0 { + out.WriteByte('m') + } + + return out.String() +} + +func parseAccess(access string) (byte, error) { + var mask byte + + for _, r := range access { + switch r { + case 'r': + mask |= accRead + case 'w': + mask |= accWrite + case 'm': + mask |= accMknod + default: + return 0, fmt.Errorf("%w %q", errUnknownAccess, r) + } + } + + return mask, nil +} + +func parseNum(field string) (int64, error) { + if field == "*" { + return wildcardNum, nil + } + + num, err := strconv.ParseInt(field, 10, 64) + if err != nil { + return 0, fmt.Errorf("device number %q: %w", field, err) + } + + return num, nil +} + +// parseDevicesList parses devices.list ("c 1:3 rwm", "b 8:* r", "a *:* rwm"). +func parseDevicesList(data []byte) ([]listEntry, error) { + var entries []listEntry + + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + fields := strings.Fields(line) + if len(fields) != 3 || len(fields[0]) != 1 || !strings.Contains("abc", fields[0]) { + return nil, fmt.Errorf("%w: %q", errListEntry, line) + } + + numbers := strings.Split(fields[1], ":") + if len(numbers) != 2 { + return nil, fmt.Errorf("%w: %q", errListEntry, line) + } + + major, err := parseNum(numbers[0]) + if err != nil { + return nil, fmt.Errorf("%w: %q: %w", errListEntry, line, err) + } + + minor, err := parseNum(numbers[1]) + if err != nil { + return nil, fmt.Errorf("%w: %q: %w", errListEntry, line, err) + } + + access, err := parseAccess(fields[2]) + if err != nil { + return nil, fmt.Errorf("%w: %q: %w", errListEntry, line, err) + } + + entries = append( + entries, + listEntry{typ: fields[0][0], major: major, minor: minor, access: access}, + ) + } + + err := scanner.Err() + if err != nil { + return nil, fmt.Errorf("read devices.list: %w", err) + } + + return entries, nil +} + +// withoutOwned returns entries with the owned bits removed from the exact +// (non-wildcard) entries they appear in. devices.list merges the runtime's +// access bits and ours into one line per device, so this is what separates +// the runtime's baseline from the daemon's grants. Wildcard entries are +// always baseline: the daemon never writes one. +func withoutOwned(entries []listEntry, owned map[devBit]struct{}) []listEntry { + out := make([]listEntry, 0, len(entries)) + + for _, entry := range entries { + if entry.typ != 'a' && entry.major != wildcardNum && entry.minor != wildcardNum { + for bit := range owned { + if bit.typ == entry.typ && bit.major == entry.major && bit.minor == entry.minor { + entry.access &^= bit.access + } + } + } + + if entry.access != 0 { + out = append(out, entry) + } + } + + return out +} + +// expandRules turns allow rules into single-access-bit units. +func expandRules(rules []DeviceRule) (map[devBit]struct{}, error) { + bits := make(map[devBit]struct{}) + + for _, rule := range rules { + if !rule.Allow { + return nil, errDenyRule + } + + if (rule.Type != "c" && rule.Type != "b") || rule.Major == nil || rule.Minor == nil || + *rule.Major < 0 || *rule.Minor < 0 { + return nil, fmt.Errorf("%w: %+v", errConcreteRule, rule) + } + + access, err := parseAccess(rule.Access) + if err != nil { + return nil, err + } + + for _, one := range []byte{accRead, accWrite, accMknod} { + if access&one != 0 { + bits[devBit{typ: rule.Type[0], major: *rule.Major, minor: *rule.Minor, access: one}] = struct{}{} + } + } + } + + return bits, nil +} + +func sortedBits(set map[devBit]struct{}) []devBit { + out := make([]devBit, 0, len(set)) + for bit := range set { + out = append(out, bit) + } + + slices.SortFunc(out, compareBits) + + return out +} + +// v1files reads and writes the devices.* files relative to a cgroup +// directory descriptor. It is a seam for failure injection in tests. +type v1files interface { + read(dirFD int, name string) ([]byte, error) + write(dirFD int, name, line string) error +} + +// openatFiles is the production v1files: every access is openat on the +// directory descriptor, never a path lookup. +type openatFiles struct{} + +func (openatFiles) read(dirFD int, name string) ([]byte, error) { + fd, err := unix.Openat(dirFD, name, unix.O_RDONLY|unix.O_CLOEXEC, 0) + if err != nil { + return nil, fmt.Errorf("open %s: %w", name, err) + } + + file := os.NewFile(uintptr(fd), name) + defer file.Close() + + data, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("read %s: %w", name, err) + } + + return data, nil +} + +func (openatFiles) write(dirFD int, name, line string) error { + fd, err := unix.Openat(dirFD, name, unix.O_WRONLY|unix.O_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("open %s: %w", name, err) + } + + file := os.NewFile(uintptr(fd), name) + defer file.Close() + + _, err = file.WriteString(line) + if err != nil { + return fmt.Errorf("write %q to %s: %w", line, name, err) + } + + return nil +} + +func (c *cgroupv1) readList(dirFD int) ([]listEntry, error) { + data, err := c.files.read(dirFD, devicesList) + if err != nil { + return nil, err + } + + return parseDevicesList(data) +} + +// SetDeviceRules makes the daemon-owned device exceptions of the cgroup +// equal exactly rules. +// +// The runtime's own exceptions (the baseline) are what devices.list shows +// minus the bits the ledger says this daemon added. Only bits outside the +// baseline become owned, so a grant the runtime already made is never +// recorded and therefore never revoked. Revocations are written before +// grants and a failed revocation stops the call: going from A to B with a +// failed revoke of A must not end as A plus B. Every successful write +// updates the ledger at once, so a partial failure leaves it matching the +// files, and a retry recomputes from devices.list. +func (c *cgroupv1) SetDeviceRules(handle *CgroupHandle, rules []DeviceRule) error { + desired, err := expandRules(rules) + if err != nil { + return err + } + + id := handle.Identity() + + entries, err := c.readList(handle.fd) + if err != nil { + return err + } + + ownedOld := c.ledger.owned(id) + baseline := withoutOwned(entries, ownedOld) + + ownedNew := make(map[devBit]struct{}) + + for bit := range desired { + if !covered(baseline, bit) { + ownedNew[bit] = struct{}{} + } + } + + // Owned bits that the baseline now covers by itself (the runtime granted + // them since) are forgotten, not denied: denying them would take away + // the runtime's grant. + forgotten := make(map[devBit]struct{}) + + for _, bit := range sortedBits(ownedOld) { + if _, keep := ownedNew[bit]; keep { + continue + } + + if covered(baseline, bit) { + c.ledger.remove(id, bit) + forgotten[bit] = struct{}{} + + continue + } + + err = c.files.write(handle.fd, devicesDeny, bit.line()) + if err != nil { + return fmt.Errorf("revoke %q: %w", bit.line(), err) + } + + c.ledger.remove(id, bit) + } + + var errs []error + + for _, bit := range sortedBits(ownedNew) { + // Written whenever the bit is not effective now, not merely when the + // ledger lacks it: an external writer may have removed an owned bit, + // and an identical Set must restore it. + if !covered(entries, bit) { + err = c.files.write(handle.fd, devicesAllow, bit.line()) + if err != nil { + errs = append(errs, fmt.Errorf("grant %q: %w", bit.line(), err)) + + continue + } + } + + c.ledger.add(id, bit) + } + + errs = append(errs, c.verify(handle.fd, ownedOld, ownedNew, forgotten)) + + return errors.Join(errs...) +} + +// verify re-reads devices.list and checks that, among the bits the daemon +// owned before or owns now, exactly the owned-now bits are effective. +func (c *cgroupv1) verify(dirFD int, ownedOld, ownedNew, forgotten map[devBit]struct{}) error { + after, err := c.readList(dirFD) + if err != nil { + return fmt.Errorf("verify: %w", err) + } + + checked := make(map[devBit]struct{}, len(ownedOld)+len(ownedNew)) + + for bit := range ownedOld { + if _, gone := forgotten[bit]; !gone { + checked[bit] = struct{}{} + } + } + + for bit := range ownedNew { + checked[bit] = struct{}{} + } + + var mismatched []string + + for _, bit := range sortedBits(checked) { + _, want := ownedNew[bit] + if covered(after, bit) != want { + mismatched = append(mismatched, fmt.Sprintf("%s (want %t)", bit.line(), want)) + } + } + + if len(mismatched) > 0 { + return fmt.Errorf("%w: %s", ErrDeviceRulesDrift, strings.Join(mismatched, ", ")) + } + + return nil +} diff --git a/internal/cgroup/v1set_test.go b/internal/cgroup/v1set_test.go new file mode 100644 index 0000000..5dd5f97 --- /dev/null +++ b/internal/cgroup/v1set_test.go @@ -0,0 +1,601 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cgroup + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "syscall" + "testing" +) + +// devicesModel stands in for the kernel behind a cgroup v1 directory made +// of t.TempDir() files. Reads and writes go through the real openat code +// (openatFiles) on the directory descriptor; after every successful write +// the model applies cgroup v1 exception semantics (allow ORs access into the +// exact entry, deny clears it, wildcard entries are never touched by an +// exact write) and rewrites devices.list. +type devicesModel struct { + t *testing.T + dir string + entries []listEntry + + // fail maps "allow ", "deny " or "read" to an injected error. + fail map[string]error + // ignoreWrites makes the kernel accept writes without effect. + ignoreWrites bool + writes []string +} + +func newDevicesModel(t *testing.T, list string) *devicesModel { + t.Helper() + + entries, err := parseDevicesList([]byte(list)) + if err != nil { + t.Fatal(err) + } + + model := &devicesModel{t: t, dir: t.TempDir(), entries: entries, fail: map[string]error{}} + + for _, name := range []string{devicesAllow, devicesDeny} { + err = os.WriteFile(filepath.Join(model.dir, name), nil, 0o600) + if err != nil { + t.Fatal(err) + } + } + + model.sync() + + return model +} + +func formatEntry(entry listEntry) string { + num := func(n int64) string { + if n == wildcardNum { + return "*" + } + + return strconv.FormatInt(n, 10) + } + + return fmt.Sprintf( + "%c %s:%s %s", + entry.typ, + num(entry.major), + num(entry.minor), + accessString(entry.access), + ) +} + +func (m *devicesModel) sync() { + m.t.Helper() + + lines := make([]string, 0, len(m.entries)) + for _, entry := range m.entries { + lines = append(lines, formatEntry(entry)+"\n") + } + + err := os.WriteFile(filepath.Join(m.dir, devicesList), []byte(strings.Join(lines, "")), 0o600) + if err != nil { + m.t.Fatal(err) + } +} + +func (m *devicesModel) list() string { + lines := make([]string, 0, len(m.entries)) + for _, entry := range m.entries { + lines = append(lines, formatEntry(entry)) + } + + return strings.Join(lines, ", ") +} + +// apply mutates the exact entry for line, as the kernel does. +func (m *devicesModel) apply(allow bool, line string) { + m.t.Helper() + + parsed, err := parseDevicesList([]byte(line)) + if err != nil || len(parsed) != 1 { + m.t.Fatalf("bad rule line %q: %v", line, err) + } + + rule := parsed[0] + + idx := slices.IndexFunc(m.entries, func(e listEntry) bool { + return e.typ == rule.typ && e.major == rule.major && e.minor == rule.minor + }) + + switch { + case allow && idx < 0: + m.entries = append(m.entries, rule) + case allow: + m.entries[idx].access |= rule.access + case idx >= 0: + m.entries[idx].access &^= rule.access + if m.entries[idx].access == 0 { + m.entries = slices.Delete(m.entries, idx, idx+1) + } + } + + m.sync() +} + +func (m *devicesModel) read(dirFD int, name string) ([]byte, error) { + err := m.fail["read"] + if err != nil { + return nil, err + } + + return openatFiles{}.read(dirFD, name) +} + +func (m *devicesModel) write(dirFD int, name, line string) error { + verb := map[string]string{devicesAllow: "allow", devicesDeny: "deny"}[name] + call := verb + " " + line + m.writes = append(m.writes, call) + + failErr := m.fail[call] + if failErr != nil { + return failErr + } + + err := openatFiles{}.write(dirFD, name, line) + if err != nil { + return err + } + + if !m.ignoreWrites { + m.apply(name == devicesAllow, line) + } + + return nil +} + +func (m *devicesModel) writesWith(verb string) []string { + var out []string + + for _, call := range m.writes { + if strings.HasPrefix(call, verb+" ") { + out = append(out, call) + } + } + + return out +} + +// setup opens the model's directory as a real cgroup handle. +func (m *devicesModel) setup(ledger *Ledger) (*cgroupv1, *CgroupHandle) { + m.t.Helper() + + handle, err := OpenCgroup(m.dir) + if err != nil { + m.t.Fatal(err) + } + + m.t.Cleanup(func() { _ = handle.Close() }) + + return &cgroupv1{ledger: ledger, files: m}, handle +} + +func rule(typ string, major, minor int64, access string) DeviceRule { + return DeviceRule{Allow: true, Type: typ, Major: &major, Minor: &minor, Access: access} +} + +func bit(typ byte, major, minor int64, access byte) devBit { + return devBit{typ: typ, major: major, minor: minor, access: access} +} + +func ownedLines(ledger *Ledger, id Identity) []string { + owned := ledger.owned(id) + out := make([]string, 0, len(owned)) + + for _, one := range sortedBits(owned) { + out = append(out, one.line()) + } + + return out +} + +func TestCovered_Table(t *testing.T) { + cases := []struct { + list string + bit devBit + want bool + }{ + {"c 10:200 rw", bit('c', 10, 200, accRead), true}, + {"c 10:200 rw", bit('c', 10, 200, accMknod), false}, + {"c 10:200 rw", bit('c', 10, 201, accRead), false}, + {"c 10:200 rw", bit('b', 10, 200, accRead), false}, + {"a *:* rwm", bit('b', 8, 0, accWrite), true}, + {"a *:* m", bit('c', 1, 3, accRead), false}, + {"c 1:* m", bit('c', 1, 3, accMknod), true}, + {"c 1:* m", bit('c', 1, 3, accRead), false}, + {"c 1:* m", bit('c', 2, 3, accMknod), false}, + {"b 8:* r", bit('b', 8, 16, accRead), true}, + {"b 8:* r", bit('b', 8, 16, accWrite), false}, + {"c *:* m", bit('c', 99, 1, accMknod), true}, + {"c *:5 w", bit('c', 99, 5, accWrite), true}, + } + + for _, tc := range cases { + entries, err := parseDevicesList([]byte(tc.list)) + if err != nil { + t.Fatal(err) + } + + if got := covered(entries, tc.bit); got != tc.want { + t.Errorf("covered(%q, %s) = %t, want %t", tc.list, tc.bit.line(), got, tc.want) + } + } +} + +func TestParseDevicesList_RejectsMalformed(t *testing.T) { + for _, bad := range []string{"x 1:3 rwm", "c 1 rwm", "c a:3 rwm", "c 1:3 rwx", "c 1:3"} { + _, err := parseDevicesList([]byte(bad)) + if !errors.Is(err, errListEntry) { + t.Errorf("parse %q: err = %v, want errListEntry", bad, err) + } + } +} + +// TestSet_AccessSubsetAndSuperset checks that only the bits the baseline +// lacks are written and owned: a superset request over a partial baseline +// owns the difference, a subset request owns nothing. +func TestSet_AccessSubsetAndSuperset(t *testing.T) { + model := newDevicesModel(t, "c 1:* m\nc 10:200 r\n") + ledger := &Ledger{} + cg, handle := model.setup(ledger) + + err := cg.SetDeviceRules( + handle, + []DeviceRule{rule("c", 10, 200, "rw"), rule("c", 1, 3, "rwm"), rule("c", 1, 5, "m")}, + ) + if err != nil { + t.Fatal(err) + } + + want := []string{"c 1:3 r", "c 1:3 w", "c 10:200 w"} + if got := ownedLines(ledger, handle.Identity()); !slices.Equal(got, want) { + t.Errorf("owned = %q, want %q", got, want) + } + + if got := model.writesWith( + "allow", + ); !slices.Equal( + got, + []string{"allow c 1:3 r", "allow c 1:3 w", "allow c 10:200 w"}, + ) { + t.Errorf("allow writes = %q", got) + } +} + +func TestSet_GrantTwiceThenEmptyRevokesAndKeepsBaseline(t *testing.T) { + model := newDevicesModel(t, "c 10:200 m\n") + ledger := &Ledger{} + cg, handle := model.setup(ledger) + rules := []DeviceRule{rule("c", 10, 200, "rwm")} + + for range 2 { + err := cg.SetDeviceRules(handle, rules) + if err != nil { + t.Fatal(err) + } + } + + if got := model.list(); got != "c 10:200 rwm" { + t.Fatalf("after grants list = %q", got) + } + + if got := model.writesWith("allow"); len(got) != 2 { + t.Errorf("allow writes = %q, want r and w once each", got) + } + + err := cg.SetDeviceRules(handle, nil) + if err != nil { + t.Fatal(err) + } + + if got := model.list(); got != "c 10:200 m" { + t.Errorf("after Set(empty) list = %q, want baseline m only", got) + } + + if got := ownedLines(ledger, handle.Identity()); len(got) != 0 { + t.Errorf("owned after revoke = %q", got) + } + + if got := model.writesWith( + "deny", + ); !slices.Equal( + got, + []string{"deny c 10:200 r", "deny c 10:200 w"}, + ) { + t.Errorf("deny writes = %q", got) + } +} + +func TestSet_RedundantGrantIsNotRecordedOrRevoked(t *testing.T) { + model := newDevicesModel(t, "c 1:3 rwm\n") + ledger := &Ledger{} + cg, handle := model.setup(ledger) + + err := cg.SetDeviceRules(handle, []DeviceRule{rule("c", 1, 3, "rwm")}) + if err != nil { + t.Fatal(err) + } + + if got := ownedLines(ledger, handle.Identity()); len(got) != 0 { + t.Errorf("owned = %q, want nothing (runtime already grants /dev/null)", got) + } + + err = cg.SetDeviceRules(handle, nil) + if err != nil { + t.Fatal(err) + } + + if len(model.writes) != 0 { + t.Errorf("writes = %q, want none", model.writes) + } + + if got := model.list(); got != "c 1:3 rwm" { + t.Errorf("list = %q, baseline touched", got) + } +} + +// assertLedgerMatchesFiles checks that the ledger holds exactly the bits +// that are effective and not part of baselineList. +func assertLedgerMatchesFiles( + t *testing.T, + model *devicesModel, + ledger *Ledger, + id Identity, + baselineList string, +) { + t.Helper() + + baseline, err := parseDevicesList([]byte(baselineList)) + if err != nil { + t.Fatal(err) + } + + effective := make(map[devBit]struct{}) + + for _, entry := range model.entries { + for _, one := range []byte{accRead, accWrite, accMknod} { + candidate := devBit{typ: entry.typ, major: entry.major, minor: entry.minor, access: one} + if entry.access&one != 0 && !covered(baseline, candidate) { + effective[candidate] = struct{}{} + } + } + } + + effectiveOwned := make([]string, 0, len(effective)) + for _, owned := range sortedBits(effective) { + effectiveOwned = append(effectiveOwned, owned.line()) + } + + if got := ownedLines(ledger, id); !slices.Equal(got, effectiveOwned) { + t.Errorf( + "ledger %q does not match effective non-baseline bits %q (list %q)", + got, + effectiveOwned, + model.list(), + ) + } +} + +// TestSet_InjectedFailureAtEachStepKeepsLedgerConsistent runs an A to B +// transition with every write, in turn, failing, and checks that the +// ledger still describes the files after the failed call. +func TestSet_InjectedFailureAtEachStepKeepsLedgerConsistent(t *testing.T) { + const baselineList = "c 1:3 rwm\n" + + steps := []string{ + "deny c 10:200 r", "deny c 10:200 w", "deny c 10:200 m", + "allow c 10:201 r", "allow c 10:201 w", "allow c 10:201 m", + } + + for _, failing := range steps { + t.Run(failing, func(t *testing.T) { + model := newDevicesModel(t, baselineList) + ledger := &Ledger{} + cg, handle := model.setup(ledger) + + err := cg.SetDeviceRules(handle, []DeviceRule{rule("c", 10, 200, "rwm")}) + if err != nil { + t.Fatal(err) + } + + model.fail[failing] = errInjected + + err = cg.SetDeviceRules(handle, []DeviceRule{rule("c", 10, 201, "rwm")}) + if !errors.Is(err, errInjected) { + t.Fatalf("err = %v, want injected", err) + } + + assertLedgerMatchesFiles(t, model, ledger, handle.Identity(), baselineList) + + // A retry without the fault converges on B. + delete(model.fail, failing) + + err = cg.SetDeviceRules(handle, []DeviceRule{rule("c", 10, 201, "rwm")}) + if err != nil { + t.Fatalf("retry: %v", err) + } + + if got := model.list(); got != "c 1:3 rwm, c 10:201 rwm" { + t.Errorf("after retry list = %q", got) + } + + assertLedgerMatchesFiles(t, model, ledger, handle.Identity(), baselineList) + }) + } +} + +func TestSet_DenyFailureSkipsEveryGrant(t *testing.T) { + model := newDevicesModel(t, "") + ledger := &Ledger{} + cg, handle := model.setup(ledger) + + err := cg.SetDeviceRules(handle, []DeviceRule{rule("c", 10, 200, "rw")}) + if err != nil { + t.Fatal(err) + } + + model.writes = nil + model.fail["deny c 10:200 w"] = errInjected + + err = cg.SetDeviceRules(handle, []DeviceRule{rule("c", 10, 201, "rwm")}) + if !errors.Is(err, errInjected) { + t.Fatalf("err = %v, want injected", err) + } + + if got := model.writesWith("allow"); len(got) != 0 { + t.Errorf("allow writes after a failed revoke = %q, want none", got) + } + + if got := ownedLines(ledger, handle.Identity()); !slices.Equal(got, []string{"c 10:200 w"}) { + t.Errorf("owned = %q, want the bit whose deny failed", got) + } +} + +func TestSet_RestoresExternallyRemovedOwnedBit(t *testing.T) { + model := newDevicesModel(t, "") + ledger := &Ledger{} + cg, handle := model.setup(ledger) + rules := []DeviceRule{rule("c", 10, 200, "rw")} + + err := cg.SetDeviceRules(handle, rules) + if err != nil { + t.Fatal(err) + } + + model.apply(false, "c 10:200 r") + model.writes = nil + + err = cg.SetDeviceRules(handle, rules) + if err != nil { + t.Fatal(err) + } + + if !slices.Equal(model.writes, []string{"allow c 10:200 r"}) { + t.Errorf("writes = %q, want the removed bit restored", model.writes) + } + + if got := model.list(); got != "c 10:200 rw" { + t.Errorf("list = %q", got) + } +} + +func TestSet_VerifyMismatchIsReported(t *testing.T) { + model := newDevicesModel(t, "") + model.ignoreWrites = true + cg, handle := model.setup(&Ledger{}) + + err := cg.SetDeviceRules(handle, []DeviceRule{rule("c", 10, 200, "r")}) + if !errors.Is(err, ErrDeviceRulesDrift) { + t.Fatalf("err = %v, want ErrDeviceRulesDrift", err) + } +} + +func TestSet_RejectsRulesThatAreNotConcreteAllows(t *testing.T) { + model := newDevicesModel(t, "") + cg, handle := model.setup(&Ledger{}) + + deny := rule("c", 10, 200, "r") + deny.Allow = false + + for _, bad := range []DeviceRule{deny, rule("a", 10, 200, "r"), rule("c", -1, 200, "r"), rule("c", 10, 200, "x")} { + err := cg.SetDeviceRules(handle, []DeviceRule{bad}) + if err == nil { + t.Errorf("rule %+v accepted", bad) + } + } + + if len(model.writes) != 0 { + t.Errorf("writes = %q, want none", model.writes) + } +} + +// TestSet_LedgerIsKeyedByInode models a cgroup removed and recreated at the +// same path: the new directory (other inode) owns nothing, so an empty Set +// on it revokes nothing, while the old identity keeps its record. +func TestSet_LedgerIsKeyedByInode(t *testing.T) { + ledger := &Ledger{} + + oldModel := newDevicesModel(t, "") + oldCg, oldHandle := oldModel.setup(ledger) + + err := oldCg.SetDeviceRules(oldHandle, []DeviceRule{rule("c", 10, 200, "r")}) + if err != nil { + t.Fatal(err) + } + + newModel := newDevicesModel(t, "c 10:200 r\n") + newCg, newHandle := newModel.setup(ledger) + newHandle.identity.Path = oldHandle.Identity().Path + + if newHandle.Identity() == oldHandle.Identity() { + t.Fatal("test needs distinct inodes") + } + + err = newCg.SetDeviceRules(newHandle, nil) + if err != nil { + t.Fatal(err) + } + + if len(newModel.writes) != 0 { + t.Errorf( + "recreated cgroup got writes %q; it must start with nothing owned", + newModel.writes, + ) + } + + if got := ownedLines(ledger, oldHandle.Identity()); !slices.Equal(got, []string{"c 10:200 r"}) { + t.Errorf("old identity owned = %q", got) + } +} + +func TestOpenCgroup_RecordsInode(t *testing.T) { + dir := t.TempDir() + + handle, err := OpenCgroup(dir) + if err != nil { + t.Fatal(err) + } + defer handle.Close() + + info, err := os.Stat(dir) + if err != nil { + t.Fatal(err) + } + + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("no stat_t") + } + + if handle.Identity() != (Identity{Path: dir, Inode: stat.Ino}) { + t.Errorf("identity = %+v, want path %q inode %d", handle.Identity(), dir, stat.Ino) + } +} diff --git a/internal/cgroup/v2.go b/internal/cgroup/v2.go index 85a9bf8..57cb5a4 100644 --- a/internal/cgroup/v2.go +++ b/internal/cgroup/v2.go @@ -150,7 +150,10 @@ func scanProcCgroupV2(r io.Reader, path string, prefix string) (string, error) { return "", errNoCgroupV2Entry } -// AddDeviceRules adds a set of device rules for the device cgroup at cgroupPath. +// SetDeviceRules adds rules to the device filter of the cgroup behind handle. +// +// It is still grant-only: rules are prepended to the attached programs, and +// earlier grants are not replaced or revoked. An empty rule set is a no-op. // // Every attached program is replaced by a copy with the rules prepended. // Unlike NVIDIA's upstream code, which detaches every program before @@ -159,15 +162,12 @@ func scanProcCgroupV2(r io.Reader, path string, prefix string) (string, error) { // the container unfiltered in between, and permanently so if the attach // failed or the daemon died. Programs are therefore replaced pairwise, each // new program attached (or atomically swapped in) before its original goes. -func (c *cgroupv2) AddDeviceRules(cgroupPath string, rules []DeviceRule) error { - // Open the cgroup path. - dirFD, err := unix.Open(cgroupPath, unix.O_DIRECTORY|unix.O_RDONLY|unix.O_CLOEXEC, 0) - if err != nil { - return fmt.Errorf("unable to open the cgroup path: %w", err) +func (c *cgroupv2) SetDeviceRules(handle *CgroupHandle, rules []DeviceRule) error { + if len(rules) == 0 { + return nil } - defer unix.Close(dirFD) - return c.addDeviceRules(dirFD, rules) + return c.addDeviceRules(handle.fd, rules) } func (c *cgroupv2) addDeviceRules(dirFD int, rules []DeviceRule) error { diff --git a/internal/processor/applyerr_test.go b/internal/processor/applyerr_test.go index 12da6d4..56e6bdc 100644 --- a/internal/processor/applyerr_test.go +++ b/internal/processor/applyerr_test.go @@ -22,7 +22,10 @@ import ( "context" "errors" "fmt" + "os" + "path/filepath" "strings" + "syscall" "testing" "github.com/moby/moby/api/types/container" @@ -37,22 +40,39 @@ import ( type failingCgroup struct { cgroup.Interface - err error - calls int + err error + calls int + identity cgroup.Identity } -func (f *failingCgroup) AddDeviceRules(string, []cgroup.DeviceRule) error { +func (f *failingCgroup) SetDeviceRules(handle *cgroup.CgroupHandle, _ []cgroup.DeviceRule) error { f.calls++ + f.identity = handle.Identity() return f.err } +// hostRootWithCgroup returns a host root holding the cgroup directory that +// buildProcRoot's /proc resolves to, so the processor can open it. +func hostRootWithCgroup(t *testing.T) string { + t.Helper() + + root := t.TempDir() + + err := os.MkdirAll(filepath.Join(root, "sys", "fs", "cgroup", "docker", "testcontainer"), 0o755) + if err != nil { + t.Fatal(err) + } + + return root +} + func devNullProcessor(t *testing.T, fake *failingCgroup) *Processor { t.Helper() const pid = 70 - real2, err := cgroup.New(2) + real2, err := cgroup.New(2, nil) if err != nil { t.Fatal(err) } @@ -67,9 +87,38 @@ func devNullProcessor(t *testing.T, fake *failingCgroup) *Processor { }, }}, Cfg: newStore(policy.ModeAll, false), - HostRoot: "/host", + HostRoot: hostRootWithCgroup(t), ProcRoot: buildProcRoot(t, pid), - newCgroup: func(int) (cgroup.Interface, error) { return fake, nil }, + newCgroup: func(int, *cgroup.Ledger) (cgroup.Interface, error) { return fake, nil }, + } +} + +// TestProcessContainer_SetsRulesThroughOpenedHandle checks that the +// processor opens the resolved cgroup directory itself and hands the cgroup +// API a handle carrying that directory's inode. +func TestProcessContainer_SetsRulesThroughOpenedHandle(t *testing.T) { + fake := &failingCgroup{} + proc := devNullProcessor(t, fake) + + err := proc.ProcessContainer(context.Background(), "abc") + if err != nil { + t.Fatal(err) + } + + wantPath := filepath.Join(proc.HostRoot, "sys", "fs", "cgroup", "docker", "testcontainer") + + info, err := os.Stat(wantPath) + if err != nil { + t.Fatal(err) + } + + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("no stat_t") + } + + if fake.identity != (cgroup.Identity{Path: wantPath, Inode: stat.Ino}) { + t.Errorf("handle identity = %+v, want %s inode %d", fake.identity, wantPath, stat.Ino) } } @@ -85,7 +134,7 @@ func TestProcessContainer_UnsupportedAttachModeSkipsContainer(t *testing.T) { } if fake.calls != 1 { - t.Fatalf("AddDeviceRules calls = %d, want 1", fake.calls) + t.Fatalf("SetDeviceRules calls = %d, want 1", fake.calls) } out := buf.String() diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 2929511..5bf06a5 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -85,9 +85,13 @@ type Processor struct { // CallTimeout. mu sync.Mutex + // ledger records the cgroup v1 grants this processor made, so a later + // Set can revoke them without touching the runtime's own exceptions. + ledger cgroup.Ledger + // newCgroup builds the cgroup API for a version; nil means cgroup.New. // Tests replace it to observe or fail the mutation without a kernel. - newCgroup func(version int) (cgroup.Interface, error) + newCgroup func(version int, ledger *cgroup.Ledger) (cgroup.Interface, error) // afterCompute, when set, runs under mu after the rules are computed // and before they are applied. Tests use it to hold a worker mid-flight. @@ -175,7 +179,7 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er newCgroup = cgroup.New } - api, apiErr := newCgroup(cgroupVersion) + api, apiErr := newCgroup(cgroupVersion, &p.ledger) if apiErr != nil { return fmt.Errorf("init cgroup api (version=%d): %w", cgroupVersion, apiErr) } @@ -312,8 +316,10 @@ func collectContainerRules( return result } -// applyRulesToCgroup logs and (unless dryRun) attaches the collected device -// rules to the cgroup at cgroupPath via a single AddDeviceRules call. +// applyRulesToCgroup logs and (unless dryRun) sets the collected device rules +// on the cgroup at cgroupPath. The directory is opened once here; the cgroup +// API then works on that descriptor only, so the rules cannot land in a +// different cgroup recreated at the same path. func (p *Processor) applyRulesToCgroup( api cgroup.Interface, rules []cgroup.DeviceRule, @@ -349,9 +355,21 @@ func (p *Processor) applyRulesToCgroup( return nil } - err := api.AddDeviceRules(cgroupPath, rules) + handle, err := cgroup.OpenCgroup(cgroupPath) + if err != nil { + return fmt.Errorf("set device rules: %w", err) + } + + defer func() { + closeErr := handle.Close() + if closeErr != nil { + log.Warn("close cgroup handle", "cgroup", cgroupPath, "err", closeErr) + } + }() + + err = api.SetDeviceRules(handle, rules) if err != nil { - return fmt.Errorf("add device rules: %w", err) + return fmt.Errorf("set device rules: %w", err) } return nil diff --git a/internal/processor/processor_test.go b/internal/processor/processor_test.go index ec41152..f2afa01 100644 --- a/internal/processor/processor_test.go +++ b/internal/processor/processor_test.go @@ -305,7 +305,7 @@ func TestProcessContainer_DevMountFilterApplied(t *testing.T) { err := proc.ProcessContainer(context.Background(), "abc") if err == nil { t.Fatal( - "ProcessContainer should return error when AddDeviceRules fails on fake cgroup path", + "ProcessContainer should return error when the cgroup path cannot be opened", ) } } diff --git a/test/integration/daemon_test.go b/test/integration/daemon_test.go index 3400483..4f8187f 100644 --- a/test/integration/daemon_test.go +++ b/test/integration/daemon_test.go @@ -57,7 +57,7 @@ const ( // that bind-mounts /dev/null, and asserts the daemon grants that one device. // // No BPF syscalls or elevated privileges are required: -dry-run skips -// AddDeviceRules and logs intent instead. +// SetDeviceRules and logs intent instead. func TestDaemon_DryRun_DetectsDeviceMount(t *testing.T) { cli := requireDocker(t) ensureTestImage(t, cli) From 276b8b10fff3bf79041fe30cdd9c1a8d236ec8fb Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 09:54:00 +0200 Subject: [PATCH 06/37] feat(cgroup)!: replace and revoke cgroup v2 grants through validated owned blocks Every program the daemon attaches is now HEADER, init, rule blocks, TRAILER, original. Header and trailer are dead stores to R0 carrying a magic, a format version, a per-lineage 64-bit nonce, a 128-bit SHA-256 digest of the original and the block length. A program counts as owned only if the whole wrapper regenerates byte for byte from the rules and nonce parsed out of it; anything that starts like a header but fails is ErrOwnedBlockConflict and nothing is changed (reason owned_block_conflict, ERROR with the cgroup and the remedy: restart the container). Unmarked programs are never interpreted. SetDeviceRules strips validated wrappers back to the original before wrapping again, so grants replace each other instead of stacking, with no daemon state. Empty rules restore the bare original under an owned wrapper and leave unmarked programs alone. Wrappers are named sda_devfilter and are swapped in through the pairwise swap primitive; every load happens before any attach. A reloadability gate limits wrapping to the instruction subset device filters use (context loads at 0/4/8, mov/and/or/lsh/rsh, in-range conditional jumps and ja, exit; no calls, maps, 64-bit immediates, stores, atomics or may_goto) with map IDs known and empty on Linux 4.16+. Golden fixtures of real runc 1.5, crun 1.28 and systemd 259 device filters captured on Linux 7.0 prove they pass the gate, reload unchanged, and that the kernel's dump of the wrapper strips back to the exact original; a test-only interpreter checks the verdicts. BREAKING CHANGE: device filters outside the wrappable instruction subset (or on kernels older than 4.16) are no longer wrapped; the container is retried with reason program_not_wrappable. Upgrade note: grants attached by earlier versions carry no marker and look like the runtime's own filter, so they persist until the container restarts; restart containers processed by earlier versions. --- README.md | 21 + internal/cgroup/bpfinterp_test.go | 179 ++++++ internal/cgroup/fixture_capture_test.go | 160 +++++ internal/cgroup/fixtures_test.go | 53 ++ internal/cgroup/owned.go | 549 +++++++++++++++++ internal/cgroup/owned_test.go | 581 ++++++++++++++++++ internal/cgroup/testdata/devfilter/README.md | 39 ++ internal/cgroup/testdata/devfilter/crun.orig | 77 +++ .../cgroup/testdata/devfilter/crun.reloaded | 77 +++ .../cgroup/testdata/devfilter/crun.wrapped | 109 ++++ internal/cgroup/testdata/devfilter/runc.orig | 69 +++ .../cgroup/testdata/devfilter/runc.reloaded | 69 +++ .../cgroup/testdata/devfilter/runc.wrapped | 101 +++ .../cgroup/testdata/devfilter/systemd.orig | 59 ++ .../testdata/devfilter/systemd.reloaded | 59 ++ .../cgroup/testdata/devfilter/systemd.wrapped | 91 +++ internal/cgroup/v2.go | 188 ++++-- internal/cgroup/v2ops.go | 46 +- internal/cgroup/v2swap_test.go | 36 +- internal/processor/applyerr_test.go | 27 + internal/processor/processor.go | 81 ++- 21 files changed, 2590 insertions(+), 81 deletions(-) create mode 100644 internal/cgroup/bpfinterp_test.go create mode 100644 internal/cgroup/fixture_capture_test.go create mode 100644 internal/cgroup/fixtures_test.go create mode 100644 internal/cgroup/owned.go create mode 100644 internal/cgroup/owned_test.go create mode 100644 internal/cgroup/testdata/devfilter/README.md create mode 100644 internal/cgroup/testdata/devfilter/crun.orig create mode 100644 internal/cgroup/testdata/devfilter/crun.reloaded create mode 100644 internal/cgroup/testdata/devfilter/crun.wrapped create mode 100644 internal/cgroup/testdata/devfilter/runc.orig create mode 100644 internal/cgroup/testdata/devfilter/runc.reloaded create mode 100644 internal/cgroup/testdata/devfilter/runc.wrapped create mode 100644 internal/cgroup/testdata/devfilter/systemd.orig create mode 100644 internal/cgroup/testdata/devfilter/systemd.reloaded create mode 100644 internal/cgroup/testdata/devfilter/systemd.wrapped diff --git a/README.md b/README.md index ad71bfb..26ace13 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,27 @@ than removing it first, so the container is never left unfiltered. That needs: programs, and such policies are unsupported. - A device filter attached at all. A container whose cgroup has none (for example after `systemctl daemon-reload` wiped it) is not granted anything. +- A device filter made only of the instructions device filters normally use + (context loads, simple ALU, jumps inside the program, exit; no helper calls, + maps or 64-bit immediates) on Linux 4.16 or newer. The daemon rebuilds the + runtime's filter from the kernel's dump of it, and only this subset is known + to load back unchanged; the filters runc, crun and systemd attach are all + inside it. A filter outside it is left untouched and the container is + retried with reason `program_not_wrappable`. + +The daemon marks every program it attaches with a header naming it as its own +(`bpftool` shows it as `sda_devfilter`) and, on the next change, strips that +wrapper back to the runtime's original before wrapping it again. Grants +therefore replace each other instead of piling up, and survive a daemon restart +without any saved state. A program that starts like such a header but does not +check out completely is never modified: the daemon logs an error with reason +`owned_block_conflict` and the container has to be restarted. + +> **Upgrading from versions without these markers:** programs attached by +> earlier versions look exactly like the runtime's own filter, so the grants +> they carry are kept (and wrapped again) until the container is restarted. +> Restart containers that were processed by an earlier version to drop grants +> the new version cannot see. Atomic replacement (`BPF_F_REPLACE`) is used when the kernel supports it (Linux 5.5+, detected on first use); otherwise each new program is attached diff --git a/internal/cgroup/bpfinterp_test.go b/internal/cgroup/bpfinterp_test.go new file mode 100644 index 0000000..f318a94 --- /dev/null +++ b/internal/cgroup/bpfinterp_test.go @@ -0,0 +1,179 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cgroup + +import ( + "encoding/binary" + "testing" + + "github.com/cilium/ebpf/asm" + "golang.org/x/sys/unix" +) + +// devRequest is a bpf_cgroup_dev_ctx: the access a process asks for. +type devRequest struct { + typ uint32 // unix.BPF_DEVCG_DEV_CHAR or unix.BPF_DEVCG_DEV_BLOCK + access uint32 // unix.BPF_DEVCG_ACC_* mask + major uint32 + minor uint32 +} + +func charReq(major, minor uint32, access uint32) devRequest { + return devRequest{typ: unix.BPF_DEVCG_DEV_CHAR, access: access, major: major, minor: minor} +} + +func (r devRequest) ctx() []byte { + buf := make([]byte, 12) + binary.LittleEndian.PutUint32(buf[0:], r.access<<16|r.typ) + binary.LittleEndian.PutUint32(buf[4:], r.major) + binary.LittleEndian.PutUint32(buf[8:], r.minor) + + return buf +} + +// runDeviceFilter interprets the subset of eBPF that device filters use +// (context loads, 32/64-bit mov/and/or/lsh/rsh, conditional jumps, ja, +// exit) and returns the program's verdict for req: 1 allows, 0 denies. It +// exists because counting instructions cannot catch a wrong access check. +func runDeviceFilter(t *testing.T, raw []byte, req devRequest) uint64 { + t.Helper() + + insts, err := fromCanonical(raw) + if err != nil { + t.Fatal(err) + } + + ctx := req.ctx() + + var regs [11]uint64 + + for pc, steps := 0, 0; ; pc, steps = pc+1, steps+1 { + if pc < 0 || pc >= len(insts) || steps > 10*len(insts) { + t.Fatalf("program left its bounds at pc %d", pc) + } + + ins := insts[pc] + op := ins.OpCode + + //nolint:exhaustive // the interpreter covers the wrappable subset only + switch op.Class() { + case asm.LdXClass: + if ins.Src != asm.R1 { + t.Fatalf("pc %d: load from non-context register", pc) + } + + off := int(ins.Offset) + + //nolint:exhaustive // device filter context fields are at most 32 bits + switch op.Size() { + case asm.Word: + regs[ins.Dst] = uint64(binary.LittleEndian.Uint32(ctx[off:])) + case asm.Half: + regs[ins.Dst] = uint64(binary.LittleEndian.Uint16(ctx[off:])) + case asm.Byte: + regs[ins.Dst] = uint64(ctx[off]) + default: + t.Fatalf("pc %d: unsupported load size", pc) + } + + case asm.ALUClass, asm.ALU64Class: + src := uint64(ins.Constant) + if op.Source() == asm.RegSource { + src = regs[ins.Src] + } + + dst := regs[ins.Dst] + + //nolint:exhaustive // the interpreter covers the wrappable subset only + switch op.ALUOp() { + case asm.Mov: + dst = src + case asm.And: + dst &= src + case asm.Or: + dst |= src + case asm.LSh: + dst <<= src & 63 + case asm.RSh: + if op.Class() == asm.ALUClass { + dst = uint64(uint32(dst) >> (src & 31)) + } else { + dst >>= src & 63 + } + default: + t.Fatalf("pc %d: unsupported ALU op %v", pc, op) + } + + if op.Class() == asm.ALUClass { + dst = uint64(uint32(dst)) + } + + regs[ins.Dst] = dst + + case asm.JumpClass, asm.Jump32Class: + //nolint:exhaustive // conditional jumps are handled below + switch op.JumpOp() { + case asm.Exit: + return regs[asm.R0] + case asm.Ja: + pc += int(ins.Offset) + + continue + } + + left, right := regs[ins.Dst], uint64(ins.Constant) + if op.Source() == asm.RegSource { + right = regs[ins.Src] + } + + if op.Class() == asm.Jump32Class { + left, right = uint64(uint32(left)), uint64(uint32(right)) + } + + var taken bool + + //nolint:exhaustive // the interpreter covers the unsigned jumps device filters use + switch op.JumpOp() { + case asm.JEq: + taken = left == right + case asm.JNE: + taken = left != right + case asm.JGT: + taken = left > right + case asm.JGE: + taken = left >= right + case asm.JLT: + taken = left < right + case asm.JLE: + taken = left <= right + case asm.JSet: + taken = left&right != 0 + default: + t.Fatalf("pc %d: unsupported jump %v", pc, op) + } + + if taken { + pc += int(ins.Offset) + } + + default: + t.Fatalf("pc %d: unsupported instruction class %v", pc, op) + } + } +} diff --git a/internal/cgroup/fixture_capture_test.go b/internal/cgroup/fixture_capture_test.go new file mode 100644 index 0000000..04cd26e --- /dev/null +++ b/internal/cgroup/fixture_capture_test.go @@ -0,0 +1,160 @@ +//go:build linux && fixturecapture + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cgroup + +import ( + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/sys/unix" +) + +// TestCaptureFixtures records the golden fixtures under testdata/devfilter. +// It needs CAP_SYS_ADMIN/CAP_BPF and is not part of the normal test run; see +// testdata/devfilter/README.md for how to run it. For every device filter +// attached to SDA_CAPTURE_CGROUP it writes: +// +// - .orig: the translated dump, as the daemon reads it; +// - .reloaded: the dump of that original loaded back unchanged; +// - .wrapped: the dump of the wrapper around it for fixtureRules +// with fixtureNonce. +// +// Nothing is attached or detached: programs are only loaded and dumped. +func TestCaptureFixtures(t *testing.T) { + cgroupPath := os.Getenv("SDA_CAPTURE_CGROUP") + outDir := os.Getenv("SDA_CAPTURE_OUT") + name := os.Getenv("SDA_CAPTURE_NAME") + + if cgroupPath == "" || outDir == "" || name == "" { + t.Skip("set SDA_CAPTURE_CGROUP, SDA_CAPTURE_OUT and SDA_CAPTURE_NAME") + } + + handle, err := OpenCgroup(cgroupPath) + if err != nil { + t.Fatal(err) + } + defer handle.Close() + + ops := kernelOps{} + + progs, total, inaccessible, flags, err := ops.query(handle.fd) + if err != nil { + t.Fatal(err) + } + defer closeAll(progs) + + var uname unix.Utsname + + _ = unix.Uname(&uname) + t.Logf("kernel %s: %d programs (%d inaccessible), attach flags %s", + unix.ByteSliceToString(uname.Release[:]), total, inaccessible, describeAttachFlags(flags)) + + for idx, prog := range progs { + info, infoErr := prog.(kernelProg).Info() + if infoErr != nil { + t.Fatal(infoErr) + } + + insts, meta, instErr := ops.instructions(prog) + if instErr != nil { + t.Fatal(instErr) + } + + raw, rawErr := canonicalBytes(insts) + if rawErr != nil { + t.Fatal(rawErr) + } + + base := filepath.Join(outDir, fmt.Sprintf("%s-%d", name, idx)) + t.Logf( + "program %d: name %q, %d instructions, maps known %t count %d, gate: %v", + idx, + info.Name, + len(insts), + meta.mapIDsKnown, + meta.mapCount, + checkWrappable(insts, meta), + ) + t.Logf("%v", insts) + + writeFixture(t, base+".orig", raw) + writeFixture(t, base+".reloaded", loadAndDump(t, raw, "")) + + wrapped, emitErr := emitOwned(fixtureRules(), fixtureNonce, raw) + if emitErr != nil { + t.Fatal(emitErr) + } + + writeFixture(t, base+".wrapped", loadAndDump(t, wrapped, ownedProgramName)) + } +} + +func loadAndDump(t *testing.T, raw []byte, name string) []byte { + t.Helper() + + c := &cgroupv2{ops: kernelOps{}} + + handle, err := c.loadCanonical(raw, name) + if err != nil { + t.Fatal(err) + } + defer handle.Close() + + info, err := handle.(kernelProg).Info() + if err != nil { + t.Fatal(err) + } + + insts, err := info.Instructions() + if err != nil { + t.Fatal(err) + } + + dumped, err := canonicalBytes(insts) + if err != nil { + t.Fatal(err) + } + + return dumped +} + +func writeFixture(t *testing.T, path string, raw []byte) { + t.Helper() + + err := os.WriteFile(path, []byte(encodeFixture(raw)), 0o644) + if err != nil { + t.Fatal(err) + } +} + +// encodeFixture writes one 8-byte instruction per line as hex. +func encodeFixture(raw []byte) string { + var out strings.Builder + + for off := 0; off < len(raw); off += 8 { + out.WriteString(hex.EncodeToString(raw[off:min(off+8, len(raw))])) + out.WriteByte('\n') + } + + return out.String() +} diff --git a/internal/cgroup/fixtures_test.go b/internal/cgroup/fixtures_test.go new file mode 100644 index 0000000..baf9733 --- /dev/null +++ b/internal/cgroup/fixtures_test.go @@ -0,0 +1,53 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cgroup + +import ( + "encoding/hex" + "os" + "strings" + "testing" +) + +// fixtureNonce and fixtureRules are the wrapper parameters the captured +// .wrapped fixtures were produced with. +const fixtureNonce uint64 = 0x0123456789abcdef + +func fixtureRules() []DeviceRule { + return []DeviceRule{ + rule("c", 10, 200, "rwm"), + rule("b", 8, 0, "r"), + } +} + +func readFixture(t *testing.T, path string) []byte { + t.Helper() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + raw, err := hex.DecodeString(strings.Join(strings.Fields(string(data)), "")) + if err != nil { + t.Fatalf("%s: %v", path, err) + } + + return raw +} diff --git a/internal/cgroup/owned.go b/internal/cgroup/owned.go new file mode 100644 index 0000000..098d59f --- /dev/null +++ b/internal/cgroup/owned.go @@ -0,0 +1,549 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cgroup + +import ( + "bytes" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "slices" + + "github.com/cilium/ebpf/asm" + "golang.org/x/sys/unix" +) + +// An owned device filter is the runtime's original program wrapped by this +// daemon: +// +// HEADER, init, rule blocks, TRAILER, original +// +// HEADER and TRAILER are dead stores to R0 (R0 is written again before every +// exit), so they cost nothing at run time but survive in the kernel's +// translated dump. The header carries magic_lo, magic_hi, format_version, +// nonce_lo, nonce_hi, digest_0..digest_3 and block_len: a random 64-bit +// nonce per wrapper lineage, the first 128 bits of the SHA-256 of the +// original's canonical bytes, and the instruction index of the trailer. The +// trailer repeats magic_lo, magic_hi, nonce_lo and nonce_hi. +// +// Ownership is never inferred from a partial match: a program is owned only +// when the whole wrapper regenerates byte for byte from the rules and nonce +// parsed out of it and the original that follows it (which also checks the +// digest, the block length and the trailer). A program that starts like a +// header but fails that is a conflict and is never modified. +const ( + ownedMagicLo uint32 = 0x5f414453 // "SDA_" little-endian + ownedMagicHi uint32 = 0x4b4c4244 // "DBLK" little-endian + ownedFormatVersion uint32 = 1 + + ownedHeaderLen = 10 + ownedTrailerLen = 4 + ownedInitLen = 5 + + // ownedProgramName names wrappers for bpftool users. + ownedProgramName = "sda_devfilter" +) + +// Header field positions. +const ( + hdrMagicLo = iota + hdrMagicHi + hdrVersion + hdrNonceLo + hdrNonceHi + hdrDigest0 + hdrDigest1 + hdrDigest2 + hdrDigest3 + hdrBlockLen +) + +var ( + // ErrOwnedBlockConflict reports a device filter that starts like one of + // this daemon's wrappers but does not validate as one. Nothing is changed: + // acting on a partial match could strip a runtime restriction or keep a + // stale grant. Restarting the container replaces the program. + ErrOwnedBlockConflict = errors.New("device filter carries an invalid swarm-device-access block") + + // ErrProgramNotWrappable reports an original device filter outside the + // instruction subset whose translated dump can be loaded back unchanged. + ErrProgramNotWrappable = errors.New("device filter program cannot be wrapped safely") +) + +// hostByteOrder is the byte order of kernel instruction dumps. cilium only +// accepts the concrete binary.LittleEndian or binary.BigEndian values, not +// binary.NativeEndian. +var hostByteOrder = func() binary.ByteOrder { + if binary.NativeEndian.Uint16([]byte{1, 0}) == 1 { + return binary.LittleEndian + } + + return binary.BigEndian +}() + +// ownedProgram is a validated wrapper taken apart. +type ownedProgram struct { + nonce uint64 + rules []DeviceRule + original []byte // canonical bytes of the wrapped original +} + +// canonicalBytes encodes instructions as raw kernel bytes with symbols and +// other metadata stripped, the form every comparison and digest uses. +func canonicalBytes(insts asm.Instructions) ([]byte, error) { + var buf bytes.Buffer + + for idx, ins := range insts { + ins.Metadata = asm.Metadata{} + + _, err := ins.Marshal(&buf, hostByteOrder) + if err != nil { + return nil, fmt.Errorf("encode instruction %d: %w", idx, err) + } + } + + return buf.Bytes(), nil +} + +// fromCanonical decodes raw kernel bytes into instructions without symbols. +func fromCanonical(raw []byte) (asm.Instructions, error) { + insts, err := asm.AppendInstructions(nil, bytes.NewReader(raw), hostByteOrder, "linux") + if err != nil { + return nil, fmt.Errorf("decode instructions: %w", err) + } + + return insts, nil +} + +func markerInst(value uint32) asm.Instruction { + return asm.Mov.Imm32(asm.R0, int32(value)) +} + +// markerValue returns the value of a dead-store marker instruction. +func markerValue(ins asm.Instruction) (uint32, bool) { + if ins.OpCode != asm.Mov.Op32(asm.ImmSource) || ins.Dst != asm.R0 || ins.Src != asm.R0 || + ins.Offset != 0 { + return 0, false + } + + return uint32(ins.Constant), true +} + +func digestOf(original []byte) [4]uint32 { + sum := sha256.Sum256(original) + + var digest [4]uint32 + for idx := range digest { + digest[idx] = binary.LittleEndian.Uint32(sum[idx*4:]) + } + + return digest +} + +func newNonce() (uint64, error) { + var raw [8]byte + + _, err := rand.Read(raw[:]) + if err != nil { + return 0, fmt.Errorf("generate nonce: %w", err) + } + + return binary.LittleEndian.Uint64(raw[:]), nil +} + +// emitOwned builds the wrapper for rules around the canonical original. +func emitOwned(rules []DeviceRule, nonce uint64, original []byte) ([]byte, error) { + origInsts, err := fromCanonical(original) + if err != nil { + return nil, err + } + + if len(origInsts) == 0 { + return nil, errNoOriginalProgram + } + + labelPrefix, err := randomLabelPrefix() + if err != nil { + return nil, fmt.Errorf("generate label prefix: %w", err) + } + + body := &program{} + body.init() + + for _, dev := range slices.Backward(rules) { + err = body.appendDevice(dev, labelPrefix) + if err != nil { + return nil, err + } + } + + nonceLo, nonceHi := uint32(nonce), uint32(nonce>>32) + digest := digestOf(original) + blockLen := ownedHeaderLen + len(body.insts) + + header := asm.Instructions{ + markerInst(ownedMagicLo), markerInst(ownedMagicHi), markerInst(ownedFormatVersion), + markerInst(nonceLo), markerInst(nonceHi), + markerInst(digest[0]), markerInst(digest[1]), markerInst(digest[2]), markerInst(digest[3]), + markerInst(uint32(blockLen)), + } + + // The last rule block falls through to the trailer, which in turn falls + // through to the original. + fallThrough := fmt.Sprintf("%s-block-%d", labelPrefix, body.blockID) + trailer := asm.Instructions{ + markerInst(ownedMagicLo).WithSymbol(fallThrough), markerInst(ownedMagicHi), + markerInst(nonceLo), markerInst(nonceHi), + } + + all := slices.Concat(header, body.insts, trailer, origInsts) + + var buf bytes.Buffer + + err = all.Marshal(&buf, hostByteOrder) + if err != nil { + return nil, fmt.Errorf("encode wrapper: %w", err) + } + + return buf.Bytes(), nil +} + +// parseOwned takes a program's canonical bytes apart. owned is false, with no +// error, when the program does not start like a header: it is then an opaque +// original, whatever else it contains. A program that starts like a header +// but does not validate completely returns ErrOwnedBlockConflict. +func parseOwned(raw []byte) (*ownedProgram, bool, error) { + insts, err := fromCanonical(raw) + if err != nil { + return nil, false, err + } + + if len(insts) == 0 { + return nil, false, nil + } + + first, isMarker := markerValue(insts[0]) + if !isMarker || first != ownedMagicLo { + return nil, false, nil + } + + owned, err := validateOwned(insts, raw) + if err != nil { + return nil, false, fmt.Errorf("%w: %w", ErrOwnedBlockConflict, err) + } + + return owned, true, nil +} + +var ( + errHeader = errors.New("malformed header") + errBlockLen = errors.New("block length out of range") + errTrailer = errors.New("trailer missing or not matching the header") + errRuleBlock = errors.New("rule block does not parse") + errDigest = errors.New("digest does not match the wrapped original") + errRegenerated = errors.New("block does not regenerate byte for byte") +) + +func validateOwned(insts asm.Instructions, raw []byte) (*ownedProgram, error) { + if len(insts) < ownedHeaderLen { + return nil, errHeader + } + + var header [ownedHeaderLen]uint32 + + for idx := range header { + value, ok := markerValue(insts[idx]) + if !ok { + return nil, fmt.Errorf("%w: field %d", errHeader, idx) + } + + header[idx] = value + } + + if header[hdrMagicHi] != ownedMagicHi || header[hdrVersion] != ownedFormatVersion { + return nil, fmt.Errorf("%w: magic or version", errHeader) + } + + blockLen := int(header[hdrBlockLen]) + if blockLen < ownedHeaderLen+ownedInitLen || blockLen+ownedTrailerLen >= len(insts) { + return nil, fmt.Errorf("%w: %d", errBlockLen, blockLen) + } + + want := []uint32{ownedMagicLo, ownedMagicHi, header[hdrNonceLo], header[hdrNonceHi]} + for idx, value := range want { + got, ok := markerValue(insts[blockLen+idx]) + if !ok || got != value { + return nil, errTrailer + } + } + + rules, err := parseRuleBlocks(insts[ownedHeaderLen+ownedInitLen : blockLen]) + if err != nil { + return nil, err + } + + original, err := canonicalBytes(insts[blockLen+ownedTrailerLen:]) + if err != nil { + return nil, err + } + + if digestOf( + original, + ) != [4]uint32{ + header[hdrDigest0], + header[hdrDigest1], + header[hdrDigest2], + header[hdrDigest3], + } { + return nil, errDigest + } + + nonce := uint64(header[hdrNonceLo]) | uint64(header[hdrNonceHi])<<32 + + regenerated, err := emitOwned(rules, nonce, original) + if err != nil { + return nil, fmt.Errorf("%w: %w", errRegenerated, err) + } + + if !bytes.Equal(regenerated, raw) { + return nil, errRegenerated + } + + return &ownedProgram{nonce: nonce, rules: rules, original: original}, nil +} + +func isJumpImm(ins asm.Instruction, op asm.JumpOp, dst asm.Register) bool { + return ins.OpCode == op.Op(asm.ImmSource) && ins.Dst == dst +} + +// parseRuleBlocks reads back the rule blocks appendDevice emits. It only +// has to recover candidate rules: validateOwned regenerates the block from +// them and compares bytes, so anything this accepts loosely is still +// rejected unless it is exactly what the generator produces. +func parseRuleBlocks(insts asm.Instructions) ([]DeviceRule, error) { + var rules []DeviceRule + + idx := 0 + next := func() (asm.Instruction, bool) { + if idx >= len(insts) { + return asm.Instruction{}, false + } + + idx++ + + return insts[idx-1], true + } + + for idx < len(insts) { + major, minor := int64(wildcardNum), int64(wildcardNum) + rule := DeviceRule{Type: "a", Access: "rwm", Major: &major, Minor: &minor} + + if isJumpImm(insts[idx], asm.JNE, asm.R2) { + switch insts[idx].Constant { + case int64(unix.BPF_DEVCG_DEV_CHAR): + rule.Type = "c" + case int64(unix.BPF_DEVCG_DEV_BLOCK): + rule.Type = "b" + default: + return nil, fmt.Errorf("%w: device type %d", errRuleBlock, insts[idx].Constant) + } + + idx++ + } + + if idx < len(insts) && insts[idx].OpCode == asm.Mov.Op32(asm.RegSource) && + insts[idx].Dst == asm.R6 { + idx++ + + and, ok := next() + if !ok || and.OpCode != asm.And.Op32(asm.ImmSource) || and.Dst != asm.R6 { + return nil, fmt.Errorf("%w: access mask", errRuleBlock) + } + + rule.Access = bpfAccessString(and.Constant) + + _, ok = next() + if !ok { + return nil, fmt.Errorf("%w: access check", errRuleBlock) + } + } + + if idx < len(insts) && isJumpImm(insts[idx], asm.JNE, asm.R4) { + major = insts[idx].Constant + idx++ + } + + if idx < len(insts) && isJumpImm(insts[idx], asm.JNE, asm.R5) { + minor = insts[idx].Constant + idx++ + } + + verdict, ok := next() + if !ok { + return nil, fmt.Errorf("%w: missing verdict", errRuleBlock) + } + + value, isMarker := markerValue(verdict) + if !isMarker || value > 1 { + return nil, fmt.Errorf("%w: verdict", errRuleBlock) + } + + rule.Allow = value == 1 + + exit, ok := next() + if !ok || exit.OpCode != asm.Exit.Op(asm.ImmSource) { + return nil, fmt.Errorf("%w: missing exit", errRuleBlock) + } + + rules = append(rules, rule) + } + + // appendDevice is fed the rules last to first. + slices.Reverse(rules) + + return rules, nil +} + +// bpfAccessString turns a BPF_DEVCG_ACC_* mask into "rwm" letters. +func bpfAccessString(mask int64) string { + var out []byte + + if mask&unix.BPF_DEVCG_ACC_READ != 0 { + out = append(out, 'r') + } + + if mask&unix.BPF_DEVCG_ACC_WRITE != 0 { + out = append(out, 'w') + } + + if mask&unix.BPF_DEVCG_ACC_MKNOD != 0 { + out = append(out, 'm') + } + + return string(out) +} + +// progMeta is what the gate needs to know about a program besides its code. +type progMeta struct { + // mapIDsKnown reports whether the kernel told us the program's map IDs. + mapIDsKnown bool + mapCount int +} + +// checkWrappable is the reloadability gate. A translated dump is only +// guaranteed to load back as the same program when it uses no maps, calls +// or wide loads (whose kernel-side rewrites the dump does not undo), so +// wrapping is limited to the instruction set device filters actually use: +// context loads, simple ALU, conditional and unconditional jumps within the +// program, and exit. +func checkWrappable(insts asm.Instructions, meta progMeta) error { + if !meta.mapIDsKnown { + return fmt.Errorf("%w: map IDs unavailable", ErrProgramNotWrappable) + } + + if meta.mapCount > 0 { + return fmt.Errorf("%w: program uses %d maps", ErrProgramNotWrappable, meta.mapCount) + } + + if len(insts) == 0 { + return fmt.Errorf("%w: empty program", ErrProgramNotWrappable) + } + + for idx, ins := range insts { + err := checkInstruction(ins) + if err != nil { + return fmt.Errorf("%w: instruction %d (%v): %w", ErrProgramNotWrappable, idx, ins, err) + } + + if ins.OpCode.Class().IsJump() && ins.OpCode.JumpOp() != asm.Exit { + target := idx + int(ins.Offset) + 1 + if target < 0 || target >= len(insts) { + return fmt.Errorf( + "%w: instruction %d jumps out of the program", + ErrProgramNotWrappable, + idx, + ) + } + } + } + + return nil +} + +var errOutsideSubset = errors.New("outside the wrappable instruction subset") + +func checkInstruction(ins asm.Instruction) error { + opCode := ins.OpCode + + //nolint:exhaustive // an allow list: every other class is rejected by the default branch + switch opCode.Class() { + case asm.LdXClass: + if opCode.Mode() != asm.MemMode || ins.Src != asm.R1 { + return errOutsideSubset + } + + switch ins.Offset { + case 0, 4, 8: + return nil + default: + return errOutsideSubset + } + + case asm.ALUClass, asm.ALU64Class: + //nolint:exhaustive // an allow list: every other ALU op is rejected by the default branch + switch opCode.ALUOp() { + case asm.Mov, asm.And, asm.Or, asm.RSh, asm.LSh: + if ins.Offset != 0 { + // A non-zero offset turns Mov into a sign-extending move. + return errOutsideSubset + } + + return nil + default: + return errOutsideSubset + } + + case asm.JumpClass, asm.Jump32Class: + return checkJump(opCode) + + default: + return errOutsideSubset + } +} + +func checkJump(opCode asm.OpCode) error { + //nolint:exhaustive // an allow list: Call, JCOND (may_goto) and anything unknown are rejected + switch opCode.JumpOp() { + case asm.JEq, asm.JNE, asm.JGT, asm.JGE, asm.JLT, asm.JLE, asm.JSet, + asm.JSGT, asm.JSGE, asm.JSLT, asm.JSLE: + return nil + case asm.Exit, asm.Ja: + // JMP32 Exit is not an instruction, and JMP32 Ja is a long jump with + // the offset in the immediate. + if opCode.Class() != asm.JumpClass { + return errOutsideSubset + } + + return nil + default: + return errOutsideSubset + } +} diff --git a/internal/cgroup/owned_test.go b/internal/cgroup/owned_test.go new file mode 100644 index 0000000..2596020 --- /dev/null +++ b/internal/cgroup/owned_test.go @@ -0,0 +1,581 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cgroup + +import ( + "bytes" + "errors" + "path/filepath" + "slices" + "testing" + + "github.com/cilium/ebpf/asm" + "golang.org/x/sys/unix" +) + +var runtimeFixtures = []string{"runc", "crun", "systemd"} + +func mustCanonical(t *testing.T, insts asm.Instructions) []byte { + t.Helper() + + raw, err := canonicalBytes(insts) + if err != nil { + t.Fatal(err) + } + + return raw +} + +func mustEmit(t *testing.T, rules []DeviceRule, nonce uint64, original []byte) []byte { + t.Helper() + + raw, err := emitOwned(rules, nonce, original) + if err != nil { + t.Fatal(err) + } + + return raw +} + +func knownMeta() progMeta { + return progMeta{mapIDsKnown: true} +} + +// allowNullOriginal is a runtime-style original that allows exactly +// c 1:3 (any access) and denies everything else. +func allowNullOriginal() asm.Instructions { + return asm.Instructions{ + asm.LoadMem(asm.R2, asm.R1, 0, asm.Word), + asm.And.Imm32(asm.R2, 0xffff), + asm.LoadMem(asm.R4, asm.R1, 4, asm.Word), + asm.LoadMem(asm.R5, asm.R1, 8, asm.Word), + { + OpCode: asm.JNE.Op(asm.ImmSource), + Dst: asm.R2, + Offset: 4, + Constant: int64(unix.BPF_DEVCG_DEV_CHAR), + }, + {OpCode: asm.JNE.Op(asm.ImmSource), Dst: asm.R4, Offset: 3, Constant: 1}, + {OpCode: asm.JNE.Op(asm.ImmSource), Dst: asm.R5, Offset: 2, Constant: 3}, + asm.Mov.Imm32(asm.R0, 1), + asm.Return(), + asm.Mov.Imm32(asm.R0, 0), + asm.Return(), + } +} + +// TestFixtures_PassGateAndRoundTrip checks, for real runc, crun and systemd +// device filters, that the original passes the gate, reloads to the same +// bytes, is not mistaken for a wrapper, and that the kernel's dump of the +// daemon's wrapper around it strips back to exactly the original. +func TestFixtures_PassGateAndRoundTrip(t *testing.T) { + for _, name := range runtimeFixtures { + t.Run(name, func(t *testing.T) { + base := filepath.Join("testdata", "devfilter", name) + orig := readFixture(t, base+".orig") + reloaded := readFixture(t, base+".reloaded") + wrapped := readFixture(t, base+".wrapped") + + origInsts, err := fromCanonical(orig) + if err != nil { + t.Fatal(err) + } + + err = checkWrappable(origInsts, knownMeta()) + if err != nil { + t.Fatalf("gate rejects the %s original: %v", name, err) + } + + if !bytes.Equal(reloaded, orig) { + t.Fatal("reloading the original changed its dump") + } + + _, owned, err := parseOwned(orig) + if err != nil || owned { + t.Fatalf("original parsed as owned=%t err=%v, want an opaque original", owned, err) + } + + if !bytes.Equal(mustEmit(t, fixtureRules(), fixtureNonce, orig), wrapped) { + t.Fatal("kernel dump of the wrapper differs from the emitted wrapper") + } + + parsed, owned, err := parseOwned(wrapped) + if err != nil || !owned { + t.Fatalf("wrapper dump: owned=%t err=%v", owned, err) + } + + if !bytes.Equal(parsed.original, orig) { + t.Error("stripping the wrapper dump does not return the original") + } + + if parsed.nonce != fixtureNonce { + t.Errorf("nonce = %#x, want %#x", parsed.nonce, fixtureNonce) + } + }) + } +} + +func TestParseOwned_ValidatedBlockRoundTrips(t *testing.T) { + orig := mustCanonical(t, allowNullOriginal()) + rules := []DeviceRule{rule("c", 10, 200, "rw"), rule("b", 8, 0, "rwm"), rule("c", 189, 3, "m")} + + parsed, owned, err := parseOwned(mustEmit(t, rules, 42, orig)) + if err != nil || !owned { + t.Fatalf("owned=%t err=%v", owned, err) + } + + if parsed.nonce != 42 || !bytes.Equal(parsed.original, orig) { + t.Errorf( + "parsed nonce %d original equal %t", + parsed.nonce, + bytes.Equal(parsed.original, orig), + ) + } + + if !bytes.Equal(mustEmit(t, parsed.rules, 42, orig), mustEmit(t, rules, 42, orig)) { + t.Errorf("parsed rules %+v do not regenerate the block", parsed.rules) + } +} + +func TestParseOwned_AbsentOrMarkerless(t *testing.T) { + // A program that starts with the very init sequence the wrapper uses, + // but has no header, is an opaque original. + initOnly := &program{} + initOnly.init() + + cases := map[string]asm.Instructions{ + "runtime original": allowNullOriginal(), + "starts with init": slices.Concat( + initOnly.insts, + asm.Instructions{asm.Mov.Imm32(asm.R0, 1), asm.Return()}, + ), + "starts with a non-magic dead store": { + asm.Mov.Imm32(asm.R0, 7), asm.Mov.Imm32(asm.R0, 0), asm.Return(), + }, + } + + for name, insts := range cases { + parsed, owned, err := parseOwned(mustCanonical(t, insts)) + if err != nil || owned || parsed != nil { + t.Errorf("%s: owned=%t err=%v, want an opaque original", name, owned, err) + } + } + + runcOrig := readFixture(t, filepath.Join("testdata", "devfilter", "runc.orig")) + + _, owned, err := parseOwned(runcOrig) + if err != nil || owned { + t.Errorf("runc original: owned=%t err=%v", owned, err) + } +} + +// corrupt returns the wrapper with instruction idx replaced. +func corrupt(t *testing.T, raw []byte, idx int, ins asm.Instruction) []byte { + t.Helper() + + insts, err := fromCanonical(raw) + if err != nil { + t.Fatal(err) + } + + insts[idx] = ins + + return mustCanonical(t, insts) +} + +func TestParseOwned_ConflictsNeverValidate(t *testing.T) { + orig := mustCanonical(t, allowNullOriginal()) + rules := []DeviceRule{rule("c", 10, 200, "rwm")} + wrapped := mustEmit(t, rules, 99, orig) + + insts, err := fromCanonical(wrapped) + if err != nil { + t.Fatal(err) + } + + blockLen, _ := markerValue(insts[hdrBlockLen]) + trailerAt := int(blockLen) + + cases := map[string][]byte{ + "header without trailer": corrupt(t, wrapped, trailerAt, asm.Mov.Imm32(asm.R0, 0)), + "trailer nonce differs": corrupt(t, wrapped, trailerAt+2, markerInst(98)), + "wrong block_len": corrupt(t, wrapped, hdrBlockLen, markerInst(blockLen+1)), + "block_len past the end": corrupt(t, wrapped, hdrBlockLen, markerInst(uint32(len(insts)))), + "wrong version": corrupt(t, wrapped, hdrVersion, markerInst(2)), + "digest mismatch": corrupt(t, wrapped, len(insts)-2, asm.Mov.Imm32(asm.R0, 1)), + "header field not marker": corrupt(t, wrapped, hdrNonceHi, asm.Mov.Imm32(asm.R1, 0)), + // Parses as a rule block but a jump no longer lands where the + // generator puts it. + "non-regenerable block": corrupt( + t, + wrapped, + ownedHeaderLen+ownedInitLen, + asm.Instruction{ + OpCode: asm.JNE.Op(asm.ImmSource), + Dst: asm.R2, + Offset: 1, + Constant: int64(unix.BPF_DEVCG_DEV_CHAR), + }, + ), + "unparseable block": corrupt( + t, + wrapped, + ownedHeaderLen+ownedInitLen, + asm.Mov.Imm32(asm.R7, 0), + ), + "truncated header": mustCanonical( + t, + asm.Instructions{markerInst(ownedMagicLo), asm.Mov.Imm32(asm.R0, 0), asm.Return()}, + ), + } + + for name, raw := range cases { + _, owned, parseErr := parseOwned(raw) + if owned || !errors.Is(parseErr, ErrOwnedBlockConflict) { + t.Errorf("%s: owned=%t err=%v, want ErrOwnedBlockConflict", name, owned, parseErr) + } + } +} + +func TestCheckWrappable_Subset(t *testing.T) { + ok := allowNullOriginal() + + err := checkWrappable(ok, knownMeta()) + if err != nil { + t.Fatalf("minimal original rejected: %v", err) + } + + withTail := func(insts ...asm.Instruction) asm.Instructions { + return slices.Concat( + asm.Instructions{asm.Mov.Imm32(asm.R0, 0)}, + insts, + asm.Instructions{asm.Return()}, + ) + } + + cases := map[string]struct { + insts asm.Instructions + meta progMeta + }{ + "map IDs unknown": {ok, progMeta{}}, + "uses a map": {ok, progMeta{mapIDsKnown: true, mapCount: 1}}, + "empty": {nil, knownMeta()}, + "helper call": {withTail(asm.FnGetCurrentPidTgid.Call()), knownMeta()}, + "64-bit immediate": {withTail(asm.LoadImm(asm.R3, 1<<40, asm.DWord)), knownMeta()}, + "stack store": {withTail(asm.StoreMem(asm.RFP, -8, asm.R2, asm.Word)), knownMeta()}, + "load from R2": {withTail(asm.LoadMem(asm.R3, asm.R2, 0, asm.Word)), knownMeta()}, + "ctx load at 12": {withTail(asm.LoadMem(asm.R3, asm.R1, 12, asm.Word)), knownMeta()}, + "alu add": {withTail(asm.Add.Imm(asm.R3, 1)), knownMeta()}, + "jump out of program": { + withTail(asm.Instruction{OpCode: asm.Ja.Op(asm.ImmSource), Offset: 5}), + knownMeta(), + }, + "jump before start": { + withTail(asm.Instruction{OpCode: asm.JEq.Op(asm.ImmSource), Dst: asm.R2, Offset: -5}), + knownMeta(), + }, + } + + for name, tc := range cases { + err = checkWrappable(tc.insts, tc.meta) + if !errors.Is(err, ErrProgramNotWrappable) { + t.Errorf("%s: err = %v, want ErrProgramNotWrappable", name, err) + } + } +} + +func TestKernelAtLeast(t *testing.T) { + cases := map[string]bool{ + "4.16.0": true, + "4.15.18": false, + "5.4.0-100-generic": true, + "3.99": false, + "junk": false, + } + + for release, want := range cases { + if got := kernelAtLeast(release, 4, 16); got != want { + t.Errorf("kernelAtLeast(%q) = %t, want %t", release, got, want) + } + } +} + +func TestWrapper_Verdicts(t *testing.T) { + orig := mustCanonical(t, allowNullOriginal()) + wrapped := mustEmit( + t, + []DeviceRule{rule("c", 10, 200, "rwm"), rule("c", 10, 201, "r")}, + 1, + orig, + ) + + const ( + read = unix.BPF_DEVCG_ACC_READ + write = unix.BPF_DEVCG_ACC_WRITE + ) + + cases := []struct { + name string + req devRequest + want uint64 + }{ + {"exact device allowed", charReq(10, 200, read|write), 1}, + {"other minor denied", charReq(10, 202, read), 0}, + { + "other type denied", + devRequest{typ: unix.BPF_DEVCG_DEV_BLOCK, access: read, major: 10, minor: 200}, + 0, + }, + {"r rule allows r", charReq(10, 201, read), 1}, + {"r rule with rw request denied", charReq(10, 201, read|write), 0}, + {"fall-through to original allows", charReq(1, 3, read|write), 1}, + {"fall-through to original denies", charReq(1, 5, read), 0}, + } + + for _, tc := range cases { + if got := runDeviceFilter(t, wrapped, tc.req); got != tc.want { + t.Errorf("%s: verdict %d, want %d", tc.name, got, tc.want) + } + } + + // The same holds on top of a real runtime filter: /dev/null (c 1:3) is + // in every runtime's default set, and the wrapper keeps it. + runcOrig := readFixture(t, filepath.Join("testdata", "devfilter", "runc.orig")) + runcWrapped := mustEmit(t, []DeviceRule{rule("c", 10, 250, "rwm")}, 1, runcOrig) + + if got := runDeviceFilter(t, runcWrapped, charReq(1, 3, read|write)); got != 1 { + t.Errorf("runc fall-through for /dev/null: verdict %d, want 1", got) + } + + if got := runDeviceFilter(t, runcWrapped, charReq(10, 250, read)); got != 1 { + t.Errorf("runc wrapper grant: verdict %d, want 1", got) + } + + if got := runDeviceFilter(t, runcOrig, charReq(10, 250, read)); got != 0 { + t.Errorf("runc original must deny c 10:250 for this test to mean anything, got %d", got) + } +} + +// ownedOps is a fakeOps whose attached programs carry the given code. +func ownedOps(t *testing.T, programs map[string][]byte) *fakeOps { + t.Helper() + + names := make([]string, 0, len(programs)) + for name := range programs { + names = append(names, name) + } + + slices.Sort(names) + + ops := newFakeOps(names...) + + for name, raw := range programs { + insts, err := fromCanonical(raw) + if err != nil { + t.Fatal(err) + } + + ops.insts[name] = insts + } + + return ops +} + +func setRules(ops *fakeOps, rules []DeviceRule) error { + c := &cgroupv2{ops: ops, replace: probeIn(replaceSupported)} + + return c.setDeviceRules(3, "/sys/fs/cgroup/test", rules) +} + +func attachedRaw(t *testing.T, ops *fakeOps, name string) []byte { + t.Helper() + + insts, _, err := ops.instructions(&fakeProg{name: name}) + if err != nil { + t.Fatal(err) + } + + return mustCanonical(t, insts) +} + +func TestSet_ApplyTwiceKeepsLengthAndLineage(t *testing.T) { + orig := mustCanonical(t, allowNullOriginal()) + ops := ownedOps(t, map[string][]byte{"O1": orig}) + rules := []DeviceRule{rule("c", 10, 200, "rwm")} + + err := setRules(ops, rules) + if err != nil { + t.Fatal(err) + } + + first := attachedRaw(t, ops, ops.attached[0]) + + err = setRules(ops, rules) + if err != nil { + t.Fatal(err) + } + + second := attachedRaw(t, ops, ops.attached[0]) + + if len(second) != len(first) { + t.Errorf( + "program grew from %d to %d bytes on a second identical apply", + len(first), + len(second), + ) + } + + firstParsed, _, _ := parseOwned(first) + secondParsed, owned, err := parseOwned(second) + + if err != nil || !owned || secondParsed.nonce != firstParsed.nonce || + !bytes.Equal(secondParsed.original, orig) { + t.Errorf( + "second apply did not re-wrap the same original under the same nonce (err %v)", + err, + ) + } + + if len(ops.attached) != 1 { + t.Errorf("attached = %q, want one program", ops.attached) + } +} + +func TestSet_WrapperIsNamedAndRestoreIsBare(t *testing.T) { + orig := mustCanonical(t, allowNullOriginal()) + ops := ownedOps(t, map[string][]byte{"O1": orig}) + + err := setRules(ops, []DeviceRule{rule("c", 10, 200, "rwm")}) + if err != nil { + t.Fatal(err) + } + + if ops.names["N1"] != ownedProgramName { + t.Errorf("wrapper name = %q, want %q", ops.names["N1"], ownedProgramName) + } + + err = setRules(ops, nil) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops, "load N1", "replace O1->N1", "load N2", "replace N1->N2") + + if !bytes.Equal(attachedRaw(t, ops, "N2"), orig) { + t.Error("empty rules did not restore the bare original") + } + + if ops.names["N2"] == ownedProgramName { + t.Error("restored original must not carry the wrapper name") + } + + ops.assertAllClosed(t) +} + +func TestSet_EmptyRulesLeaveUnmarkedProgramsAlone(t *testing.T) { + ops := ownedOps(t, map[string][]byte{"O1": mustCanonical(t, allowNullOriginal())}) + + err := setRules(ops, nil) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops) + assertAttached(t, ops, "O1") + ops.assertAllClosed(t) +} + +func TestSet_EmptyRulesWithNothingAttachedIsANoOp(t *testing.T) { + ops := newFakeOps() + + err := setRules(ops, nil) + if err != nil { + t.Fatalf("err = %v, want a no-op", err) + } + + assertCalls(t, ops) +} + +func TestSet_ConflictMutatesNothing(t *testing.T) { + orig := mustCanonical(t, allowNullOriginal()) + wrapped := mustEmit(t, []DeviceRule{rule("c", 10, 200, "rwm")}, 5, orig) + + insts, err := fromCanonical(wrapped) + if err != nil { + t.Fatal(err) + } + + blockLen, _ := markerValue(insts[hdrBlockLen]) + broken := corrupt(t, wrapped, int(blockLen), asm.Mov.Imm32(asm.R0, 0)) + + // O1 is fine; O2 conflicts. Nothing may happen to either. + ops := ownedOps(t, map[string][]byte{"O1": orig, "O2": broken}) + + for _, rules := range [][]DeviceRule{{rule("c", 10, 200, "rwm")}, nil} { + err = setRules(ops, rules) + if !errors.Is(err, ErrOwnedBlockConflict) { + t.Fatalf("err = %v, want ErrOwnedBlockConflict", err) + } + } + + assertCalls(t, ops) + assertAttached(t, ops, "O1", "O2") + ops.assertAllClosed(t) +} + +func TestSet_UnwrappableProgramMutatesNothing(t *testing.T) { + ops := ownedOps(t, map[string][]byte{"O1": mustCanonical(t, allowNullOriginal())}) + ops.meta = progMeta{mapIDsKnown: true, mapCount: 1} + + err := setRules(ops, []DeviceRule{rule("c", 10, 200, "rwm")}) + if !errors.Is(err, ErrProgramNotWrappable) { + t.Fatalf("err = %v, want ErrProgramNotWrappable", err) + } + + assertCalls(t, ops) + ops.assertAllClosed(t) +} + +func TestSet_StripsOnlyValidatedBlocks(t *testing.T) { + orig := mustCanonical(t, allowNullOriginal()) + older := mustEmit(t, []DeviceRule{rule("c", 10, 200, "rwm")}, 7, orig) + ops := ownedOps(t, map[string][]byte{"O1": older}) + + err := setRules(ops, []DeviceRule{rule("b", 8, 0, "r")}) + if err != nil { + t.Fatal(err) + } + + parsed, owned, err := parseOwned(attachedRaw(t, ops, "N1")) + if err != nil || !owned { + t.Fatalf("owned=%t err=%v", owned, err) + } + + if !bytes.Equal(parsed.original, orig) || parsed.nonce != 7 { + t.Error("new wrapper does not wrap the stripped original under the old nonce") + } + + if got := runDeviceFilter( + t, + attachedRaw(t, ops, "N1"), + charReq(10, 200, unix.BPF_DEVCG_ACC_READ), + ); got != 0 { + t.Error("grant from the replaced wrapper survived") + } +} diff --git a/internal/cgroup/testdata/devfilter/README.md b/internal/cgroup/testdata/devfilter/README.md new file mode 100644 index 0000000..637a2b1 --- /dev/null +++ b/internal/cgroup/testdata/devfilter/README.md @@ -0,0 +1,39 @@ +# Device filter fixtures + +Real `BPF_CGROUP_DEVICE` programs, as the daemon reads them back from the +kernel (translated dump, canonical bytes, one 8-byte instruction per line in +hex). `owned_test.go` uses them to prove that the programs container runtimes +attach pass the reloadability gate and survive a wrap/strip round trip. + +| Fixture | Attached by | +| --- | --- | +| `systemd` | systemd 259 (`sd_devices`) for a Docker 29 container (systemd cgroup driver) | +| `runc` | runc 1.5.1 for the same container | +| `crun` | crun 1.28 under podman 5.8 (cgroupfs manager) | + +Captured on Linux 7.0 (x86_64). For each fixture: + +- `.orig` is the program as dumped from the container's cgroup; +- `.reloaded` is the dump after loading `.orig` back unchanged, which + must equal `.orig` for the gate's premise to hold; +- `.wrapped` is the dump after loading the daemon's wrapper around + `.orig` for `fixtureRules` with `fixtureNonce` (`fixtures_test.go`). + +## Capturing on another kernel or runtime + +The capture only loads and dumps programs; it never attaches or detaches +anything. It needs a privileged container on the host that runs the target +container. + +```bash +CGO_ENABLED=0 go test -c -tags fixturecapture -o /tmp/sda-capture ./internal/cgroup +docker run --rm --privileged --pid=host --cgroupns=host \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /tmp/sda-capture:/capture:ro -v "$PWD/out:/out" \ + -e SDA_CAPTURE_CGROUP=/sys/fs/cgroup/ \ + -e SDA_CAPTURE_OUT=/out -e SDA_CAPTURE_NAME= \ + alpine:3.22 /capture -test.run TestCaptureFixtures -test.v +``` + +One set of files is written per attached program (`-.*`). Rename +them, add them to the fixture list in `owned_test.go`, and note the versions +here. diff --git a/internal/cgroup/testdata/devfilter/crun.orig b/internal/cgroup/testdata/devfilter/crun.orig new file mode 100644 index 0000000..1b28ebb --- /dev/null +++ b/internal/cgroup/testdata/devfilter/crun.orig @@ -0,0 +1,77 @@ +6112000000000000 +54020000ffff0000 +6113000000000000 +7403000010000000 +6114040000000000 +6115080000000000 +5502070002000000 +bc31000000000000 +5401000000000000 +5d31040000000000 +5504030000000000 +5505020000000000 +b700000001000000 +9500000000000000 +5502040002000000 +550403000a000000 +55050200c8000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020002000000 +b700000001000000 +9500000000000000 +5502030002000000 +5504020088000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020001000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020009000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020005000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020000000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020007000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020008000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020003000000 +b700000001000000 +9500000000000000 +5502050001000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +b700000001000000 +9500000000000000 +5502050002000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +b700000001000000 +9500000000000000 +b700000000000000 +9500000000000000 diff --git a/internal/cgroup/testdata/devfilter/crun.reloaded b/internal/cgroup/testdata/devfilter/crun.reloaded new file mode 100644 index 0000000..1b28ebb --- /dev/null +++ b/internal/cgroup/testdata/devfilter/crun.reloaded @@ -0,0 +1,77 @@ +6112000000000000 +54020000ffff0000 +6113000000000000 +7403000010000000 +6114040000000000 +6115080000000000 +5502070002000000 +bc31000000000000 +5401000000000000 +5d31040000000000 +5504030000000000 +5505020000000000 +b700000001000000 +9500000000000000 +5502040002000000 +550403000a000000 +55050200c8000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020002000000 +b700000001000000 +9500000000000000 +5502030002000000 +5504020088000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020001000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020009000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020005000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020000000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020007000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020008000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020003000000 +b700000001000000 +9500000000000000 +5502050001000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +b700000001000000 +9500000000000000 +5502050002000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +b700000001000000 +9500000000000000 +b700000000000000 +9500000000000000 diff --git a/internal/cgroup/testdata/devfilter/crun.wrapped b/internal/cgroup/testdata/devfilter/crun.wrapped new file mode 100644 index 0000000..5eb510e --- /dev/null +++ b/internal/cgroup/testdata/devfilter/crun.wrapped @@ -0,0 +1,109 @@ +b40000005344415f +b400000044424c4b +b400000001000000 +b4000000efcdab89 +b400000067452301 +b4000000ef4711dc +b40000000ebf58c3 +b400000038ea7554 +b4000000b268381b +b40000001c000000 +6912000000000000 +6113000000000000 +7403000010000000 +6114040000000000 +6115080000000000 +5502070001000000 +bc36000000000000 +5406000002000000 +5d36040000000000 +5504030008000000 +5505020000000000 +b400000001000000 +9500000000000000 +5502040002000000 +550403000a000000 +55050200c8000000 +b400000001000000 +9500000000000000 +b40000005344415f +b400000044424c4b +b4000000efcdab89 +b400000067452301 +6112000000000000 +54020000ffff0000 +6113000000000000 +7403000010000000 +6114040000000000 +6115080000000000 +5502070002000000 +bc31000000000000 +5401000000000000 +5d31040000000000 +5504030000000000 +5505020000000000 +b700000001000000 +9500000000000000 +5502040002000000 +550403000a000000 +55050200c8000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020002000000 +b700000001000000 +9500000000000000 +5502030002000000 +5504020088000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020001000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020009000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020005000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020000000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020007000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020008000000 +b700000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020003000000 +b700000001000000 +9500000000000000 +5502050001000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +b700000001000000 +9500000000000000 +5502050002000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +b700000001000000 +9500000000000000 +b700000000000000 +9500000000000000 diff --git a/internal/cgroup/testdata/devfilter/runc.orig b/internal/cgroup/testdata/devfilter/runc.orig new file mode 100644 index 0000000..55ee203 --- /dev/null +++ b/internal/cgroup/testdata/devfilter/runc.orig @@ -0,0 +1,69 @@ +6112000000000000 +54020000ffff0000 +6113000000000000 +7403000010000000 +6114040000000000 +6115080000000000 +5502050001000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +b400000001000000 +9500000000000000 +5502050002000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020003000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020005000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020007000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020008000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020009000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020000000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020001000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020002000000 +b400000001000000 +9500000000000000 +5502040002000000 +550403000a000000 +55050200c8000000 +b400000001000000 +9500000000000000 +5502030002000000 +5504020088000000 +b400000001000000 +9500000000000000 +b400000000000000 +9500000000000000 diff --git a/internal/cgroup/testdata/devfilter/runc.reloaded b/internal/cgroup/testdata/devfilter/runc.reloaded new file mode 100644 index 0000000..55ee203 --- /dev/null +++ b/internal/cgroup/testdata/devfilter/runc.reloaded @@ -0,0 +1,69 @@ +6112000000000000 +54020000ffff0000 +6113000000000000 +7403000010000000 +6114040000000000 +6115080000000000 +5502050001000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +b400000001000000 +9500000000000000 +5502050002000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020003000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020005000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020007000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020008000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020009000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020000000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020001000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020002000000 +b400000001000000 +9500000000000000 +5502040002000000 +550403000a000000 +55050200c8000000 +b400000001000000 +9500000000000000 +5502030002000000 +5504020088000000 +b400000001000000 +9500000000000000 +b400000000000000 +9500000000000000 diff --git a/internal/cgroup/testdata/devfilter/runc.wrapped b/internal/cgroup/testdata/devfilter/runc.wrapped new file mode 100644 index 0000000..2729f34 --- /dev/null +++ b/internal/cgroup/testdata/devfilter/runc.wrapped @@ -0,0 +1,101 @@ +b40000005344415f +b400000044424c4b +b400000001000000 +b4000000efcdab89 +b400000067452301 +b4000000b8f4fb5f +b400000008605bc5 +b400000094fd71f0 +b4000000ee3ae2a0 +b40000001c000000 +6912000000000000 +6113000000000000 +7403000010000000 +6114040000000000 +6115080000000000 +5502070001000000 +bc36000000000000 +5406000002000000 +5d36040000000000 +5504030008000000 +5505020000000000 +b400000001000000 +9500000000000000 +5502040002000000 +550403000a000000 +55050200c8000000 +b400000001000000 +9500000000000000 +b40000005344415f +b400000044424c4b +b4000000efcdab89 +b400000067452301 +6112000000000000 +54020000ffff0000 +6113000000000000 +7403000010000000 +6114040000000000 +6115080000000000 +5502050001000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +b400000001000000 +9500000000000000 +5502050002000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020003000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020005000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020007000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020008000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030001000000 +5505020009000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020000000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020001000000 +b400000001000000 +9500000000000000 +5502040002000000 +5504030005000000 +5505020002000000 +b400000001000000 +9500000000000000 +5502040002000000 +550403000a000000 +55050200c8000000 +b400000001000000 +9500000000000000 +5502030002000000 +5504020088000000 +b400000001000000 +9500000000000000 +b400000000000000 +9500000000000000 diff --git a/internal/cgroup/testdata/devfilter/systemd.orig b/internal/cgroup/testdata/devfilter/systemd.orig new file mode 100644 index 0000000..55aaa24 --- /dev/null +++ b/internal/cgroup/testdata/devfilter/systemd.orig @@ -0,0 +1,59 @@ +6112000000000000 +54020000ffff0000 +6113000000000000 +7403000010000000 +6114040000000000 +6115080000000000 +5502020002000000 +5504010088000000 +0500300000000000 +5502030002000000 +550402000a000000 +55050100c8000000 +05002c0000000000 +5502030002000000 +5504020005000000 +5505010002000000 +0500280000000000 +5502030002000000 +5504020005000000 +5505010001000000 +0500240000000000 +5502030002000000 +5504020005000000 +5505010000000000 +0500200000000000 +5502030002000000 +5504020001000000 +5505010009000000 +05001c0000000000 +5502030002000000 +5504020001000000 +5505010008000000 +0500180000000000 +5502030002000000 +5504020001000000 +5505010007000000 +0500140000000000 +5502030002000000 +5504020001000000 +5505010005000000 +0500100000000000 +5502030002000000 +5504020001000000 +5505010003000000 +05000c0000000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +5502010002000000 +0500070000000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +5502010001000000 +0500020000000000 +b700000000000000 +0500010000000000 +b700000001000000 +9500000000000000 diff --git a/internal/cgroup/testdata/devfilter/systemd.reloaded b/internal/cgroup/testdata/devfilter/systemd.reloaded new file mode 100644 index 0000000..55aaa24 --- /dev/null +++ b/internal/cgroup/testdata/devfilter/systemd.reloaded @@ -0,0 +1,59 @@ +6112000000000000 +54020000ffff0000 +6113000000000000 +7403000010000000 +6114040000000000 +6115080000000000 +5502020002000000 +5504010088000000 +0500300000000000 +5502030002000000 +550402000a000000 +55050100c8000000 +05002c0000000000 +5502030002000000 +5504020005000000 +5505010002000000 +0500280000000000 +5502030002000000 +5504020005000000 +5505010001000000 +0500240000000000 +5502030002000000 +5504020005000000 +5505010000000000 +0500200000000000 +5502030002000000 +5504020001000000 +5505010009000000 +05001c0000000000 +5502030002000000 +5504020001000000 +5505010008000000 +0500180000000000 +5502030002000000 +5504020001000000 +5505010007000000 +0500140000000000 +5502030002000000 +5504020001000000 +5505010005000000 +0500100000000000 +5502030002000000 +5504020001000000 +5505010003000000 +05000c0000000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +5502010002000000 +0500070000000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +5502010001000000 +0500020000000000 +b700000000000000 +0500010000000000 +b700000001000000 +9500000000000000 diff --git a/internal/cgroup/testdata/devfilter/systemd.wrapped b/internal/cgroup/testdata/devfilter/systemd.wrapped new file mode 100644 index 0000000..6f367ee --- /dev/null +++ b/internal/cgroup/testdata/devfilter/systemd.wrapped @@ -0,0 +1,91 @@ +b40000005344415f +b400000044424c4b +b400000001000000 +b4000000efcdab89 +b400000067452301 +b4000000461f9f51 +b400000062fd8042 +b400000065faeed5 +b4000000f32ce751 +b40000001c000000 +6912000000000000 +6113000000000000 +7403000010000000 +6114040000000000 +6115080000000000 +5502070001000000 +bc36000000000000 +5406000002000000 +5d36040000000000 +5504030008000000 +5505020000000000 +b400000001000000 +9500000000000000 +5502040002000000 +550403000a000000 +55050200c8000000 +b400000001000000 +9500000000000000 +b40000005344415f +b400000044424c4b +b4000000efcdab89 +b400000067452301 +6112000000000000 +54020000ffff0000 +6113000000000000 +7403000010000000 +6114040000000000 +6115080000000000 +5502020002000000 +5504010088000000 +0500300000000000 +5502030002000000 +550402000a000000 +55050100c8000000 +05002c0000000000 +5502030002000000 +5504020005000000 +5505010002000000 +0500280000000000 +5502030002000000 +5504020005000000 +5505010001000000 +0500240000000000 +5502030002000000 +5504020005000000 +5505010000000000 +0500200000000000 +5502030002000000 +5504020001000000 +5505010009000000 +05001c0000000000 +5502030002000000 +5504020001000000 +5505010008000000 +0500180000000000 +5502030002000000 +5504020001000000 +5505010007000000 +0500140000000000 +5502030002000000 +5504020001000000 +5505010005000000 +0500100000000000 +5502030002000000 +5504020001000000 +5505010003000000 +05000c0000000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +5502010002000000 +0500070000000000 +bc31000000000000 +5401000001000000 +5d31020000000000 +5502010001000000 +0500020000000000 +b700000000000000 +0500010000000000 +b700000001000000 +9500000000000000 diff --git a/internal/cgroup/v2.go b/internal/cgroup/v2.go index 57cb5a4..176222e 100644 --- a/internal/cgroup/v2.go +++ b/internal/cgroup/v2.go @@ -29,8 +29,9 @@ import ( "strings" "github.com/cilium/ebpf" - "github.com/cilium/ebpf/asm" "golang.org/x/sys/unix" + + "github.com/leinardi/swarm-device-access/internal/logger" ) const ( @@ -150,12 +151,22 @@ func scanProcCgroupV2(r io.Reader, path string, prefix string) (string, error) { return "", errNoCgroupV2Entry } -// SetDeviceRules adds rules to the device filter of the cgroup behind handle. +// SetDeviceRules makes this daemon's grants in the device filters of the +// cgroup behind handle equal exactly rules. +// +// Each attached program is either one of this daemon's wrappers (see +// owned.go), which is stripped back to the runtime original it wraps, or an +// unmarked program, which is the original itself. Non-empty rules replace +// every program with a fresh wrapper around its original; empty rules +// replace every wrapper with its bare original and leave unmarked programs +// alone, so a cgroup this daemon never touched is never touched. Nothing +// but the programs themselves carries state, so this works the same after +// a daemon restart. // -// It is still grant-only: rules are prepended to the attached programs, and -// earlier grants are not replaced or revoked. An empty rule set is a no-op. +// Unmarked programs are never interpreted: a wrapper written by a version +// that predates the ownership markers looks exactly like a runtime program, +// so its grants are kept until the container restarts. // -// Every attached program is replaced by a copy with the rules prepended. // Unlike NVIDIA's upstream code, which detaches every program before // attaching the replacements because it runs strictly before the container // starts, this daemon mutates live containers: detaching first would leave @@ -163,14 +174,17 @@ func scanProcCgroupV2(r io.Reader, path string, prefix string) (string, error) { // failed or the daemon died. Programs are therefore replaced pairwise, each // new program attached (or atomically swapped in) before its original goes. func (c *cgroupv2) SetDeviceRules(handle *CgroupHandle, rules []DeviceRule) error { - if len(rules) == 0 { - return nil - } + return c.setDeviceRules(handle.fd, handle.Identity().Path, rules) +} - return c.addDeviceRules(handle.fd, rules) +// replacement is the planned new program for one attached program. +type replacement struct { + old progHandle + raw []byte + name string } -func (c *cgroupv2) addDeviceRules(dirFD int, rules []DeviceRule) error { +func (c *cgroupv2) setDeviceRules(dirFD int, cgroupPath string, rules []DeviceRule) error { // Find any existing eBPF device filter programs attached to this cgroup. oldProgs, total, inaccessible, attachFlags, err := c.ops.query(dirFD) if err != nil { @@ -184,6 +198,8 @@ func (c *cgroupv2) addDeviceRules(dirFD int, rules []DeviceRule) error { switch { case inaccessible > 0: return fmt.Errorf("%w: %d of %d programs", ErrFiltersInaccessible, inaccessible, total) + case total == 0 && len(rules) == 0: + return nil case total == 0: return ErrFilterMissing case attachFlags&unix.BPF_F_ALLOW_MULTI == 0: @@ -194,28 +210,35 @@ func (c *cgroupv2) addDeviceRules(dirFD int, rules []DeviceRule) error { ) } - // Build every replacement before touching the cgroup, so a generation or - // load failure mutates nothing. - newProgs := make([]progHandle, 0, len(oldProgs)) - defer func() { closeAll(newProgs) }() + // Plan every replacement before touching the cgroup: a conflict, an + // unwrappable program or a load failure anywhere mutates nothing. + plans := make([]replacement, 0, len(oldProgs)) for _, oldProg := range oldProgs { var ( - oldInsts asm.Instructions - newProg progHandle + plan replacement + keep bool ) - oldInsts, err = c.ops.instructions(oldProg) + plan, keep, err = c.planReplacement(oldProg, cgroupPath, rules) if err != nil { return err } - newProg, err = c.generateNewProgram(rules, oldInsts) + if !keep { + plans = append(plans, plan) + } + } + + newProgs := make([]progHandle, 0, len(plans)) + defer func() { closeAll(newProgs) }() + + for _, plan := range plans { + var newProg progHandle + + newProg, err = c.loadCanonical(plan.raw, plan.name) if err != nil { - return fmt.Errorf( - "unable to generate new device filter program from existing programs: %w", - err, - ) + return err } newProgs = append(newProgs, newProg) @@ -225,13 +248,13 @@ func (c *cgroupv2) addDeviceRules(dirFD int, rules []DeviceRule) error { // every program, so a half-replaced set is narrower, never wider (an // original not yet replaced masks the new grant). A failure therefore // stops here without undoing earlier pairs; a retry completes the rest. - for idx := range oldProgs { - err = c.swap(dirFD, oldProgs[idx], newProgs[idx]) + for idx, plan := range plans { + err = c.swap(dirFD, plan.old, newProgs[idx]) if err != nil { return fmt.Errorf( "replace device filter program %d of %d: %w", idx+1, - len(oldProgs), + len(plans), err, ) } @@ -240,6 +263,97 @@ func (c *cgroupv2) addDeviceRules(dirFD int, rules []DeviceRule) error { return nil } +// planReplacement decides what replaces oldProg. keep reports that oldProg +// stays as it is (an unmarked program under empty rules). +func (c *cgroupv2) planReplacement( + oldProg progHandle, + cgroupPath string, + rules []DeviceRule, +) (replacement, bool, error) { + insts, meta, err := c.ops.instructions(oldProg) + if err != nil { + return replacement{}, false, err + } + + raw, err := canonicalBytes(insts) + if err != nil { + return replacement{}, false, err + } + + owned, isOwned, err := parseOwned(raw) + if err != nil { + return replacement{}, false, err + } + + if !isOwned && len(rules) == 0 { + return replacement{}, true, nil + } + + original := raw + if isOwned { + original = owned.original + } + + origInsts, err := fromCanonical(original) + if err != nil { + return replacement{}, false, err + } + + err = checkWrappable(origInsts, meta) + if err != nil { + return replacement{}, false, err + } + + if len(rules) == 0 { + return replacement{old: oldProg, raw: original}, false, nil + } + + var nonce uint64 + + if isOwned { + nonce = owned.nonce + } else { + nonce, err = newNonce() + if err != nil { + return replacement{}, false, err + } + + logger.L(). + Info("wrapping device filter with no prior owned block; pre-upgrade grants, if any, are not managed", + "cgroup", cgroupPath) + } + + wrapped, err := emitOwned(rules, nonce, original) + if err != nil { + return replacement{}, false, fmt.Errorf( + "unable to generate new device filter program: %w", + err, + ) + } + + return replacement{old: oldProg, raw: wrapped, name: ownedProgramName}, false, nil +} + +// loadCanonical loads a program from canonical bytes. A verifier rejection +// is returned as is, before anything is attached. +// +//nolint:ireturn // progHandle hides the kernel handle so the swap logic can be tested with fakes +func (c *cgroupv2) loadCanonical(raw []byte, name string) (progHandle, error) { + insts, err := fromCanonical(raw) + if err != nil { + return nil, err + } + + spec := &ebpf.ProgramSpec{ + Name: name, + Type: ebpf.CGroupDevice, + Instructions: insts, + License: bpfProgramLicense, + } + + return c.ops.load(spec) +} + // swap replaces oldProg with newProg on dirFD: atomically when the kernel // supports BPF_F_REPLACE, otherwise by attaching newProg before detaching // oldProg. If that detach fails, newProg is detached again so the pair is @@ -281,30 +395,6 @@ func (c *cgroupv2) swap(dirFD int, oldProg, newProg progHandle) error { return detachErr } -//nolint:ireturn // progHandle hides the kernel handle so the swap logic can be tested with fakes -func (c *cgroupv2) generateNewProgram( - rules []DeviceRule, - oldInsts asm.Instructions, -) (progHandle, error) { - // Prepend instructions for the new devices to the original set of instructions. - newInsts, err := PrependDeviceFilter(rules, oldInsts) - if err != nil { - return nil, fmt.Errorf( - "unable to prepend new device filters to the original device filters program: %w", - err, - ) - } - - // Generate new eBPF program for the merged device filter instructions. - spec := &ebpf.ProgramSpec{ - Type: ebpf.CGroupDevice, - Instructions: newInsts, - License: bpfProgramLicense, - } - - return c.ops.load(spec) -} - func closeAll(progs []progHandle) { for _, prog := range progs { prog.Close() diff --git a/internal/cgroup/v2ops.go b/internal/cgroup/v2ops.go index 603a659..425c8d5 100644 --- a/internal/cgroup/v2ops.go +++ b/internal/cgroup/v2ops.go @@ -68,7 +68,9 @@ type v2ops interface { // replacement of that program. attach(prog progHandle, dirFD int, flags uint32, replace progHandle) error detach(prog progHandle, dirFD int) error - instructions(prog progHandle) (asm.Instructions, error) + // instructions returns the program's translated instructions and what + // the reloadability gate needs to know about its maps. + instructions(prog progHandle) (asm.Instructions, progMeta, error) } // kernelProg adapts *ebpf.Program to progHandle. @@ -164,15 +166,19 @@ func (kernelOps) detach(prog progHandle, dirFD int) error { return DetachCgroupDeviceFilter(kprog, dirFD) } -func (kernelOps) instructions(prog progHandle) (asm.Instructions, error) { +func (kernelOps) instructions(prog progHandle) (asm.Instructions, progMeta, error) { kprog, err := kernelProgram(prog) if err != nil { - return nil, err + return nil, progMeta{}, err + } + + if !kernelDumpsSanitized() { + return nil, progMeta{}, fmt.Errorf("%w: kernel older than 4.16", ErrProgramNotWrappable) } info, err := kprog.Info() if err != nil { - return nil, fmt.Errorf( + return nil, progMeta{}, fmt.Errorf( "unable to get Info() of the original device filters program: %w", err, ) @@ -180,13 +186,41 @@ func (kernelOps) instructions(prog progHandle) (asm.Instructions, error) { insts, err := info.Instructions() if err != nil { - return nil, fmt.Errorf( + return nil, progMeta{}, fmt.Errorf( "unable to get the instructions of the original device filters program: %w", err, ) } - return insts, nil + mapIDs, known := info.MapIDs() + + return insts, progMeta{mapIDsKnown: known, mapCount: len(mapIDs)}, nil +} + +// kernelDumpsSanitized reports whether the running kernel is 4.16 or newer, +// the first release whose translated dumps of unprivileged-style programs +// are sanitized consistently enough to be loaded back. +func kernelDumpsSanitized() bool { + var uname unix.Utsname + + err := unix.Uname(&uname) + if err != nil { + return false + } + + return kernelAtLeast(unix.ByteSliceToString(uname.Release[:]), 4, 16) +} + +// kernelAtLeast parses the leading "major.minor" of a kernel release string. +func kernelAtLeast(release string, wantMajor, wantMinor int) bool { + var major, minor int + + _, err := fmt.Sscanf(release, "%d.%d", &major, &minor) + if err != nil { + return false + } + + return major > wantMajor || (major == wantMajor && minor >= wantMinor) } // Replace-support states for replaceProbe. diff --git a/internal/cgroup/v2swap_test.go b/internal/cgroup/v2swap_test.go index e9f3f48..c4cafd9 100644 --- a/internal/cgroup/v2swap_test.go +++ b/internal/cgroup/v2swap_test.go @@ -61,10 +61,28 @@ type fakeOps struct { calls []string opened []*fakeProg loads int + + // insts holds each program's instructions; a name without an entry is + // a runtime original (runtimeOriginal). + insts map[string]asm.Instructions + names map[string]string // program name each loaded program was given + meta progMeta +} + +// runtimeOriginal is a minimal runtime device filter: deny everything. +func runtimeOriginal() asm.Instructions { + return asm.Instructions{asm.Mov.Imm32(asm.R0, 0), asm.Return()} } func newFakeOps(attached ...string) *fakeOps { - return &fakeOps{attached: attached, flags: unix.BPF_F_ALLOW_MULTI, errs: map[string]error{}} + return &fakeOps{ + attached: attached, + flags: unix.BPF_F_ALLOW_MULTI, + errs: map[string]error{}, + insts: map[string]asm.Instructions{}, + names: map[string]string{}, + meta: progMeta{mapIDsKnown: true}, + } } func (f *fakeOps) open(name string) *fakeProg { @@ -93,7 +111,7 @@ func nameOf(handle progHandle) string { } //nolint:ireturn // implements v2ops.load -func (f *fakeOps) load(*ebpf.ProgramSpec) (progHandle, error) { +func (f *fakeOps) load(spec *ebpf.ProgramSpec) (progHandle, error) { f.loads++ name := fmt.Sprintf("N%d", f.loads) @@ -105,6 +123,9 @@ func (f *fakeOps) load(*ebpf.ProgramSpec) (progHandle, error) { return nil, err } + f.insts[name] = slices.Clone(spec.Instructions) + f.names[name] = spec.Name + return f.open(name), nil } @@ -159,8 +180,13 @@ func (f *fakeOps) detach(prog progHandle, _ int) error { return nil } -func (f *fakeOps) instructions(progHandle) (asm.Instructions, error) { - return asm.Instructions{asm.Mov.Imm32(asm.R0, 0), asm.Return()}, nil +func (f *fakeOps) instructions(prog progHandle) (asm.Instructions, progMeta, error) { + insts, ok := f.insts[nameOf(prog)] + if !ok { + insts = runtimeOriginal() + } + + return slices.Clone(insts), f.meta, nil } func (f *fakeOps) assertAllClosed(t *testing.T) { @@ -189,7 +215,7 @@ func probeIn(state int32) *replaceProbe { func run(ops *fakeOps, probe *replaceProbe) error { c := &cgroupv2{ops: ops, replace: probe} - return c.addDeviceRules(3, testRules()) + return c.setDeviceRules(3, "/sys/fs/cgroup/test", testRules()) } func assertCalls(t *testing.T, ops *fakeOps, want ...string) { diff --git a/internal/processor/applyerr_test.go b/internal/processor/applyerr_test.go index 56e6bdc..a93c9bf 100644 --- a/internal/processor/applyerr_test.go +++ b/internal/processor/applyerr_test.go @@ -156,3 +156,30 @@ func TestProcessContainer_InaccessibleFiltersIsRetryableError(t *testing.T) { t.Errorf("error %q does not carry the reason", err) } } + +func TestProcessContainer_OwnedBlockConflictIsLoggedWithRemedy(t *testing.T) { + buf := captureLogger(t) + fake := &failingCgroup{err: fmt.Errorf("%w: trailer missing", cgroup.ErrOwnedBlockConflict)} + + err := devNullProcessor(t, fake).ProcessContainer(context.Background(), "abc") + if !errors.Is(err, cgroup.ErrOwnedBlockConflict) || + !strings.Contains(err.Error(), "owned_block_conflict") { + t.Fatalf("err = %v, want a retryable ErrOwnedBlockConflict carrying the reason", err) + } + + out := buf.String() + if !strings.Contains(out, "level=ERROR") || !strings.Contains(out, "restart the container") || + !strings.Contains(out, "cgroup=") { + t.Errorf("expected ERROR with cgroup and remedy, got:\n%s", out) + } +} + +func TestProcessContainer_NotWrappableIsRetryableError(t *testing.T) { + fake := &failingCgroup{err: fmt.Errorf("%w: uses maps", cgroup.ErrProgramNotWrappable)} + + err := devNullProcessor(t, fake).ProcessContainer(context.Background(), "abc") + if !errors.Is(err, cgroup.ErrProgramNotWrappable) || + !strings.Contains(err.Error(), "program_not_wrappable") { + t.Fatalf("err = %v, want ErrProgramNotWrappable with its reason", err) + } +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 5bf06a5..bdca0b1 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -215,28 +215,8 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er if len(collected.granted) > 0 { applyErr := p.applyRulesToCgroup(api, collected.granted, cgroupPath, pid, cfg.DryRun) - if errors.Is(applyErr, cgroup.ErrUnsupportedAttachMode) { - // Permanent for this container: the runtime chose the attach mode. - // Nothing was attached or detached, so it keeps only the runtime's - // own device filter. - log.Error("container skipped: device filter attach mode not supported; "+ - "the runtime must attach device filters with BPF_F_ALLOW_MULTI", - "id", containerID, "cgroup", cgroupPath, "err", applyErr) - p.Metrics.RecordContainerSkipped("unsupported_attach_mode") - - return nil - } - - if errors.Is(applyErr, cgroup.ErrFiltersInaccessible) { - return fmt.Errorf( - "container %q (reason filters_inaccessible, retryable): %w", - containerID, - applyErr, - ) - } - if applyErr != nil { - return applyErr + return p.classifyApplyError(containerID, cgroupPath, applyErr) } } @@ -263,6 +243,65 @@ func (p *Processor) callContext(ctx context.Context) (context.Context, context.C return context.WithTimeout(ctx, p.CallTimeout) } +// classifyApplyError turns a cgroup mutation failure into the container's +// outcome: nil when the container is skipped for good, otherwise an error +// that names the reason, so the caller retries it. +func (p *Processor) classifyApplyError(containerID, cgroupPath string, applyErr error) error { + log := logger.L() + + switch { + case errors.Is(applyErr, cgroup.ErrUnsupportedAttachMode): + // Permanent for this container: the runtime chose the attach mode. + // Nothing was attached or detached, so it keeps only the runtime's + // own device filter. + log.Error("container skipped: device filter attach mode not supported; "+ + "the runtime must attach device filters with BPF_F_ALLOW_MULTI", + "id", containerID, "cgroup", cgroupPath, "err", applyErr) + p.Metrics.RecordContainerSkipped("unsupported_attach_mode") + + return nil + + case errors.Is(applyErr, cgroup.ErrOwnedBlockConflict): + // Nothing was changed. Only a fresh device filter (a container + // restart) clears it; retrying keeps the container pending. + log.Error( + "device filter carries an invalid swarm-device-access block; "+ + "nothing was changed, restart the container to replace its device filter", + "id", + containerID, + "cgroup", + cgroupPath, + "reason", + "owned_block_conflict", + "err", + applyErr, + ) + + return fmt.Errorf( + "container %q (reason owned_block_conflict, retryable): %w", + containerID, + applyErr, + ) + + case errors.Is(applyErr, cgroup.ErrProgramNotWrappable): + return fmt.Errorf( + "container %q (reason program_not_wrappable, retryable): %w", + containerID, + applyErr, + ) + + case errors.Is(applyErr, cgroup.ErrFiltersInaccessible): + return fmt.Errorf( + "container %q (reason filters_inaccessible, retryable): %w", + containerID, + applyErr, + ) + + default: + return applyErr + } +} + // containerRules aggregates the per-mount results for one container. type containerRules struct { granted []cgroup.DeviceRule // deduplicated across mounts From b4f8aa4c315ad0358175d1eeb95c0435b7718f1c Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 10:04:40 +0200 Subject: [PATCH 07/37] feat(cgroup): converge cgroup v2 device filters and rebuild wiped ones Attached programs are classified into wrapper lineages (validated nonce plus byte-identical innermost original) and bare runtime programs. With grants, each lineage ends as one wrapper equal to the desired one (an equal member is kept, otherwise the first is replaced) and its other members are detached; each bare program is wrapped as its own pair under a new nonce, so identical runtime programs keep their count and the leftover of a failed rollback ends as one redundant, identical wrapper. Without grants, lineages go back to their bare original, and a lineage whose original is already attached bare only has its wrappers detached, which is how an interrupted restore converges. A wrapper over a wrapper is re-emitted as one. Kernel program IDs are never used, every comparison is on canonical bytes, and every attach precedes the detach it enables. A per-cgroup FilterCache (process memory, keyed by path plus inode, with Forget for lifecycle ends) records the runtime originals and lineage nonces. When systemd's daemon-reload has detached every filter, the cached originals are attached again, bare or wrapped under their nonce, restoring the runtime's own restrictions as well; a rebuild that stops halfway stays marked in the cache so the next pass reattaches the lineages still missing instead of forgetting them. With nothing cached, ErrFilterMissing: the processor treats a privileged container as a no-op (INFO, skipped as privileged_no_filter) and anything else as retryable with reason filter_missing and a warning to restart the container, never a silent success. --- README.md | 19 +- internal/cgroup/api.go | 10 +- internal/cgroup/api_test.go | 28 +- internal/cgroup/converge_test.go | 489 +++++++++++++++++++++++++++ internal/cgroup/owned_test.go | 22 +- internal/cgroup/v2.go | 170 ++++------ internal/cgroup/v2plan.go | 491 ++++++++++++++++++++++++++++ internal/cgroup/v2swap_test.go | 5 +- internal/processor/applyerr_test.go | 43 ++- internal/processor/processor.go | 39 ++- 10 files changed, 1171 insertions(+), 145 deletions(-) create mode 100644 internal/cgroup/converge_test.go create mode 100644 internal/cgroup/v2plan.go diff --git a/README.md b/README.md index 26ace13..d61e723 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,16 @@ than removing it first, so the container is never left unfiltered. That needs: changed and the container is retried: an unreadable program cannot be checked or safely replaced. This is stricter than runc, which ignores such programs, and such policies are unsupported. -- A device filter attached at all. A container whose cgroup has none (for - example after `systemctl daemon-reload` wiped it) is not granted anything. +- A device filter attached at all, or one this daemon has seen before. + `systemctl daemon-reload` can detach every device filter of a container; the + daemon remembers (in memory) the runtime filters it has seen per cgroup and + reattaches them, with its grants if the container has any. This puts the + runtime's own restrictions back too, which the reload would otherwise have + lifted. A daemon that did not see the container before the reload (for + example one started after it) cannot rebuild the filter: an unprivileged + container with no filter is retried with reason `filter_missing` and a + warning to restart the container, and is never reported as done. Privileged + containers have no device filter and are left alone. - A device filter made only of the instructions device filters normally use (context loads, simple ALU, jumps inside the program, exit; no helper calls, maps or 64-bit immediates) on Linux 4.16 or newer. The daemon rebuilds the @@ -91,6 +99,13 @@ without any saved state. A program that starts like such a header but does not check out completely is never modified: the daemon logs an error with reason `owned_block_conflict` and the container has to be restarted. +Every change attaches the new program before detaching the one it replaces, so +an interrupted change leaves the container narrower, never wider, and the next +pass finishes it. The daemon never tracks kernel program IDs. When it cannot +tell a runtime program from the leftover of a change that failed halfway, it +wraps both, which leaves one redundant, identical wrapper: the same grants, +never wider, and stable from then on. + > **Upgrading from versions without these markers:** programs attached by > earlier versions look exactly like the runtime's own filter, so the grants > they carry are kept (and wrapped again) until the container is restarted. diff --git a/internal/cgroup/api.go b/internal/cgroup/api.go index 60b0906..1796b5d 100644 --- a/internal/cgroup/api.go +++ b/internal/cgroup/api.go @@ -50,11 +50,12 @@ type Interface interface { } // New returns the cgroup API for version. ledger records the cgroup v1 -// grants this daemon made; it must outlive the call (one per Processor) -// and is required for version 1. +// grants this daemon made and cache the cgroup v2 runtime originals it has +// seen; both must outlive the call (one per Processor). ledger is required +// for version 1; a nil cache disables rebuilding wiped cgroup v2 filters. // //nolint:ireturn // intentional: callers use the interface -func New(version int, ledger *Ledger) (Interface, error) { +func New(version int, ledger *Ledger, cache *FilterCache) (Interface, error) { switch version { case 1: if ledger == nil { @@ -63,7 +64,7 @@ func New(version int, ledger *Ledger) (Interface, error) { return &cgroupv1{ledger: ledger, files: openatFiles{}}, nil case 2: - return &cgroupv2{ops: kernelOps{}, replace: processReplaceProbe}, nil + return &cgroupv2{ops: kernelOps{}, replace: processReplaceProbe, cache: cache}, nil default: return nil, fmt.Errorf( //nolint:err113 // dynamic content "invalid cgroup version %d", @@ -82,6 +83,7 @@ type ( cgroupv2 struct { ops v2ops replace *replaceProbe + cache *FilterCache } ) diff --git a/internal/cgroup/api_test.go b/internal/cgroup/api_test.go index 068b888..e7382db 100644 --- a/internal/cgroup/api_test.go +++ b/internal/cgroup/api_test.go @@ -94,48 +94,48 @@ func TestDeviceRule_JSON_RoundTrip(t *testing.T) { } func TestNew_InvalidVersion(t *testing.T) { - _, err := New(-1, &Ledger{}) + _, err := New(-1, &Ledger{}, nil) if err == nil { - t.Error("New(-1, &Ledger{}) must return an error") + t.Error("New(-1, &Ledger{}, nil) must return an error") } - _, err = New(99, &Ledger{}) + _, err = New(99, &Ledger{}, nil) if err == nil { - t.Error("New(99, &Ledger{}) must return an error") + t.Error("New(99, &Ledger{}, nil) must return an error") } } func TestNew_ValidVersions(t *testing.T) { - v1, err := New(1, &Ledger{}) + v1, err := New(1, &Ledger{}, nil) if err != nil { - t.Errorf("New(1, &Ledger{}) returned err: %v", err) + t.Errorf("New(1, &Ledger{}, nil) returned err: %v", err) } if v1 == nil { - t.Error("New(1, &Ledger{}) returned nil Interface") + t.Error("New(1, &Ledger{}, nil) returned nil Interface") } - v2, err := New(2, nil) + v2, err := New(2, nil, nil) if err != nil { - t.Errorf("New(2, nil) returned err: %v", err) + t.Errorf("New(2, nil, nil) returned err: %v", err) } if v2 == nil { - t.Error("New(2, nil) returned nil Interface") + t.Error("New(2, nil, nil) returned nil Interface") } } func TestNew_V1RequiresLedger(t *testing.T) { - _, err := New(1, nil) + _, err := New(1, nil, nil) if !errors.Is(err, errMissingLedger) { - t.Fatalf("New(1, nil) err = %v, want errMissingLedger", err) + t.Fatalf("New(1, nil, nil) err = %v, want errMissingLedger", err) } } func TestNew_InvalidVersionIncludesValue(t *testing.T) { - _, err := New(42, &Ledger{}) + _, err := New(42, &Ledger{}, nil) if err == nil { - t.Fatal("New(42, &Ledger{}) must return an error") + t.Fatal("New(42, &Ledger{}, nil) must return an error") } if !strings.Contains(err.Error(), "42") { diff --git a/internal/cgroup/converge_test.go b/internal/cgroup/converge_test.go new file mode 100644 index 0000000..a9b4360 --- /dev/null +++ b/internal/cgroup/converge_test.go @@ -0,0 +1,489 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cgroup + +import ( + "bytes" + "errors" + "slices" + "strconv" + "testing" + + "golang.org/x/sys/unix" +) + +// pass runs one SetDeviceRules pass with the given replace support and +// cache, and clears the recorded calls first so each pass is asserted alone. +func pass(ops *fakeOps, replace int32, cache *FilterCache, rules []DeviceRule) error { + ops.calls = nil + c := &cgroupv2{ops: ops, replace: probeIn(replace), cache: cache} + + return c.setDeviceRules(3, testIdentity, rules) +} + +// attachedPrograms parses every attached program: wrappers by lineage +// nonce, bare programs by their bytes. +func attachedPrograms( + t *testing.T, + ops *fakeOps, +) (nonces []uint64, originals [][]byte, bares [][]byte) { + t.Helper() + + for _, name := range ops.attached { + raw := attachedRaw(t, ops, name) + + parsed, owned, err := parseOwned(raw) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + + if owned { + nonces = append(nonces, parsed.nonce) + originals = append(originals, parsed.original) + } else { + bares = append(bares, raw) + } + } + + return nonces, originals, bares +} + +func grantRules() []DeviceRule { + return []DeviceRule{rule("c", 10, 200, "rwm")} +} + +// TestConverge_WrapRollbackFailureThenRetry: the first wrap attaches N1 but +// neither the original's detach nor the rollback succeed, leaving O1 and N1 +// attached. The retry keeps N1 and wraps O1 as its own pair: two identical +// wrappers (one redundant, not wider), no bare program left, and nothing +// changes on later passes. +func TestConverge_WrapRollbackFailureThenRetry(t *testing.T) { + ops := ownedOps(t, map[string][]byte{"O1": mustCanonical(t, allowNullOriginal())}) + ops.errs["detach O1"] = errInjected + ops.errs["detach N1"] = errRollback + + err := pass(ops, replaceUnsupported, nil, grantRules()) + if !errors.Is(err, errRollback) { + t.Fatalf("err = %v, want the joined rollback failure", err) + } + + assertAttached(t, ops, "O1", "N1") + + ops.errs = map[string]error{} + + err = pass(ops, replaceUnsupported, nil, grantRules()) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops, "load N2", "attach N2", "detach O1") + + nonces, originals, bares := attachedPrograms(t, ops) + if len(bares) != 0 || len(nonces) != 2 || nonces[0] == nonces[1] || + !bytes.Equal(originals[0], originals[1]) { + t.Fatalf( + "want two identical-original wrappers with distinct nonces and no bare, got nonces %v bares %d", + nonces, + len(bares), + ) + } + + for range 2 { + err = pass(ops, replaceUnsupported, nil, grantRules()) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops) + } + + ops.assertAllClosed(t) +} + +// TestConverge_RollbackSucceededThenRetryIsOneProgramPerOriginal is the 0.1 +// case whose rollback worked: pair 1 replaced, pair 2 back to its original; +// the retry ends with exactly one wrapper per runtime original. +func TestConverge_RollbackSucceededThenRetryIsOneProgramPerOriginal(t *testing.T) { + origA := mustCanonical(t, allowNullOriginal()) + origB := mustCanonical(t, runtimeOriginal()) + ops := ownedOps(t, map[string][]byte{"O1": origA, "O2": origB}) + ops.errs["detach O2"] = errInjected + + err := pass(ops, replaceUnsupported, nil, grantRules()) + if !errors.Is(err, errInjected) { + t.Fatalf("err = %v", err) + } + + assertAttached(t, ops, "O2", "N1") + + ops.errs = map[string]error{} + + err = pass(ops, replaceUnsupported, nil, grantRules()) + if err != nil { + t.Fatal(err) + } + + _, originals, bares := attachedPrograms(t, ops) + if len(bares) != 0 || len(originals) != 2 || bytes.Equal(originals[0], originals[1]) { + t.Fatalf( + "want one wrapper per original, got %d wrappers %d bares", + len(originals), + len(bares), + ) + } + + err = pass(ops, replaceUnsupported, nil, grantRules()) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops) +} + +func TestConverge_IdenticalBaresBecomeTwoWrappersEachByItsOwnPair(t *testing.T) { + orig := mustCanonical(t, allowNullOriginal()) + ops := ownedOps(t, map[string][]byte{"O1": orig, "O2": orig}) + + err := pass(ops, replaceUnsupported, nil, grantRules()) + if err != nil { + t.Fatal(err) + } + + // Each bare program is detached only right after its own wrapper is in. + assertCalls(t, ops, "load N1", "load N2", "attach N1", "detach O1", "attach N2", "detach O2") + + nonces, _, bares := attachedPrograms(t, ops) + if len(bares) != 0 || len(nonces) != 2 || nonces[0] == nonces[1] { + t.Fatalf("want two wrappers with distinct nonces, got %v (bares %d)", nonces, len(bares)) + } +} + +func wrapperOf(t *testing.T, rules []DeviceRule, nonce uint64) []byte { + t.Helper() + + return mustEmit(t, rules, nonce, mustCanonical(t, allowNullOriginal())) +} + +func TestConverge_EmptyRestoreDetachFailureThenRetry(t *testing.T) { + orig := mustCanonical(t, allowNullOriginal()) + ops := ownedOps(t, map[string][]byte{"W1": wrapperOf(t, grantRules(), 11)}) + ops.errs["detach W1"] = errInjected + + err := pass(ops, replaceUnsupported, nil, nil) + if !errors.Is(err, errInjected) { + t.Fatalf("err = %v", err) + } + + // The rollback worked: only the wrapper is attached. + assertAttached(t, ops, "W1") + + ops.errs = map[string]error{} + + err = pass(ops, replaceUnsupported, nil, nil) + if err != nil { + t.Fatal(err) + } + + nonces, _, bares := attachedPrograms(t, ops) + if len(nonces) != 0 || len(bares) != 1 || !bytes.Equal(bares[0], orig) { + t.Fatalf("want a single bare original, got %d wrappers %d bares", len(nonces), len(bares)) + } +} + +// TestConverge_EmptyRestoreRollbackFailureThenRetry: the bare original is +// attached, the wrapper's detach fails and so does the rollback of the +// original. The retry sees the original already bare and only detaches the +// wrapper (removing a wrapper next to its own original only narrows). +func TestConverge_EmptyRestoreRollbackFailureThenRetry(t *testing.T) { + orig := mustCanonical(t, allowNullOriginal()) + ops := ownedOps(t, map[string][]byte{"W1": wrapperOf(t, grantRules(), 11)}) + ops.errs["detach W1"] = errInjected + ops.errs["detach N1"] = errRollback + + err := pass(ops, replaceUnsupported, nil, nil) + if !errors.Is(err, errRollback) { + t.Fatalf("err = %v", err) + } + + assertAttached(t, ops, "W1", "N1") + + ops.errs = map[string]error{} + + err = pass(ops, replaceUnsupported, nil, nil) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops, "detach W1") + assertAttached(t, ops, "N1") + + if !bytes.Equal(attachedRaw(t, ops, "N1"), orig) { + t.Error("remaining program is not the bare original") + } + + err = pass(ops, replaceUnsupported, nil, nil) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops) +} + +// TestConverge_EmptyRestoreKeepsIdenticalRuntimeProgramCount checks that +// one bare program satisfies one lineage: two identical originals, each +// wrapped, are restored as two bare programs. +func TestConverge_EmptyRestoreKeepsIdenticalRuntimeProgramCount(t *testing.T) { + ops := ownedOps(t, map[string][]byte{ + "W1": wrapperOf(t, grantRules(), 11), + "W2": wrapperOf(t, grantRules(), 12), + }) + + err := pass(ops, replaceSupported, nil, nil) + if err != nil { + t.Fatal(err) + } + + nonces, _, bares := attachedPrograms(t, ops) + if len(nonces) != 0 || len(bares) != 2 { + t.Fatalf("want two bare originals, got %d wrappers %d bares", len(nonces), len(bares)) + } +} + +func TestConverge_StaleSameNonceWrappersLeaveOne(t *testing.T) { + older := []DeviceRule{rule("c", 10, 201, "r")} + + t.Run("none equal to desired", func(t *testing.T) { + ops := ownedOps( + t, + map[string][]byte{"W1": wrapperOf(t, older, 11), "W2": wrapperOf(t, nil, 11)}, + ) + + err := pass(ops, replaceUnsupported, nil, grantRules()) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops, "load N1", "attach N1", "detach W1", "detach W2") + assertAttached(t, ops, "N1") + }) + + t.Run("one already equal", func(t *testing.T) { + ops := ownedOps( + t, + map[string][]byte{"W1": wrapperOf(t, older, 11), "W2": wrapperOf(t, grantRules(), 11)}, + ) + + err := pass(ops, replaceUnsupported, nil, grantRules()) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops, "detach W1") + assertAttached(t, ops, "W2") + }) +} + +func TestConverge_IdenticalWrappersWithDifferentNoncesBothRemain(t *testing.T) { + ops := ownedOps(t, map[string][]byte{ + "W1": wrapperOf(t, grantRules(), 11), + "W2": wrapperOf(t, grantRules(), 12), + }) + + err := pass(ops, replaceUnsupported, nil, grantRules()) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops) + assertAttached(t, ops, "W1", "W2") +} + +func TestConverge_WrapperOverWrapperIsNormalised(t *testing.T) { + orig := mustCanonical(t, allowNullOriginal()) + inner := mustEmit(t, []DeviceRule{rule("c", 10, 201, "r")}, 21, orig) + ops := ownedOps(t, map[string][]byte{"W1": mustEmit(t, grantRules(), 22, inner)}) + + err := pass(ops, replaceSupported, nil, grantRules()) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(attachedRaw(t, ops, "N1"), mustEmit(t, grantRules(), 22, orig)) { + t.Error( + "wrapper over a wrapper was not re-emitted as one wrapper over the innermost original", + ) + } +} + +func TestConverge_WipeAfterWrapRebuildsSameLineage(t *testing.T) { + orig := mustCanonical(t, allowNullOriginal()) + cache := &FilterCache{} + ops := ownedOps(t, map[string][]byte{"O1": orig}) + + err := pass(ops, replaceSupported, cache, grantRules()) + if err != nil { + t.Fatal(err) + } + + nonces, _, _ := attachedPrograms(t, ops) + + ops.attached = nil // systemd daemon-reload + + err = pass(ops, replaceSupported, cache, grantRules()) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops, "load N2", "attach N2") + + rebuiltNonces, originals, _ := attachedPrograms(t, ops) + if len(rebuiltNonces) != 1 || rebuiltNonces[0] != nonces[0] || + !bytes.Equal(originals[0], orig) { + t.Fatalf("rebuilt lineage %v, want nonce %v around the original", rebuiltNonces, nonces) + } + + raw := attachedRaw(t, ops, "N2") + if runDeviceFilter(t, raw, charReq(10, 200, unix.BPF_DEVCG_ACC_READ)) != 1 || + runDeviceFilter(t, raw, charReq(10, 201, unix.BPF_DEVCG_ACC_READ)) != 0 { + t.Error("rebuilt wrapper does not carry exactly the desired grant over the original") + } +} + +func TestConverge_WipeWithEmptyRulesRestoresBareOriginal(t *testing.T) { + orig := mustCanonical(t, allowNullOriginal()) + cache := &FilterCache{} + ops := ownedOps(t, map[string][]byte{"O1": orig}) + + err := pass(ops, replaceSupported, cache, grantRules()) + if err != nil { + t.Fatal(err) + } + + ops.attached = nil + + err = pass(ops, replaceSupported, cache, nil) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops, "load N2", "attach N2") + + if !bytes.Equal(attachedRaw(t, ops, "N2"), orig) { + t.Error("wipe with empty rules did not restore the bare runtime original") + } +} + +func TestConverge_ForgottenCacheCannotRebuild(t *testing.T) { + cache := &FilterCache{} + ops := ownedOps(t, map[string][]byte{"O1": mustCanonical(t, allowNullOriginal())}) + + err := pass(ops, replaceSupported, cache, grantRules()) + if err != nil { + t.Fatal(err) + } + + cache.Forget(testIdentity) + + ops.attached = nil + + err = pass(ops, replaceSupported, cache, grantRules()) + if !errors.Is(err, ErrFilterMissing) { + t.Fatalf("err = %v, want ErrFilterMissing", err) + } + + assertCalls(t, ops) +} + +func TestConverge_CacheIsKeyedByIdentity(t *testing.T) { + cache := &FilterCache{} + cache.set(testIdentity, cacheEntry{lineages: []cachedLineage{{original: []byte{1}}}}) + + restarted := testIdentity + restarted.Inode++ + + if got := cache.get(restarted).lineages; len(got) != 0 { + t.Errorf("a new cgroup inode at the same path sees %d cached lineages, want none", len(got)) + } +} + +// TestConverge_PartialRebuildResumes: the cgroup had two runtime programs, +// the wipe removed both, and the rebuild attaches the first but fails on the +// second. The retry sees one program attached (not a wiped cgroup) and must +// still reattach the lost lineage instead of forgetting it. +func TestConverge_PartialRebuildResumes(t *testing.T) { + origA := mustCanonical(t, allowNullOriginal()) + origB := mustCanonical(t, runtimeOriginal()) + cache := &FilterCache{} + ops := ownedOps(t, map[string][]byte{"O1": origA, "O2": origB}) + + for _, rules := range [][]DeviceRule{grantRules(), nil} { + err := pass(ops, replaceSupported, cache, grantRules()) + if err != nil { + t.Fatal(err) + } + + ops.attached = nil + ops.errs = map[string]error{} + failing := "attach N" + strconv.Itoa(ops.loads+2) + ops.errs[failing] = errInjected + + err = pass(ops, replaceSupported, cache, rules) + if !errors.Is(err, errInjected) { + t.Fatalf("rules %v: err = %v, want the injected %s failure", rules, err, failing) + } + + if len(ops.attached) != 1 { + t.Fatalf("rules %v: attached = %q, want the first lineage only", rules, ops.attached) + } + + ops.errs = map[string]error{} + + err = pass(ops, replaceSupported, cache, rules) + if err != nil { + t.Fatal(err) + } + + var got [][]byte + + nonces, originals, bares := attachedPrograms(t, ops) + got = append(append(got, originals...), bares...) + + if len(got) != 2 || + !slices.ContainsFunc(got, func(b []byte) bool { return bytes.Equal(b, origA) }) || + !slices.ContainsFunc(got, func(b []byte) bool { return bytes.Equal(b, origB) }) { + t.Fatalf( + "rules %v: after the retry want both runtime originals, got %d wrappers %d bares", + rules, + len(nonces), + len(bares), + ) + } + + if cache.get(testIdentity).rebuilding { + t.Errorf("rules %v: rebuild still pending after a complete pass", rules) + } + + err = pass(ops, replaceSupported, cache, rules) + if err != nil { + t.Fatal(err) + } + + assertCalls(t, ops) + } +} diff --git a/internal/cgroup/owned_test.go b/internal/cgroup/owned_test.go index 2596020..29a73db 100644 --- a/internal/cgroup/owned_test.go +++ b/internal/cgroup/owned_test.go @@ -400,7 +400,7 @@ func ownedOps(t *testing.T, programs map[string][]byte) *fakeOps { func setRules(ops *fakeOps, rules []DeviceRule) error { c := &cgroupv2{ops: ops, replace: probeIn(replaceSupported)} - return c.setDeviceRules(3, "/sys/fs/cgroup/test", rules) + return c.setDeviceRules(3, testIdentity, rules) } func attachedRaw(t *testing.T, ops *fakeOps, name string) []byte { @@ -501,15 +501,21 @@ func TestSet_EmptyRulesLeaveUnmarkedProgramsAlone(t *testing.T) { ops.assertAllClosed(t) } -func TestSet_EmptyRulesWithNothingAttachedIsANoOp(t *testing.T) { - ops := newFakeOps() +func TestSet_NothingAttachedAndNothingCachedIsFilterMissing(t *testing.T) { + for _, rules := range [][]DeviceRule{nil, {rule("c", 10, 200, "rwm")}} { + ops := newFakeOps() + + err := setRules(ops, rules) + if !errors.Is(err, ErrFilterMissing) { + t.Fatalf( + "rules %v: err = %v, want ErrFilterMissing (never a silent success)", + rules, + err, + ) + } - err := setRules(ops, nil) - if err != nil { - t.Fatalf("err = %v, want a no-op", err) + assertCalls(t, ops) } - - assertCalls(t, ops) } func TestSet_ConflictMutatesNothing(t *testing.T) { diff --git a/internal/cgroup/v2.go b/internal/cgroup/v2.go index 176222e..dfd9b07 100644 --- a/internal/cgroup/v2.go +++ b/internal/cgroup/v2.go @@ -155,36 +155,34 @@ func scanProcCgroupV2(r io.Reader, path string, prefix string) (string, error) { // cgroup behind handle equal exactly rules. // // Each attached program is either one of this daemon's wrappers (see -// owned.go), which is stripped back to the runtime original it wraps, or an -// unmarked program, which is the original itself. Non-empty rules replace -// every program with a fresh wrapper around its original; empty rules -// replace every wrapper with its bare original and leave unmarked programs -// alone, so a cgroup this daemon never touched is never touched. Nothing -// but the programs themselves carries state, so this works the same after -// a daemon restart. +// owned.go), stripped back to the runtime original it wraps, or an unmarked +// program, which is a runtime original itself. Non-empty rules leave one +// wrapper per lineage around its original; empty rules put the bare +// originals back and leave unmarked programs alone, so a cgroup this daemon +// never touched is never touched. The programs carry all the state this +// needs, so it works the same after a daemon restart; the one exception is +// a cgroup whose filters were all detached (systemd daemon-reload), which +// is rebuilt from FilterCache when this process saw it before. // // Unmarked programs are never interpreted: a wrapper written by a version // that predates the ownership markers looks exactly like a runtime program, -// so its grants are kept until the container restarts. +// so its grants are kept until the container restarts. Kernel program IDs +// are reused and are never used as identity; every equality decision +// compares full canonical instruction bytes. // // Unlike NVIDIA's upstream code, which detaches every program before // attaching the replacements because it runs strictly before the container // starts, this daemon mutates live containers: detaching first would leave // the container unfiltered in between, and permanently so if the attach -// failed or the daemon died. Programs are therefore replaced pairwise, each -// new program attached (or atomically swapped in) before its original goes. +// failed or the daemon died. Every new program is therefore attached (or +// atomically swapped in) before anything it replaces is detached. func (c *cgroupv2) SetDeviceRules(handle *CgroupHandle, rules []DeviceRule) error { - return c.setDeviceRules(handle.fd, handle.Identity().Path, rules) + return c.setDeviceRules(handle.fd, handle.Identity(), rules) } -// replacement is the planned new program for one attached program. -type replacement struct { - old progHandle - raw []byte - name string -} +const multiFlag = unix.BPF_F_ALLOW_MULTI -func (c *cgroupv2) setDeviceRules(dirFD int, cgroupPath string, rules []DeviceRule) error { +func (c *cgroupv2) setDeviceRules(dirFD int, id Identity, rules []DeviceRule) error { // Find any existing eBPF device filter programs attached to this cgroup. oldProgs, total, inaccessible, attachFlags, err := c.ops.query(dirFD) if err != nil { @@ -198,10 +196,8 @@ func (c *cgroupv2) setDeviceRules(dirFD int, cgroupPath string, rules []DeviceRu switch { case inaccessible > 0: return fmt.Errorf("%w: %d of %d programs", ErrFiltersInaccessible, inaccessible, total) - case total == 0 && len(rules) == 0: - return nil case total == 0: - return ErrFilterMissing + return c.rebuild(dirFD, id, rules) case attachFlags&unix.BPF_F_ALLOW_MULTI == 0: return fmt.Errorf( "%w: attach mode %s", @@ -210,128 +206,88 @@ func (c *cgroupv2) setDeviceRules(dirFD int, cgroupPath string, rules []DeviceRu ) } - // Plan every replacement before touching the cgroup: a conflict, an - // unwrappable program or a load failure anywhere mutates nothing. - plans := make([]replacement, 0, len(oldProgs)) + // Classify and plan everything before touching the cgroup: a conflict, + // an unwrappable program or a load failure anywhere mutates nothing. + classified := make([]attachedProg, 0, len(oldProgs)) for _, oldProg := range oldProgs { - var ( - plan replacement - keep bool - ) + var prog attachedProg - plan, keep, err = c.planReplacement(oldProg, cgroupPath, rules) + prog, err = c.classify(oldProg) if err != nil { return err } - if !keep { - plans = append(plans, plan) - } + classified = append(classified, prog) } - newProgs := make([]progHandle, 0, len(plans)) - defer func() { closeAll(newProgs) }() + groups, bares := groupWrappers(classified) - for _, plan := range plans { - var newProg progHandle + var plan passPlan - newProg, err = c.loadCanonical(plan.raw, plan.name) + if len(rules) == 0 { + plan = planRestore(groups, bares) + } else { + plan, err = planGrant(groups, bares, rules, id.Path) if err != nil { return err } - - newProgs = append(newProgs, newProg) } - // Pairs are independent: under BPF_F_ALLOW_MULTI the verdict is the AND of - // every program, so a half-replaced set is narrower, never wider (an - // original not yet replaced masks the new grant). A failure therefore - // stops here without undoing earlier pairs; a retry completes the rest. - for idx, plan := range plans { - err = c.swap(dirFD, plan.old, newProgs[idx]) + // A rebuild that stopped halfway left some lineages attached and others + // not; the missing ones were lost to the wipe, so they are attached + // again rather than dropped from the cache. + entry := c.cache.get(id) + if entry.rebuilding { + err = plan.addRebuild(missingLineages(entry.lineages, groups, bares), rules) if err != nil { - return fmt.Errorf( - "replace device filter program %d of %d: %w", - idx+1, - len(plans), - err, - ) + return err } } - return nil + return c.commit(dirFD, id, plan, entry.rebuilding) } -// planReplacement decides what replaces oldProg. keep reports that oldProg -// stays as it is (an unmarked program under empty rules). -func (c *cgroupv2) planReplacement( - oldProg progHandle, - cgroupPath string, - rules []DeviceRule, -) (replacement, bool, error) { - insts, meta, err := c.ops.instructions(oldProg) - if err != nil { - return replacement{}, false, err - } +// commit records plan's lineages and executes it. rebuilding stays set in +// the cache until a pass that reattaches every lineage has succeeded. +func (c *cgroupv2) commit(dirFD int, id Identity, plan passPlan, rebuilding bool) error { + c.cache.set(id, cacheEntry{lineages: plan.lineages, rebuilding: rebuilding}) - raw, err := canonicalBytes(insts) + err := c.execute(dirFD, plan) if err != nil { - return replacement{}, false, err + return err } - owned, isOwned, err := parseOwned(raw) - if err != nil { - return replacement{}, false, err + if rebuilding { + c.cache.set(id, cacheEntry{lineages: plan.lineages}) } - if !isOwned && len(rules) == 0 { - return replacement{}, true, nil - } + return nil +} - original := raw - if isOwned { - original = owned.original +// rebuild handles a cgroup with no device filter at all. That is what +// systemd's daemon-reload leaves behind, so it is not a no-op for a +// container that had one: its runtime originals are attached again from +// FilterCache. Without a cached entry nothing can be rebuilt, and +// ErrFilterMissing lets the caller decide (a privileged container never had +// a filter; anything else must not be reported as done). +func (c *cgroupv2) rebuild(dirFD int, id Identity, rules []DeviceRule) error { + entry := c.cache.get(id) + if len(entry.lineages) == 0 { + return ErrFilterMissing } - origInsts, err := fromCanonical(original) - if err != nil { - return replacement{}, false, err - } + var plan passPlan - err = checkWrappable(origInsts, meta) + err := plan.addRebuild(entry.lineages, rules) if err != nil { - return replacement{}, false, err + return err } - if len(rules) == 0 { - return replacement{old: oldProg, raw: original}, false, nil - } - - var nonce uint64 - - if isOwned { - nonce = owned.nonce - } else { - nonce, err = newNonce() - if err != nil { - return replacement{}, false, err - } - - logger.L(). - Info("wrapping device filter with no prior owned block; pre-upgrade grants, if any, are not managed", - "cgroup", cgroupPath) - } - - wrapped, err := emitOwned(rules, nonce, original) - if err != nil { - return replacement{}, false, fmt.Errorf( - "unable to generate new device filter program: %w", - err, - ) - } + logger.L().Info("device filter missing; reattaching the cached runtime originals", + "cgroup", id.Path, "programs", len(entry.lineages), "with_grants", len(rules) > 0) - return replacement{old: oldProg, raw: wrapped, name: ownedProgramName}, false, nil + return c.commit(dirFD, id, plan, true) } // loadCanonical loads a program from canonical bytes. A verifier rejection diff --git a/internal/cgroup/v2plan.go b/internal/cgroup/v2plan.go new file mode 100644 index 0000000..b255cf0 --- /dev/null +++ b/internal/cgroup/v2plan.go @@ -0,0 +1,491 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cgroup + +import ( + "bytes" + "fmt" + "slices" + "sync" + + "github.com/leinardi/swarm-device-access/internal/logger" +) + +// FilterCache remembers, per cgroup identity, the runtime originals (and +// the wrapper nonce of each lineage) that were attached the last time this +// daemon looked. systemd's daemon-reload can detach every device filter of +// a cgroup; the programs themselves then carry nothing to recover from, and +// this cache is the only way to put the runtime's restrictions back, with +// or without this daemon's grants. It lives only as long as the process: a +// daemon that did not see a cgroup before its filter was wiped cannot +// rebuild it. +// +// Entries are keyed by Identity, so a container restarted with the same ID +// (a new cgroup inode) starts empty; Forget drops an entry when its +// lifecycle ends. The zero value is ready to use. +type FilterCache struct { + mu sync.Mutex + entries map[Identity]cacheEntry +} + +// cacheEntry is what is known about one cgroup. rebuilding is set while a +// rebuild has not completed: until it has, lineages absent from the cgroup +// were lost to the wipe (not removed by the runtime) and are reattached by +// the next pass instead of being dropped from the entry. +type cacheEntry struct { + lineages []cachedLineage + rebuilding bool +} + +// cachedLineage is one runtime original and the nonce of the wrapper +// lineage around it (0 when it has never been wrapped). +type cachedLineage struct { + nonce uint64 + original []byte + meta progMeta +} + +// Forget drops what is cached for id. +func (f *FilterCache) Forget(id Identity) { + if f == nil { + return + } + + f.mu.Lock() + defer f.mu.Unlock() + + delete(f.entries, id) +} + +func (f *FilterCache) get(id Identity) cacheEntry { + if f == nil { + return cacheEntry{} + } + + f.mu.Lock() + defer f.mu.Unlock() + + entry := f.entries[id] + entry.lineages = slices.Clone(entry.lineages) + + return entry +} + +func (f *FilterCache) set(id Identity, entry cacheEntry) { + if f == nil { + return + } + + f.mu.Lock() + defer f.mu.Unlock() + + if f.entries == nil { + f.entries = make(map[Identity]cacheEntry) + } + + f.entries[id] = entry +} + +// attachedProg is one attached program, classified. +type attachedProg struct { + handle progHandle + raw []byte + meta progMeta + + // For validated wrappers: the lineage nonce and the innermost original. + owned bool + nonce uint64 + original []byte +} + +// wrapperGroup holds the wrappers of one lineage: same validated nonce and +// byte-identical original. Two identical runtime originals wrapped +// separately have different nonces and are never merged. +type wrapperGroup struct { + nonce uint64 + original []byte + meta progMeta + members []attachedProg +} + +// step is one mutation. old nil attaches newIdx on its own (rebuild); newIdx +// -1 detaches old; otherwise newIdx replaces old as a pair. +type step struct { + old progHandle + newIdx int +} + +// passPlan is the complete set of changes for one cgroup, built before +// anything is loaded or attached. +type passPlan struct { + loads [][]byte + names []string + steps []step + lineages []cachedLineage +} + +func (p *passPlan) load(raw []byte, name string) int { + p.loads = append(p.loads, raw) + p.names = append(p.names, name) + + return len(p.loads) - 1 +} + +func (p *passPlan) pair(old progHandle, raw []byte, name string) { + p.steps = append(p.steps, step{old: old, newIdx: p.load(raw, name)}) +} + +func (p *passPlan) detach(old progHandle) { + p.steps = append(p.steps, step{old: old, newIdx: -1}) +} + +// classify parses one attached program. Wrappers are stripped to the +// innermost original, so a wrapper around a wrapper is re-emitted as one. +func (c *cgroupv2) classify(prog progHandle) (attachedProg, error) { + insts, meta, err := c.ops.instructions(prog) + if err != nil { + return attachedProg{}, err + } + + raw, err := canonicalBytes(insts) + if err != nil { + return attachedProg{}, err + } + + owned, isOwned, err := parseOwned(raw) + if err != nil { + return attachedProg{}, err + } + + if !isOwned { + return attachedProg{handle: prog, raw: raw, meta: meta, original: raw}, nil + } + + original := owned.original + + for { + inner, innerOwned, innerErr := parseOwned(original) + if innerErr != nil { + return attachedProg{}, innerErr + } + + if !innerOwned { + break + } + + original = inner.original + } + + err = gate(original, meta) + if err != nil { + return attachedProg{}, err + } + + return attachedProg{ + handle: prog, raw: raw, meta: meta, + owned: true, nonce: owned.nonce, original: original, + }, nil +} + +func gate(original []byte, meta progMeta) error { + insts, err := fromCanonical(original) + if err != nil { + return err + } + + return checkWrappable(insts, meta) +} + +// groupWrappers splits classified programs into wrapper lineages (in order +// of first appearance) and bare programs. +func groupWrappers(progs []attachedProg) ([]wrapperGroup, []attachedProg) { + var ( + groups []wrapperGroup + bares []attachedProg + ) + + for _, prog := range progs { + if !prog.owned { + bares = append(bares, prog) + + continue + } + + idx := slices.IndexFunc(groups, func(g wrapperGroup) bool { + return g.nonce == prog.nonce && bytes.Equal(g.original, prog.original) + }) + if idx < 0 { + groups = append( + groups, + wrapperGroup{nonce: prog.nonce, original: prog.original, meta: prog.meta}, + ) + idx = len(groups) - 1 + } + + groups[idx].members = append(groups[idx].members, prog) + } + + return groups, bares +} + +// planGrant plans a pass for non-empty rules. Each wrapper lineage ends as +// exactly one wrapper equal to the desired one (an existing equal member is +// kept, otherwise the first member is replaced by it), and the other +// members of the lineage are detached after it is in place. Each bare +// program is wrapped as its own pair under a new nonce, even when an +// identical original is already wrapped: without IDs a second runtime +// program and the leftover of a failed rollback look the same, and wrapping +// both only costs one redundant, identical wrapper. +func planGrant( + groups []wrapperGroup, + bares []attachedProg, + rules []DeviceRule, + cgroupPath string, +) (passPlan, error) { + var plan passPlan + + for _, group := range groups { + desired, err := emitOwned(rules, group.nonce, group.original) + if err != nil { + return passPlan{}, err + } + + keep := slices.IndexFunc( + group.members, + func(m attachedProg) bool { return bytes.Equal(m.raw, desired) }, + ) + if keep < 0 { + plan.pair(group.members[0].handle, desired, ownedProgramName) + + keep = 0 + } + + for idx, member := range group.members { + if idx != keep { + plan.detach(member.handle) + } + } + + plan.lineages = append( + plan.lineages, + cachedLineage{nonce: group.nonce, original: group.original, meta: group.meta}, + ) + } + + for _, bare := range bares { + err := gate(bare.raw, bare.meta) + if err != nil { + return passPlan{}, err + } + + nonce, err := newNonce() + if err != nil { + return passPlan{}, err + } + + wrapped, err := emitOwned(rules, nonce, bare.raw) + if err != nil { + return passPlan{}, err + } + + logger.L(). + Info("wrapping device filter with no prior owned block; pre-upgrade grants, if any, are not managed", + "cgroup", cgroupPath) + + plan.pair(bare.handle, wrapped, ownedProgramName) + plan.lineages = append( + plan.lineages, + cachedLineage{nonce: nonce, original: bare.raw, meta: bare.meta}, + ) + } + + return plan, nil +} + +// planRestore plans a pass for empty rules: every wrapper lineage ends as +// its bare original. A lineage whose original is already attached bare +// (from before this pass; one bare satisfies one lineage, so identical +// runtime programs keep their count) only has its wrappers detached: that +// is how a restore whose detach failed converges on the next pass. Any +// other lineage gets its original attached in place of its first wrapper, +// then the rest detached. Bare programs are left alone. +func planRestore(groups []wrapperGroup, bares []attachedProg) passPlan { + var plan passPlan + + available := make([]bool, len(bares)) + for idx := range available { + available[idx] = true + } + + for _, group := range groups { + match := -1 + + for idx, bare := range bares { + if available[idx] && bytes.Equal(bare.raw, group.original) { + match = idx + + break + } + } + + members := group.members + + if match >= 0 { + available[match] = false + } else { + plan.pair(members[0].handle, group.original, "") + members = members[1:] + } + + for _, member := range members { + plan.detach(member.handle) + } + + plan.lineages = append( + plan.lineages, + cachedLineage{nonce: group.nonce, original: group.original, meta: group.meta}, + ) + } + + for idx, bare := range bares { + if available[idx] { + plan.lineages = append( + plan.lineages, + cachedLineage{original: bare.raw, meta: bare.meta}, + ) + } + } + + return plan +} + +// addRebuild adds, for each cached lineage, an attach of its original: +// bare for empty rules, wrapped under the lineage nonce otherwise. +func (p *passPlan) addRebuild(cached []cachedLineage, rules []DeviceRule) error { + for _, lineage := range cached { + err := gate(lineage.original, lineage.meta) + if err != nil { + return err + } + + if len(rules) == 0 { + p.steps = append(p.steps, step{newIdx: p.load(lineage.original, "")}) + p.lineages = append(p.lineages, lineage) + + continue + } + + if lineage.nonce == 0 { + lineage.nonce, err = newNonce() + if err != nil { + return err + } + } + + wrapped, err := emitOwned(rules, lineage.nonce, lineage.original) + if err != nil { + return err + } + + p.steps = append(p.steps, step{newIdx: p.load(wrapped, ownedProgramName)}) + p.lineages = append(p.lineages, lineage) + } + + return nil +} + +// missingLineages returns the cached lineages the cgroup does not show: no +// wrapper of the same nonce and original, and no bare copy of the original +// left over for it (each attached program accounts for one lineage). +func missingLineages( + cached []cachedLineage, + groups []wrapperGroup, + bares []attachedProg, +) []cachedLineage { + groupUsed := make([]bool, len(groups)) + bareUsed := make([]bool, len(bares)) + + var missing []cachedLineage + + for _, lineage := range cached { + found := false + + for idx, group := range groups { + if !groupUsed[idx] && group.nonce == lineage.nonce && + bytes.Equal(group.original, lineage.original) { + groupUsed[idx], found = true, true + + break + } + } + + for idx, bare := range bares { + if found { + break + } + + if !bareUsed[idx] && bytes.Equal(bare.raw, lineage.original) { + bareUsed[idx], found = true, true + } + } + + if !found { + missing = append(missing, lineage) + } + } + + return missing +} + +// execute loads every new program, then applies the steps in order. It +// stops at the first failure: attach always precedes detach, and under +// BPF_F_ALLOW_MULTI every intermediate state is at most as wide as the +// desired one, so the next pass continues from wherever this one stopped. +func (c *cgroupv2) execute(dirFD int, plan passPlan) error { + loaded := make([]progHandle, 0, len(plan.loads)) + defer func() { closeAll(loaded) }() + + for idx, raw := range plan.loads { + prog, err := c.loadCanonical(raw, plan.names[idx]) + if err != nil { + return err + } + + loaded = append(loaded, prog) + } + + for idx, st := range plan.steps { + var err error + + switch { + case st.old == nil: + err = c.ops.attach(loaded[st.newIdx], dirFD, multiFlag, nil) + case st.newIdx < 0: + err = c.ops.detach(st.old, dirFD) + default: + err = c.swap(dirFD, st.old, loaded[st.newIdx]) + } + + if err != nil { + return fmt.Errorf("device filter change %d of %d: %w", idx+1, len(plan.steps), err) + } + } + + return nil +} diff --git a/internal/cgroup/v2swap_test.go b/internal/cgroup/v2swap_test.go index c4cafd9..de70d2e 100644 --- a/internal/cgroup/v2swap_test.go +++ b/internal/cgroup/v2swap_test.go @@ -205,6 +205,9 @@ func testRules() []DeviceRule { return []DeviceRule{{Allow: true, Type: "c", Major: &major, Minor: &minor, Access: "rwm"}} } +// testIdentity is the cgroup the fake ops stand for. +var testIdentity = Identity{Path: "/sys/fs/cgroup/test", Inode: 4242} + func probeIn(state int32) *replaceProbe { probe := &replaceProbe{} probe.state.Store(state) @@ -215,7 +218,7 @@ func probeIn(state int32) *replaceProbe { func run(ops *fakeOps, probe *replaceProbe) error { c := &cgroupv2{ops: ops, replace: probe} - return c.setDeviceRules(3, "/sys/fs/cgroup/test", testRules()) + return c.setDeviceRules(3, testIdentity, testRules()) } func assertCalls(t *testing.T, ops *fakeOps, want ...string) { diff --git a/internal/processor/applyerr_test.go b/internal/processor/applyerr_test.go index a93c9bf..ce88d0a 100644 --- a/internal/processor/applyerr_test.go +++ b/internal/processor/applyerr_test.go @@ -72,7 +72,7 @@ func devNullProcessor(t *testing.T, fake *failingCgroup) *Processor { const pid = 70 - real2, err := cgroup.New(2, nil) + real2, err := cgroup.New(2, nil, nil) if err != nil { t.Fatal(err) } @@ -86,10 +86,12 @@ func devNullProcessor(t *testing.T, fake *failingCgroup) *Processor { {Source: "/dev/null", Destination: "/dev/null", Type: mount.TypeBind}, }, }}, - Cfg: newStore(policy.ModeAll, false), - HostRoot: hostRootWithCgroup(t), - ProcRoot: buildProcRoot(t, pid), - newCgroup: func(int, *cgroup.Ledger) (cgroup.Interface, error) { return fake, nil }, + Cfg: newStore(policy.ModeAll, false), + HostRoot: hostRootWithCgroup(t), + ProcRoot: buildProcRoot(t, pid), + newCgroup: func(int, *cgroup.Ledger, *cgroup.FilterCache) (cgroup.Interface, error) { + return fake, nil + }, } } @@ -183,3 +185,34 @@ func TestProcessContainer_NotWrappableIsRetryableError(t *testing.T) { t.Fatalf("err = %v, want ErrProgramNotWrappable with its reason", err) } } + +func TestProcessContainer_FilterMissing(t *testing.T) { + for _, privileged := range []bool{true, false} { + buf := captureLogger(t) + fake := &failingCgroup{err: cgroup.ErrFilterMissing} + proc := devNullProcessor(t, fake) + + insp, ok := proc.Inspector.(*fakeInspector) + if !ok { + t.Fatal("unexpected inspector") + } + + insp.result.HostConfig = &container.HostConfig{Privileged: privileged} + + err := proc.ProcessContainer(context.Background(), "abc") + + switch { + case privileged && err != nil: + t.Errorf("privileged: err = %v, want a no-op", err) + case privileged && !strings.Contains(buf.String(), "privileged container has no device filter"): + t.Errorf("privileged: expected INFO, got:\n%s", buf.String()) + case !privileged && (!errors.Is(err, cgroup.ErrFilterMissing) || !strings.Contains(err.Error(), "filter_missing")): + t.Errorf( + "unprivileged: err = %v, want a retryable ErrFilterMissing with its reason", + err, + ) + case !privileged && !strings.Contains(buf.String(), "restart the container"): + t.Errorf("unprivileged: expected WARN with remedy, got:\n%s", buf.String()) + } + } +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index bdca0b1..fe824d1 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -89,9 +89,13 @@ type Processor struct { // Set can revoke them without touching the runtime's own exceptions. ledger cgroup.Ledger + // filterCache remembers the cgroup v2 runtime device filters seen per + // cgroup, so a filter wiped by systemd's daemon-reload can be rebuilt. + filterCache cgroup.FilterCache + // newCgroup builds the cgroup API for a version; nil means cgroup.New. // Tests replace it to observe or fail the mutation without a kernel. - newCgroup func(version int, ledger *cgroup.Ledger) (cgroup.Interface, error) + newCgroup func(version int, ledger *cgroup.Ledger, cache *cgroup.FilterCache) (cgroup.Interface, error) // afterCompute, when set, runs under mu after the rules are computed // and before they are applied. Tests use it to hold a worker mid-flight. @@ -179,7 +183,7 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er newCgroup = cgroup.New } - api, apiErr := newCgroup(cgroupVersion, &p.ledger) + api, apiErr := newCgroup(cgroupVersion, &p.ledger, &p.filterCache) if apiErr != nil { return fmt.Errorf("init cgroup api (version=%d): %w", cgroupVersion, apiErr) } @@ -216,7 +220,9 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er if len(collected.granted) > 0 { applyErr := p.applyRulesToCgroup(api, collected.granted, cgroupPath, pid, cfg.DryRun) if applyErr != nil { - return p.classifyApplyError(containerID, cgroupPath, applyErr) + privileged := info.HostConfig != nil && info.HostConfig.Privileged + + return p.classifyApplyError(containerID, cgroupPath, privileged, applyErr) } } @@ -246,10 +252,35 @@ func (p *Processor) callContext(ctx context.Context) (context.Context, context.C // classifyApplyError turns a cgroup mutation failure into the container's // outcome: nil when the container is skipped for good, otherwise an error // that names the reason, so the caller retries it. -func (p *Processor) classifyApplyError(containerID, cgroupPath string, applyErr error) error { +func (p *Processor) classifyApplyError( + containerID, cgroupPath string, + privileged bool, + applyErr error, +) error { log := logger.L() switch { + case errors.Is(applyErr, cgroup.ErrFilterMissing) && privileged: + // A privileged container runs without a device filter by design; + // there is nothing to add grants to and nothing to restrict. + log.Info("privileged container has no device filter; nothing to do", + "id", containerID, "cgroup", cgroupPath) + p.Metrics.RecordContainerSkipped("privileged_no_filter") + + return nil + + case errors.Is(applyErr, cgroup.ErrFilterMissing): + // Most likely wiped by systemd's daemon-reload before this process + // saw the cgroup, so the runtime's own filter cannot be rebuilt. + log.Warn("device filter missing and no cached original; restart the container", + "id", containerID, "cgroup", cgroupPath, "reason", "filter_missing") + + return fmt.Errorf( + "container %q (reason filter_missing, retryable): %w", + containerID, + applyErr, + ) + case errors.Is(applyErr, cgroup.ErrUnsupportedAttachMode): // Permanent for this container: the runtime chose the attach mode. // Nothing was attached or detached, so it keeps only the runtime's From 58634a67b785b7a6b87dc26695864523048619a8 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 10:28:00 +0200 Subject: [PATCH 08/37] feat(processor): reconcile the full desired device set with process identity pinning Processor.Reconcile (formerly ProcessContainer) computes the complete device set the current config allows a container and always applies it, including the empty set: a container disabled or not opted in by policy, one with invalid labels and one with any unresolved device all get the empty set, so an earlier grant is revoked instead of kept. An incomplete device set returns a retryable incomplete_device_set error. The per-container mutex is held from the config load through SetDeviceRules. The container process is pinned with a pidfd, /proc//cgroup and mountinfo are read through one directory descriptor (cgroup.ParseProcCgroup), and immediately before the mutation the pinned process must be alive, a second bounded inspect must report the same pid, StartedAt and running state, and, for grants only, the pid must be listed in cgroup.procs read through the cgroup handle (CgroupHandle.HasProcess). The verified identity is recorded per lifecycle (container ID plus StartedAt) before any mutation and kept across failed applies. A container that is no longer running, or that Docker cannot inspect, has the empty set applied in the cgroup its lifecycle was last verified in, only if the re-opened path still has the recorded inode; a removed or recreated cgroup releases the entry without mutation (Ledger.Forget, FilterCache.Forget). An inspect failure keeps the container pending; one caused by the caller cancelling (daemon shutdown) revokes nothing, while a timeout does. Entries of exited lifecycles are retained until their cgroup is verified gone; the periodic sweep that does so lands with the lifecycle records. Dry-run pins nothing, reads no /proc entry and builds no cgroup API; it logs the rules and a would-set line per container. README documents the reconcile semantics, the revoke-then-regrant cost of transient errors and the Linux 5.3 pidfd requirement. --- .agents/skills/trust-boundary/SKILL.md | 22 +- README.md | 24 +- docs/architecture.md | 7 +- internal/cgroup/api.go | 57 +++ internal/cgroup/handle.go | 24 ++ internal/cgroup/ledger.go | 9 + internal/daemon/daemon.go | 4 +- internal/daemon/daemon_test.go | 2 +- internal/daemon/events.go | 2 +- internal/daemon/timeout_test.go | 8 +- internal/processor/applyerr_test.go | 105 +++-- internal/processor/pin.go | 137 ++++++ internal/processor/processor.go | 313 ++++++++------ internal/processor/processor_test.go | 121 +++--- internal/processor/reconcile.go | 312 ++++++++++++++ internal/processor/reconcile_test.go | 565 +++++++++++++++++++++++++ internal/processor/serialize_test.go | 27 +- 17 files changed, 1474 insertions(+), 265 deletions(-) create mode 100644 internal/processor/pin.go create mode 100644 internal/processor/reconcile.go create mode 100644 internal/processor/reconcile_test.go diff --git a/.agents/skills/trust-boundary/SKILL.md b/.agents/skills/trust-boundary/SKILL.md index db4d7b7..3febba1 100644 --- a/.agents/skills/trust-boundary/SKILL.md +++ b/.agents/skills/trust-boundary/SKILL.md @@ -67,7 +67,7 @@ flags, and exits `1` with `invalid config: …` on failure; an unreadable or unp ## 3. Labels are untrusted input Anyone who can `docker service create` or `docker run` sets `swarm-device-access.*` labels. The -container's own labels are the **only** label input: `Processor.ProcessContainer` +container's own labels are the **only** label input: `Processor.Reconcile` (`internal/processor/processor.go`) parses `Config.Labels` from the container inspect and nothing else. Swarm service labels (`deploy.labels:`) are ignored, and the processor never calls `ServiceInspect` or `Info` (neither is part of `DockerInspector`), so a decision cannot depend on @@ -75,9 +75,10 @@ which node runs the daemon or on whether a service inspect happened to succeed. as attacker-controlled. - `policy.ParseContainer` rejects a non-boolean `enable` and any malformed glob in `device-allow` - or `device-deny`, naming the label. `ProcessContainer` then **skips the container** (warns, - records `invalid_labels`) rather than falling back to the global policy — a typo in a narrowing - label must never widen access to "whatever the global allows". + or `device-deny`, naming the label. `Reconcile` then gives the container the **empty** device + set (warns, records `invalid_labels`, revokes any earlier grant) rather than falling back to the + global policy — a typo in a narrowing label must never widen access to "whatever the global + allows", nor keep what an earlier, valid label granted. - Labels can only narrow: `policy.Global.DeviceAllowed` checks the global deny, the container deny, the global allow and the container allow, in that order. **Deny wins over allow**, and a per-container allow can never re-admit a path the global allow excludes. @@ -85,13 +86,13 @@ as attacker-controlled. "allow nothing" — that is documented in the README label table; keep it that way or change both. Covered by `TestParseContainer`, `TestDeviceAllowed`, `TestExplicitlyAllowed`, -`TestProcessContainer_IgnoresServiceLevelLabels` and -`TestProcessContainer_WarnsOnUnknownContainerLabel`. +`TestReconcile_IgnoresServiceLevelLabels` and +`TestReconcile_WarnsOnUnknownContainerLabel`. -- [ ] New label parsing returns an error on malformed input, and the caller skips the container. +- [ ] New label parsing returns an error on malformed input, and the caller applies the empty set. - [ ] A new label can only narrow; a test proves deny still beats allow with it set. - [ ] Unknown `swarm-device-access.*` keys on the container keep being reported - (`policy.UnknownLabels`, warned by `ProcessContainer`), not silently accepted. + (`policy.UnknownLabels`, warned by `Reconcile`), not silently accepted. - [ ] No second label source (service, node, stack) is added back: a label that only some nodes can read, or that disappears when an API call fails, turns a lost narrowing label into wider access. @@ -108,8 +109,9 @@ for a character or block device. Unresolvable symlinks get two different treatme warning only when an explicit allow glob names it (`skipUnresolvable`, the fix merged in commit `244d2a5`); - a mount **source** that is itself an unresolvable symlink returns an error in - `MountResult.Errs`, which `ProcessContainer` logs as a rule failure — no rule is emitted for it - either way. + `MountResult.Errs`, which `Reconcile` logs as a rule failure. Any entry in `Errs` makes the + container's whole desired set empty (reason `incomplete_device_set`) until a retry resolves every + device: a partly resolved set is never applied. Covered by `TestIsDeviceMountSource`, `TestCollectMountRules_*` and `TestCollectMountRules_UnresolvableSymlinks_ExplicitAllow` / `_NoAllowGlobs`. diff --git a/README.md b/README.md index d61e723..6060918 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,11 @@ uses the host Docker socket to launch the real privileged daemon container. ### Host Requirements +On every host the daemon needs Linux 5.3 or newer: it pins each running +container's process with `pidfd_open(2)` before changing its cgroup (see +below). On an older kernel no running container is changed, not even to +revoke a grant, and each attempt fails with a `pidfd_open` error. + On cgroup v2 hosts the daemon adds its grants to the device filter the container runtime already attached, replacing that program in place rather than removing it first, so the container is never left unfiltered. That needs: @@ -124,6 +129,23 @@ exceptions granted by the previous instance are then indistinguishable from the runtime's and are not revoked when policy narrows, until the container is restarted. cgroup v2 hosts do not have this gap. +Every time the daemon looks at a container (start, unpause, startup, systemd +reload) it sets the container's grants to the complete set the current +configuration allows, including none at all. A container that policy disables +or does not opt in, one with invalid labels, and one where any mounted device +cannot be resolved all get the empty set, so an earlier grant is revoked rather +than kept. The price is that a transient error (an unreadable device node, for +example) revokes the container's grants until the retry that grants them again. + +The daemon pins the container's process with a pidfd (Linux 5.3+) before it +resolves the cgroup, and immediately before changing anything checks that the +process is still alive, that Docker still reports it as the container's +running process with the same start time and, for a grant, that it is still in +that cgroup. A recycled pid therefore never directs grants to another cgroup. +Once a container has exited, or when Docker cannot be asked about it, its +grants are revoked in the cgroup the daemon last verified for it, and only if +that directory is still the same one (same inode) and was not recreated. + ### Docker Compose for Swarm ```yaml @@ -191,7 +213,7 @@ value is set in both places, the CLI flag wins. | `-log-format` | `text` | `text`, `json`, `plain` | | `-log-time` | `false` | Include timestamps in log lines | | `-docker-socket` | `/var/run/docker.sock` | Path to the Docker daemon's UNIX socket | -| `-dry-run` | `false` | Log device rules that would be applied without writing to the cgroup | +| `-dry-run` | `false` | Log the device set each container would get, without reading its `/proc` entry or its cgroup and without writing anything | | `-policy-mode` | `opt-in` | `opt-in`: only `enable=true` containers. `all`: unless `enable=false`. | | `-device-allow` | `""` | Glob for `/dev/...` paths to allow, repeatable. Empty means allow all. | | `-device-deny` | `""` | Glob for `/dev/...` paths to deny, repeatable. Deny takes priority over allow. | diff --git a/docs/architecture.md b/docs/architecture.md index 30000be..ea5742d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -237,11 +237,12 @@ When `-debug-addr=:6060` is set, the standard Go pprof endpoints are available a ## Dry-run mode -`-dry-run` logs what rules _would_ be written (at `Info` level) without calling `bpf(2)` or writing to `devices.allow`. Useful for auditing policy and -CI smoke tests. +`-dry-run` logs the complete device set each container _would_ get (at `Info` level) without pinning its process, reading its `/proc` entry, +querying its cgroup, calling `bpf(2)` or writing to `devices.allow`/`devices.deny`. Useful for auditing policy and CI smoke tests. ``` -level=INFO msg="dry-run: would add device rule" pid=1234 cgroup=/host/sys/fs/cgroup/docker/abc type=c major=195 minor=0 +level=INFO msg="dry-run: would add device rule" id=abc type=c major=195 minor=0 +level=INFO msg="dry-run: would set device rules" id=abc rules=1 ``` ## Troubleshooting diff --git a/internal/cgroup/api.go b/internal/cgroup/api.go index 1796b5d..4f61b02 100644 --- a/internal/cgroup/api.go +++ b/internal/cgroup/api.go @@ -20,6 +20,7 @@ package cgroup import ( "bufio" + "bytes" "errors" "fmt" "io" @@ -105,6 +106,62 @@ func GetDeviceCGroupVersion(rootPath string, pid int) (int, error) { return scanCGroupVersion(file, path) } +// ProcCgroup is where a process's device cgroup lives, as described by its +// /proc//cgroup and /proc//mountinfo. +type ProcCgroup struct { + Version int + MountPrefix string // root of the cgroup mount, from mountinfo + MountPoint string // where the hierarchy is mounted, from mountinfo + Root string // the process's cgroup below MountPrefix +} + +// ParseProcCgroup resolves the device cgroup from the contents of a +// process's cgroup and mountinfo files. Taking contents rather than a pid +// lets the caller read them through a /proc/ descriptor it has pinned. +func ParseProcCgroup(cgroupData, mountinfoData []byte) (ProcCgroup, error) { + const cgroupName, mountinfoName = "cgroup", "mountinfo" + + version, err := scanCGroupVersion(bytes.NewReader(cgroupData), cgroupName) + if err != nil { + return ProcCgroup{}, err + } + + result := ProcCgroup{Version: version} + + switch version { + case 1: + result.MountPrefix, result.MountPoint, err = scanMountInfoV1( + bytes.NewReader(mountinfoData), + mountinfoName, + ) + if err == nil { + result.Root, err = scanProcCgroupV1( + bytes.NewReader(cgroupData), + cgroupName, + result.MountPrefix, + ) + } + default: + result.MountPrefix, result.MountPoint, err = scanMountInfoV2( + bytes.NewReader(mountinfoData), + mountinfoName, + ) + if err == nil { + result.Root, err = scanProcCgroupV2( + bytes.NewReader(cgroupData), + cgroupName, + result.MountPrefix, + ) + } + } + + if err != nil { + return ProcCgroup{}, err + } + + return result, nil +} + // scanCGroupVersion parses the cgroup hierarchy file and returns 1 (v1) or 2 (v2). // Extracted from GetDeviceCGroupVersion to allow unit testing without a real /proc. func scanCGroupVersion(r io.Reader, path string) (int, error) { diff --git a/internal/cgroup/handle.go b/internal/cgroup/handle.go index 58821c2..1f4406b 100644 --- a/internal/cgroup/handle.go +++ b/internal/cgroup/handle.go @@ -20,6 +20,11 @@ package cgroup import ( "fmt" + "io" + "os" + "slices" + "strconv" + "strings" "golang.org/x/sys/unix" ) @@ -76,3 +81,22 @@ func (h *CgroupHandle) Close() error { return nil } + +// HasProcess reports whether pid is listed in the cgroup's cgroup.procs, +// read through the handle's descriptor. +func (h *CgroupHandle) HasProcess(pid int) (bool, error) { + fd, err := unix.Openat(h.fd, "cgroup.procs", unix.O_RDONLY|unix.O_CLOEXEC, 0) + if err != nil { + return false, fmt.Errorf("open cgroup.procs of %q: %w", h.identity.Path, err) + } + + file := os.NewFile(uintptr(fd), "cgroup.procs") + defer file.Close() + + data, err := io.ReadAll(file) + if err != nil { + return false, fmt.Errorf("read cgroup.procs of %q: %w", h.identity.Path, err) + } + + return slices.Contains(strings.Fields(string(data)), strconv.Itoa(pid)), nil +} diff --git a/internal/cgroup/ledger.go b/internal/cgroup/ledger.go index 94c85f2..3332b8c 100644 --- a/internal/cgroup/ledger.go +++ b/internal/cgroup/ledger.go @@ -34,6 +34,15 @@ type Ledger struct { entries map[Identity]map[devBit]struct{} } +// Forget drops the record for id. Only call it once the grants it lists are +// known to be gone (revoked, or the cgroup no longer exists). +func (l *Ledger) Forget(id Identity) { + l.mu.Lock() + defer l.mu.Unlock() + + delete(l.entries, id) +} + // owned returns a copy of the bits recorded for id. func (l *Ledger) owned(id Identity) map[devBit]struct{} { l.mu.Lock() diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 9aacefa..5dae5e2 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -156,10 +156,10 @@ func processExistingContainers( return nil } -// processorApply adapts Processor.ProcessContainer to applyFn. +// processorApply adapts Processor.Reconcile to applyFn. func processorApply(proc *processor.Processor) applyFn { return func(ctx context.Context, id string) error { - return proc.ProcessContainer(ctx, id) + return proc.Reconcile(ctx, id) } } diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index 1fd45eb..801fe65 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -143,7 +143,7 @@ func (f *fakeDocker) snapshot() (calls, sinces []string, listTime time.Time) { } // recordingInspector is a processor.DockerInspector that records inspected IDs -// and reports containers without a live pid, so ProcessContainer succeeds +// and reports containers without a live pid, so Reconcile succeeds // without touching cgroups. type recordingInspector struct { mu sync.Mutex diff --git a/internal/daemon/events.go b/internal/daemon/events.go index f97698b..9d6cfca 100644 --- a/internal/daemon/events.go +++ b/internal/daemon/events.go @@ -45,7 +45,7 @@ const ( var errSubscribeTimeout = errors.New("docker event stream not established before timeout") // applyFn is the per-container rule-application callback injected into consumeEvents. -// In production this wraps Processor.ProcessContainer; in tests it is replaced by a fake. +// In production this wraps Processor.Reconcile; in tests it is replaced by a fake. type applyFn func(ctx context.Context, id string) error // eventListOptions returns the Docker event subscription options: "start" diff --git a/internal/daemon/timeout_test.go b/internal/daemon/timeout_test.go index c7fe794..cd065cf 100644 --- a/internal/daemon/timeout_test.go +++ b/internal/daemon/timeout_test.go @@ -132,10 +132,10 @@ func TestProcessOne_HungContainerInspectIsBounded(t *testing.T) { } } -// TestProcessContainer_HungInspectIsBoundedPerCall calls the processor +// TestReconcile_HungInspectIsBoundedPerCall calls the processor // directly with an unbounded context: its own per-call timeout must bound // ContainerInspect. -func TestProcessContainer_HungInspectIsBoundedPerCall(t *testing.T) { +func TestReconcile_HungInspectIsBoundedPerCall(t *testing.T) { insp := &hangingInspector{} proc := &processor.Processor{ Inspector: insp, @@ -143,8 +143,8 @@ func TestProcessContainer_HungInspectIsBoundedPerCall(t *testing.T) { CallTimeout: testCallTimeout, } - assertBounded(t, "ProcessContainer", func() { - _ = proc.ProcessContainer(context.Background(), "c1") + assertBounded(t, "Reconcile", func() { + _ = proc.Reconcile(context.Background(), "c1") }) got := insp.observed() diff --git a/internal/processor/applyerr_test.go b/internal/processor/applyerr_test.go index ce88d0a..d9a3f03 100644 --- a/internal/processor/applyerr_test.go +++ b/internal/processor/applyerr_test.go @@ -24,6 +24,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "syscall" "testing" @@ -35,79 +36,131 @@ import ( "github.com/leinardi/swarm-device-access/internal/policy" ) -// failingCgroup is a cgroup.Interface whose mutation returns err. The path -// lookups delegate to the real v2 implementation, which reads the fake /proc. +// failingCgroup is a cgroup.Interface that records every SetDeviceRules +// call and returns err. The processor resolves cgroup paths itself, so the +// embedded Interface is never called. type failingCgroup struct { cgroup.Interface err error calls int identity cgroup.Identity + rules [][]cgroup.DeviceRule } -func (f *failingCgroup) SetDeviceRules(handle *cgroup.CgroupHandle, _ []cgroup.DeviceRule) error { +func (f *failingCgroup) SetDeviceRules( + handle *cgroup.CgroupHandle, + rules []cgroup.DeviceRule, +) error { f.calls++ f.identity = handle.Identity() + f.rules = append(f.rules, rules) return f.err } +// fakePinner pins nothing: every pinned process is alive unless gone is set. +type fakePinner struct { + pins []int + gone bool +} + +//nolint:ireturn // implements processPinner +func (f *fakePinner) pin(pid int) (pinnedProcess, error) { + f.pins = append(f.pins, pid) + + return fakePinned{gone: f.gone}, nil +} + +type fakePinned struct{ gone bool } + +func (fakePinned) Close() error { return nil } + +func (f fakePinned) alive() error { + if f.gone { + return errPinnedProcessGone + } + + return nil +} + +// testCgroupDir is where buildProcRoot's /proc places the container, below +// a host root. +func testCgroupDir(hostRoot string) string { + return filepath.Join(hostRoot, "sys", "fs", "cgroup", "docker", "testcontainer") +} + // hostRootWithCgroup returns a host root holding the cgroup directory that -// buildProcRoot's /proc resolves to, so the processor can open it. -func hostRootWithCgroup(t *testing.T) string { +// buildProcRoot's /proc resolves to, with pid listed in its cgroup.procs. +func hostRootWithCgroup(t *testing.T, pid int) string { t.Helper() root := t.TempDir() - err := os.MkdirAll(filepath.Join(root, "sys", "fs", "cgroup", "docker", "testcontainer"), 0o755) + err := os.MkdirAll(testCgroupDir(root), 0o755) if err != nil { t.Fatal(err) } + writeProcs(t, root, pid) + return root } -func devNullProcessor(t *testing.T, fake *failingCgroup) *Processor { +// writeProcs replaces the test cgroup's cgroup.procs with pids. +func writeProcs(t *testing.T, hostRoot string, pids ...int) { t.Helper() - const pid = 70 + var content strings.Builder + for _, pid := range pids { + content.WriteString(strconv.Itoa(pid) + "\n") + } - real2, err := cgroup.New(2, nil, nil) + err := os.WriteFile( + filepath.Join(testCgroupDir(hostRoot), "cgroup.procs"), + []byte(content.String()), + 0o600, + ) if err != nil { t.Fatal(err) } +} - fake.Interface = real2 +func devNullProcessor(t *testing.T, fake *failingCgroup) *Processor { + t.Helper() + + const pid = 70 return &Processor{ Inspector: &fakeInspector{result: container.InspectResponse{ - State: &container.State{Pid: pid}, + State: &container.State{Running: true, Pid: pid}, Mounts: []container.MountPoint{ {Source: "/dev/null", Destination: "/dev/null", Type: mount.TypeBind}, }, }}, Cfg: newStore(policy.ModeAll, false), - HostRoot: hostRootWithCgroup(t), + HostRoot: hostRootWithCgroup(t, pid), ProcRoot: buildProcRoot(t, pid), + pinner: &fakePinner{}, newCgroup: func(int, *cgroup.Ledger, *cgroup.FilterCache) (cgroup.Interface, error) { return fake, nil }, } } -// TestProcessContainer_SetsRulesThroughOpenedHandle checks that the +// TestReconcile_SetsRulesThroughOpenedHandle checks that the // processor opens the resolved cgroup directory itself and hands the cgroup // API a handle carrying that directory's inode. -func TestProcessContainer_SetsRulesThroughOpenedHandle(t *testing.T) { +func TestReconcile_SetsRulesThroughOpenedHandle(t *testing.T) { fake := &failingCgroup{} proc := devNullProcessor(t, fake) - err := proc.ProcessContainer(context.Background(), "abc") + err := proc.Reconcile(context.Background(), "abc") if err != nil { t.Fatal(err) } - wantPath := filepath.Join(proc.HostRoot, "sys", "fs", "cgroup", "docker", "testcontainer") + wantPath := testCgroupDir(proc.HostRoot) info, err := os.Stat(wantPath) if err != nil { @@ -124,13 +177,13 @@ func TestProcessContainer_SetsRulesThroughOpenedHandle(t *testing.T) { } } -func TestProcessContainer_UnsupportedAttachModeSkipsContainer(t *testing.T) { +func TestReconcile_UnsupportedAttachModeSkipsContainer(t *testing.T) { buf := captureLogger(t) fake := &failingCgroup{ err: fmt.Errorf("%w: attach mode none (exclusive)", cgroup.ErrUnsupportedAttachMode), } - err := devNullProcessor(t, fake).ProcessContainer(context.Background(), "abc") + err := devNullProcessor(t, fake).Reconcile(context.Background(), "abc") if err != nil { t.Fatalf("unsupported attach mode must skip, not fail: %v", err) } @@ -146,10 +199,10 @@ func TestProcessContainer_UnsupportedAttachModeSkipsContainer(t *testing.T) { } } -func TestProcessContainer_InaccessibleFiltersIsRetryableError(t *testing.T) { +func TestReconcile_InaccessibleFiltersIsRetryableError(t *testing.T) { fake := &failingCgroup{err: fmt.Errorf("%w: 1 of 2 programs", cgroup.ErrFiltersInaccessible)} - err := devNullProcessor(t, fake).ProcessContainer(context.Background(), "abc") + err := devNullProcessor(t, fake).Reconcile(context.Background(), "abc") if !errors.Is(err, cgroup.ErrFiltersInaccessible) { t.Fatalf("err = %v, want ErrFiltersInaccessible", err) } @@ -159,11 +212,11 @@ func TestProcessContainer_InaccessibleFiltersIsRetryableError(t *testing.T) { } } -func TestProcessContainer_OwnedBlockConflictIsLoggedWithRemedy(t *testing.T) { +func TestReconcile_OwnedBlockConflictIsLoggedWithRemedy(t *testing.T) { buf := captureLogger(t) fake := &failingCgroup{err: fmt.Errorf("%w: trailer missing", cgroup.ErrOwnedBlockConflict)} - err := devNullProcessor(t, fake).ProcessContainer(context.Background(), "abc") + err := devNullProcessor(t, fake).Reconcile(context.Background(), "abc") if !errors.Is(err, cgroup.ErrOwnedBlockConflict) || !strings.Contains(err.Error(), "owned_block_conflict") { t.Fatalf("err = %v, want a retryable ErrOwnedBlockConflict carrying the reason", err) @@ -176,17 +229,17 @@ func TestProcessContainer_OwnedBlockConflictIsLoggedWithRemedy(t *testing.T) { } } -func TestProcessContainer_NotWrappableIsRetryableError(t *testing.T) { +func TestReconcile_NotWrappableIsRetryableError(t *testing.T) { fake := &failingCgroup{err: fmt.Errorf("%w: uses maps", cgroup.ErrProgramNotWrappable)} - err := devNullProcessor(t, fake).ProcessContainer(context.Background(), "abc") + err := devNullProcessor(t, fake).Reconcile(context.Background(), "abc") if !errors.Is(err, cgroup.ErrProgramNotWrappable) || !strings.Contains(err.Error(), "program_not_wrappable") { t.Fatalf("err = %v, want ErrProgramNotWrappable with its reason", err) } } -func TestProcessContainer_FilterMissing(t *testing.T) { +func TestReconcile_FilterMissing(t *testing.T) { for _, privileged := range []bool{true, false} { buf := captureLogger(t) fake := &failingCgroup{err: cgroup.ErrFilterMissing} @@ -199,7 +252,7 @@ func TestProcessContainer_FilterMissing(t *testing.T) { insp.result.HostConfig = &container.HostConfig{Privileged: privileged} - err := proc.ProcessContainer(context.Background(), "abc") + err := proc.Reconcile(context.Background(), "abc") switch { case privileged && err != nil: diff --git a/internal/processor/pin.go b/internal/processor/pin.go new file mode 100644 index 0000000..43f1cdc --- /dev/null +++ b/internal/processor/pin.go @@ -0,0 +1,137 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processor + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + + "golang.org/x/sys/unix" + + "github.com/leinardi/swarm-device-access/internal/cgroup" +) + +var ( + // errPinnedProcessGone reports that the process pinned for a container + // exited before the mutation. + errPinnedProcessGone = errors.New("pinned container process exited") + // errLifecycleChanged reports that a second inspect no longer shows the + // container running with the pinned pid and start time. + errLifecycleChanged = errors.New("container changed between inspect and mutation") + // errNotInCgroup reports that the pinned pid is not in the cgroup the + // processor resolved for it. + errNotInCgroup = errors.New("container process is not in the resolved cgroup") +) + +// pinnedProcess is a process held by a pidfd: while it is alive its pid +// cannot be reused, so a pid read once stays meaningful. +type pinnedProcess interface { + // alive returns nil when the pinned process has not exited. + alive() error + Close() error +} + +// processPinner opens pidfds. It is a seam: tests cannot pin arbitrary pids. +type processPinner interface { + pin(pid int) (pinnedProcess, error) +} + +// pidfdPinner is the production processPinner (Linux 5.3+). +type pidfdPinner struct{} + +type pidfd int + +//nolint:ireturn // pinnedProcess hides the pidfd so tests can fake process lifetimes +func (pidfdPinner) pin(pid int) (pinnedProcess, error) { + fd, err := unix.PidfdOpen(pid, 0) + if err != nil { + return nil, fmt.Errorf("pidfd_open(%d): %w", pid, err) + } + + return pidfd(fd), nil +} + +func (fd pidfd) Close() error { + err := unix.Close(int(fd)) + if err != nil { + return fmt.Errorf("close pidfd: %w", err) + } + + return nil +} + +func (fd pidfd) alive() error { + err := unix.PidfdSendSignal(int(fd), 0, nil, 0) + if err != nil { + return fmt.Errorf("%w: %w", errPinnedProcessGone, err) + } + + return nil +} + +// readProcCgroup reads cgroup and mountinfo of pid through one descriptor on +// /proc/. The directory descriptor refers to the process it was opened +// for, so a later pid reuse cannot redirect the second read. +func readProcCgroup(procRoot string, pid int) (cgroup.ProcCgroup, error) { + dirPath := filepath.Join(procRoot, "proc", strconv.Itoa(pid)) + + dirFD, err := unix.Open(dirPath, unix.O_DIRECTORY|unix.O_RDONLY|unix.O_CLOEXEC, 0) + if err != nil { + return cgroup.ProcCgroup{}, fmt.Errorf("open %q: %w", dirPath, err) + } + defer unix.Close(dirFD) + + cgroupData, err := readAt(dirFD, "cgroup") + if err != nil { + return cgroup.ProcCgroup{}, err + } + + mountinfoData, err := readAt(dirFD, "mountinfo") + if err != nil { + return cgroup.ProcCgroup{}, err + } + + resolved, err := cgroup.ParseProcCgroup(cgroupData, mountinfoData) + if err != nil { + return cgroup.ProcCgroup{}, fmt.Errorf("resolve cgroup of pid %d: %w", pid, err) + } + + return resolved, nil +} + +func readAt(dirFD int, name string) ([]byte, error) { + fd, err := unix.Openat(dirFD, name, unix.O_RDONLY|unix.O_CLOEXEC, 0) + if err != nil { + return nil, fmt.Errorf("open %s: %w", name, err) + } + + file := os.NewFile(uintptr(fd), name) + defer file.Close() + + data, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("read %s: %w", name, err) + } + + return data, nil +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index fe824d1..636db8a 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -93,44 +93,173 @@ type Processor struct { // cgroup, so a filter wiped by systemd's daemon-reload can be rebuilt. filterCache cgroup.FilterCache + // known maps each container lifecycle to the cgroup it was last + // verified in (guarded by mu). It is how grants are revoked once the + // process is gone or Docker cannot be asked: see revokeKnown. + known map[lifecycleKey]knownCgroup + + // pinner pins container processes; nil means pidfds. Tests replace it. + pinner processPinner + // newCgroup builds the cgroup API for a version; nil means cgroup.New. // Tests replace it to observe or fail the mutation without a kernel. newCgroup func(version int, ledger *cgroup.Ledger, cache *cgroup.FilterCache) (cgroup.Interface, error) - // afterCompute, when set, runs under mu after the rules are computed - // and before they are applied. Tests use it to hold a worker mid-flight. + // afterCompute, when set, runs under mu after the desired rules are + // computed and before they are applied. Tests use it to hold a worker + // mid-flight. afterCompute func() } -// ProcessContainer inspects a container and applies cgroup BPF device-allow -// rules for every bind mount sourced from /dev/... +// lifecycleKey names one run of a container. A restart under the same ID +// has a new StartedAt, so what is known about the old run never applies to +// the new one. +type lifecycleKey struct { + containerID string + startedAt string +} + +// knownCgroup is the cgroup a lifecycle was last verified in: the identity +// of the directory its handle was opened on, the cgroup version, and +// whether the container is privileged (it then has no device filter). +type knownCgroup struct { + identity cgroup.Identity + version int + privileged bool +} + +// desiredSet is the outcome of evaluating a container's labels, the +// current policy and its /dev mounts. +type desiredSet struct { + // rules is what the container's cgroup must hold. It is empty whenever + // the set cannot be established with confidence. + rules []cgroup.DeviceRule + enabled bool + collected containerRules + // incomplete is non-nil when some device of an enabled container could + // not be resolved; the container is then retried. + incomplete error +} + +// Reconcile makes the device grants of a container's cgroup equal exactly +// what the current config allows it. Start, unpause, startup enumeration, +// systemd re-apply and reload all run this one idempotent path. // -//nolint:cyclop,gocyclo,funlen // inherent: policy-check + inspect + version-detect + path-resolve + mount-filter + collect + apply -func (p *Processor) ProcessContainer(ctx context.Context, containerID string) error { +// Every outcome that cannot establish the desired set with confidence +// (policy disabled or not opted in, invalid labels, an unresolved device) +// is the empty set, and the empty set is applied like any other: returning +// early would keep grants the current config no longer allows. The cost is +// that a transient error revokes the container's grants until a retry +// grants them again. +// +// The container's process is pinned (see applyPinned), so a recycled pid +// cannot direct the rules to another cgroup. A container that is no longer +// running, or that Docker cannot report on, has its grants revoked in the +// cgroup its lifecycle was last verified in. +func (p *Processor) Reconcile(ctx context.Context, containerID string) error { log := logger.L() - inspectCtx, cancelInspect := p.callContext(ctx) - inspected, inspectErr := p.Inspector.ContainerInspect( - inspectCtx, - containerID, - client.ContainerInspectOptions{}, - ) + info, inspectErr := p.inspect(ctx, containerID) - cancelInspect() + p.mu.Lock() + defer p.mu.Unlock() + + cfg := p.Cfg.Load() if inspectErr != nil { - return fmt.Errorf("inspect container %q: %w", containerID, inspectErr) - } + // The caller canceled (daemon shutdown), not Docker: nothing would + // retry and grant again, so a running container keeps its grants. + // A deadline is Docker not answering in time, and does revoke. + if errors.Is(ctx.Err(), context.Canceled) { + return inspectErr + } - info := inspected.Container + return p.revokeAfterInspectFailure(containerID, cfg.DryRun, inspectErr) + } - if info.State == nil || info.State.Pid == 0 { + if info.State == nil || !info.State.Running || info.State.Pid == 0 { log.Debug("container has no live pid; skipping", "id", containerID) p.Metrics.RecordContainerSkipped("no_pid") - return nil + if cfg.DryRun { + return nil + } + + return p.revokeKnown( + lifecycleKey{containerID: containerID, startedAt: startedAt(info.State)}, + ) + } + + desired := p.computeDesired(containerID, &info, cfg) + + if p.afterCompute != nil { + p.afterCompute() + } + + if cfg.DryRun { + p.logDryRun(containerID, desired.rules) + } else { + privileged := info.HostConfig != nil && info.HostConfig.Privileged + + cgroupPath, applyErr := p.applyPinned( + ctx, + containerID, + info.State, + privileged, + desired.rules, + ) + if applyErr != nil { + // A container skipped for good still reports an incomplete set. + return errors.Join( + p.classifyApplyError(containerID, cgroupPath, privileged, applyErr), + desired.incomplete, + ) + } } + if desired.enabled && desired.collected.devMounts > 0 { + log.Info("container processed", + "id", containerID, + "pid", info.State.Pid, + "devices_granted", len(desired.rules), + "skipped", desired.collected.skipped, + "errors", len(desired.collected.deviceErrs), + "dry_run", cfg.DryRun, + ) + } + + return desired.incomplete +} + +// inspect runs one bounded ContainerInspect. +func (p *Processor) inspect( + ctx context.Context, + containerID string, +) (container.InspectResponse, error) { + callCtx, cancel := p.callContext(ctx) + defer cancel() + + inspected, err := p.Inspector.ContainerInspect( + callCtx, + containerID, + client.ContainerInspectOptions{}, + ) + if err != nil { + return container.InspectResponse{}, fmt.Errorf("inspect container %q: %w", containerID, err) + } + + return inspected.Container, nil +} + +// computeDesired evaluates the container's labels and /dev mounts under +// cfg. It never touches the container's cgroup. +func (p *Processor) computeDesired( + containerID string, + info *container.InspectResponse, + cfg config.Runtime, +) desiredSet { + log := logger.L() + var containerLabels map[string]string if info.Config != nil { containerLabels = info.Config.Labels @@ -143,18 +272,13 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er "id", containerID, "label", unknownKey) } - p.mu.Lock() - defer p.mu.Unlock() - - cfg := p.Cfg.Load() - cpol, parseErr := policy.ParseContainer(containerLabels) if parseErr != nil { log.Warn("container skipped: invalid policy labels", "id", containerID, "err", parseErr) p.Metrics.RecordContainerSkipped("invalid_labels") - return nil + return desiredSet{} } if !cfg.Policy.Enabled(cpol) { @@ -164,44 +288,12 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er ) p.Metrics.RecordContainerSkipped("policy") - return nil + return desiredSet{} } p.Metrics.RecordContainerScanned() - pid := info.State.Pid - - cgroupVersion, versionErr := cgroup.GetDeviceCGroupVersion(p.ProcRoot, pid) - if versionErr != nil { - return fmt.Errorf("detect cgroup version for pid %d: %w", pid, versionErr) - } - - log.Debug("cgroup version detected", "pid", pid, "version", cgroupVersion) - - newCgroup := p.newCgroup - if newCgroup == nil { - newCgroup = cgroup.New - } - - api, apiErr := newCgroup(cgroupVersion, &p.ledger, &p.filterCache) - if apiErr != nil { - return fmt.Errorf("init cgroup api (version=%d): %w", cgroupVersion, apiErr) - } - - cgroupPrefix, sysfsPath, mountErr := api.GetDeviceCGroupMountPath(p.ProcRoot, pid) - if mountErr != nil { - return fmt.Errorf("resolve cgroup mount path: %w", mountErr) - } - - cgroupRoot, rootErr := api.GetDeviceCGroupRootPath(p.ProcRoot, cgroupPrefix, pid) - if rootErr != nil { - return fmt.Errorf("resolve cgroup root path: %w", rootErr) - } - - cgroupPath := hostCGroupPath(p.HostRoot, sysfsPath, cgroupPrefix, cgroupRoot) - log.Debug("cgroup path resolved", "pid", pid, "path", cgroupPath) - - collected := collectContainerRules(containerID, pid, info.Mounts, cfg.Policy, cpol) + collected := collectContainerRules(containerID, info.State.Pid, info.Mounts, cfg.Policy, cpol) for _, deviceErr := range collected.deviceErrs { log.Warn("device rule failed", "id", containerID, "err", deviceErr) @@ -209,35 +301,41 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er p.Metrics.AddDeviceFilesDiscovered(len(collected.granted)) - if len(collected.deviceErrs) > 0 { - p.Metrics.AddRuleFailures(len(collected.deviceErrs)) - } - - if p.afterCompute != nil { - p.afterCompute() + if len(collected.deviceErrs) == 0 { + return desiredSet{rules: collected.granted, enabled: true, collected: collected} } - if len(collected.granted) > 0 { - applyErr := p.applyRulesToCgroup(api, collected.granted, cgroupPath, pid, cfg.DryRun) - if applyErr != nil { - privileged := info.HostConfig != nil && info.HostConfig.Privileged + p.Metrics.AddRuleFailures(len(collected.deviceErrs)) + log.Warn("device set incomplete; no devices are granted until every device resolves", + "id", containerID, "errors", len(collected.deviceErrs), "reason", "incomplete_device_set") - return p.classifyApplyError(containerID, cgroupPath, privileged, applyErr) - } + return desiredSet{ + enabled: true, + collected: collected, + incomplete: fmt.Errorf( + "container %q (reason incomplete_device_set, retryable): %w", + containerID, + errors.Join(collected.deviceErrs...), + ), } +} - if collected.devMounts > 0 { - log.Info("container processed", +// logDryRun reports what a real run would set. Dry-run stays unprivileged: +// it reads neither /proc nor the cgroup. +func (p *Processor) logDryRun(containerID string, rules []cgroup.DeviceRule) { + log := logger.L() + + for _, rule := range rules { + log.Info("dry-run: would add device rule", "id", containerID, - "pid", pid, - "devices_granted", len(collected.granted), - "skipped", collected.skipped, - "errors", len(collected.deviceErrs), - "dry_run", cfg.DryRun, + "type", rule.Type, + "major", *rule.Major, + "minor", *rule.Minor, ) } - return nil + log.Info("dry-run: would set device rules", "id", containerID, "rules", len(rules)) + p.Metrics.AddDryRunSkips(len(rules)) } // callContext derives the context for one Docker call. @@ -386,65 +484,6 @@ func collectContainerRules( return result } -// applyRulesToCgroup logs and (unless dryRun) sets the collected device rules -// on the cgroup at cgroupPath. The directory is opened once here; the cgroup -// API then works on that descriptor only, so the rules cannot land in a -// different cgroup recreated at the same path. -func (p *Processor) applyRulesToCgroup( - api cgroup.Interface, - rules []cgroup.DeviceRule, - cgroupPath string, - pid int, - dryRun bool, -) error { - log := logger.L() - - for _, rule := range rules { - if dryRun { - log.Info("dry-run: would add device rule", - "pid", pid, - "cgroup", cgroupPath, - "type", rule.Type, - "major", *rule.Major, - "minor", *rule.Minor, - ) - } else { - log.Debug("adding device rule", - "pid", pid, - "cgroup", cgroupPath, - "type", rule.Type, - "major", *rule.Major, - "minor", *rule.Minor, - ) - } - } - - if dryRun { - p.Metrics.AddDryRunSkips(len(rules)) - - return nil - } - - handle, err := cgroup.OpenCgroup(cgroupPath) - if err != nil { - return fmt.Errorf("set device rules: %w", err) - } - - defer func() { - closeErr := handle.Close() - if closeErr != nil { - log.Warn("close cgroup handle", "cgroup", cgroupPath, "err", closeErr) - } - }() - - err = api.SetDeviceRules(handle, rules) - if err != nil { - return fmt.Errorf("set device rules: %w", err) - } - - return nil -} - func hostCGroupPath(hostRoot, sysfsPath, cgroupPrefix, cgroupRoot string) string { return filepath.Join(hostRoot, sysfsPath, cgroupPrefix, cgroupRoot) } diff --git a/internal/processor/processor_test.go b/internal/processor/processor_test.go index f2afa01..3571664 100644 --- a/internal/processor/processor_test.go +++ b/internal/processor/processor_test.go @@ -77,7 +77,7 @@ const ( ) // buildProcRoot creates a minimal /proc//{cgroup,mountinfo} structure -// under a temp dir so ProcessContainer can resolve the cgroup path without a +// under a temp dir so Reconcile can resolve the cgroup path without a // real /proc filesystem. // @@ -198,9 +198,9 @@ func TestHostCGroupPath(t *testing.T) { } } -// ---- ProcessContainer tests ---- +// ---- Reconcile tests ---- -func TestProcessContainer_InspectError(t *testing.T) { +func TestReconcile_InspectError(t *testing.T) { insp := &fakeInspector{err: errDaemonUnavail} proc := &Processor{ Inspector: insp, @@ -209,13 +209,13 @@ func TestProcessContainer_InspectError(t *testing.T) { ProcRoot: "/", } - err := proc.ProcessContainer(context.Background(), "abc") + err := proc.Reconcile(context.Background(), "abc") if err == nil { t.Fatal("expected error from inspect failure, got nil") } } -func TestProcessContainer_NilState(t *testing.T) { +func TestReconcile_NilState(t *testing.T) { insp := &fakeInspector{result: container.InspectResponse{ State: nil, }} @@ -226,14 +226,14 @@ func TestProcessContainer_NilState(t *testing.T) { ProcRoot: "/", } - err := proc.ProcessContainer(context.Background(), "abc") + err := proc.Reconcile(context.Background(), "abc") if err != nil { t.Fatalf("expected nil error for nil state, got %v", err) } } -func TestProcessContainer_ZeroPid(t *testing.T) { - state := &container.State{Pid: 0} +func TestReconcile_ZeroPid(t *testing.T) { + state := &container.State{Running: true, Pid: 0} insp := &fakeInspector{result: container.InspectResponse{ State: state, }} @@ -244,18 +244,18 @@ func TestProcessContainer_ZeroPid(t *testing.T) { ProcRoot: "/", } - err := proc.ProcessContainer(context.Background(), "abc") + err := proc.Reconcile(context.Background(), "abc") if err != nil { t.Fatalf("expected nil error for pid=0, got %v", err) } } -func TestProcessContainer_NoDevMounts(t *testing.T) { +func TestReconcile_NoDevMounts(t *testing.T) { const pid = 42 root := buildProcRoot(t, pid) - state := &container.State{Pid: pid} + state := &container.State{Running: true, Pid: pid} insp := &fakeInspector{result: container.InspectResponse{ State: state, Mounts: []container.MountPoint{ @@ -263,31 +263,44 @@ func TestProcessContainer_NoDevMounts(t *testing.T) { {Source: "/var/log", Destination: "/logs", Type: mount.TypeBind}, }, }} + fake := &failingCgroup{} proc := &Processor{ Inspector: insp, Cfg: newStore(policy.ModeAll, false), - HostRoot: "/host", + HostRoot: hostRootWithCgroup(t, pid), ProcRoot: root, + pinner: &fakePinner{}, + newCgroup: func(int, *cgroup.Ledger, *cgroup.FilterCache) (cgroup.Interface, error) { + return fake, nil + }, } buf := captureLogger(t) - err := proc.ProcessContainer(context.Background(), "abc") + err := proc.Reconcile(context.Background(), "abc") if err != nil { t.Fatalf("expected nil error for container with no /dev mounts, got %v", err) } + if fake.calls != 1 || len(fake.rules[0]) != 0 { + t.Errorf( + "SetDeviceRules calls = %d with %v, want one call with the empty set", + fake.calls, + fake.rules, + ) + } + if strings.Contains(buf.String(), "container processed") { t.Errorf("expected no summary for container without /dev mounts, got: %s", buf.String()) } } -func TestProcessContainer_DevMountFilterApplied(t *testing.T) { +func TestReconcile_DevMountFilterApplied(t *testing.T) { const pid = 43 root := buildProcRoot(t, pid) - state := &container.State{Pid: pid} + state := &container.State{Running: true, Pid: pid} insp := &fakeInspector{result: container.InspectResponse{ State: state, Mounts: []container.MountPoint{ @@ -300,22 +313,23 @@ func TestProcessContainer_DevMountFilterApplied(t *testing.T) { Cfg: newStore(policy.ModeAll, false), HostRoot: "/host", ProcRoot: root, + pinner: &fakePinner{}, } - err := proc.ProcessContainer(context.Background(), "abc") + err := proc.Reconcile(context.Background(), "abc") if err == nil { t.Fatal( - "ProcessContainer should return error when the cgroup path cannot be opened", + "Reconcile should return error when the cgroup path cannot be opened", ) } } -func TestProcessContainer_DevMount_DryRunNoError(t *testing.T) { +func TestReconcile_DevMount_DryRunNoError(t *testing.T) { const pid = 44 root := buildProcRoot(t, pid) - state := &container.State{Pid: pid} + state := &container.State{Running: true, Pid: pid} insp := &fakeInspector{result: container.InspectResponse{ State: state, Mounts: []container.MountPoint{ @@ -329,18 +343,18 @@ func TestProcessContainer_DevMount_DryRunNoError(t *testing.T) { ProcRoot: root, } - err := proc.ProcessContainer(context.Background(), "abc") + err := proc.Reconcile(context.Background(), "abc") if err != nil { - t.Fatalf("dry-run ProcessContainer should not error: %v", err) + t.Fatalf("dry-run Reconcile should not error: %v", err) } } -func TestProcessContainer_DeduplicatesDuplicateMounts(t *testing.T) { +func TestReconcile_DeduplicatesDuplicateMounts(t *testing.T) { const pid = 45 root := buildProcRoot(t, pid) - state := &container.State{Pid: pid} + state := &container.State{Running: true, Pid: pid} insp := &fakeInspector{result: container.InspectResponse{ State: state, Mounts: []container.MountPoint{ @@ -355,43 +369,18 @@ func TestProcessContainer_DeduplicatesDuplicateMounts(t *testing.T) { ProcRoot: root, } - err := proc.ProcessContainer(context.Background(), "abc") + err := proc.Reconcile(context.Background(), "abc") if err != nil { t.Fatalf("dry-run with duplicate mounts should not error: %v", err) } } -func TestProcessContainer_OptInSkipsUnlabelled(t *testing.T) { - const pid = 50 - - root := buildProcRoot(t, pid) - - state := &container.State{Pid: pid} - insp := &fakeInspector{result: container.InspectResponse{ - State: state, - Mounts: []container.MountPoint{ - {Source: "/dev/null", Destination: "/dev/null", Type: mount.TypeBind}, - }, - }} - proc := &Processor{ - Inspector: insp, - Cfg: newStore(policy.ModeOptIn, false), - HostRoot: "/host", - ProcRoot: root, - } - - err := proc.ProcessContainer(context.Background(), "abc") - if err != nil { - t.Fatalf("opt-in skip should return nil, got: %v", err) - } -} - -func TestProcessContainer_OptInProcessesEnabled(t *testing.T) { +func TestReconcile_OptInProcessesEnabled(t *testing.T) { const pid = 51 root := buildProcRoot(t, pid) - state := &container.State{Pid: pid} + state := &container.State{Running: true, Pid: pid} insp := &fakeInspector{result: container.InspectResponse{ State: state, Config: &container.Config{ @@ -408,7 +397,7 @@ func TestProcessContainer_OptInProcessesEnabled(t *testing.T) { ProcRoot: root, } - err := proc.ProcessContainer(context.Background(), "abc") + err := proc.Reconcile(context.Background(), "abc") if err != nil { t.Fatalf("opt-in dry-run with enable=true should not error: %v", err) } @@ -775,14 +764,14 @@ func TestCollectMountRules_SingleFileNonDevice(t *testing.T) { } } -// TestProcessContainer_DevDirectorySummary checks that a whole-/dev mount is +// TestReconcile_DevDirectorySummary checks that a whole-/dev mount is // processed without a container-level error and emits one INFO summary. -func TestProcessContainer_DevDirectorySummary(t *testing.T) { +func TestReconcile_DevDirectorySummary(t *testing.T) { const pid = 46 root := buildProcRoot(t, pid) - state := &container.State{Pid: pid} + state := &container.State{Running: true, Pid: pid} insp := &fakeInspector{result: container.InspectResponse{ State: state, Mounts: []container.MountPoint{ @@ -805,9 +794,9 @@ func TestProcessContainer_DevDirectorySummary(t *testing.T) { buf := captureLogger(t) - err := proc.ProcessContainer(context.Background(), "abc") + err := proc.Reconcile(context.Background(), "abc") if err != nil { - t.Fatalf("ProcessContainer returned error: %v", err) + t.Fatalf("Reconcile returned error: %v", err) } logOutput := buf.String() @@ -850,18 +839,18 @@ func captureLogger(t *testing.T) *bytes.Buffer { return &buf } -// TestProcessContainer_IgnoresServiceLevelLabels checks that policy comes +// TestReconcile_IgnoresServiceLevelLabels checks that policy comes // from the container's own labels only: the parent service is never // inspected, and a container whose only opt-in is a service-level (deploy.labels) // label is skipped. -func TestProcessContainer_IgnoresServiceLevelLabels(t *testing.T) { +func TestReconcile_IgnoresServiceLevelLabels(t *testing.T) { buf := captureLogger(t) const pid = 51 inspector := &fakeInspector{ result: container.InspectResponse{ - State: &container.State{Pid: pid}, + State: &container.State{Running: true, Pid: pid}, Config: &container.Config{Labels: map[string]string{ "com.docker.swarm.service.id": "svc456", }}, @@ -877,9 +866,9 @@ func TestProcessContainer_IgnoresServiceLevelLabels(t *testing.T) { Cfg: newStore(policy.ModeOptIn, true), } - err := proc.ProcessContainer(context.Background(), "abc123") + err := proc.Reconcile(context.Background(), "abc123") if err != nil { - t.Fatalf("ProcessContainer: %v", err) + t.Fatalf("Reconcile: %v", err) } if inspector.serviceCalls != 0 { @@ -891,12 +880,12 @@ func TestProcessContainer_IgnoresServiceLevelLabels(t *testing.T) { } } -func TestProcessContainer_WarnsOnUnknownContainerLabel(t *testing.T) { +func TestReconcile_WarnsOnUnknownContainerLabel(t *testing.T) { buf := captureLogger(t) inspector := &fakeInspector{ result: container.InspectResponse{ - State: &container.State{Pid: 51}, + State: &container.State{Running: true, Pid: 51}, Config: &container.Config{Labels: map[string]string{ policy.LabelPrefix + "enabled": "true", }}, @@ -905,9 +894,9 @@ func TestProcessContainer_WarnsOnUnknownContainerLabel(t *testing.T) { proc := &Processor{Inspector: inspector, Cfg: newStore(policy.ModeOptIn, true)} - err := proc.ProcessContainer(context.Background(), "abc123") + err := proc.Reconcile(context.Background(), "abc123") if err != nil { - t.Fatalf("ProcessContainer: %v", err) + t.Fatalf("Reconcile: %v", err) } if !strings.Contains(buf.String(), "unrecognized swarm-device-access label on container") { diff --git a/internal/processor/reconcile.go b/internal/processor/reconcile.go new file mode 100644 index 0000000..308433d --- /dev/null +++ b/internal/processor/reconcile.go @@ -0,0 +1,312 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processor + +import ( + "context" + "errors" + "fmt" + "io/fs" + + "github.com/moby/moby/api/types/container" + + "github.com/leinardi/swarm-device-access/internal/cgroup" + "github.com/leinardi/swarm-device-access/internal/logger" +) + +// applyPinned sets rules on the cgroup of the container's running process. +// It returns the cgroup path once resolved, for error reporting. +// +// The pid from inspect can be reused by another process before the cgroup +// is resolved, so the process is pinned: a pidfd is opened first, /proc/ +// is read through one directory descriptor, and immediately before the +// mutation the pinned process must still be alive and a second inspect must +// report the same pid, start time and running state. While the pinned +// process is alive its pid cannot belong to anyone else, so the container +// still reporting that pid proves the reads were of the container's own +// process. A grant additionally requires the pid to be listed in the +// cgroup's cgroup.procs; an empty set is only ever narrowing. +func (p *Processor) applyPinned( + ctx context.Context, + containerID string, + state *container.State, + privileged bool, + rules []cgroup.DeviceRule, +) (string, error) { + log := logger.L() + pid := state.Pid + + pinned, err := p.processPinner().pin(pid) + if err != nil { + return "", fmt.Errorf("container %q: %w", containerID, err) + } + + defer func() { + closeErr := pinned.Close() + if closeErr != nil { + log.Warn("close pidfd", "id", containerID, "err", closeErr) + } + }() + + resolved, err := readProcCgroup(p.ProcRoot, pid) + if err != nil { + return "", fmt.Errorf("container %q: %w", containerID, err) + } + + cgroupPath := hostCGroupPath( + p.HostRoot, + resolved.MountPoint, + resolved.MountPrefix, + resolved.Root, + ) + log.Debug("cgroup path resolved", "pid", pid, "version", resolved.Version, "path", cgroupPath) + + api, err := p.cgroupAPI(resolved.Version) + if err != nil { + return cgroupPath, err + } + + handle, err := cgroup.OpenCgroup(cgroupPath) + if err != nil { + return cgroupPath, fmt.Errorf("set device rules: %w", err) + } + + defer closeHandle(handle) + + err = p.verifyPinned(ctx, containerID, state, pinned, handle, len(rules) > 0) + if err != nil { + return cgroupPath, err + } + + // Recorded after verification, so an identity resolved through a + // recycled pid is never kept, and before the mutation, so a failed or + // partial one can still be revoked here later. + p.recordKnown( + lifecycleKey{containerID: containerID, startedAt: state.StartedAt}, + knownCgroup{identity: handle.Identity(), version: resolved.Version, privileged: privileged}, + ) + + for _, rule := range rules { + log.Debug("setting device rule", + "pid", pid, "cgroup", cgroupPath, + "type", rule.Type, "major", *rule.Major, "minor", *rule.Minor) + } + + err = api.SetDeviceRules(handle, rules) + if err != nil { + return cgroupPath, fmt.Errorf("set device rules: %w", err) + } + + return cgroupPath, nil +} + +// verifyPinned is the check made immediately before a mutation; see +// applyPinned. +func (p *Processor) verifyPinned( + ctx context.Context, + containerID string, + state *container.State, + pinned pinnedProcess, + handle *cgroup.CgroupHandle, + grants bool, +) error { + again, err := p.inspect(ctx, containerID) + if err != nil { + return fmt.Errorf("verify container %q: %w", containerID, err) + } + + if again.State == nil || !again.State.Running || + again.State.Pid != state.Pid || again.State.StartedAt != state.StartedAt { + return fmt.Errorf("container %q pid %d: %w", containerID, state.Pid, errLifecycleChanged) + } + + err = pinned.alive() + if err != nil { + return fmt.Errorf("container %q pid %d: %w", containerID, state.Pid, err) + } + + if !grants { + return nil + } + + member, err := handle.HasProcess(state.Pid) + if err != nil { + return fmt.Errorf("verify container %q: %w", containerID, err) + } + + if !member { + return fmt.Errorf("container %q pid %d: %w", containerID, state.Pid, errNotInCgroup) + } + + return nil +} + +// revokeAfterInspectFailure handles a container Docker cannot report on. +// Without a pid there is no cgroup to resolve, so every lifecycle of the +// container with a known cgroup is revoked there, and the container stays +// pending either way. Dry-run never mutates. +func (p *Processor) revokeAfterInspectFailure( + containerID string, + dryRun bool, + inspectErr error, +) error { + errs := []error{inspectErr} + + if dryRun { + return inspectErr + } + + for key, known := range p.known { + if key.containerID == containerID { + errs = append(errs, p.revokeAt(key, known, true)) + } + } + + return errors.Join(errs...) +} + +// revokeKnown applies the empty set to the cgroup key was last verified in, +// if any. It serves a lifecycle whose process is gone: nothing can be +// pinned any more, and the cgroup may already have been removed. +func (p *Processor) revokeKnown(key lifecycleKey) error { + known, ok := p.known[key] + if !ok { + return nil + } + + return p.revokeAt(key, known, false) +} + +// revokeAt re-opens the recorded path and applies the empty set only if it +// is still the same directory (same inode). A cgroup that is gone, or was +// recreated at the same path, holds none of the lifecycle's grants: the +// entry is released and nothing is mutated. Otherwise the entry is kept +// even after the revoke, until the cgroup is verified gone. mayRun reports +// that the container may still be running (Docker could not say). +func (p *Processor) revokeAt(key lifecycleKey, known knownCgroup, mayRun bool) error { + log := logger.L() + + handle, err := cgroup.OpenCgroup(known.identity.Path) + if errors.Is(err, fs.ErrNotExist) { + p.release(key, known, "cgroup removed") + + return nil + } + + if err != nil { + return fmt.Errorf("revoke container %q: %w", key.containerID, err) + } + + defer closeHandle(handle) + + if handle.Identity() != known.identity { + p.release(key, known, "cgroup recreated") + + return nil + } + + api, err := p.cgroupAPI(known.version) + if err != nil { + return err + } + + err = api.SetDeviceRules(handle, nil) + + switch { + case errors.Is(err, cgroup.ErrFilterMissing) && mayRun && !known.privileged: + // Nothing attached holds a grant, but a container that may still be + // running is unfiltered. The container stays pending either way. + log.Warn("device filter missing and no cached original; restart the container", + "id", key.containerID, "cgroup", known.identity.Path, "reason", "filter_missing") + case errors.Is(err, cgroup.ErrFilterMissing): + // An exited or privileged container has nothing attached, so + // nothing to revoke. + case err != nil: + return fmt.Errorf( + "revoke container %q in %q: %w", + key.containerID, + known.identity.Path, + err, + ) + } + + log.Debug("device grants revoked", "id", key.containerID, "cgroup", known.identity.Path) + + return nil +} + +func (p *Processor) recordKnown(key lifecycleKey, known knownCgroup) { + if p.known == nil { + p.known = make(map[lifecycleKey]knownCgroup) + } + + p.known[key] = known +} + +// release forgets a lifecycle whose cgroup is verified gone, together with +// what the cgroup API remembered for that directory. +func (p *Processor) release(key lifecycleKey, known knownCgroup, why string) { + logger.L().Debug("container cgroup gone; released", + "id", key.containerID, "cgroup", known.identity.Path, "why", why) + + delete(p.known, key) + p.ledger.Forget(known.identity) + p.filterCache.Forget(known.identity) +} + +//nolint:ireturn // processPinner is the seam that lets tests fake pidfds +func (p *Processor) processPinner() processPinner { + if p.pinner == nil { + return pidfdPinner{} + } + + return p.pinner +} + +//nolint:ireturn // intentional: callers use the interface +func (p *Processor) cgroupAPI(version int) (cgroup.Interface, error) { + newCgroup := p.newCgroup + if newCgroup == nil { + newCgroup = cgroup.New + } + + api, err := newCgroup(version, &p.ledger, &p.filterCache) + if err != nil { + return nil, fmt.Errorf("init cgroup api (version=%d): %w", version, err) + } + + return api, nil +} + +func closeHandle(handle *cgroup.CgroupHandle) { + err := handle.Close() + if err != nil { + logger.L().Warn("close cgroup handle", "cgroup", handle.Identity().Path, "err", err) + } +} + +// startedAt is the lifecycle's start time, empty when inspect reported no +// state. +func startedAt(state *container.State) string { + if state == nil { + return "" + } + + return state.StartedAt +} diff --git a/internal/processor/reconcile_test.go b/internal/processor/reconcile_test.go new file mode 100644 index 0000000..5aacfce --- /dev/null +++ b/internal/processor/reconcile_test.go @@ -0,0 +1,565 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processor + +import ( + "context" + "errors" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/mount" + "github.com/moby/moby/client" + + "github.com/leinardi/swarm-device-access/internal/cgroup" + "github.com/leinardi/swarm-device-access/internal/policy" +) + +const ( + reconcilePid = 80 + reconcileStarted = "2026-09-27T10:00:00.000000000Z" +) + +type inspectReply struct { + resp container.InspectResponse + err error +} + +// seqInspector answers the n-th ContainerInspect with replies[n], repeating +// the last reply once they run out. +type seqInspector struct { + replies []inspectReply + calls int +} + +func (s *seqInspector) ContainerInspect( + _ context.Context, + _ string, + _ client.ContainerInspectOptions, +) (client.ContainerInspectResult, error) { + reply := s.replies[min(s.calls, len(s.replies)-1)] + s.calls++ + + return client.ContainerInspectResult{Container: reply.resp}, reply.err +} + +// then makes every later inspect answer with replies, in order. +func (s *seqInspector) then(replies ...inspectReply) { + s.replies = replies + s.calls = 0 +} + +func runningDevNull(started string) inspectReply { + return inspectReply{resp: container.InspectResponse{ + State: &container.State{Running: true, Pid: reconcilePid, StartedAt: started}, + Mounts: []container.MountPoint{ + {Source: "/dev/null", Destination: "/dev/null", Type: mount.TypeBind}, + }, + }} +} + +func exited() inspectReply { + return inspectReply{resp: container.InspectResponse{ + State: &container.State{Running: false, StartedAt: reconcileStarted}, + }} +} + +type reconcileEnv struct { + proc *Processor + insp *seqInspector + fake *failingCgroup + pinner *fakePinner + apis int +} + +func newReconcileEnv(t *testing.T, mode policy.Mode, dryRun bool) *reconcileEnv { + t.Helper() + + env := &reconcileEnv{ + insp: &seqInspector{ + replies: []inspectReply{runningDevNull(reconcileStarted)}, + }, + fake: &failingCgroup{}, + pinner: &fakePinner{}, + } + env.proc = &Processor{ + Inspector: env.insp, + Cfg: newStore(mode, dryRun), + HostRoot: hostRootWithCgroup(t, reconcilePid), + ProcRoot: buildProcRoot(t, reconcilePid), + pinner: env.pinner, + newCgroup: func(int, *cgroup.Ledger, *cgroup.FilterCache) (cgroup.Interface, error) { + env.apis++ + + return env.fake, nil + }, + } + + return env +} + +func (e *reconcileEnv) reconcile() error { + return e.proc.Reconcile(context.Background(), "abc") +} + +// grantOnce runs one successful reconcile that grants /dev/null, leaving the +// lifecycle's cgroup known. +func (e *reconcileEnv) grantOnce(t *testing.T) { + t.Helper() + + err := e.reconcile() + if err != nil { + t.Fatal(err) + } + + if e.fake.calls != 1 || len(e.fake.rules[0]) != 1 { + t.Fatalf( + "grant: SetDeviceRules calls = %d with %v, want one call with /dev/null", + e.fake.calls, + e.fake.rules, + ) + } + + if len(e.proc.known) != 1 { + t.Fatalf("known = %v, want the lifecycle recorded", e.proc.known) + } +} + +// TestReconcile_PidRecycledBeforePinDoesNotMutate: the container exits right +// after the first inspect and its pid goes to an unrelated process in +// another cgroup. The pin and the /proc reads then describe that process, +// but the second inspect no longer reports the pid, so nothing is mutated +// and the foreign cgroup is not recorded. +func TestReconcile_PidRecycledBeforePinDoesNotMutate(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + + other := filepath.Join(env.proc.HostRoot, "sys", "fs", "cgroup", "system.slice", "other") + + err := os.MkdirAll(other, 0o755) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile( + filepath.Join(other, "cgroup.procs"), + []byte(strconv.Itoa(reconcilePid)+"\n"), + 0o600, + ) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile( + filepath.Join(env.proc.ProcRoot, "proc", strconv.Itoa(reconcilePid), "cgroup"), + []byte("0::/system.slice/other\n"), + 0o600, + ) + if err != nil { + t.Fatal(err) + } + + env.insp.then(runningDevNull(reconcileStarted), exited()) + + err = env.reconcile() + if !errors.Is(err, errLifecycleChanged) { + t.Fatalf("err = %v, want errLifecycleChanged", err) + } + + if env.fake.calls != 0 || len(env.proc.known) != 0 { + t.Errorf("SetDeviceRules calls = %d, known = %v; want no mutation and nothing recorded", + env.fake.calls, env.proc.known) + } +} + +func TestReconcile_StartedAtChangedDoesNotMutate(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.insp.then( + runningDevNull(reconcileStarted), + runningDevNull("2026-09-27T10:00:05.000000000Z"), + ) + + err := env.reconcile() + if !errors.Is(err, errLifecycleChanged) { + t.Fatalf("err = %v, want errLifecycleChanged", err) + } + + if env.fake.calls != 0 { + t.Errorf("SetDeviceRules calls = %d, want none", env.fake.calls) + } +} + +func TestReconcile_PinnedProcessGoneDoesNotMutate(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.pinner.gone = true + + err := env.reconcile() + if !errors.Is(err, errPinnedProcessGone) { + t.Fatalf("err = %v, want errPinnedProcessGone", err) + } + + if env.fake.calls != 0 { + t.Errorf("SetDeviceRules calls = %d, want none", env.fake.calls) + } +} + +// TestReconcile_MembershipGatesGrantsOnly: a pid missing from cgroup.procs +// blocks a grant, but not the empty set, which can only narrow. +func TestReconcile_MembershipGatesGrantsOnly(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + writeProcs(t, env.proc.HostRoot) + + err := env.reconcile() + if !errors.Is(err, errNotInCgroup) || env.fake.calls != 0 { + t.Fatalf( + "grant: err = %v, calls = %d; want errNotInCgroup and no mutation", + err, + env.fake.calls, + ) + } + + env.proc.Cfg = newStore(policy.ModeOptIn, false) + + err = env.reconcile() + if err != nil || env.fake.calls != 1 || len(env.fake.rules[0]) != 0 { + t.Fatalf( + "empty set: err = %v, rules = %v; want one empty SetDeviceRules", + err, + env.fake.rules, + ) + } +} + +// TestReconcile_RevokeAfterExitWithMatchingInode: once the container has +// exited nothing can be pinned; the empty set goes to the cgroup the +// lifecycle was verified in, still the same directory, and the entry is +// kept until the cgroup is verified gone. +func TestReconcile_RevokeAfterExitWithMatchingInode(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.grantOnce(t) + + granted := env.fake.identity + + writeProcs(t, env.proc.HostRoot) + env.insp.then(exited()) + + err := env.reconcile() + if err != nil { + t.Fatal(err) + } + + if env.fake.calls != 2 || len(env.fake.rules[1]) != 0 || env.fake.identity != granted { + t.Fatalf("revoke: calls = %d, rules = %v, identity = %+v; want an empty set on %+v", + env.fake.calls, env.fake.rules, env.fake.identity, granted) + } + + if len(env.pinner.pins) != 1 { + t.Errorf("pins = %v, want only the grant to pin", env.pinner.pins) + } + + if len(env.proc.known) != 1 { + t.Errorf("known = %v, want the entry kept until the cgroup is gone", env.proc.known) + } +} + +// TestReconcile_InodeMismatchReleasesWithoutMutation: the cgroup was +// recreated at the same path, so it holds none of the lifecycle's grants. +func TestReconcile_InodeMismatchReleasesWithoutMutation(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.grantOnce(t) + + // Build the replacement before removing the original, so the new + // directory cannot reuse the old inode number. + dir := testCgroupDir(env.proc.HostRoot) + + err := os.Mkdir(dir+".new", 0o755) + if err != nil { + t.Fatal(err) + } + + err = os.RemoveAll(dir) + if err != nil { + t.Fatal(err) + } + + err = os.Rename(dir+".new", dir) + if err != nil { + t.Fatal(err) + } + + env.insp.then(exited()) + + err = env.reconcile() + if err != nil { + t.Fatal(err) + } + + if env.fake.calls != 1 { + t.Errorf("SetDeviceRules calls = %d, want only the grant", env.fake.calls) + } + + if len(env.proc.known) != 0 { + t.Errorf("known = %v, want the entry released", env.proc.known) + } +} + +func TestReconcile_ExitAfterCgroupRemovedReleases(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.grantOnce(t) + + err := os.RemoveAll(testCgroupDir(env.proc.HostRoot)) + if err != nil { + t.Fatal(err) + } + + env.insp.then(exited()) + + err = env.reconcile() + if err != nil || env.fake.calls != 1 || len(env.proc.known) != 0 { + t.Fatalf("err = %v, calls = %d, known = %v; want release without mutation", + err, env.fake.calls, env.proc.known) + } +} + +// TestReconcile_InspectFailureRevokesKnownIdentity: without a pid the +// container's grants are revoked where its lifecycle was verified, and the +// container stays pending. With no known identity it is pending only. +func TestReconcile_InspectFailureRevokesKnownIdentity(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.insp.then(inspectReply{err: errDaemonUnavail}) + + err := env.reconcile() + if !errors.Is(err, errDaemonUnavail) || env.fake.calls != 0 { + t.Fatalf( + "unknown: err = %v, calls = %d; want pending with no mutation", + err, + env.fake.calls, + ) + } + + env.insp.then(runningDevNull(reconcileStarted)) + env.grantOnce(t) + env.insp.then(inspectReply{err: errDaemonUnavail}) + + err = env.reconcile() + if !errors.Is(err, errDaemonUnavail) { + t.Fatalf("known: err = %v, want the inspect error (container stays pending)", err) + } + + if env.fake.calls != 2 || len(env.fake.rules[1]) != 0 { + t.Fatalf("known: rules = %v, want the empty set applied", env.fake.rules) + } + + if len(env.proc.known) != 1 { + t.Errorf("known = %v, want the entry kept (not-found alone never releases)", env.proc.known) + } +} + +func TestReconcile_DisabledContainerSetsEmptySet(t *testing.T) { + buf := captureLogger(t) + env := newReconcileEnv(t, policy.ModeOptIn, false) + + err := env.reconcile() + if err != nil { + t.Fatal(err) + } + + if env.fake.calls != 1 || len(env.fake.rules[0]) != 0 { + t.Fatalf("rules = %v, want one SetDeviceRules with the empty set", env.fake.rules) + } + + if strings.Contains(buf.String(), "container processed") { + t.Errorf( + "a container skipped by policy must not also be reported processed:\n%s", + buf.String(), + ) + } +} + +func TestReconcile_InvalidLabelsSetEmptySet(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + reply := runningDevNull(reconcileStarted) + reply.resp.Config = &container.Config{Labels: map[string]string{policy.LabelEnable: "maybe"}} + env.insp.then(reply) + + err := env.reconcile() + if err != nil || env.fake.calls != 1 || len(env.fake.rules[0]) != 0 { + t.Fatalf("err = %v, rules = %v; want the empty set applied", err, env.fake.rules) + } +} + +// TestReconcile_IncompleteDeviceSetSetsEmptySet: one unresolved device +// empties the whole set; the container is retried. +func TestReconcile_IncompleteDeviceSetSetsEmptySet(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + reply := runningDevNull(reconcileStarted) + reply.resp.Mounts = append( + reply.resp.Mounts, + container.MountPoint{ + Source: "/dev/sda-does-not-exist", + Destination: "/dev/x", + Type: mount.TypeBind, + }, + ) + env.insp.then(reply) + + err := env.reconcile() + if err == nil || !strings.Contains(err.Error(), "incomplete_device_set") { + t.Fatalf("err = %v, want a retryable incomplete_device_set error", err) + } + + if env.fake.calls != 1 || len(env.fake.rules[0]) != 0 { + t.Fatalf("rules = %v, want the empty set applied", env.fake.rules) + } +} + +// TestReconcile_DryRunIssuesNoCgroupCalls: dry-run computes and reports the +// set but pins nothing, reads no /proc and builds no cgroup API. +func TestReconcile_DryRunIssuesNoCgroupCalls(t *testing.T) { + buf := captureLogger(t) + env := newReconcileEnv(t, policy.ModeAll, true) + env.proc.ProcRoot = filepath.Join(t.TempDir(), "absent") + + err := env.reconcile() + if err != nil { + t.Fatal(err) + } + + env.insp.then(exited()) + + err = env.reconcile() + if err != nil { + t.Fatal(err) + } + + if env.apis != 0 || env.fake.calls != 0 || len(env.pinner.pins) != 0 { + t.Errorf("apis = %d, calls = %d, pins = %v; want no cgroup or pid access", + env.apis, env.fake.calls, env.pinner.pins) + } + + if !strings.Contains(buf.String(), "dry-run: would set device rules") { + t.Errorf("expected the would-set line, got:\n%s", buf.String()) + } +} + +// TestReconcile_CancelledCallerKeepsGrants: an inspect that fails because +// the caller gave up (daemon shutdown) is not Docker failing; nothing would +// retry, so a running container must keep its grants. +func TestReconcile_CancelledCallerKeepsGrants(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.grantOnce(t) + env.insp.then(inspectReply{err: context.Canceled}) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := env.proc.Reconcile(ctx, "abc") + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + + if env.fake.calls != 1 { + t.Errorf("SetDeviceRules calls = %d, want only the grant", env.fake.calls) + } +} + +// TestReconcile_EmptySetWithMissingFilterIsNotDone: an unprivileged +// container with no device filter is never reported done, whatever its +// desired set. +func TestReconcile_EmptySetWithMissingFilterIsNotDone(t *testing.T) { + env := newReconcileEnv(t, policy.ModeOptIn, false) + env.fake.err = cgroup.ErrFilterMissing + + err := env.reconcile() + if !errors.Is(err, cgroup.ErrFilterMissing) || + !strings.Contains(err.Error(), "filter_missing") { + t.Fatalf("err = %v, want a retryable filter_missing error", err) + } +} + +// TestReconcile_SkippedContainerStillReportsIncompleteSet: a container +// skipped for good (privileged, no filter) keeps its incomplete-set error. +func TestReconcile_SkippedContainerStillReportsIncompleteSet(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.fake.err = cgroup.ErrFilterMissing + reply := runningDevNull(reconcileStarted) + reply.resp.HostConfig = &container.HostConfig{Privileged: true} + reply.resp.Mounts = append( + reply.resp.Mounts, + container.MountPoint{ + Source: "/dev/sda-does-not-exist", + Destination: "/dev/x", + Type: mount.TypeBind, + }, + ) + env.insp.then(reply) + + err := env.reconcile() + if err == nil || !strings.Contains(err.Error(), "incomplete_device_set") { + t.Fatalf("err = %v, want the incomplete_device_set error", err) + } +} + +// TestReconcile_DeadlineExceededRevokes: an inspect that ran out of time is +// Docker not answering, even when the caller's own deadline has passed too; +// the known grants are revoked. +func TestReconcile_DeadlineExceededRevokes(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.grantOnce(t) + env.insp.then(inspectReply{err: context.DeadlineExceeded}) + + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancel() + + err := env.proc.Reconcile(ctx, "abc") + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v, want context.DeadlineExceeded (container stays pending)", err) + } + + if env.fake.calls != 2 || len(env.fake.rules[1]) != 0 { + t.Fatalf("rules = %v, want the empty set applied", env.fake.rules) + } +} + +// TestReconcile_InspectFailureOfPrivilegedDoesNotWarn: a privileged +// container never has a device filter, so a missing one is not reported. +func TestReconcile_InspectFailureOfPrivilegedDoesNotWarn(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + reply := runningDevNull(reconcileStarted) + reply.resp.HostConfig = &container.HostConfig{Privileged: true} + env.insp.then(reply) + env.grantOnce(t) + + buf := captureLogger(t) + env.fake.err = cgroup.ErrFilterMissing + env.insp.then(inspectReply{err: errDaemonUnavail}) + + err := env.reconcile() + if !errors.Is(err, errDaemonUnavail) { + t.Fatalf("err = %v, want the inspect error", err) + } + + if strings.Contains(buf.String(), "restart the container") { + t.Errorf("privileged container must not get the filter_missing warning:\n%s", buf.String()) + } +} diff --git a/internal/processor/serialize_test.go b/internal/processor/serialize_test.go index 2d529f0..fc032d5 100644 --- a/internal/processor/serialize_test.go +++ b/internal/processor/serialize_test.go @@ -48,15 +48,18 @@ func (c *countingInspector) ContainerInspect( return client.ContainerInspectResult{Container: c.result}, nil } -// TestProcessContainer_SerializesConfigLoadThroughApply holds one worker +// TestReconcile_SerializesConfigLoadThroughApply holds one worker // between computing its rules and applying them, then starts a second worker // for the same container and publishes a new config. The second worker must // not load the config or compute anything until the first has finished, so -// no computation can be overtaken by an older one. -func TestProcessContainer_SerializesConfigLoadThroughApply(t *testing.T) { +// no computation can be overtaken by an older one, and must then compute +// under the new config. +func TestReconcile_SerializesConfigLoadThroughApply(t *testing.T) { const pid = 60 - insp := &countingInspector{result: container.InspectResponse{State: &container.State{Pid: pid}}} + insp := &countingInspector{ + result: container.InspectResponse{State: &container.State{Running: true, Pid: pid}}, + } store := newStore(policy.ModeAll, true) proc := &Processor{ Inspector: insp, @@ -87,11 +90,11 @@ func TestProcessContainer_SerializesConfigLoadThroughApply(t *testing.T) { var workers sync.WaitGroup - workers.Go(func() { _ = proc.ProcessContainer(context.Background(), "c1") }) + workers.Go(func() { _ = proc.Reconcile(context.Background(), "c1") }) <-firstInside - workers.Go(func() { _ = proc.ProcessContainer(context.Background(), "c1") }) + workers.Go(func() { _ = proc.Reconcile(context.Background(), "c1") }) // Positive signal first: the second worker has finished its inspect, so it // is at (or past) the lock. @@ -117,13 +120,9 @@ func TestProcessContainer_SerializesConfigLoadThroughApply(t *testing.T) { close(release) workers.Wait() - if got := computed.Load(); got != 1 { - // The second worker loads the new opt-in config and skips the - // unlabeled container before computing anything. - t.Fatalf("computed = %d after release, want 1 (second run must see the new config)", got) - } - - if len(modes) != 1 || modes[0] != policy.ModeAll { - t.Fatalf("modes seen by computing workers = %v, want [all]", modes) + // The second worker computes (the empty set, for an unlabeled container + // under opt-in) from the config published while it waited. + if len(modes) != 2 || modes[0] != policy.ModeAll || modes[1] != policy.ModeOptIn { + t.Fatalf("modes seen by computing workers = %v, want [all opt-in]", modes) } } From 0e2282cf0dabdb6201d834cb1848ddd3650987ec Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 10:44:53 +0200 Subject: [PATCH 09/37] feat(daemon): run every reconciliation through one coordinator A single coordinator goroutine now owns reconciliation passes: it tracks the latest config generation and a trigger epoch bumped by every request, lists containers with a bounded ContainerList (retrying the whole pass with minBackoff/maxBackoff when the list fails), retries failed containers from a pending set with per-container backoff until they succeed, disappear (Processor reports ErrContainerGone) or the daemon stops, and owns the processed dedup map (with its TTL) that the event consumer reaches only through mutex-guarded methods. A newer request supersedes an in-flight pass after its current container, re-tags pending entries and coalesces into one trailing pass; results from an older generation or epoch never clear the gauges or log completion. Run waits for the coordinator before returning. New gauges sda_reconcile_pending_containers and sda_reload_incomplete; the daemon warns while either is non-zero and logs config reload complete once both are zero for the latest request. config.NewStore now returns the Store and a Publisher, and every publication bumps a generation. Processor.PublishAndReconcile publishes under the processor lock and hands the generation to the coordinator; it is the only publication path after startup (enforced by a source test). SIGHUP (now registered before startup work), systemd reloads and startup all request passes; the startup pass reconciles every running container, so a live start replaces or strips grants a previous instance left, while a dry-run start warns that it cannot. README, SECURITY.md and the architecture metrics table document the passes, the gauges and that restarting into dry-run is not a cleanup path; a new enforcement test (SDA_IT_ENFORCE=1) covers the kill-and-restart cases. --- README.md | 18 +- SECURITY.md | 4 + cmd/swarm-device-access/config.go | 32 +- cmd/swarm-device-access/main.go | 27 +- docs/architecture.md | 2 + go.mod | 2 +- internal/config/config.go | 65 +++- internal/config/publish_test.go | 122 +++++++ internal/daemon/coordinator.go | 491 +++++++++++++++++++++++++ internal/daemon/coordinator_test.go | 513 +++++++++++++++++++++++++++ internal/daemon/daemon.go | 121 ++----- internal/daemon/daemon_test.go | 84 +++-- internal/daemon/events.go | 46 +-- internal/daemon/events_test.go | 89 ++--- internal/daemon/timeout_test.go | 31 +- internal/observability/metrics.go | 36 ++ internal/processor/processor.go | 10 +- internal/processor/processor_test.go | 12 +- internal/processor/publish.go | 61 ++++ internal/processor/reconcile.go | 30 +- internal/processor/reconcile_test.go | 62 ++++ internal/processor/serialize_test.go | 79 ++++- test/integration/enforce_test.go | 87 ++++- 23 files changed, 1733 insertions(+), 291 deletions(-) create mode 100644 internal/config/publish_test.go create mode 100644 internal/daemon/coordinator.go create mode 100644 internal/daemon/coordinator_test.go create mode 100644 internal/processor/publish.go diff --git a/README.md b/README.md index 6060918..054f481 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ value is set in both places, the CLI flag wins. | `-log-format` | `text` | `text`, `json`, `plain` | | `-log-time` | `false` | Include timestamps in log lines | | `-docker-socket` | `/var/run/docker.sock` | Path to the Docker daemon's UNIX socket | -| `-dry-run` | `false` | Log the device set each container would get, without reading its `/proc` entry or its cgroup and without writing anything | +| `-dry-run` | `false` | Log the device set each container would get, without reading its `/proc` entry or its cgroup and without writing anything. Not a cleanup path: grants left by a previous live run stay (see [Config File](#config-file)) | | `-policy-mode` | `opt-in` | `opt-in`: only `enable=true` containers. `all`: unless `enable=false`. | | `-device-allow` | `""` | Glob for `/dev/...` paths to allow, repeatable. Empty means allow all. | | `-device-deny` | `""` | Glob for `/dev/...` paths to deny, repeatable. Deny takes priority over allow. | @@ -237,6 +237,22 @@ kill -HUP $(docker inspect --format '{{.State.Pid}}' swarm-device-access) Some settings are only read at startup and still require a restart: `docker-socket`, `metrics-addr`, and `debug-addr`. +A reload, a daemon start and a `systemctl daemon-reload` each reconcile every +running container under the current config, so narrowing the policy revokes +grants the new policy no longer allows. Until a pass has reached every running +container, `sda_reload_incomplete` is 1; containers whose reconcile failed are +retried with backoff and counted in `sda_reconcile_pending_containers`. The +daemon warns while either is non-zero and logs `config reload complete` once +both are zero for the latest config. + +Restarting the daemon **live** is also a cleanup: the startup pass replaces or +strips the grants a previous instance left on running containers (on cgroup v1 +only within the limits described under [Host Requirements](#host-requirements)). +Restarting into **dry-run** is not: dry-run never looks at cgroups, so grants a +previous live run left stay in place until a live start or a container +restart. The daemon logs `dry-run: grants left by a previous live run cannot be +detected or cleaned in dry-run` at every dry-run start. + ### Container Labels Consumer services opt in and narrow their allowed device set with labels: diff --git a/SECURITY.md b/SECURITY.md index 9a605d7..243999e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -58,4 +58,8 @@ Expected response time: acknowledgment within 7 days, patch or mitigation plan w restarted. Within one daemon run, narrowing revokes exactly what was granted and never touches the runtime's own exceptions. cgroup v1 is deprecated by Docker and systemd; cgroup v2 hosts are not affected by this gap. +- **Restarting into dry-run does not revoke anything.** Dry-run never queries or mutates cgroups + (it runs unprivileged in tests), so grants left by a previous live run stay until a live start + or a container restart; the daemon warns about this at every dry-run start. Restart live with a + narrower policy, or restart the affected containers, to take grants back. - The cgroup BPF code in `internal/cgroup/` is derived from [NVIDIA's container toolkit](https://github.com/NVIDIA/libnvidia-container) (Apache 2.0). Security issues in that code should also be reported to NVIDIA. diff --git a/cmd/swarm-device-access/config.go b/cmd/swarm-device-access/config.go index af08d86..33db14f 100644 --- a/cmd/swarm-device-access/config.go +++ b/cmd/swarm-device-access/config.go @@ -22,20 +22,19 @@ import ( "context" "flag" "os" - "os/signal" - "syscall" "github.com/leinardi/swarm-device-access/internal/config" "github.com/leinardi/swarm-device-access/internal/logger" "github.com/leinardi/swarm-device-access/internal/policy" + "github.com/leinardi/swarm-device-access/internal/processor" ) // applyFileConfig merges the file config into the CLI flags (for flags not -// explicitly set by the user) and stores the result in store. -// Must be called after flag.Parse() and after store is initialized. +// explicitly set by the user) and returns the resulting startup Runtime. +// Must be called after flag.Parse(). // //nolint:gocyclo,cyclop // complexity is inherent: merges many independent optional config fields -func applyFileConfig(fileCfg *config.FileSchema, store *config.Store) { +func applyFileConfig(fileCfg *config.FileSchema) config.Runtime { cliSet := make(map[string]bool) flag.Visit(func(f *flag.Flag) { cliSet[f.Name] = true }) @@ -80,27 +79,27 @@ func applyFileConfig(fileCfg *config.FileSchema, store *config.Store) { deviceDeny = fileCfg.DeviceDeny } - store.Set(config.Runtime{ + return config.Runtime{ DryRun: *dryRun, Policy: policy.Global{ Mode: policy.Mode(*policyMode), DeviceAllow: append([]string(nil), deviceAllow...), DeviceDeny: append([]string(nil), deviceDeny...), }, - }) + } } // watchSIGHUP blocks until ctx is done, reloading the config file and -// updating store + logger on each SIGHUP. Settings that require a -// restart (docker-socket, metrics-addr, debug-addr) are not reloaded. +// updating the logger on each SIGHUP, then publishing the new config through +// proc, which reconciles every running container under it. Settings that +// require a restart (docker-socket, metrics-addr, debug-addr) are not +// reloaded. +// +// sigCh must already be registered for SIGHUP (see run), so a SIGHUP during +// startup is queued instead of terminating the process. // //nolint:gocyclo,cyclop,gocognit // complexity is inherent: handles SIGHUP, reload, merge, validate, and logger update -func watchSIGHUP(ctx context.Context, store *config.Store) { - sigCh := make(chan os.Signal, 1) - - signal.Notify(sigCh, syscall.SIGHUP) - defer signal.Stop(sigCh) - +func watchSIGHUP(ctx context.Context, sigCh <-chan os.Signal, proc *processor.Processor) { for { select { case <-ctx.Done(): @@ -177,12 +176,13 @@ func watchSIGHUP(ctx context.Context, store *config.Store) { continue } - store.Set(config.Runtime{ + generation := proc.PublishAndReconcile(ctx, config.Runtime{ DryRun: newDryRun, Policy: newPolicy, }) logger.L().Info("config reloaded", + "generation", generation, "dry_run", newDryRun, "policy_mode", newPolicy.Mode, "device_allow", newPolicy.DeviceAllow, diff --git a/cmd/swarm-device-access/main.go b/cmd/swarm-device-access/main.go index 56ef8b3..61cfe05 100644 --- a/cmd/swarm-device-access/main.go +++ b/cmd/swarm-device-access/main.go @@ -54,8 +54,6 @@ func run() int { return 0 } - store := config.NewStore() - // Load config file; CLI flags override file values. fileCfg, fileErr := config.LoadFile(*configFile) if fileErr != nil { @@ -64,7 +62,14 @@ func run() int { return 1 } - applyFileConfig(&fileCfg, store) + store, publisher := config.NewStore(applyFileConfig(&fileCfg)) + + // Registered before anything slow, so a SIGHUP during startup is queued + // for watchSIGHUP instead of terminating the process (its default). + sighup := make(chan os.Signal, 1) + + signal.Notify(sighup, syscall.SIGHUP) + defer signal.Stop(sighup) startupValidationErr := store.Load().Policy.Validate() if startupValidationErr != nil { @@ -89,6 +94,14 @@ func run() int { "device_deny", cfg.Policy.DeviceDeny, ) + if cfg.DryRun { + // Dry-run has no cgroup view and no saved ownership state, so it + // cannot even count what a previous live run left behind. + log.Warn( + "dry-run: grants left by a previous live run cannot be detected or cleaned in dry-run", + ) + } + // Lift RLIMIT_MEMLOCK once for the process so BPF_PROG_LOAD does not fail // on kernels that still charge BPF memory to it (no-op from Linux 5.11, // where memcg accounting replaced it). The limit is not inherited by @@ -109,9 +122,6 @@ func run() int { ) defer cancelRoot() - // SIGHUP: reload the config file and update hot settings + logger. - go watchSIGHUP(rootCtx, store) - // API version negotiation is the client default; it runs lazily on the first request. cli, err := client.New(client.WithHost("unix://" + *dockerSocket)) if err != nil { @@ -136,12 +146,17 @@ func run() int { proc := &processor.Processor{ Inspector: cli, Cfg: store, + Publisher: publisher, Metrics: recorder, HostRoot: hostRootPath, ProcRoot: "/", CallTimeout: daemon.DockerCallTimeout, } + // SIGHUP: reload the config file, update the logger and publish the + // new config through the processor. + go watchSIGHUP(rootCtx, sighup, proc) + runErr := daemon.Run(rootCtx, daemon.Options{ Docker: cli, Proc: proc, diff --git a/docs/architecture.md b/docs/architecture.md index ea5742d..da4473c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -230,6 +230,8 @@ Metrics exposed: | `sda_rule_failures_total` | counter | — | Per-device rule failures (including walk errors); the per-device failure signal | | `sda_dry_run_skips_total` | counter | — | Rules skipped in dry-run mode | | `sda_last_event_timestamp_seconds` | gauge | — | Unix timestamp of last container processed successfully (event, startup or reload) | +| `sda_reconcile_pending_containers` | gauge | — | Containers whose last reconcile failed and are retried with backoff | +| `sda_reload_incomplete` | gauge | — | 1 until a pass for the latest config and trigger has reached every running container, else 0 | ### pprof debug server (`--debug-addr`) diff --git a/go.mod b/go.mod index eac3458..e3cd2b3 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.8 require ( github.com/cilium/ebpf v0.22.0 + github.com/containerd/errdefs v1.0.0 github.com/godbus/dbus/v5 v5.2.2 github.com/moby/moby/api v1.56.0 github.com/moby/moby/client v0.6.0 @@ -16,7 +17,6 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-connections v0.7.0 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index 0d0cfe4..aef3c39 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -19,6 +19,7 @@ package config import ( + "sync" "sync/atomic" "github.com/leinardi/swarm-device-access/internal/policy" @@ -31,29 +32,61 @@ type Runtime struct { Policy policy.Global } -// Store provides atomic read/write access to the current Runtime. +// snapshot is one published Runtime and its generation. +type snapshot struct { + runtime Runtime + generation uint64 +} + +// Store provides atomic read access to the current Runtime. Only the +// Publisher returned with it can replace the Runtime, so whoever holds the +// Publisher (the processor, after startup) is the one path that publishes. type Store struct { - current atomic.Pointer[Runtime] + current atomic.Pointer[snapshot] +} + +// Publisher replaces the Runtime of the Store it was created with. +type Publisher struct { + store *Store + // mu orders publications, so generations are handed out in the order + // the snapshots become visible. + mu sync.Mutex } -// NewStore returns an initialized Store with a zero-value Runtime. -// Call Set to populate it after parsing the initial configuration. -func NewStore() *Store { - return &Store{} +// NewStore returns a Store holding initial as generation 1, and the +// Publisher for it. +func NewStore(initial Runtime) (*Store, *Publisher) { + store := &Store{} + store.current.Store(&snapshot{runtime: initial, generation: 1}) + + return store, &Publisher{store: store} } -// Load returns the current Runtime. Returns a zero-value Runtime if Set has -// not been called yet. +// Load returns the current Runtime. func (s *Store) Load() Runtime { - ptr := s.current.Load() - if ptr == nil { - return Runtime{} - } + return s.current.Load().runtime +} - return *ptr +// Snapshot returns the current Runtime and its generation. +func (s *Store) Snapshot() (rt Runtime, generation uint64) { + current := s.current.Load() + + return current.runtime, current.generation +} + +// Generation returns the generation of the current Runtime. It starts at 1 +// and grows by one with every publication. +func (s *Store) Generation() uint64 { + return s.current.Load().generation } -// Set atomically replaces the current Runtime. -func (s *Store) Set(rt Runtime) { - s.current.Store(&rt) +// Publish atomically replaces the Runtime and returns its generation. +func (p *Publisher) Publish(rt Runtime) uint64 { + p.mu.Lock() + defer p.mu.Unlock() + + generation := p.store.Generation() + 1 + p.store.current.Store(&snapshot{runtime: rt, generation: generation}) + + return generation } diff --git a/internal/config/publish_test.go b/internal/config/publish_test.go new file mode 100644 index 0000000..785611c --- /dev/null +++ b/internal/config/publish_test.go @@ -0,0 +1,122 @@ +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package config + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "strings" + "testing" +) + +func TestPublisher_GenerationsGrow(t *testing.T) { + t.Parallel() + + store, publisher := NewStore(Runtime{}) + if store.Generation() != 1 { + t.Fatalf("initial generation = %d, want 1", store.Generation()) + } + + gen := publisher.Publish(Runtime{DryRun: true}) + + rt, snapGen := store.Snapshot() + if gen != 2 || snapGen != 2 || !rt.DryRun { + t.Errorf( + "after Publish: returned %d, snapshot %d %+v; want 2, 2, dry-run", + gen, + snapGen, + rt, + ) + } +} + +// TestPublisher_OnlyProcessorPublishes enforces that, outside tests, the +// config changes only through Processor.PublishAndReconcile: every other +// path would publish without taking the processor's lock or requesting a +// reconciliation pass. +func TestPublisher_OnlyProcessorPublishes(t *testing.T) { + t.Parallel() + + root := filepath.Join("..", "..") + fset := token.NewFileSet() + + var ( + offenders []string + allowed int + ) + + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + + if entry.IsDir() { + if strings.HasPrefix(entry.Name(), ".") && path != root { + return filepath.SkipDir + } + + return nil + } + + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + + file, parseErr := parser.ParseFile(fset, path, nil, 0) + if parseErr != nil { + return fmt.Errorf("parse %s: %w", path, parseErr) + } + + ast.Inspect(file, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok || sel.Sel.Name != "Publish" { + return true + } + + if filepath.ToSlash(filepath.Dir(path)) == "../../internal/processor" { + allowed++ + } else { + offenders = append(offenders, fset.Position(call.Pos()).String()) + } + + return true + }) + + return nil + }) + if err != nil { + t.Fatal(err) + } + + // Guards the scan itself: it must see the one allowed call. + if allowed != 1 { + t.Errorf("found %d Publish calls in internal/processor, want 1", allowed) + } + + if len(offenders) > 0 { + t.Errorf("config published outside Processor.PublishAndReconcile: %v", offenders) + } +} diff --git a/internal/daemon/coordinator.go b/internal/daemon/coordinator.go new file mode 100644 index 0000000..0e7d64e --- /dev/null +++ b/internal/daemon/coordinator.go @@ -0,0 +1,491 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package daemon + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/moby/moby/client" + + "github.com/leinardi/swarm-device-access/internal/logger" + "github.com/leinardi/swarm-device-access/internal/observability" + "github.com/leinardi/swarm-device-access/internal/processor" +) + +// processedTTL bounds how long an enumerated container's entry suppresses a +// start event that is not newer than its inspect. After 2×maxBackoff the +// overlap window between enumeration and the event stream has certainly +// passed; a remaining entry belongs to a container that exited before its +// event was consumed. +const processedTTL = 2 * maxBackoff + +// passKey is what a pass reconciles for: a config generation and a trigger +// epoch. Every request bumps the epoch, so two systemd reloads under one +// generation are still two passes (the second can wipe what the first +// restored). +type passKey struct { + generation uint64 + epoch uint64 +} + +// pendingEntry is a container whose last reconcile failed. +type pendingEntry struct { + // key is the request the entry must be reconciled for; it is re-tagged + // with every newer request, so an older result cannot clear it. + key passKey + backoff time.Duration + due time.Time +} + +// passOutcome is how a pass ended. +type passOutcome int + +const ( + passDone passOutcome = iota + passListFailed + passSuperseded +) + +// coordinator is the single owner of reconciliation passes. Startup, +// SIGHUP (through Processor.PublishAndReconcile) and systemd reloads only +// request a pass; one goroutine (run) runs them, retries the containers +// that failed, maintains the pending and reload-incomplete gauges and logs +// completion. Requests arriving while a pass runs coalesce: the pass +// finishes its current container, then starts again under the newest +// request, so no result is reported on behalf of an older one. +// +// It also owns the processed map the event consumer uses to skip start +// events already covered by an enumeration. +type coordinator struct { + docker dockerAPI + apply applyFn + metrics *observability.Recorder + timeout time.Duration + + minBackoff time.Duration + maxBackoff time.Duration + now func() time.Time + + // wake has room for one signal; a request never blocks on it. + wake chan struct{} + // passEnded receives nothing; it is closed after the first pass + // attempt so Run can start consuming events after the enumeration. + passEnded chan struct{} + + mu sync.Mutex + // latest is the newest request; dirty means no pass has started for it. + latest passKey + dirty bool + // incomplete is set from a request until a pass for the latest request + // has visited every listed container. + incomplete bool + // announced is the request whose completion was last logged. + announced passKey + pending map[string]*pendingEntry + processed map[string]time.Time + firstDone bool +} + +func newCoordinator( + docker dockerAPI, + apply applyFn, + metrics *observability.Recorder, + timeout time.Duration, +) *coordinator { + return &coordinator{ + docker: docker, + apply: apply, + metrics: metrics, + timeout: timeout, + minBackoff: minBackoff, + maxBackoff: maxBackoff, + now: time.Now, + wake: make(chan struct{}, 1), + passEnded: make(chan struct{}), + pending: make(map[string]*pendingEntry), + processed: make(map[string]time.Time), + } +} + +// request asks for a pass under generation (or the newest generation +// already requested, whichever is higher). It never blocks. +func (c *coordinator) request(generation uint64) { + c.mu.Lock() + + c.latest.epoch++ + c.latest.generation = max(c.latest.generation, generation) + c.dirty = true + c.incomplete = true + + for _, entry := range c.pending { + entry.key = c.latest + } + + c.metrics.SetReloadIncomplete(true) + c.mu.Unlock() + + select { + case c.wake <- struct{}{}: + default: + } +} + +// requestPass adapts request to processor.PassRequester. +func (c *coordinator) requestPass(_ context.Context, generation uint64) { + c.request(generation) +} + +// awaitFirstPass blocks until the first pass has ended (completed, failed +// to list, or been superseded) or ctx is done. +func (c *coordinator) awaitFirstPass(ctx context.Context) { + select { + case <-c.passEnded: + case <-ctx.Done(): + } +} + +// run executes passes and retries until ctx is done. +func (c *coordinator) run(ctx context.Context) { + listBackoff := c.minBackoff + + var listRetry <-chan time.Time + + prune := time.NewTicker(processedTTL) + defer prune.Stop() + + for { + if ctx.Err() != nil { + return + } + + key, start := c.takeRequest() + if start { + switch c.pass(ctx, key) { + case passListFailed: + listRetry = time.After(listBackoff) + listBackoff = min(listBackoff*2, c.maxBackoff) + case passDone: + listRetry = nil + listBackoff = c.minBackoff + case passSuperseded: + } + + continue + } + + var retry <-chan time.Time + if due, ok := c.nextDue(); ok { + retry = time.After(max(due.Sub(c.now()), 0)) + } + + select { + case <-ctx.Done(): + return + case <-c.wake: + case <-listRetry: + listRetry = nil + + c.redo() + case <-retry: + c.retryDue(ctx) + case <-prune.C: + c.pruneProcessed() + } + } +} + +// takeRequest returns the latest request if no pass has started for it. +func (c *coordinator) takeRequest() (passKey, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.dirty { + return passKey{}, false + } + + c.dirty = false + + return c.latest, true +} + +// redo marks the latest request as needing a pass again (its list failed). +func (c *coordinator) redo() { + c.mu.Lock() + c.dirty = true + c.mu.Unlock() +} + +// pass lists the running containers and reconciles each under key. +func (c *coordinator) pass(ctx context.Context, key passKey) passOutcome { + defer c.markFirstPassEnded() + + log := logger.L() + + listCtx, cancelList := context.WithTimeout(ctx, c.timeout) + list, err := c.docker.ContainerList(listCtx, client.ContainerListOptions{}) + + cancelList() + + if err != nil { + log.Warn("could not list containers; reconciliation incomplete, retrying", + "err", fmt.Errorf("list containers: %w", err), + "generation", key.generation) + c.warnIncomplete() + + return passListFailed + } + + log.Debug("reconciling running containers", + "count", len(list.Items), "generation", key.generation, "epoch", key.epoch) + + for idx := range list.Items { + if ctx.Err() != nil || c.superseded(key) { + return passSuperseded + } + + containerID := list.Items[idx].ID + inspectedAt := c.now() + + applyErr := processOne(ctx, containerID, c.metrics, c.apply, c.timeout, + "could not process running container") + c.settle(containerID, key, applyErr) + + if applyErr == nil { + c.markProcessed(containerID, inspectedAt) + } + } + + c.finishPass(key) + + return passDone +} + +// retryDue reconciles every pending container whose backoff has elapsed. +func (c *coordinator) retryDue(ctx context.Context) { + for _, containerID := range c.dueIDs() { + if ctx.Err() != nil { + return + } + + key := c.current() + applyErr := processOne(ctx, containerID, c.metrics, c.apply, c.timeout, + "could not reconcile pending container") + c.settle(containerID, key, applyErr) + } + + c.mu.Lock() + c.reportLocked() + c.mu.Unlock() +} + +// eventApplied records the outcome of an event-driven reconcile that +// started under key: a failure makes the container pending. +func (c *coordinator) eventApplied(containerID string, key passKey, applyErr error) { + c.settle(containerID, key, applyErr) + + if applyErr != nil { + // Wake run so the new entry's retry is scheduled. + select { + case c.wake <- struct{}{}: + default: + } + } +} + +// settle records one reconcile that started under key. Success clears the +// container's pending entry only if no newer request re-tagged it since; +// a container Docker no longer knows is dropped; any other failure is +// (re)scheduled with backoff. +func (c *coordinator) settle(containerID string, key passKey, applyErr error) { + c.mu.Lock() + defer c.mu.Unlock() + + entry, isPending := c.pending[containerID] + + switch { + case applyErr == nil || errors.Is(applyErr, processor.ErrContainerGone): + if isPending && entry.key == key { + delete(c.pending, containerID) + } else if isPending { + entry.due = c.now() + } + case isPending: + entry.backoff = min(entry.backoff*2, c.maxBackoff) + entry.due = c.now().Add(entry.backoff) + default: + c.pending[containerID] = &pendingEntry{ + key: c.latest, + backoff: c.minBackoff, + due: c.now().Add(c.minBackoff), + } + } + + c.metrics.SetPendingContainers(len(c.pending)) +} + +// finishPass records that a pass for key visited every listed container. +func (c *coordinator) finishPass(key passKey) { + c.mu.Lock() + defer c.mu.Unlock() + + if key == c.latest { + c.incomplete = false + c.metrics.SetReloadIncomplete(false) + } + + c.reportLocked() +} + +// reportLocked logs completion once per request when nothing is left for +// the latest request, and warns while anything is. +func (c *coordinator) reportLocked() { + log := logger.L() + + if c.incomplete || len(c.pending) > 0 { + log.Warn("reconciliation incomplete; retrying", + "pending_containers", len(c.pending), + "reload_incomplete", c.incomplete, + "generation", c.latest.generation) + + return + } + + if c.announced == c.latest { + return + } + + c.announced = c.latest + log.Info("config reload complete", + "generation", c.latest.generation, "epoch", c.latest.epoch) +} + +func (c *coordinator) warnIncomplete() { + c.mu.Lock() + defer c.mu.Unlock() + + c.reportLocked() +} + +func (c *coordinator) superseded(key passKey) bool { + c.mu.Lock() + defer c.mu.Unlock() + + return c.latest != key +} + +func (c *coordinator) current() passKey { + c.mu.Lock() + defer c.mu.Unlock() + + return c.latest +} + +func (c *coordinator) nextDue() (time.Time, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + var ( + next time.Time + found bool + ) + + for _, entry := range c.pending { + if !found || entry.due.Before(next) { + next, found = entry.due, true + } + } + + return next, found +} + +func (c *coordinator) dueIDs() []string { + c.mu.Lock() + defer c.mu.Unlock() + + now := c.now() + + var ids []string + + for containerID, entry := range c.pending { + if !entry.due.After(now) { + ids = append(ids, containerID) + } + } + + return ids +} + +func (c *coordinator) markFirstPassEnded() { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.firstDone { + c.firstDone = true + close(c.passEnded) + } +} + +// markProcessed records that containerID was reconciled from an inspect +// made at inspectedAt. +func (c *coordinator) markProcessed(containerID string, inspectedAt time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + + c.processed[containerID] = inspectedAt +} + +// skipEvent reports whether an event for containerID at eventTime is +// already covered by an enumeration: Docker emits "start" after the +// container is running, so an event not newer than the inspect was visible +// to it. The entry is consumed by the first event for the container either +// way, and ignored once older than processedTTL. +func (c *coordinator) skipEvent(containerID string, eventTime time.Time) bool { + c.mu.Lock() + defer c.mu.Unlock() + + inspectedAt, found := c.processed[containerID] + if !found { + return false + } + + delete(c.processed, containerID) + + if c.now().Sub(inspectedAt) > processedTTL { + return false + } + + return !eventTime.After(inspectedAt) +} + +func (c *coordinator) pruneProcessed() { + c.mu.Lock() + defer c.mu.Unlock() + + now := c.now() + + for containerID, inspectedAt := range c.processed { + if now.Sub(inspectedAt) > processedTTL { + delete(c.processed, containerID) + } + } +} diff --git a/internal/daemon/coordinator_test.go b/internal/daemon/coordinator_test.go new file mode 100644 index 0000000..d118c7e --- /dev/null +++ b/internal/daemon/coordinator_test.go @@ -0,0 +1,513 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package daemon + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/events" + "github.com/moby/moby/client" + "github.com/prometheus/client_golang/prometheus" + + "github.com/leinardi/swarm-device-access/internal/config" + "github.com/leinardi/swarm-device-access/internal/policy" + "github.com/leinardi/swarm-device-access/internal/processor" +) + +const testBackoff = 5 * time.Millisecond + +var errListFailed = errors.New("list failed") + +// recordingApply records the containers it is called for; gate, when set, +// holds the first call until it is closed. +type recordingApply struct { + mu sync.Mutex + ids []string + fail map[string]int // remaining failures per container + gate chan struct{} + entered chan struct{} +} + +func (r *recordingApply) apply(_ context.Context, id string) error { + r.mu.Lock() + r.ids = append(r.ids, id) + first := len(r.ids) == 1 + failing := r.fail[id] > 0 + + if failing { + r.fail[id]-- + } + r.mu.Unlock() + + if first && r.gate != nil { + close(r.entered) + <-r.gate + } + + if failing { + return errApplyFailed + } + + return nil +} + +func (r *recordingApply) seen() []string { + r.mu.Lock() + defer r.mu.Unlock() + + return slices.Clone(r.ids) +} + +func gatedApply() *recordingApply { + return &recordingApply{gate: make(chan struct{}), entered: make(chan struct{})} +} + +// startCoordinator runs a coordinator with test backoffs until the test +// ends. +func startCoordinator(t *testing.T, docker dockerAPI, apply applyFn) *coordinator { + t.Helper() + + coord := newCoordinator(docker, apply, sharedRecorder(), DockerCallTimeout) + coord.minBackoff = testBackoff + coord.maxBackoff = 4 * testBackoff + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + + go func() { + defer close(done) + + coord.run(ctx) + }() + + t.Cleanup(func() { + cancel() + <-done + }) + + return coord +} + +func (c *coordinator) state() (pending int, incomplete bool) { + c.mu.Lock() + defer c.mu.Unlock() + + return len(c.pending), c.incomplete +} + +func (c *coordinator) settled() bool { + pending, incomplete := c.state() + + return pending == 0 && !incomplete +} + +func (f *fakeDocker) listCalls() int { + f.mu.Lock() + defer f.mu.Unlock() + + count := 0 + + for _, call := range f.calls { + if call == "list" { + count++ + } + } + + return count +} + +// completions returns the generation of every "config reload complete" line. +func completions(logOutput string) []string { + var gens []string + + for line := range strings.SplitSeq(logOutput, "\n") { + if !strings.Contains(line, `msg="config reload complete"`) { + continue + } + + for field := range strings.FieldsSeq(line) { + if gen, ok := strings.CutPrefix(field, "generation="); ok { + gens = append(gens, gen) + } + } + } + + return gens +} + +func gaugeValue(t *testing.T, name string) float64 { + t.Helper() + + families, err := prometheus.DefaultGatherer.Gather() + if err != nil { + t.Fatalf("gather metrics: %v", err) + } + + for _, family := range families { + if family.GetName() == name { + return family.GetMetric()[0].GetGauge().GetValue() + } + } + + t.Fatalf("metric %s not registered", name) + + return 0 +} + +// logBuffer is a concurrency-safe log sink: the coordinator logs from its +// own goroutine while the test reads. +type logBuffer struct { + mu sync.Mutex + buf strings.Builder +} + +func (l *logBuffer) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + + written, err := l.buf.Write(p) + if err != nil { + return written, fmt.Errorf("log buffer: %w", err) + } + + return written, nil +} + +func (l *logBuffer) String() string { + l.mu.Lock() + defer l.mu.Unlock() + + return l.buf.String() +} + +func TestCoordinator_ListFailsThenSucceeds(t *testing.T) { + logs := captureSafeLogs(t) + docker := &fakeDocker{containers: []container.Summary{{ID: "c1"}}, listFailures: 2} + recorder := &recordingApply{} + coord := startCoordinator(t, docker, recorder.apply) + + coord.request(1) + waitFor(t, func() bool { return coord.settled() && len(completions(logs.String())) == 1 }) + + if got := docker.listCalls(); got != 3 { + t.Errorf("list calls = %d, want 3 (two failures, then the pass)", got) + } + + if got := recorder.seen(); !slices.Equal(got, []string{"c1"}) { + t.Errorf("applied = %v, want [c1]", got) + } + + if !strings.Contains(logs.String(), "reconciliation incomplete") { + t.Errorf("expected a WARN while the list failed:\n%s", logs.String()) + } + + if gaugeValue(t, "sda_reload_incomplete") != 0 { + t.Error("sda_reload_incomplete did not return to 0") + } +} + +func TestCoordinator_PendingGaugeReturnsToZero(t *testing.T) { + logs := captureSafeLogs(t) + docker := &fakeDocker{containers: []container.Summary{{ID: "c1"}, {ID: "c2"}}} + recorder := &recordingApply{fail: map[string]int{"c2": 2}} + coord := startCoordinator(t, docker, recorder.apply) + + coord.request(1) + waitFor(t, func() bool { + pending, _ := coord.state() + + return pending == 1 + }) + + if gens := completions(logs.String()); len(gens) != 0 { + t.Fatalf("completion logged with a container pending: %v", gens) + } + + waitFor(t, func() bool { return coord.settled() && len(completions(logs.String())) == 1 }) + + if gaugeValue(t, "sda_reconcile_pending_containers") != 0 { + t.Error("sda_reconcile_pending_containers did not return to 0") + } + + if got := recorder.seen(); !slices.Equal(got, []string{"c1", "c2", "c2", "c2"}) { + t.Errorf("applied = %v, want c2 retried until it succeeds", got) + } +} + +// TestCoordinator_OverlappingPublishesCompleteOnlyNewest: a publication +// arriving while the pass for the previous one runs supersedes it; only +// the newer generation is reported complete. +func TestCoordinator_OverlappingPublishesCompleteOnlyNewest(t *testing.T) { + logs := captureSafeLogs(t) + docker := &fakeDocker{containers: []container.Summary{{ID: "c1"}, {ID: "c2"}}} + recorder := gatedApply() + coord := startCoordinator(t, docker, recorder.apply) + + coord.request(1) + <-recorder.entered + coord.request(2) + close(recorder.gate) + + waitFor(t, func() bool { return coord.settled() && len(completions(logs.String())) > 0 }) + + if gens := completions(logs.String()); !slices.Equal(gens, []string{"2"}) { + t.Errorf("completed generations = %v, want [2]", gens) + } + + // The superseded pass stopped after its current container. + if got := recorder.seen(); !slices.Equal(got, []string{"c1", "c1", "c2"}) { + t.Errorf("applied = %v, want [c1 c1 c2]", got) + } +} + +// TestCoordinator_OlderResultCannotClearGauges: a success that started +// before a newer request neither clears the re-tagged pending entry nor +// completes the reload. +func TestCoordinator_OlderResultCannotClearGauges(t *testing.T) { + logs := captureSafeLogs(t) + coord := newCoordinator(nil, noopApply, sharedRecorder(), DockerCallTimeout) + + coord.request(1) + older := coord.current() + coord.settle("c1", older, errApplyFailed) + coord.request(2) + + coord.settle("c1", older, nil) + coord.finishPass(older) + + if pending, incomplete := coord.state(); pending != 1 || !incomplete { + t.Fatalf( + "pending = %d, incomplete = %v after an older result; want 1, true", + pending, + incomplete, + ) + } + + if gaugeValue(t, "sda_reconcile_pending_containers") != 1 || + gaugeValue(t, "sda_reload_incomplete") != 1 { + t.Error("an older result changed the gauges") + } + + if gens := completions(logs.String()); len(gens) != 0 { + t.Fatalf("an older result logged completion: %v", gens) + } + + newer := coord.current() + coord.settle("c1", newer, nil) + coord.finishPass(newer) + + if !coord.settled() || !slices.Equal(completions(logs.String()), []string{"2"}) { + t.Errorf("newer result did not complete generation 2:\n%s", logs.String()) + } +} + +// TestCoordinator_SameGenerationTriggersRunTrailingPass: two systemd reloads +// under one generation while a pass runs coalesce into one trailing pass. +func TestCoordinator_SameGenerationTriggersRunTrailingPass(t *testing.T) { + docker := &fakeDocker{containers: []container.Summary{{ID: "c1"}}} + recorder := gatedApply() + coord := startCoordinator(t, docker, recorder.apply) + + coord.request(1) + <-recorder.entered + coord.request(1) + coord.request(1) + close(recorder.gate) + + waitFor(t, func() bool { return coord.settled() && len(recorder.seen()) == 2 }) + + // Nothing else is pending, so no further pass may start. + time.Sleep(10 * testBackoff) + + if got := docker.listCalls(); got != 2 { + t.Errorf("list calls = %d, want 2 (the pass and one trailing pass)", got) + } +} + +// TestPublishAndReconcile_VisitsRunningContainers: a SIGHUP publication +// reaches every running container through the coordinator. +func TestPublishAndReconcile_VisitsRunningContainers(t *testing.T) { + store, publisher := config.NewStore(config.Runtime{}) + proc := &processor.Processor{Inspector: &recordingInspector{}, Cfg: store, Publisher: publisher} + docker := &fakeDocker{containers: []container.Summary{{ID: "a"}, {ID: "b"}}} + recorder := &recordingApply{} + coord := startCoordinator(t, docker, recorder.apply) + + proc.SetPassRequester(coord.requestPass) + + gen := proc.PublishAndReconcile(context.Background(), config.Runtime{ + Policy: policy.Global{Mode: policy.ModeOptIn}, + }) + + waitFor(t, func() bool { return coord.settled() && len(recorder.seen()) == 2 }) + + if gen != 2 || coord.current().generation != 2 { + t.Errorf( + "published generation %d, coordinator at %d; want 2", + gen, + coord.current().generation, + ) + } + + if got := recorder.seen(); !slices.Equal(got, []string{"a", "b"}) { + t.Errorf("visited = %v, want [a b]", got) + } +} + +// TestCoordinator_DedupSharedWithConsumer runs passes (which write the +// processed map) while one event consumer reads and consumes it, without a +// reconnect; run under -race. +func TestCoordinator_DedupSharedWithConsumer(t *testing.T) { + docker := &fakeDocker{containers: []container.Summary{{ID: "a"}, {ID: "b"}, {ID: "c"}}} + coord := startCoordinator(t, docker, noopApply) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + msgs, errs := makeChans(0, 0) + consumed := make(chan struct{}) + + go func() { + defer close(consumed) + + backoff := minBackoff + + consumeEvents( + ctx, + msgs, + errs, + coord, + &backoff, + new(int64), + nil, + noopApply, + DockerCallTimeout, + ) + }() + + for round := range 50 { + coord.request(1) + + msgs <- events.Message{ + Actor: events.Actor{ID: []string{"a", "b", "c"}[round%3]}, + TimeNano: time.Now().UnixNano(), + } + } + + waitFor(t, coord.settled) + cancel() + <-consumed +} + +// captureSafeLogs is captureLogs with a sink that tolerates the +// coordinator goroutine writing while the test reads. +func captureSafeLogs(t *testing.T) *logBuffer { + t.Helper() + + var buf logBuffer + + setTestLogger(t, &buf) + + return &buf +} + +// TestCoordinator_GoneContainerLeavesPending: a container Docker no longer +// knows ends its retries instead of keeping the reload incomplete forever. +func TestCoordinator_GoneContainerLeavesPending(t *testing.T) { + coord := newCoordinator(nil, noopApply, sharedRecorder(), DockerCallTimeout) + coord.request(1) + key := coord.current() + + gone := fmt.Errorf("container %q: %w", "c1", processor.ErrContainerGone) + + coord.settle("c1", key, errApplyFailed) + coord.settle("c1", key, gone) + coord.settle("c2", key, gone) + coord.finishPass(key) + + if !coord.settled() { + pending, incomplete := coord.state() + t.Fatalf( + "pending = %d, incomplete = %v; want the gone containers dropped", + pending, + incomplete, + ) + } +} + +// TestCoordinator_DedupAcrossReconnectUnderRace drives the event loop +// through a stream close and a re-subscription while passes run, so the +// consumer and the coordinator touch the processed map concurrently on both +// sides of a reconnect; run under -race. +func TestCoordinator_DedupAcrossReconnectUnderRace(t *testing.T) { + msgs := make(chan events.Message) + docker := &fakeDocker{ + containers: []container.Summary{{ID: "a"}, {ID: "b"}}, + firstMsgs: msgs, + firstErrs: make(chan error), + } + coord := startCoordinator(t, docker, noopApply) + opts := Options{ + Docker: docker, + Proc: &processor.Processor{Inspector: &recordingInspector{}, Cfg: newTestStore()}, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan struct{}) + + go func() { + defer close(done) + + listenEvents(ctx, opts, coord, time.Now(), client.EventsResult{}, func() {}) + }() + + for round := range 20 { + coord.request(1) + + msgs <- events.Message{ + Actor: events.Actor{ID: []string{"a", "b"}[round%2]}, + TimeNano: time.Now().UnixNano(), + } + } + + close(msgs) + + // While the loop backs off and re-subscribes, keep the passes going. + waitForWithin(t, 5*minBackoff, func() bool { + coord.request(1) + + started, _ := docker.eventsCalls() + + return started >= 2 + }) + + waitFor(t, coord.settled) + cancel() + <-done +} diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 5dae5e2..5af2879 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -20,7 +20,7 @@ package daemon import ( "context" - "fmt" + "sync" "time" "github.com/moby/moby/client" @@ -61,9 +61,9 @@ func (o *Options) timeout() time.Duration { return DockerCallTimeout } -// Run subscribes to the Docker event stream, enumerates existing containers, -// subscribes to the systemd reload signal, and then blocks on the event stream -// until ctx is done. +// Run subscribes to the Docker event stream, reconciles every running +// container, subscribes to the systemd reload signal, and then blocks on the +// event stream until ctx is done. // // The event stream is opened before the enumeration, with Since set to the // time just before subscribing, so a container started while the list is @@ -72,7 +72,11 @@ func Run(ctx context.Context, opts Options) error { return run(ctx, opts, startReloadWatcher) } -func run(ctx context.Context, opts Options, startWatcher func(context.Context, Options)) error { +func run( + ctx context.Context, + opts Options, + startWatcher func(context.Context, Options, func()), +) error { log := logger.L() since := time.Now() @@ -87,71 +91,32 @@ func run(ctx context.Context, opts Options, startWatcher func(context.Context, O log.Warn("could not subscribe to docker events; will retry", "err", subErr) } - processed := make(map[string]time.Time) - - processErr := processExistingContainers( - ctx, - opts.Docker, - processed, - opts.Metrics, - processorApply(opts.Proc), - opts.timeout(), - ) - if processErr != nil { - log.Warn("could not enumerate existing containers", "err", processErr) - } - - startWatcher(ctx, opts) - - listenEvents(ctx, opts, processed, since, stream, cancelStream) - - return nil -} - -// processExistingContainers iterates the currently running containers and -// applies device rules to each one that bind-mounts /dev/... paths. For every -// container processed successfully, processed records the time captured just -// before it was inspected, so later events for an earlier run can be skipped. -func processExistingContainers( - ctx context.Context, - cli dockerAPI, - processed map[string]time.Time, - metrics *observability.Recorder, - apply applyFn, - timeout time.Duration, -) error { - log := logger.L() + coord := newCoordinator(opts.Docker, processorApply(opts.Proc), opts.Metrics, opts.timeout()) - listCtx, cancelList := context.WithTimeout(ctx, timeout) - list, err := cli.ContainerList(listCtx, client.ContainerListOptions{}) + // Every later publication goes to the coordinator; installed before + // the startup request, so no generation published from here on is + // missed. + opts.Proc.SetPassRequester(coord.requestPass) - cancelList() + // Waited for before Run returns, so shutdown does not cut off a + // reconcile in the middle of its cgroup mutation. + var coordinator sync.WaitGroup + defer coordinator.Wait() - if err != nil { - return fmt.Errorf("list containers: %w", err) - } + coordinator.Go(func() { coord.run(ctx) }) - containers := list.Items + // The startup pass reconciles every running container under the + // startup config. In live mode that replaces or strips grants a + // previous instance left behind. + coord.request(opts.Proc.Cfg.Generation()) + coord.awaitFirstPass(ctx) - log.Debug("enumerating running containers", "count", len(containers)) + startWatcher(ctx, opts, func() { + opts.Metrics.IncReloadReapply() + coord.request(opts.Proc.Cfg.Generation()) + }) - for idx := range containers { - startedAt := time.Now() - - processErr := processOne( - ctx, - containers[idx].ID, - metrics, - apply, - timeout, - "could not process running container", - ) - if processErr != nil { - continue - } - - processed[containers[idx].ID] = startedAt - } + listenEvents(ctx, opts, coord, since, stream, cancelStream) return nil } @@ -164,10 +129,11 @@ func processorApply(proc *processor.Processor) applyFn { } // startReloadWatcher tries to subscribe to systemd's DBus Reloading signal so -// that when daemon-reload wipes the cgroup BPF programs, we re-apply rules to -// every running container. DBus is optional — on hosts without systemd or -// without the DBus socket mounted, this logs a warning and returns. -func startReloadWatcher(ctx context.Context, opts Options) { +// that when daemon-reload wipes the cgroup BPF programs, trigger requests a +// pass that re-applies rules to every running container. DBus is optional — +// on hosts without systemd or without the DBus socket mounted, this logs a +// warning and returns. +func startReloadWatcher(ctx context.Context, _ Options, trigger func()) { log := logger.L() watcher, err := systemd.Open() @@ -185,23 +151,6 @@ func startReloadWatcher(ctx context.Context, opts Options) { } }() - watcher.Watch(ctx, func() { - opts.Metrics.IncReloadReapply() - - fresh := make(map[string]time.Time) - - processErr := processExistingContainers( - ctx, - opts.Docker, - fresh, - opts.Metrics, - processorApply(opts.Proc), - opts.timeout(), - ) - if processErr != nil { - log.Warn("could not re-apply rules after systemd reload", - "err", processErr) - } - }) + watcher.Watch(ctx, trigger) }() } diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index 801fe65..b075f39 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -23,6 +23,7 @@ import ( "context" "errors" "fmt" + "io" "log/slog" "slices" "strings" @@ -56,6 +57,7 @@ type fakeDocker struct { firstErrs chan error onList func() hangList bool + listFailures int hangEvents map[int]bool eventsReturned int eventsCtx context.Context //nolint:containedctx // records the stream context so tests can assert its lifetime @@ -76,6 +78,18 @@ func (f *fakeDocker) ContainerList( return client.ContainerListResult{}, fmt.Errorf("hung list: %w", ctx.Err()) } + f.mu.Lock() + + failing := f.listFailures > 0 + if failing { + f.listFailures-- + } + f.mu.Unlock() + + if failing { + return client.ContainerListResult{}, errListFailed + } + if f.onList != nil { f.onList() } @@ -192,7 +206,14 @@ func waitForWithin(t *testing.T, limit time.Duration, cond func() bool) { t.Fatal("condition not met before deadline") } -func noopWatcher(context.Context, Options) {} +func noopWatcher(context.Context, Options, func()) {} + +// newTestStore returns a store holding the zero config. +func newTestStore() *config.Store { + store, _ := config.NewStore(config.Runtime{}) + + return store +} // TestRun_SubscribesBeforeEnumerating checks that the event stream is opened // (with Since not after the list time) before containers are listed, and that @@ -214,7 +235,7 @@ func TestRun_SubscribesBeforeEnumerating(t *testing.T) { insp := &recordingInspector{} opts := Options{ Docker: docker, - Proc: &processor.Processor{Inspector: insp, Cfg: config.NewStore()}, + Proc: &processor.Processor{Inspector: insp, Cfg: newTestStore()}, } done := make(chan struct{}) @@ -265,7 +286,7 @@ func TestListenEvents_ReconnectSinceAfterLastEvent(t *testing.T) { insp := &recordingInspector{} opts := Options{ Docker: docker, - Proc: &processor.Processor{Inspector: insp, Cfg: config.NewStore()}, + Proc: &processor.Processor{Inspector: insp, Cfg: newTestStore()}, } done := make(chan struct{}) @@ -276,7 +297,7 @@ func TestListenEvents_ReconnectSinceAfterLastEvent(t *testing.T) { listenEvents( ctx, opts, - map[string]time.Time{}, + testCoordinator(nil), time.Now().Add(-time.Hour), client.EventsResult{Messages: msgs, Err: make(chan error)}, func() {}, @@ -316,7 +337,7 @@ func TestListenEvents_ReconnectBeforeAnyEventUsesInitialSince(t *testing.T) { initial := time.Now().Add(-time.Hour) opts := Options{ Docker: docker, - Proc: &processor.Processor{Inspector: &recordingInspector{}, Cfg: config.NewStore()}, + Proc: &processor.Processor{Inspector: &recordingInspector{}, Cfg: newTestStore()}, } done := make(chan struct{}) @@ -327,7 +348,7 @@ func TestListenEvents_ReconnectBeforeAnyEventUsesInitialSince(t *testing.T) { listenEvents( ctx, opts, - map[string]time.Time{}, + testCoordinator(nil), initial, client.EventsResult{Messages: msgs, Err: make(chan error)}, func() {}, @@ -410,15 +431,23 @@ func readMetrics(t *testing.T) metricsSnapshot { func captureLogs(t *testing.T) *bytes.Buffer { t.Helper() - prev := logger.L() - var buf bytes.Buffer - logger.Set(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))) - t.Cleanup(func() { logger.Set(prev) }) + + setTestLogger(t, &buf) return &buf } +// setTestLogger sends logger.L() to sink until the test ends. +func setTestLogger(t *testing.T, sink io.Writer) { + t.Helper() + + prev := logger.L() + + logger.Set(slog.New(slog.NewTextHandler(sink, &slog.HandlerOptions{Level: slog.LevelDebug}))) + t.Cleanup(func() { logger.Set(prev) }) +} + func assertFailureReported(t *testing.T, before, after metricsSnapshot, logOutput, wantMsg string) { t.Helper() @@ -479,21 +508,13 @@ func TestProcessOne_StartupFailureMatchesEventPath(t *testing.T) { before := readMetrics(t) docker := &fakeDocker{containers: []container.Summary{{ID: "bad"}}} - processed := map[string]time.Time{} + coord := newCoordinator(docker, failingApply, metrics, DockerCallTimeout) - err := processExistingContainers( - context.Background(), - docker, - processed, - metrics, - failingApply, - DockerCallTimeout, - ) - if err != nil { - t.Fatalf("processExistingContainers: %v", err) + if outcome := coord.pass(context.Background(), coord.current()); outcome != passDone { + t.Fatalf("pass outcome = %v, want passDone", outcome) } - if _, recorded := processed["bad"]; recorded { + if _, recorded := coord.processed["bad"]; recorded { t.Error("failed container must not be recorded as processed") } @@ -526,9 +547,8 @@ func TestProcessOne_StartupFailureMatchesEventPath(t *testing.T) { ctx, msgs, errs, - map[string]time.Time{}, + testCoordinator(nil), &backoff, - nil, new(int64), metrics, failingApply, @@ -545,18 +565,10 @@ func TestProcessOne_StartupSuccessRecordsMetrics(t *testing.T) { before := readMetrics(t) docker := &fakeDocker{containers: []container.Summary{{ID: "good"}}} - processed := map[string]time.Time{} + coord := newCoordinator(docker, noopApply, metrics, DockerCallTimeout) - err := processExistingContainers( - context.Background(), - docker, - processed, - metrics, - noopApply, - DockerCallTimeout, - ) - if err != nil { - t.Fatalf("processExistingContainers: %v", err) + if outcome := coord.pass(context.Background(), coord.current()); outcome != passDone { + t.Fatalf("pass outcome = %v, want passDone", outcome) } after := readMetrics(t) @@ -576,7 +588,7 @@ func TestProcessOne_StartupSuccessRecordsMetrics(t *testing.T) { t.Error("last_event_timestamp not set after successful startup apply") } - if _, recorded := processed["good"]; !recorded { + if _, recorded := coord.processed["good"]; !recorded { t.Error("successful container should be recorded as processed") } } diff --git a/internal/daemon/events.go b/internal/daemon/events.go index 9d6cfca..61ac9e5 100644 --- a/internal/daemon/events.go +++ b/internal/daemon/events.go @@ -88,7 +88,7 @@ func resubscribeSince(initial time.Time, lastEventNano int64) string { func listenEvents( ctx context.Context, opts Options, - processed map[string]time.Time, + coord *coordinator, since time.Time, stream client.EventsResult, cancelStream context.CancelFunc, @@ -99,12 +99,6 @@ func listenEvents( var lastEventNano int64 - // The processed map guards the overlap window between startup enumeration - // and the live event stream. After 2×maxBackoff (60s) the window has - // certainly passed; any remaining entries are from containers that exited - // before producing a start event and will never be drained normally. - clearProcessed := time.After(2 * maxBackoff) - for { if ctx.Err() != nil { cancelStream() @@ -144,9 +138,8 @@ func listenEvents( ctx, stream.Messages, stream.Err, - processed, + coord, &backoff, - clearProcessed, &lastEventNano, opts.Metrics, processorApply(opts.Proc), @@ -224,18 +217,16 @@ func subscribe( // caller should reconnect, false on context cancellation. lastEventNano is // updated with the timestamp of every received event. // -// An event is skipped only when processed holds an entry for the container -// and the event is not newer than that entry: Docker emits "start" after the -// container is running, so an earlier event was already visible to the -// startup inspect. A newer event (restart, unpause) has a new cgroup and is -// applied. The entry is removed on the first event for that ID either way. +// An event already covered by an enumeration is skipped (see +// coordinator.skipEvent); a newer one (restart, unpause) has a new cgroup and +// is applied. A failed apply leaves the container to the coordinator's +// retries. func consumeEvents( ctx context.Context, msgs <-chan events.Message, errs <-chan error, - processed map[string]time.Time, + coord *coordinator, backoff *time.Duration, - clearProcessed <-chan time.Time, lastEventNano *int64, metrics *observability.Recorder, apply applyFn, @@ -248,16 +239,6 @@ func consumeEvents( case <-ctx.Done(): return false - case <-clearProcessed: - // Overlap window expired; discard any startup entries that were - // never matched by a live event (containers that exited during the - // window). Reassign to a nil channel so the case never fires again. - for key := range processed { - delete(processed, key) - } - - clearProcessed = nil - case streamErr := <-errs: if streamErr == nil { continue @@ -304,16 +285,12 @@ func consumeEvents( metrics.RecordEvent(string(msg.Action)) - recordedAt, alreadyProcessed := processed[msg.Actor.ID] - if alreadyProcessed { - delete(processed, msg.Actor.ID) - - if !time.Unix(0, msg.TimeNano).After(recordedAt) { - continue - } + if coord.skipEvent(msg.Actor.ID, time.Unix(0, msg.TimeNano)) { + continue } - _ = processOne( + key := coord.current() + applyErr := processOne( ctx, msg.Actor.ID, metrics, @@ -321,6 +298,7 @@ func consumeEvents( timeout, "could not process container", ) + coord.eventApplied(msg.Actor.ID, key, applyErr) } } } diff --git a/internal/daemon/events_test.go b/internal/daemon/events_test.go index de84806..e55b7c4 100644 --- a/internal/daemon/events_test.go +++ b/internal/daemon/events_test.go @@ -98,9 +98,8 @@ func TestConsumeEvents_ContextCancelledReturnsNoReconnect(t *testing.T) { ctx, msgs, errs, - map[string]time.Time{}, + testCoordinator(nil), &backoff, - nil, new(int64), nil, noopApply, @@ -124,9 +123,8 @@ func TestConsumeEvents_StreamErrorReturnsReconnect(t *testing.T) { ctx, msgs, errs, - map[string]time.Time{}, + testCoordinator(nil), &backoff, - nil, new(int64), nil, noopApply, @@ -154,9 +152,8 @@ func TestConsumeEvents_ContextErrFromStreamErrorNoReconnect(t *testing.T) { ctx, msgs, errs, - map[string]time.Time{}, + testCoordinator(nil), &backoff, - nil, new(int64), nil, noopApply, @@ -195,9 +192,8 @@ func TestConsumeEvents_ArbitraryStreamErrorAfterCancelNoReconnect(t *testing.T) ctx, msgs, errs, - map[string]time.Time{}, + testCoordinator(nil), &backoff, - nil, new(int64), nil, noopApply, @@ -248,9 +244,8 @@ func TestConsumeEvents_ChannelCloseReturnsReconnect(t *testing.T) { ctx, msgs, errs, - map[string]time.Time{}, + testCoordinator(nil), &backoff, - nil, new(int64), nil, noopApply, @@ -290,9 +285,8 @@ func TestConsumeEvents_EventCallsApply(t *testing.T) { ctx, msgs, errs, - map[string]time.Time{}, + testCoordinator(nil), &backoff, - nil, new(int64), nil, apply, @@ -333,9 +327,8 @@ func TestConsumeEvents_DeduplicatesProcessedIDs(t *testing.T) { ctx, msgs, errs, - processed, + testCoordinator(processed), &backoff, - nil, new(int64), nil, apply, @@ -351,41 +344,25 @@ func TestConsumeEvents_DeduplicatesProcessedIDs(t *testing.T) { } } -func TestConsumeEvents_ClearProcessedOnTTL(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - msgs, errs := makeChans(0, 0) - backoff := minBackoff - processed := map[string]time.Time{ - "stale-a": time.Now(), - "stale-b": time.Now(), +// TestCoordinator_ProcessedEntriesExpire checks that an enumeration entry +// older than processedTTL neither suppresses an event nor survives a prune. +func TestCoordinator_ProcessedEntriesExpire(t *testing.T) { + inspectedAt := time.Now() + coord := testCoordinator(map[string]time.Time{ + "stale-a": inspectedAt, + "stale-b": inspectedAt, + "fresh": inspectedAt.Add(processedTTL), + }) + coord.now = func() time.Time { return inspectedAt.Add(processedTTL + time.Second) } + + if coord.skipEvent("stale-a", inspectedAt.Add(-time.Second)) { + t.Error("an expired entry must not suppress an event") } - // Fire the TTL immediately via a closed channel. - clearCh := make(chan time.Time) - close(clearCh) - - go func() { - time.Sleep(20 * time.Millisecond) - cancel() - }() - - consumeEvents( - ctx, - msgs, - errs, - processed, - &backoff, - clearCh, - new(int64), - nil, - noopApply, - DockerCallTimeout, - ) + coord.pruneProcessed() - if len(processed) != 0 { - t.Errorf("processed map has %d entries after TTL, want 0", len(processed)) + if _, ok := coord.processed["stale-b"]; ok || len(coord.processed) != 1 { + t.Errorf("processed = %v after prune, want only the fresh entry", coord.processed) } } @@ -407,9 +384,8 @@ func TestConsumeEvents_BackoffResetsOnSuccessfulEvent(t *testing.T) { ctx, msgs, errs, - map[string]time.Time{}, + testCoordinator(nil), &backoff, - nil, new(int64), nil, noopApply, @@ -465,9 +441,8 @@ func TestConsumeEvents_RestartWithinWindow(t *testing.T) { ctx, msgs, errs, - processed, + testCoordinator(processed), &backoff, - nil, new(int64), nil, apply, @@ -507,9 +482,8 @@ func TestConsumeEvents_TracksLastEventNano(t *testing.T) { ctx, msgs, errs, - map[string]time.Time{}, + testCoordinator(nil), &backoff, - nil, &lastEventNano, nil, noopApply, @@ -546,3 +520,14 @@ func TestResubscribeSince(t *testing.T) { t.Errorf("resubscribeSince = %v, want %v", parsed, last.Add(time.Nanosecond)) } } + +// testCoordinator returns a coordinator for consumer tests, with processed +// as its enumeration entries when non-nil. +func testCoordinator(processed map[string]time.Time) *coordinator { + coord := newCoordinator(nil, noopApply, nil, DockerCallTimeout) + if processed != nil { + coord.processed = processed + } + + return coord +} diff --git a/internal/daemon/timeout_test.go b/internal/daemon/timeout_test.go index cd065cf..0f023d1 100644 --- a/internal/daemon/timeout_test.go +++ b/internal/daemon/timeout_test.go @@ -30,7 +30,6 @@ import ( "github.com/moby/moby/api/types/container" "github.com/moby/moby/client" - "github.com/leinardi/swarm-device-access/internal/config" "github.com/leinardi/swarm-device-access/internal/processor" ) @@ -89,30 +88,24 @@ func assertBounded(t *testing.T, name string, fn func()) { } } -func TestProcessExistingContainers_HungContainerListIsBounded(t *testing.T) { +func TestCoordinatorPass_HungContainerListIsBounded(t *testing.T) { docker := &fakeDocker{hangList: true} + coord := newCoordinator(docker, noopApply, nil, testCallTimeout) - var err error + var outcome passOutcome - assertBounded(t, "processExistingContainers", func() { - err = processExistingContainers( - context.Background(), - docker, - map[string]time.Time{}, - nil, - noopApply, - testCallTimeout, - ) + assertBounded(t, "coordinator pass", func() { + outcome = coord.pass(context.Background(), coord.current()) }) - if !errors.Is(err, context.DeadlineExceeded) { - t.Fatalf("err = %v, want context.DeadlineExceeded", err) + if outcome != passListFailed { + t.Fatalf("outcome = %v, want passListFailed", outcome) } } func TestProcessOne_HungContainerInspectIsBounded(t *testing.T) { insp := &hangingInspector{} - proc := &processor.Processor{Inspector: insp, Cfg: config.NewStore()} + proc := &processor.Processor{Inspector: insp, Cfg: newTestStore()} var err error @@ -139,7 +132,7 @@ func TestReconcile_HungInspectIsBoundedPerCall(t *testing.T) { insp := &hangingInspector{} proc := &processor.Processor{ Inspector: insp, - Cfg: config.NewStore(), + Cfg: newTestStore(), CallTimeout: testCallTimeout, } @@ -216,7 +209,7 @@ func TestRun_InitialEventsHangStillEnumeratesAndResubscribes(t *testing.T) { insp := &recordingInspector{} opts := Options{ Docker: docker, - Proc: &processor.Processor{Inspector: insp, Cfg: config.NewStore()}, + Proc: &processor.Processor{Inspector: insp, Cfg: newTestStore()}, callTimeout: testCallTimeout, } @@ -260,7 +253,7 @@ func TestListenEvents_ReconnectEventsHangRetries(t *testing.T) { docker := &fakeDocker{hangEvents: map[int]bool{1: true}} opts := Options{ Docker: docker, - Proc: &processor.Processor{Inspector: &recordingInspector{}, Cfg: config.NewStore()}, + Proc: &processor.Processor{Inspector: &recordingInspector{}, Cfg: newTestStore()}, callTimeout: testCallTimeout, } @@ -274,7 +267,7 @@ func TestListenEvents_ReconnectEventsHangRetries(t *testing.T) { listenEvents( ctx, opts, - map[string]time.Time{}, + testCoordinator(nil), time.Now(), client.EventsResult{}, func() {}, diff --git a/internal/observability/metrics.go b/internal/observability/metrics.go index 486e6a0..86817ba 100644 --- a/internal/observability/metrics.go +++ b/internal/observability/metrics.go @@ -39,6 +39,8 @@ type Recorder struct { ruleFailures prometheus.Counter dryRunSkips prometheus.Counter lastEventTimestamp prometheus.Gauge + pendingContainers prometheus.Gauge + reloadIncomplete prometheus.Gauge } // NewRecorder registers all metric collectors against Prometheus' default @@ -101,6 +103,16 @@ func NewRecorder() *Recorder { Name: "sda_last_event_timestamp_seconds", Help: "Unix timestamp of the last container processed successfully (event, startup or reload).", }), + + pendingContainers: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "sda_reconcile_pending_containers", + Help: "Containers whose last reconciliation failed and that are being retried with backoff.", + }), + + reloadIncomplete: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "sda_reload_incomplete", + Help: "1 until a reconciliation pass for the latest config and trigger has visited every running container, else 0.", + }), } } @@ -207,3 +219,27 @@ func (rec *Recorder) SetLastEvent(eventTime time.Time) { rec.lastEventTimestamp.Set(float64(eventTime.Unix())) } + +// SetPendingContainers records how many containers await a reconcile retry. +func (rec *Recorder) SetPendingContainers(count int) { + if rec == nil { + return + } + + rec.pendingContainers.Set(float64(count)) +} + +// SetReloadIncomplete records whether the latest requested reconciliation +// pass has yet to visit every running container. +func (rec *Recorder) SetReloadIncomplete(incomplete bool) { + if rec == nil { + return + } + + value := 0.0 + if incomplete { + value = 1 + } + + rec.reloadIncomplete.Set(value) +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 636db8a..97fd747 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -57,8 +57,9 @@ type deviceRuleKey struct { } // Processor applies cgroup BPF device-allow rules to containers that -// bind-mount /dev/... paths. Inspector and Cfg are required; Metrics may be -// nil (calls become no-ops). HostRoot is the container-internal path to the +// bind-mount /dev/... paths. Inspector and Cfg are required; Publisher is +// required for PublishAndReconcile; Metrics may be nil (calls become +// no-ops). HostRoot is the container-internal path to the // host root (typically "/host"). ProcRoot is used for /proc lookups ("/" in // production, temp dir in tests). CallTimeout bounds each Docker call the // processor makes, independent of the caller's context, so every entry point @@ -70,6 +71,7 @@ type deviceRuleKey struct { type Processor struct { Inspector DockerInspector Cfg *config.Store + Publisher *config.Publisher Metrics *observability.Recorder HostRoot string ProcRoot string @@ -98,6 +100,10 @@ type Processor struct { // process is gone or Docker cannot be asked: see revokeKnown. known map[lifecycleKey]knownCgroup + // requestPass receives each generation PublishAndReconcile publishes + // (guarded by mu). + requestPass PassRequester + // pinner pins container processes; nil means pidfds. Tests replace it. pinner processPinner diff --git a/internal/processor/processor_test.go b/internal/processor/processor_test.go index 3571664..a599ed9 100644 --- a/internal/processor/processor_test.go +++ b/internal/processor/processor_test.go @@ -119,10 +119,13 @@ func buildProcRoot( var errDaemonUnavail = errors.New("daemon unavailable") func newStore(mode policy.Mode, dryRun bool) *config.Store { - s := config.NewStore() - s.Set(config.Runtime{Policy: policy.Global{Mode: mode}, DryRun: dryRun}) + store, _ := newPublishedStore(mode, dryRun) - return s + return store +} + +func newPublishedStore(mode policy.Mode, dryRun bool) (*config.Store, *config.Publisher) { + return config.NewStore(config.Runtime{Policy: policy.Global{Mode: mode}, DryRun: dryRun}) } func TestIsDeviceMountSource(t *testing.T) { @@ -779,8 +782,7 @@ func TestReconcile_DevDirectorySummary(t *testing.T) { }, }} - store := config.NewStore() - store.Set(config.Runtime{ + store, _ := config.NewStore(config.Runtime{ Policy: policy.Global{Mode: policy.ModeAll, DeviceAllow: []string{"/dev/null"}}, DryRun: true, }) diff --git a/internal/processor/publish.go b/internal/processor/publish.go new file mode 100644 index 0000000..70f7aba --- /dev/null +++ b/internal/processor/publish.go @@ -0,0 +1,61 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processor + +import ( + "context" + + "github.com/leinardi/swarm-device-access/internal/config" +) + +// PassRequester asks for a reconciliation pass over every running container +// under config generation (or a newer one). It must not block on the pass. +type PassRequester func(ctx context.Context, generation uint64) + +// SetPassRequester installs the function PublishAndReconcile hands each new +// generation to (the daemon's reconciliation coordinator). +func (p *Processor) SetPassRequester(request PassRequester) { + p.mu.Lock() + defer p.mu.Unlock() + + p.requestPass = request +} + +// PublishAndReconcile publishes rt as the new config and requests a pass +// that reconciles every running container under it. It is the only way the +// config changes after startup. +// +// The publication takes the same lock as Reconcile, so no container is +// between loading the config and mutating its cgroup when the generation +// changes: a worker that computed under the old config finishes first, and +// the pass then overwrites its result. The pass runs after the lock is +// released and takes it per container, so it cannot deadlock against this +// call. +func (p *Processor) PublishAndReconcile(ctx context.Context, rt config.Runtime) uint64 { + p.mu.Lock() + generation := p.Publisher.Publish(rt) + request := p.requestPass + p.mu.Unlock() + + if request != nil { + request(ctx, generation) + } + + return generation +} diff --git a/internal/processor/reconcile.go b/internal/processor/reconcile.go index 308433d..42f9557 100644 --- a/internal/processor/reconcile.go +++ b/internal/processor/reconcile.go @@ -24,12 +24,17 @@ import ( "fmt" "io/fs" + cerrdefs "github.com/containerd/errdefs" "github.com/moby/moby/api/types/container" "github.com/leinardi/swarm-device-access/internal/cgroup" "github.com/leinardi/swarm-device-access/internal/logger" ) +// ErrContainerGone reports that Docker no longer knows the container and no +// revoke is outstanding for it, so there is nothing left to retry. +var ErrContainerGone = errors.New("container no longer exists") + // applyPinned sets rules on the cgroup of the container's running process. // It returns the cgroup path once resolved, for error reporting. // @@ -159,26 +164,31 @@ func (p *Processor) verifyPinned( // revokeAfterInspectFailure handles a container Docker cannot report on. // Without a pid there is no cgroup to resolve, so every lifecycle of the -// container with a known cgroup is revoked there, and the container stays -// pending either way. Dry-run never mutates. +// container with a known cgroup is revoked there. The container stays +// pending, unless Docker reported it does not exist and every revoke +// succeeded: the error then wraps ErrContainerGone. Dry-run never mutates, +// so for it a container Docker does not know is simply gone. func (p *Processor) revokeAfterInspectFailure( containerID string, dryRun bool, inspectErr error, ) error { - errs := []error{inspectErr} - - if dryRun { - return inspectErr - } + var errs []error for key, known := range p.known { - if key.containerID == containerID { - errs = append(errs, p.revokeAt(key, known, true)) + if key.containerID == containerID && !dryRun { + revokeErr := p.revokeAt(key, known, true) + if revokeErr != nil { + errs = append(errs, revokeErr) + } } } - return errors.Join(errs...) + if len(errs) == 0 && cerrdefs.IsNotFound(inspectErr) { + return fmt.Errorf("%w: %w", ErrContainerGone, inspectErr) + } + + return errors.Join(append([]error{inspectErr}, errs...)...) } // revokeKnown applies the empty set to the cgroup key was last verified in, diff --git a/internal/processor/reconcile_test.go b/internal/processor/reconcile_test.go index 5aacfce..f4bde1c 100644 --- a/internal/processor/reconcile_test.go +++ b/internal/processor/reconcile_test.go @@ -21,6 +21,7 @@ package processor import ( "context" "errors" + "fmt" "os" "path/filepath" "strconv" @@ -28,6 +29,7 @@ import ( "testing" "time" + cerrdefs "github.com/containerd/errdefs" "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/mount" "github.com/moby/moby/client" @@ -563,3 +565,63 @@ func TestReconcile_InspectFailureOfPrivilegedDoesNotWarn(t *testing.T) { t.Errorf("privileged container must not get the filter_missing warning:\n%s", buf.String()) } } + +var errRevokeFailed = errors.New("revoke failed") + +// TestReconcile_NotFoundIsContainerGone: a container Docker does not know +// is reported gone, so it is not retried forever, but only once nothing is +// left to revoke for it. +func TestReconcile_NotFoundIsContainerGone(t *testing.T) { + notFound := inspectReply{err: fmt.Errorf("no such container: %w", cerrdefs.ErrNotFound)} + + t.Run("unknown", func(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.insp.then(notFound) + + err := env.reconcile() + if !errors.Is(err, ErrContainerGone) || env.fake.calls != 0 { + t.Fatalf( + "err = %v, calls = %d; want ErrContainerGone and no mutation", + err, + env.fake.calls, + ) + } + }) + + t.Run("dry-run", func(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, true) + env.insp.then(notFound) + + err := env.reconcile() + if !errors.Is(err, ErrContainerGone) { + t.Fatalf("err = %v, want ErrContainerGone", err) + } + }) + + t.Run("known and revoked", func(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.grantOnce(t) + env.insp.then(notFound) + + err := env.reconcile() + if !errors.Is(err, ErrContainerGone) || env.fake.calls != 2 || len(env.fake.rules[1]) != 0 { + t.Fatalf( + "err = %v, rules = %v; want the revoke, then ErrContainerGone", + err, + env.fake.rules, + ) + } + }) + + t.Run("known and revoke failed", func(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.grantOnce(t) + env.insp.then(notFound) + env.fake.err = errRevokeFailed + + err := env.reconcile() + if errors.Is(err, ErrContainerGone) || !errors.Is(err, errRevokeFailed) { + t.Fatalf("err = %v, want the revoke failure and the container kept pending", err) + } + }) +} diff --git a/internal/processor/serialize_test.go b/internal/processor/serialize_test.go index fc032d5..25d20f5 100644 --- a/internal/processor/serialize_test.go +++ b/internal/processor/serialize_test.go @@ -60,7 +60,7 @@ func TestReconcile_SerializesConfigLoadThroughApply(t *testing.T) { insp := &countingInspector{ result: container.InspectResponse{State: &container.State{Running: true, Pid: pid}}, } - store := newStore(policy.ModeAll, true) + store, publisher := newPublishedStore(policy.ModeAll, true) proc := &Processor{ Inspector: insp, Cfg: store, @@ -107,7 +107,10 @@ func TestReconcile_SerializesConfigLoadThroughApply(t *testing.T) { time.Sleep(time.Millisecond) } - store.Set(config.Runtime{Policy: policy.Global{Mode: policy.ModeOptIn}, DryRun: true}) + // Published directly, not through PublishAndReconcile: this test is + // about the lock ordering between workers, and the publication must + // land while the first worker holds the lock. + publisher.Publish(config.Runtime{Policy: policy.Global{Mode: policy.ModeOptIn}, DryRun: true}) // Negative assertion: nothing can be polled for "did not happen", so give // the second worker a bounded window to (wrongly) get past the lock. @@ -126,3 +129,75 @@ func TestReconcile_SerializesConfigLoadThroughApply(t *testing.T) { t.Fatalf("modes seen by computing workers = %v, want [all opt-in]", modes) } } + +// TestPublishAndReconcile_StalePolicyRace pauses a worker after it computed +// a grant under the old config, publishes a narrower config through +// PublishAndReconcile, then releases the worker. The publication waits for +// the worker, and the pass it requests runs afterwards, so the cgroup ends +// with the narrower (empty) set rather than the stale grant. +func TestPublishAndReconcile_StalePolicyRace(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + + store, publisher := newPublishedStore(policy.ModeAll, false) + env.proc.Cfg = store + env.proc.Publisher = publisher + + var ( + held = make(chan struct{}) + release = make(chan struct{}) + once sync.Once + ) + + env.proc.afterCompute = func() { + once.Do(func() { + close(held) + <-release + }) + } + + // The pass the coordinator would run: reconcile the one container. + passErrs := make(chan error, 1) + + env.proc.SetPassRequester(func(ctx context.Context, _ uint64) { + passErrs <- env.proc.Reconcile(ctx, "abc") + }) + + var workers sync.WaitGroup + + workers.Go(func() { _ = env.reconcile() }) + + <-held + + published := make(chan uint64, 1) + + workers.Go(func() { + published <- env.proc.PublishAndReconcile( + context.Background(), + config.Runtime{Policy: policy.Global{Mode: policy.ModeOptIn}}, + ) + }) + + // Negative assertion: the publication must wait for the worker. + select { + case <-published: + t.Fatal("PublishAndReconcile published while a worker held the lock") + case <-time.After(50 * time.Millisecond): + } + + close(release) + workers.Wait() + + if gen := <-published; gen != 2 { + t.Errorf("generation = %d, want 2", gen) + } + + passErr := <-passErrs + if passErr != nil { + t.Fatalf("pass: %v", passErr) + } + + last := env.fake.rules[len(env.fake.rules)-1] + if env.fake.calls != 2 || len(env.fake.rules[0]) != 1 || len(last) != 0 { + t.Fatalf("rules = %v, want the stale grant replaced by the empty set", env.fake.rules) + } +} diff --git a/test/integration/enforce_test.go b/test/integration/enforce_test.go index ca3d2dd..ffe3c36 100644 --- a/test/integration/enforce_test.go +++ b/test/integration/enforce_test.go @@ -53,8 +53,9 @@ const ( // mounted under /var/run inside the container. containerDBusSocket = "/var/run/dbus/system_bus_socket" - msgWatcherStarted = "systemd reload watcher started" - msgEPERM = "Operation not permitted" + msgWatcherStarted = "systemd reload watcher started" + msgDryRunLeftovers = "dry-run: grants left by a previous live run cannot be detected or cleaned in dry-run" + msgEPERM = "Operation not permitted" // wipeTimeout bounds how long daemon-reload may take to remove the // device program after systemctl returns. @@ -136,6 +137,78 @@ func TestEnforce_GrantsDeviceAccess(t *testing.T) { }) } +// TestEnforce_RestartReconcilesLeftoverGrants kills a daemon that granted a +// device, so it cannot clean up, then restarts it. A dry-run restart must +// leave the grant alone (dry-run never touches cgroups) and say so; a live +// restart under a policy that denies the device must revoke it. +// +// It attaches BPF programs on this host, so it only runs with +// SDA_IT_ENFORCE=1. +func TestEnforce_RestartReconcilesLeftoverGrants(t *testing.T) { + if !enforceRequested() { + t.Skipf("set %s=1 to run: it attaches BPF programs on this host", envEnforce) + } + + cli := requireDocker(t) + ensureTestImage(t, cli) + + ctx := testCtx(t) + image := requireDaemonImage(ctx, t, cli) + + requireEnforceHost(t) + + device := pickDevice(ctx, t, cli) + target := startTestContainer( + ctx, + t, + cli, + map[string]string{policy.LabelEnable: "true"}, + []string{device + ":" + device}, + ) + + firstID, first := startDaemonContainer(ctx, t, cli, image) + requireProcessed(ctx, t, first, target, 1, 0) + requireAllowed(ctx, t, cli, target, device, "after the first daemon granted it") + killDaemonContainer(ctx, t, cli, firstID, first) + + dryID, dry := startDaemonContainer(ctx, t, cli, image, "-dry-run") + dry.wait(ctx, t, "the dry-run leftover warning", withMsg(msgDryRunLeftovers)) + requireProcessed(ctx, t, dry, target, 1, 0) + requireAllowed(ctx, t, cli, target, device, "after a dry-run restart (not a cleanup path)") + killDaemonContainer(ctx, t, cli, dryID, dry) + + _, narrow := startDaemonContainer(ctx, t, cli, image, "-device-deny="+device) + requireProcessed(ctx, t, narrow, target, 0, 1) + requireDenied(ctx, t, cli, target, device, "after a live restart with a policy that denies it") +} + +// killDaemonContainer kills the daemon with SIGKILL, so it cannot clean up, +// and waits until its log stream ends. +func killDaemonContainer( + ctx context.Context, + t *testing.T, + cli *dockerclient.Client, + daemonID string, + proc *daemonProc, +) { + t.Helper() + + _, err := cli.ContainerKill(ctx, daemonID, dockerclient.ContainerKillOptions{Signal: "KILL"}) + if err != nil { + t.Fatalf("kill daemon container %s: %v", shortID(daemonID), err) + } + + select { + case <-proc.exited: + case <-ctx.Done(): + t.Fatalf( + "daemon container %s log stream did not end: %v", + shortID(daemonID), + context.Cause(ctx), + ) + } +} + // testReloadReapply first proves that daemon-reload wipes the device program // on this host, then that the daemon puts it back. Without the wipe check a // host that never wipes would pass the re-apply assertion without the daemon @@ -341,20 +414,24 @@ func pickDevice(ctx context.Context, t *testing.T, cli *dockerclient.Client) str // ---- daemon container ---- // startDaemonContainer runs the daemon image with the runtime options of -// .mk/docker-run.mk, waits until it subscribes to Docker events and returns -// its ID and log collector. It is removed when the test ends. +// .mk/docker-run.mk and extraArgs, waits until it subscribes to Docker events +// and returns its ID and log collector. It is removed when the test ends. func startDaemonContainer( ctx context.Context, t *testing.T, cli *dockerclient.Client, image string, + extraArgs ...string, ) (string, *daemonProc) { t.Helper() daemonID := startContainer(ctx, t, cli, &container.Config{ Image: image, - Cmd: []string{"-policy-mode=opt-in", "-log-format=json", "-log-level=debug"}, + Cmd: append( + []string{"-policy-mode=opt-in", "-log-format=json", "-log-level=debug"}, + extraArgs..., + ), }, &container.HostConfig{ Privileged: true, From 0d1a6a38f6d791a22c042e4652cb7b4b7883c21f Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 11:04:16 +0200 Subject: [PATCH 10/37] feat(daemon): track container lifecycles and release them only when verified gone The coordinator now keeps a per-container history of runs (processor.Lifecycles, installed into the processor, which records into it): provisional when an inspect failed, verified once the run's start time and cgroup identity were verified against the pinned process, terminal once the run ended. A restart under the same ID adds a record, so late cleanup for an old run never reaches the new one. The processor's per-lifecycle identity map is replaced by these records. die and destroy are added to the event subscription. The coordinator adjudicates every event: the consumer only submits it. A die or destroy resolves to the verified run with the latest start not after the event time (else the provisional records), whose cgroup gets the empty set; the record stays until its cgroup is verified gone. Before handling a container the coordinator reserves it, so work requested meanwhile (an event, a pass, a cleanup) coalesces into one follow-up; a start event not newer than a running reconcile is covered by it, while a cleanup covers none. Pending retries skip reserved containers, and completion waits for running reconciles. Records, with the cgroup v1 ledger entry and the cached cgroup v2 filters of their cgroup, are released only when the cgroup path no longer exists or its inode changed; an inspect not-found never releases one. A periodic sweep re-checks every record, retries revokes of ended runs that failed, and drops records that never named a cgroup once ended or stale. --- README.md | 11 +- docs/architecture.md | 4 +- internal/daemon/coordinator.go | 287 ++++++++++++++++++++--- internal/daemon/coordinator_test.go | 219 +++++++++++++++++- internal/daemon/daemon.go | 3 + internal/daemon/daemon_test.go | 4 +- internal/daemon/events.go | 28 +-- internal/daemon/events_test.go | 56 ++--- internal/processor/lifecycle.go | 333 +++++++++++++++++++++++++++ internal/processor/lifecycle_test.go | 268 +++++++++++++++++++++ internal/processor/processor.go | 30 +-- internal/processor/reconcile.go | 233 +++++++++++++++---- internal/processor/reconcile_test.go | 44 +++- 13 files changed, 1345 insertions(+), 175 deletions(-) create mode 100644 internal/processor/lifecycle.go create mode 100644 internal/processor/lifecycle_test.go diff --git a/README.md b/README.md index 054f481..f9b23e2 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,9 @@ At runtime the daemon: - Processes already-running containers at startup, so daemon restarts do not leave existing tasks without device access. -- Subscribes to Docker `start` and `unpause` events, with reconnect and backoff - if the Docker event stream drops. +- Subscribes to Docker `start` and `unpause` events (to grant) and `die` and + `destroy` events (to clean up after a run), with reconnect and backoff if the + Docker event stream drops. - Inspects each eligible container for bind mounts whose source is under `/dev`. - Walks directory mounts such as `/dev/bus/usb` and applies one rule per device @@ -146,6 +147,12 @@ Once a container has exited, or when Docker cannot be asked about it, its grants are revoked in the cgroup the daemon last verified for it, and only if that directory is still the same one (same inode) and was not recreated. +The daemon keeps a short history of each container's runs. A `die` or +`destroy` event revokes the grants of the run it ended (the latest run that +started before the event, so a late event for an earlier run never touches a +container restarted under the same ID). What the daemon remembers about a run +is dropped only once its cgroup is verified gone; a periodic sweep checks. + ### Docker Compose for Swarm ```yaml diff --git a/docs/architecture.md b/docs/architecture.md index da4473c..a4f8b31 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -14,7 +14,7 @@ The daemon watches Docker for container-start events and, for each new container ``` Docker daemon | - | container start / unpause events (Unix socket) + | container start / unpause / die / destroy events (Unix socket) v +---------------------------------------------------+ | cmd/swarm-device-access/main.go | @@ -219,7 +219,7 @@ Metrics exposed: | Metric | Type | Labels | Description | | --- | --- | --- | --- | -| `sda_events_total` | counter | `event` | Docker events received (`start`, `unpause`) | +| `sda_events_total` | counter | `event` | Docker events received (`start`, `unpause`, `die`, `destroy`) | | `sda_rules_applied_total` | counter | `result` | Containers handled: `ok` = no container-level failure (includes policy, label and no-pid skips and partial per-device failures); `error` = container-level failure | | `sda_reload_reapplies_total` | counter | — | Re-applies after systemd daemon-reload | | `sda_docker_reconnects_total` | counter | — | Docker event stream reconnects | diff --git a/internal/daemon/coordinator.go b/internal/daemon/coordinator.go index 0e7d64e..86df734 100644 --- a/internal/daemon/coordinator.go +++ b/internal/daemon/coordinator.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/moby/moby/api/types/events" "github.com/moby/moby/client" "github.com/leinardi/swarm-device-access/internal/logger" @@ -39,6 +40,10 @@ import ( // event was consumed. const processedTTL = 2 * maxBackoff +// sweepInterval is how often the lifecycle history is re-checked for runs +// whose cgroup is gone. +const sweepInterval = time.Minute + // passKey is what a pass reconciles for: a config generation and a trigger // epoch. Every request bumps the epoch, so two systemd reloads under one // generation are still two passes (the second can wipe what the first @@ -74,14 +79,29 @@ const ( // finishes its current container, then starts again under the newest // request, so no result is reported on behalf of an older one. // -// It also owns the processed map the event consumer uses to skip start -// events already covered by an enumeration. +// It also adjudicates and applies container events: the event consumer +// only submits them (handleEvent). Every reconcile or cleanup of a +// container first reserves it, so work requested for a container that is +// already being handled coalesces into one follow-up instead of running +// twice; a start event not newer than the running apply is already covered +// by it. The coordinator keeps the lifecycle history of container runs +// (shared with the processor, which records into it) and sweeps it +// periodically, and owns the processed map that skips start events an +// enumeration already covered. type coordinator struct { docker dockerAPI apply applyFn metrics *observability.Recorder timeout time.Duration + // terminate and sweep are Processor.Terminate and Processor.Sweep; + // nil in tests that do not exercise them. + terminate func(containerID string, eventTime time.Time) error + sweep func() int + // lifecycles is the history of container runs, installed into the + // processor by Run. + lifecycles *processor.Lifecycles + minBackoff time.Duration maxBackoff time.Duration now func() time.Time @@ -104,6 +124,24 @@ type coordinator struct { pending map[string]*pendingEntry processed map[string]time.Time firstDone bool + // inFlight holds a reservation for every container being handled. + inFlight map[string]*reservation +} + +// reservation marks a container as being handled and collects the work +// requested for it meanwhile. +type reservation struct { + // applyStart is when the current reconcile began; a start event not + // newer than it is covered by that reconcile. Zero while the holder + // only cleans up, which covers no start event. + applyStart time.Time + // running is set while the holder's work includes a reconcile, whose + // outcome completion must wait for. + running bool + // reconcile asks for another reconcile after the current work. + reconcile bool + // terminateAt is the latest end-of-run event received meanwhile. + terminateAt time.Time } func newCoordinator( @@ -124,6 +162,8 @@ func newCoordinator( passEnded: make(chan struct{}), pending: make(map[string]*pendingEntry), processed: make(map[string]time.Time), + inFlight: make(map[string]*reservation), + lifecycles: &processor.Lifecycles{}, } } @@ -173,6 +213,9 @@ func (c *coordinator) run(ctx context.Context) { prune := time.NewTicker(processedTTL) defer prune.Stop() + sweep := time.NewTicker(sweepInterval) + defer sweep.Stop() + for { if ctx.Err() != nil { return @@ -210,6 +253,8 @@ func (c *coordinator) run(ctx context.Context) { c.retryDue(ctx) case <-prune.C: c.pruneProcessed() + case <-sweep.C: + c.runSweep() } } } @@ -263,16 +308,12 @@ func (c *coordinator) pass(ctx context.Context, key passKey) passOutcome { return passSuperseded } - containerID := list.Items[idx].ID - inspectedAt := c.now() - - applyErr := processOne(ctx, containerID, c.metrics, c.apply, c.timeout, - "could not process running container") - c.settle(containerID, key, applyErr) - - if applyErr == nil { - c.markProcessed(containerID, inspectedAt) - } + _ = c.reconcileContainer( + ctx, + list.Items[idx].ID, + time.Time{}, + "could not process running container", + ) } c.finishPass(key) @@ -287,10 +328,12 @@ func (c *coordinator) retryDue(ctx context.Context) { return } - key := c.current() - applyErr := processOne(ctx, containerID, c.metrics, c.apply, c.timeout, - "could not reconcile pending container") - c.settle(containerID, key, applyErr) + _ = c.reconcileContainer( + ctx, + containerID, + time.Time{}, + "could not reconcile pending container", + ) } c.mu.Lock() @@ -298,17 +341,182 @@ func (c *coordinator) retryDue(ctx context.Context) { c.mu.Unlock() } -// eventApplied records the outcome of an event-driven reconcile that -// started under key: a failure makes the container pending. -func (c *coordinator) eventApplied(containerID string, key passKey, applyErr error) { - c.settle(containerID, key, applyErr) +// handleEvent adjudicates one container event from the stream. start and +// unpause reconcile the container unless an enumeration already covered +// the event; die and destroy end a run and clean up after it. +func (c *coordinator) handleEvent(ctx context.Context, msg *events.Message) { + eventTime := time.Unix(0, msg.TimeNano) + containerID := msg.Actor.ID - if applyErr != nil { - // Wake run so the new entry's retry is scheduled. - select { - case c.wake <- struct{}{}: - default: + switch msg.Action { //nolint:exhaustive // the stream is filtered to these four actions + case events.ActionDie, events.ActionDestroy: + c.terminateContainer(ctx, containerID, eventTime) + default: + if c.skipEvent(containerID, eventTime) { + return + } + + applyErr := c.reconcileContainer(ctx, containerID, eventTime, "could not process container") + if applyErr != nil { + // Wake run so the new pending entry's retry is scheduled. + select { + case c.wake <- struct{}{}: + default: + } + } + } +} + +// reconcileContainer reconciles one container unless it is already being +// handled, in which case the work is left to the holder (see reserve). It +// returns the error of the last reconcile it ran. +func (c *coordinator) reconcileContainer( + ctx context.Context, + containerID string, + eventTime time.Time, + logMsg string, +) error { + if !c.reserve(containerID, eventTime, false) { + return nil + } + + // Only an enumeration (no event) marks the container processed: its + // inspect is what covers start events that are delivered later. + return c.hold(ctx, containerID, true, time.Time{}, eventTime.IsZero(), logMsg) +} + +// terminateContainer cleans up after a run that ended at eventTime. +func (c *coordinator) terminateContainer( + ctx context.Context, + containerID string, + eventTime time.Time, +) { + if !c.reserve(containerID, eventTime, true) { + return + } + + _ = c.hold(ctx, containerID, false, eventTime, false, "could not process container") +} + +// hold does the work for a reserved container, then whatever was requested +// for it meanwhile, until nothing is left; it then releases the +// reservation. A cleanup runs before a reconcile, so the reconcile sees +// the final state. +func (c *coordinator) hold( + ctx context.Context, + containerID string, + reconcile bool, + terminateAt time.Time, + enumerated bool, + logMsg string, +) error { + var lastErr error + + for { + if !terminateAt.IsZero() && c.terminate != nil { + termErr := c.terminate(containerID, terminateAt) + if termErr != nil { + logger.L(). + Warn("could not clean up after container", "id", containerID, "err", termErr) + } + } + + if reconcile { + key := c.current() + started := c.now() + + lastErr = processOne(ctx, containerID, c.metrics, c.apply, c.timeout, logMsg) + c.settle(containerID, key, lastErr) + + if lastErr == nil && enumerated { + c.markProcessed(containerID, started) + } + } + + reconcile, terminateAt = c.next(containerID) + if !reconcile && terminateAt.IsZero() { + return lastErr + } + } +} + +// reserve claims containerID for the caller and reports whether it got +// it. If the container is already being handled, the request is left to +// the holder: an end-of-run event is queued; a reconcile is queued unless +// it is a start event not newer than the running apply, which that apply +// already covers. +func (c *coordinator) reserve(containerID string, eventTime time.Time, terminal bool) bool { + c.mu.Lock() + defer c.mu.Unlock() + + held, busy := c.inFlight[containerID] + if !busy { + fresh := &reservation{running: !terminal} + if !terminal { + fresh.applyStart = c.now() } + + c.inFlight[containerID] = fresh + + return true + } + + switch { + case terminal: + if eventTime.After(held.terminateAt) { + held.terminateAt = eventTime + } + case eventTime.IsZero() || held.applyStart.IsZero() || eventTime.After(held.applyStart): + held.reconcile = true + } + + return false +} + +// next returns the work queued for a reserved container, or releases the +// reservation when there is none. +func (c *coordinator) next(containerID string) (bool, time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + + held := c.inFlight[containerID] + if !held.reconcile && held.terminateAt.IsZero() { + delete(c.inFlight, containerID) + + // Its retries were held back while the container was reserved. + if _, isPending := c.pending[containerID]; isPending { + select { + case c.wake <- struct{}{}: + default: + } + } + + c.announceLocked() + + return false, time.Time{} + } + + reconcile, terminateAt := held.reconcile, held.terminateAt + held.reconcile, held.terminateAt = false, time.Time{} + held.running = reconcile + held.applyStart = time.Time{} + + if reconcile { + held.applyStart = c.now() + } + + return reconcile, terminateAt +} + +// runSweep re-checks the lifecycle history (Processor.Sweep). +func (c *coordinator) runSweep() { + if c.sweep == nil { + return + } + + released := c.sweep() + if released > 0 { + logger.L().Debug("lifecycle sweep released records", "released", released) } } @@ -370,12 +578,25 @@ func (c *coordinator) reportLocked() { return } - if c.announced == c.latest { + c.announceLocked() +} + +// announceLocked logs completion once per request, when nothing is left +// for the latest request: no pass outstanding, nothing pending, and no +// follow-up queued behind a reservation. +func (c *coordinator) announceLocked() { + if c.incomplete || len(c.pending) > 0 || c.announced == c.latest { return } + for _, held := range c.inFlight { + if held.reconcile || held.running { + return + } + } + c.announced = c.latest - log.Info("config reload complete", + logger.L().Info("config reload complete", "generation", c.latest.generation, "epoch", c.latest.epoch) } @@ -409,7 +630,13 @@ func (c *coordinator) nextDue() (time.Time, bool) { found bool ) - for _, entry := range c.pending { + for containerID, entry := range c.pending { + // A reserved container is retried by its holder's settle, or + // once the reservation ends (next wakes run). + if _, reserved := c.inFlight[containerID]; reserved { + continue + } + if !found || entry.due.Before(next) { next, found = entry.due, true } @@ -427,6 +654,10 @@ func (c *coordinator) dueIDs() []string { var ids []string for containerID, entry := range c.pending { + if _, reserved := c.inFlight[containerID]; reserved { + continue + } + if !entry.due.After(now) { ids = append(ids, containerID) } diff --git a/internal/daemon/coordinator_test.go b/internal/daemon/coordinator_test.go index d118c7e..51ee01a 100644 --- a/internal/daemon/coordinator_test.go +++ b/internal/daemon/coordinator_test.go @@ -404,8 +404,6 @@ func TestCoordinator_DedupSharedWithConsumer(t *testing.T) { &backoff, new(int64), nil, - noopApply, - DockerCallTimeout, ) }() @@ -511,3 +509,220 @@ func TestCoordinator_DedupAcrossReconnectUnderRace(t *testing.T) { cancel() <-done } + +func startEvent(containerID string, at time.Time) *events.Message { + return &events.Message{ + Action: events.ActionStart, + Actor: events.Actor{ID: containerID}, + TimeNano: at.UnixNano(), + } +} + +// TestCoordinator_EventDuringEnumerationAppliesOnce: a start event for the +// container an enumeration is applying, and not newer than that apply, +// coalesces into it (exactly one apply); a newer event (a restart) queues +// exactly one follow-up. +func TestCoordinator_EventDuringEnumerationAppliesOnce(t *testing.T) { + for _, tc := range []struct { + name string + delay time.Duration + want int + }{ + {"event covered by the apply", -time.Second, 1}, + {"newer event", time.Hour, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + docker := &fakeDocker{containers: []container.Summary{{ID: "c1"}}} + recorder := gatedApply() + coord := startCoordinator(t, docker, recorder.apply) + + coord.request(1) + <-recorder.entered + + coord.handleEvent(context.Background(), startEvent("c1", time.Now().Add(tc.delay))) + close(recorder.gate) + + waitFor(t, func() bool { return coord.settled() && coord.idle() }) + + if got := len(recorder.seen()); got != tc.want { + t.Errorf("applies = %d, want %d", got, tc.want) + } + }) + } +} + +// TestCoordinator_DieEventTerminates: die and destroy are adjudicated by +// the coordinator; one arriving while the container is being reconciled +// runs after that reconcile, not alongside it. +func TestCoordinator_DieEventTerminates(t *testing.T) { + docker := &fakeDocker{containers: []container.Summary{{ID: "c1"}}} + recorder := gatedApply() + coord := startCoordinator(t, docker, recorder.apply) + + var ( + mu sync.Mutex + terminated []string + ) + + coord.terminate = func(containerID string, _ time.Time) error { + mu.Lock() + defer mu.Unlock() + + terminated = append(terminated, containerID) + + return nil + } + + coord.request(1) + <-recorder.entered + + dieAt := time.Now() + coord.handleEvent(context.Background(), &events.Message{ + Action: events.ActionDie, Actor: events.Actor{ID: "c1"}, TimeNano: dieAt.UnixNano(), + }) + + mu.Lock() + early := len(terminated) + mu.Unlock() + + if early != 0 { + t.Fatal("die was handled while the container was still being reconciled") + } + + close(recorder.gate) + waitFor(t, coord.idle) + + coord.handleEvent(context.Background(), &events.Message{ + Action: events.ActionDestroy, + Actor: events.Actor{ID: "c2"}, + TimeNano: time.Now().UnixNano(), + }) + + mu.Lock() + defer mu.Unlock() + + if !slices.Equal(terminated, []string{"c1", "c2"}) { + t.Errorf("terminated = %v, want the queued die for c1, then the destroy for c2", terminated) + } +} + +// idle reports whether no container is being handled. +func (c *coordinator) idle() bool { + c.mu.Lock() + defer c.mu.Unlock() + + return len(c.inFlight) == 0 +} + +func warnCount(logOutput string) int { + return strings.Count(logOutput, `msg="reconciliation incomplete; retrying"`) +} + +// TestCoordinator_ReservedPendingDoesNotSpin: a pending container that +// another holder is handling is not retried (nor its due time polled) +// until the reservation ends; then it is retried once. +func TestCoordinator_ReservedPendingDoesNotSpin(t *testing.T) { + logs := captureSafeLogs(t) + recorder := &recordingApply{} + coord := startCoordinator(t, &fakeDocker{}, recorder.apply) + + if !coord.reserve("c1", time.Time{}, false) { + t.Fatal("could not reserve c1") + } + + coord.settle("c1", coord.current(), errApplyFailed) + + coord.wake <- struct{}{} + + time.Sleep(20 * testBackoff) + + if got := warnCount(logs.String()); got > 1 || len(recorder.seen()) != 0 { + t.Fatalf( + "while reserved: %d warnings, applies %v; want no retry and no spinning", + got, + recorder.seen(), + ) + } + + if again, _ := coord.next("c1"); again { + t.Fatal("a skipped retry must not queue work on the holder") + } + + waitFor(t, coord.settled) + + if got := recorder.seen(); !slices.Equal(got, []string{"c1"}) { + t.Errorf("applies = %v, want one retry after the reservation ended", got) + } +} + +// TestCoordinator_StartDuringCleanupQueuesReconcile: a cleanup covers no +// start event, so a start (a restart) arriving while a die is being +// handled is reconciled afterwards, whatever its timestamp. +func TestCoordinator_StartDuringCleanupQueuesReconcile(t *testing.T) { + recorder := &recordingApply{} + coord := startCoordinator(t, &fakeDocker{}, recorder.apply) + + entered, gate := make(chan struct{}), make(chan struct{}) + coord.terminate = func(string, time.Time) error { + close(entered) + <-gate + + return nil + } + + done := make(chan struct{}) + + go func() { + defer close(done) + + coord.handleEvent(context.Background(), &events.Message{ + Action: events.ActionDie, + Actor: events.Actor{ID: "c1"}, + TimeNano: time.Now().UnixNano(), + }) + }() + + <-entered + coord.handleEvent(context.Background(), startEvent("c1", time.Now().Add(-time.Second))) + close(gate) + <-done + + if got := recorder.seen(); !slices.Equal(got, []string{"c1"}) { + t.Errorf("applies = %v, want the start reconciled after the cleanup", got) + } +} + +// TestCoordinator_NoCompletionWhileReconcileRunning: a pass that finishes +// while another reconcile is still running must not report completion +// until that reconcile has settled. +func TestCoordinator_NoCompletionWhileReconcileRunning(t *testing.T) { + logs := captureSafeLogs(t) + recorder := gatedApply() + coord := startCoordinator(t, &fakeDocker{}, recorder.apply) + + done := make(chan struct{}) + + go func() { + defer close(done) + + coord.handleEvent(context.Background(), startEvent("c1", time.Now())) + }() + + <-recorder.entered + coord.request(1) + + waitFor(t, func() bool { + _, incomplete := coord.state() + + return !incomplete + }) + + if gens := completions(logs.String()); len(gens) != 0 { + t.Fatalf("completion logged while a reconcile was running: %v", gens) + } + + close(recorder.gate) + <-done + + waitFor(t, func() bool { return len(completions(logs.String())) == 1 }) +} diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 5af2879..79c622d 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -92,6 +92,9 @@ func run( } coord := newCoordinator(opts.Docker, processorApply(opts.Proc), opts.Metrics, opts.timeout()) + coord.terminate = opts.Proc.Terminate + coord.sweep = opts.Proc.Sweep + opts.Proc.SetLifecycles(coord.lifecycles) // Every later publication goes to the coordinator; installed before // the startup request, so no generation published from here on is diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index b075f39..884a599 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -547,12 +547,10 @@ func TestProcessOne_StartupFailureMatchesEventPath(t *testing.T) { ctx, msgs, errs, - testCoordinator(nil), + testCoordinatorWith(failingApply, metrics, nil), &backoff, new(int64), metrics, - failingApply, - DockerCallTimeout, ) assertFailureReported(t, before, readMetrics(t), buf.String(), "could not process container") diff --git a/internal/daemon/events.go b/internal/daemon/events.go index 61ac9e5..0eb26bb 100644 --- a/internal/daemon/events.go +++ b/internal/daemon/events.go @@ -44,17 +44,18 @@ const ( var errSubscribeTimeout = errors.New("docker event stream not established before timeout") -// applyFn is the per-container rule-application callback injected into consumeEvents. -// In production this wraps Processor.Reconcile; in tests it is replaced by a fake. +// applyFn is the per-container reconcile the coordinator runs. In +// production it wraps Processor.Reconcile; in tests it is replaced by a fake. type applyFn func(ctx context.Context, id string) error // eventListOptions returns the Docker event subscription options: "start" -// and "unpause" container events at or after since. +// and "unpause" container events, which reconcile a container, and "die" +// and "destroy", which end a run and clean up after it, at or after since. func eventListOptions(since string) client.EventsListOptions { return client.EventsListOptions{ Since: since, // make, not the zero value: a nil client.Filters panics on Add. - Filters: make(client.Filters).Add("event", "start", "unpause"), + Filters: make(client.Filters).Add("event", "start", "unpause", "die", "destroy"), } } @@ -142,8 +143,6 @@ func listenEvents( &backoff, &lastEventNano, opts.Metrics, - processorApply(opts.Proc), - timeout, ) // The stream context lives exactly as long as this subscription: a @@ -229,8 +228,6 @@ func consumeEvents( backoff *time.Duration, lastEventNano *int64, metrics *observability.Recorder, - apply applyFn, - timeout time.Duration, ) bool { log := logger.L() @@ -285,20 +282,7 @@ func consumeEvents( metrics.RecordEvent(string(msg.Action)) - if coord.skipEvent(msg.Actor.ID, time.Unix(0, msg.TimeNano)) { - continue - } - - key := coord.current() - applyErr := processOne( - ctx, - msg.Actor.ID, - metrics, - apply, - timeout, - "could not process container", - ) - coord.eventApplied(msg.Actor.ID, key, applyErr) + coord.handleEvent(ctx, &msg) } } } diff --git a/internal/daemon/events_test.go b/internal/daemon/events_test.go index e55b7c4..864b87d 100644 --- a/internal/daemon/events_test.go +++ b/internal/daemon/events_test.go @@ -29,6 +29,8 @@ import ( "github.com/moby/moby/api/types/events" "github.com/moby/moby/client" + + "github.com/leinardi/swarm-device-access/internal/observability" ) var errTransportEOF = errors.New("transport EOF") @@ -98,12 +100,10 @@ func TestConsumeEvents_ContextCancelledReturnsNoReconnect(t *testing.T) { ctx, msgs, errs, - testCoordinator(nil), + testCoordinatorWith(noopApply, nil, nil), &backoff, new(int64), nil, - noopApply, - DockerCallTimeout, ) if got { t.Error( @@ -123,12 +123,10 @@ func TestConsumeEvents_StreamErrorReturnsReconnect(t *testing.T) { ctx, msgs, errs, - testCoordinator(nil), + testCoordinatorWith(noopApply, nil, nil), &backoff, new(int64), nil, - noopApply, - DockerCallTimeout, ) if !got { t.Error("consumeEvents should return true (reconnect) on stream error") @@ -152,12 +150,10 @@ func TestConsumeEvents_ContextErrFromStreamErrorNoReconnect(t *testing.T) { ctx, msgs, errs, - testCoordinator(nil), + testCoordinatorWith(noopApply, nil, nil), &backoff, new(int64), nil, - noopApply, - DockerCallTimeout, ) if got { t.Error("consumeEvents should return false when stream error is context.Canceled") @@ -192,12 +188,10 @@ func TestConsumeEvents_ArbitraryStreamErrorAfterCancelNoReconnect(t *testing.T) ctx, msgs, errs, - testCoordinator(nil), + testCoordinatorWith(noopApply, nil, nil), &backoff, new(int64), nil, - noopApply, - DockerCallTimeout, ) if got { t.Error("consumeEvents should return false when the context is already canceled") @@ -227,7 +221,7 @@ func TestEventListOptions(t *testing.T) { t.Errorf("Since = %q, want %q", got.Since, since) } - want := client.Filters{"event": {"start": true, "unpause": true}} + want := client.Filters{"event": {"start": true, "unpause": true, "die": true, "destroy": true}} if !maps.EqualFunc(got.Filters, want, maps.Equal) { t.Errorf("Filters = %v, want %v", got.Filters, want) } @@ -244,12 +238,10 @@ func TestConsumeEvents_ChannelCloseReturnsReconnect(t *testing.T) { ctx, msgs, errs, - testCoordinator(nil), + testCoordinatorWith(noopApply, nil, nil), &backoff, new(int64), nil, - noopApply, - DockerCallTimeout, ) if !got { t.Error("consumeEvents should return true (reconnect) on channel close") @@ -285,12 +277,10 @@ func TestConsumeEvents_EventCallsApply(t *testing.T) { ctx, msgs, errs, - testCoordinator(nil), + testCoordinatorWith(apply, nil, nil), &backoff, new(int64), nil, - apply, - DockerCallTimeout, ) if called.Load() != 2 { @@ -327,12 +317,10 @@ func TestConsumeEvents_DeduplicatesProcessedIDs(t *testing.T) { ctx, msgs, errs, - testCoordinator(processed), + testCoordinatorWith(apply, nil, processed), &backoff, new(int64), nil, - apply, - DockerCallTimeout, ) if called.Load() != 0 { @@ -384,12 +372,10 @@ func TestConsumeEvents_BackoffResetsOnSuccessfulEvent(t *testing.T) { ctx, msgs, errs, - testCoordinator(nil), + testCoordinatorWith(noopApply, nil, nil), &backoff, new(int64), nil, - noopApply, - DockerCallTimeout, ) if backoff != minBackoff { @@ -441,12 +427,10 @@ func TestConsumeEvents_RestartWithinWindow(t *testing.T) { ctx, msgs, errs, - testCoordinator(processed), + testCoordinatorWith(apply, nil, processed), &backoff, new(int64), nil, - apply, - DockerCallTimeout, ) if called.Load() != tc.wantApply { @@ -482,12 +466,10 @@ func TestConsumeEvents_TracksLastEventNano(t *testing.T) { ctx, msgs, errs, - testCoordinator(nil), + testCoordinatorWith(noopApply, nil, nil), &backoff, &lastEventNano, nil, - noopApply, - DockerCallTimeout, ) if lastEventNano != 300 { @@ -524,7 +506,17 @@ func TestResubscribeSince(t *testing.T) { // testCoordinator returns a coordinator for consumer tests, with processed // as its enumeration entries when non-nil. func testCoordinator(processed map[string]time.Time) *coordinator { - coord := newCoordinator(nil, noopApply, nil, DockerCallTimeout) + return testCoordinatorWith(noopApply, nil, processed) +} + +// testCoordinatorWith is testCoordinator with the reconcile the events +// trigger and the recorder it reports to. +func testCoordinatorWith( + apply applyFn, + metrics *observability.Recorder, + processed map[string]time.Time, +) *coordinator { + coord := newCoordinator(nil, apply, metrics, DockerCallTimeout) if processed != nil { coord.processed = processed } diff --git a/internal/processor/lifecycle.go b/internal/processor/lifecycle.go new file mode 100644 index 0000000..1cf6500 --- /dev/null +++ b/internal/processor/lifecycle.go @@ -0,0 +1,333 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processor + +import ( + "slices" + "sync" + "time" + + "github.com/leinardi/swarm-device-access/internal/cgroup" +) + +// LifecycleState is how much is known about one run of a container. +type LifecycleState int + +const ( + // LifecycleProvisional: an inspect failed; only the ID and the time of + // the observation are known. + LifecycleProvisional LifecycleState = iota + // LifecycleVerified: the start time and the cgroup identity were + // verified against the pinned process. + LifecycleVerified + // LifecycleTerminal: the run ended (die, destroy, or seen not running). + LifecycleTerminal +) + +func (s LifecycleState) String() string { + switch s { + case LifecycleProvisional: + return "provisional" + case LifecycleVerified: + return "verified" + case LifecycleTerminal: + return "terminal" + default: + return "unknown" + } +} + +// maxRecordsPerContainer bounds one container's history; the oldest +// records without an identity go first. +const maxRecordsPerContainer = 8 + +// LifecycleRecord is one run of a container. +type LifecycleRecord struct { + ContainerID string + // StartedAt is Docker's State.StartedAt, empty while unknown. + StartedAt string + // Identity is the cgroup the run was verified in; zero while unknown. + Identity cgroup.Identity + Version int + Privileged bool + State LifecycleState + FirstSeen time.Time + // Observed is the last time an inspect failure refreshed the record. + Observed time.Time + // Revoked is set once the empty set was applied at Identity after the + // run ended; the record is kept until the cgroup is verified gone. + Revoked bool +} + +// HasIdentity reports whether the record names a cgroup. +func (r *LifecycleRecord) HasIdentity() bool { + return r.Identity.Path != "" +} + +// started parses StartedAt; ok is false when it is unknown. +func (r *LifecycleRecord) started() (time.Time, bool) { + if r.StartedAt == "" { + return time.Time{}, false + } + + parsed, err := time.Parse(time.RFC3339Nano, r.StartedAt) + if err != nil { + return time.Time{}, false + } + + return parsed, true +} + +// Lifecycles is the per-container history of runs. A restart under the same +// ID adds a record rather than replacing one, so cleanup that arrives late +// for an old run never reaches the new one. Records are released only once +// their cgroup is verified gone; an inspect that cannot find the container +// never removes one. The zero value is ready to use. +type Lifecycles struct { + mu sync.Mutex + byID map[string][]*LifecycleRecord + now func() time.Time +} + +// Records returns copies of the container's records, oldest first. +func (l *Lifecycles) Records(containerID string) []LifecycleRecord { + l.mu.Lock() + defer l.mu.Unlock() + + out := make([]LifecycleRecord, 0, len(l.byID[containerID])) + for _, rec := range l.byID[containerID] { + out = append(out, *rec) + } + + return out +} + +func (l *Lifecycles) clock() time.Time { + if l.now == nil { + return time.Now() + } + + return l.now() +} + +// all returns copies of every record. +func (l *Lifecycles) all() []LifecycleRecord { + l.mu.Lock() + defer l.mu.Unlock() + + var out []LifecycleRecord + + for _, recs := range l.byID { + for _, rec := range recs { + out = append(out, *rec) + } + } + + return out +} + +// provisional creates or refreshes the provisional record of a container an +// inspect could not report on. +func (l *Lifecycles) provisional(containerID string) { + l.mu.Lock() + defer l.mu.Unlock() + + now := l.clock() + + for _, rec := range l.byID[containerID] { + if rec.State == LifecycleProvisional { + rec.Observed = now + + return + } + } + + l.appendLocked(&LifecycleRecord{ + ContainerID: containerID, + State: LifecycleProvisional, + FirstSeen: now, + Observed: now, + }) +} + +// verify records the cgroup a run was verified in. It upgrades the record +// of the same run, else a provisional record, else adds a record. +func (l *Lifecycles) verify(verified *LifecycleRecord) { + l.mu.Lock() + defer l.mu.Unlock() + + recs := l.byID[verified.ContainerID] + + idx := slices.IndexFunc(recs, func(rec *LifecycleRecord) bool { + return rec.StartedAt == verified.StartedAt + }) + if idx < 0 { + idx = slices.IndexFunc(recs, func(rec *LifecycleRecord) bool { + return rec.State == LifecycleProvisional + }) + } + + if idx < 0 { + added := *verified + added.State = LifecycleVerified + added.FirstSeen = l.clock() + l.appendLocked(&added) + + return + } + + rec := recs[idx] + rec.StartedAt = verified.StartedAt + rec.Identity = verified.Identity + rec.Version = verified.Version + rec.Privileged = verified.Privileged + rec.Revoked = false + + if rec.State != LifecycleTerminal { + rec.State = LifecycleVerified + } +} + +// lookup returns the record of one run. +func (l *Lifecycles) lookup(containerID, startedAt string) (LifecycleRecord, bool) { + l.mu.Lock() + defer l.mu.Unlock() + + for _, rec := range l.byID[containerID] { + if rec.StartedAt == startedAt { + return *rec, true + } + } + + return LifecycleRecord{}, false +} + +// terminate resolves an event that ended a run at eventTime (die, +// destroy). It marks terminal the verified record with the latest start +// time not after eventTime and returns it; with no such record, every +// provisional record becomes terminal and is returned instead. +func (l *Lifecycles) terminate(containerID string, eventTime time.Time) []LifecycleRecord { + l.mu.Lock() + defer l.mu.Unlock() + + var ( + match *LifecycleRecord + matchStart time.Time + provisional []*LifecycleRecord + ) + + for _, rec := range l.byID[containerID] { + if rec.State == LifecycleProvisional { + provisional = append(provisional, rec) + + continue + } + + started, ok := rec.started() + if !ok || !rec.HasIdentity() || started.After(eventTime) { + continue + } + + if match == nil || started.After(matchStart) { + match, matchStart = rec, started + } + } + + if match != nil { + match.State = LifecycleTerminal + + return []LifecycleRecord{*match} + } + + out := make([]LifecycleRecord, 0, len(provisional)) + for _, rec := range provisional { + rec.State = LifecycleTerminal + out = append(out, *rec) + } + + return out +} + +// markTerminal marks one run ended. +func (l *Lifecycles) markTerminal(containerID, startedAt string) { + l.update(containerID, startedAt, func(rec *LifecycleRecord) { rec.State = LifecycleTerminal }) +} + +// markRevoked records that the empty set was applied for an ended run. +func (l *Lifecycles) markRevoked(containerID, startedAt string) { + l.update(containerID, startedAt, func(rec *LifecycleRecord) { + if rec.State == LifecycleTerminal { + rec.Revoked = true + } + }) +} + +func (l *Lifecycles) update(containerID, startedAt string, change func(*LifecycleRecord)) { + l.mu.Lock() + defer l.mu.Unlock() + + for _, rec := range l.byID[containerID] { + if rec.StartedAt == startedAt { + change(rec) + } + } +} + +// release removes the record released is a copy of. Records are told apart +// by start time and creation time: records that never learned their start +// time all share an empty StartedAt. +func (l *Lifecycles) release(released *LifecycleRecord) { + l.mu.Lock() + defer l.mu.Unlock() + + containerID := released.ContainerID + + recs := slices.DeleteFunc(l.byID[containerID], func(rec *LifecycleRecord) bool { + return rec.StartedAt == released.StartedAt && rec.FirstSeen.Equal(released.FirstSeen) + }) + if len(recs) == 0 { + delete(l.byID, containerID) + + return + } + + l.byID[containerID] = recs +} + +func (l *Lifecycles) appendLocked(rec *LifecycleRecord) { + if l.byID == nil { + l.byID = make(map[string][]*LifecycleRecord) + } + + l.byID[rec.ContainerID] = append(l.byID[rec.ContainerID], rec) + recs := l.byID[rec.ContainerID] + + // Over the bound, drop the oldest records that hold no identity: they + // name no cgroup, so nothing can be left to revoke through them. + for len(recs) > maxRecordsPerContainer { + idx := slices.IndexFunc(recs, func(r *LifecycleRecord) bool { return !r.HasIdentity() }) + if idx < 0 { + break + } + + recs = slices.Delete(recs, idx, idx+1) + } + + l.byID[rec.ContainerID] = recs +} diff --git a/internal/processor/lifecycle_test.go b/internal/processor/lifecycle_test.go new file mode 100644 index 0000000..ad6ebc9 --- /dev/null +++ b/internal/processor/lifecycle_test.go @@ -0,0 +1,268 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processor + +import ( + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/leinardi/swarm-device-access/internal/policy" +) + +const restartedAt = "2026-09-27T10:05:00.000000000Z" + +func mustTime(t *testing.T, value string) time.Time { + t.Helper() + + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + t.Fatal(err) + } + + return parsed +} + +// TestLifecycle_InspectFailureThenSuccessUpgradesRecord: an inspect +// failure leaves a provisional record; the next successful reconcile +// upgrades that same record to verified with the run's cgroup. +func TestLifecycle_InspectFailureThenSuccessUpgradesRecord(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.insp.then(inspectReply{err: errDaemonUnavail}) + + _ = env.reconcile() + + recs := env.proc.lifecycleStore().Records("abc") + if len(recs) != 1 || recs[0].State != LifecycleProvisional || recs[0].HasIdentity() { + t.Fatalf("records = %+v, want one provisional record without a cgroup", recs) + } + + env.insp.then(runningDevNull(reconcileStarted)) + + err := env.reconcile() + if err != nil { + t.Fatal(err) + } + + recs = env.proc.lifecycleStore().Records("abc") + if len(recs) != 1 || recs[0].State != LifecycleVerified || + recs[0].StartedAt != reconcileStarted || recs[0].Identity != env.fake.identity { + t.Fatalf("records = %+v, want the provisional record upgraded to verified", recs) + } +} + +// TestTerminate_StaleDieAfterRestartLeavesNewRun: the container restarted +// under the same ID into a new cgroup; a die event for the old run, +// delivered late, revokes the old run's cgroup only. +func TestTerminate_StaleDieAfterRestartLeavesNewRun(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.grantOnce(t) + + oldIdentity := env.fake.identity + + // The restart: a new cgroup directory and start time. + newDir := filepath.Join(env.proc.HostRoot, "sys", "fs", "cgroup", "docker", "restarted") + + err := os.MkdirAll(newDir, 0o755) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile( + filepath.Join(newDir, "cgroup.procs"), + []byte(strconv.Itoa(reconcilePid)+"\n"), + 0o600, + ) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile( + filepath.Join(env.proc.ProcRoot, "proc", strconv.Itoa(reconcilePid), "cgroup"), + []byte("0::/docker/restarted\n"), + 0o600, + ) + if err != nil { + t.Fatal(err) + } + + env.insp.then(runningDevNull(restartedAt)) + + err = env.reconcile() + if err != nil { + t.Fatal(err) + } + + newIdentity := env.fake.identity + + err = env.proc.Terminate("abc", mustTime(t, restartedAt).Add(-time.Minute)) + if err != nil { + t.Fatal(err) + } + + if env.fake.calls != 3 || len(env.fake.rules[2]) != 0 || env.fake.identity != oldIdentity { + t.Fatalf("calls = %d, last identity = %+v; want one empty set on the old run's cgroup %+v", + env.fake.calls, env.fake.identity, oldIdentity) + } + + newRun, ok := env.proc.lifecycleStore().lookup("abc", restartedAt) + if !ok || newRun.State != LifecycleVerified || newRun.Identity != newIdentity { + t.Errorf("new run = %+v, want it verified and untouched", newRun) + } +} + +// TestTerminate_DieRevokesAndRetainsUntilGone: a die event while the +// cgroup still exists revokes there and keeps the record; the sweep +// releases it only once the cgroup is gone. +func TestTerminate_DieRevokesAndRetainsUntilGone(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.grantOnce(t) + + err := env.proc.Terminate("abc", mustTime(t, reconcileStarted).Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + + if env.fake.calls != 2 || len(env.fake.rules[1]) != 0 { + t.Fatalf("rules = %v, want the empty set applied", env.fake.rules) + } + + recs := verifiedRecords(env.proc) + if len(recs) != 1 || recs[0].State != LifecycleTerminal || !recs[0].Revoked { + t.Fatalf("records = %+v, want the run kept as terminal and revoked", recs) + } + + if released := env.proc.Sweep(); released != 0 || len(verifiedRecords(env.proc)) != 1 { + t.Fatalf("sweep released %d with the cgroup present, want 0", released) + } + + err = os.RemoveAll(testCgroupDir(env.proc.HostRoot)) + if err != nil { + t.Fatal(err) + } + + if released := env.proc.Sweep(); released != 1 || len(verifiedRecords(env.proc)) != 0 { + t.Fatalf("sweep released %d after the cgroup was removed, want 1", released) + } + + if env.fake.calls != 2 { + t.Errorf("sweep mutated: calls = %d, want 2", env.fake.calls) + } +} + +// TestTerminate_ProvisionalOnly: without a verified run the provisional +// record becomes terminal, nothing is mutated, and the sweep drops it. +func TestTerminate_ProvisionalOnly(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.insp.then(inspectReply{err: errDaemonUnavail}) + + _ = env.reconcile() + + err := env.proc.Terminate("abc", time.Now()) + if err != nil { + t.Fatal(err) + } + + recs := env.proc.lifecycleStore().Records("abc") + if len(recs) != 1 || recs[0].State != LifecycleTerminal || env.fake.calls != 0 { + t.Fatalf("records = %+v, calls = %d; want the provisional record terminal and no mutation", + recs, env.fake.calls) + } + + if released := env.proc.Sweep(); released != 1 || + len(env.proc.lifecycleStore().Records("abc")) != 0 { + t.Fatalf("sweep released %d, want the terminal provisional record dropped", released) + } +} + +// TestSweep_DropsDeadEntries: the sweep releases runs whose cgroup is gone +// or recreated and stale provisional records, and keeps live runs. +func TestSweep_DropsDeadEntries(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.grantOnce(t) + + store := env.proc.lifecycleStore() + now := time.Now() + store.now = func() time.Time { return now } + + store.provisional("gone") + + if released := env.proc.Sweep(); released != 0 { + t.Fatalf("sweep released %d live or fresh records, want 0", released) + } + + store.now = func() time.Time { return now.Add(provisionalTTL + time.Second) } + + if released := env.proc.Sweep(); released != 1 || len(store.Records("gone")) != 0 { + t.Fatalf("sweep released %d, want the stale provisional record dropped", released) + } + + // Recreated: same path, new inode. + dir := testCgroupDir(env.proc.HostRoot) + + err := os.Mkdir(dir+".new", 0o755) + if err != nil { + t.Fatal(err) + } + + err = os.RemoveAll(dir) + if err != nil { + t.Fatal(err) + } + + err = os.Rename(dir+".new", dir) + if err != nil { + t.Fatal(err) + } + + if released := env.proc.Sweep(); released != 1 || len(verifiedRecords(env.proc)) != 0 { + t.Fatalf("sweep released %d, want the run with a recreated cgroup released", released) + } + + if env.fake.calls != 1 { + t.Errorf("sweep mutated: calls = %d, want only the grant", env.fake.calls) + } +} + +// TestSweep_ReleasesOnlyTheEndedRecord: records without a start time all +// share an empty StartedAt; dropping an ended one must keep a fresh one. +func TestSweep_ReleasesOnlyTheEndedRecord(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + store := env.proc.lifecycleStore() + + now := time.Now() + store.now = func() time.Time { return now } + store.provisional("abc") + + _ = store.terminate("abc", now) + + store.now = func() time.Time { return now.Add(time.Second) } + store.provisional("abc") + + if released := env.proc.Sweep(); released != 1 { + t.Fatalf("sweep released %d, want 1", released) + } + + recs := store.Records("abc") + if len(recs) != 1 || recs[0].State != LifecycleProvisional { + t.Fatalf("records = %+v, want only the fresh provisional record", recs) + } +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 97fd747..a93738d 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -95,10 +95,11 @@ type Processor struct { // cgroup, so a filter wiped by systemd's daemon-reload can be rebuilt. filterCache cgroup.FilterCache - // known maps each container lifecycle to the cgroup it was last - // verified in (guarded by mu). It is how grants are revoked once the - // process is gone or Docker cannot be asked: see revokeKnown. - known map[lifecycleKey]knownCgroup + // lifecycles is the history of container runs and the cgroups they + // were verified in (written under mu). It is how grants are revoked + // once the process is gone or Docker cannot be asked: see revokeAt. + // The daemon's coordinator installs its own; nil means a private one. + lifecycles *Lifecycles // requestPass receives each generation PublishAndReconcile publishes // (guarded by mu). @@ -117,23 +118,6 @@ type Processor struct { afterCompute func() } -// lifecycleKey names one run of a container. A restart under the same ID -// has a new StartedAt, so what is known about the old run never applies to -// the new one. -type lifecycleKey struct { - containerID string - startedAt string -} - -// knownCgroup is the cgroup a lifecycle was last verified in: the identity -// of the directory its handle was opened on, the cgroup version, and -// whether the container is privileged (it then has no device filter). -type knownCgroup struct { - identity cgroup.Identity - version int - privileged bool -} - // desiredSet is the outcome of evaluating a container's labels, the // current policy and its /dev mounts. type desiredSet struct { @@ -191,9 +175,7 @@ func (p *Processor) Reconcile(ctx context.Context, containerID string) error { return nil } - return p.revokeKnown( - lifecycleKey{containerID: containerID, startedAt: startedAt(info.State)}, - ) + return p.revokeEnded(containerID, startedAt(info.State)) } desired := p.computeDesired(containerID, &info, cfg) diff --git a/internal/processor/reconcile.go b/internal/processor/reconcile.go index 42f9557..c12f727 100644 --- a/internal/processor/reconcile.go +++ b/internal/processor/reconcile.go @@ -23,9 +23,11 @@ import ( "errors" "fmt" "io/fs" + "time" cerrdefs "github.com/containerd/errdefs" "github.com/moby/moby/api/types/container" + "golang.org/x/sys/unix" "github.com/leinardi/swarm-device-access/internal/cgroup" "github.com/leinardi/swarm-device-access/internal/logger" @@ -102,10 +104,13 @@ func (p *Processor) applyPinned( // Recorded after verification, so an identity resolved through a // recycled pid is never kept, and before the mutation, so a failed or // partial one can still be revoked here later. - p.recordKnown( - lifecycleKey{containerID: containerID, startedAt: state.StartedAt}, - knownCgroup{identity: handle.Identity(), version: resolved.Version, privileged: privileged}, - ) + p.lifecycleStore().verify(&LifecycleRecord{ + ContainerID: containerID, + StartedAt: state.StartedAt, + Identity: handle.Identity(), + Version: resolved.Version, + Privileged: privileged, + }) for _, rule := range rules { log.Debug("setting device rule", @@ -163,11 +168,13 @@ func (p *Processor) verifyPinned( } // revokeAfterInspectFailure handles a container Docker cannot report on. -// Without a pid there is no cgroup to resolve, so every lifecycle of the -// container with a known cgroup is revoked there. The container stays -// pending, unless Docker reported it does not exist and every revoke -// succeeded: the error then wraps ErrContainerGone. Dry-run never mutates, -// so for it a container Docker does not know is simply gone. +// The failure creates or refreshes a provisional record. Without a pid +// there is no cgroup to resolve, so every run of the container with a +// verified cgroup is revoked there. The container stays pending, unless +// Docker reported it does not exist and every revoke succeeded: the error +// then wraps ErrContainerGone (the records stay until the sweep verifies +// their cgroups gone). Dry-run never mutates, so for it a container Docker +// does not know is simply gone. func (p *Processor) revokeAfterInspectFailure( containerID string, dryRun bool, @@ -175,9 +182,18 @@ func (p *Processor) revokeAfterInspectFailure( ) error { var errs []error - for key, known := range p.known { - if key.containerID == containerID && !dryRun { - revokeErr := p.revokeAt(key, known, true) + if !dryRun { + store := p.lifecycleStore() + store.provisional(containerID) + + recs := store.Records(containerID) + for idx := range recs { + if !recs[idx].HasIdentity() { + continue + } + + // An ended run is not running, whatever Docker can say. + revokeErr := p.revokeAt(&recs[idx], recs[idx].State != LifecycleTerminal) if revokeErr != nil { errs = append(errs, revokeErr) } @@ -191,47 +207,156 @@ func (p *Processor) revokeAfterInspectFailure( return errors.Join(append([]error{inspectErr}, errs...)...) } -// revokeKnown applies the empty set to the cgroup key was last verified in, -// if any. It serves a lifecycle whose process is gone: nothing can be -// pinned any more, and the cgroup may already have been removed. -func (p *Processor) revokeKnown(key lifecycleKey) error { - known, ok := p.known[key] - if !ok { +// revokeEnded applies the empty set for a run whose process is gone, in +// the cgroup it was verified in, if any: nothing can be pinned any more, +// and the cgroup may already have been removed. +func (p *Processor) revokeEnded(containerID, startedAt string) error { + store := p.lifecycleStore() + + rec, ok := store.lookup(containerID, startedAt) + if !ok || !rec.HasIdentity() { return nil } - return p.revokeAt(key, known, false) + store.markTerminal(containerID, startedAt) + + rec.State = LifecycleTerminal + + return p.revokeAt(&rec, false) +} + +// Terminate handles an event that ended a run of the container at +// eventTime (die, destroy). The event names no run, so it is resolved +// against the history: the verified run with the latest start not after +// eventTime (never a newer run under the same ID), else the provisional +// records. The empty set is applied in that run's cgroup; the record stays +// until the sweep verifies the cgroup gone. Dry-run never mutates. +func (p *Processor) Terminate(containerID string, eventTime time.Time) error { + p.mu.Lock() + defer p.mu.Unlock() + + ended := p.lifecycleStore().terminate(containerID, eventTime) + + if p.Cfg.Load().DryRun { + return nil + } + + var errs []error + + for idx := range ended { + if ended[idx].HasIdentity() { + errs = append(errs, p.revokeAt(&ended[idx], false)) + } + } + + return errors.Join(errs...) +} + +// provisionalTTL is how long a provisional record without a cgroup is kept +// before the sweep drops it: it names no cgroup, so nothing can be revoked +// through it. +const provisionalTTL = 10 * time.Minute + +// Sweep re-checks every lifecycle record: a record whose cgroup no longer +// exists, or was recreated with another inode, is released together with +// what the cgroup API remembered for it; an ended run whose revoke has not +// succeeded yet is revoked again; records that never named a cgroup are +// dropped once ended or stale. It returns how many records it released. +func (p *Processor) Sweep() int { + p.mu.Lock() + defer p.mu.Unlock() + + store := p.lifecycleStore() + now := store.clock() + dryRun := p.Cfg.Load().DryRun + released := 0 + + recs := store.all() + for idx := range recs { + rec := &recs[idx] + + if !rec.HasIdentity() { + lastSeen := rec.FirstSeen + if rec.Observed.After(lastSeen) { + lastSeen = rec.Observed + } + + if rec.State == LifecycleTerminal || now.Sub(lastSeen) > provisionalTTL { + store.release(rec) + + released++ + } + + continue + } + + gone, err := cgroupGone(rec.Identity) + + switch { + case err != nil: + logger.L().Warn("could not check container cgroup", "id", rec.ContainerID, "err", err) + case gone: + p.release(rec, "cgroup gone") + + released++ + case rec.State == LifecycleTerminal && !rec.Revoked && !dryRun: + revokeErr := p.revokeAt(rec, false) + if revokeErr != nil { + logger.L(). + Warn("could not revoke ended container", "id", rec.ContainerID, "err", revokeErr) + } + } + } + + return released +} + +// cgroupGone reports whether the directory named by id no longer exists or +// was recreated (another inode). +func cgroupGone(identity cgroup.Identity) (bool, error) { + var stat unix.Stat_t + + err := unix.Stat(identity.Path, &stat) + if errors.Is(err, unix.ENOENT) { + return true, nil + } + + if err != nil { + return false, fmt.Errorf("stat cgroup %q: %w", identity.Path, err) + } + + return stat.Ino != identity.Inode, nil } // revokeAt re-opens the recorded path and applies the empty set only if it // is still the same directory (same inode). A cgroup that is gone, or was -// recreated at the same path, holds none of the lifecycle's grants: the -// entry is released and nothing is mutated. Otherwise the entry is kept -// even after the revoke, until the cgroup is verified gone. mayRun reports -// that the container may still be running (Docker could not say). -func (p *Processor) revokeAt(key lifecycleKey, known knownCgroup, mayRun bool) error { +// recreated at the same path, holds none of the run's grants: the record is +// released and nothing is mutated. Otherwise the record is kept even after +// the revoke, until the sweep verifies the cgroup gone. mayRun reports that +// the container may still be running (Docker could not say). +func (p *Processor) revokeAt(rec *LifecycleRecord, mayRun bool) error { log := logger.L() - handle, err := cgroup.OpenCgroup(known.identity.Path) + handle, err := cgroup.OpenCgroup(rec.Identity.Path) if errors.Is(err, fs.ErrNotExist) { - p.release(key, known, "cgroup removed") + p.release(rec, "cgroup removed") return nil } if err != nil { - return fmt.Errorf("revoke container %q: %w", key.containerID, err) + return fmt.Errorf("revoke container %q: %w", rec.ContainerID, err) } defer closeHandle(handle) - if handle.Identity() != known.identity { - p.release(key, known, "cgroup recreated") + if handle.Identity() != rec.Identity { + p.release(rec, "cgroup recreated") return nil } - api, err := p.cgroupAPI(known.version) + api, err := p.cgroupAPI(rec.Version) if err != nil { return err } @@ -239,45 +364,57 @@ func (p *Processor) revokeAt(key lifecycleKey, known knownCgroup, mayRun bool) e err = api.SetDeviceRules(handle, nil) switch { - case errors.Is(err, cgroup.ErrFilterMissing) && mayRun && !known.privileged: + case errors.Is(err, cgroup.ErrFilterMissing) && mayRun && !rec.Privileged: // Nothing attached holds a grant, but a container that may still be // running is unfiltered. The container stays pending either way. log.Warn("device filter missing and no cached original; restart the container", - "id", key.containerID, "cgroup", known.identity.Path, "reason", "filter_missing") + "id", rec.ContainerID, "cgroup", rec.Identity.Path, "reason", "filter_missing") case errors.Is(err, cgroup.ErrFilterMissing): // An exited or privileged container has nothing attached, so // nothing to revoke. case err != nil: return fmt.Errorf( "revoke container %q in %q: %w", - key.containerID, - known.identity.Path, + rec.ContainerID, + rec.Identity.Path, err, ) } - log.Debug("device grants revoked", "id", key.containerID, "cgroup", known.identity.Path) + p.lifecycleStore().markRevoked(rec.ContainerID, rec.StartedAt) + log.Debug("device grants revoked", "id", rec.ContainerID, "cgroup", rec.Identity.Path) return nil } -func (p *Processor) recordKnown(key lifecycleKey, known knownCgroup) { - if p.known == nil { - p.known = make(map[lifecycleKey]knownCgroup) - } +// release forgets a run whose cgroup is verified gone, together with what +// the cgroup API remembered for that directory (the v1 ledger and the +// cached v2 filters). +func (p *Processor) release(rec *LifecycleRecord, why string) { + logger.L().Debug("container cgroup gone; released", + "id", rec.ContainerID, "cgroup", rec.Identity.Path, "why", why) - p.known[key] = known + p.lifecycleStore().release(rec) + p.ledger.Forget(rec.Identity) + p.filterCache.Forget(rec.Identity) } -// release forgets a lifecycle whose cgroup is verified gone, together with -// what the cgroup API remembered for that directory. -func (p *Processor) release(key lifecycleKey, known knownCgroup, why string) { - logger.L().Debug("container cgroup gone; released", - "id", key.containerID, "cgroup", known.identity.Path, "why", why) +// SetLifecycles installs the lifecycle history the daemon's coordinator +// keeps; until then the processor keeps a private one. +func (p *Processor) SetLifecycles(lifecycles *Lifecycles) { + p.mu.Lock() + defer p.mu.Unlock() + + p.lifecycles = lifecycles +} + +// lifecycleStore returns the lifecycle history; callers hold mu. +func (p *Processor) lifecycleStore() *Lifecycles { + if p.lifecycles == nil { + p.lifecycles = &Lifecycles{} + } - delete(p.known, key) - p.ledger.Forget(known.identity) - p.filterCache.Forget(known.identity) + return p.lifecycles } //nolint:ireturn // processPinner is the seam that lets tests fake pidfds diff --git a/internal/processor/reconcile_test.go b/internal/processor/reconcile_test.go index f4bde1c..03f9b7f 100644 --- a/internal/processor/reconcile_test.go +++ b/internal/processor/reconcile_test.go @@ -143,8 +143,8 @@ func (e *reconcileEnv) grantOnce(t *testing.T) { ) } - if len(e.proc.known) != 1 { - t.Fatalf("known = %v, want the lifecycle recorded", e.proc.known) + if len(verifiedRecords(e.proc)) != 1 { + t.Fatalf("known = %v, want the lifecycle recorded", verifiedRecords(e.proc)) } } @@ -188,9 +188,9 @@ func TestReconcile_PidRecycledBeforePinDoesNotMutate(t *testing.T) { t.Fatalf("err = %v, want errLifecycleChanged", err) } - if env.fake.calls != 0 || len(env.proc.known) != 0 { + if env.fake.calls != 0 || len(verifiedRecords(env.proc)) != 0 { t.Errorf("SetDeviceRules calls = %d, known = %v; want no mutation and nothing recorded", - env.fake.calls, env.proc.known) + env.fake.calls, verifiedRecords(env.proc)) } } @@ -279,8 +279,11 @@ func TestReconcile_RevokeAfterExitWithMatchingInode(t *testing.T) { t.Errorf("pins = %v, want only the grant to pin", env.pinner.pins) } - if len(env.proc.known) != 1 { - t.Errorf("known = %v, want the entry kept until the cgroup is gone", env.proc.known) + if len(verifiedRecords(env.proc)) != 1 { + t.Errorf( + "known = %v, want the entry kept until the cgroup is gone", + verifiedRecords(env.proc), + ) } } @@ -320,8 +323,8 @@ func TestReconcile_InodeMismatchReleasesWithoutMutation(t *testing.T) { t.Errorf("SetDeviceRules calls = %d, want only the grant", env.fake.calls) } - if len(env.proc.known) != 0 { - t.Errorf("known = %v, want the entry released", env.proc.known) + if len(verifiedRecords(env.proc)) != 0 { + t.Errorf("known = %v, want the entry released", verifiedRecords(env.proc)) } } @@ -337,9 +340,9 @@ func TestReconcile_ExitAfterCgroupRemovedReleases(t *testing.T) { env.insp.then(exited()) err = env.reconcile() - if err != nil || env.fake.calls != 1 || len(env.proc.known) != 0 { + if err != nil || env.fake.calls != 1 || len(verifiedRecords(env.proc)) != 0 { t.Fatalf("err = %v, calls = %d, known = %v; want release without mutation", - err, env.fake.calls, env.proc.known) + err, env.fake.calls, verifiedRecords(env.proc)) } } @@ -372,8 +375,11 @@ func TestReconcile_InspectFailureRevokesKnownIdentity(t *testing.T) { t.Fatalf("known: rules = %v, want the empty set applied", env.fake.rules) } - if len(env.proc.known) != 1 { - t.Errorf("known = %v, want the entry kept (not-found alone never releases)", env.proc.known) + if len(verifiedRecords(env.proc)) != 1 { + t.Errorf( + "known = %v, want the entry kept (not-found alone never releases)", + verifiedRecords(env.proc), + ) } } @@ -625,3 +631,17 @@ func TestReconcile_NotFoundIsContainerGone(t *testing.T) { } }) } + +// verifiedRecords returns the lifecycle records that name a cgroup. +func verifiedRecords(proc *Processor) []LifecycleRecord { + var out []LifecycleRecord + + recs := proc.lifecycleStore().all() + for idx := range recs { + if recs[idx].HasIdentity() { + out = append(out, recs[idx]) + } + } + + return out +} From a063b0437e68579a9ab637d28ce69bd187045d08 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 11:10:45 +0200 Subject: [PATCH 11/37] fix(config): reject unknown keys, wrong shapes and extra documents in the config file config.LoadFile now makes two passes over the same bytes. The first walks the parsed nodes, following aliases, and requires every known key to have exactly its shape: dry-run and log-time a YAML bool (a quoted yes is no longer coerced), policy-mode, log-format, log-level and docker-socket a non-empty string, metrics-addr and debug-addr a string that may be empty, device-allow and device-deny a list of non-empty strings. An explicit null, also behind an alias, is an error instead of silently meaning not set, and [null] can no longer decode to [""], which read as allow-all. Merge keys are refused. The second pass decodes with unknown keys rejected (a device_deny typo no longer drops the deny list) and requires a single document. log-format, log-level and policy-mode are validated when the file is loaded and again, on the effective values, at startup and before a SIGHUP reload is applied; the undocumented log-level aliases warning, fatal and panic are retired (README flag section). The README documents the strict file rules, and the trust-boundary checklist is updated. --- .agents/skills/trust-boundary/SKILL.md | 16 +- README.md | 16 ++ cmd/swarm-device-access/config.go | 20 ++ cmd/swarm-device-access/flags.go | 2 +- cmd/swarm-device-access/main.go | 8 +- internal/config/loader.go | 291 ++++++++++++++++++++++++- internal/config/loader_test.go | 211 ++++++++++++++++++ 7 files changed, 553 insertions(+), 11 deletions(-) create mode 100644 internal/config/loader_test.go diff --git a/.agents/skills/trust-boundary/SKILL.md b/.agents/skills/trust-boundary/SKILL.md index 3febba1..d02b778 100644 --- a/.agents/skills/trust-boundary/SKILL.md +++ b/.agents/skills/trust-boundary/SKILL.md @@ -52,17 +52,19 @@ fixed set, it must be rejected at load time, naming the key and the bad value. `policy.ParseMode` rejects an unknown `policy-mode`, and `policy.Global.Validate` wraps glob errors as `device-allow: …` / `device-deny: …` via `policy.ValidateGlobs`. `run` in -`cmd/swarm-device-access/main.go` calls `Validate` after `applyFileConfig` merges the file into the -flags, and exits `1` with `invalid config: …` on failure; an unreadable or unparsable file from -`config.LoadFile` (`internal/config/loader.go`) makes it exit `1` with `config file error: …`. Covered by -`TestGlobalValidate` and `TestValidateGlobs`. +`cmd/swarm-device-access/main.go` calls `Validate` and `config.ValidateEnums` (log-format, log-level, +policy-mode) on the effective values after `applyFileConfig` merges the file into the flags, and exits +`1` with `invalid config: …` on failure. `config.LoadFile` (`internal/config/loader.go`) is strict: it +checks every known key's exact YAML shape (following aliases; explicit `null`, a quoted `"yes"` for a +bool and `[null]` in a list are errors), refuses merge keys, unknown keys and a second document, and +validates the enums; any failure makes the daemon exit `1` with `config file error: …`. Covered by +`TestGlobalValidate`, `TestValidateGlobs`, `TestLoadFile_*` and `TestValidateEnums`. - [ ] A new enum-like key or flag is checked in `Validate` (or an equivalent load-time check) with a table test covering an unknown value. - [ ] The error names the key, so an operator can fix it without reading Go. -- [ ] **Known gap:** `log-format` and `log-level` are not validated — `logger.Configure` falls back - to `text` and `INFO` on anything it does not recognize. That cannot widen device access, but - it is the pattern this item forbids; do not copy it for a key that can. +- [ ] A new file key gets an entry in `keyShapes` (`internal/config/loader.go`), so an explicit + `null` or a wrong type is rejected instead of decoding as "not set". ## 3. Labels are untrusted input diff --git a/README.md b/README.md index f9b23e2..fd00ee3 100644 --- a/README.md +++ b/README.md @@ -229,12 +229,28 @@ value is set in both places, the CLI flag wins. | `-config` | `""` | Path to a YAML config file. CLI flags override file values. Reload with SIGHUP. | | `-help` | | Print this flag list and exit | +`-log-level`, `-log-format` and `-policy-mode` (and their config file keys) +accept only the values listed; anything else is an error at startup and makes a +reload keep the previous config. The undocumented `-log-level` aliases +`warning`, `fatal` and `panic`, which earlier versions mapped to `warn` and +`error`, are no longer accepted. + ### Config File All CLI flags can be represented in YAML and loaded with `-config`. See [`deployments/docker/config.yaml`](deployments/docker/config.yaml) for an example. +The file is checked strictly: an unknown key (for example `device_deny` for +`device-deny`), a second YAML document, a YAML merge key (`<<`) or a value of +the wrong type is an error. `dry-run` and `log-time` must be `true` or +`false` (not `"yes"`); `policy-mode`, `log-format`, `log-level` and +`docker-socket` must be non-empty strings; `metrics-addr` and `debug-addr` must +be strings, and an empty one disables the listener; `device-allow` and +`device-deny` must be lists of non-empty strings. An explicit `null` (also +through an alias) is an error, never a silent "not set": leave a key out to +keep the flag's value. + Send `SIGHUP` to reload the config file without restarting the daemon: ```bash diff --git a/cmd/swarm-device-access/config.go b/cmd/swarm-device-access/config.go index 33db14f..cf23e58 100644 --- a/cmd/swarm-device-access/config.go +++ b/cmd/swarm-device-access/config.go @@ -136,6 +136,26 @@ func watchSIGHUP(ctx context.Context, sigCh <-chan os.Signal, proc *processor.Pr effectiveLogTime = *fileCfg.LogTime } + // Checked again on the effective values before anything is + // applied, so a bad value never reaches the logger or the + // published config. + effectivePolicyMode := *policyMode + if !cliSet["policy-mode"] && fileCfg.PolicyMode != "" { + effectivePolicyMode = fileCfg.PolicyMode + } + + enumErr := config.ValidateEnums( + effectiveLogFormat, + effectiveLogLevel, + effectivePolicyMode, + "setting", + ) + if enumErr != nil { + log.Error("config reload: invalid setting; keeping previous config", "err", enumErr) + + continue + } + logger.Configure(effectiveLogFormat, effectiveLogLevel, effectiveLogTime) newDryRun := *dryRun diff --git a/cmd/swarm-device-access/flags.go b/cmd/swarm-device-access/flags.go index 8c418a4..706b3b1 100644 --- a/cmd/swarm-device-access/flags.go +++ b/cmd/swarm-device-access/flags.go @@ -37,7 +37,7 @@ func (f *stringSliceFlag) Set(val string) error { var ( logFormat = flag.String("log-format", "text", "Either json, text or plain") - logLevel = flag.String("log-level", "info", "Either debug, info, warn, error, fatal, panic") + logLevel = flag.String("log-level", "info", "Either debug, info, warn or error") logTime = flag.Bool("log-time", false, "Include timestamp in logs") dockerSocket = flag.String( "docker-socket", diff --git a/cmd/swarm-device-access/main.go b/cmd/swarm-device-access/main.go index 61cfe05..5a7aa3f 100644 --- a/cmd/swarm-device-access/main.go +++ b/cmd/swarm-device-access/main.go @@ -23,6 +23,7 @@ package main import ( "context" + "errors" "flag" "fmt" "os" @@ -71,7 +72,12 @@ func run() int { signal.Notify(sighup, syscall.SIGHUP) defer signal.Stop(sighup) - startupValidationErr := store.Load().Policy.Validate() + // The file's values were checked when it was loaded; the effective + // values (flags included) are checked here. + startupValidationErr := errors.Join( + config.ValidateEnums(*logFormat, *logLevel, *policyMode, "setting"), + store.Load().Policy.Validate(), + ) if startupValidationErr != nil { fmt.Fprintf(os.Stderr, "invalid config: %v\n", startupValidationErr) diff --git a/internal/config/loader.go b/internal/config/loader.go index fe770bb..08384d9 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -17,10 +17,16 @@ package config import ( + "bytes" + "errors" "fmt" + "io" "os" + "slices" "gopkg.in/yaml.v3" + + "github.com/leinardi/swarm-device-access/internal/policy" ) // FileSchema mirrors the CLI flags that can be set via the config file. @@ -41,8 +47,77 @@ type FileSchema struct { DebugAddr string `yaml:"debug-addr"` } +var ( + errShape = errors.New("wrong shape") + errMergeKey = errors.New("YAML merge keys are not supported") + errMultipleDocs = errors.New("multiple YAML documents") + errUnknownValue = errors.New("unknown value") + errAliasCycle = errors.New("alias cycle") + errDocumentNotMap = errors.New("the document must be a mapping") +) + +// LogFormats and LogLevels are the accepted log-format and log-level values. +// +//nolint:gochecknoglobals // read-only enum lists +var ( + LogFormats = []string{"text", "json", "plain"} + LogLevels = []string{"debug", "info", "warn", "error"} +) + +// keyShape is what a config key's value must look like. +type keyShape int + +const ( + shapeBool keyShape = iota + shapeNonEmptyString + shapeString + shapeNonEmptyStringSeq +) + +func (s keyShape) String() string { + switch s { + case shapeBool: + return "a boolean (true or false)" + case shapeNonEmptyString: + return "a non-empty string" + case shapeString: + return "a string (empty disables it)" + case shapeNonEmptyStringSeq: + return "a list of non-empty strings" + default: + return "unknown" + } +} + +// keyShapes lists every key the file may set. An explicit null never +// matches a shape, so it cannot silently stand for "not set" and fall back +// to the flag's value. +// +//nolint:gochecknoglobals // read-only table +var keyShapes = map[string]keyShape{ + "dry-run": shapeBool, + "log-time": shapeBool, + "policy-mode": shapeNonEmptyString, + "log-format": shapeNonEmptyString, + "log-level": shapeNonEmptyString, + "docker-socket": shapeNonEmptyString, + "metrics-addr": shapeString, + "debug-addr": shapeString, + "device-allow": shapeNonEmptyStringSeq, + "device-deny": shapeNonEmptyStringSeq, +} + // LoadFile reads and parses the YAML config file at path. // Returns a zero-value FileSchema without error when path is empty. +// +// The file is checked strictly, in two passes over the same bytes. The +// first walks the parsed nodes (following aliases) and requires every +// known key to have exactly its shape: yaml.v3 would otherwise turn a +// quoted "yes" into true, an explicit null into "not set", and [null] into +// [""], which reads as allow-all. Merge keys are refused, since they could +// smuggle a value past that check. The second decodes with unknown keys +// rejected and requires the file to hold a single document. Enum values +// are validated last. func LoadFile(path string) (FileSchema, error) { if path == "" { return FileSchema{}, nil @@ -53,12 +128,224 @@ func LoadFile(path string) (FileSchema, error) { return FileSchema{}, fmt.Errorf("read config file %q: %w", path, err) } - var cfg FileSchema + var doc yaml.Node - err = yaml.Unmarshal(data, &cfg) + err = yaml.Unmarshal(data, &doc) if err != nil { return FileSchema{}, fmt.Errorf("parse config file %q: %w", path, err) } + if isEmptyDocument(&doc) { + // Still exactly one document: "---" followed by a second document + // is not an empty file. + err = skipFirstDocument(data) + if err != nil { + return FileSchema{}, fmt.Errorf("config file %q: %w", path, err) + } + + return FileSchema{}, nil + } + + err = checkShapes(&doc) + if err != nil { + return FileSchema{}, fmt.Errorf("config file %q: %w", path, err) + } + + cfg, err := decodeStrict(data) + if err != nil { + return FileSchema{}, fmt.Errorf("config file %q: %w", path, err) + } + + err = cfg.validateEnums() + if err != nil { + return FileSchema{}, fmt.Errorf("config file %q: %w", path, err) + } + return cfg, nil } + +// decodeStrict is the second pass: unknown keys are errors and the file +// must hold exactly one document. +func decodeStrict(data []byte) (FileSchema, error) { + var cfg FileSchema + + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + + err := dec.Decode(&cfg) + if err != nil { + return FileSchema{}, fmt.Errorf("decode: %w", err) + } + + err = singleDocument(dec) + if err != nil { + return FileSchema{}, err + } + + return cfg, nil +} + +// singleDocument requires dec, past its first document, to be at the end +// of the input. +func singleDocument(dec *yaml.Decoder) error { + var rest yaml.Node + + err := dec.Decode(&rest) + + switch { + case errors.Is(err, io.EOF): + return nil + case err != nil: + return fmt.Errorf("parse trailing content: %w", err) + default: + return errMultipleDocs + } +} + +// skipFirstDocument checks that data holds at most one document. +func skipFirstDocument(data []byte) error { + dec := yaml.NewDecoder(bytes.NewReader(data)) + + var first yaml.Node + + err := dec.Decode(&first) + if errors.Is(err, io.EOF) { + return nil + } + + if err != nil { + return fmt.Errorf("decode: %w", err) + } + + return singleDocument(dec) +} + +// isEmptyDocument reports whether the first document is empty: no content +// at all, or a bare "---" (an implicit null). +func isEmptyDocument(doc *yaml.Node) bool { + if doc.Kind == 0 || len(doc.Content) == 0 { + return true + } + + root := doc.Content[0] + + return root.Kind == yaml.ScalarNode && root.Tag == "!!null" && root.Value == "" +} + +// checkShapes is the first pass over the parsed document. +func checkShapes(doc *yaml.Node) error { + root, err := deref(doc.Content[0], map[*yaml.Node]bool{}) + if err != nil { + return err + } + + if root.Kind != yaml.MappingNode { + return errDocumentNotMap + } + + for idx := 0; idx+1 < len(root.Content); idx += 2 { + keyNode, valueNode := root.Content[idx], root.Content[idx+1] + + if keyNode.Tag == "!!merge" || keyNode.Value == "<<" { + return errMergeKey + } + + shape, known := keyShapes[keyNode.Value] + if !known { + continue // the strict decode reports it as unknown + } + + value, derefErr := deref(valueNode, map[*yaml.Node]bool{}) + if derefErr != nil { + return fmt.Errorf("key %q: %w", keyNode.Value, derefErr) + } + + if !hasShape(value, shape) { + return fmt.Errorf("key %q must be %s: %w", keyNode.Value, shape, errShape) + } + } + + return nil +} + +// hasShape reports whether node (aliases already followed) has shape. +func hasShape(node *yaml.Node, shape keyShape) bool { + switch shape { + case shapeBool: + return node.Kind == yaml.ScalarNode && node.Tag == "!!bool" + case shapeNonEmptyString: + return node.Kind == yaml.ScalarNode && node.Tag == "!!str" && node.Value != "" + case shapeString: + return node.Kind == yaml.ScalarNode && node.Tag == "!!str" + case shapeNonEmptyStringSeq: + if node.Kind != yaml.SequenceNode { + return false + } + + for _, elem := range node.Content { + item, err := deref(elem, map[*yaml.Node]bool{}) + if err != nil || !hasShape(item, shapeNonEmptyString) { + return false + } + } + + return true + default: + return false + } +} + +// deref follows aliases to the node they name. +func deref(node *yaml.Node, visited map[*yaml.Node]bool) (*yaml.Node, error) { + for node.Kind == yaml.AliasNode { + if visited[node] { + return nil, errAliasCycle + } + + visited[node] = true + node = node.Alias + } + + return node, nil +} + +// validateEnums checks the enumerated values the file sets. +func (f *FileSchema) validateEnums() error { + return ValidateEnums(f.LogFormat, f.LogLevel, f.PolicyMode, "key") +} + +// ValidateEnums checks log-format, log-level and policy-mode values; +// empty values are not set and pass. source names what set them in the +// error ("key" for the file, "flag" for the effective settings). +func ValidateEnums(logFormat, logLevel, policyMode, source string) error { + if logFormat != "" && !slices.Contains(LogFormats, logFormat) { + return fmt.Errorf( + "%s %q: %w %q, must be one of %v", + source, + "log-format", + errUnknownValue, + logFormat, + LogFormats, + ) + } + + if logLevel != "" && !slices.Contains(LogLevels, logLevel) { + return fmt.Errorf( + "%s %q: %w %q, must be one of %v", + source, + "log-level", + errUnknownValue, + logLevel, + LogLevels, + ) + } + + if policyMode != "" { + _, err := policy.ParseMode(policyMode) + if err != nil { + return fmt.Errorf("%s %q: %w", source, "policy-mode", err) + } + } + + return nil +} diff --git a/internal/config/loader_test.go b/internal/config/loader_test.go new file mode 100644 index 0000000..b42307d --- /dev/null +++ b/internal/config/loader_test.go @@ -0,0 +1,211 @@ +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package config + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func writeConfig(t *testing.T, content string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "config.yaml") + + err := os.WriteFile(path, []byte(content), 0o600) + if err != nil { + t.Fatal(err) + } + + return path +} + +func TestLoadFile_ValidFile(t *testing.T) { + t.Parallel() + + path := writeConfig(t, ` +log-format: json +log-level: debug +log-time: true +docker-socket: /run/docker.sock +dry-run: false +policy-mode: all +device-allow: + - /dev/nvidia* + - /dev/dri/* +device-deny: [] +metrics-addr: "" +debug-addr: "127.0.0.1:6060" +`) + + cfg, err := LoadFile(path) + if err != nil { + t.Fatal(err) + } + + yes, no := true, false + want := FileSchema{ + LogFormat: "json", + LogLevel: "debug", + LogTime: &yes, + DockerSocket: "/run/docker.sock", + DryRun: &no, + PolicyMode: "all", + DeviceAllow: []string{"/dev/nvidia*", "/dev/dri/*"}, + DeviceDeny: []string{}, + DebugAddr: "127.0.0.1:6060", + } + + if !reflect.DeepEqual(cfg, want) { + t.Errorf("cfg = %+v, want %+v", cfg, want) + } +} + +func TestLoadFile_EmptyInputs(t *testing.T) { + t.Parallel() + + cfg, err := LoadFile("") + if err != nil || !reflect.DeepEqual(cfg, FileSchema{}) { + t.Errorf("empty path: cfg = %+v, err = %v; want the zero config", cfg, err) + } + + for name, content := range map[string]string{ + "empty file": "", + "only comments": "# nothing set\n", + "bare doc marker": "---\n", + "shipped template": "# log-level: info\n# device-deny: []\n", + } { + cfg, err = LoadFile(writeConfig(t, content)) + if err != nil || !reflect.DeepEqual(cfg, FileSchema{}) { + t.Errorf("%s: cfg = %+v, err = %v; want the zero config", name, cfg, err) + } + } +} + +func TestLoadFile_ShippedExample(t *testing.T) { + t.Parallel() + + _, err := LoadFile(filepath.Join("..", "..", "deployments", "docker", "config.yaml")) + if err != nil { + t.Fatalf("the shipped example must load: %v", err) + } +} + +func TestLoadFile_UnreadablePath(t *testing.T) { + t.Parallel() + + _, err := LoadFile(filepath.Join(t.TempDir(), "missing.yaml")) + if err == nil || !strings.Contains(err.Error(), "read config file") { + t.Fatalf("err = %v, want a read error", err) + } +} + +// TestLoadFile_Rejects covers every file the loader must refuse, with a +// fragment the error must contain. +func TestLoadFile_Rejects(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name, content, want string + }{ + {"unknown key", "device_deny:\n - /dev/sda\n", "field device_deny not found"}, + {"second document", "dry-run: true\n---\npolicy-mode: all\n", "multiple YAML documents"}, + {"empty second document", "dry-run: true\n---\n", "multiple YAML documents"}, + {"document after an empty one", "---\n---\ndry-run: true\n", "multiple YAML documents"}, + {"malformed trailing content", "dry-run: true\n---\n: : [\n", "trailing content"}, + {"duplicate key", "policy-mode: all\npolicy-mode: opt-in\n", "already defined"}, + {"root not a mapping", "- dry-run\n", "must be a mapping"}, + {"explicit null document", "~\n", "must be a mapping"}, + + {"dry-run as a string", "dry-run: \"yes\"\n", `key "dry-run" must be a boolean`}, + {"dry-run null", "dry-run: null\n", `key "dry-run" must be a boolean`}, + {"policy-mode null", "policy-mode: ~\n", `key "policy-mode" must be a non-empty string`}, + {"policy-mode empty", "policy-mode: \"\"\n", `key "policy-mode" must be a non-empty string`}, + {"device-allow null", "device-allow:\n", `key "device-allow" must be a list`}, + {"device-deny null", "device-deny: null\n", `key "device-deny" must be a list`}, + {"device-allow null element", "device-allow: [null]\n", `key "device-allow" must be a list`}, + {"device-deny empty element", "device-deny: [\"\"]\n", `key "device-deny" must be a list`}, + {"device-deny scalar", "device-deny: /dev/sda\n", `key "device-deny" must be a list`}, + {"log-time null", "log-time: null\n", `key "log-time" must be a boolean`}, + {"metrics-addr number", "metrics-addr: 9090\n", `key "metrics-addr" must be a string`}, + + {"null behind an alias", "x: &n null\ndevice-deny: *n\n", `key "device-deny" must be a list`}, + {"null element behind an alias", "x: &n null\ndevice-allow: [*n]\n", `key "device-allow" must be a list`}, + {"inline merge", "<<: {device-deny: null}\n", "YAML merge keys are not supported"}, + {"aliased merge", "base: &b {dry-run: false}\n<<: *b\n", "YAML merge keys are not supported"}, + + {"unknown log-format", "log-format: xml\n", `key "log-format": unknown value "xml"`}, + {"retired log-level alias", "log-level: warning\n", `key "log-level": unknown value "warning"`}, + {"unknown policy-mode", "policy-mode: everything\n", `key "policy-mode"`}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + path := writeConfig(t, tc.content) + + _, err := LoadFile(path) + if err == nil { + t.Fatalf("LoadFile accepted %q", tc.content) + } + + if !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), path) { + t.Errorf("err = %q, want it to name the file and contain %q", err, tc.want) + } + }) + } +} + +// TestLoadFile_LogTimeAbsentVsNull documents the chosen behavior: an +// absent log-time leaves the flag's value in force, an explicit null is an +// error like any other key (not a silent "unset"). +func TestLoadFile_LogTimeAbsentVsNull(t *testing.T) { + t.Parallel() + + cfg, err := LoadFile(writeConfig(t, "log-level: info\n")) + if err != nil || cfg.LogTime != nil { + t.Errorf("absent: LogTime = %v, err = %v; want not set", cfg.LogTime, err) + } + + _, err = LoadFile(writeConfig(t, "log-time: null\n")) + if err == nil { + t.Error("explicit null: want an error") + } +} + +func TestValidateEnums(t *testing.T) { + t.Parallel() + + err := ValidateEnums("plain", "warn", "opt-in", "flag") + if err != nil { + t.Errorf("valid values: %v", err) + } + + err = ValidateEnums("", "", "", "flag") + if err != nil { + t.Errorf("unset values: %v", err) + } + + for _, alias := range []string{"warning", "fatal", "panic"} { + err = ValidateEnums("", alias, "", "flag") + if err == nil || !strings.Contains(err.Error(), `flag "log-level"`) { + t.Errorf("log-level %q: err = %v, want it rejected naming the flag", alias, err) + } + } +} From 60f8b950369971679af0e2ec4689b9316ede55a5 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 11:15:57 +0200 Subject: [PATCH 12/37] fix(config): recompute every setting on reload instead of falling back to the startup file Right after flag.Parse the flag values and the set of flags given on the command line are snapshotted into one settings struct (list flags deep-copied). One pure mergeSettings(flags, cliSet, file) returns the complete effective settings and replaces the duplicated merges of applyFileConfig and watchSIGHUP, which wrote the startup file's values into the flag globals so that a key later removed from the file kept its old file value (policy-mode: all stayed all). Startup consumes every field. A SIGHUP reload validates the merged settings (enums and policy) before anything is applied, applies only the hot-reloadable ones (logging, dry-run, policy), logs and ignores a changed docker-socket, metrics-addr or debug-addr, and publishes through Processor.PublishAndReconcile. A reload may turn dry-run off but never on: that is rejected with dry-run cannot be enabled on a running daemon, and the previous config is kept. Any rejected reload leaves the store, the logger and the in-force settings untouched. SIGHUP is registered in run() and the channel passed in. README (Config File, -dry-run row) and the trust-boundary checklist describe the reload rules; config_test.go replaces the empty main_test.go. --- .agents/skills/trust-boundary/SKILL.md | 28 +-- README.md | 19 +- cmd/swarm-device-access/config.go | 315 ++++++++++++++----------- cmd/swarm-device-access/config_test.go | 278 ++++++++++++++++++++++ cmd/swarm-device-access/main.go | 49 ++-- cmd/swarm-device-access/main_test.go | 21 -- 6 files changed, 508 insertions(+), 202 deletions(-) create mode 100644 cmd/swarm-device-access/config_test.go delete mode 100644 cmd/swarm-device-access/main_test.go diff --git a/.agents/skills/trust-boundary/SKILL.md b/.agents/skills/trust-boundary/SKILL.md index d02b778..10b3e5d 100644 --- a/.agents/skills/trust-boundary/SKILL.md +++ b/.agents/skills/trust-boundary/SKILL.md @@ -136,22 +136,22 @@ Covered by `TestIsDeviceMountSource`, `TestCollectMountRules_*` and ## 5. Hot reload never widens access on a parse error -`watchSIGHUP` (`cmd/swarm-device-access/config.go`) re-reads the file on `SIGHUP`. On a read or -parse error from `config.LoadFile` it logs `config reload failed` and keeps the running config; on -a `policy.Global.Validate` failure it logs `config reload: invalid policy; keeping previous config` -and keeps it too. `config.Store.Set` swaps the whole `config.Runtime` (policy and `dry-run`) -atomically, so no reader sees half a policy. - -- [ ] Every new reloadable key is validated before `store.Set`, and a failure `continue`s without - touching the store. +`watchSIGHUP` (`cmd/swarm-device-access/config.go`) calls `reloader.reload` on `SIGHUP`. It +re-reads the file and recomputes every setting with the pure `mergeSettings(flags, cliSet, file)` +from the flag snapshot taken right after `flag.Parse`, so a key removed from the file falls back to +the command line or the default. Validation (`settings.validate`: enums and policy) and the dry-run +rule (a reload may turn `dry-run` off, never on) run before anything is applied; any failure is +logged as `config reload rejected; keeping previous config` and leaves the store, the logger and +the in-force settings untouched. On success the logger is reconfigured and the runtime is published +through `Processor.PublishAndReconcile`, which swaps the whole `config.Runtime` (policy and +`dry-run`) atomically and reconciles every running container. Restart-only settings changed in the +file are logged and ignored. Covered by `TestReload_*` and `TestMergeSettings_*`. + +- [ ] Every new setting goes through `settings`/`mergeSettings`, is checked in + `settings.validate`, and a failure returns before `configureLogger` or `publish`. - [ ] A reload that fails leaves the previous policy and `dry-run` in force — never a zero value, never a partially merged one. -- [ ] **Known gap:** `watchSIGHUP` calls `logger.Configure` with the file's `log-format`, - `log-level` and `log-time` before `Validate` runs, so a reload rejected for its policy still - changes logging. Device access is unaffected, but do not move any access-relevant setting - ahead of validation the same way. -- [ ] **Known gap:** there is no test for the reload path. A change to `watchSIGHUP` adds one that - sends an invalid file and asserts the previous policy is still loaded. +- [ ] A new restart-only setting is added to `settings.coldChanges`. ## 6. Least privilege in the image and the deployment diff --git a/README.md b/README.md index fd00ee3..720e46e 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ value is set in both places, the CLI flag wins. | `-log-format` | `text` | `text`, `json`, `plain` | | `-log-time` | `false` | Include timestamps in log lines | | `-docker-socket` | `/var/run/docker.sock` | Path to the Docker daemon's UNIX socket | -| `-dry-run` | `false` | Log the device set each container would get, without reading its `/proc` entry or its cgroup and without writing anything. Not a cleanup path: grants left by a previous live run stay (see [Config File](#config-file)) | +| `-dry-run` | `false` | Log the device set each container would get, without reading its `/proc` entry or its cgroup and without writing anything. Not a cleanup path: grants left by a previous live run stay, and a reload can turn it off but not on (see [Config File](#config-file)) | | `-policy-mode` | `opt-in` | `opt-in`: only `enable=true` containers. `all`: unless `enable=false`. | | `-device-allow` | `""` | Glob for `/dev/...` paths to allow, repeatable. Empty means allow all. | | `-device-deny` | `""` | Glob for `/dev/...` paths to deny, repeatable. Deny takes priority over allow. | @@ -258,7 +258,22 @@ kill -HUP $(docker inspect --format '{{.State.Pid}}' swarm-device-access) ``` Some settings are only read at startup and still require a restart: -`docker-socket`, `metrics-addr`, and `debug-addr`. +`docker-socket`, `metrics-addr`, and `debug-addr`. A reload that changes one of +them logs that it was ignored and applies the rest. + +A reload computes every setting afresh from the command line, the file and the +defaults, in that order: a key removed from the file falls back to the flag +given on the command line, or to the default, not to what the file said +before. A reload that fails for any reason (unreadable or invalid file, invalid +value or policy) changes nothing, logging included, and the previous config +stays in force. + +`dry-run` can be turned off by a reload, and the daemon then applies the +current policy for real. It cannot be turned on: a reload that would enable it +is rejected with `dry-run cannot be enabled on a running daemon; to stop +enforcing, restart the daemon live with a narrower policy, or restart the +affected containers`, and the previous config is kept, because dry-run would +stop revoking while every grant stayed in place. A reload, a daemon start and a `systemctl daemon-reload` each reconcile every running container under the current config, so narrowing the policy revokes diff --git a/cmd/swarm-device-access/config.go b/cmd/swarm-device-access/config.go index cf23e58..4523c72 100644 --- a/cmd/swarm-device-access/config.go +++ b/cmd/swarm-device-access/config.go @@ -20,194 +20,221 @@ package main import ( "context" + "errors" "flag" + "fmt" "os" + "slices" "github.com/leinardi/swarm-device-access/internal/config" "github.com/leinardi/swarm-device-access/internal/logger" "github.com/leinardi/swarm-device-access/internal/policy" - "github.com/leinardi/swarm-device-access/internal/processor" ) -// applyFileConfig merges the file config into the CLI flags (for flags not -// explicitly set by the user) and returns the resulting startup Runtime. -// Must be called after flag.Parse(). -// -//nolint:gocyclo,cyclop // complexity is inherent: merges many independent optional config fields -func applyFileConfig(fileCfg *config.FileSchema) config.Runtime { - cliSet := make(map[string]bool) - - flag.Visit(func(f *flag.Flag) { cliSet[f.Name] = true }) - - if !cliSet["log-format"] && fileCfg.LogFormat != "" { - *logFormat = fileCfg.LogFormat - } - - if !cliSet["log-level"] && fileCfg.LogLevel != "" { - *logLevel = fileCfg.LogLevel - } +var errDryRunEnable = errors.New( + "dry-run cannot be enabled on a running daemon; to stop enforcing, restart the daemon " + + "live with a narrower policy, or restart the affected containers", +) - if !cliSet["log-time"] && fileCfg.LogTime != nil { - *logTime = *fileCfg.LogTime - } +// settings is every value the flags and the config file control. +type settings struct { + LogFormat string + LogLevel string + LogTime bool + DockerSocket string + MetricsAddr string + DebugAddr string + DryRun bool + PolicyMode string + DeviceAllow []string + DeviceDeny []string +} - if !cliSet["docker-socket"] && fileCfg.DockerSocket != "" { - *dockerSocket = fileCfg.DockerSocket - } +// flagSettings snapshots the parsed flags (defaults included) and the set +// of flags given on the command line. Call it right after flag.Parse; the +// slices are copied, so nothing later aliases the flag variables. +func flagSettings() (flags settings, cliSet map[string]bool) { + cliSet = make(map[string]bool) - if !cliSet["metrics-addr"] && fileCfg.MetricsAddr != "" { - *metricsAddr = fileCfg.MetricsAddr - } + flag.Visit(func(f *flag.Flag) { cliSet[f.Name] = true }) - if !cliSet["debug-addr"] && fileCfg.DebugAddr != "" { - *debugAddr = fileCfg.DebugAddr - } + return settings{ + LogFormat: *logFormat, + LogLevel: *logLevel, + LogTime: *logTime, + DockerSocket: *dockerSocket, + MetricsAddr: *metricsAddr, + DebugAddr: *debugAddr, + DryRun: *dryRun, + PolicyMode: *policyMode, + DeviceAllow: slices.Clone(deviceAllow), + DeviceDeny: slices.Clone(deviceDeny), + }, cliSet +} - if !cliSet["dry-run"] && fileCfg.DryRun != nil { - *dryRun = *fileCfg.DryRun +// mergeSettings returns the effective settings: a flag given on the command +// line wins, then a key set in the file, then the flag's default (whatever +// defaults holds). It is pure and never aliases file's slices, so a reload +// that drops a key from the file falls back to the command line or the +// default, not to what an earlier file said. +// +//nolint:gocritic // hugeParam: settings is passed by value on purpose, the result must not share it +func mergeSettings(defaults settings, cliSet map[string]bool, file config.FileSchema) settings { + merged := defaults + merged.DeviceAllow = slices.Clone(defaults.DeviceAllow) + merged.DeviceDeny = slices.Clone(defaults.DeviceDeny) + + fileString := func(name, value string, target *string) { + if !cliSet[name] && value != "" { + *target = value + } } - if !cliSet["policy-mode"] && fileCfg.PolicyMode != "" { - *policyMode = fileCfg.PolicyMode + fileBool := func(name string, value *bool, target *bool) { + if !cliSet[name] && value != nil { + *target = *value + } } - if !cliSet["device-allow"] && fileCfg.DeviceAllow != nil { - deviceAllow = fileCfg.DeviceAllow + fileList := func(name string, value []string, target *[]string) { + if !cliSet[name] && value != nil { + *target = slices.Clone(value) + } } - if !cliSet["device-deny"] && fileCfg.DeviceDeny != nil { - deviceDeny = fileCfg.DeviceDeny - } + fileString("log-format", file.LogFormat, &merged.LogFormat) + fileString("log-level", file.LogLevel, &merged.LogLevel) + fileBool("log-time", file.LogTime, &merged.LogTime) + fileString("docker-socket", file.DockerSocket, &merged.DockerSocket) + fileString("metrics-addr", file.MetricsAddr, &merged.MetricsAddr) + fileString("debug-addr", file.DebugAddr, &merged.DebugAddr) + fileBool("dry-run", file.DryRun, &merged.DryRun) + fileString("policy-mode", file.PolicyMode, &merged.PolicyMode) + fileList("device-allow", file.DeviceAllow, &merged.DeviceAllow) + fileList("device-deny", file.DeviceDeny, &merged.DeviceDeny) + + return merged +} - return config.Runtime{ - DryRun: *dryRun, - Policy: policy.Global{ - Mode: policy.Mode(*policyMode), - DeviceAllow: append([]string(nil), deviceAllow...), - DeviceDeny: append([]string(nil), deviceDeny...), - }, +// policy returns the global policy the settings describe. +func (s *settings) policy() policy.Global { + return policy.Global{ + Mode: policy.Mode(s.PolicyMode), + DeviceAllow: slices.Clone(s.DeviceAllow), + DeviceDeny: slices.Clone(s.DeviceDeny), } } -// watchSIGHUP blocks until ctx is done, reloading the config file and -// updating the logger on each SIGHUP, then publishing the new config through -// proc, which reconciles every running container under it. Settings that -// require a restart (docker-socket, metrics-addr, debug-addr) are not -// reloaded. -// -// sigCh must already be registered for SIGHUP (see run), so a SIGHUP during -// startup is queued instead of terminating the process. -// -//nolint:gocyclo,cyclop,gocognit // complexity is inherent: handles SIGHUP, reload, merge, validate, and logger update -func watchSIGHUP(ctx context.Context, sigCh <-chan os.Signal, proc *processor.Processor) { - for { - select { - case <-ctx.Done(): - return - - case <-sigCh: - log := logger.L() - log.Info("SIGHUP received; reloading config", "config_file", *configFile) - - fileCfg, err := config.LoadFile(*configFile) - if err != nil { - log.Error("config reload failed", "err", err) +// runtime returns the hot-reloadable part the processor reads. +func (s *settings) runtime() config.Runtime { + return config.Runtime{DryRun: s.DryRun, Policy: s.policy()} +} - continue - } +// validate checks every enumerated value and the policy; it runs before +// anything (the logger included) is configured from the settings. +func (s *settings) validate() error { + return errors.Join( + config.ValidateEnums(s.LogFormat, s.LogLevel, s.PolicyMode, "setting"), + s.policy().Validate(), + ) +} - cliSet := make(map[string]bool) +// coldChanges names the restart-only settings that differ between s and +// next. +func (s *settings) coldChanges(next *settings) []string { + var changed []string - flag.Visit(func(f *flag.Flag) { cliSet[f.Name] = true }) + if next.DockerSocket != s.DockerSocket { + changed = append(changed, "docker-socket") + } - effectiveLogFormat := *logFormat - effectiveLogLevel := *logLevel - effectiveLogTime := *logTime + if next.MetricsAddr != s.MetricsAddr { + changed = append(changed, "metrics-addr") + } - if !cliSet["log-format"] && fileCfg.LogFormat != "" { - effectiveLogFormat = fileCfg.LogFormat - } + if next.DebugAddr != s.DebugAddr { + changed = append(changed, "debug-addr") + } - if !cliSet["log-level"] && fileCfg.LogLevel != "" { - effectiveLogLevel = fileCfg.LogLevel - } + return changed +} - if !cliSet["log-time"] && fileCfg.LogTime != nil { - effectiveLogTime = *fileCfg.LogTime - } +// reloader applies a re-read config file on SIGHUP. +type reloader struct { + path string + flags settings + cliSet map[string]bool + // current is what is in force: the startup settings, then those of + // the last reload that succeeded. + current settings + + configureLogger func(format, level string, includeTime bool) + publish func(ctx context.Context, rt config.Runtime) uint64 +} - // Checked again on the effective values before anything is - // applied, so a bad value never reaches the logger or the - // published config. - effectivePolicyMode := *policyMode - if !cliSet["policy-mode"] && fileCfg.PolicyMode != "" { - effectivePolicyMode = fileCfg.PolicyMode - } +// reload re-reads the file and applies the hot-reloadable settings (logging, +// dry-run, policy). On any error nothing is applied: the store, the logger +// and current keep their previous values. A changed restart-only setting +// is logged and ignored. dry-run may be turned off but not on: turning it +// on would leave every grant in place while no longer enforcing narrower +// policy. +func (r *reloader) reload(ctx context.Context) error { + file, err := config.LoadFile(r.path) + if err != nil { + return fmt.Errorf("config reload failed: %w", err) + } - enumErr := config.ValidateEnums( - effectiveLogFormat, - effectiveLogLevel, - effectivePolicyMode, - "setting", - ) - if enumErr != nil { - log.Error("config reload: invalid setting; keeping previous config", "err", enumErr) + next := mergeSettings(r.flags, r.cliSet, file) - continue - } + err = next.validate() + if err != nil { + return fmt.Errorf("config reload: invalid setting: %w", err) + } - logger.Configure(effectiveLogFormat, effectiveLogLevel, effectiveLogTime) + if next.DryRun && !r.current.DryRun { + return fmt.Errorf("config reload: %w", errDryRunEnable) + } - newDryRun := *dryRun - newPolicyMode := *policyMode + for _, key := range r.current.coldChanges(&next) { + logger.L().Info("config reload: setting requires a restart; ignored", "key", key) + } - newDeviceAllow := append([]string(nil), deviceAllow...) - newDeviceDeny := append([]string(nil), deviceDeny...) + next.DockerSocket = r.current.DockerSocket + next.MetricsAddr = r.current.MetricsAddr + next.DebugAddr = r.current.DebugAddr - if !cliSet["dry-run"] && fileCfg.DryRun != nil { - newDryRun = *fileCfg.DryRun - } + r.configureLogger(next.LogFormat, next.LogLevel, next.LogTime) - if !cliSet["policy-mode"] && fileCfg.PolicyMode != "" { - newPolicyMode = fileCfg.PolicyMode - } + generation := r.publish(ctx, next.runtime()) + r.current = next - if !cliSet["device-allow"] && fileCfg.DeviceAllow != nil { - newDeviceAllow = fileCfg.DeviceAllow - } + logger.L().Info("config reloaded", + "generation", generation, + "dry_run", next.DryRun, + "policy_mode", next.PolicyMode, + "device_allow", next.DeviceAllow, + "device_deny", next.DeviceDeny, + ) - if !cliSet["device-deny"] && fileCfg.DeviceDeny != nil { - newDeviceDeny = fileCfg.DeviceDeny - } + return nil +} - newPolicy := policy.Global{ - Mode: policy.Mode(newPolicyMode), - DeviceAllow: newDeviceAllow, - DeviceDeny: newDeviceDeny, - } +// watchSIGHUP blocks until ctx is done, reloading the config on each +// SIGHUP. sigCh must already be registered for SIGHUP (see run), so a +// SIGHUP during startup is queued instead of terminating the process. +func watchSIGHUP(ctx context.Context, sigCh <-chan os.Signal, reload *reloader) { + for { + select { + case <-ctx.Done(): + return - valErr := newPolicy.Validate() - if valErr != nil { - log.Error( - "config reload: invalid policy; keeping previous config", - "err", valErr, - ) + case <-sigCh: + logger.L().Info("SIGHUP received; reloading config", "config_file", reload.path) - continue + err := reload.reload(ctx) + if err != nil { + logger.L().Error("config reload rejected; keeping previous config", "err", err) } - - generation := proc.PublishAndReconcile(ctx, config.Runtime{ - DryRun: newDryRun, - Policy: newPolicy, - }) - - logger.L().Info("config reloaded", - "generation", generation, - "dry_run", newDryRun, - "policy_mode", newPolicy.Mode, - "device_allow", newPolicy.DeviceAllow, - "device_deny", newPolicy.DeviceDeny, - ) } } } diff --git a/cmd/swarm-device-access/config_test.go b/cmd/swarm-device-access/config_test.go new file mode 100644 index 0000000..2e52ff0 --- /dev/null +++ b/cmd/swarm-device-access/config_test.go @@ -0,0 +1,278 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "bytes" + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" + + "github.com/leinardi/swarm-device-access/internal/config" + "github.com/leinardi/swarm-device-access/internal/logger" + "github.com/leinardi/swarm-device-access/internal/policy" + "github.com/leinardi/swarm-device-access/internal/processor" +) + +// defaultSettings are the flag defaults, as flagSettings returns them +// without any flag on the command line. +func defaultSettings() settings { + return settings{ + LogFormat: "text", + LogLevel: "info", + DockerSocket: "/var/run/docker.sock", + PolicyMode: string(policy.ModeOptIn), + } +} + +func TestMergeSettings_Precedence(t *testing.T) { + t.Parallel() + + file := config.FileSchema{ + PolicyMode: "all", + LogLevel: "debug", + DeviceDeny: []string{"/dev/sda"}, + } + + cli := defaultSettings() + cli.LogLevel = "warn" + + merged := mergeSettings(cli, map[string]bool{"log-level": true}, file) + + if merged.LogLevel != "warn" { + t.Errorf("log-level = %q, want the command line's warn", merged.LogLevel) + } + + if merged.PolicyMode != "all" || !slices.Equal(merged.DeviceDeny, []string{"/dev/sda"}) { + t.Errorf( + "merged = %+v, want the file's policy-mode and device-deny over the defaults", + merged, + ) + } + + if merged.LogFormat != "text" || merged.DockerSocket != "/var/run/docker.sock" { + t.Errorf("merged = %+v, want defaults where neither sets a value", merged) + } +} + +func TestMergeSettings_DoesNotAlias(t *testing.T) { + t.Parallel() + + defaults := defaultSettings() + defaults.DeviceAllow = []string{"/dev/null"} + + file := config.FileSchema{DeviceDeny: []string{"/dev/sda"}} + + merged := mergeSettings(defaults, map[string]bool{}, file) + merged.DeviceAllow[0] = "/dev/changed" + file.DeviceDeny[0] = "/dev/changed" + + if defaults.DeviceAllow[0] != "/dev/null" || merged.DeviceDeny[0] != "/dev/sda" { + t.Error("merged settings share a slice with the defaults or the file") + } +} + +// reloadEnv is a reloader wired to a real store and processor publication, +// with the logger configuration recorded instead of applied. +type reloadEnv struct { + reloader *reloader + store *config.Store + path string + loggerCfg []string + logs *bytes.Buffer +} + +func newReloadEnv( + t *testing.T, + flags *settings, + cliSet map[string]bool, + initialFile string, +) *reloadEnv { + t.Helper() + + env := &reloadEnv{path: filepath.Join(t.TempDir(), "config.yaml")} + env.writeFile(t, initialFile) + + file, err := config.LoadFile(env.path) + if err != nil { + t.Fatal(err) + } + + current := mergeSettings(*flags, cliSet, file) + + store, publisher := config.NewStore(current.runtime()) + proc := &processor.Processor{Cfg: store, Publisher: publisher} + + env.store = store + env.reloader = &reloader{ + path: env.path, + flags: *flags, + cliSet: cliSet, + current: current, + configureLogger: func(format, level string, _ bool) { + env.loggerCfg = append(env.loggerCfg, format+"/"+level) + }, + publish: proc.PublishAndReconcile, + } + + prev := logger.L() + env.logs = &bytes.Buffer{} + logger.Set(slog.New(slog.NewTextHandler(env.logs, nil))) + t.Cleanup(func() { logger.Set(prev) }) + + return env +} + +func (e *reloadEnv) writeFile(t *testing.T, content string) { + t.Helper() + + err := os.WriteFile(e.path, []byte(content), 0o600) + if err != nil { + t.Fatal(err) + } +} + +func (e *reloadEnv) reload(t *testing.T, content string) error { + t.Helper() + + e.writeFile(t, content) + + return e.reloader.reload(context.Background()) +} + +func TestReload_RemovedKeyFallsBackToFlagOrDefault(t *testing.T) { + cli := defaultSettings() + cli.DeviceDeny = []string{"/dev/sdb"} + + env := newReloadEnv(t, &cli, map[string]bool{"device-deny": true}, "policy-mode: all\n") + + if got := env.store.Load().Policy.Mode; got != policy.ModeAll { + t.Fatalf("startup policy-mode = %q, want all from the file", got) + } + + err := env.reload(t, "log-level: info\n") + if err != nil { + t.Fatal(err) + } + + got := env.store.Load().Policy + if got.Mode != policy.ModeOptIn || !slices.Equal(got.DeviceDeny, []string{"/dev/sdb"}) { + t.Errorf( + "after the key was removed: policy = %+v, want the default opt-in and the CLI deny", + got, + ) + } +} + +func TestReload_InvalidKeepsStoreAndLogger(t *testing.T) { + env := newReloadEnv(t, new(defaultSettings()), map[string]bool{}, "policy-mode: all\n") + before, beforeGen := env.store.Snapshot() + + for name, content := range map[string]string{ + "unparsable file": "policy-mode: [\n", + "invalid glob": "log-level: debug\ndevice-allow: [\"/dev/[\"]\n", + "unknown value": "log-level: loud\n", + } { + err := env.reload(t, content) + if err == nil { + t.Fatalf("%s: reload accepted it", name) + } + + after, afterGen := env.store.Snapshot() + if afterGen != beforeGen || !reflect.DeepEqual(after, before) { + t.Errorf("%s: store changed to %+v", name, after) + } + } + + if len(env.loggerCfg) != 0 { + t.Errorf("logger reconfigured by rejected reloads: %v", env.loggerCfg) + } +} + +func TestReload_ColdFieldChangeIgnored(t *testing.T) { + env := newReloadEnv( + t, + new(defaultSettings()), + map[string]bool{}, + "metrics-addr: \"127.0.0.1:9090\"\n", + ) + + err := env.reload(t, "metrics-addr: \"127.0.0.1:9999\"\npolicy-mode: all\n") + if err != nil { + t.Fatal(err) + } + + if env.reloader.current.MetricsAddr != "127.0.0.1:9090" { + t.Errorf("metrics-addr = %q, want the startup value kept", env.reloader.current.MetricsAddr) + } + + if env.store.Load().Policy.Mode != policy.ModeAll { + t.Error("the hot settings of the same file were not applied") + } + + if !strings.Contains(env.logs.String(), "setting requires a restart; ignored") || + !strings.Contains(env.logs.String(), "key=metrics-addr") { + t.Errorf("expected an INFO naming the ignored key, got:\n%s", env.logs.String()) + } +} + +func TestReload_DryRunTransitions(t *testing.T) { + t.Run("true to false is applied", func(t *testing.T) { + env := newReloadEnv(t, new(defaultSettings()), map[string]bool{}, "dry-run: true\n") + + err := env.reload(t, "dry-run: false\n") + if err != nil { + t.Fatal(err) + } + + if env.store.Load().DryRun { + t.Error("dry-run still on after a reload turned it off") + } + }) + + t.Run("false to true is rejected", func(t *testing.T) { + env := newReloadEnv( + t, + new(defaultSettings()), + map[string]bool{}, + "dry-run: false\npolicy-mode: all\n", + ) + before, beforeGen := env.store.Snapshot() + + err := env.reload(t, "dry-run: true\npolicy-mode: opt-in\n") + if !errors.Is(err, errDryRunEnable) { + t.Fatalf("err = %v, want errDryRunEnable", err) + } + + after, afterGen := env.store.Snapshot() + if afterGen != beforeGen || !reflect.DeepEqual(after, before) || len(env.loggerCfg) != 0 { + t.Errorf( + "rejected reload changed the store to %+v or the logger %v", + after, + env.loggerCfg, + ) + } + }) +} diff --git a/cmd/swarm-device-access/main.go b/cmd/swarm-device-access/main.go index 5a7aa3f..dce78a7 100644 --- a/cmd/swarm-device-access/main.go +++ b/cmd/swarm-device-access/main.go @@ -23,7 +23,6 @@ package main import ( "context" - "errors" "flag" "fmt" "os" @@ -55,6 +54,15 @@ func run() int { return 0 } + // Registered before anything slow, so a SIGHUP during startup is queued + // for watchSIGHUP instead of terminating the process (its default). + sighup := make(chan os.Signal, 1) + + signal.Notify(sighup, syscall.SIGHUP) + defer signal.Stop(sighup) + + flags, cliSet := flagSettings() + // Load config file; CLI flags override file values. fileCfg, fileErr := config.LoadFile(*configFile) if fileErr != nil { @@ -63,28 +71,20 @@ func run() int { return 1 } - store, publisher := config.NewStore(applyFileConfig(&fileCfg)) - - // Registered before anything slow, so a SIGHUP during startup is queued - // for watchSIGHUP instead of terminating the process (its default). - sighup := make(chan os.Signal, 1) - - signal.Notify(sighup, syscall.SIGHUP) - defer signal.Stop(sighup) + effective := mergeSettings(flags, cliSet, fileCfg) // The file's values were checked when it was loaded; the effective - // values (flags included) are checked here. - startupValidationErr := errors.Join( - config.ValidateEnums(*logFormat, *logLevel, *policyMode, "setting"), - store.Load().Policy.Validate(), - ) + // values (flags included) are checked here, before anything uses them. + startupValidationErr := effective.validate() if startupValidationErr != nil { fmt.Fprintf(os.Stderr, "invalid config: %v\n", startupValidationErr) return 1 } - logger.Configure(*logFormat, *logLevel, *logTime) + logger.Configure(effective.LogFormat, effective.LogLevel, effective.LogTime) + + store, publisher := config.NewStore(effective.runtime()) log := logger.L() @@ -129,7 +129,7 @@ func run() int { defer cancelRoot() // API version negotiation is the client default; it runs lazily on the first request. - cli, err := client.New(client.WithHost("unix://" + *dockerSocket)) + cli, err := client.New(client.WithHost("unix://" + effective.DockerSocket)) if err != nil { log.Error("docker client init failed", "err", err) @@ -141,12 +141,12 @@ func run() int { // Start optional observability servers before the main loop so they are // reachable during startup enumeration. - if *metricsAddr != "" { - observability.StartMetricsServer(rootCtx, *metricsAddr) + if effective.MetricsAddr != "" { + observability.StartMetricsServer(rootCtx, effective.MetricsAddr) } - if *debugAddr != "" { - observability.StartDebugServer(rootCtx, *debugAddr) + if effective.DebugAddr != "" { + observability.StartDebugServer(rootCtx, effective.DebugAddr) } proc := &processor.Processor{ @@ -161,7 +161,14 @@ func run() int { // SIGHUP: reload the config file, update the logger and publish the // new config through the processor. - go watchSIGHUP(rootCtx, sighup, proc) + go watchSIGHUP(rootCtx, sighup, &reloader{ + path: *configFile, + flags: flags, + cliSet: cliSet, + current: effective, + configureLogger: logger.Configure, + publish: proc.PublishAndReconcile, + }) runErr := daemon.Run(rootCtx, daemon.Options{ Docker: cli, diff --git a/cmd/swarm-device-access/main_test.go b/cmd/swarm-device-access/main_test.go deleted file mode 100644 index 7832cc5..0000000 --- a/cmd/swarm-device-access/main_test.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build linux - -/* - * Copyright 2026 Roberto Leinardi. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Tests have moved to internal/daemon and internal/processor. - -package main From 2a2ee3815c0daaa3da92211f1b34f3e66e5c796e Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 11:34:30 +0200 Subject: [PATCH 13/37] fix(processor): identify device candidates by descriptor and sysfs DEVNAME Every name under a /dev mount (the source, walked entries and symlinks) now goes through one evaluation on file descriptors: a lexical gate (clean, under /dev), openat2 beneath a /dev descriptor opened once per pass with RESOLVE_BENEATH and RESOLVE_NO_MAGICLINKS, a bounded by-hand follow of absolute symlinks back into /dev (EXDEV), fstat for type and major:minor, and the canonical name from the device's sysfs uevent DEVNAME. Names that lead outside /dev are skipped with a WARN and counted in sda_device_candidates_skipped_total{reason}. Policy splits DeviceAllowed into Denied, checked on the alias, the resolved node and the canonical name, and Authorized, checked on the canonical and resolved names. Candidates are aggregated per device across all mounts: a device is granted only when authorized with no deny vote under any name, replacing the first-wins dedup. A candidate whose identity cannot be established and that policy would otherwise grant empties the whole desired set with a retryable error. The daemon refuses to start without openat2 RESOLVE_BENEATH; the documented kernel requirement is now Linux 5.6. --- README.md | 21 +- cmd/swarm-device-access/main.go | 46 +- docs/architecture.md | 53 +- internal/observability/metrics.go | 15 + internal/policy/policy.go | 29 +- internal/policy/policy_test.go | 34 ++ internal/processor/devfs.go | 268 ++++++++ internal/processor/processor.go | 92 ++- internal/processor/processor_test.go | 361 ----------- internal/processor/rules.go | 595 +++++++++++++----- internal/processor/rules_test.go | 879 +++++++++++++++++++++++++++ 11 files changed, 1811 insertions(+), 582 deletions(-) create mode 100644 internal/processor/devfs.go create mode 100644 internal/processor/rules_test.go diff --git a/README.md b/README.md index 720e46e..e77f9ac 100644 --- a/README.md +++ b/README.md @@ -58,10 +58,11 @@ uses the host Docker socket to launch the real privileged daemon container. ### Host Requirements -On every host the daemon needs Linux 5.3 or newer: it pins each running -container's process with `pidfd_open(2)` before changing its cgroup (see -below). On an older kernel no running container is changed, not even to -revoke a grant, and each attempt fails with a `pidfd_open` error. +On every host the daemon needs Linux 5.6 or newer. It resolves every device +path beneath `/dev` with `openat2(2)` and `RESOLVE_BENEATH` (Linux 5.6), and +refuses to start with `unsupported kernel` when that is missing. It also pins +each running container's process with `pidfd_open(2)` (Linux 5.3) before +changing its cgroup (see below). On cgroup v2 hosts the daemon adds its grants to the device filter the container runtime already attached, replacing that program in place rather @@ -324,6 +325,18 @@ If you bind-mount a directory (for example `source: /dev/dri`), the `device-allow` glob is evaluated **per child node** inside that directory — write the glob against the children, e.g. `/dev/dri/*` or `/dev/dri/renderD128`. +A device can be reached under several names: the mounted name (an alias such +as `/dev/disk/by-id/usb-...`), the node it resolves to, and its kernel name +(`DEVNAME` in sysfs, e.g. `/dev/sda`). Deny globs are checked against all three, +and a match on any name of a device, in any of the container's mounts, denies +it. Allow globs must match the kernel name and the resolved node; an alias +that no allow glob names does not deny anything. A node planted elsewhere +under `/dev` is therefore judged as the device it really is. A name that +leads outside `/dev` is skipped with a warning; a device whose identity cannot +be established (no or ambiguous `DEVNAME` in sysfs, an unreadable node) leaves +the container with no grants until a retry succeeds, unless policy already +excludes it under its other names. + Global `-device-allow` and `-device-deny` define the broadest access the daemon may grant. Per-container labels can only narrow that access. Deny rules always win. diff --git a/cmd/swarm-device-access/main.go b/cmd/swarm-device-access/main.go index dce78a7..06d6e9a 100644 --- a/cmd/swarm-device-access/main.go +++ b/cmd/swarm-device-access/main.go @@ -45,6 +45,34 @@ func main() { os.Exit(run()) } +// startupSettings loads the config file, merges it with the flags and checks +// the result and the kernel before anything uses them. +func startupSettings(flags *settings, cliSet map[string]bool) (settings, error) { + // CLI flags override file values. + fileCfg, err := config.LoadFile(*configFile) + if err != nil { + return settings{}, fmt.Errorf("config file error: %w", err) + } + + effective := mergeSettings(*flags, cliSet, fileCfg) + + // The file's values were checked when it was loaded; the effective + // values (flags included) are checked here, before anything uses them. + err = effective.validate() + if err != nil { + return settings{}, fmt.Errorf("invalid config: %w", err) + } + + // Device paths are resolved beneath /dev with openat2; without it no + // path can be contained, so refuse to start rather than fall back. + err = processor.ProbeOpenat2() + if err != nil { + return settings{}, fmt.Errorf("unsupported kernel: %w", err) + } + + return effective, nil +} + func run() int { flag.Parse() @@ -63,21 +91,9 @@ func run() int { flags, cliSet := flagSettings() - // Load config file; CLI flags override file values. - fileCfg, fileErr := config.LoadFile(*configFile) - if fileErr != nil { - fmt.Fprintf(os.Stderr, "config file error: %v\n", fileErr) - - return 1 - } - - effective := mergeSettings(flags, cliSet, fileCfg) - - // The file's values were checked when it was loaded; the effective - // values (flags included) are checked here, before anything uses them. - startupValidationErr := effective.validate() - if startupValidationErr != nil { - fmt.Fprintf(os.Stderr, "invalid config: %v\n", startupValidationErr) + effective, startupErr := startupSettings(&flags, cliSet) + if startupErr != nil { + fmt.Fprintln(os.Stderr, startupErr) return 1 } diff --git a/docs/architecture.md b/docs/architecture.md index a4f8b31..ba4671b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -113,25 +113,35 @@ rather than replacing it. For every bind mount whose source is `/dev` or lives under `/dev/`, the processor collects one device rule per device node: -- **Single-file mount** (e.g. `/dev/nvidia0`): policy is checked against the (resolved) path. A missing, unresolvable or non-device file is an error, - because it was mounted explicitly. -- **Directory mount** (e.g. `/dev`, `/dev/dri`): the tree is walked (depth-capped). Allow/deny globs match the **resolved target** of each entry, - never the symlink name, so a link such as `/dev/disk/by-id/...` cannot bypass a deny on `/dev/sda`. - -Symlinks are resolved in the daemon's own view. Because the daemon bind-mounts the host `/dev`, links that stay inside `/dev` (`disk/by-id/*`, -`dri/by-path/*`, `char/*`) resolve correctly. Links that leave `/dev` (`/dev/log -> /run/systemd/journal/dev-log`, `/dev/initctl -> /run/initctl`, -`/dev/stdin -> /proc/self/fd/0`) often dangle inside the daemon container; they never point to device nodes. Such unresolvable symlinks are -**skipped, not errors**: - -- `Warn` `device symlink matches allow policy but cannot be resolved` — only when an explicit allow glob (global or label) matches the link path, since - that likely means an intended device is missing. -- `Debug` `unresolvable symlink skipped` — every other case, including all mounts without allow globs. - -Entries that pass policy but are not character or block devices (regular files, sockets, pipes) are skipped at `Debug` (`non-device entry skipped`). -Other failures on entries that pass policy (e.g. a `stat` error on a device node), and errors reading the tree itself during the walk (e.g. an -unreadable subdirectory, reported before any policy check), are per-device errors: each is logged once as -`Warn` `device rule failed` and counted in `sda_rule_failures_total`, while the remaining rules are still applied. Only container-level failures -(inspect, cgroup detection/path resolution, `AddDeviceRules`) make the container fail. +- **Single-file mount** (e.g. `/dev/nvidia0`): the source is one candidate. A missing or non-device source is an error, because it was mounted + explicitly. +- **Directory mount** (e.g. `/dev`, `/dev/dri`): the tree is walked (depth-capped) only to enumerate names; each entry is one candidate named + after the mount source. + +`/dev` is opened once per pass. Each candidate goes through one evaluation on file descriptors, so an entry swapped between enumeration and use +is judged by what it is when opened, not by its name: + +1. The name must be clean (no `..`, `//` or trailing `/`) and under `/dev`; otherwise `Warn` `device path resolves outside /dev` + (`reason=outside_dev`) and it is skipped. +2. It is opened with `openat2(O_PATH, RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS)` beneath the `/dev` descriptor: symlinks are followed, but no + step may leave `/dev`. The kernel also refuses an absolute symlink back into `/dev` (`EXDEV`); the link itself is then read and, if its + target is under `/dev`, followed by hand (at most 40 hops). A link that leaves `/dev` (`/dev/log -> /run/...`, `/dev/stdin -> + /proc/self/fd/0`) is skipped as `outside_dev`; a dangling name is skipped as `dangling` (`Warn` `device symlink matches allow policy but + cannot be resolved` when an explicit allow glob names it, `Debug` otherwise). +3. `fstat` on the descriptor gives the type and `major:minor`; a directory or a non-device entry is skipped at `Debug`. +4. The device's kernel name is read from `/dev/{char,block}/:/uevent`: exactly one `DEVNAME=` line with a clean relative + value. The canonical name is `/dev/` plus that value. sysfs is `/host/sys` when mounted, else `/sys`. +5. Deny globs are checked on the alias, the resolved node (from `/proc/self/fd`) and the canonical name; allow globs on the canonical and the + resolved name. + +Candidates are aggregated per device across all of the container's mounts: a device is granted once, only when it is authorized and no candidate +that resolves to it is denied. A deny on one name that suppresses an otherwise allowed device is logged at `Info` with `denied_by`. + +A candidate whose identity cannot be established (another `openat2` error, `fstat` failure, a missing, unreadable or ambiguous `DEVNAME`) and +that policy would otherwise grant, and errors reading the tree itself during the walk, are per-device errors: each is logged as `Warn` `device +rule failed` and counted in `sda_rule_failures_total`, and the container's whole desired set becomes empty with a retryable error until every +device resolves. A device without a `DEVNAME` that its other names already exclude (e.g. `/dev/pts/*` under an allow list that does not name +it) is simply not granted. Each container with at least one `/dev` mount produces one summary line with counts only: @@ -139,8 +149,8 @@ Each container with at least one `/dev` mount produces one summary line with cou level=INFO msg="container processed" id=abc pid=1234 devices_granted=12 skipped=87 errors=0 dry_run=false ``` -`devices_granted` is the number of deduplicated rules, `skipped` counts entries excluded by policy, unresolvable symlinks, symlinks to directories -and non-device entries, and `errors` counts per-device failures. +`devices_granted` is the number of granted devices, `skipped` counts candidates excluded by policy, names outside `/dev`, dangling names, +symlinks to directories and non-device entries, and `errors` counts per-device failures. ## Package layout @@ -227,6 +237,7 @@ Metrics exposed: | `sda_containers_scanned_total` | counter | — | Containers that passed policy and were processed | | `sda_containers_skipped_total` | counter | `reason` | Containers skipped (`policy`, `invalid_labels`, `no_pid`) | | `sda_device_files_discovered_total` | counter | — | Device files found across all processed containers | +| `sda_device_candidates_skipped_total` | counter | `reason` | Names under a `/dev` mount that name no device: `outside_dev`, `dangling`, `not_device` | | `sda_rule_failures_total` | counter | — | Per-device rule failures (including walk errors); the per-device failure signal | | `sda_dry_run_skips_total` | counter | — | Rules skipped in dry-run mode | | `sda_last_event_timestamp_seconds` | gauge | — | Unix timestamp of last container processed successfully (event, startup or reload) | diff --git a/internal/observability/metrics.go b/internal/observability/metrics.go index 86817ba..3b80485 100644 --- a/internal/observability/metrics.go +++ b/internal/observability/metrics.go @@ -36,6 +36,7 @@ type Recorder struct { containersScanned prometheus.Counter containersSkipped *prometheus.CounterVec deviceFilesDiscovered prometheus.Counter + candidatesSkipped *prometheus.CounterVec ruleFailures prometheus.Counter dryRunSkips prometheus.Counter lastEventTimestamp prometheus.Gauge @@ -89,6 +90,11 @@ func NewRecorder() *Recorder { Help: "Device files for which rules were collected.", }), + candidatesSkipped: promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "sda_device_candidates_skipped_total", + Help: "Names under a /dev mount that name no device (outside_dev, dangling, not_device).", + }, []string{"reason"}), + ruleFailures: promauto.NewCounter(prometheus.CounterOpts{ Name: "sda_rule_failures_total", Help: "Per-device rule failures (entries that passed policy but could not be turned into a rule, and walk errors).", @@ -193,6 +199,15 @@ func (rec *Recorder) AddDeviceFilesDiscovered(count int) { rec.deviceFilesDiscovered.Add(float64(count)) } +// AddCandidatesSkipped adds count to the skipped-candidates counter for reason. +func (rec *Recorder) AddCandidatesSkipped(reason string, count int) { + if rec == nil { + return + } + + rec.candidatesSkipped.WithLabelValues(reason).Add(float64(count)) +} + // AddRuleFailures adds count to the rule-failures counter. func (rec *Recorder) AddRuleFailures(count int) { if rec == nil { diff --git a/internal/policy/policy.go b/internal/policy/policy.go index 03c9752..d18a3b3 100644 --- a/internal/policy/policy.go +++ b/internal/policy/policy.go @@ -196,30 +196,31 @@ func (g Global) Enabled(cpol Container) bool { } // DeviceAllowed reports whether path is permitted by the combined global and -// per-container allow/deny policy. +// per-container allow/deny policy: not Denied and Authorized. // // Global is the maximum allowed access; per-container labels can only narrow // it further. Deny always wins over allow. -// - func (g Global) DeviceAllowed(cpol Container, path string) bool { - if matchAny(g.DeviceDeny, path) { - return false - } + return !g.Denied(cpol, path) && g.Authorized(cpol, path) +} - if matchAny(cpol.DeviceDeny, path) { - return false - } +// Denied reports whether path matches a global or a per-container deny glob. +// A device is checked under every name it was found by, and one match on +// any of them denies it. +func (g Global) Denied(cpol Container, path string) bool { + return matchAny(g.DeviceDeny, path) || matchAny(cpol.DeviceDeny, path) +} +// Authorized reports whether path passes the allow lists: the global one and +// the per-container one, each allowing everything when empty. A device must +// be authorized under its canonical and its resolved name; missing the allow +// list under some other alias is not a reason to deny it. +func (g Global) Authorized(cpol Container, path string) bool { if len(g.DeviceAllow) > 0 && !matchAny(g.DeviceAllow, path) { return false } - if len(cpol.DeviceAllow) > 0 && !matchAny(cpol.DeviceAllow, path) { - return false - } - - return true + return len(cpol.DeviceAllow) == 0 || matchAny(cpol.DeviceAllow, path) } // ExplicitlyAllowed reports whether path is matched by at least one explicit diff --git a/internal/policy/policy_test.go b/internal/policy/policy_test.go index 27cd99f..495999e 100644 --- a/internal/policy/policy_test.go +++ b/internal/policy/policy_test.go @@ -146,6 +146,40 @@ func TestDeviceAllowed(t *testing.T) { } } +// TestDeniedAuthorized checks the two halves DeviceAllowed is made of: a +// deny glob is a veto on its own, and the allow lists never deny. +func TestDeniedAuthorized(t *testing.T) { + t.Parallel() + + g := policy.Global{ + DeviceAllow: []string{"/dev/dri/*"}, + DeviceDeny: []string{"/dev/disk/by-id/*"}, + } + c := policy.Container{DeviceDeny: []string{"/dev/dri/card1"}} + + for _, tc := range []struct { + path string + denied, authorized bool + }{ + {"/dev/dri/card0", false, true}, + {"/dev/dri/card1", true, true}, + {"/dev/disk/by-id/usb-x", true, false}, + {"/dev/sda", false, false}, + } { + if got := g.Denied(c, tc.path); got != tc.denied { + t.Errorf("Denied(%q) = %v, want %v", tc.path, got, tc.denied) + } + + if got := g.Authorized(c, tc.path); got != tc.authorized { + t.Errorf("Authorized(%q) = %v, want %v", tc.path, got, tc.authorized) + } + } + + if !(policy.Global{}).Authorized(policy.Container{}, "/dev/anything") { + t.Error("empty allow lists must authorize everything") + } +} + // TestExplicitlyAllowed checks that only a matching explicit allow glob (with no // matching deny glob) counts; the empty-allow "allow everything" default does not. func TestExplicitlyAllowed(t *testing.T) { diff --git a/internal/processor/devfs.go b/internal/processor/devfs.go new file mode 100644 index 0000000..5fec7a9 --- /dev/null +++ b/internal/processor/devfs.go @@ -0,0 +1,268 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processor + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "golang.org/x/sys/unix" +) + +// devRoot is where the daemon sees the host's /dev (bind-mounted in). +const devRoot = "/dev" + +// errOpenat2Unsupported reports a kernel without openat2 RESOLVE_BENEATH. +var errOpenat2Unsupported = errors.New( + "openat2 with RESOLVE_BENEATH is not supported; Linux 5.6 or newer is required", +) + +// nodeKind is the file type of a resolved /dev entry. +type nodeKind int + +const ( + nodeOther nodeKind = iota + nodeChar + nodeBlock + nodeDir +) + +// deviceID is a device number and its type ("c" or "b"). +type deviceID struct { + typ string + major int64 + minor int64 +} + +func (d deviceID) String() string { + return fmt.Sprintf("%s %d:%d", d.typ, d.major, d.minor) +} + +// sysfsClass is the /sys/dev directory for the device's type. +func (d deviceID) sysfsClass() string { + if d.typ == "b" { + return "block" + } + + return "char" +} + +// devNode is a /dev entry opened by descriptor: its identity comes from the +// descriptor, never from a second lookup of the path. +type devNode interface { + // stat returns the node's type and, for a device, its number. + stat() (nodeKind, deviceID, error) + // resolved is the /dev path the descriptor refers to; informational + // and for policy only. + resolved() string + Close() error +} + +// devFS resolves names under /dev without leaving it. It is the seam that +// lets unit tests script identities: an unprivileged test cannot mknod. +type devFS interface { + // root is the directory that stands for /dev, for enumerating names. + root() string + // openBeneath opens the entry at rel (relative to /dev), following + // symlinks but refusing any step outside /dev (unix.EXDEV). + openBeneath(rel string) (devNode, error) + // readLink returns the target of the symlink at rel itself. + readLink(rel string) (string, error) + // readUevent returns the sysfs uevent of a device. + readUevent(id deviceID) ([]byte, error) + Close() error +} + +// realDevFS is the production devFS: openat2 beneath a /dev descriptor +// opened once per pass, and sysfs under sysfsRoot. +type realDevFS struct { + dir string + fd int + sysfsRoot string +} + +// openDevFS opens dir (the /dev directory) for one pass. +func openDevFS(dir, sysfsRoot string) (*realDevFS, error) { + fd, err := unix.Open(dir, unix.O_PATH|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + return nil, fmt.Errorf("open %s: %w", dir, err) + } + + return &realDevFS{dir: dir, fd: fd, sysfsRoot: sysfsRoot}, nil +} + +// sysfsRootFor returns where sysfs is read from: /sys, the +// documented mount, when it has device entries, otherwise /sys (a daemon run +// straight on the host). /sys/dev lists the kernel's devices whatever the +// mount namespace, so both give the same answer. +func sysfsRootFor(hostRoot string) string { + mounted := filepath.Join(hostRoot, "sys") + + _, err := os.Stat(filepath.Join(mounted, "dev")) + if err == nil { + return mounted + } + + return "/sys" +} + +// ProbeOpenat2 fails when the kernel cannot resolve paths beneath a +// directory (openat2 RESOLVE_BENEATH, Linux 5.6). Without it no device path +// can be contained to /dev, so the daemon refuses to start. +func ProbeOpenat2() error { + probeFd, err := unix.Openat2(unix.AT_FDCWD, ".", &unix.OpenHow{ + Flags: unix.O_PATH | unix.O_CLOEXEC, + Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_MAGICLINKS, + }) + if errors.Is(err, unix.ENOSYS) || errors.Is(err, unix.EINVAL) { + return fmt.Errorf("%w: %w", errOpenat2Unsupported, err) + } + + if err != nil { + return fmt.Errorf("probe openat2: %w", err) + } + + unix.Close(probeFd) + + return nil +} + +func (r *realDevFS) Close() error { + err := unix.Close(r.fd) + if err != nil { + return fmt.Errorf("close %s: %w", r.dir, err) + } + + return nil +} + +func (r *realDevFS) root() string { return r.dir } + +//nolint:ireturn // devNode hides the descriptor so tests can script identities +func (r *realDevFS) openBeneath(rel string) (devNode, error) { + fd, err := r.openat2(rel, 0) + if err != nil { + return nil, err + } + + return &realDevNode{fd: fd, dir: r.dir}, nil +} + +func (r *realDevFS) readLink(rel string) (string, error) { + linkFd, err := r.openat2(rel, unix.O_NOFOLLOW) + if err != nil { + return "", err + } + defer unix.Close(linkFd) + + buf := make([]byte, unix.PathMax) + + n, err := unix.Readlinkat(linkFd, "", buf) + if err != nil { + return "", fmt.Errorf("readlink %s: %w", rel, err) + } + + return string(buf[:n]), nil +} + +func (r *realDevFS) readUevent(id deviceID) ([]byte, error) { + path := filepath.Join(r.sysfsRoot, "dev", id.sysfsClass(), + strconv.FormatInt(id.major, 10)+":"+strconv.FormatInt(id.minor, 10), "uevent") + + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read %s: %w", path, err) + } + + return data, nil +} + +// openat2 opens rel beneath the /dev descriptor with O_PATH, which never +// opens the device driver. The kernel follows symlinks but refuses, as one +// atomic lookup, any step that leaves /dev: an absolute target, ".." past +// the root, or a magic link. +func (r *realDevFS) openat2(rel string, flags uint64) (int, error) { + nodeFd, err := unix.Openat2(r.fd, rel, &unix.OpenHow{ + Flags: unix.O_PATH | unix.O_CLOEXEC | flags, + Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_MAGICLINKS, + }) + if err != nil { + return -1, fmt.Errorf("openat2 %s: %w", rel, err) + } + + return nodeFd, nil +} + +type realDevNode struct { + fd int + dir string +} + +func (n *realDevNode) Close() error { + err := unix.Close(n.fd) + if err != nil { + return fmt.Errorf("close device node: %w", err) + } + + return nil +} + +func (n *realDevNode) stat() (nodeKind, deviceID, error) { + var info unix.Stat_t + + err := unix.Fstat(n.fd, &info) + if err != nil { + return nodeOther, deviceID{}, fmt.Errorf("fstat: %w", err) + } + + device := deviceID{major: int64(unix.Major(info.Rdev)), minor: int64(unix.Minor(info.Rdev))} + + switch info.Mode & unix.S_IFMT { + case unix.S_IFCHR: + device.typ = "c" + + return nodeChar, device, nil + case unix.S_IFBLK: + device.typ = "b" + + return nodeBlock, device, nil + case unix.S_IFDIR: + return nodeDir, deviceID{}, nil + default: + return nodeOther, deviceID{}, nil + } +} + +func (n *realDevNode) resolved() string { + target, err := os.Readlink("/proc/self/fd/" + strconv.Itoa(n.fd)) + if err != nil { + return "" + } + + rel, err := filepath.Rel(n.dir, target) + if err != nil || rel == ".." || strings.HasPrefix(rel, "../") { + return "" + } + + return filepath.Join(devRoot, rel) +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index a93738d..fbb9f05 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "path/filepath" + "slices" "sync" "time" @@ -48,14 +49,6 @@ type DockerInspector interface { ) (client.ContainerInspectResult, error) } -// deviceRuleKey is the deduplication key for cgroup device rules collected -// within a single container processing pass. -type deviceRuleKey struct { - typ string - major int64 - minor int64 -} - // Processor applies cgroup BPF device-allow rules to containers that // bind-mount /dev/... paths. Inspector and Cfg are required; Publisher is // required for PublishAndReconcile; Metrics may be nil (calls become @@ -112,6 +105,11 @@ type Processor struct { // Tests replace it to observe or fail the mutation without a kernel. newCgroup func(version int, ledger *cgroup.Ledger, cache *cgroup.FilterCache) (cgroup.Interface, error) + // devfs opens the view of /dev and sysfs for one pass; nil means the + // real /dev and HostRoot's sysfs. Tests replace it to script device + // identities. + devfs func() (devFS, error) + // afterCompute, when set, runs under mu after the desired rules are // computed and before they are applied. Tests use it to hold a worker // mid-flight. @@ -281,7 +279,7 @@ func (p *Processor) computeDesired( p.Metrics.RecordContainerScanned() - collected := collectContainerRules(containerID, info.State.Pid, info.Mounts, cfg.Policy, cpol) + collected := p.collectDevices(containerID, info.State.Pid, info.Mounts, cfg.Policy, cpol) for _, deviceErr := range collected.deviceErrs { log.Warn("device rule failed", "id", containerID, "err", deviceErr) @@ -421,25 +419,70 @@ func (p *Processor) classifyApplyError( // containerRules aggregates the per-mount results for one container. type containerRules struct { - granted []cgroup.DeviceRule // deduplicated across mounts + granted []cgroup.DeviceRule // one rule per granted device skipped int deviceErrs []error devMounts int } -// collectContainerRules walks every /dev mount of a container and merges the -// results. Per-device errors are collected, not returned, so one bad entry -// does not prevent the remaining rules from being applied. -func collectContainerRules( +// collectDevices opens /dev for this pass and collects the container's +// rules. Failing to open it leaves every device unresolved. +func (p *Processor) collectDevices( containerID string, pid int, mounts []container.MountPoint, gpol policy.Global, cpol policy.Container, ) containerRules { - var result containerRules + if !slices.ContainsFunc(mounts, func(mnt container.MountPoint) bool { + return IsMountSource(mnt.Source) + }) { + return containerRules{} + } + + open := p.devfs + if open == nil { + open = func() (devFS, error) { + return openDevFS(devRoot, sysfsRootFor(p.HostRoot)) + } + } + + dev, err := open() + if err != nil { + return containerRules{ + devMounts: 1, + deviceErrs: []error{fmt.Errorf("%w: %w", errUnresolved, err)}, + } + } + + defer func() { + closeErr := dev.Close() + if closeErr != nil { + logger.L().Warn("close /dev", "err", closeErr) + } + }() - seen := make(map[deviceRuleKey]struct{}) + return collectContainerRules(dev, containerID, pid, mounts, gpol, cpol, p.Metrics) +} + +// collectContainerRules evaluates every /dev mount of a container and +// aggregates the candidates per device across all of them: a device is +// granted only when it is authorized and no name it was found by is denied. +// Per-device errors are collected; any error leaves the set unknown (see +// computeDesired). +func collectContainerRules( + dev devFS, + containerID string, + pid int, + mounts []container.MountPoint, + gpol policy.Global, + cpol policy.Container, + metrics *observability.Recorder, +) containerRules { + var ( + result containerRules + devices aggregation + ) for _, mnt := range mounts { if !IsMountSource(mnt.Source) { @@ -455,20 +498,23 @@ func collectContainerRules( "destination", mnt.Destination, ) - mountResult := CollectMountRules(mnt.Source, gpol, cpol) + mountResult := CollectMountRules(dev, mnt.Source, gpol, cpol) result.skipped += mountResult.Skipped result.deviceErrs = append(result.deviceErrs, mountResult.Errs...) - for _, rule := range mountResult.Rules { - key := deviceRuleKey{rule.Type, *rule.Major, *rule.Minor} - if _, dup := seen[key]; !dup { - seen[key] = struct{}{} + for reason, count := range mountResult.Outcomes { + metrics.AddCandidatesSkipped(reason, count) + } - result.granted = append(result.granted, rule) - } + for idx := range mountResult.Candidates { + devices.add(&mountResult.Candidates[idx]) } } + granted, excluded := devices.emit(containerID) + result.granted = granted + result.skipped += excluded + return result } diff --git a/internal/processor/processor_test.go b/internal/processor/processor_test.go index a599ed9..901bfaa 100644 --- a/internal/processor/processor_test.go +++ b/internal/processor/processor_test.go @@ -406,367 +406,6 @@ func TestReconcile_OptInProcessesEnabled(t *testing.T) { } } -// ---- CollectMountRules tests ---- - -func TestCollectMountRules_ExcludedByPolicy(t *testing.T) { - t.Parallel() - - gpol := policy.Global{Mode: policy.ModeAll, DeviceDeny: []string{"/dev/null"}} - result := CollectMountRules("/dev/null", gpol, policy.Container{}) - rules, errs := result.Rules, result.Errs - - if len(rules) != 0 || len(errs) != 0 { - t.Errorf("expected no rules/errors for denied path, got rules=%v errs=%v", rules, errs) - } - - if result.Skipped != 1 { - t.Errorf("expected Skipped=1 for denied path, got %d", result.Skipped) - } -} - -func TestCollectMountRules_File(t *testing.T) { - t.Parallel() - - gpol := policy.Global{Mode: policy.ModeAll} - result := CollectMountRules("/dev/null", gpol, policy.Container{}) - rules, errs := result.Rules, result.Errs - - if len(errs) != 0 { - t.Fatalf("unexpected errors: %v", errs) - } - - if len(rules) != 1 { - t.Fatalf("expected 1 rule, got %d", len(rules)) - } - - if !rules[0].Allow || rules[0].Access != "rwm" { - t.Errorf("rule has unexpected allow/access: %+v", rules[0]) - } -} - -func TestCollectMountRules_BadPath(t *testing.T) { - t.Parallel() - - gpol := policy.Global{Mode: policy.ModeAll} - result := CollectMountRules("/dev/nonexistent-device-xyzzy", gpol, policy.Container{}) - rules, errs := result.Rules, result.Errs - - if len(rules) != 0 { - t.Errorf("expected no rules for bad path, got %v", rules) - } - - if len(errs) == 0 { - t.Error("expected errors for bad path, got none") - } -} - -// TestCollectMountRules_DirectoryMount_NoChildrenMatch checks that a WARN is emitted -// when a directory mount has children but none match the allow/deny policy. -func TestCollectMountRules_DirectoryMount_NoChildrenMatch(t *testing.T) { - dir := t.TempDir() - - err := os.WriteFile(filepath.Join(dir, "card0"), []byte{}, 0o600) - if err != nil { - t.Fatalf("create card0: %v", err) - } - - err = os.WriteFile(filepath.Join(dir, "renderD128"), []byte{}, 0o600) - if err != nil { - t.Fatalf("create renderD128: %v", err) - } - - gpol := policy.Global{ - Mode: policy.ModeAll, - DeviceAllow: []string{filepath.Join(dir, "nonexistent")}, - } - - buf := captureLogger(t) - - result := CollectMountRules(dir, gpol, policy.Container{}) - rules, errs := result.Rules, result.Errs - - if len(rules) != 0 { - t.Errorf("expected no rules, got %v", rules) - } - - if len(errs) != 0 { - t.Errorf("expected no errors, got %v", errs) - } - - logOutput := buf.String() - if !strings.Contains(logOutput, "mount excluded: no children matched") { - t.Errorf("expected WARN log about no children matched, got: %s", logOutput) - } - - if !strings.Contains(logOutput, "card0") || !strings.Contains(logOutput, "renderD128") { - t.Errorf("expected child names in WARN log, got: %s", logOutput) - } -} - -// TestCollectMountRules_DirectoryMount_SymlinkToDirSkipped checks that a symlink inside -// a directory mount that points to another directory is not recursed into. -func TestCollectMountRules_DirectoryMount_SymlinkToDirSkipped(t *testing.T) { - dir := t.TempDir() - subDir := filepath.Join(dir, "subdir") - - err := os.Mkdir(subDir, 0o755) - if err != nil { - t.Fatalf("create subdir: %v", err) - } - - err = os.Symlink(subDir, filepath.Join(dir, "linktodir")) - if err != nil { - t.Fatalf("create symlink: %v", err) - } - - gpol := policy.Global{Mode: policy.ModeAll} - - buf := captureLogger(t) - - result := CollectMountRules(dir, gpol, policy.Container{}) - rules, errs := result.Rules, result.Errs - - if len(rules) != 0 { - t.Errorf("expected no rules for dir-only mount, got %v", rules) - } - - if len(errs) != 0 { - t.Errorf("unexpected errors: %v", errs) - } - - if !strings.Contains(buf.String(), "symlink to directory skipped") { - t.Errorf("expected debug log about skipped dir symlink, got: %s", buf.String()) - } -} - -// TestCollectMountRules_DirectoryMount_SymlinkToDevice is the core regression test: -// a directory mount whose children include a symlink to a real device gets a cgroup rule -// injected even though the allow-glob targets the resolved path, not the mount source. -func TestCollectMountRules_DirectoryMount_SymlinkToDevice(t *testing.T) { - dir := t.TempDir() - - err := os.Symlink("/dev/null", filepath.Join(dir, "null")) - if err != nil { - t.Fatalf("create symlink: %v", err) - } - - gpol := policy.Global{ - Mode: policy.ModeAll, - DeviceAllow: []string{"/dev/null"}, - } - - result := CollectMountRules(dir, gpol, policy.Container{}) - rules, errs := result.Rules, result.Errs - - if len(errs) != 0 { - t.Fatalf("unexpected errors: %v", errs) - } - - if len(rules) != 1 { - t.Fatalf("expected 1 rule for /dev/null, got %d", len(rules)) - } - - if !rules[0].Allow || rules[0].Access != "rwm" { - t.Errorf("rule has unexpected allow/access: %+v", rules[0]) - } -} - -// globQuote backslash-escapes glob metacharacters so a t.TempDir() path can be -// used as a literal prefix inside a filepath.Match pattern. -func globQuote(s string) string { - var sb strings.Builder - - for _, r := range s { - if strings.ContainsRune(`\*?[`, r) { - sb.WriteRune('\\') - } - - sb.WriteRune(r) - } - - return sb.String() -} - -// buildSymlinkTree creates a directory with valid device symlinks, dangling -// symlinks and a regular file, mirroring what a host /dev bind mount looks like -// inside the daemon container (e.g. /dev/log -> /run/... does not resolve). -func buildSymlinkTree(t *testing.T) string { - t.Helper() - - dir := t.TempDir() - - links := map[string]string{ - "null": "/dev/null", - "zero": "/dev/zero", - "log": "/run/systemd/journal/dev-log-xyzzy", - "nullish": filepath.Join(dir, "missing-xyzzy"), - } - - for name, target := range links { - err := os.Symlink(target, filepath.Join(dir, name)) - if err != nil { - t.Fatalf("create symlink %s: %v", name, err) - } - } - - err := os.WriteFile(filepath.Join(dir, "regular.txt"), []byte("not a device"), 0o600) - if err != nil { - t.Fatalf("create regular.txt: %v", err) - } - - return dir -} - -func ruleSet(rules []cgroup.DeviceRule) map[string]struct{} { - set := make(map[string]struct{}, len(rules)) - for _, rule := range rules { - set[rule.Type+" "+strconv.FormatInt(*rule.Major, 10)+":"+strconv.FormatInt(*rule.Minor, 10)] = struct{}{} - } - - return set -} - -func warnLines(logOutput string) []string { - var lines []string - - for line := range strings.SplitSeq(logOutput, "\n") { - if strings.Contains(line, "level=WARN") { - lines = append(lines, line) - } - } - - return lines -} - -func TestGlobQuote(t *testing.T) { - t.Parallel() - - quoted := globQuote(`/tmp/a*b?c[d\e`) - if quoted != `/tmp/a\*b\?c\[d\\e` { - t.Fatalf("globQuote = %q", quoted) - } - - ok, err := filepath.Match(quoted, `/tmp/a*b?c[d\e`) - if err != nil || !ok { - t.Fatalf("quoted pattern does not match literal path: ok=%v err=%v", ok, err) - } -} - -// TestCollectMountRules_UnresolvableSymlinks_ExplicitAllow checks that dangling -// symlinks are skipped (never errors) and only the one matched by an explicit -// allow glob produces a WARN. -func TestCollectMountRules_UnresolvableSymlinks_ExplicitAllow(t *testing.T) { - dir := buildSymlinkTree(t) - - gpol := policy.Global{ - Mode: policy.ModeAll, - DeviceAllow: []string{"/dev/null", globQuote(dir) + "/nullish"}, - } - - buf := captureLogger(t) - - result := CollectMountRules(dir, gpol, policy.Container{}) - - if len(result.Errs) != 0 { - t.Fatalf("unexpected errors: %v", result.Errs) - } - - got := ruleSet(result.Rules) - if len(got) != 1 || len(result.Rules) != 1 { - t.Fatalf("expected exactly {c 1:3}, got %v", got) - } - - if _, ok := got["c 1:3"]; !ok { - t.Fatalf("expected {c 1:3}, got %v", got) - } - - // zero (excluded by policy), log, nullish (unresolvable), regular.txt (excluded by policy). - if result.Skipped != 4 { - t.Errorf("expected Skipped=4, got %d", result.Skipped) - } - - warns := warnLines(buf.String()) - if len(warns) != 1 { - t.Fatalf("expected exactly one WARN, got %d: %v", len(warns), warns) - } - - if !strings.Contains(warns[0], "path="+filepath.Join(dir, "nullish")) || - !strings.Contains(warns[0], "device symlink matches allow policy but cannot be resolved") { - t.Errorf("expected WARN for nullish, got: %s", warns[0]) - } - - if strings.Contains(warns[0], "path="+filepath.Join(dir, "log")) { - t.Errorf("unexpected WARN for log: %s", warns[0]) - } -} - -// TestCollectMountRules_UnresolvableSymlinks_NoAllowGlobs checks that without -// allow globs dangling symlinks and non-device entries are skipped silently. -func TestCollectMountRules_UnresolvableSymlinks_NoAllowGlobs(t *testing.T) { - dir := buildSymlinkTree(t) - - gpol := policy.Global{Mode: policy.ModeAll} - - buf := captureLogger(t) - - result := CollectMountRules(dir, gpol, policy.Container{}) - - if len(result.Errs) != 0 { - t.Fatalf("unexpected errors: %v", result.Errs) - } - - got := ruleSet(result.Rules) - _, hasNull := got["c 1:3"] - _, hasZero := got["c 1:5"] - - if len(got) != 2 || !hasNull || !hasZero { - t.Fatalf("expected {c 1:3, c 1:5}, got %v", got) - } - - // log, nullish (unresolvable), regular.txt (non-device). - if result.Skipped != 3 { - t.Errorf("expected Skipped=3, got %d", result.Skipped) - } - - logOutput := buf.String() - - warns := warnLines(logOutput) - if len(warns) != 0 { - t.Errorf("expected no WARN, got: %v", warns) - } - - if !strings.Contains(logOutput, "unresolvable symlink skipped") { - t.Errorf("expected DEBUG about unresolvable symlink, got: %s", logOutput) - } - - if !strings.Contains(logOutput, "non-device entry skipped") { - t.Errorf("expected DEBUG about non-device entry, got: %s", logOutput) - } -} - -// TestCollectMountRules_SingleFileNonDevice checks that an explicitly mounted -// single file that is not a device is still reported as an error. -func TestCollectMountRules_SingleFileNonDevice(t *testing.T) { - t.Parallel() - - file := filepath.Join(t.TempDir(), "regular.txt") - - err := os.WriteFile(file, []byte("not a device"), 0o600) - if err != nil { - t.Fatalf("create file: %v", err) - } - - result := CollectMountRules(file, policy.Global{Mode: policy.ModeAll}, policy.Container{}) - - if len(result.Rules) != 0 || len(result.Errs) != 1 { - t.Fatalf("expected one error and no rules, got rules=%v errs=%v", result.Rules, result.Errs) - } - - if !errors.Is(result.Errs[0], errNotDevice) { - t.Errorf("expected errNotDevice, got %v", result.Errs[0]) - } -} - // TestReconcile_DevDirectorySummary checks that a whole-/dev mount is // processed without a container-level error and emits one INFO summary. func TestReconcile_DevDirectorySummary(t *testing.T) { diff --git a/internal/processor/rules.go b/internal/processor/rules.go index 8cbe6a7..9b41db4 100644 --- a/internal/processor/rules.go +++ b/internal/processor/rules.go @@ -19,10 +19,11 @@ package processor import ( + "bufio" + "bytes" "errors" "fmt" "io/fs" - "os" "path/filepath" "strings" @@ -36,22 +37,65 @@ import ( const ( maxDirDepth = 8 maxLogChildren = 32 + // maxLinkHops bounds the absolute symlinks followed by hand (see + // evaluateCandidate), like the kernel's own symlink limit. + maxLinkHops = 40 deviceAccessAll = "rwm" ) -// errNotDevice is returned (wrapped) by getDeviceInfo when the path exists but -// is neither a character nor a block device. -var errNotDevice = errors.New("neither character nor block device") +var ( + // errNotDevice is returned (wrapped) for a mount source that exists but + // is neither a character nor a block device. + errNotDevice = errors.New("neither character nor block device") + // errUnresolved marks a candidate whose device identity could not be + // established; the container's whole device set is then unknown. + errUnresolved = errors.New("device identity could not be established") + // errBadDevname reports a sysfs uevent without exactly one valid DEVNAME. + errBadDevname = errors.New("uevent has no single valid DEVNAME") + // errTooManyLinks reports a chain of absolute symlinks over maxLinkHops. + errTooManyLinks = errors.New("too many levels of symbolic links") +) + +// Candidate skip reasons, also the sda_device_candidates_skipped_total +// reason label. +const ( + skipOutsideDev = "outside_dev" + skipDangling = "dangling" + skipNotDevice = "not_device" +) + +// decision is the policy outcome for one candidate. +type decision struct { + // deniedBy is the name that matched a deny glob; empty when none did. + deniedBy string + // authorized is set when the canonical and the resolved name both pass + // the allow lists. + authorized bool +} + +// candidate is one name found under a /dev mount (the mount source itself, +// or an entry of a directory mount) and what it resolved to. +type candidate struct { + alias string // the name as found + resolved string // the /dev path its descriptor refers to + canonical string // /dev/ + the device's sysfs DEVNAME + id deviceID + kind nodeKind + skip string // non-empty: not a device candidate, see skip reasons + decision decision +} -// MountResult is the outcome of collecting device rules for one mount source. -// Skipped counts entries that were intentionally not granted (excluded by -// policy, unresolvable symlinks, non-device entries); Errs holds failures for -// entries that should have produced a rule. +// MountResult is the outcome of collecting the candidates of one mount +// source. Skipped counts entries that name no device (dangling or outside +// /dev symlinks, non-device entries); Errs holds failures that leave the +// container's device set unknown. type MountResult struct { - Rules []cgroup.DeviceRule - Skipped int - Errs []error + Candidates []candidate + Skipped int + Errs []error + // Outcomes counts the skipped entries per skip reason. + Outcomes map[string]int } // IsMountSource reports whether path is /dev or a path under /dev/. @@ -59,99 +103,139 @@ func IsMountSource(path string) bool { return path == "/dev" || strings.HasPrefix(path, "/dev/") } -// CollectMountRules returns the DeviceRules, skipped count and errors for a -// mount source. Directory mounts are walked; per-file policy checks apply. +// underDev is the lexical gate on a name: already clean (no "..", "//" or +// trailing "/") and /dev or below. +func underDev(path string) bool { + return filepath.Clean(path) == path && IsMountSource(path) +} + +// relToDev returns path relative to /dev ("." for /dev itself). +func relToDev(path string) string { + if path == devRoot { + return "." + } + + return strings.TrimPrefix(path, devRoot+"/") +} + +// CollectMountRules evaluates the mount source and, for a directory, every +// entry below it. Each name goes through evaluateCandidate on descriptors; +// the walk only enumerates names. func CollectMountRules( + dev devFS, mountPath string, gpol policy.Global, cpol policy.Container, ) MountResult { - linfo, err := os.Lstat(mountPath) - if err != nil { - return MountResult{Errs: []error{fmt.Errorf("stat %q: %w", mountPath, err)}} - } + result := MountResult{Outcomes: make(map[string]int)} - effectivePath := mountPath - if linfo.Mode()&os.ModeSymlink != 0 { - resolved, resolveErr := filepath.EvalSymlinks(mountPath) - if resolveErr != nil { - return MountResult{ - Errs: []error{fmt.Errorf("resolve symlink %q: %w", mountPath, resolveErr)}, - } - } + source := evaluateCandidate(dev, mountPath, gpol, cpol) - effectivePath = resolved + switch { + case source.err != nil: + result.Errs = append(result.Errs, source.err) - linfo, err = os.Lstat(effectivePath) - if err != nil { - return MountResult{Errs: []error{fmt.Errorf("stat %q: %w", effectivePath, err)}} - } - } + return result + case source.skip == skipDangling: + // Docker mounted it, so it existed; a device that went away + // leaves the set unknown until it is back or the container ends. + result.Errs = append(result.Errs, + fmt.Errorf("%w: mount source %s: %w", errUnresolved, mountPath, unix.ENOENT)) - if !linfo.IsDir() { - if !gpol.DeviceAllowed(cpol, effectivePath) { - logger.L().Debug("device mount excluded by policy", "path", effectivePath) + return result + case source.skip != "": + result.addSkip(source.skip) - return MountResult{Skipped: 1} - } + return result + case source.kind == nodeDir: + return walkMount(dev, mountPath, source.resolved, gpol, cpol, result) + case source.kind == nodeOther: + // A single file mounted from /dev must be a device. + result.Errs = append(result.Errs, fmt.Errorf("device %q is %w", mountPath, errNotDevice)) - rule, ruleErr := collectDeviceRule(effectivePath) - if ruleErr != nil { - return MountResult{Errs: []error{ruleErr}} - } + return result + default: + result.Candidates = append(result.Candidates, source.candidate) - return MountResult{Rules: []cgroup.DeviceRule{rule}} + return result } +} + +func (r *MountResult) addSkip(reason string) { + r.Skipped++ + r.Outcomes[reason]++ +} +// walkMount enumerates a directory mount. dir is the resolved directory; the +// candidates are named after the mount source (their alias). +func walkMount( + dev devFS, + source, dir string, + gpol policy.Global, + cpol policy.Container, + result MountResult, +) MountResult { state := &mountWalkState{ - mountPath: effectivePath, - gpol: gpol, - cpol: cpol, + source: source, + base: filepath.Join(dev.root(), relToDev(dir)), + dev: dev, + gpol: gpol, + cpol: cpol, + result: result, } - walkErr := filepath.WalkDir(effectivePath, state.visitEntry) + walkErr := filepath.WalkDir(state.base, state.visitEntry) if walkErr != nil { - state.errs = append(state.errs, fmt.Errorf("walk %q: %w", effectivePath, walkErr)) + state.result.Errs = append(state.result.Errs, fmt.Errorf("walk %q: %w", source, walkErr)) } - if len(state.rules) == 0 && state.policyExcluded > 0 && len(state.errs) == 0 { + granted := 0 + + for idx := range state.result.Candidates { + verdict := state.result.Candidates[idx].decision + if verdict.deniedBy == "" && verdict.authorized { + granted++ + } + } + + if granted == 0 && state.policyExcluded > 0 && len(state.result.Errs) == 0 { logger.L().Warn("mount excluded: no children matched allow/deny policy", - "path", effectivePath, + "path", source, "children_seen", state.childrenSeen, "allow_globs", append(gpol.DeviceAllow, cpol.DeviceAllow...), "deny_globs", append(gpol.DeviceDeny, cpol.DeviceDeny...), ) } - return MountResult{Rules: state.rules, Skipped: state.skipped, Errs: state.errs} + return state.result } type mountWalkState struct { - mountPath string - gpol policy.Global - cpol policy.Container - rules []cgroup.DeviceRule - skipped int - // policyExcluded counts entries rejected by allow/deny globs (a subset of skipped). + source string + base string + dev devFS + gpol policy.Global + cpol policy.Container + result MountResult + // policyExcluded counts device candidates the policy did not grant. policyExcluded int - errs []error childrenSeen []string } func (s *mountWalkState) visitEntry(walkedPath string, entry fs.DirEntry, entryErr error) error { if entryErr != nil { - s.errs = append(s.errs, entryErr) + s.result.Errs = append(s.result.Errs, entryErr) return nil } if entry.IsDir() { - if walkedPath == s.mountPath { + if walkedPath == s.base { return nil } depth := strings.Count( - strings.TrimPrefix(walkedPath, s.mountPath), + strings.TrimPrefix(walkedPath, s.base), string(filepath.Separator), ) if depth > maxDirDepth { @@ -163,140 +247,363 @@ func (s *mountWalkState) visitEntry(walkedPath string, entry fs.DirEntry, entryE return nil } - if entry.Type()&os.ModeSymlink != 0 { - s.visitSymlink(walkedPath) + s.trackChild(walkedPath) - return nil - } + alias := s.source + strings.TrimPrefix(walkedPath, s.base) + + found := evaluateCandidate(s.dev, alias, s.gpol, s.cpol) - s.visitDeviceCandidate(walkedPath) + switch { + case found.err != nil: + s.result.Errs = append(s.result.Errs, found.err) + case found.skip != "": + if found.skip == skipDangling { + skipUnresolvable(alias, s.gpol, s.cpol) + } + + s.result.addSkip(found.skip) + case found.kind == nodeDir: + logger.L().Debug("symlink to directory skipped", "path", alias, "target", found.resolved) + s.result.addSkip(skipNotDevice) + case found.kind == nodeOther: + logger.L().Debug("non-device entry skipped", "path", alias) + s.result.addSkip(skipNotDevice) + default: + if found.decision.deniedBy != "" || !found.decision.authorized { + s.policyExcluded++ + } + + s.result.Candidates = append(s.result.Candidates, found.candidate) + } return nil } -// visitSymlink resolves a symlink found during a directory walk. A link that -// cannot be resolved is skipped, never an error: links leaving /dev (e.g. -// /dev/log -> /run/...) routinely dangle inside the daemon container. Policy -// globs still match the resolved target; the link path only decides whether -// the failure is worth a WARN (an explicit allow glob matches the link). -func (s *mountWalkState) visitSymlink(symlinkPath string) { - realPath, resolveErr := filepath.EvalSymlinks(symlinkPath) - if resolveErr != nil { - s.skipUnresolvable( - symlinkPath, - fmt.Errorf("resolve symlink %q: %w", symlinkPath, resolveErr), - ) +func (s *mountWalkState) trackChild(path string) { + if len(s.childrenSeen) < maxLogChildren { + s.childrenSeen = append(s.childrenSeen, filepath.Base(path)) + } +} - return +// evaluated is a candidate, or why it is not one. +type evaluated struct { + candidate + + // err is set when the name exists but its identity could not be + // established. + err error +} + +// evaluateCandidate establishes what alias names, on descriptors rather +// than re-walked paths, so an entry swapped between enumeration and use +// cannot lend its name to another device: +// +// 1. alias must be clean and under /dev; +// 2. it is opened beneath /dev (openat2 RESOLVE_BENEATH): symlinks are +// followed but nothing outside /dev is reached. A planted absolute link +// into /dev (-> /dev/sda) is followed by hand, bounded; any other escape +// is skipped as outside_dev, a dangling name as dangling; +// 3. fstat on the descriptor gives the type and major:minor; +// 4. sysfs gives the canonical name (DEVNAME); +// 5. policy is evaluated on the alias, the resolved and the canonical name +// (evaluateIdentity). +// +// A name that exists but whose identity cannot be established returns err: +// the caller must then treat the container's whole device set as unknown. +func evaluateCandidate( + dev devFS, + alias string, + gpol policy.Global, + cpol policy.Container, +) evaluated { + found := evaluated{candidate: candidate{alias: alias}} + + if !underDev(alias) { + logger.L(). + Warn("device path resolves outside /dev", "path", alias, "reason", skipOutsideDev) + found.skip = skipOutsideDev + + return found } - targetInfo, statErr := os.Stat(realPath) - if statErr != nil { - s.skipUnresolvable(symlinkPath, fmt.Errorf("stat symlink target %q: %w", realPath, statErr)) + node, skip, err := openCandidate(dev, alias) + if err != nil || skip != "" { + found.skip = skip + found.err = err - return + return found } - if targetInfo.IsDir() { - logger.L().Debug("symlink to directory skipped", "path", symlinkPath, "target", realPath) + defer func() { + closeErr := node.Close() + if closeErr != nil { + logger.L().Warn("close device node", "path", alias, "err", closeErr) + } + }() - s.skipped++ + found.kind, found.id, err = node.stat() + if err != nil { + found.err = unresolvedErr(alias, "", nil, err) - return + return found } - s.visitDeviceCandidate(realPath) + found.resolved = node.resolved() + if found.resolved == "" { + found.resolved = alias + } + + if found.kind != nodeChar && found.kind != nodeBlock { + return found + } + + found.canonical, err = canonicalName(dev, found.id) + if err == nil { + found.decision = evaluateIdentity(alias, found.resolved, found.canonical, gpol, cpol) + + return found + } + + // Without a canonical name the device can never be authorized, so the + // candidate only matters if its other names would grant it: then the + // set is unknown. Excluded by policy under those names, it is simply + // not granted. + found.decision = evaluateIdentity(alias, found.resolved, found.resolved, gpol, cpol) + if found.decision.deniedBy == "" && found.decision.authorized { + found.err = unresolvedErr(alias, found.resolved, &found.id, err) + } + + found.decision.authorized = false + + return found } -func (s *mountWalkState) skipUnresolvable(symlinkPath string, err error) { - s.skipped++ +// openCandidate opens alias beneath /dev, following a planted absolute +// symlink into /dev by hand (up to maxLinkHops). skip is set for a name +// that names nothing under /dev. +// +//nolint:ireturn // devNode is the devFS seam +func openCandidate(dev devFS, alias string) (devNode, string, error) { + rel := relToDev(alias) + + for range maxLinkHops { + node, err := dev.openBeneath(rel) + + switch { + case err == nil: + return node, "", nil + case errors.Is(err, unix.ENOENT): + return nil, skipDangling, nil + case !errors.Is(err, unix.EXDEV): + return nil, "", unresolvedErr(alias, "", nil, err) + } - if s.gpol.ExplicitlyAllowed(s.cpol, symlinkPath) { - logger.L().Warn("device symlink matches allow policy but cannot be resolved", - "path", symlinkPath, - "err", err, - ) + next, target, err := escapeHop(dev, rel) + if err != nil { + return nil, "", unresolvedErr(alias, "", nil, err) + } - return + if next == "" { + logger.L().Warn("device path resolves outside /dev", + "path", alias, "target", target, "reason", skipOutsideDev) + + return nil, skipOutsideDev, nil + } + + rel = next } - logger.L().Debug("unresolvable symlink skipped", "path", symlinkPath, "err", err) + return nil, "", unresolvedErr(alias, "", nil, errTooManyLinks) } -// visitDeviceCandidate applies policy to a walked entry (or resolved symlink -// target) and collects its device rule. Entries that pass policy but are not -// character/block devices are skipped; other stat failures are errors. -func (s *mountWalkState) visitDeviceCandidate(filePath string) { - s.trackChild(filePath) +// escapeHop handles a lookup of rel that left /dev (EXDEV). The kernel +// refuses an absolute symlink even when it points back into /dev, so the +// first component that escapes is found and its link read: an absolute +// target inside /dev replaces that prefix and next is the path to retry. +// next is empty when the escape really leaves /dev; target is then what +// it pointed at. +func escapeHop(dev devFS, rel string) (next, target string, err error) { + parts := strings.Split(rel, "/") + + for idx := range parts { + prefix := strings.Join(parts[:idx+1], "/") + + node, openErr := dev.openBeneath(prefix) + if openErr == nil { + closeErr := node.Close() + if closeErr != nil { + logger.L().Warn("close device node", "path", prefix, "err", closeErr) + } - if !s.gpol.DeviceAllowed(s.cpol, filePath) { - logger.L().Debug("device file excluded by policy", "path", filePath) + continue + } - s.skipped++ - s.policyExcluded++ + if !errors.Is(openErr, unix.EXDEV) { + return "", "", openErr + } - return + target, err = dev.readLink(prefix) + if err != nil { + return "", "", err + } + + cleaned := filepath.Clean(target) + if !filepath.IsAbs(target) || !underDev(cleaned) { + return "", target, nil + } + + return filepath.Join(append([]string{relToDev(cleaned)}, parts[idx+1:]...)...), target, nil } - rule, ruleErr := collectDeviceRule(filePath) - if errors.Is(ruleErr, errNotDevice) { - logger.L().Debug("non-device entry skipped", "path", filePath) + // Every prefix opens on its own: the whole name no longer escapes. + return rel, "", nil +} - s.skipped++ +// canonicalName reads the device's DEVNAME from sysfs. The uevent must hold +// exactly one DEVNAME line whose value is a clean relative path without +// "..". +func canonicalName(dev devFS, id deviceID) (string, error) { + data, err := dev.readUevent(id) + if err != nil { + return "", err + } - return + var names []string + + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + if value, ok := strings.CutPrefix(scanner.Text(), "DEVNAME="); ok { + names = append(names, value) + } } - if ruleErr != nil { - s.errs = append(s.errs, fmt.Errorf("device rule for %q: %w", filePath, ruleErr)) + if len(names) != 1 { + return "", fmt.Errorf("%w: %d DEVNAME lines", errBadDevname, len(names)) + } - return + name := names[0] + if name == "" || filepath.IsAbs(name) || filepath.Clean(name) != name || + name == ".." || strings.HasPrefix(name, "../") { + return "", fmt.Errorf("%w: %q", errBadDevname, name) } - s.rules = append(s.rules, rule) + return devRoot + "/" + name, nil } -func (s *mountWalkState) trackChild(path string) { - if len(s.childrenSeen) < maxLogChildren { - s.childrenSeen = append(s.childrenSeen, filepath.Base(path)) +// evaluateIdentity applies policy to one candidate. Deny globs are checked +// on every name the device was found by: any match is a deny vote. The +// allow lists must pass for the canonical and the resolved name; an alias +// missing them is not a vote either way (a by-path or by-id link is not +// what authorizes a device). +func evaluateIdentity( + alias, resolved, canonical string, + gpol policy.Global, + cpol policy.Container, +) decision { + for _, name := range []string{alias, resolved, canonical} { + if gpol.Denied(cpol, name) { + return decision{deniedBy: name} + } } + + return decision{authorized: gpol.Authorized(cpol, canonical) && gpol.Authorized(cpol, resolved)} } -// collectDeviceRule returns the DeviceRule for a single (non-directory) device file. -func collectDeviceRule(devicePath string) (cgroup.DeviceRule, error) { - deviceType, major, minor, err := getDeviceInfo(devicePath) - if err != nil { - return cgroup.DeviceRule{}, err +// unresolvedErr names a candidate whose identity could not be established. +func unresolvedErr(alias, resolved string, id *deviceID, cause error) error { + switch { + case id != nil: + return fmt.Errorf("%w: %s (resolved %s, %s): %w", errUnresolved, alias, resolved, id, cause) + default: + return fmt.Errorf("%w: %s: %w", errUnresolved, alias, cause) + } +} + +// skipUnresolvable logs a dangling name: a WARN when an explicit allow glob +// names it, since the operator expected a device there. +func skipUnresolvable(alias string, gpol policy.Global, cpol policy.Container) { + if gpol.ExplicitlyAllowed(cpol, alias) { + logger.L().Warn("device symlink matches allow policy but cannot be resolved", "path", alias) + + return } - return cgroup.DeviceRule{ - Allow: true, - Access: deviceAccessAll, - Type: deviceType, - Major: &major, - Minor: &minor, - }, nil + logger.L().Debug("unresolvable symlink skipped", "path", alias) +} + +// deviceVotes aggregates every candidate that resolved to one device. +type deviceVotes struct { + id deviceID + candidates int + authorized bool + deniedBy string } -func getDeviceInfo(devicePath string) (deviceType string, major, minor int64, statErr error) { - var stat unix.Stat_t +// aggregation collects candidates per device across all mounts of a +// container, so one name's deny vote holds against every other name of the +// same device. +type aggregation struct { + order []deviceID + votes map[deviceID]*deviceVotes +} - statErr = unix.Stat(devicePath, &stat) - if statErr != nil { - return "", -1, -1, fmt.Errorf("stat %q: %w", devicePath, statErr) +func (a *aggregation) add(found *candidate) { + if a.votes == nil { + a.votes = make(map[deviceID]*deviceVotes) } - switch stat.Mode & unix.S_IFMT { - case unix.S_IFBLK: - deviceType = "b" - case unix.S_IFCHR: - deviceType = "c" - default: - return "", -1, -1, fmt.Errorf("device %q is %w", devicePath, errNotDevice) + votes, ok := a.votes[found.id] + if !ok { + votes = &deviceVotes{id: found.id} + a.votes[found.id] = votes + a.order = append(a.order, found.id) } - major = int64(unix.Major(stat.Rdev)) - minor = int64(unix.Minor(stat.Rdev)) + votes.candidates++ + + if found.decision.deniedBy != "" && votes.deniedBy == "" { + votes.deniedBy = found.decision.deniedBy + } + + if found.decision.authorized { + votes.authorized = true + } +} + +// emit returns the rules for every device that is authorized and has no +// deny vote, in the order devices were first seen, and how many candidates +// were not granted. +func (a *aggregation) emit(containerID string) (rules []cgroup.DeviceRule, skipped int) { + for _, device := range a.order { + votes := a.votes[device] + + switch { + case votes.deniedBy != "" && votes.authorized: + logger.L().Info("device denied by policy under one of its names", + "id", containerID, "device", device.String(), "denied_by", votes.deniedBy) + + skipped += votes.candidates + case votes.deniedBy != "": + logger.L().Debug("device excluded by policy", + "id", containerID, "device", device.String(), "denied_by", votes.deniedBy) + + skipped += votes.candidates + case !votes.authorized: + logger.L().Debug("device not in the allow list", + "id", containerID, "device", device.String()) + + skipped += votes.candidates + default: + major, minor := device.major, device.minor + rules = append(rules, cgroup.DeviceRule{ + Allow: true, + Access: deviceAccessAll, + Type: device.typ, + Major: &major, + Minor: &minor, + }) + } + } - return deviceType, major, minor, nil + return rules, skipped } diff --git a/internal/processor/rules_test.go b/internal/processor/rules_test.go new file mode 100644 index 0000000..87ac6f6 --- /dev/null +++ b/internal/processor/rules_test.go @@ -0,0 +1,879 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package processor + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/mount" + "golang.org/x/sys/unix" + + "github.com/leinardi/swarm-device-access/internal/cgroup" + "github.com/leinardi/swarm-device-access/internal/config" + "github.com/leinardi/swarm-device-access/internal/policy" +) + +// fakeDevFS scripts what each name under /dev resolves to, as openat2 +// beneath /dev would. Every registered name also exists as a file (or a +// directory) under dir, so directory mounts can be enumerated. +type fakeDevFS struct { + t *testing.T + dir string + nodes map[string]fakeNode + errs map[string]error + links map[string]string + uevents map[deviceID]string +} + +type fakeNode struct { + kind nodeKind + id deviceID + path string // the resolved /dev path +} + +func (fakeNode) Close() error { return nil } +func (n fakeNode) stat() (nodeKind, deviceID, error) { return n.kind, n.id, nil } +func (n fakeNode) resolved() string { return n.path } + +func newFakeDevFS(t *testing.T) *fakeDevFS { + t.Helper() + + return &fakeDevFS{ + t: t, + dir: t.TempDir(), + nodes: make(map[string]fakeNode), + errs: make(map[string]error), + links: make(map[string]string), + uevents: make(map[deviceID]string), + } +} + +func (*fakeDevFS) Close() error { return nil } +func (f *fakeDevFS) root() string { return f.dir } + +//nolint:ireturn // the devFS seam returns the interface +func (f *fakeDevFS) openBeneath(rel string) (devNode, error) { + // A component that fails the lookup fails every name below it. + parts := strings.Split(rel, "/") + for idx := range parts { + if err, ok := f.errs[strings.Join(parts[:idx+1], "/")]; ok { + return nil, err + } + } + + node, ok := f.nodes[rel] + if !ok { + return nil, fmt.Errorf("openat2 %s: %w", rel, unix.ENOENT) + } + + return node, nil +} + +func (f *fakeDevFS) readLink(rel string) (string, error) { + target, ok := f.links[rel] + if !ok { + return "", unix.EINVAL + } + + return target, nil +} + +func (f *fakeDevFS) readUevent(id deviceID) ([]byte, error) { + data, ok := f.uevents[id] + if !ok { + return nil, fmt.Errorf("read uevent %s: %w", id, unix.ENOENT) + } + + return []byte(data), nil +} + +// touch creates the name under dir so a walk enumerates it. +func (f *fakeDevFS) touch(rel string) { + f.t.Helper() + + path := filepath.Join(f.dir, rel) + + err := os.MkdirAll(filepath.Dir(path), 0o755) + if err == nil { + err = os.WriteFile(path, nil, 0o600) + } + + if err != nil { + f.t.Fatal(err) + } +} + +// device registers a device node at rel with its sysfs DEVNAME. +func (f *fakeDevFS) device(rel string, id deviceID, devname string) { + f.touch(rel) + f.nodes[rel] = fakeNode{kind: kindOf(id), id: id, path: devRoot + "/" + rel} + f.uevents[id] = "MAJOR=" + strconv.FormatInt(id.major, 10) + "\nDEVNAME=" + devname + "\n" +} + +// alias registers a symlink at rel the kernel follows (relative, inside +// /dev) to the node registered at target. +func (f *fakeDevFS) alias(rel, target string) { + f.touch(rel) + f.nodes[rel] = f.nodes[target] +} + +// absLink registers an absolute symlink at rel: openat2 beneath /dev +// refuses it with EXDEV, and readLink returns target. +func (f *fakeDevFS) absLink(rel, target string) { + f.touch(rel) + f.errs[rel] = fmt.Errorf("openat2 %s: %w", rel, unix.EXDEV) + f.links[rel] = target +} + +// directory registers rel as a directory. +func (f *fakeDevFS) directory(rel string) { + f.t.Helper() + + err := os.MkdirAll(filepath.Join(f.dir, rel), 0o755) + if err != nil { + f.t.Fatal(err) + } + + f.nodes[rel] = fakeNode{kind: nodeDir, path: devRoot + "/" + rel} +} + +func kindOf(id deviceID) nodeKind { + if id.typ == "b" { + return nodeBlock + } + + return nodeChar +} + +var ( + devSda = deviceID{typ: "b", major: 8, minor: 0} + devCard = deviceID{typ: "c", major: 226, minor: 0} + devRend = deviceID{typ: "c", major: 226, minor: 128} + devNull = deviceID{typ: "c", major: 1, minor: 3} +) + +func collectFrom(dev devFS, gpol policy.Global, sources ...string) containerRules { + mounts := make([]container.MountPoint, 0, len(sources)) + for _, source := range sources { + mounts = append( + mounts, + container.MountPoint{Source: source, Destination: source, Type: mount.TypeBind}, + ) + } + + return collectContainerRules(dev, "abc", 1, mounts, gpol, policy.Container{}, nil) +} + +func ruleSet(rules []cgroup.DeviceRule) map[string]struct{} { + set := make(map[string]struct{}, len(rules)) + for _, rule := range rules { + set[rule.Type+" "+strconv.FormatInt(*rule.Major, 10)+":"+strconv.FormatInt(*rule.Minor, 10)] = struct{}{} + } + + return set +} + +func warnLines(logOutput string) []string { + var lines []string + + for line := range strings.SplitSeq(logOutput, "\n") { + if strings.Contains(line, "level=WARN") { + lines = append(lines, line) + } + } + + return lines +} + +// wantRules fails unless the collection has no errors and grants exactly +// the listed devices. +func wantRules(t *testing.T, got containerRules, want ...deviceID) { + t.Helper() + + if len(got.deviceErrs) != 0 { + t.Fatalf("unexpected errors: %v", got.deviceErrs) + } + + set := ruleSet(got.granted) + if len(set) != len(want) || len(got.granted) != len(want) { + t.Fatalf("granted %v, want %v", set, want) + } + + for _, id := range want { + if _, ok := set[id.typ+" "+strconv.FormatInt(id.major, 10)+":"+strconv.FormatInt(id.minor, 10)]; !ok { + t.Fatalf("granted %v, want %v", set, want) + } + } + + for _, rule := range got.granted { + if !rule.Allow || rule.Access != "rwm" { + t.Errorf("rule has unexpected allow/access: %+v", rule) + } + } +} + +// wantUnresolved fails unless the collection reports an unresolved identity +// naming every fragment; computeDesired then empties the whole set (see +// TestProcessor_UnresolvedCandidateEmptiesTheSet). +func wantUnresolved(t *testing.T, got containerRules, fragments ...string) { + t.Helper() + + if len(got.deviceErrs) == 0 { + t.Fatal("want an unresolved-identity error, got none") + } + + joined := errors.Join(got.deviceErrs...) + if !errors.Is(joined, errUnresolved) { + t.Errorf("err = %v, want errUnresolved", joined) + } + + for _, fragment := range fragments { + if !strings.Contains(joined.Error(), fragment) { + t.Errorf("err = %v, want it to contain %q", joined, fragment) + } + } +} + +func TestEvaluateIdentity(t *testing.T) { + t.Parallel() + + const ( + alias = "/dev/disk/by-id/usb-x" + resolved = "/dev/sda" + canonical = "/dev/sda" + ) + + for _, tc := range []struct { + name string + gpol policy.Global + cpol policy.Container + want decision + }{ + {"no policy", policy.Global{}, policy.Container{}, decision{authorized: true}}, + { + "alias denied", + policy.Global{DeviceDeny: []string{"/dev/disk/by-id/*"}}, + policy.Container{}, + decision{deniedBy: alias}, + }, + { + "canonical denied by the container", + policy.Global{}, + policy.Container{DeviceDeny: []string{"/dev/sd*"}}, + decision{deniedBy: resolved}, + }, + { + "alias allowed but not the device", + policy.Global{DeviceAllow: []string{"/dev/disk/by-id/*"}}, + policy.Container{}, + decision{}, + }, + { + "device allowed, alias not listed", + policy.Global{DeviceAllow: []string{"/dev/sda"}}, + policy.Container{}, + decision{authorized: true}, + }, + { + "container allow list narrows", + policy.Global{DeviceAllow: []string{"/dev/sd*"}}, + policy.Container{DeviceAllow: []string{"/dev/sdb"}}, + decision{}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got := evaluateIdentity(alias, resolved, canonical, tc.gpol, tc.cpol) + if got != tc.want { + t.Errorf("decision = %+v, want %+v", got, tc.want) + } + }) + } + + // Authorized needs both the canonical and the resolved name. + got := evaluateIdentity("/dev/x", "/dev/dri/card0", "/dev/sda", + policy.Global{DeviceAllow: []string{"/dev/dri/*"}}, policy.Container{}) + if got.authorized { + t.Error("authorized on the resolved name alone") + } +} + +func TestCanonicalName(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name, uevent, want string + }{ + {"plain", "MAJOR=8\nDEVNAME=sda\nDEVTYPE=disk\n", "/dev/sda"}, + {"nested", "DEVNAME=dri/card0\n", "/dev/dri/card0"}, + {"missing", "MAJOR=8\n", ""}, + {"two lines", "DEVNAME=sda\nDEVNAME=sdb\n", ""}, + {"empty", "DEVNAME=\n", ""}, + {"absolute", "DEVNAME=/dev/sda\n", ""}, + {"dot-dot", "DEVNAME=../sda\n", ""}, + {"inner dot-dot", "DEVNAME=dri/../../sda\n", ""}, + {"unclean", "DEVNAME=dri//card0\n", ""}, + {"trailing slash", "DEVNAME=sda/\n", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.uevents[devSda] = tc.uevent + + got, err := canonicalName(dev, devSda) + if tc.want == "" { + if !errors.Is(err, errBadDevname) { + t.Errorf("canonicalName = %q, %v; want errBadDevname", got, err) + } + + return + } + + if err != nil || got != tc.want { + t.Errorf("canonicalName = %q, %v; want %q", got, err, tc.want) + } + }) + } +} + +// A node planted under /dev/shm by anyone who can write there is judged by +// its sysfs identity, not by where it was planted. +func TestCollect_PlantedNodeUnderShmJudgedAsSda(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.directory("shm") + dev.device("shm/evil", devSda, "sda") + + allowShm := policy.Global{Mode: policy.ModeAll, DeviceAllow: []string{"/dev/shm/*"}} + wantRules(t, collectFrom(dev, allowShm, "/dev/shm")) + + denySda := policy.Global{Mode: policy.ModeAll, DeviceDeny: []string{"/dev/sda"}} + wantRules(t, collectFrom(dev, denySda, "/dev/shm")) +} + +func TestCollect_SymlinkOutsideDevRejected(t *testing.T) { + dev := newFakeDevFS(t) + dev.directory("dri") + dev.absLink("dri/x", "/tmp/x") + dev.absLink("dri/up", "/dev/../tmp/x") + + buf := captureLogger(t) + + got := collectFrom(dev, policy.Global{Mode: policy.ModeAll}, "/dev/dri") + wantRules(t, got) + + if got.skipped != 2 { + t.Errorf("skipped = %d, want 2", got.skipped) + } + + if strings.Count(buf.String(), "device path resolves outside /dev") != 2 || + !strings.Contains(buf.String(), "reason=outside_dev") { + t.Errorf("want one outside_dev WARN per link, got:\n%s", buf.String()) + } +} + +func TestCollect_RelativeSymlinkPolicyOnAllThreeNames(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.device("sda", devSda, "sda") + dev.directory("disk/by-id") + dev.alias("disk/by-id/usb-x", "sda") // ../../sda + + const source = "/dev/disk/by-id/usb-x" + + wantRules(t, collectFrom(dev, policy.Global{Mode: policy.ModeAll}, source), devSda) + + for _, deny := range []string{"/dev/disk/by-id/*", "/dev/sda"} { + gpol := policy.Global{Mode: policy.ModeAll, DeviceDeny: []string{deny}} + wantRules(t, collectFrom(dev, gpol, source)) + } + + aliasOnly := policy.Global{Mode: policy.ModeAll, DeviceAllow: []string{"/dev/disk/by-id/*"}} + wantRules(t, collectFrom(dev, aliasOnly, source)) + + canonical := policy.Global{Mode: policy.ModeAll, DeviceAllow: []string{"/dev/sda"}} + wantRules(t, collectFrom(dev, canonical, source), devSda) +} + +// A deny on one name holds against the device under every other name, even +// one that is walked separately and allowed. +func TestCollect_AliasDenySuppressesDeviceAcrossMounts(t *testing.T) { + dev := newFakeDevFS(t) + dev.device("sda", devSda, "sda") + dev.directory("disk/by-id") + dev.alias("disk/by-id/usb-x", "sda") + + gpol := policy.Global{ + Mode: policy.ModeAll, + DeviceAllow: []string{"/dev/sda"}, + DeviceDeny: []string{"/dev/disk/by-id/*"}, + } + + buf := captureLogger(t) + + for _, order := range [][]string{ + {"/dev/sda", "/dev/disk/by-id"}, + {"/dev/disk/by-id", "/dev/sda"}, + } { + got := collectFrom(dev, gpol, order...) + wantRules(t, got) + + if got.skipped != 2 { + t.Errorf("%v: skipped = %d, want both candidates", order, got.skipped) + } + } + + if !strings.Contains(buf.String(), "device denied by policy under one of its names") || + !strings.Contains(buf.String(), "denied_by=/dev/disk/by-id/usb-x") { + t.Errorf("want an INFO naming the denying alias, got:\n%s", buf.String()) + } +} + +func TestCollect_ByPathAliasGrantedByCanonical(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.directory("dri") + dev.device("dri/card0", devCard, "dri/card0") + dev.device("dri/renderD128", devRend, "dri/renderD128") + dev.directory("dri/by-path") + dev.alias("dri/by-path/pci-0000:00:02.0-card", "dri/card0") + + gpol := policy.Global{Mode: policy.ModeAll, DeviceAllow: []string{"/dev/dri/*"}} + + got := collectFrom(dev, gpol, "/dev/dri") + wantRules(t, got, devCard, devRend) + + if got.skipped != 0 { + t.Errorf("skipped = %d, want 0", got.skipped) + } +} + +// openat2 refuses an absolute symlink even into /dev; the retry follows it +// by hand so a planted /dev/evil -> /dev/sda is still judged under both +// names. +func TestCollect_AbsoluteSymlinkUnderDeniedAliasVetoed(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.device("sda", devSda, "sda") + dev.directory("evil") + dev.absLink("evil/link", "/dev/sda") + + gpol := policy.Global{Mode: policy.ModeAll, DeviceDeny: []string{"/dev/evil/*"}} + wantRules(t, collectFrom(dev, gpol, "/dev/sda", "/dev/evil")) + + wantRules(t, collectFrom(dev, policy.Global{Mode: policy.ModeAll}, "/dev/evil"), devSda) +} + +// A directory mounted through an absolute symlink into /dev is walked, and +// each entry is still followed through the link and judged under its alias. +func TestCollect_DirectoryBehindAbsoluteSymlink(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.directory("dri") + dev.device("dri/card0", devCard, "dri/card0") + dev.absLink("gpu", "/dev/dri") + + allow := policy.Global{Mode: policy.ModeAll, DeviceAllow: []string{"/dev/dri/*"}} + wantRules(t, collectFrom(dev, allow, "/dev/gpu"), devCard) + + deny := policy.Global{Mode: policy.ModeAll, DeviceDeny: []string{"/dev/gpu/*"}} + wantRules(t, collectFrom(dev, deny, "/dev/gpu")) +} + +func TestCollect_AbsoluteSymlinkLoopUnresolved(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.absLink("a", "/dev/b") + dev.absLink("b", "/dev/a") + + wantUnresolved(t, collectFrom(dev, policy.Global{Mode: policy.ModeAll}, "/dev/a"), + "/dev/a", "too many levels") +} + +func TestCollect_DotDotSourceRejected(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.device("sda", devSda, "sda") + + for _, source := range []string{"/dev/../etc/passwd", "/dev//sda", "/dev/sda/"} { + got := collectFrom(dev, policy.Global{Mode: policy.ModeAll}, source) + wantRules(t, got) + + if got.skipped != 1 { + t.Errorf("%s: skipped = %d, want 1", source, got.skipped) + } + } +} + +func TestCollect_UeventProblemsEmptyTheSet(t *testing.T) { + t.Parallel() + + for name, uevent := range map[string]string{ + "missing": "", + "no DEVNAME": "MAJOR=8\n", + "two DEVNAME": "DEVNAME=sda\nDEVNAME=sdb\n", + "DEVNAME=../sda": "DEVNAME=../sda\n", + "absolute": "DEVNAME=/dev/sda\n", + "empty DEVNAME": "DEVNAME=\n", + "unclean DEVNAME": "DEVNAME=./sda\n", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.device("null", devNull, "null") + dev.device("sda", devSda, "sda") + + if uevent == "" { + delete(dev.uevents, devSda) + } else { + dev.uevents[devSda] = uevent + } + + got := collectFrom(dev, policy.Global{Mode: policy.ModeAll}, "/dev/null", "/dev/sda") + wantUnresolved(t, got, "/dev/sda", "b 8:0") + }) + } +} + +// Without a canonical name a device cannot be authorized; when its other +// names already exclude it the set is still known. +func TestCollect_UeventMissingButExcludedByPolicy(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.directory("pts") + dev.device("pts/0", deviceID{typ: "c", major: 136, minor: 0}, "") + delete(dev.uevents, deviceID{typ: "c", major: 136, minor: 0}) + dev.device("null", devNull, "null") + + notAllowed := policy.Global{Mode: policy.ModeAll, DeviceAllow: []string{"/dev/null"}} + wantRules(t, collectFrom(dev, notAllowed, "/dev/null", "/dev/pts"), devNull) + + denied := policy.Global{Mode: policy.ModeAll, DeviceDeny: []string{"/dev/pts/*"}} + wantRules(t, collectFrom(dev, denied, "/dev/null", "/dev/pts"), devNull) + + wantUnresolved( + t, + collectFrom(dev, policy.Global{Mode: policy.ModeAll}, "/dev/pts"), + "/dev/pts/0", + ) +} + +// The walk only enumerates names: an entry swapped after enumeration is +// judged by what its descriptor is, not by the name it had. +func TestCollect_EntryReplacedAfterEnumerationFdIdentityWins(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.device("sda", devSda, "sda") + dev.directory("dri") + dev.touch("dri/card0") + // By the time it is opened, dri/card0 is sda. + dev.nodes["dri/card0"] = fakeNode{kind: nodeBlock, id: devSda, path: "/dev/dri/card0"} + + gpol := policy.Global{Mode: policy.ModeAll, DeviceAllow: []string{"/dev/dri/*"}} + wantRules(t, collectFrom(dev, gpol, "/dev/dri")) + + all := policy.Global{Mode: policy.ModeAll} + got := collectFrom(dev, all, "/dev/dri") + wantRules(t, got, devSda) +} + +func TestCollect_OtherOpenErrorUnresolved(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.directory("dri") + dev.touch("dri/card0") + dev.errs["dri/card0"] = fmt.Errorf("openat2 dri/card0: %w", unix.EACCES) + + wantUnresolved( + t, + collectFrom(dev, policy.Global{Mode: policy.ModeAll}, "/dev/dri"), + "/dev/dri/card0", + ) +} + +func TestCollect_MissingSourceUnresolved(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + + wantUnresolved( + t, + collectFrom(dev, policy.Global{Mode: policy.ModeAll}, "/dev/ttyUSB0"), + "/dev/ttyUSB0", + ) +} + +func TestCollect_SingleFileNonDevice(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.touch("regular") + dev.nodes["regular"] = fakeNode{kind: nodeOther, path: "/dev/regular"} + + got := collectFrom(dev, policy.Global{Mode: policy.ModeAll}, "/dev/regular") + if len(got.granted) != 0 || len(got.deviceErrs) != 1 || + !errors.Is(got.deviceErrs[0], errNotDevice) { + t.Fatalf("want one errNotDevice, got rules=%v errs=%v", got.granted, got.deviceErrs) + } +} + +// Dangling names and non-device entries of a directory mount are skipped, +// never errors; only a dangling name an explicit allow glob matches WARNs. +func TestCollect_DirectorySkips(t *testing.T) { + dev := newFakeDevFS(t) + dev.directory("d") + dev.device("null", devNull, "null") + dev.alias("d/null", "null") + dev.touch("d/nullish") // dangling: not registered + dev.touch("d/regular") + dev.nodes["d/regular"] = fakeNode{kind: nodeOther, path: "/dev/d/regular"} + dev.directory("d/sub") + dev.touch("d/linktodir") + dev.nodes["d/linktodir"] = fakeNode{kind: nodeDir, path: "/dev/d/sub"} + + buf := captureLogger(t) + + gpol := policy.Global{ + Mode: policy.ModeAll, + DeviceAllow: []string{"/dev/null", "/dev/d/nullish"}, + } + + got := collectFrom(dev, gpol, "/dev/d") + wantRules(t, got, devNull) + + if got.skipped != 3 { + t.Errorf("skipped = %d, want nullish, regular and linktodir", got.skipped) + } + + warns := warnLines(buf.String()) + if len(warns) != 1 || + !strings.Contains(warns[0], "device symlink matches allow policy but cannot be resolved") || + !strings.Contains(warns[0], "path=/dev/d/nullish") { + t.Errorf("want one WARN for nullish, got %v", warns) + } + + for _, debug := range []string{"non-device entry skipped", "symlink to directory skipped"} { + if !strings.Contains(buf.String(), debug) { + t.Errorf("want DEBUG %q, got:\n%s", debug, buf.String()) + } + } +} + +func TestCollect_DirectoryNoChildrenMatchWarns(t *testing.T) { + dev := newFakeDevFS(t) + dev.directory("dri") + dev.device("dri/card0", devCard, "dri/card0") + dev.device("dri/renderD128", devRend, "dri/renderD128") + + buf := captureLogger(t) + + gpol := policy.Global{Mode: policy.ModeAll, DeviceAllow: []string{"/dev/dri/nonexistent"}} + wantRules(t, collectFrom(dev, gpol, "/dev/dri")) + + logOutput := buf.String() + if !strings.Contains(logOutput, "mount excluded: no children matched") || + !strings.Contains(logOutput, "card0") || !strings.Contains(logOutput, "renderD128") { + t.Errorf("want a WARN naming the children, got:\n%s", logOutput) + } +} + +func TestProcessor_DevFSOpenFailureEmptiesTheSet(t *testing.T) { + t.Parallel() + + proc := &Processor{devfs: func() (devFS, error) { return nil, unix.ENOSYS }} + info := &container.InspectResponse{ + State: &container.State{Running: true, Pid: 1}, + Mounts: []container.MountPoint{{Source: "/dev/null", Destination: "/dev/null"}}, + } + + desired := proc.computeDesired( + "abc", + info, + config.Runtime{Policy: policy.Global{Mode: policy.ModeAll}}, + ) + if len(desired.rules) != 0 || desired.incomplete == nil || + !strings.Contains(desired.incomplete.Error(), "retryable") { + t.Errorf("desired = %+v, want an empty set and a retryable error", desired) + } +} + +func TestProcessor_UnresolvedCandidateEmptiesTheSet(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.device("null", devNull, "null") + dev.device("sda", devSda, "sda") + delete(dev.uevents, devSda) + + proc := &Processor{devfs: func() (devFS, error) { return dev, nil }} + info := &container.InspectResponse{ + State: &container.State{Running: true, Pid: 1}, + Mounts: []container.MountPoint{ + {Source: "/dev/null", Destination: "/dev/null"}, + {Source: "/dev/sda", Destination: "/dev/sda"}, + }, + } + + desired := proc.computeDesired( + "abc", + info, + config.Runtime{Policy: policy.Global{Mode: policy.ModeAll}}, + ) + if len(desired.rules) != 0 || desired.incomplete == nil { + t.Fatalf("desired = %+v, want an empty set and an error", desired) + } + + msg := desired.incomplete.Error() + for _, fragment := range []string{"incomplete_device_set, retryable", "/dev/sda", "b 8:0"} { + if !strings.Contains(msg, fragment) { + t.Errorf("err = %q, want %q", msg, fragment) + } + } +} + +func TestProbeOpenat2(t *testing.T) { + t.Parallel() + + err := ProbeOpenat2() + if err != nil && !errors.Is(err, errOpenat2Unsupported) { + t.Errorf("ProbeOpenat2 = %v, want nil or errOpenat2Unsupported", err) + } +} + +// realDev opens the host's /dev for tests that need a real device node. +func realDev(t *testing.T) *realDevFS { + t.Helper() + + err := ProbeOpenat2() + if err != nil { + t.Skipf("openat2 unavailable: %v", err) + } + + dev, err := openDevFS(devRoot, "/sys") + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { _ = dev.Close() }) + + return dev +} + +func TestCollect_RealDevNull(t *testing.T) { + t.Parallel() + + dev := realDev(t) + + wantRules(t, collectFrom(dev, policy.Global{Mode: policy.ModeAll}, "/dev/null"), devNull) + + denied := policy.Global{Mode: policy.ModeAll, DeviceDeny: []string{"/dev/null"}} + + got := collectFrom(dev, denied, "/dev/null") + wantRules(t, got) + + if got.skipped != 1 { + t.Errorf("skipped = %d, want 1", got.skipped) + } +} + +// TestRealDevFS_Openat2Containment runs the real openat2 against regular +// files: RESOLVE_BENEATH refuses an absolute symlink and a relative one that +// climbs out, and readLink returns the link itself. +func TestRealDevFS_Openat2Containment(t *testing.T) { + t.Parallel() + + err := ProbeOpenat2() + if err != nil { + t.Skipf("openat2 unavailable: %v", err) + } + + outside := filepath.Join(t.TempDir(), "secret") + root := t.TempDir() + + for _, step := range []error{ + os.WriteFile(outside, nil, 0o600), + os.WriteFile(filepath.Join(root, "inside"), nil, 0o600), + os.Symlink(outside, filepath.Join(root, "abs")), + os.Symlink("../"+filepath.Base(filepath.Dir(outside))+"/secret", filepath.Join(root, "climb")), + os.Symlink("inside", filepath.Join(root, "rel")), + } { + if step != nil { + t.Fatal(step) + } + } + + dev, err := openDevFS(root, "/sys") + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { _ = dev.Close() }) + + for _, name := range []string{"abs", "climb"} { + node, openErr := dev.openBeneath(name) + if !errors.Is(openErr, unix.EXDEV) { + if node != nil { + _ = node.Close() + } + + t.Errorf("openBeneath(%s) = %v, want EXDEV", name, openErr) + } + } + + target, err := dev.readLink("abs") + if err != nil || target != outside { + t.Errorf("readLink(abs) = %q, %v; want %q", target, err, outside) + } + + node, err := dev.openBeneath("rel") + if err != nil { + t.Fatal(err) + } + + defer func() { _ = node.Close() }() + + kind, _, err := node.stat() + if err != nil || kind != nodeOther { + t.Errorf("stat = %v, %v; want a regular file", kind, err) + } + + if node.resolved() != devRoot+"/inside" { + t.Errorf("resolved = %q, want /dev/inside", node.resolved()) + } +} From 63b7ad915b00415d78f5e754f63f3a552aa8734e Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 11:42:14 +0200 Subject: [PATCH 14/37] fix(processor): empty the device set when a mount cannot be enumerated completely A directory mount is walked only to enumerate names, and every name goes through evaluateCandidate. A root that cannot be opened, a directory that cannot be read, any walk error, or more than 4096 entries in one mount (mount too large; narrow the bind mount) now makes the container's whole desired set empty with a retryable error, like an unresolved candidate: an alias that was not seen might deny a device that was. The silent depth cap is gone and there is no subtree exception. The enforcement test now creates, inside the opted-in container, a node with the granted device's major and the next minor and one of the other device type, and requires both to stay denied. README, SECURITY.md and docs/architecture.md state that the canonical identity is the security authority, that alias-level deny is a best-effort veto, recommend deny globs on kernel names such as /dev/sd* and /dev/nvme*, and explain how to see a device that lacks DEVNAME and why whole-/dev mounts can hit the entry cap. The trust-boundary skill records the containment and planted-node gaps as closed. --- .agents/skills/trust-boundary/SKILL.md | 63 ++++++------ README.md | 27 ++++- SECURITY.md | 12 +++ docs/architecture.md | 10 +- internal/processor/devfs.go | 10 ++ internal/processor/rules.go | 50 ++++++---- internal/processor/rules_test.go | 131 +++++++++++++++++++++++-- test/integration/enforce_test.go | 58 ++++++++++- 8 files changed, 300 insertions(+), 61 deletions(-) diff --git a/.agents/skills/trust-boundary/SKILL.md b/.agents/skills/trust-boundary/SKILL.md index 10b3e5d..9cac854 100644 --- a/.agents/skills/trust-boundary/SKILL.md +++ b/.agents/skills/trust-boundary/SKILL.md @@ -103,36 +103,41 @@ Covered by `TestParseContainer`, `TestDeviceAllowed`, `TestExplicitlyAllowed`, `collectContainerRules` (`internal/processor/processor.go`) passes a mount to `processor.CollectMountRules` only when `processor.IsMountSource` says its `Source` is `/dev` or -under `/dev/`. `CollectMountRules` (`internal/processor/rules.go`) resolves a symlinked mount -source, walks directory mounts, applies `DeviceAllowed` to each resolved entry, and emits a rule only -for a character or block device. Unresolvable symlinks get two different treatments: - -- a dangling or unresolvable symlink **found while walking a directory mount** is skipped, with a - warning only when an explicit allow glob names it (`skipUnresolvable`, the fix merged in commit - `244d2a5`); -- a mount **source** that is itself an unresolvable symlink returns an error in - `MountResult.Errs`, which `Reconcile` logs as a rule failure. Any entry in `Errs` makes the - container's whole desired set empty (reason `incomplete_device_set`) until a retry resolves every - device: a partly resolved set is never applied. -Covered by `TestIsDeviceMountSource`, `TestCollectMountRules_*` and -`TestCollectMountRules_UnresolvableSymlinks_ExplicitAllow` / `_NoAllowGlobs`. +under `/dev/`. From there every name (the source, each walked entry, each symlink) goes through one +`evaluateCandidate` (`internal/processor/rules.go`) on file descriptors, never re-walked paths: + +- a lexical gate: the name must be clean and under `/dev`, so `/dev/../etc/x` is skipped + (`outside_dev`); +- `openat2` beneath a `/dev` descriptor with `RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS` (the `devFS` + seam in `devfs.go`): no symlink step can leave `/dev`. An absolute link back into `/dev` is + followed by hand, bounded to 40 hops; any other escape is skipped with a warning. The daemon + refuses to start without `openat2` (`ProbeOpenat2`); +- `fstat` on that descriptor gives the device number, and sysfs `DEVNAME` gives the canonical name, + so a node planted under `/dev/shm` is judged as the device it is; +- `Denied` is checked on the alias, the resolved and the canonical name, `Authorized` on the + canonical and the resolved name; candidates are aggregated per device across all mounts, and one + deny vote suppresses the device. + +A dangling name found while walking is skipped (a warning only when an explicit allow glob names it). +Everything that leaves the set unknown is an entry in `MountResult.Errs` and makes the container's +whole desired set empty (reason `incomplete_device_set`) until a retry succeeds: a missing mount +source, an identity that cannot be established for a candidate policy would otherwise grant, any +walk error, and a directory mount over `maxMountEntries`. A partly known set is never applied. +Covered by `TestIsDeviceMountSource`, `TestEvaluateIdentity`, `TestCanonicalName`, `TestCollect_*`, +`TestProcessor_*EmptiesTheSet` and `TestRealDevFS_Openat2Containment`. - [ ] No new code path turns a non-`/dev` source into a rule. -- [ ] A new resolution step (symlink, bind, overlay) fails only that entry on error — a skip or a - per-mount error in `MountResult.Errs` — and never falls back to the unresolved path. -- [ ] **Known gap:** `IsMountSource` is a lexical prefix check on the unresolved `Source`, and - the mount `Source` comes from whoever writes the service spec — the same untrusted deployer - who sets the labels — so `/dev/../etc` passes it. The resolved targets in - `CollectMountRules` and `visitSymlink` are policy-checked but not re-checked against - `/dev`, and `/dev/shm` and `/dev/mqueue` are world-writable (mode 1777) on normal hosts, so - a non-root host user or a container that bind-mounts `/dev/shm` can plant symlinks the walk - follows. What bounds this today is not who can write to `/dev`; it is that - `DeviceAllowed` runs on the **resolved** path, so allow globs anchored at `/dev/...` reject a - target they do not name, and that only character and block device nodes become rules. With - no allow globs configured (the default) that bound is only as tight as the deployer's - ability to mount the target directly. A change that touches resolution must not widen - this; the right fix is `filepath.Clean` plus an `IsMountSource` check on every resolved - path. +- [ ] A new resolution step goes through `evaluateCandidate`; it skips a name that names nothing + under `/dev`, and anything it cannot establish is an error that empties the set — never a + fallback to path-only policy and never a silent skip. +- [ ] Policy keeps judging the canonical identity: deny on every name, allow on the canonical and + resolved names. An alias-level deny is only a reconciliation-time veto (a container can hide + the alias between passes); the durable boundary is a glob on the canonical name. +- [x] **Closed gap (containment):** mount sources and symlink targets are contained to `/dev` by + the lexical gate and `openat2 RESOLVE_BENEATH`, not by a prefix check on the unresolved path. +- [x] **Closed gap (planted nodes):** a node planted in a world-writable `/dev` directory + (`/dev/shm`, `/dev/mqueue`) is judged by its sysfs `DEVNAME`, so `/dev/shm/x` made as `b 8:0` + is denied by a deny on `/dev/sd*` and not authorized by an allow on `/dev/shm/*`. ## 5. Hot reload never widens access on a parse error @@ -188,6 +193,6 @@ a static binary onto `dhi.io/static` with `USER 0` and nothing else in the runti - [ ] Empty and unknown mode deny; a test covers both - [ ] Enum-like config and flags validated at load, by name - [ ] Malformed labels skip the container; labels only narrow; deny beats allow -- [ ] Only resolved `/dev` device nodes become rules; unresolvable symlinks are skipped +- [ ] Only device nodes resolved beneath `/dev` become rules; an unknown identity or walk empties the set - [ ] SIGHUP reload keeps the previous policy and `dry-run` on any read, parse or validation error - [ ] No privilege or mount beyond the README's documented set; DBus mount stays optional diff --git a/README.md b/README.md index e77f9ac..b2c9e51 100644 --- a/README.md +++ b/README.md @@ -302,6 +302,9 @@ Consumer services opt in and narrow their allowed device set with labels: | `swarm-device-access.device-allow` | Comma-separated globs | Allow only matching `/dev/...` paths. Empty means inherit. | | `swarm-device-access.device-deny` | Comma-separated globs | Deny matching `/dev/...` paths. Deny overrides allow. | +Write deny globs against kernel device names (`/dev/sd*`, `/dev/nvme*`), not +against aliases such as `/dev/disk/by-id/*`: see [Security](#-security). + Declare these labels under top-level `labels:` in your service definition. Docker copies top-level labels into each task container, so the daemon can read them on every node — including worker nodes. @@ -335,7 +338,18 @@ under `/dev` is therefore judged as the device it really is. A name that leads outside `/dev` is skipped with a warning; a device whose identity cannot be established (no or ambiguous `DEVNAME` in sysfs, an unreadable node) leaves the container with no grants until a retry succeeds, unless policy already -excludes it under its other names. +excludes it under its other names. The same holds when a directory mount +cannot be enumerated completely: a read error, or more than 4096 entries +(`mount too large; narrow the bind mount`). A whole-`/dev` mount counts +everything under it, including files programs keep in `/dev/shm`, so on a busy +host it can hit that limit: mount the device directories you need instead. + +Some pseudo devices have no `DEVNAME`; `/dev/pts/*` nodes, for example, have +no `/sys/dev/char/136:0` entry. To see why a device blocks a container, look +up its numbers (`ls -l /dev/pts/0` shows `136, 0`) and check +`ls /sys/dev/char/136:0` (`/sys/dev/block/M:m` for block devices). Blocking +such a device is intentional: bind-mount only the devices you need, or +exclude the rest with an allow list. Global `-device-allow` and `-device-deny` define the broadest access the daemon may grant. Per-container labels can only narrow that access. Deny rules always @@ -355,6 +369,17 @@ If every workload on the node is trusted, `-policy-mode=all` can be used to process all containers unless they explicitly set `swarm-device-access.enable: "false"`. +The security authority is a device's canonical identity: its device number, +named by its kernel name (`DEVNAME` in sysfs). Cgroup rules grant a +`major:minor`, not a path. A deny glob that matches only an alias (a +`/dev/disk/by-id/...` link, a node someone planted under `/dev/shm`) vetoes the +device while that alias is present when the daemon evaluates the container, +but a container that can change the directory it mounts can hide the alias +before a later pass. Treat alias-level deny as a best-effort veto, and write +the denies you rely on against kernel names, for example `/dev/sd*` and +`/dev/nvme*`. Allow globs are always checked against the kernel name and the +node a name resolves to, so an alias alone never grants a device. + See [SECURITY.md](SECURITY.md) for the threat model and disclosure policy. ## 🧪 Verifying Device Passthrough diff --git a/SECURITY.md b/SECURITY.md index 243999e..1391aac 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -25,6 +25,18 @@ are ignored, and the daemon does not inspect services or the node's Swarm role, so a decision cannot differ between manager and worker nodes or change after a daemon restart. +**Device identity.** A mount source, and every name under a mounted +directory, is resolved beneath `/dev` with `openat2(RESOLVE_BENEATH)`, so no +path or symlink can lead the daemon outside `/dev`. Each device is identified +by the device number of the opened node and its kernel name from sysfs +(`DEVNAME`), and policy is applied to that identity: a node planted in a +world-writable `/dev` directory is judged as the device it is. A device whose +identity cannot be established, or a directory mount that cannot be +enumerated completely, leaves the container with no grants until a retry +succeeds. Deny globs also apply to aliases, but only as a best-effort veto at +evaluation time; the boundary to rely on is a glob on the kernel name (see the +README's Security section). + **Opt-in default.** The daemon defaults to `-policy-mode=opt-in`, processing only containers that carry `swarm-device-access.enable: "true"`. This minimises blast radius — containers that happen to bind-mount `/dev/...` paths are not silently diff --git a/docs/architecture.md b/docs/architecture.md index ba4671b..a30ee73 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -33,6 +33,10 @@ Docker daemon | ├─ New(version) → Interface | | ├─ GetDeviceCGroupMountPath(procRootPath, pid) | | └─ for each /dev/... mount: | +| enumerate names (complete, ≤ 4096) | +| each name: openat2 beneath /dev → fstat | +| → sysfs DEVNAME → deny/allow on names | +| aggregate per device across mounts | | applyMount → applyDeviceRules | | └─ api.AddDeviceRules(cgroupPath, rules) | +---------------------------------------------------+ @@ -115,8 +119,10 @@ For every bind mount whose source is `/dev` or lives under `/dev/`, the processo - **Single-file mount** (e.g. `/dev/nvidia0`): the source is one candidate. A missing or non-device source is an error, because it was mounted explicitly. -- **Directory mount** (e.g. `/dev`, `/dev/dri`): the tree is walked (depth-capped) only to enumerate names; each entry is one candidate named - after the mount source. +- **Directory mount** (e.g. `/dev`, `/dev/dri`): the tree is walked only to enumerate names; each entry is one candidate named after the mount + source. Enumeration must be complete: a root that cannot be opened, a directory that cannot be read, any walk error, or more than 4096 + entries in one mount (`mount too large; narrow the bind mount`) makes the container's whole desired set empty with a retryable error, since + an alias that was not seen might deny a device that was. There is no depth cap and no subtree exception. `/dev` is opened once per pass. Each candidate goes through one evaluation on file descriptors, so an entry swapped between enumeration and use is judged by what it is when opened, not by its name: diff --git a/internal/processor/devfs.go b/internal/processor/devfs.go index 5fec7a9..380cd7d 100644 --- a/internal/processor/devfs.go +++ b/internal/processor/devfs.go @@ -21,6 +21,7 @@ package processor import ( "errors" "fmt" + "io/fs" "os" "path/filepath" "strconv" @@ -90,6 +91,10 @@ type devFS interface { readLink(rel string) (string, error) // readUevent returns the sysfs uevent of a device. readUevent(id deviceID) ([]byte, error) + // walk enumerates the names under base (a path under root), like + // filepath.WalkDir. It is for names only: every name is then resolved + // with openBeneath. + walk(base string, visit fs.WalkDirFunc) error Close() error } @@ -185,6 +190,11 @@ func (r *realDevFS) readLink(rel string) (string, error) { return string(buf[:n]), nil } +func (*realDevFS) walk(base string, visit fs.WalkDirFunc) error { + //nolint:wrapcheck // WalkDir only returns what visit returned + return filepath.WalkDir(base, visit) +} + func (r *realDevFS) readUevent(id deviceID) ([]byte, error) { path := filepath.Join(r.sysfsRoot, "dev", id.sysfsClass(), strconv.FormatInt(id.major, 10)+":"+strconv.FormatInt(id.minor, 10), "uevent") diff --git a/internal/processor/rules.go b/internal/processor/rules.go index 9b41db4..820afcb 100644 --- a/internal/processor/rules.go +++ b/internal/processor/rules.go @@ -35,8 +35,11 @@ import ( ) const ( - maxDirDepth = 8 - maxLogChildren = 32 + // maxMountEntries bounds the names enumerated under one directory + // mount. A walk that cannot see every name cannot know that no alias + // denies a device it did see, so overflow is an error, not a cutoff. + maxMountEntries = 4096 + maxLogChildren = 32 // maxLinkHops bounds the absolute symlinks followed by hand (see // evaluateCandidate), like the kernel's own symlink limit. maxLinkHops = 40 @@ -55,6 +58,11 @@ var ( errBadDevname = errors.New("uevent has no single valid DEVNAME") // errTooManyLinks reports a chain of absolute symlinks over maxLinkHops. errTooManyLinks = errors.New("too many levels of symbolic links") + // errIncompleteWalk marks a directory mount whose names could not all + // be enumerated; like an unresolved candidate, it leaves the set unknown. + errIncompleteWalk = errors.New("device mount enumeration incomplete") + // errMountTooLarge reports a directory mount over maxMountEntries. + errMountTooLarge = errors.New("mount too large; narrow the bind mount") ) // Candidate skip reasons, also the sda_device_candidates_skipped_total @@ -184,9 +192,10 @@ func walkMount( result: result, } - walkErr := filepath.WalkDir(state.base, state.visitEntry) + walkErr := dev.walk(state.base, state.visitEntry) if walkErr != nil { - state.result.Errs = append(state.result.Errs, fmt.Errorf("walk %q: %w", source, walkErr)) + state.result.Errs = append(state.result.Errs, + fmt.Errorf("%w: %s: %w", errIncompleteWalk, source, walkErr)) } granted := 0 @@ -220,37 +229,40 @@ type mountWalkState struct { // policyExcluded counts device candidates the policy did not grant. policyExcluded int childrenSeen []string + // entries counts every name enumerated, directories included. + entries int } func (s *mountWalkState) visitEntry(walkedPath string, entry fs.DirEntry, entryErr error) error { + alias := s.source + strings.TrimPrefix(walkedPath, s.base) + + // The root failing to open, a directory that cannot be read, any walk + // error: some name was not seen, so the set is unknown. if entryErr != nil { - s.result.Errs = append(s.result.Errs, entryErr) + s.result.Errs = append(s.result.Errs, + fmt.Errorf("%w: %s: %w", errIncompleteWalk, alias, entryErr)) return nil } - if entry.IsDir() { - if walkedPath == s.base { - return nil - } + if walkedPath == s.base { + return nil + } - depth := strings.Count( - strings.TrimPrefix(walkedPath, s.base), - string(filepath.Separator), - ) - if depth > maxDirDepth { - logger.L().Debug("walk depth cap reached", "path", walkedPath) + s.entries++ + if s.entries > maxMountEntries { + s.result.Errs = append(s.result.Errs, fmt.Errorf("%w: %s: %w: more than %d entries", + errIncompleteWalk, s.source, errMountTooLarge, maxMountEntries)) - return filepath.SkipDir - } + return filepath.SkipAll + } + if entry.IsDir() { return nil } s.trackChild(walkedPath) - alias := s.source + strings.TrimPrefix(walkedPath, s.base) - found := evaluateCandidate(s.dev, alias, s.gpol, s.cpol) switch { diff --git a/internal/processor/rules_test.go b/internal/processor/rules_test.go index 87ac6f6..0cc531a 100644 --- a/internal/processor/rules_test.go +++ b/internal/processor/rules_test.go @@ -21,6 +21,7 @@ package processor import ( "errors" "fmt" + "io/fs" "os" "path/filepath" "strconv" @@ -46,6 +47,8 @@ type fakeDevFS struct { errs map[string]error links map[string]string uevents map[deviceID]string + // walkErrs injects an error for a name (relative to dir) into the walk. + walkErrs map[string]error } type fakeNode struct { @@ -62,12 +65,13 @@ func newFakeDevFS(t *testing.T) *fakeDevFS { t.Helper() return &fakeDevFS{ - t: t, - dir: t.TempDir(), - nodes: make(map[string]fakeNode), - errs: make(map[string]error), - links: make(map[string]string), - uevents: make(map[deviceID]string), + t: t, + dir: t.TempDir(), + nodes: make(map[string]fakeNode), + errs: make(map[string]error), + links: make(map[string]string), + uevents: make(map[deviceID]string), + walkErrs: make(map[string]error), } } @@ -110,6 +114,18 @@ func (f *fakeDevFS) readUevent(id deviceID) ([]byte, error) { return []byte(data), nil } +func (f *fakeDevFS) walk(base string, visit fs.WalkDirFunc) error { + //nolint:wrapcheck // WalkDir only returns what visit returned + return filepath.WalkDir(base, func(path string, entry fs.DirEntry, err error) error { + rel, relErr := filepath.Rel(f.dir, path) + if injected, ok := f.walkErrs[rel]; ok && relErr == nil && err == nil { + return visit(path, entry, injected) + } + + return visit(path, entry, err) + }) +} + // touch creates the name under dir so a walk enumerates it. func (f *fakeDevFS) touch(rel string) { f.t.Helper() @@ -241,13 +257,19 @@ func wantRules(t *testing.T, got containerRules, want ...deviceID) { func wantUnresolved(t *testing.T, got containerRules, fragments ...string) { t.Helper() + wantIncomplete(t, got, errUnresolved, fragments...) +} + +func wantIncomplete(t *testing.T, got containerRules, sentinel error, fragments ...string) { + t.Helper() + if len(got.deviceErrs) == 0 { - t.Fatal("want an unresolved-identity error, got none") + t.Fatalf("want a %v error, got none", sentinel) } joined := errors.Join(got.deviceErrs...) - if !errors.Is(joined, errUnresolved) { - t.Errorf("err = %v, want errUnresolved", joined) + if !errors.Is(joined, sentinel) { + t.Errorf("err = %v, want %v", joined, sentinel) } for _, fragment := range fragments { @@ -695,6 +717,97 @@ func TestCollect_DirectorySkips(t *testing.T) { } } +// Every way a directory mount's names can go unseen leaves the set unknown: +// an alias that was not enumerated might deny a device that was. +func TestCollect_IncompleteEnumeration(t *testing.T) { + t.Parallel() + + for name, tc := range map[string]struct { + setup func(dev *fakeDevFS) + want error + }{ + "root cannot be opened": {func(dev *fakeDevFS) { + dev.directory("dri") + dev.device("dri/card0", devCard, "dri/card0") + dev.errs["dri"] = fmt.Errorf("openat2 dri: %w", unix.EACCES) + }, errUnresolved}, + "root vanished before the walk": {func(dev *fakeDevFS) { + dev.nodes["dri"] = fakeNode{kind: nodeDir, path: "/dev/dri"} + }, errIncompleteWalk}, + "directory read error": {func(dev *fakeDevFS) { + dev.directory("dri") + dev.directory("dri/by-path") + dev.walkErrs["dri/by-path"] = unix.EACCES + }, errIncompleteWalk}, + "entry error": {func(dev *fakeDevFS) { + dev.directory("dri") + dev.touch("dri/renderD128") + dev.walkErrs["dri/renderD128"] = unix.EIO + }, errIncompleteWalk}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.device("null", devNull, "null") + tc.setup(dev) + + got := collectFrom(dev, policy.Global{Mode: policy.ModeAll}, "/dev/null", "/dev/dri") + wantIncomplete(t, got, tc.want, "/dev/dri") + + proc := &Processor{devfs: func() (devFS, error) { return dev, nil }} + info := &container.InspectResponse{ + State: &container.State{Running: true, Pid: 1}, + Mounts: []container.MountPoint{ + {Source: "/dev/null", Destination: "/dev/null"}, + {Source: "/dev/dri", Destination: "/dev/dri"}, + }, + } + + desired := proc.computeDesired( + "abc", + info, + config.Runtime{Policy: policy.Global{Mode: policy.ModeAll}}, + ) + if len(desired.rules) != 0 || !errors.Is(desired.incomplete, tc.want) { + t.Errorf("desired = %+v, want an empty set and a %v error", desired, tc.want) + } + }) + } +} + +func TestCollect_EntryCapOverflowIsAnError(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.directory("big") + + for idx := range maxMountEntries + 1 { + dev.device( + "big/n"+strconv.Itoa(idx), + deviceID{typ: "c", major: 250, minor: int64(idx)}, + "n", + ) + } + + got := collectFrom(dev, policy.Global{Mode: policy.ModeAll}, "/dev/big") + wantIncomplete(t, got, errMountTooLarge, "/dev/big", "narrow the bind mount") + + // At the cap exactly, the mount is enumerated in full. + delete(dev.nodes, "big/n0") + + err := os.Remove(filepath.Join(dev.dir, "big", "n0")) + if err != nil { + t.Fatal(err) + } + + got = collectFrom(dev, policy.Global{Mode: policy.ModeAll}, "/dev/big") + if len(got.deviceErrs) != 0 || len(got.granted) != maxMountEntries { + t.Errorf("at the cap: granted %d, errs %v; want %d and none", + len(got.granted), got.deviceErrs, maxMountEntries) + } +} + func TestCollect_DirectoryNoChildrenMatchWarns(t *testing.T) { dev := newFakeDevFS(t) dev.directory("dri") diff --git a/test/integration/enforce_test.go b/test/integration/enforce_test.go index ffe3c36..e0280b9 100644 --- a/test/integration/enforce_test.go +++ b/test/integration/enforce_test.go @@ -26,6 +26,7 @@ import ( "io" "os" "os/exec" + "strconv" "strings" "testing" "time" @@ -33,6 +34,7 @@ import ( "github.com/moby/moby/api/pkg/stdcopy" "github.com/moby/moby/api/types/container" dockerclient "github.com/moby/moby/client" + "golang.org/x/sys/unix" "github.com/leinardi/swarm-device-access/internal/policy" ) @@ -127,6 +129,7 @@ func TestEnforce_GrantsDeviceAccess(t *testing.T) { requireProcessed(ctx, t, proc, target, 1, 0) requireAllowed(ctx, t, cli, target, device, "labeled container after the daemon granted it") + requireNoOverGrant(ctx, t, cli, target, device) sibling := startTestContainer(ctx, t, cli, nil, []string{bind}) requireSkipped(ctx, t, proc, sibling, msgSkippedPolicy) @@ -524,8 +527,61 @@ func probeOpen( ) (exitCode int, stderrText string) { t.Helper() + return execIn(ctx, t, cli, containerID, "sh", "-c", `exec 3< "$1"`, "probe", device) +} + +// requireNoOverGrant creates, inside the opted-in container, nodes the daemon +// must not have granted along with device: the same major with the next +// minor, and the same numbers as the other device type (a block node when +// device is a character device). Opening each must still be denied. +func requireNoOverGrant( + ctx context.Context, + t *testing.T, + cli *dockerclient.Client, + containerID, device string, +) { + t.Helper() + + var st unix.Stat_t + + err := unix.Stat(device, &st) + if err != nil { + t.Fatalf("stat %s: %v", device, err) + } + + major, minor := strconv.FormatUint(uint64(unix.Major(st.Rdev)), 10), unix.Minor(st.Rdev) + + typ, other := "c", "b" + if st.Mode&unix.S_IFMT == unix.S_IFBLK { + typ, other = "b", "c" + } + + for _, node := range [][]string{ + {"/tmp/sda-next-minor", typ, major, strconv.FormatUint(uint64(minor)+1, 10)}, + {"/tmp/sda-other-type", other, major, strconv.FormatUint(uint64(minor), 10)}, + } { + exitCode, stderr := execIn(ctx, t, cli, containerID, append([]string{"mknod"}, node...)...) + if exitCode != 0 { + t.Fatalf("mknod %v in the opted-in container: exit %d: %s", node, exitCode, stderr) + } + + requireDenied(ctx, t, cli, containerID, node[0], + fmt.Sprintf("node %s %s:%s next to the granted %s", node[1], node[2], node[3], device)) + } +} + +// execIn runs cmd inside containerID and returns its exit code and stderr. +func execIn( + ctx context.Context, + t *testing.T, + cli *dockerclient.Client, + containerID string, + cmd ...string, +) (exitCode int, stderrText string) { + t.Helper() + created, err := cli.ExecCreate(ctx, containerID, dockerclient.ExecCreateOptions{ - Cmd: []string{"sh", "-c", `exec 3< "$1"`, "probe", device}, + Cmd: cmd, AttachStdout: true, AttachStderr: true, }) From 6bc430b53accd209caa3e7cf44765b86adeb1495 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 11:43:13 +0200 Subject: [PATCH 15/37] ci(ci): run go vet and the unit tests on pull requests Add a test job to the CI workflow with the same steps as the release workflow's test job: setup-go pinned by SHA with Go from go.mod, then make go-vet and make go-test. Until now vet and unit tests only ran at release time. --- .github/workflows/ci.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 671316c..cc0d125 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -74,6 +74,27 @@ jobs: - name: Audit Go dependencies run: make audit-deps + # The same steps as the release workflow's test job, so a pull request + # cannot merge with a failing vet or unit test and only find out at release. + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + + - name: Vet + run: make go-vet + + - name: Run the test suite + run: make go-test + markdownlint: runs-on: ubuntu-latest From a6fb727ebbcd3eade835bce0f4460b8e305934a1 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 11:43:53 +0200 Subject: [PATCH 16/37] fix(examples): place daemon flags after the image name In every example compose file the commented daemon flags (-device-allow, -policy-mode) sat before the image in the docker run argument list, so uncommenting one made it a docker run flag instead of reaching the daemon. They now follow the image, as in deployments/docker/docker-compose.yaml, with a note saying why. --- examples/audio-passthrough/docker-compose.yaml | 3 ++- examples/gpu-passthrough/docker-compose.yaml | 3 ++- examples/smartctl-exporter/docker-compose.yaml | 3 ++- examples/usb-passthrough/docker-compose.yaml | 3 ++- examples/v4l2-passthrough/docker-compose.yaml | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/examples/audio-passthrough/docker-compose.yaml b/examples/audio-passthrough/docker-compose.yaml index 6a987c8..ae84d76 100644 --- a/examples/audio-passthrough/docker-compose.yaml +++ b/examples/audio-passthrough/docker-compose.yaml @@ -41,9 +41,10 @@ services: - /var/run/docker.sock:/var/run/docker.sock - -v - /dev:/dev + - ghcr.io/leinardi/swarm-device-access:latest + # Daemon flags go after the image name; before it they are docker run flags. # Restrict to ALSA sound devices only: # - -device-allow=/dev/snd/* - - ghcr.io/leinardi/swarm-device-access:latest entrypoint: docker deploy: mode: global diff --git a/examples/gpu-passthrough/docker-compose.yaml b/examples/gpu-passthrough/docker-compose.yaml index d184388..01fceb2 100644 --- a/examples/gpu-passthrough/docker-compose.yaml +++ b/examples/gpu-passthrough/docker-compose.yaml @@ -49,12 +49,13 @@ services: # Enable reload-after-daemon-reload (optional): # - -v # - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket + - ghcr.io/leinardi/swarm-device-access:latest + # Daemon flags go after the image name; before it they are docker run flags. # Policy mode: opt-in (default) processes only labelled containers. # Use -policy-mode=all to process every container on the node. # - -policy-mode=opt-in # Limit scope to NVIDIA devices only (optional): # - -device-allow=/dev/nvidia* - - ghcr.io/leinardi/swarm-device-access:latest entrypoint: docker deploy: mode: global diff --git a/examples/smartctl-exporter/docker-compose.yaml b/examples/smartctl-exporter/docker-compose.yaml index 0e89569..a3e7424 100644 --- a/examples/smartctl-exporter/docker-compose.yaml +++ b/examples/smartctl-exporter/docker-compose.yaml @@ -60,9 +60,10 @@ services: # Enable reload-after-daemon-reload (optional): # - -v # - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket + - ghcr.io/leinardi/swarm-device-access:latest + # Daemon flags go after the image name; before it they are docker run flags. # Restrict to disk devices only (optional): # - -device-allow=/dev/sd*,/dev/nvme*,/dev/sg*,/dev/disk/* - - ghcr.io/leinardi/swarm-device-access:latest entrypoint: docker deploy: mode: global diff --git a/examples/usb-passthrough/docker-compose.yaml b/examples/usb-passthrough/docker-compose.yaml index 6a1129e..ef4eefb 100644 --- a/examples/usb-passthrough/docker-compose.yaml +++ b/examples/usb-passthrough/docker-compose.yaml @@ -29,9 +29,10 @@ services: - /var/run/docker.sock:/var/run/docker.sock - -v - /dev:/dev + - ghcr.io/leinardi/swarm-device-access:latest + # Daemon flags go after the image name; before it they are docker run flags. # Restrict to USB bus devices only: # - -device-allow=/dev/bus/usb/* - - ghcr.io/leinardi/swarm-device-access:latest entrypoint: docker deploy: mode: global diff --git a/examples/v4l2-passthrough/docker-compose.yaml b/examples/v4l2-passthrough/docker-compose.yaml index b0c7268..d364df1 100644 --- a/examples/v4l2-passthrough/docker-compose.yaml +++ b/examples/v4l2-passthrough/docker-compose.yaml @@ -29,9 +29,10 @@ services: - /var/run/docker.sock:/var/run/docker.sock - -v - /dev:/dev + - ghcr.io/leinardi/swarm-device-access:latest + # Daemon flags go after the image name; before it they are docker run flags. # Restrict to V4L2 devices only: # - -device-allow=/dev/video* - - ghcr.io/leinardi/swarm-device-access:latest entrypoint: docker deploy: mode: global From 4b951dd5f367dfd687b94a5739585b63c860bd0b Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 11:44:24 +0200 Subject: [PATCH 17/37] fix(examples): match USB device nodes in the USB passthrough example USB device nodes live at /dev/bus/usb//, and a glob * does not cross a slash, so /dev/bus/usb/* matched only the bus directories and granted no device. The flag comment and the label now use /dev/bus/usb/*/*. --- examples/usb-passthrough/docker-compose.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/usb-passthrough/docker-compose.yaml b/examples/usb-passthrough/docker-compose.yaml index ef4eefb..23146e5 100644 --- a/examples/usb-passthrough/docker-compose.yaml +++ b/examples/usb-passthrough/docker-compose.yaml @@ -31,8 +31,9 @@ services: - /dev:/dev - ghcr.io/leinardi/swarm-device-access:latest # Daemon flags go after the image name; before it they are docker run flags. - # Restrict to USB bus devices only: - # - -device-allow=/dev/bus/usb/* + # Restrict to USB bus devices only. Nodes are /dev/bus/usb//, + # and * does not match across "/", so the glob needs both levels: + # - -device-allow=/dev/bus/usb/*/* entrypoint: docker deploy: mode: global @@ -48,7 +49,7 @@ services: # Use top-level labels: so they reach the container; deploy.labels: are ignored. labels: swarm-device-access.enable: "true" - swarm-device-access.device-allow: "/dev/bus/usb/*" + swarm-device-access.device-allow: "/dev/bus/usb/*/*" deploy: mode: replicated replicas: 1 From 77b9f013f7a6f0ac454bbe25c40303c6dd01df4d Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 11:45:11 +0200 Subject: [PATCH 18/37] fix(examples): use one glob per -device-allow flag The -device-allow and -device-deny flags take one glob each and are not split on commas, so the smartctl-exporter example's comma list would have been one glob that matches nothing. It now repeats the flag per glob, and the README flag table says one glob per flag, unlike the comma-separated labels. --- README.md | 4 ++-- examples/smartctl-exporter/docker-compose.yaml | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b2c9e51..5b84ea4 100644 --- a/README.md +++ b/README.md @@ -223,8 +223,8 @@ value is set in both places, the CLI flag wins. | `-docker-socket` | `/var/run/docker.sock` | Path to the Docker daemon's UNIX socket | | `-dry-run` | `false` | Log the device set each container would get, without reading its `/proc` entry or its cgroup and without writing anything. Not a cleanup path: grants left by a previous live run stay, and a reload can turn it off but not on (see [Config File](#config-file)) | | `-policy-mode` | `opt-in` | `opt-in`: only `enable=true` containers. `all`: unless `enable=false`. | -| `-device-allow` | `""` | Glob for `/dev/...` paths to allow, repeatable. Empty means allow all. | -| `-device-deny` | `""` | Glob for `/dev/...` paths to deny, repeatable. Deny takes priority over allow. | +| `-device-allow` | `""` | Glob for `/dev/...` paths to allow, one glob per flag; repeat the flag for more (not comma-split, unlike the labels). Empty means allow all. | +| `-device-deny` | `""` | Glob for `/dev/...` paths to deny, one glob per flag; repeat the flag for more (not comma-split, unlike the labels). Deny takes priority over allow. | | `-metrics-addr` | `""` | `host:port` for Prometheus `/metrics`, `/healthz`, `/readyz`. Empty disables it. | | `-debug-addr` | `""` | `host:port` for pprof `/debug/pprof/*`. Empty disables it. | | `-config` | `""` | Path to a YAML config file. CLI flags override file values. Reload with SIGHUP. | diff --git a/examples/smartctl-exporter/docker-compose.yaml b/examples/smartctl-exporter/docker-compose.yaml index a3e7424..fb4d87f 100644 --- a/examples/smartctl-exporter/docker-compose.yaml +++ b/examples/smartctl-exporter/docker-compose.yaml @@ -62,8 +62,12 @@ services: # - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket - ghcr.io/leinardi/swarm-device-access:latest # Daemon flags go after the image name; before it they are docker run flags. - # Restrict to disk devices only (optional): - # - -device-allow=/dev/sd*,/dev/nvme*,/dev/sg*,/dev/disk/* + # Restrict to disk devices only (optional). One glob per flag: flags are + # not split on commas (only the labels are). + # - -device-allow=/dev/sd* + # - -device-allow=/dev/nvme* + # - -device-allow=/dev/sg* + # - -device-allow=/dev/disk/* entrypoint: docker deploy: mode: global From 87a1418945a11b7e2150d38d2abcbd13027a9382 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 11:45:42 +0200 Subject: [PATCH 19/37] fix(examples): mount the DBus socket at /var/run inside the container The gpu-passthrough and smartctl-exporter examples mounted the host DBus socket at /run/dbus/system_bus_socket inside the daemon container, but the daemon connects to /var/run/dbus/system_bus_socket and dhi.io/static has no /var/run -> /run symlink, so reload handling stayed disabled. Both now mount it at /var/run/dbus/system_bus_socket, as the reference compose file does. --- examples/gpu-passthrough/docker-compose.yaml | 5 +++-- examples/smartctl-exporter/docker-compose.yaml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/gpu-passthrough/docker-compose.yaml b/examples/gpu-passthrough/docker-compose.yaml index 01fceb2..b8bea10 100644 --- a/examples/gpu-passthrough/docker-compose.yaml +++ b/examples/gpu-passthrough/docker-compose.yaml @@ -46,9 +46,10 @@ services: - /var/run/docker.sock:/var/run/docker.sock - -v - /dev:/dev - # Enable reload-after-daemon-reload (optional): + # Enable reload-after-daemon-reload (optional). The container-side path + # must be under /var/run: dhi.io/static has no /var/run -> /run symlink. # - -v - # - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket + # - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket - ghcr.io/leinardi/swarm-device-access:latest # Daemon flags go after the image name; before it they are docker run flags. # Policy mode: opt-in (default) processes only labelled containers. diff --git a/examples/smartctl-exporter/docker-compose.yaml b/examples/smartctl-exporter/docker-compose.yaml index fb4d87f..2e93f17 100644 --- a/examples/smartctl-exporter/docker-compose.yaml +++ b/examples/smartctl-exporter/docker-compose.yaml @@ -57,9 +57,10 @@ services: - /var/run/docker.sock:/var/run/docker.sock - -v - /dev:/dev - # Enable reload-after-daemon-reload (optional): + # Enable reload-after-daemon-reload (optional). The container-side path + # must be under /var/run: dhi.io/static has no /var/run -> /run symlink. # - -v - # - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket + # - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket - ghcr.io/leinardi/swarm-device-access:latest # Daemon flags go after the image name; before it they are docker run flags. # Restrict to disk devices only (optional). One glob per flag: flags are From 0dad707cdc2f689ad3fe2c48bace1166e4927604 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 11:52:28 +0200 Subject: [PATCH 20/37] fix(deployments): name the daemon container and forward stop signals to it The Swarm wrapper ran a bare docker run with no --name, so the README's SIGHUP recipe could not find the daemon, and a killed wrapper orphaned the daemon while Swarm started a second one. The wrapper is now an sh -c script in the README, deployments/docker/docker-compose.yaml and all five examples. It sets a TERM and INT trap first, removes a stale swarm-device-access container, starts docker run --name=swarm-device-access in the background (a shell blocked on a foreground child defers its traps), and waits on it; the trap runs docker stop -t 10 on the daemon, waits for it and exits with its status. stop_grace_period is 30s so the stop can finish. The docker run arguments are built with one set -- line each, options before the image and daemon flags after it, so an optional mount or flag is enabled by uncommenting one line and cannot land on the wrong side of the image. The SIGHUP recipe becomes docker kill -s HUP swarm-device-access, and the commented config example mounts the directory, since a single-file mount keeps the old content after an editor replaces the file. docs/testing.md adds the manual Swarm check: kill -9 the wrapper task and confirm the replacement starts without a name conflict and exactly one daemon container runs. That check needs a Swarm host and was not run here; the script's signal handling and argument building were checked with a stub docker under busybox sh. --- README.md | 89 ++++++++++++++----- deployments/docker/docker-compose.yaml | 75 ++++++++++------ docs/testing.md | 8 +- .../audio-passthrough/docker-compose.yaml | 65 ++++++++++---- examples/gpu-passthrough/docker-compose.yaml | 75 ++++++++++------ .../smartctl-exporter/docker-compose.yaml | 77 ++++++++++------ examples/usb-passthrough/docker-compose.yaml | 67 +++++++++----- examples/v4l2-passthrough/docker-compose.yaml | 65 ++++++++++---- 8 files changed, 361 insertions(+), 160 deletions(-) diff --git a/README.md b/README.md index 5b84ea4..b1ef666 100644 --- a/README.md +++ b/README.md @@ -160,28 +160,51 @@ is dropped only once its cgroup is verified gone; a periodic sweep checks. services: swarm-device-access: image: docker:29 - entrypoint: docker # Swarm rejects privileged/cgroup/pid/userns on services. This wrapper # launches the actual daemon with `docker run`, where those flags are valid. + entrypoint: ["sh", "-c"] command: - - run - - -i - - --rm - - --privileged - - --cgroupns=host - - --pid=host - - --userns=host - - -v - - /sys:/host/sys - - -v - - /var/run/docker.sock:/var/run/docker.sock - - -v - - /dev:/dev - # Optional: reapply device rules after systemctl daemon-reload. - # NOTE: dhi.io/static has no /var/run→/run symlink; use /var/run/dbus/... inside the container. - # - -v - # - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket - - ghcr.io/leinardi/swarm-device-access:latest + - | + # docker run options, before the image. Uncomment a line to add it. + set -- --rm --name=swarm-device-access \ + --privileged --cgroupns=host --pid=host --userns=host \ + -v /sys:/host/sys \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /dev:/dev + # Reapply device rules after systemctl daemon-reload, which wipes cgroup + # BPF programs (without it the daemon warns and skips reload handling). + # The container-side path must be under /var/run: dhi.io/static has no + # /var/run -> /run symlink. + # set -- "$$@" -v /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket + # A config file for -config below. Mount the directory, not the file: + # an editor that replaces the file would leave a file mount on the old + # content, and a reload would silently apply it. + # set -- "$$@" -v /etc/swarm-device-access:/etc/swarm-device-access:ro + set -- "$$@" ghcr.io/leinardi/swarm-device-access:latest + # Daemon flags, after the image; before it they are docker run flags. + # Reload the config file with: docker kill -s HUP swarm-device-access + # set -- "$$@" -config /etc/swarm-device-access/config.yaml + + # On SIGTERM or SIGINT from Swarm, stop the daemon and exit with its + # status. The trap is set before the daemon starts, and the daemon runs + # in the background: a shell waiting on a foreground child only runs + # its traps after the child exits. + pid= + stop_daemon() { + docker stop -t 10 swarm-device-access >/dev/null 2>&1 + wait $$pid + exit $$? + } + trap stop_daemon TERM INT + + # Clear a daemon container left behind by a wrapper that was killed. + docker rm -f swarm-device-access >/dev/null 2>&1 || true + docker run "$$@" & + pid=$$! + wait "$$pid" + exit $$? + # Leaves time for docker stop -t 10 before Swarm kills the wrapper. + stop_grace_period: 30s volumes: - /var/run/docker.sock:/var/run/docker.sock deploy: @@ -210,6 +233,26 @@ services: The daemon compose file is also available at [`deployments/docker/docker-compose.yaml`](deployments/docker/docker-compose.yaml). +The wrapper is a small `sh` script rather than a bare `docker run`, so that +there is exactly one daemon container per node, always named +`swarm-device-access`: + +- It builds the `docker run` arguments with one `set -- "$$@" ...` line each: + options before the image, daemon flags after it. To enable an optional mount + or flag, uncomment its line; it cannot end up on the wrong side of the image. +- It sets a `SIGTERM`/`SIGINT` trap, then removes any `swarm-device-access` + container a killed wrapper left behind, so the replacement task does not + start a second daemon next to an orphan or, normally, hit a name conflict (if + it does, Swarm restarts the task and the next attempt clears it). +- It starts `docker run` in the background and waits on it, because a shell + blocked on a foreground child runs its traps only after the child exits. +- On `SIGTERM` from Swarm the trap runs `docker stop -t 10 + swarm-device-access`, waits for the daemon to exit, and exits with its + status. `stop_grace_period: 30s` gives it time to do so. + +Compose interpolates `$` in the file, so the script writes `$$` for a literal +`$`. + ## ⚙️ Configuration You can configure the daemon with CLI flags, a YAML config file, or both. When a @@ -255,9 +298,15 @@ keep the flag's value. Send `SIGHUP` to reload the config file without restarting the daemon: ```bash -kill -HUP $(docker inspect --format '{{.State.Pid}}' swarm-device-access) +docker kill -s HUP swarm-device-access ``` +Mount the directory holding the file into the daemon container and pass +`-config` after the image name; the compose files above show both as commented +lines. Mount the directory rather than the file: editors and tools that write a +new file and rename it over the old one would leave a single-file mount on the +old content, and the reload would silently apply that. + Some settings are only read at startup and still require a restart: `docker-socket`, `metrics-addr`, and `debug-addr`. A reload that changes one of them logs that it was ignored and applies the rest. diff --git a/deployments/docker/docker-compose.yaml b/deployments/docker/docker-compose.yaml index a2d4de8..d45d0cf 100644 --- a/deployments/docker/docker-compose.yaml +++ b/deployments/docker/docker-compose.yaml @@ -6,35 +6,51 @@ services: # The wrapper only needs the host Docker socket to dispatch `docker run`. - /var/run/docker.sock:/var/run/docker.sock + # Swarm rejects privileged/cgroup/pid/userns on services. This wrapper + # launches the actual daemon with `docker run`, where those flags are valid. command: - - run - - -i - - --rm - - --privileged - - --cgroupns=host - - --pid=host - - --userns=host - - -v - - /sys:/host/sys - - -v - - /var/run/docker.sock:/var/run/docker.sock - - -v - - /dev:/dev - # Optional: gives the daemon access to the host's system DBus so it can - # detect systemctl daemon-reload (which wipes cgroup BPF programs) and - # re-apply device rules. Without this mount, reload handling is skipped - # and the daemon logs a warning at startup. - # NOTE: dhi.io/static has no /var/run→/run symlink; the container-side - # path must be /var/run/dbus/system_bus_socket (not /run/dbus/...). - # - -v - # - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket - - ghcr.io/leinardi/swarm-device-access:latest - - "-log-level" - - "info" - - "-log-format" - - "text" - - entrypoint: docker + - | + # docker run options, before the image. Uncomment a line to add it. + set -- --rm --name=swarm-device-access \ + --privileged --cgroupns=host --pid=host --userns=host \ + -v /sys:/host/sys \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /dev:/dev + # Reapply device rules after systemctl daemon-reload, which wipes cgroup + # BPF programs (without it the daemon warns and skips reload handling). + # The container-side path must be under /var/run: dhi.io/static has no + # /var/run -> /run symlink. + # set -- "$$@" -v /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket + # A config file for -config below. Mount the directory, not the file: + # an editor that replaces the file would leave a file mount on the old + # content, and a reload would silently apply it. + # set -- "$$@" -v /etc/swarm-device-access:/etc/swarm-device-access:ro + set -- "$$@" ghcr.io/leinardi/swarm-device-access:latest + # Daemon flags, after the image; before it they are docker run flags. + # Reload the config file with: docker kill -s HUP swarm-device-access + # set -- "$$@" -config /etc/swarm-device-access/config.yaml + set -- "$$@" -log-level info -log-format text + + # On SIGTERM or SIGINT from Swarm, stop the daemon and exit with its + # status. The trap is set before the daemon starts, and the daemon runs + # in the background: a shell waiting on a foreground child only runs + # its traps after the child exits. + pid= + stop_daemon() { + docker stop -t 10 swarm-device-access >/dev/null 2>&1 + wait $$pid + exit $$? + } + trap stop_daemon TERM INT + + # Clear a daemon container left behind by a wrapper that was killed. + docker rm -f swarm-device-access >/dev/null 2>&1 || true + docker run "$$@" & + pid=$$! + wait "$$pid" + exit $$? + + entrypoint: [ "sh", "-c" ] restart: unless-stopped @@ -42,3 +58,6 @@ services: mode: global restart_policy: condition: any + + # Leaves time for docker stop -t 10 before Swarm kills the wrapper. + stop_grace_period: 30s diff --git a/docs/testing.md b/docs/testing.md index cfe5a12..5e110a2 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -112,6 +112,12 @@ or deployment behavior: 7. If `/run/dbus/system_bus_socket` is mounted and systemd is available, run `systemctl daemon-reload` and verify the daemon logs that it re-applied rules. -8. Verify observability by starting with `-metrics-addr :9090` and +8. On a Swarm node, `kill -9` the wrapper task's container and confirm that + Swarm's replacement task starts without a name conflict and that exactly one + `swarm-device-access` daemon container is running afterwards + (`docker ps --filter 'name=^swarm-device-access$'`; without the anchors the + filter also matches the wrapper task's container). + +9. Verify observability by starting with `-metrics-addr :9090` and `-debug-addr :6060`, then checking `/healthz`, `/readyz`, `/metrics`, and `/debug/pprof/`. diff --git a/examples/audio-passthrough/docker-compose.yaml b/examples/audio-passthrough/docker-compose.yaml index ae84d76..77c5a23 100644 --- a/examples/audio-passthrough/docker-compose.yaml +++ b/examples/audio-passthrough/docker-compose.yaml @@ -26,26 +26,53 @@ services: image: docker:29 volumes: - /var/run/docker.sock:/var/run/docker.sock + # Swarm rejects privileged/cgroup/pid/userns on services. This wrapper + # launches the actual daemon with `docker run`, where those flags are valid. + entrypoint: ["sh", "-c"] command: - - run - - -i - - --rm - - --name=swarm-device-access - - --privileged - - --cgroupns=host - - --pid=host - - --userns=host - - -v - - /sys:/host/sys - - -v - - /var/run/docker.sock:/var/run/docker.sock - - -v - - /dev:/dev - - ghcr.io/leinardi/swarm-device-access:latest - # Daemon flags go after the image name; before it they are docker run flags. - # Restrict to ALSA sound devices only: - # - -device-allow=/dev/snd/* - entrypoint: docker + - | + # docker run options, before the image. Uncomment a line to add it. + set -- --rm --name=swarm-device-access \ + --privileged --cgroupns=host --pid=host --userns=host \ + -v /sys:/host/sys \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /dev:/dev + # Reapply device rules after systemctl daemon-reload, which wipes cgroup + # BPF programs (without it the daemon warns and skips reload handling). + # The container-side path must be under /var/run: dhi.io/static has no + # /var/run -> /run symlink. + # set -- "$$@" -v /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket + # A config file for -config below. Mount the directory, not the file: + # an editor that replaces the file would leave a file mount on the old + # content, and a reload would silently apply it. + # set -- "$$@" -v /etc/swarm-device-access:/etc/swarm-device-access:ro + set -- "$$@" ghcr.io/leinardi/swarm-device-access:latest + # Daemon flags, after the image; before it they are docker run flags. + # Reload the config file with: docker kill -s HUP swarm-device-access + # set -- "$$@" -config /etc/swarm-device-access/config.yaml + # Restrict to ALSA sound devices only: + # set -- "$$@" -device-allow '/dev/snd/*' + + # On SIGTERM or SIGINT from Swarm, stop the daemon and exit with its + # status. The trap is set before the daemon starts, and the daemon runs + # in the background: a shell waiting on a foreground child only runs + # its traps after the child exits. + pid= + stop_daemon() { + docker stop -t 10 swarm-device-access >/dev/null 2>&1 + wait $$pid + exit $$? + } + trap stop_daemon TERM INT + + # Clear a daemon container left behind by a wrapper that was killed. + docker rm -f swarm-device-access >/dev/null 2>&1 || true + docker run "$$@" & + pid=$$! + wait "$$pid" + exit $$? + # Leaves time for docker stop -t 10 before Swarm kills the wrapper. + stop_grace_period: 30s deploy: mode: global restart_policy: diff --git a/examples/gpu-passthrough/docker-compose.yaml b/examples/gpu-passthrough/docker-compose.yaml index b8bea10..339607c 100644 --- a/examples/gpu-passthrough/docker-compose.yaml +++ b/examples/gpu-passthrough/docker-compose.yaml @@ -31,33 +31,56 @@ services: image: docker:29 volumes: - /var/run/docker.sock:/var/run/docker.sock + # Swarm rejects privileged/cgroup/pid/userns on services. This wrapper + # launches the actual daemon with `docker run`, where those flags are valid. + entrypoint: ["sh", "-c"] command: - - run - - -i - - --rm - - --name=swarm-device-access - - --privileged - - --cgroupns=host - - --pid=host - - --userns=host - - -v - - /sys:/host/sys - - -v - - /var/run/docker.sock:/var/run/docker.sock - - -v - - /dev:/dev - # Enable reload-after-daemon-reload (optional). The container-side path - # must be under /var/run: dhi.io/static has no /var/run -> /run symlink. - # - -v - # - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket - - ghcr.io/leinardi/swarm-device-access:latest - # Daemon flags go after the image name; before it they are docker run flags. - # Policy mode: opt-in (default) processes only labelled containers. - # Use -policy-mode=all to process every container on the node. - # - -policy-mode=opt-in - # Limit scope to NVIDIA devices only (optional): - # - -device-allow=/dev/nvidia* - entrypoint: docker + - | + # docker run options, before the image. Uncomment a line to add it. + set -- --rm --name=swarm-device-access \ + --privileged --cgroupns=host --pid=host --userns=host \ + -v /sys:/host/sys \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /dev:/dev + # Reapply device rules after systemctl daemon-reload, which wipes cgroup + # BPF programs (without it the daemon warns and skips reload handling). + # The container-side path must be under /var/run: dhi.io/static has no + # /var/run -> /run symlink. + # set -- "$$@" -v /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket + # A config file for -config below. Mount the directory, not the file: + # an editor that replaces the file would leave a file mount on the old + # content, and a reload would silently apply it. + # set -- "$$@" -v /etc/swarm-device-access:/etc/swarm-device-access:ro + set -- "$$@" ghcr.io/leinardi/swarm-device-access:latest + # Daemon flags, after the image; before it they are docker run flags. + # Reload the config file with: docker kill -s HUP swarm-device-access + # set -- "$$@" -config /etc/swarm-device-access/config.yaml + # Policy mode: opt-in (the default) processes only labelled containers; + # all processes every container on the node: + # set -- "$$@" -policy-mode opt-in + # Limit scope to NVIDIA devices only: + # set -- "$$@" -device-allow '/dev/nvidia*' + + # On SIGTERM or SIGINT from Swarm, stop the daemon and exit with its + # status. The trap is set before the daemon starts, and the daemon runs + # in the background: a shell waiting on a foreground child only runs + # its traps after the child exits. + pid= + stop_daemon() { + docker stop -t 10 swarm-device-access >/dev/null 2>&1 + wait $$pid + exit $$? + } + trap stop_daemon TERM INT + + # Clear a daemon container left behind by a wrapper that was killed. + docker rm -f swarm-device-access >/dev/null 2>&1 || true + docker run "$$@" & + pid=$$! + wait "$$pid" + exit $$? + # Leaves time for docker stop -t 10 before Swarm kills the wrapper. + stop_grace_period: 30s deploy: mode: global restart_policy: diff --git a/examples/smartctl-exporter/docker-compose.yaml b/examples/smartctl-exporter/docker-compose.yaml index 2e93f17..5c83608 100644 --- a/examples/smartctl-exporter/docker-compose.yaml +++ b/examples/smartctl-exporter/docker-compose.yaml @@ -42,34 +42,57 @@ services: image: docker:29 volumes: - /var/run/docker.sock:/var/run/docker.sock + # Swarm rejects privileged/cgroup/pid/userns on services. This wrapper + # launches the actual daemon with `docker run`, where those flags are valid. + entrypoint: ["sh", "-c"] command: - - run - - -i - - --rm - - --name=swarm-device-access - - --privileged - - --cgroupns=host - - --pid=host - - --userns=host - - -v - - /sys:/host/sys - - -v - - /var/run/docker.sock:/var/run/docker.sock - - -v - - /dev:/dev - # Enable reload-after-daemon-reload (optional). The container-side path - # must be under /var/run: dhi.io/static has no /var/run -> /run symlink. - # - -v - # - /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket - - ghcr.io/leinardi/swarm-device-access:latest - # Daemon flags go after the image name; before it they are docker run flags. - # Restrict to disk devices only (optional). One glob per flag: flags are - # not split on commas (only the labels are). - # - -device-allow=/dev/sd* - # - -device-allow=/dev/nvme* - # - -device-allow=/dev/sg* - # - -device-allow=/dev/disk/* - entrypoint: docker + - | + # docker run options, before the image. Uncomment a line to add it. + set -- --rm --name=swarm-device-access \ + --privileged --cgroupns=host --pid=host --userns=host \ + -v /sys:/host/sys \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /dev:/dev + # Reapply device rules after systemctl daemon-reload, which wipes cgroup + # BPF programs (without it the daemon warns and skips reload handling). + # The container-side path must be under /var/run: dhi.io/static has no + # /var/run -> /run symlink. + # set -- "$$@" -v /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket + # A config file for -config below. Mount the directory, not the file: + # an editor that replaces the file would leave a file mount on the old + # content, and a reload would silently apply it. + # set -- "$$@" -v /etc/swarm-device-access:/etc/swarm-device-access:ro + set -- "$$@" ghcr.io/leinardi/swarm-device-access:latest + # Daemon flags, after the image; before it they are docker run flags. + # Reload the config file with: docker kill -s HUP swarm-device-access + # set -- "$$@" -config /etc/swarm-device-access/config.yaml + # Restrict to disk devices only, one glob per flag: flags are not split + # on commas (only the labels are): + # set -- "$$@" -device-allow '/dev/sd*' + # set -- "$$@" -device-allow '/dev/nvme*' + # set -- "$$@" -device-allow '/dev/sg*' + # set -- "$$@" -device-allow '/dev/disk/*' + + # On SIGTERM or SIGINT from Swarm, stop the daemon and exit with its + # status. The trap is set before the daemon starts, and the daemon runs + # in the background: a shell waiting on a foreground child only runs + # its traps after the child exits. + pid= + stop_daemon() { + docker stop -t 10 swarm-device-access >/dev/null 2>&1 + wait $$pid + exit $$? + } + trap stop_daemon TERM INT + + # Clear a daemon container left behind by a wrapper that was killed. + docker rm -f swarm-device-access >/dev/null 2>&1 || true + docker run "$$@" & + pid=$$! + wait "$$pid" + exit $$? + # Leaves time for docker stop -t 10 before Swarm kills the wrapper. + stop_grace_period: 30s deploy: mode: global restart_policy: diff --git a/examples/usb-passthrough/docker-compose.yaml b/examples/usb-passthrough/docker-compose.yaml index 23146e5..e78d26f 100644 --- a/examples/usb-passthrough/docker-compose.yaml +++ b/examples/usb-passthrough/docker-compose.yaml @@ -14,27 +14,54 @@ services: image: docker:29 volumes: - /var/run/docker.sock:/var/run/docker.sock + # Swarm rejects privileged/cgroup/pid/userns on services. This wrapper + # launches the actual daemon with `docker run`, where those flags are valid. + entrypoint: ["sh", "-c"] command: - - run - - -i - - --rm - - --name=swarm-device-access - - --privileged - - --cgroupns=host - - --pid=host - - --userns=host - - -v - - /sys:/host/sys - - -v - - /var/run/docker.sock:/var/run/docker.sock - - -v - - /dev:/dev - - ghcr.io/leinardi/swarm-device-access:latest - # Daemon flags go after the image name; before it they are docker run flags. - # Restrict to USB bus devices only. Nodes are /dev/bus/usb//, - # and * does not match across "/", so the glob needs both levels: - # - -device-allow=/dev/bus/usb/*/* - entrypoint: docker + - | + # docker run options, before the image. Uncomment a line to add it. + set -- --rm --name=swarm-device-access \ + --privileged --cgroupns=host --pid=host --userns=host \ + -v /sys:/host/sys \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /dev:/dev + # Reapply device rules after systemctl daemon-reload, which wipes cgroup + # BPF programs (without it the daemon warns and skips reload handling). + # The container-side path must be under /var/run: dhi.io/static has no + # /var/run -> /run symlink. + # set -- "$$@" -v /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket + # A config file for -config below. Mount the directory, not the file: + # an editor that replaces the file would leave a file mount on the old + # content, and a reload would silently apply it. + # set -- "$$@" -v /etc/swarm-device-access:/etc/swarm-device-access:ro + set -- "$$@" ghcr.io/leinardi/swarm-device-access:latest + # Daemon flags, after the image; before it they are docker run flags. + # Reload the config file with: docker kill -s HUP swarm-device-access + # set -- "$$@" -config /etc/swarm-device-access/config.yaml + # Restrict to USB bus devices only. Nodes are /dev/bus/usb//, + # and * does not match across "/", so the glob needs both levels: + # set -- "$$@" -device-allow '/dev/bus/usb/*/*' + + # On SIGTERM or SIGINT from Swarm, stop the daemon and exit with its + # status. The trap is set before the daemon starts, and the daemon runs + # in the background: a shell waiting on a foreground child only runs + # its traps after the child exits. + pid= + stop_daemon() { + docker stop -t 10 swarm-device-access >/dev/null 2>&1 + wait $$pid + exit $$? + } + trap stop_daemon TERM INT + + # Clear a daemon container left behind by a wrapper that was killed. + docker rm -f swarm-device-access >/dev/null 2>&1 || true + docker run "$$@" & + pid=$$! + wait "$$pid" + exit $$? + # Leaves time for docker stop -t 10 before Swarm kills the wrapper. + stop_grace_period: 30s deploy: mode: global restart_policy: diff --git a/examples/v4l2-passthrough/docker-compose.yaml b/examples/v4l2-passthrough/docker-compose.yaml index d364df1..16b6ac6 100644 --- a/examples/v4l2-passthrough/docker-compose.yaml +++ b/examples/v4l2-passthrough/docker-compose.yaml @@ -14,26 +14,53 @@ services: image: docker:29 volumes: - /var/run/docker.sock:/var/run/docker.sock + # Swarm rejects privileged/cgroup/pid/userns on services. This wrapper + # launches the actual daemon with `docker run`, where those flags are valid. + entrypoint: ["sh", "-c"] command: - - run - - -i - - --rm - - --name=swarm-device-access - - --privileged - - --cgroupns=host - - --pid=host - - --userns=host - - -v - - /sys:/host/sys - - -v - - /var/run/docker.sock:/var/run/docker.sock - - -v - - /dev:/dev - - ghcr.io/leinardi/swarm-device-access:latest - # Daemon flags go after the image name; before it they are docker run flags. - # Restrict to V4L2 devices only: - # - -device-allow=/dev/video* - entrypoint: docker + - | + # docker run options, before the image. Uncomment a line to add it. + set -- --rm --name=swarm-device-access \ + --privileged --cgroupns=host --pid=host --userns=host \ + -v /sys:/host/sys \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /dev:/dev + # Reapply device rules after systemctl daemon-reload, which wipes cgroup + # BPF programs (without it the daemon warns and skips reload handling). + # The container-side path must be under /var/run: dhi.io/static has no + # /var/run -> /run symlink. + # set -- "$$@" -v /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket + # A config file for -config below. Mount the directory, not the file: + # an editor that replaces the file would leave a file mount on the old + # content, and a reload would silently apply it. + # set -- "$$@" -v /etc/swarm-device-access:/etc/swarm-device-access:ro + set -- "$$@" ghcr.io/leinardi/swarm-device-access:latest + # Daemon flags, after the image; before it they are docker run flags. + # Reload the config file with: docker kill -s HUP swarm-device-access + # set -- "$$@" -config /etc/swarm-device-access/config.yaml + # Restrict to V4L2 devices only: + # set -- "$$@" -device-allow '/dev/video*' + + # On SIGTERM or SIGINT from Swarm, stop the daemon and exit with its + # status. The trap is set before the daemon starts, and the daemon runs + # in the background: a shell waiting on a foreground child only runs + # its traps after the child exits. + pid= + stop_daemon() { + docker stop -t 10 swarm-device-access >/dev/null 2>&1 + wait $$pid + exit $$? + } + trap stop_daemon TERM INT + + # Clear a daemon container left behind by a wrapper that was killed. + docker rm -f swarm-device-access >/dev/null 2>&1 || true + docker run "$$@" & + pid=$$! + wait "$$pid" + exit $$? + # Leaves time for docker stop -t 10 before Swarm kills the wrapper. + stop_grace_period: 30s deploy: mode: global restart_policy: From 0f69c7b41b079a40b4c3338ba72409f1539cabed Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 11:57:07 +0200 Subject: [PATCH 21/37] docs(architecture): describe the reconcile path and fix drift from the code AGENTS.md, docs/architecture.md and the tracked .serena/memories/core.md described an event loop in main.go with processExistingContainers, processContainer and AddDeviceRules, none of which exist. They now describe internal/daemon (Run, listenEvents, consumeEvents, processOne and the coordinator that owns passes, retries, reservations and the time-stamped processed dedup with processedTTL), internal/processor (Reconcile, CollectMountRules, applyPinned and SetDeviceRules through a cgroup handle), the config, policy, daemon, processor and observability packages, and the current v1 and v2 file names, with the reconcile path described once. CONTRIBUTING.md drops the link to the missing UPSTREAM_AUDIT.md and says go-build targets the host's GOARCH and docker-build is single-arch. AGENTS.md and CONTRIBUTING.md say release notes come from gh release create --generate-notes (pull request titles), not commit messages. SECURITY.md notes that Swarm supports cap_add since Engine 20.10 and why the wrapper is still needed. Metrics and debug examples bind 127.0.0.1, and log examples use the current setting device rule message. --- .serena/memories/core.md | 48 +++++++---- AGENTS.md | 50 ++++++----- CONTRIBUTING.md | 12 +-- README.md | 2 +- SECURITY.md | 2 +- docs/architecture.md | 182 +++++++++++++++++++++++++-------------- docs/testing.md | 6 +- 7 files changed, 188 insertions(+), 114 deletions(-) diff --git a/.serena/memories/core.md b/.serena/memories/core.md index 8ca8df1..c354c0c 100644 --- a/.serena/memories/core.md +++ b/.serena/memories/core.md @@ -1,36 +1,45 @@ # Core — swarm-device-access -Linux-only daemon that injects cgroup BPF device-allow rules into Docker Swarm containers +Linux-only daemon that sets cgroup device-allow rules for Docker Swarm containers that bind-mount `/dev/...` paths. Solves the Docker Swarm gap where `devices:` / `device_cgroup_rules:` are silently ignored. ## Source map ``` -cmd/swarm-device-access/ # main binary (all files: //go:build linux) - main.go # run() entrypoint, event loop +cmd/swarm-device-access/ # main binary (//go:build linux) + main.go # run(): flags, settings, openat2 probe, store, processor, daemon.Run flags.go # CLI flag definitions - config.go # YAML config load + SIGHUP reload - containers.go # processExistingContainers, processContainer - devices.go # device detection / rule application - events.go # listenEvents with exponential-backoff reconnect - metrics.go # Prometheus + health endpoints - http.go # pprof debug endpoints - systemd.go # startReloadWatcher (DBus Reloading signal) + config.go # settings merge/validate, SIGHUP reloader version.go # version/commit/date (filled by ldflags) internal/ - cgroup/ # BPF/cgroup device rules (NVIDIA-derived, Apache 2.0) - api.go # Interface: GetDeviceCGroupMountPath/RootPath, AddDeviceRules - v1.go # cgroup v1: writes to devices.allow - v2.go + ebpf.go # cgroup v2: BPF_CGROUP_DEVICE via cilium/ebpf, BPF_F_ALLOW_MULTI + config/ # strict YAML loader; runtime Store/Publisher with generations + policy/ # mode, label parsing, globs: Enabled, Denied, Authorized (cross-platform) + daemon/ # event loop and coordinator + daemon.go # Run: subscribe, coordinator, startup pass, reload watcher, listenEvents + events.go # listenEvents/consumeEvents (reconnect backoff), processOne + coordinator.go # passes, retries, reservations, processed dedup (processedTTL), sweep + processor/ # one idempotent Reconcile path + processor.go # Reconcile, computeDesired, collectContainerRules + rules.go # CollectMountRules, evaluateCandidate, aggregation per device + devfs.go # devFS seam: openat2 beneath /dev, fstat, sysfs uevent; ProbeOpenat2 + reconcile.go # applyPinned, Terminate, Sweep, revokes + pin.go # pidfd pinning, /proc//cgroup + lifecycle.go # per-container run history + publish.go # PublishAndReconcile + cgroup/ # device rules (parts NVIDIA-derived, Apache 2.0) + api.go # Interface: GetDeviceCGroupMountPath/RootPath, SetDeviceRules(handle, rules); New + handle.go # CgroupHandle (directory fd + identity) + v1.go, v1set.go, ledger.go # cgroup v1: devices.allow/devices.deny, owned-exception ledger + v2.go, v2plan.go, v2ops.go, owned.go, ebpf.go # cgroup v2: owned wrapper around the runtime's BPF filter + observability/ # Prometheus Recorder (nil-safe), health/ready, pprof logger/ # slog wrapper (text/json/plain handlers, L() lazy-init) - policy/ # Container/global policy, mode parsing, label parsing (cross-platform) systemd/ # DBus Reloading signal watcher -test/integration/ # daemon integration tests (Linux only) -deployments/docker/ # Dockerfile -docs/ # architecture.md with sequence diagram +test/integration/ # daemon integration tests (build tag integration) +deployments/docker/ # Dockerfile, compose file with the sh wrapper, example config +docs/ # architecture.md, testing.md, release.md ``` ## Key invariants @@ -39,6 +48,7 @@ docs/ # architecture.md with sequence diagram - `internal/cgroup/` retains NVIDIA's Apache 2.0 copyright header — do not relicense or reformat. - `hostRootPath = "/host"` is the in-container view of host root; cgroup paths join against it. - Version strings live in `version.go`, set via `-ldflags -X main.version=...`. -- Container dedup: `processed map[string]struct{}` bridges startup enumeration ↔ live event stream. +- Every trigger runs `Processor.Reconcile`; anything uncertain applies the empty set (fail closed). +- Container dedup: the coordinator's `processed map[string]time.Time` bridges enumeration and the event stream. See `mem:tech_stack`, `mem:conventions`, `mem:suggested_commands`, `mem:task_completion`. diff --git a/AGENTS.md b/AGENTS.md index bedd415..ce62a8c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,42 +47,52 @@ The Makefile pulls shared snippets from `leinardi/make-common@v1` into `.mk/` on See [`docs/architecture.md`](docs/architecture.md) for the sequence diagram, BPF program structure, package layout, and troubleshooting guide. -Three layers, all under `cmd/swarm-device-access` + `internal/`: +Four layers, all under `cmd/swarm-device-access` + `internal/`: -**1. Event loop (`cmd/swarm-device-access/main.go`)** +**1. Entrypoint and config (`cmd/swarm-device-access/`, `internal/config/`, `internal/policy/`)** -`run()` performs: +`main.go` parses flags, merges them with the config file (`config.go`: `flagSettings`, `mergeSettings`, `validate`), refuses to start +without `openat2` (`processor.ProbeOpenat2`), creates the `config.Store`/`Publisher`, and wires the processor and `daemon.Run`. `SIGHUP` +re-reads the file (`reloader.reload`) and publishes through `Processor.PublishAndReconcile`. `internal/config` is the strict YAML loader and +the generation-numbered runtime store; `internal/policy` holds mode, label parsing and the glob predicates (`Denied`, `Authorized`). -1. `processExistingContainers` — enumerates running containers at startup and applies rules to each (closes the "daemon restart loses state" gap). -2. `startReloadWatcher` — best-effort subscribe to systemd DBus `Reloading` signal so that `systemctl daemon-reload` (which wipes cgroup BPF programs) - triggers a full re-apply. -3. `listenEvents` — subscribes to Docker `start` + `unpause` events, reconnects with exponential backoff (`minBackoff`/`maxBackoff`) instead of - `log.Fatal` on stream errors. +**2. Event loop and coordinator (`internal/daemon/`)** -A `processed map[string]struct{}` deduplicates the overlap window between the startup enumeration and the live event stream. +`Run` subscribes to Docker events (`start`, `unpause`, `die`, `destroy`) **before** the first pass, starts the coordinator, requests the +startup pass, starts the systemd reload watcher, then consumes events (`listenEvents` → `consumeEvents`, reconnecting with +`minBackoff`/`maxBackoff` backoff). The coordinator (`coordinator.go`) is the single owner of reconciliation passes: startup, `SIGHUP` and +systemd reloads only request one; it retries failed containers with backoff, reserves each container so concurrent work coalesces, sweeps +the lifecycle history, and owns the `processed map[string]time.Time` dedup (entries expire after `processedTTL`) that skips start events an +enumeration already covered. Every reconcile goes through `processOne` for metrics and logging. -For each container, `processContainer` reads its PID, detects cgroup version via `cgroup.GetDeviceCGroupVersion`, resolves the host cgroup path ( -`host/sys/...`), walks every `mount.Source` under `/dev`, and calls `cgroup.AddDeviceRules` with the major/minor pair from `unix.Stat`. +**3. Reconcile (`internal/processor/`)** -**2. Cgroup BPF (`internal/cgroup/`)** +`Processor.Reconcile(id)` is the one idempotent path: inspect, compute the complete desired set (`computeDesired`: policy, then +`CollectMountRules` per `/dev` mount, which resolves each name beneath `/dev` with `openat2`, identifies it by `fstat` and sysfs `DEVNAME`, +and aggregates per device), then apply it (`applyPinned`: pidfd pin, `/proc//cgroup`, `cgroup.OpenCgroup`, re-verify, +`SetDeviceRules`). Anything uncertain yields the empty set. `Terminate` and `Sweep` revoke and release ended runs (`lifecycle.go`). -`api.go` defines the `Interface` (`GetDeviceCGroupMountPath`, `GetDeviceCGroupRootPath`, `AddDeviceRules`) and a `New(version)` factory. `v1.go` -writes to `devices.allow`. `v2.go` + `ebpf.go` compile and attach a `BPF_CGROUP_DEVICE` program (cilium/ebpf) via `BPF_F_ALLOW_MULTI` so it composes -with the container runtime's existing filter rather than replacing it. **This code is preserved from NVIDIA's k8s-device-plugin (Apache 2.0) — touch -with care; the upstream PRs went into runc/containerd long ago.** +**4. Cgroup (`internal/cgroup/`)** -**3. Glue (`internal/logger/`, `internal/systemd/`)** +`api.go` defines the `Interface` (`GetDeviceCGroupMountPath`, `GetDeviceCGroupRootPath`, `SetDeviceRules(handle, rules)`) and the +`New(version, ledger, cache)` factory; `handle.go` holds the directory-fd `CgroupHandle`. `v1.go`/`v1set.go`/`ledger.go` write +`devices.allow` and `devices.deny` and track the daemon's own exceptions. `v2.go`, `v2plan.go`, `v2ops.go`, `owned.go` and `ebpf.go` wrap the +runtime's `BPF_CGROUP_DEVICE` filter in an owned block and swap it in with `BPF_F_ALLOW_MULTI` (or `BPF_F_REPLACE`). **Parts of this code are +preserved from NVIDIA (Apache 2.0) — touch with care.** + +**Glue (`internal/logger/`, `internal/systemd/`, `internal/observability/`)** - `logger` — slog wrapper with `text`/`json`/`plain` handlers, `-log-time` strips timestamps via a `ReplaceAttr`. `L()` lazy-inits a default INFO text handler so packages can log without explicit wiring. - `systemd` — DBus `Reloading` signal watcher. Triggers re-apply on the **completion** edge only (signal body `active=false`). The start edge (`true`) is mid-reload and races the cgroup wipe. Gracefully degrades when DBus is unreachable. +- `observability` — Prometheus `Recorder` (nil-safe), `/healthz`, `/readyz` and pprof servers. ## Runtime requirements The daemon **must** run with `privileged: true`, `cgroup: host`, `pid: host`, `userns_mode: host`, and bind mounts for `/var/run/docker.sock` and -`/sys → /host/sys`. The `hostRootPath = "/host"` constant in `main.go` is the inside-container view of the host root; cgroup paths are joined against -it. The DBus socket mount is optional — enables reload handling. Mount as `-v /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket`; the container-side path must be under `/var/run/` because `dhi.io/static` has no `/var/run → /run` symlink. +`/sys → /host/sys`. The `hostRootPath = "/host"` constant in `main.go` is the inside-container view of the host root; cgroup paths (and sysfs, when +mounted there) are joined against it. The DBus socket mount is optional — enables reload handling. Mount as `-v /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket`; the container-side path must be under `/var/run/` because `dhi.io/static` has no `/var/run → /run` symlink. ## Conventions worth knowing @@ -117,7 +127,7 @@ Skills live in `.agents/skills/` (symlinked as `.claude/skills`). Load them befo All commits MUST be Conventional Commits 1.0.0 **with a scope**: `()[!]: `, optional blank-line body and footers. Enforced by the `conventional-pre-commit` `commit-msg` hook (`--force-scope`). Types: `feat`, `fix`, `docs`, `test`, `refactor`, `perf`, `build`, `ci`, `chore`, `style`, `revert`. Breaking changes use `!` before `:` or a `BREAKING CHANGE:` footer. Release notes are -generated from these messages. Examples: `fix(processor): skip unresolvable symlinks`, `ci(dependabot): add dhi registry`. +not built from these messages: `gh release create --generate-notes` lists the merged pull requests by title. Examples: `fix(processor): skip unresolvable symlinks`, `ci(dependabot): add dhi registry`. Release versions are derived from the commit types since the last tag (`feat` minor, `fix` patch, `!`/`BREAKING CHANGE` major; anything else bumps nothing), so a wrong type ships a wrong version. PRs land as merge commits, so every commit counts, not just the diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c369ccb..f9693fa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,8 +10,8 @@ ## Building ```bash -make go-build # cross-compiles to linux/amd64 into ./dist/ -make docker-build # builds the multi-arch container image +make go-build # builds for Linux (GOOS=linux, the host's GOARCH) into ./dist/ +make docker-build # builds the container image for the host's architecture (single-arch) ``` ## Testing @@ -51,7 +51,7 @@ The files in `internal/cgroup/` carry a `Copyright (c) 2021, NVIDIA CORPORATION` - Do not remove or modify the NVIDIA copyright header. - Do not relicense this code. - Keep diffs against the upstream (`NVIDIA/libnvidia-container` `src/nvcgo/internal/cgroup/`) minimal. -- Before proposing a change, check whether the upstream already fixed it. See [`UPSTREAM_AUDIT.md`](UPSTREAM_AUDIT.md) for the current audit state. +- Before proposing a change, check whether the upstream already fixed it. - If you fix a bug that also exists upstream, note the upstream file and line in your PR description, and consider sending a patch upstream. - If golangci-lint flags something in this directory, check whether the linter exclusion in `.golangci.yaml` covers it before adding a `//nolint` directive. @@ -59,7 +59,9 @@ The files in `internal/cgroup/` carry a `Copyright (c) 2021, NVIDIA CORPORATION` All commits must follow [Conventional Commits 1.0.0](https://www.conventionalcommits.org/en/v1.0.0/) with a scope: `()[!]: `. The `conventional-pre-commit` hook enforces this on `commit-msg`, so install the hooks with -`make pre-commit-install` (it installs both the `pre-commit` and `commit-msg` hooks). Release notes are generated from these messages. +`make pre-commit-install` (it installs both the `pre-commit` and `commit-msg` hooks). Release notes are not generated from these +messages: the release workflow runs `gh release create --generate-notes`, which lists the merged pull requests by title, so write PR +titles for readers of the release notes. Common types: `feat`, `fix`, `docs`, `test`, `refactor`, `perf`, `build`, `ci`, `chore`, `style`, `revert`. Use a lower-case, imperative description: @@ -90,7 +92,7 @@ After any change to the device-apply path, verify on a live host: docker run --rm -v /dev/null:/dev/null alpine sh -c 'echo ok > /dev/null && echo "device access works"' # Watch the daemon's logs for the apply sequence -docker logs 2>&1 | grep -E 'device mount detected|adding device rule' +docker logs swarm-device-access 2>&1 | grep -E 'device mount detected|setting device rule' ``` For GPU passthrough, bind-mount `/dev/nvidia0` and run `nvidia-smi` inside the container. diff --git a/README.md b/README.md index b1ef666..ddc282e 100644 --- a/README.md +++ b/README.md @@ -462,7 +462,7 @@ With debug logs enabled, successful processing looks like this: ```text level=DEBUG device mount detected id=abc123 source=/dev/ttyS0 ... -level=DEBUG adding device rule pid=1234 type=c major=4 minor=64 +level=DEBUG setting device rule pid=1234 cgroup=/host/sys/fs/cgroup/... type=c major=4 minor=64 ``` ## 🛠️ Development diff --git a/SECURITY.md b/SECURITY.md index 1391aac..21d62fe 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -62,7 +62,7 @@ Expected response time: acknowledgment within 7 days, patch or mitigation plan w ## Known design constraints - The daemon must run as root. Dropping to a minimal capability set (`CAP_BPF`, `CAP_PERFMON`, `CAP_SYS_ADMIN`, `CAP_SYS_RESOURCE`) is theoretically possible but has not been tested across kernel versions and is not supported at this time. -- The daemon uses `privileged: true` in the reference Swarm deployment. This is required because Swarm rejects `--cap-add` on service definitions. In non-Swarm deployments, explicit capabilities can be used instead. +- The daemon uses `privileged: true` in the reference Swarm deployment, started by a wrapper service with `docker run`. Swarm services do support `cap_add` (Docker Engine 20.10 and newer), but not the host cgroup, PID and user namespaces the daemon also needs, which is why the wrapper exists. With the wrapper, or outside Swarm, explicit capabilities could be used instead of `privileged`; that is not tested or supported (see above). - **cgroup v1: grants made by a previous daemon instance are not revoked.** On cgroup v1 the daemon can only tell its own `devices.allow` exceptions from the runtime's through an in-memory ledger. A restarted daemon starts with an empty ledger, so exceptions its predecessor added look like the diff --git a/docs/architecture.md b/docs/architecture.md index a30ee73..caea5f1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -4,10 +4,11 @@ `swarm-device-access` is a privileged Linux daemon that fills a gap in Docker Swarm: Swarm services silently ignore `devices:` and `device_cgroup_rules:` in compose specs, yet the kernel still enforces cgroup device controls. A container that bind-mounts `/dev/nvidia0` can _see_ -the device file but cannot open it (`EACCES`). +the device file but cannot open it (`EPERM`). -The daemon watches Docker for container-start events and, for each new container that bind-mounts something under `/dev/`, attaches an extra BPF -`BPF_CGROUP_DEVICE` program to the container's cgroup that allows read/write/mknod on the mounted device's major/minor pair. +The daemon watches Docker for container events and, for every running container that bind-mounts something under `/dev/`, makes the device grants +in the container's cgroup equal exactly what the current policy allows it: on cgroup v2 by wrapping the runtime's `BPF_CGROUP_DEVICE` filter, on +cgroup v1 through `devices.allow` and `devices.deny`. ## High-level flow @@ -16,72 +17,107 @@ Docker daemon | | container start / unpause / die / destroy events (Unix socket) v -+---------------------------------------------------+ -| cmd/swarm-device-access/main.go | -| | -| run() | -| ├─ Events(Since=now) ← subscribe | -| ├─ processExistingContainers() ← startup | -| ├─ startReloadWatcher() ← goroutine | -| └─ listenEvents() ← main loop | -| │ | -| │ per container-start event | -| ▼ | -| processContainer(id, procRootPath, dryRun) | -| ├─ ContainerInspect → PID, Mounts | -| ├─ GetDeviceCGroupVersion(procRootPath, pid) | -| ├─ New(version) → Interface | -| ├─ GetDeviceCGroupMountPath(procRootPath, pid) | -| └─ for each /dev/... mount: | -| enumerate names (complete, ≤ 4096) | -| each name: openat2 beneath /dev → fstat | -| → sysfs DEVNAME → deny/allow on names | -| aggregate per device across mounts | -| applyMount → applyDeviceRules | -| └─ api.AddDeviceRules(cgroupPath, rules) | -+---------------------------------------------------+ ++---------------------------------------------------------------+ +| cmd/swarm-device-access/main.go | +| flags + config file → config.Store; SIGHUP → reloader | +| processor.Processor; daemon.Run | ++---------------------------------------------------------------+ + | + v ++---------------------------------------------------------------+ +| internal/daemon | +| Run | +| ├─ subscribe (Events, Since=now) ← before the listing | +| ├─ coordinator.run ← one goroutine | +| │ passes: list running containers, reconcile each | +| │ (startup, SIGHUP via PublishAndReconcile, systemd | +| │ reload), retries with backoff, lifecycle sweep | +| ├─ startReloadWatcher (systemd DBus) ← requests a pass | +| └─ listenEvents → consumeEvents ← reconnect loop | +| └─ coordinator.handleEvent | +| start/unpause → reconcileContainer → processOne | +| die/destroy → terminateContainer | ++---------------------------------------------------------------+ + | + v ++---------------------------------------------------------------+ +| internal/processor: Processor.Reconcile(id) | +| ContainerInspect → labels, State, Mounts | +| computeDesired: policy.Enabled, CollectMountRules per /dev | +| mount (openat2 beneath /dev → fstat → sysfs DEVNAME → | +| deny/allow on every name), aggregate per device | +| applyPinned: pidfd → /proc//cgroup → OpenCgroup → | +| re-verify → cgroup.Interface.SetDeviceRules(handle, rules) | ++---------------------------------------------------------------+ | | | cgroup v1 | cgroup v2 v v -devices.allow write BPF_CGROUP_DEVICE attach -(cgroupv1.go) (cgroupv2.go + ebpf.go) +devices.allow / devices.deny owned wrapper around the runtime's +(v1.go, v1set.go, ledger.go) BPF_CGROUP_DEVICE filter + (v2.go, v2plan.go, v2ops.go, owned.go, ebpf.go) ``` +### Reconciliation + +`Processor.Reconcile` is the one idempotent path every trigger runs: a start or unpause event, the startup pass, a systemd reload and a config +reload. It computes the complete desired set for the container under the current config and sets exactly that set, including none at all: a +container that policy disables or does not opt in, one with invalid labels, and one whose device set cannot be established all get the empty +set, so an earlier grant is revoked rather than kept. A container that is not running, or that Docker cannot report on, has its grants revoked in +the cgroup its lifecycle was last verified in (`Processor.Terminate` for `die` and `destroy`, a periodic `Processor.Sweep` for records whose +cgroup is gone). + +Before changing a cgroup the processor pins the container's process with a pidfd, resolves the cgroup from `/proc//cgroup`, opens it as a +directory handle, and re-checks that the process is alive, that Docker still reports it as the container's process with the same start time and, +for a grant, that it is in that cgroup. All mutation goes through the handle, never the path again. + +### Coordinator + +The coordinator (`internal/daemon/coordinator.go`) is the single owner of reconciliation passes. Startup, `SIGHUP` (through +`Processor.PublishAndReconcile`, which publishes the new config generation) and systemd reloads only request a pass; one goroutine runs them. +A request arriving while a pass runs makes that pass stop after its current container and start again under the newest request. Containers +whose reconcile failed are retried with backoff (`sda_reconcile_pending_containers`); `sda_reload_incomplete` is 1 until a pass for the latest +request has visited every running container, and `config reload complete` is logged when it has. + +Events go through the coordinator too (`handleEvent`). Every reconcile or cleanup of a container first reserves it, so work requested for a +container that is already being handled coalesces into one follow-up instead of running twice. + ### Startup back-fill -On daemon start, `processExistingContainers` calls `ContainerList` and applies rules to every already-running container. Without this, containers that -started before the daemon would have no device access until their next restart. +On daemon start, the first pass lists the running containers and reconciles each one. Without this, containers that started before the daemon +would have no device access until their next restart, and grants a previous instance left behind would stay. -The Docker event stream is opened **before** the enumeration, with `Since` set to the time captured just before subscribing. A container that starts +The Docker event stream is opened **before** the listing, with `Since` set to the time captured just before subscribing. A container that starts while the list is being processed is therefore still delivered as a `start` event instead of falling into the gap between the list and the -subscription (the typical node-boot case). The client delivers events on an unbuffered channel, so events that arrive during the enumeration wait -until the event loop starts consuming them; nothing is dropped. +subscription (the typical node-boot case). The client delivers events on an unbuffered channel, so events that arrive during the listing wait +until the event loop starts consuming them; nothing is dropped. `Run` starts consuming events once the first pass has ended. -A `processed map[string]time.Time` records, for every container processed successfully at startup, the time captured just before it was inspected. -An event for a container in the map is skipped only if its timestamp is not newer than that time: Docker emits `start` after the container is -running, so an older event was already visible to the startup inspect. A newer event (for example `docker restart` or `unpause` shortly after the -daemon started) belongs to a new run with a new cgroup and is applied. The entry is removed on the first event for that ID either way, and the whole -map is cleared after 60s to bound memory. Event time and daemon time come from the same host clock. +The coordinator's `processed map[string]time.Time` records, for every container a pass reconciled successfully, the time captured just before +it was inspected. A `start` or `unpause` event for a container in the map is skipped only if its timestamp is not newer than that time: Docker +emits `start` after the container is running, so an older event was already visible to the inspect. A newer event (for example `docker restart` +shortly after the daemon started) belongs to a new run with a new cgroup and is applied. The entry is removed by the first event for that ID +either way, and entries older than `processedTTL` (2 × the 30 s maximum backoff) are ignored and pruned. Event time and daemon time come from +the same host clock. -Startup and event-driven processing share the same code path (`processOne`), so both record the same metrics and log a `Warn` on failure -(`could not process running container` at startup, `could not process container` for events). +Passes, retries and events share the same per-container code path (`processOne`), so all of them record the same metrics and log a `Warn` on +failure (`could not process running container` in a pass, `could not reconcile pending container` on retry, `could not process container` for +events). ### Event loop reconnect `listenEvents` wraps `consumeEvents` in a reconnect loop with exponential backoff (`1s`→`30s`). If the Docker event stream drops (daemon restart, -socket error, channel close), the loop reconnects rather than exiting — replacing the upstream `log.Fatal(err)` pattern. +socket error, channel close), the loop reconnects rather than exiting. -On re-subscription `Since` is set to one nanosecond after the last received event (or the original startup time if no event arrived yet), so events -emitted during the disconnect are delivered while the last event is not replayed. Replaying it would re-apply rules, and on cgroup v2 every apply -prepends the rules again, growing the attached programs. +On re-subscription `Since` is set to one nanosecond after the last received event (or the original startup time if no event arrived yet), so +events emitted during the disconnect are delivered while the last event is not replayed. -Context cancellation (SIGTERM/SIGINT via `signal.NotifyContext`) exits cleanly at any point. +Context cancellation (SIGTERM/SIGINT via `signal.NotifyContext`) exits cleanly at any point; `Run` waits for the coordinator, so shutdown does +not cut off a reconcile in the middle of its cgroup mutation. ### systemd daemon-reload handling -`systemctl daemon-reload` clears all cgroup BPF programs. The optional DBus watcher (`internal/systemd/`) subscribes to -`org.freedesktop.systemd1.Manager.Reloading` and triggers a full re-apply when it receives the completion edge (`active=false`). It gracefully -degrades to a warning when the DBus socket is not mounted. +`systemctl daemon-reload` can detach cgroup BPF programs. The optional DBus watcher (`internal/systemd/`) subscribes to +`org.freedesktop.systemd1.Manager.Reloading` and requests a pass when it receives the completion edge (`active=false`). It gracefully degrades +to a warning when the DBus socket is not mounted. ## BPF program structure @@ -110,8 +146,17 @@ MOV R0, 1 ; allow RETURN ``` -Rules are prepended to any existing program (`BPF_F_ALLOW_MULTI` flag), so the daemon's rules compose with the container runtime's own device filter -rather than replacing it. +The rule blocks sit inside an owned wrapper around the runtime's original filter (`internal/cgroup/owned.go`): + +``` +HEADER, init, rule blocks, TRAILER, original +``` + +A request no rule block allows falls through to the original program, so the runtime's own restrictions still apply. The header and trailer are +dead stores to `R0` that identify the wrapper (`sda_devfilter` in `bpftool`); on the next change the daemon strips its wrapper back to the +original and wraps it again, so grants replace each other instead of piling up. The new program is attached with `BPF_F_ALLOW_MULTI` (or +`BPF_F_REPLACE` where supported) before the old one is detached. The README's Host Requirements section describes what the runtime filter must +look like for this to work. ## Device mount collection @@ -162,10 +207,14 @@ symlinks to directories and non-device entries, and `errors` counts per-device f | Package | Responsibility | | --- | --- | -| `cmd/swarm-device-access` | Daemon entrypoint, event loop, Docker client, apply pipeline | -| `internal/cgroup` | Device-rule application: cgroup v1 (`devices.allow` write) and v2 (BPF attach). Cgroup version + path detection via `/proc`. NVIDIA-derived code. | +| `cmd/swarm-device-access` | Entrypoint: flags, settings merge and validation, config file reload on `SIGHUP` (`config.go`), wiring | +| `internal/config` | Strict YAML loader, runtime config `Store` with generations and its `Publisher` | +| `internal/policy` | Cross-platform policy evaluation: mode (opt-in/all), label parsing, glob allow/deny (`Denied`, `Authorized`) | +| `internal/daemon` | Event loop (`Run`, `listenEvents`, `consumeEvents`, `processOne`) and the coordinator that owns passes, retries, the `processed` dedup and container reservations | +| `internal/processor` | `Reconcile`, `Terminate`, `Sweep`, `PublishAndReconcile`; desired-set computation (`CollectMountRules`, `devfs.go`), process pinning (`pin.go`), lifecycle history (`lifecycle.go`) | +| `internal/cgroup` | Device-rule application behind `Interface.SetDeviceRules(handle, rules)`: cgroup v1 (`v1.go`, `v1set.go`, `ledger.go`) and v2 (`v2.go`, `v2plan.go`, `v2ops.go`, `owned.go`, `ebpf.go`); cgroup handles (`handle.go`); `/proc` cgroup parsing. NVIDIA-derived code. | +| `internal/observability` | Prometheus metrics (`Recorder`, nil-safe), `/healthz`, `/readyz`, pprof servers | | `internal/logger` | `slog`-based singleton with `text`, `json`, `plain` handlers | -| `internal/policy` | Cross-platform policy evaluation: mode (opt-in/all), label parsing, glob allow/deny logic | | `internal/systemd` | DBus watcher for systemd `Reloading` signal | ## Policy model @@ -195,14 +244,15 @@ across daemon restarts. | `swarm-device-access.device-allow` | Comma-separated globs narrowing global allow. Empty = inherit. | | `swarm-device-access.device-deny` | Comma-separated globs added on top of global deny. | -**Decision rule** for a given container and device path: +**Decision rule** for a given container and device (see [Device mount collection](#device-mount-collection) for its names): ``` -enabled = (enable != false) AND (mode=all OR enable=true) -allowed = NOT global-denied - AND NOT container-denied - AND (global-allow empty OR path matches global-allow) - AND (container-allow empty OR path matches container-allow) +enabled = (enable != false) AND (mode=all OR enable=true) +Denied(p) = p matches global-deny OR p matches container-deny +Authorized(p) = (global-allow empty OR p matches global-allow) + AND (container-allow empty OR p matches container-allow) +granted = no name it was found by (alias, resolved, canonical) is Denied + AND Authorized(canonical) AND Authorized(resolved) ``` Global is the maximum allowed access; per-container labels can only narrow it. Deny always wins over allow. Invalid label values cause the container to be skipped (fail-closed). @@ -214,7 +264,7 @@ The daemon **must** run as root with: - `--privileged` (or equivalent capabilities: `CAP_BPF`, `CAP_PERFMON`, `CAP_SYS_ADMIN`, `CAP_SYS_RESOURCE`) - `--cgroupns=host`, `--pid=host`, `--userns=host` - `/sys` bind-mounted at `/host/sys` inside the container -- `/dev` bind-mounted (so device major/minor can be read via `unix.Stat`) +- `/dev` bind-mounted (device candidates are opened beneath it with `openat2` and identified with `fstat` and sysfs) - `/var/run/docker.sock` bind-mounted The DBus socket is optional — enables systemd reload handling. Mount as `-v /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket`; the container-side path must be under `/var/run/` because `dhi.io/static` has no `/var/run → /run` symlink. @@ -223,7 +273,7 @@ The DBus socket is optional — enables systemd reload handling. Mount as `-v /r ### Prometheus metrics (`--metrics-addr`) -When `-metrics-addr=:9090` is set, the daemon exposes: +When `-metrics-addr=127.0.0.1:9090` is set, the daemon exposes: | Endpoint | Description | | --- | --- | @@ -252,7 +302,7 @@ Metrics exposed: ### pprof debug server (`--debug-addr`) -When `-debug-addr=:6060` is set, the standard Go pprof endpoints are available at `/debug/pprof/*`. Only bind to localhost in production. +When `-debug-addr=127.0.0.1:6060` is set, the standard Go pprof endpoints are available at `/debug/pprof/*`. Only bind to localhost in production. ## Dry-run mode @@ -278,10 +328,12 @@ level=INFO msg="dry-run: would set device rules" id=abc rules=1 3. Confirm a rule was applied: ``` - level=DEBUG msg="adding device rule" pid=1234 type=c major=195 minor=0 + level=DEBUG msg="setting device rule" pid=1234 cgroup=/host/sys/fs/cgroup/... type=c major=195 minor=0 ``` -4. If step 2 fires but step 3 does not, check `cgroup version detected` — if it returns `-1`, the `/proc` parse failed (check `pid: host` is set). +4. If step 2 fires but step 3 does not, look for `container processed` with `errors` above 0, `device rule failed` or `device set incomplete` + (a device whose identity could not be established), and for `cgroup path resolved`; a `/proc` parse failure usually means `pid: host` is + not set. 5. If rules were applied but the container still gets `EACCES`, verify the container is on cgroup v2 and `BPF_F_ALLOW_MULTI` is supported (kernel ≥ 4.15). diff --git a/docs/testing.md b/docs/testing.md index 5e110a2..64a9e6f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -94,7 +94,7 @@ or deployment behavior: 3. Start a consumer container with `--label swarm-device-access.enable=true` and a real `/dev/...` bind mount and confirm the daemon logs `device mount detected` - and `adding device rule`. (For Swarm stacks, the equivalent placement is the + and `setting device rule`. (For Swarm stacks, the equivalent placement is the service's top-level `labels:`, which Docker copies into every task container; `docker service create --container-label` writes to the same location. Do not use `deploy.labels:` or `docker service create --label`: those are service @@ -118,6 +118,6 @@ or deployment behavior: (`docker ps --filter 'name=^swarm-device-access$'`; without the anchors the filter also matches the wrapper task's container). -9. Verify observability by starting with `-metrics-addr :9090` and - `-debug-addr :6060`, then checking `/healthz`, `/readyz`, `/metrics`, and +9. Verify observability by starting with `-metrics-addr 127.0.0.1:9090` and + `-debug-addr 127.0.0.1:6060`, then checking `/healthz`, `/readyz`, `/metrics`, and `/debug/pprof/`. From 5ffe5d471994a740226b43c6e1a542cdcccb4b3a Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 12:01:27 +0200 Subject: [PATCH 22/37] fix(systemd): accept the Reloading signal only from systemd Any client on the system bus can emit a signal named org.freedesktop.systemd1.Manager.Reloading, and the watcher counted every one. The match rules now name systemd as sender and /org/freedesktop/systemd1 as path, and each signal is also checked against the unique name that owns org.freedesktop.systemd1, resolved with GetNameOwner and updated only from the bus daemon's NameOwnerChanged, so a systemd re-exec does not silence the watcher and a peer cannot redirect it. The signal channel is registered before the owner is looked up, so an owner change that races the lookup is queued instead of dropped. Re-applies are coalesced: reloads that complete while one runs are absorbed into one trailing run after it. Tests cover foreign senders, foreign object paths, spoofed owner changes, following a real owner change, and coalescing. --- docs/architecture.md | 7 +- internal/systemd/reload.go | 192 ++++++++++++++++++++++++++++---- internal/systemd/reload_test.go | 137 +++++++++++++++++++++++ 3 files changed, 312 insertions(+), 24 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index caea5f1..2d11661 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -116,8 +116,11 @@ not cut off a reconcile in the middle of its cgroup mutation. ### systemd daemon-reload handling `systemctl daemon-reload` can detach cgroup BPF programs. The optional DBus watcher (`internal/systemd/`) subscribes to -`org.freedesktop.systemd1.Manager.Reloading` and requests a pass when it receives the completion edge (`active=false`). It gracefully degrades -to a warning when the DBus socket is not mounted. +`org.freedesktop.systemd1.Manager.Reloading` and requests a pass when it receives the completion edge (`active=false`). Any client on the +system bus can emit a signal with that name, so the watcher only counts one sent from `/org/freedesktop/systemd1` by the unique bus name that +owns `org.freedesktop.systemd1` (resolved with `GetNameOwner` at startup and followed through the bus daemon's `NameOwnerChanged`). Re-applies +are coalesced: reloads that complete while one runs lead to a single trailing run. It gracefully degrades to a warning when the DBus socket is +not mounted. ## BPF program structure diff --git a/internal/systemd/reload.go b/internal/systemd/reload.go index 59ee648..7a5397f 100644 --- a/internal/systemd/reload.go +++ b/internal/systemd/reload.go @@ -23,6 +23,7 @@ package systemd import ( "context" "fmt" + "sync" "github.com/godbus/dbus/v5" @@ -30,10 +31,17 @@ import ( ) const ( + systemdBusName = "org.freedesktop.systemd1" + systemdObjectPath = dbus.ObjectPath("/org/freedesktop/systemd1") reloadingInterface = "org.freedesktop.systemd1.Manager" reloadingMember = "Reloading" reloadingFullName = reloadingInterface + "." + reloadingMember + busDaemonName = "org.freedesktop.DBus" + busDaemonPath = dbus.ObjectPath("/org/freedesktop/DBus") + nameOwnerChangedMember = "NameOwnerChanged" + nameOwnerChangedFullName = busDaemonName + "." + nameOwnerChangedMember + signalChanBuffer = 16 ) @@ -43,32 +51,77 @@ const ( // after each reload completes so the caller can re-apply rules. type Watcher struct { conn *dbus.Conn + // sigCh is registered before owner is resolved, so an owner change + // that races the lookup is queued rather than dropped. + sigCh chan *dbus.Signal + // owner is the unique bus name systemd held when Open resolved it. + owner string } -// Open connects to the system DBus and registers a signal match for -// org.freedesktop.systemd1.Manager.Reloading. Returns an error if DBus is -// unavailable (no socket bind-mount, no systemd, daemon running off-host). -// Callers should treat this as non-fatal and continue without reload handling. +// Open connects to the system DBus, resolves the unique name that owns +// org.freedesktop.systemd1, and registers signal matches for systemd's +// Reloading signal (from systemd's name and object path only) and for +// changes of that name's owner. Returns an error if DBus is unavailable (no +// socket bind-mount, no systemd, daemon running off-host). Callers should +// treat this as non-fatal and continue without reload handling. func Open() (*Watcher, error) { conn, err := dbus.ConnectSystemBus() if err != nil { return nil, fmt.Errorf("connect system bus: %w", err) } - matchErr := conn.AddMatchSignal( - dbus.WithMatchInterface(reloadingInterface), - dbus.WithMatchMember(reloadingMember), - ) - if matchErr != nil { + watcher, err := subscribe(conn) + if err != nil { closeErr := conn.Close() if closeErr != nil { - logger.L().Warn("close dbus conn after AddMatchSignal failure", "err", closeErr) + logger.L().Warn("close dbus conn after subscription failure", "err", closeErr) } - return nil, fmt.Errorf("add match signal %s: %w", reloadingFullName, matchErr) + return nil, err + } + + return watcher, nil +} + +// subscribe adds the matches and registers the signal channel before it +// resolves systemd's owner: godbus drops signals no channel is registered +// for, so a NameOwnerChanged between the lookup and the registration would +// otherwise be lost and leave the watcher trusting a dead name. +func subscribe(conn *dbus.Conn) (*Watcher, error) { + err := conn.AddMatchSignal( + dbus.WithMatchSender(systemdBusName), + dbus.WithMatchObjectPath(systemdObjectPath), + dbus.WithMatchInterface(reloadingInterface), + dbus.WithMatchMember(reloadingMember), + ) + if err != nil { + return nil, fmt.Errorf("add match signal %s: %w", reloadingFullName, err) + } + + err = conn.AddMatchSignal( + dbus.WithMatchSender(busDaemonName), + dbus.WithMatchObjectPath(busDaemonPath), + dbus.WithMatchInterface(busDaemonName), + dbus.WithMatchMember(nameOwnerChangedMember), + dbus.WithMatchArg(0, systemdBusName), + ) + if err != nil { + return nil, fmt.Errorf("add match signal %s: %w", nameOwnerChangedFullName, err) + } + + sigCh := make(chan *dbus.Signal, signalChanBuffer) + conn.Signal(sigCh) + + var owner string + + err = conn.BusObject().Call(busDaemonName+".GetNameOwner", 0, systemdBusName).Store(&owner) + if err != nil { + conn.RemoveSignal(sigCh) + + return nil, fmt.Errorf("resolve the owner of %s: %w", systemdBusName, err) } - return &Watcher{conn: conn}, nil + return &Watcher{conn: conn, sigCh: sigCh, owner: owner}, nil } // Close releases the DBus connection. @@ -77,6 +130,8 @@ func (w *Watcher) Close() error { return nil } + w.conn.RemoveSignal(w.sigCh) + err := w.conn.Close() if err != nil { return fmt.Errorf("close dbus conn: %w", err) @@ -86,18 +141,31 @@ func (w *Watcher) Close() error { } // Watch blocks until ctx is canceled, invoking onReload on each completed -// systemd reload. The Reloading signal fires twice per reload: once with -// active=true when reload starts and once with active=false when it finishes. -// Only the completion edge triggers onReload — re-applying mid-reload races -// the cgroup wipe. +// systemd reload (see watch). func (w *Watcher) Watch(ctx context.Context, onReload func()) { + watch(ctx, w.sigCh, w.owner, onReload) +} + +// watch consumes signals until ctx is canceled or sigCh closes. The +// Reloading signal fires twice per reload: once with active=true when +// reload starts and once with active=false when it finishes. Only the +// completion edge counts (re-applying mid-reload races the cgroup wipe), +// and only from systemd: the signal must come from owner, the unique name +// of org.freedesktop.systemd1, and from systemd's object path. Any other +// client on the bus can emit a signal with the same name, so the match +// rules alone are not trusted. owner follows NameOwnerChanged from the bus +// daemon, so a systemd re-exec does not silence the watcher. +// +// onReload runs on its own goroutine and is coalesced: a reload that +// completes while it runs is absorbed into one more run after it +// finishes. watch returns only after the last run has finished. +func watch(ctx context.Context, sigCh <-chan *dbus.Signal, owner string, onReload func()) { log := logger.L() - sigCh := make(chan *dbus.Signal, signalChanBuffer) + runs := &coalescer{run: onReload} - w.conn.Signal(sigCh) - defer w.conn.RemoveSignal(sigCh) + defer runs.wait() - log.Debug("systemd reload watcher started") + log.Debug("systemd reload watcher started", "systemd", owner) for { select { @@ -111,16 +179,47 @@ func (w *Watcher) Watch(ctx context.Context, onReload func()) { return } - if !isReloadCompleted(sig) { + if newOwner, changed := ownerChange(sig); changed { + log.Debug("systemd bus name changed owner", "old", owner, "new", newOwner) + owner = newOwner + + continue + } + + if !fromSystemd(sig, owner) || !isReloadCompleted(sig) { continue } log.Info("systemd reload completed; re-applying device rules") - onReload() + runs.trigger() } } } +// fromSystemd reports whether sig was sent by owner from systemd's object +// path. An empty owner (systemd not on the bus) matches nothing. +func fromSystemd(sig *dbus.Signal, owner string) bool { + return sig != nil && owner != "" && sig.Sender == owner && sig.Path == systemdObjectPath +} + +// ownerChange returns the new owner of org.freedesktop.systemd1 when sig is +// the bus daemon's NameOwnerChanged for it. +func ownerChange(sig *dbus.Signal) (string, bool) { + if sig == nil || sig.Name != nameOwnerChangedFullName || + sig.Sender != busDaemonName || sig.Path != busDaemonPath || len(sig.Body) != 3 { + return "", false + } + + name, nameOK := sig.Body[0].(string) + newOwner, ownerOK := sig.Body[2].(string) + + if !nameOK || !ownerOK || name != systemdBusName { + return "", false + } + + return newOwner, true +} + // isReloadCompleted reports whether sig is a Reloading completion edge // (signal name matches and body[0] == false). Returns false for the start // edge, other signals, or malformed payloads. @@ -136,3 +235,52 @@ func isReloadCompleted(sig *dbus.Signal) bool { return !active } + +// coalescer runs a function on its own goroutine, one run at a time: +// triggers that arrive during a run collapse into one run after it. +type coalescer struct { + run func() + + mu sync.Mutex + running bool + pending bool + done sync.WaitGroup +} + +func (c *coalescer) trigger() { + c.mu.Lock() + defer c.mu.Unlock() + + if c.running { + c.pending = true + + return + } + + c.running = true + + c.done.Go(c.loop) +} + +func (c *coalescer) loop() { + for { + c.run() + + c.mu.Lock() + + if !c.pending { + c.running = false + c.mu.Unlock() + + return + } + + c.pending = false + c.mu.Unlock() + } +} + +// wait blocks until no run is in progress. +func (c *coalescer) wait() { + c.done.Wait() +} diff --git a/internal/systemd/reload_test.go b/internal/systemd/reload_test.go index 298e7fb..387911a 100644 --- a/internal/systemd/reload_test.go +++ b/internal/systemd/reload_test.go @@ -19,6 +19,8 @@ package systemd import ( + "context" + "sync/atomic" "testing" "github.com/godbus/dbus/v5" @@ -90,3 +92,138 @@ func TestIsReloadCompleted(t *testing.T) { }) } } + +const testOwner = ":1.1" + +func completed(sender string, path dbus.ObjectPath) *dbus.Signal { + return &dbus.Signal{Sender: sender, Path: path, Name: reloadingFullName, Body: []any{false}} +} + +func ownerChanged(sender, newOwner string) *dbus.Signal { + return &dbus.Signal{ + Sender: sender, + Path: busDaemonPath, + Name: nameOwnerChangedFullName, + Body: []any{systemdBusName, testOwner, newOwner}, + } +} + +// runWatch feeds signals one by one through an unbuffered channel, so each +// has been handled before the next is sent, and returns how many times +// onReload ran once watch has returned. +func runWatch(t *testing.T, signals ...*dbus.Signal) int { + t.Helper() + + var calls atomic.Int32 + + sigCh := make(chan *dbus.Signal) + done := make(chan struct{}) + + go func() { + defer close(done) + + watch(context.Background(), sigCh, testOwner, func() { calls.Add(1) }) + }() + + for _, sig := range signals { + sigCh <- sig + } + + close(sigCh) + <-done + + return int(calls.Load()) +} + +func TestWatch_OnlySystemdCounts(t *testing.T) { + t.Parallel() + + for name, sig := range map[string]*dbus.Signal{ + "foreign sender": completed(":1.99", systemdObjectPath), + "well-known name spoof": completed(systemdBusName, systemdObjectPath), + "foreign object path": completed(testOwner, "/org/example/evil"), + "start edge from owner": {Sender: testOwner, Path: systemdObjectPath, Name: reloadingFullName, Body: []any{true}}, + } { + if calls := runWatch(t, sig); calls != 0 { + t.Errorf("%s: onReload ran %d times, want 0", name, calls) + } + } + + // A peer claiming the name changed hands to it is not believed. + if calls := runWatch( + t, + ownerChanged(":1.99", ":1.99"), + completed(":1.99", systemdObjectPath), + ); calls != 0 { + t.Errorf("owner change by a peer: onReload ran %d times, want 0", calls) + } + + if calls := runWatch(t, completed(testOwner, systemdObjectPath)); calls != 1 { + t.Errorf("systemd's own completion: onReload ran %d times, want 1", calls) + } +} + +// A peer that claims the owner changed must not be believed; the bus +// daemon's NameOwnerChanged moves trust to systemd's new unique name. +func TestWatch_FollowsOwnerChangeFromTheBusDaemon(t *testing.T) { + t.Parallel() + + calls := runWatch(t, + ownerChanged(":1.99", ":1.99"), + completed(":1.99", systemdObjectPath), + ownerChanged(busDaemonName, ":1.5"), + completed(testOwner, systemdObjectPath), + completed(":1.5", systemdObjectPath), + ownerChanged(busDaemonName, ""), + completed(":1.5", systemdObjectPath), + ) + if calls != 1 { + t.Errorf("onReload ran %d times, want 1 (only :1.5 while it owned the name)", calls) + } +} + +// Reloads that complete while a re-apply runs collapse into one more run +// after it. +func TestWatch_CoalescesWhileRunning(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + started := make(chan struct{}) + release := make(chan struct{}) + + onReload := func() { + if calls.Add(1) == 1 { + close(started) + <-release + } + } + + sigCh := make(chan *dbus.Signal) + done := make(chan struct{}) + + go func() { + defer close(done) + + watch(context.Background(), sigCh, testOwner, onReload) + }() + + sigCh <- completed(testOwner, systemdObjectPath) + + <-started + + for range 3 { + sigCh <- completed(testOwner, systemdObjectPath) + } + + // Handled only after the third completion's trigger returned. + sigCh <- completed(":1.99", systemdObjectPath) + + close(release) + close(sigCh) + <-done + + if got := calls.Load(); got != 2 { + t.Errorf("onReload ran %d times, want 2 (the running one and one trailing run)", got) + } +} From fb65c2f0ad4132d3afa5ea47a151ea36b24888bf Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 12:05:13 +0200 Subject: [PATCH 23/37] fix(daemon): re-enumerate containers after the event stream reconnects A re-subscription only replays the events dockerd still buffers, and a restarted dockerd has none, so containers started during the gap could go unreconciled. Every successful re-subscription now requests a coordinator pass that re-lists and reconciles every running container; the coordinator's processed map skips replayed start events the pass already covered. A new test drives a reconnect with no explicit request and events arriving on the new stream while the pass runs, under -race, and checks that the pass lists and reconciles both running containers. --- docs/architecture.md | 5 ++- internal/daemon/coordinator.go | 6 ++++ internal/daemon/coordinator_test.go | 50 +++++++++++++++++++++++++++++ internal/daemon/events.go | 10 ++++++ 4 files changed, 70 insertions(+), 1 deletion(-) diff --git a/docs/architecture.md b/docs/architecture.md index 2d11661..3a4caaa 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -108,7 +108,10 @@ events). socket error, channel close), the loop reconnects rather than exiting. On re-subscription `Since` is set to one nanosecond after the last received event (or the original startup time if no event arrived yet), so -events emitted during the disconnect are delivered while the last event is not replayed. +events emitted during the disconnect are delivered while the last event is not replayed. dockerd only replays what it still buffers, and a +restarted dockerd has nothing buffered, so every successful re-subscription also requests a coordinator pass that re-lists and reconciles every +running container (`docker event stream re-subscribed; reconciling running containers`). Replayed start events the pass already covered are +skipped through the `processed` map. Context cancellation (SIGTERM/SIGINT via `signal.NotifyContext`) exits cleanly at any point; `Run` waits for the coordinator, so shutdown does not cut off a reconcile in the middle of its cgroup mutation. diff --git a/internal/daemon/coordinator.go b/internal/daemon/coordinator.go index 86df734..529eeaa 100644 --- a/internal/daemon/coordinator.go +++ b/internal/daemon/coordinator.go @@ -190,6 +190,12 @@ func (c *coordinator) request(generation uint64) { } } +// requestReenumeration asks for a pass under the newest generation already +// requested; the event loop calls it after re-subscribing. +func (c *coordinator) requestReenumeration() { + c.request(0) +} + // requestPass adapts request to processor.PassRequester. func (c *coordinator) requestPass(_ context.Context, generation uint64) { c.request(generation) diff --git a/internal/daemon/coordinator_test.go b/internal/daemon/coordinator_test.go index 51ee01a..3e0bf99 100644 --- a/internal/daemon/coordinator_test.go +++ b/internal/daemon/coordinator_test.go @@ -726,3 +726,53 @@ func TestCoordinator_NoCompletionWhileReconcileRunning(t *testing.T) { waitFor(t, func() bool { return len(completions(logs.String())) == 1 }) } + +// TestListenEvents_ReconnectReenumerates checks that a re-subscription +// requests a pass on its own (dockerd's replay buffer is empty after a +// restart), while events on the new stream are consumed concurrently with +// that pass; run under -race. +func TestListenEvents_ReconnectReenumerates(t *testing.T) { + closed := make(chan events.Message) + close(closed) + + msgs := make(chan events.Message) + docker := &fakeDocker{ + containers: []container.Summary{{ID: "a"}, {ID: "b"}}, + firstMsgs: msgs, + firstErrs: make(chan error), + } + apply := &recordingApply{} + coord := startCoordinator(t, docker, apply.apply) + opts := Options{ + Docker: docker, + Proc: &processor.Processor{Inspector: &recordingInspector{}, Cfg: newTestStore()}, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + done := make(chan struct{}) + + go func() { + defer close(done) + + listenEvents(ctx, opts, coord, time.Now(), + client.EventsResult{Messages: closed, Err: make(chan error)}, func() {}) + }() + + // The new stream delivers events while the pass the reconnect + // requested runs. + for round := range 20 { + msgs <- *startEvent([]string{"a", "b"}[round%2], time.Now()) + } + + waitForWithin(t, 5*minBackoff, func() bool { return docker.listCalls() >= 1 }) + waitFor(t, coord.settled) + cancel() + <-done + + seen := apply.seen() + if !slices.Contains(seen, "a") || !slices.Contains(seen, "b") { + t.Errorf("reconciled %v, want both running containers", seen) + } +} diff --git a/internal/daemon/events.go b/internal/daemon/events.go index 0eb26bb..2eff251 100644 --- a/internal/daemon/events.go +++ b/internal/daemon/events.go @@ -78,6 +78,8 @@ func resubscribeSince(initial time.Time, lastEventNano int64) string { } // listenEvents consumes Docker container events and applies device rules. +// Every re-subscription after the initial stream requests a coordinator +// pass (see requestReenumeration). // "start" covers fresh starts and restart's second phase; "unpause" covers // resume from a paused state if the cgroup state was cleared. stream is the // stream already opened by Run before the startup enumeration (zero value when @@ -130,6 +132,14 @@ func listenEvents( continue } + + // Since only replays what dockerd still buffers, and a + // restarted dockerd has nothing buffered, so events of the + // gap can be lost: re-list and reconcile every running + // container. The coordinator's processed map skips replayed + // start events the pass already covered. + log.Info("docker event stream re-subscribed; reconciling running containers") + coord.requestReenumeration() } observability.SetReady(true) From ee4511ecba81197ca0ac0ec0452196accb86830b Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 12:08:28 +0200 Subject: [PATCH 24/37] fix(policy): reject globs outside /dev A glob that is relative, not clean or not under /dev/ can never match a device path, so a deny written that way silently denied nothing. ValidateGlobs now rejects such patterns, so flags and the config file fail at startup or reload and a label makes the container fail closed. The README explains that * does not cross a slash, that /dev/bus/usb/* does not match /dev/bus/usb/001/002, suggests /dev/bus/usb/*/*, and states the new pattern rules. Tests cover relative, uncleaned and outside-/dev patterns in ValidateGlobs, labels and the global policy. --- README.md | 9 +++++++++ internal/policy/policy.go | 31 +++++++++++++++++++++++++------ internal/policy/policy_test.go | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ddc282e..bbea79f 100644 --- a/README.md +++ b/README.md @@ -377,6 +377,15 @@ If you bind-mount a directory (for example `source: /dev/dri`), the `device-allow` glob is evaluated **per child node** inside that directory — write the glob against the children, e.g. `/dev/dri/*` or `/dev/dri/renderD128`. +Globs use Go's `filepath.Match` syntax, and `*` does not match across a `/`: +`/dev/bus/usb/*` matches the bus directories `/dev/bus/usb/001`, not the device +node `/dev/bus/usb/001/002`. Write one `*` per level, e.g. `/dev/bus/usb/*/*`. +Every glob, in flags, the config file and labels, must be an absolute, clean +path under `/dev/` (no relative pattern, `..`, `//` or trailing `/`); anything +else is rejected at startup or reload, and in a label it makes the container +get no devices. Such a glob could never match a device path, so a deny written +that way would silently deny nothing. + A device can be reached under several names: the mounted name (an alias such as `/dev/disk/by-id/usb-...`), the node it resolves to, and its kernel name (`DEVNAME` in sysfs, e.g. `/dev/sda`). Deny globs are checked against all three, diff --git a/internal/policy/policy.go b/internal/policy/policy.go index d18a3b3..b939a15 100644 --- a/internal/policy/policy.go +++ b/internal/policy/policy.go @@ -19,6 +19,7 @@ package policy import ( + "errors" "fmt" "path/filepath" "sort" @@ -34,6 +35,15 @@ const ( LabelDeviceDeny = LabelPrefix + "device-deny" ) +// devPrefix is what every glob must start with: only paths under /dev are +// ever matched against it. +const devPrefix = "/dev/" + +var ( + errGlobNotClean = errors.New("glob pattern is not a clean path") + errGlobOutsideDev = errors.New("glob pattern must start with /dev/") +) + // Mode controls which containers the daemon processes by default. type Mode string @@ -139,13 +149,22 @@ func splitAndTrim(s string) []string { } // ValidateGlobs returns an error if any pattern in patterns is syntactically -// invalid. Malformed globs silently never match; this prevents silent -// policy misconfiguration. +// invalid, not clean, or not under /dev/. Such a glob silently never +// matches a device path, which for a deny means a device the operator meant +// to deny stays grantable; rejecting it prevents that misconfiguration. func ValidateGlobs(patterns []string) error { - for _, p := range patterns { - _, err := filepath.Match(p, "") + for _, pattern := range patterns { + _, err := filepath.Match(pattern, "") if err != nil { - return fmt.Errorf("invalid glob pattern %q: %w", p, err) + return fmt.Errorf("invalid glob pattern %q: %w", pattern, err) + } + + if filepath.Clean(pattern) != pattern { + return fmt.Errorf("%w: %q", errGlobNotClean, pattern) + } + + if !strings.HasPrefix(pattern, devPrefix) { + return fmt.Errorf("%w: %q", errGlobOutsideDev, pattern) } } @@ -153,7 +172,7 @@ func ValidateGlobs(patterns []string) error { } // Validate checks that the Global policy is well-formed: mode is valid and -// all glob lists are syntactically correct. +// every glob passes ValidateGlobs. func (g Global) Validate() error { _, err := ParseMode(string(g.Mode)) if err != nil { diff --git a/internal/policy/policy_test.go b/internal/policy/policy_test.go index 495999e..25a22cd 100644 --- a/internal/policy/policy_test.go +++ b/internal/policy/policy_test.go @@ -317,6 +317,16 @@ func TestParseContainer(t *testing.T) { map[string]string{policy.LabelDeviceDeny: "/dev/bad["}, nil, nil, nil, true, }, + { + "relative glob in device-deny fails closed", + map[string]string{policy.LabelDeviceDeny: "/dev/null, sda"}, + nil, nil, nil, true, + }, + { + "uncleaned glob in device-allow fails closed", + map[string]string{policy.LabelDeviceAllow: "/dev/dri/../sda"}, + nil, nil, nil, true, + }, } for _, tc := range cases { @@ -367,6 +377,19 @@ func TestValidateGlobs(t *testing.T) { {[]string{"/dev/nvidia*", "/dev/dri/*"}, false}, {[]string{"/dev/nvidia["}, true}, {[]string{"/dev/valid", "/dev/bad["}, true}, + {[]string{"/dev/bus/usb/*/*"}, false}, + // Relative, uncleaned and outside-/dev patterns never match a + // device path; for a deny that would fail open. + {[]string{"sda"}, true}, + {[]string{"dev/sda"}, true}, + {[]string{"*"}, true}, + {[]string{"/dev//sda"}, true}, + {[]string{"/dev/dri/"}, true}, + {[]string{"/dev/./sda"}, true}, + {[]string{"/dev/../etc/*"}, true}, + {[]string{"/etc/*"}, true}, + {[]string{"/dev"}, true}, + {[]string{"/devices/*"}, true}, } for _, tc := range cases { @@ -399,6 +422,16 @@ func TestGlobalValidate(t *testing.T) { policy.Global{Mode: policy.ModeAll, DeviceDeny: []string{"/dev/bad["}}, true, }, + { + "relative deny glob", + policy.Global{Mode: policy.ModeAll, DeviceDeny: []string{"sd*"}}, + true, + }, + { + "allow glob outside /dev", + policy.Global{Mode: policy.ModeAll, DeviceAllow: []string{"/sys/*"}}, + true, + }, } for _, tc := range cases { From 92dc9078bd1096af0df77ab3be0d1cd80e1feae8 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 12:11:03 +0200 Subject: [PATCH 25/37] ci(workflows): grant each job only the permissions it needs The CI and integration workflows ran every job with the repository's default token permissions. Both now default to no permissions at the top level, and each job states its own: the reviewdog jobs (actionlint, pre-commit-hooks, markdownlint, shellcheck, yamllint) get contents read and pull-requests write, since every one of those actions reports with -reporter=github-pr-review and none reports checks, so checks write is not granted. audit-deps, test and integration get contents read only. conventional-commits also gets contents read only: checked against the action's source at the pinned SHA, it takes no token and only prints annotations, so pull-requests write would be unused. --- .github/workflows/ci.yaml | 35 ++++++++++++++++++++++++++++++ .github/workflows/integration.yaml | 5 +++++ 2 files changed, 40 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cc0d125..673659b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,9 +8,17 @@ concurrency: group: "${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}" cancel-in-progress: true +# Nothing by default: every job states the permissions it needs. +permissions: {} + jobs: actionlint: runs-on: ubuntu-latest + # reviewdog posts findings as pull request review comments + # (-reporter=github-pr-review), not as checks. + permissions: + contents: read + pull-requests: write steps: - name: Checkout @@ -27,6 +35,11 @@ jobs: pre-commit-hooks: runs-on: ubuntu-latest + # reviewdog posts findings as pull request review comments + # (-reporter=github-pr-review), not as checks. + permissions: + contents: read + pull-requests: write steps: - name: Checkout @@ -60,6 +73,8 @@ jobs: # never be scanned there. audit-deps: runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout @@ -78,6 +93,8 @@ jobs: # cannot merge with a failing vet or unit test and only find out at release. test: runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout @@ -97,6 +114,11 @@ jobs: markdownlint: runs-on: ubuntu-latest + # reviewdog posts findings as pull request review comments + # (-reporter=github-pr-review), not as checks. + permissions: + contents: read + pull-requests: write steps: - name: Checkout @@ -113,6 +135,11 @@ jobs: shellcheck: runs-on: ubuntu-latest + # reviewdog posts findings as pull request review comments + # (-reporter=github-pr-review), not as checks. + permissions: + contents: read + pull-requests: write steps: - name: Checkout @@ -129,6 +156,11 @@ jobs: yamllint: runs-on: ubuntu-latest + # reviewdog posts findings as pull request review comments + # (-reporter=github-pr-review), not as checks. + permissions: + contents: read + pull-requests: write steps: - name: Checkout @@ -154,6 +186,9 @@ jobs: # merging would need one added here first. if: github.event_name == 'pull_request' runs-on: ubuntu-latest + # The action takes no token and only prints annotations. + permissions: + contents: read steps: - name: Checkout diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index f8e391a..d04d777 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -11,10 +11,15 @@ concurrency: group: "${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}" cancel-in-progress: true +# Nothing by default: every job states the permissions it needs. +permissions: {} + jobs: integration: runs-on: ubuntu-latest timeout-minutes: 15 + permissions: + contents: read steps: - name: Checkout From 3cfb8a27cb18118e8a9999a49f7e586cede662d5 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 12:13:56 +0200 Subject: [PATCH 26/37] ci(integration): gate enforcement on fork status and align the release run Fork pull requests get no secrets, so their integration job failed at the dhi.io login. The job now checks whether a pull request's head is another repository (head.repo.full_name differs from github.repository; head.repo.fork alone is also true for this repository's own branches if the repository is itself a GitHub fork): such a pull request skips the login and runs only the dry-run tests, under the job name integration (fork: enforcement skipped). Same-repo pull requests and pushes keep the login and SDA_IT_ENFORCE=1 mandatory, and a missing DHI_TOKEN there fails the job; the gate is never whether the secret is set, which would silently downgrade trusted runs. The release workflow's integration job now mirrors it: dhi.io login, SDA_IT_ENFORCE=1 and SDA_IT_REQUIRE_RELOAD=1, and the SKIP guard. docs/release.md and docs/testing.md describe both. Verifying both branches needs GitHub runs (one same-repo pull request, one fork pull request) and was not done here; actionlint and the YAML checks pass. --- .github/workflows/integration.yaml | 30 ++++++++++++++++++++++++++++++ .github/workflows/release.yaml | 24 ++++++++++++++++++++++-- docs/release.md | 5 +++-- docs/testing.md | 13 +++++++++++-- 4 files changed, 66 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index d04d777..5aaabe9 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -15,11 +15,23 @@ concurrency: permissions: {} jobs: + # A pull request from another repository (a fork) gets no secrets, so it + # cannot log in to dhi.io to build the daemon image the enforcement tests + # run. The gate is where the pull request comes from, never whether the + # secret is set: a same-repo pull request or a push without DHI_TOKEN is a + # misconfiguration and fails, instead of silently dropping to the dry-run + # tests. integration: + name: "${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'integration (fork: enforcement skipped)' || 'integration' }}" runs-on: ubuntu-latest timeout-minutes: 15 permissions: contents: read + env: + # A pull request whose head is another repository. Not head.repo.fork, + # which is also true for this repository's own branches when the + # repository itself is a GitHub fork. + FROM_FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} steps: - name: Checkout @@ -31,9 +43,20 @@ jobs: go-version-file: go.mod cache: true + - name: Require the dhi.io token + if: env.FROM_FORK != 'true' + env: + DHI_TOKEN: ${{ secrets.DHI_TOKEN }} + run: | + if [ -z "$DHI_TOKEN" ]; then + echo "::error::DHI_TOKEN is not set; enforcement tests are mandatory outside fork pull requests" + exit 1 + fi + # The daemon image the enforcement test runs is built from dhi.io base # images, which require a login. - name: Log in to Docker Hardened Images (dhi.io) + if: env.FROM_FORK != 'true' uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: dhi.io @@ -46,6 +69,7 @@ jobs: # enforcement flags turn every missing prerequisite into a failure, and # the SKIP check below makes sure a green run really exercised BPF. - name: Run integration tests + if: env.FROM_FORK != 'true' env: SDA_IT_ENFORCE: "1" SDA_IT_REQUIRE_RELOAD: "1" @@ -56,3 +80,9 @@ jobs: echo "::error::integration tests were skipped; SDA_IT_ENFORCE=1 requires every test to run" exit 1 fi + + # Fork pull requests: the dry-run tests only; the enforcement tests + # skip themselves without SDA_IT_ENFORCE. + - name: Run integration tests (fork, enforcement skipped) + if: env.FROM_FORK == 'true' + run: make go-test-integration diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 81e285a..42b5b8c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -137,9 +137,29 @@ jobs: go-version-file: go.mod cache: true - # Builds dist/ first, so the tests find the daemon binary and do not skip. + # The same enforcement run as integration.yaml: the daemon image the + # enforcement tests run is built from dhi.io base images. + - name: Log in to Docker Hardened Images (dhi.io) + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: dhi.io + username: ${{ vars.DHI_USERNAME }} + password: ${{ secrets.DHI_TOKEN }} + + # Builds dist/ and the daemon image first. The enforcement flags turn + # every missing prerequisite into a failure, and the SKIP check makes + # sure a green run really exercised BPF. - name: Run the integration tests - run: make go-test-integration + env: + SDA_IT_ENFORCE: "1" + SDA_IT_REQUIRE_RELOAD: "1" + run: | + set -o pipefail + make go-test-integration 2>&1 | tee integration.log + if grep -E -- '^[[:space:]]*--- SKIP' integration.log; then + echo "::error::integration tests were skipped; SDA_IT_ENFORCE=1 requires every test to run" + exit 1 + fi release: needs: diff --git a/docs/release.md b/docs/release.md index f69fe33..e370e4a 100644 --- a/docs/release.md +++ b/docs/release.md @@ -80,8 +80,9 @@ a tag happens before the tag exists, and everything after it can be finished by every other job waits on it. `workflow_dispatch` lets the caller pick any branch, and the release job holds `contents: write`, `packages: write` and an OIDC identity. 2. **Verify.** `test` (`make go-vet`, `make go-test`), `audit` (`make audit-deps`: govulncheck and the banned-module check) and - `integration` (`make go-test-integration`). They are copies of the CI jobs rather than a call into `ci.yaml`, so a release never - depends on which pull request last ran CI, and all of them take Go from `go.mod`, the toolchain the release builds with. + `integration` (`make go-test-integration` with `SDA_IT_ENFORCE=1` after a `dhi.io` login, failing if any test skipped). They are + copies of the CI jobs rather than a call into `ci.yaml` and `integration.yaml`, so a release never depends on which pull request + last ran CI, and all of them take Go from `go.mod`, the toolchain the release builds with. 3. **Resolve the version**, as in [Versioning](#versioning). 4. **Check the tag.** If `refs/tags/` already exists on the remote, the run continues only when it points at `HEAD`, as a recovery. Pointing anywhere else is a hard failure. The tag is then created **locally only**. diff --git a/docs/testing.md b/docs/testing.md index 64a9e6f..0f93c2a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -27,8 +27,8 @@ make go-test-integration SDA_IT_ENFORCE=1 # plus the enfor make go-test-integration SDA_IT_ENFORCE=1 SDA_IT_REQUIRE_RELOAD=1 ``` -The GitHub integration workflow runs the guarded enforcement test on every pull -request and push, with both variables set: +The GitHub integration workflow runs the guarded enforcement test on every push +and on every pull request from this repository, with both variables set: - `SDA_IT_ENFORCE=1` is the guard. With it set, a missing prerequisite fails the run instead of skipping it, and the workflow also fails the job if any test @@ -38,6 +38,15 @@ request and push, with both variables set: without the variable, a host where that cannot be proven skips only that subtest. +Pull requests from another repository (a fork) are the exception: GitHub gives +them no secrets, so they cannot log in to `dhi.io` to build the daemon image. +For them the job is named `integration (fork: enforcement skipped)` and runs +only the dry-run tests. The gate is where the pull request comes from, not +whether the secret is set: on a same-repository pull request or a push, a +missing `DHI_TOKEN` fails the job. A push to a branch of a contributor's fork +therefore fails that check, since the fork has no `DHI_TOKEN`. The release +workflow runs the same enforced suite. + Prerequisites for the enforcement test: - cgroup v2; From 1fb21a3218c9d2199592becb6aca4f5136c408af Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 12:15:17 +0200 Subject: [PATCH 27/37] build(docker): pin the Dockerfile frontend and build on the deps stage The syntax directive pulled docker/dockerfile:1.7 by tag only, so a rebuild of the same commit could use a different frontend; it is now pinned by its index digest like the base images. The deps stage downloaded modules but nothing used it, since build started again from the golang base; build now starts FROM deps, so the module download runs before the source is copied. Verified with make docker-build. --- deployments/docker/Dockerfile | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/deployments/docker/Dockerfile b/deployments/docker/Dockerfile index 75e7647..1644125 100644 --- a/deployments/docker/Dockerfile +++ b/deployments/docker/Dockerfile @@ -1,25 +1,29 @@ -# syntax=docker/dockerfile:1.7 +# syntax=docker/dockerfile:1.7@sha256:a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e -# Base images are pinned as tag@sha256:, so a rebuild of the same -# commit (a release recovery, for one) uses the same bases. Dependabot's docker -# ecosystem updates the tag and the digest together, through reviewed PRs. +# Base images, and the Dockerfile frontend above, are pinned as +# tag@sha256:, so a rebuild of the same commit (a release +# recovery, for one) uses the same bases and the same frontend. Dependabot's +# docker ecosystem updates the base images' tag and digest together, through +# reviewed PRs; the frontend pin is updated by hand. ######################## -# deps: cache modules +# deps: download modules in a layer keyed on go.mod and go.sum only ######################## FROM --platform=$BUILDPLATFORM dhi.io/golang:1-alpine3.23-dev@sha256:0d95d4f9889abcf60c182228d28fa83fde3e0dc2330bb1365e5960273b43ed57 AS deps WORKDIR /src RUN --mount=type=cache,target=/var/cache/apk \ apk add --no-cache git COPY go.mod go.sum ./ -# Use a named cache so we can reuse it in the build stage (no COPY needed) +# The named cache is shared with the build stage, which starts from this one. RUN --mount=type=cache,target=/go/pkg/mod,id=gomodcache \ go mod download ######################## -# build: compile static binary +# build: compile static binary, on top of deps (same base, platform and +# WORKDIR): the modules are downloaded before the source is copied, so a +# source-only change does not repeat that step ######################## -FROM --platform=$BUILDPLATFORM dhi.io/golang:1-alpine3.23-dev@sha256:0d95d4f9889abcf60c182228d28fa83fde3e0dc2330bb1365e5960273b43ed57 AS build +FROM deps AS build ARG VERSION="dev" ARG COMMIT="none" ARG DATE="unknown" From a56c131ac92744bbf620fec9184fe1144f2c55a6 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 12:19:44 +0200 Subject: [PATCH 28/37] fix(observability): fail startup when a listen address is busy The metrics and debug servers bound their address in a goroutine, so a busy or invalid address was only an error log while the daemon ran on without them. StartMetricsServer and StartDebugServer now bind synchronously and return a stop func or the error, then serve on the bound listener in a goroutine. run() exits 1 when either fails, and stops the metrics server again if the debug server then fails. The metrics server gets a 10s read, 30s write and 60s idle timeout. The debug server gets 10s read and 60s idle but no write timeout, because a CPU profile or trace streams for as long as the caller asks; the README and docs/architecture.md say it must stay on a loopback address. Tests cover a busy address, /healthz and /readyz transitions, /metrics, stop and context shutdown, a profile?seconds=1 request completing, and a failing second server releasing the first one's address. --- README.md | 4 +- cmd/swarm-device-access/main.go | 40 ++++- cmd/swarm-device-access/main_test.go | 92 ++++++++++++ docs/architecture.md | 7 +- internal/observability/http.go | 129 +++++++++++----- internal/observability/http_test.go | 214 +++++++++++++++++++++++++++ 6 files changed, 437 insertions(+), 49 deletions(-) create mode 100644 cmd/swarm-device-access/main_test.go create mode 100644 internal/observability/http_test.go diff --git a/README.md b/README.md index bbea79f..5c98aee 100644 --- a/README.md +++ b/README.md @@ -268,8 +268,8 @@ value is set in both places, the CLI flag wins. | `-policy-mode` | `opt-in` | `opt-in`: only `enable=true` containers. `all`: unless `enable=false`. | | `-device-allow` | `""` | Glob for `/dev/...` paths to allow, one glob per flag; repeat the flag for more (not comma-split, unlike the labels). Empty means allow all. | | `-device-deny` | `""` | Glob for `/dev/...` paths to deny, one glob per flag; repeat the flag for more (not comma-split, unlike the labels). Deny takes priority over allow. | -| `-metrics-addr` | `""` | `host:port` for Prometheus `/metrics`, `/healthz`, `/readyz`. Empty disables it. | -| `-debug-addr` | `""` | `host:port` for pprof `/debug/pprof/*`. Empty disables it. | +| `-metrics-addr` | `""` | `host:port` for Prometheus `/metrics`, `/healthz`, `/readyz`. Empty disables it. A busy or invalid address stops startup. | +| `-debug-addr` | `""` | `host:port` for pprof `/debug/pprof/*`. Empty disables it. Keep it on a loopback address such as `127.0.0.1:6060`: it has no write timeout, since profiles stream for as long as requested. A busy or invalid address stops startup. | | `-config` | `""` | Path to a YAML config file. CLI flags override file values. Reload with SIGHUP. | | `-help` | | Print this flag list and exit | diff --git a/cmd/swarm-device-access/main.go b/cmd/swarm-device-access/main.go index 06d6e9a..139ca5b 100644 --- a/cmd/swarm-device-access/main.go +++ b/cmd/swarm-device-access/main.go @@ -73,6 +73,36 @@ func startupSettings(flags *settings, cliSet map[string]bool) (settings, error) return effective, nil } +// startServers starts the configured metrics and debug servers. A busy or +// invalid address is an error; if the second server fails, the first is +// stopped again. The returned stop shuts down whatever was started. +func startServers(ctx context.Context, effective *settings) (stop func(), err error) { + stopMetrics := func() {} + + if effective.MetricsAddr != "" { + stopMetrics, err = observability.StartMetricsServer(ctx, effective.MetricsAddr) + if err != nil { + return nil, fmt.Errorf("start metrics server: %w", err) + } + } + + stopDebug := func() {} + + if effective.DebugAddr != "" { + stopDebug, err = observability.StartDebugServer(ctx, effective.DebugAddr) + if err != nil { + stopMetrics() + + return nil, fmt.Errorf("start debug server: %w", err) + } + } + + return func() { + stopDebug() + stopMetrics() + }, nil +} + func run() int { flag.Parse() @@ -157,13 +187,13 @@ func run() int { // Start optional observability servers before the main loop so they are // reachable during startup enumeration. - if effective.MetricsAddr != "" { - observability.StartMetricsServer(rootCtx, effective.MetricsAddr) - } + stopServers, err := startServers(rootCtx, &effective) + if err != nil { + log.Error("could not start observability server", "err", err) - if effective.DebugAddr != "" { - observability.StartDebugServer(rootCtx, effective.DebugAddr) + return 1 } + defer stopServers() proc := &processor.Processor{ Inspector: cli, diff --git a/cmd/swarm-device-access/main_test.go b/cmd/swarm-device-access/main_test.go new file mode 100644 index 0000000..08afafa --- /dev/null +++ b/cmd/swarm-device-access/main_test.go @@ -0,0 +1,92 @@ +//go:build linux + +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "net" + "testing" +) + +// freeAddr returns a loopback address nothing listens on. +func freeAddr(t *testing.T) string { + t.Helper() + + ln, err := listenTCP(t, "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + + addr := ln.Addr().String() + + err = ln.Close() + if err != nil { + t.Fatal(err) + } + + return addr +} + +// A busy debug address fails startup and releases the metrics address the +// first server had already bound. +func TestStartServers_SecondFailureStopsTheFirst(t *testing.T) { + busy, err := listenTCP(t, "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { _ = busy.Close() }) + + effective := defaultSettings() + effective.MetricsAddr = freeAddr(t) + effective.DebugAddr = busy.Addr().String() + + stop, err := startServers(context.Background(), &effective) + if err == nil { + stop() + t.Fatal("startServers accepted a busy debug address") + } + + again, err := listenTCP(t, effective.MetricsAddr) + if err != nil { + t.Fatalf("metrics address still bound after the failed start: %v", err) + } + + _ = again.Close() +} + +func TestStartServers_NoneConfigured(t *testing.T) { + effective := defaultSettings() + + stop, err := startServers(context.Background(), &effective) + if err != nil { + t.Fatal(err) + } + + stop() +} + +// listenTCP binds addr for a test. +func listenTCP(t *testing.T, addr string) (net.Listener, error) { + t.Helper() + + var config net.ListenConfig + + return config.Listen(context.Background(), "tcp", addr) //nolint:wrapcheck // test helper +} diff --git a/docs/architecture.md b/docs/architecture.md index 3a4caaa..007969a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -308,7 +308,12 @@ Metrics exposed: ### pprof debug server (`--debug-addr`) -When `-debug-addr=127.0.0.1:6060` is set, the standard Go pprof endpoints are available at `/debug/pprof/*`. Only bind to localhost in production. +When `-debug-addr=127.0.0.1:6060` is set, the standard Go pprof endpoints are available at `/debug/pprof/*`. The server has a 10 s read +timeout and a 60 s idle timeout but no write timeout, because a CPU profile or trace streams for as long as the caller asks; keep it on a +loopback address. + +Both servers bind their address before the daemon goes on, and a busy or invalid address makes the daemon exit with an error (the metrics +server is stopped again if the debug server then fails). The metrics server has a 10 s read, 30 s write and 60 s idle timeout. ## Dry-run mode diff --git a/internal/observability/http.go b/internal/observability/http.go index c186b5d..cd32ce0 100644 --- a/internal/observability/http.go +++ b/internal/observability/http.go @@ -19,8 +19,11 @@ package observability import ( "context" "errors" + "fmt" + "net" "net/http" "net/http/pprof" + "sync" "sync/atomic" "time" @@ -29,7 +32,15 @@ import ( "github.com/leinardi/swarm-device-access/internal/logger" ) -const shutdownTimeout = 5 * time.Second +const ( + shutdownTimeout = 5 * time.Second + readHeaderTimeout = 5 * time.Second + readTimeout = 10 * time.Second + idleTimeout = 60 * time.Second + // metricsWriteTimeout bounds a scrape. The debug server has none: a + // CPU profile or trace streams for as long as the caller asks. + metricsWriteTimeout = 30 * time.Second +) // ready tracks whether the daemon is subscribed to the Docker event stream. var ready atomic.Bool @@ -37,13 +48,51 @@ var ready atomic.Bool // SetReady updates the daemon readiness state reflected by the /readyz endpoint. func SetReady(val bool) { ready.Store(val) } -// StartMetricsServer starts an HTTP server on addr exposing: +// StartMetricsServer binds addr and serves, until ctx is done or stop is +// called: // // /metrics — Prometheus text format (default registry) // /healthz — 200 OK always (liveness) // /readyz — 200 OK once subscribed to Docker events (readiness) -func StartMetricsServer(ctx context.Context, addr string) { - log := logger.L() +// +// The address is bound before it returns, so a busy or invalid address is +// an error rather than a log line from a goroutine. stop shuts the server +// down and waits for it; it is safe to call more than once. +func StartMetricsServer(ctx context.Context, addr string) (stop func(), err error) { + listener, err := listen(ctx, addr) + if err != nil { + return nil, fmt.Errorf("metrics server: %w", err) + } + + return serve(ctx, "metrics", listener, newMetricsServer()), nil +} + +// StartDebugServer binds addr and serves the pprof endpoints until ctx is +// done or stop is called (see StartMetricsServer). The server has no write +// timeout, because profiles and traces stream for as long as the caller +// asks, so it must stay on a loopback address. +func StartDebugServer(ctx context.Context, addr string) (stop func(), err error) { + listener, err := listen(ctx, addr) + if err != nil { + return nil, fmt.Errorf("debug server: %w", err) + } + + return serve(ctx, "debug", listener, newDebugServer()), nil +} + +// listen binds addr now, so the caller learns about a busy address. +func listen(ctx context.Context, addr string) (net.Listener, error) { + var config net.ListenConfig + + listener, err := config.Listen(ctx, "tcp", addr) + if err != nil { + return nil, fmt.Errorf("listen on %s: %w", addr, err) + } + + return listener, nil +} + +func newMetricsServer() *http.Server { mux := http.NewServeMux() mux.Handle("/metrics", promhttp.Handler()) mux.HandleFunc("/healthz", func(resp http.ResponseWriter, _ *http.Request) { @@ -60,37 +109,16 @@ func StartMetricsServer(ctx context.Context, addr string) { } }) - srv := &http.Server{ - Addr: addr, + return &http.Server{ Handler: mux, - ReadHeaderTimeout: 5 * time.Second, + ReadHeaderTimeout: readHeaderTimeout, + ReadTimeout: readTimeout, + WriteTimeout: metricsWriteTimeout, + IdleTimeout: idleTimeout, } - - go func() { - log.Info("metrics server listening", "addr", addr) - - err := srv.ListenAndServe() - if err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Error("metrics server error", "err", err) - } - }() - - go func() { - <-ctx.Done() - - shutCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), shutdownTimeout) - defer cancel() - - err := srv.Shutdown(shutCtx) - if err != nil { - log.Warn("metrics server shutdown error", "err", err) - } - }() } -// StartDebugServer starts an HTTP server on addr exposing pprof endpoints. -func StartDebugServer(ctx context.Context, addr string) { - log := logger.L() +func newDebugServer() *http.Server { mux := http.NewServeMux() mux.HandleFunc("/debug/pprof/", pprof.Index) mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) @@ -98,30 +126,49 @@ func StartDebugServer(ctx context.Context, addr string) { mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - srv := &http.Server{ - Addr: addr, + return &http.Server{ Handler: mux, - ReadHeaderTimeout: 5 * time.Second, + ReadHeaderTimeout: readHeaderTimeout, + ReadTimeout: readTimeout, + IdleTimeout: idleTimeout, } +} + +// serve runs srv on ln until ctx is done or the returned stop is called. +func serve(ctx context.Context, name string, listener net.Listener, srv *http.Server) func() { + log := logger.L() + served := make(chan struct{}) + + log.Info(name+" server listening", "addr", listener.Addr().String()) go func() { - log.Info("debug server listening", "addr", addr) + defer close(served) - err := srv.ListenAndServe() + err := srv.Serve(listener) if err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Error("debug server error", "err", err) + log.Error(name+" server error", "err", err) } }() - go func() { - <-ctx.Done() - + stop := sync.OnceFunc(func() { shutCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), shutdownTimeout) defer cancel() err := srv.Shutdown(shutCtx) if err != nil { - log.Warn("debug server shutdown error", "err", err) + log.Warn(name+" server shutdown error", "err", err) + } + + <-served + }) + + go func() { + select { + case <-ctx.Done(): + stop() + case <-served: } }() + + return stop } diff --git a/internal/observability/http_test.go b/internal/observability/http_test.go new file mode 100644 index 0000000..bda538a --- /dev/null +++ b/internal/observability/http_test.go @@ -0,0 +1,214 @@ +/* + * Copyright 2026 Roberto Leinardi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package observability + +import ( + "context" + "io" + "net" + "net/http" + "strings" + "testing" + "time" +) + +// serveOnLoopback runs srv on a fresh loopback port and returns its base URL +// and stop func. +func serveOnLoopback(t *testing.T, name string, srv *http.Server) (baseURL string, stop func()) { + t.Helper() + + ln, err := listenTCP(t, "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + + stop = serve(context.Background(), name, ln, srv) + t.Cleanup(stop) + + return "http://" + ln.Addr().String(), stop +} + +func get(t *testing.T, url string, timeout time.Duration) (status int, body string) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) + if err != nil { + t.Fatal(err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("GET %s: %v", url, err) + } + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read %s: %v", url, err) + } + + return resp.StatusCode, string(data) +} + +func TestStartServers_BusyAddressIsAnError(t *testing.T) { + t.Parallel() + + busy, err := listenTCP(t, "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { _ = busy.Close() }) + + for name, start := range map[string]func(context.Context, string) (func(), error){ + "metrics": StartMetricsServer, + "debug": StartDebugServer, + } { + stop, err := start(context.Background(), busy.Addr().String()) + if err == nil { + stop() + t.Errorf("%s: started on a busy address", name) + } + } +} + +// Not parallel: readiness is package state. +func TestMetricsServer_Endpoints(t *testing.T) { + baseURL, _ := serveOnLoopback(t, "metrics", newMetricsServer()) + + if status, body := get( + t, + baseURL+"/healthz", + 5*time.Second, + ); status != http.StatusOK || + body != "ok" { + t.Errorf("/healthz = %d %q, want 200 ok", status, body) + } + + SetReady(false) + + if status, _ := get( + t, + baseURL+"/readyz", + 5*time.Second, + ); status != http.StatusServiceUnavailable { + t.Errorf("/readyz before ready = %d, want 503", status) + } + + SetReady(true) + t.Cleanup(func() { SetReady(false) }) + + if status, _ := get(t, baseURL+"/readyz", 5*time.Second); status != http.StatusOK { + t.Errorf("/readyz when ready = %d, want 200", status) + } + + if status, body := get(t, baseURL+"/metrics", 5*time.Second); status != http.StatusOK || + !strings.Contains(body, "go_goroutines") { + t.Errorf("/metrics = %d without go_goroutines", status) + } +} + +func TestServe_StopReturnsAndClosesTheListener(t *testing.T) { + t.Parallel() + + baseURL, stop := serveOnLoopback(t, "metrics", newMetricsServer()) + + done := make(chan struct{}) + + go func() { + defer close(done) + + stop() + stop() + }() + + select { + case <-done: + case <-time.After(2 * shutdownTimeout): + t.Fatal("stop did not return") + } + + req, err := http.NewRequestWithContext( + context.Background(), + http.MethodGet, + baseURL+"/healthz", + http.NoBody, + ) + if err != nil { + t.Fatal(err) + } + + resp, err := http.DefaultClient.Do(req) + if err == nil { + _ = resp.Body.Close() + + t.Error("server still answers after stop") + } +} + +func TestServe_ContextCancelStops(t *testing.T) { + t.Parallel() + + ln, err := listenTCP(t, "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + stop := serve(ctx, "metrics", ln, newMetricsServer()) + + cancel() + + done := make(chan struct{}) + + go func() { + defer close(done) + + stop() + }() + + select { + case <-done: + case <-time.After(2 * shutdownTimeout): + t.Fatal("stop after cancel did not return") + } +} + +// The debug server has no write timeout, so a CPU profile streams to the +// end instead of being cut off or refused. +func TestDebugServer_ProfileCompletes(t *testing.T) { + t.Parallel() + + baseURL, _ := serveOnLoopback(t, "debug", newDebugServer()) + + status, body := get(t, baseURL+"/debug/pprof/profile?seconds=1", 10*time.Second) + if status != http.StatusOK || body == "" { + t.Errorf("profile = %d with %d bytes, want 200 and a profile", status, len(body)) + } +} + +// listenTCP binds addr for a test. +func listenTCP(t *testing.T, addr string) (net.Listener, error) { + t.Helper() + + var config net.ListenConfig + + return config.Listen(context.Background(), "tcp", addr) //nolint:wrapcheck // test helper +} From 3702536e239ec4ebfc3ce1d8034c1ffcca802f34 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 12:21:41 +0200 Subject: [PATCH 29/37] fix(processor): avoid aliasing the config slice in a log field The mount-excluded WARN built its allow_globs and deny_globs fields with append on the global policy's slices, which the config store shares with every reconcile; with spare capacity, append wrote the container's globs into the stored slice's backing array. The fields now use slices.Concat. A test gives both global lists spare capacity and checks that the backing arrays are untouched after the WARN. --- internal/processor/rules.go | 7 ++++-- internal/processor/rules_test.go | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/internal/processor/rules.go b/internal/processor/rules.go index 820afcb..0e5b98f 100644 --- a/internal/processor/rules.go +++ b/internal/processor/rules.go @@ -25,6 +25,7 @@ import ( "fmt" "io/fs" "path/filepath" + "slices" "strings" "golang.org/x/sys/unix" @@ -211,8 +212,10 @@ func walkMount( logger.L().Warn("mount excluded: no children matched allow/deny policy", "path", source, "children_seen", state.childrenSeen, - "allow_globs", append(gpol.DeviceAllow, cpol.DeviceAllow...), - "deny_globs", append(gpol.DeviceDeny, cpol.DeviceDeny...), + // Concat, not append: append could write the container's + // globs into the spare capacity of the stored config slice. + "allow_globs", slices.Concat(gpol.DeviceAllow, cpol.DeviceAllow), + "deny_globs", slices.Concat(gpol.DeviceDeny, cpol.DeviceDeny), ) } diff --git a/internal/processor/rules_test.go b/internal/processor/rules_test.go index 0cc531a..8969105 100644 --- a/internal/processor/rules_test.go +++ b/internal/processor/rules_test.go @@ -826,6 +826,44 @@ func TestCollect_DirectoryNoChildrenMatchWarns(t *testing.T) { } } +// The WARN that lists the globs must not write the container's globs into +// the spare capacity of the global policy's slices, which the config store +// shares with every reconcile. +func TestCollect_ExclusionWarnDoesNotAliasTheGlobalPolicy(t *testing.T) { + dev := newFakeDevFS(t) + dev.directory("dri") + dev.device("dri/card0", devCard, "dri/card0") + + // Each list has spare capacity: its backing array is longer than it. + allowBacking := make([]string, 4) + allowBacking[0] = "/dev/dri/nonexistent" + allow := allowBacking[:1] + denyBacking := make([]string, 4) + denyBacking[0] = "/dev/dri/card0" + deny := denyBacking[:1] + + gpol := policy.Global{Mode: policy.ModeAll, DeviceAllow: allow, DeviceDeny: deny} + cpol := policy.Container{ + DeviceAllow: []string{"/dev/dri/other"}, + DeviceDeny: []string{"/dev/dri/x"}, + } + + buf := captureLogger(t) + + result := CollectMountRules(dev, "/dev/dri", gpol, cpol) + if len(result.Errs) != 0 || !strings.Contains(buf.String(), "mount excluded") { + t.Fatalf("setup: want the exclusion WARN, got errs=%v log:\n%s", result.Errs, buf.String()) + } + + if spare := allowBacking[1]; spare != "" { + t.Errorf("global allow backing array was written: %q", spare) + } + + if spare := denyBacking[1]; spare != "" { + t.Errorf("global deny backing array was written: %q", spare) + } +} + func TestProcessor_DevFSOpenFailureEmptiesTheSet(t *testing.T) { t.Parallel() From 9eafa71c4209046dc68005957852d7acf6f04c1f Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 12:24:17 +0200 Subject: [PATCH 30/37] fix(daemon): pick up reloaded log settings in long-lived goroutines The event loop, the coordinator, the systemd watcher, the observability servers and run() captured logger.L() once at start, so a SIGHUP that changed the log level or format never reached them. They now call logger.L() at each log site. A test changes the logger while the systemd watcher runs and checks that the next line goes to the new one. --- cmd/swarm-device-access/main.go | 16 +++++------ internal/daemon/coordinator.go | 10 +++---- internal/daemon/daemon.go | 10 +++---- internal/daemon/events.go | 13 ++++----- internal/observability/http.go | 7 +++-- internal/systemd/reload.go | 9 +++---- internal/systemd/reload_test.go | 47 +++++++++++++++++++++++++++++++++ 7 files changed, 72 insertions(+), 40 deletions(-) diff --git a/cmd/swarm-device-access/main.go b/cmd/swarm-device-access/main.go index 139ca5b..a9c47b5 100644 --- a/cmd/swarm-device-access/main.go +++ b/cmd/swarm-device-access/main.go @@ -132,10 +132,8 @@ func run() int { store, publisher := config.NewStore(effective.runtime()) - log := logger.L() - cfg := store.Load() - log.Info("swarm-device-access starting", + logger.L().Info("swarm-device-access starting", "version", version, "commit", commit, "date", date, @@ -149,7 +147,7 @@ func run() int { if cfg.DryRun { // Dry-run has no cgroup view and no saved ownership state, so it // cannot even count what a previous live run left behind. - log.Warn( + logger.L().Warn( "dry-run: grants left by a previous live run cannot be detected or cleaned in dry-run", ) } @@ -160,7 +158,7 @@ func run() int { // containers. memlockErr := rlimit.RemoveMemlock() if memlockErr != nil { - log.Warn( + logger.L().Warn( "could not remove RLIMIT_MEMLOCK; loading BPF device filters may fail", "err", memlockErr, @@ -177,7 +175,7 @@ func run() int { // API version negotiation is the client default; it runs lazily on the first request. cli, err := client.New(client.WithHost("unix://" + effective.DockerSocket)) if err != nil { - log.Error("docker client init failed", "err", err) + logger.L().Error("docker client init failed", "err", err) return 1 } @@ -189,7 +187,7 @@ func run() int { // reachable during startup enumeration. stopServers, err := startServers(rootCtx, &effective) if err != nil { - log.Error("could not start observability server", "err", err) + logger.L().Error("could not start observability server", "err", err) return 1 } @@ -222,12 +220,12 @@ func run() int { Metrics: recorder, }) if runErr != nil { - log.Error("daemon error", "err", runErr) + logger.L().Error("daemon error", "err", runErr) return 1 } - log.Info("swarm-device-access shutting down") + logger.L().Info("swarm-device-access shutting down") return 0 } diff --git a/internal/daemon/coordinator.go b/internal/daemon/coordinator.go index 529eeaa..d102520 100644 --- a/internal/daemon/coordinator.go +++ b/internal/daemon/coordinator.go @@ -290,15 +290,13 @@ func (c *coordinator) redo() { func (c *coordinator) pass(ctx context.Context, key passKey) passOutcome { defer c.markFirstPassEnded() - log := logger.L() - listCtx, cancelList := context.WithTimeout(ctx, c.timeout) list, err := c.docker.ContainerList(listCtx, client.ContainerListOptions{}) cancelList() if err != nil { - log.Warn("could not list containers; reconciliation incomplete, retrying", + logger.L().Warn("could not list containers; reconciliation incomplete, retrying", "err", fmt.Errorf("list containers: %w", err), "generation", key.generation) c.warnIncomplete() @@ -306,7 +304,7 @@ func (c *coordinator) pass(ctx context.Context, key passKey) passOutcome { return passListFailed } - log.Debug("reconciling running containers", + logger.L().Debug("reconciling running containers", "count", len(list.Items), "generation", key.generation, "epoch", key.epoch) for idx := range list.Items { @@ -573,10 +571,8 @@ func (c *coordinator) finishPass(key passKey) { // reportLocked logs completion once per request when nothing is left for // the latest request, and warns while anything is. func (c *coordinator) reportLocked() { - log := logger.L() - if c.incomplete || len(c.pending) > 0 { - log.Warn("reconciliation incomplete; retrying", + logger.L().Warn("reconciliation incomplete; retrying", "pending_containers", len(c.pending), "reload_incomplete", c.incomplete, "generation", c.latest.generation) diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 79c622d..00e968a 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -77,8 +77,6 @@ func run( opts Options, startWatcher func(context.Context, Options, func()), ) error { - log := logger.L() - since := time.Now() // The client delivers messages on an unbuffered channel, so events that @@ -88,7 +86,7 @@ func run( // events of the enumeration window are still replayed. stream, cancelStream, subErr := subscribe(ctx, opts.Docker, formatSince(since), opts.timeout()) if subErr != nil { - log.Warn("could not subscribe to docker events; will retry", "err", subErr) + logger.L().Warn("could not subscribe to docker events; will retry", "err", subErr) } coord := newCoordinator(opts.Docker, processorApply(opts.Proc), opts.Metrics, opts.timeout()) @@ -137,11 +135,9 @@ func processorApply(proc *processor.Processor) applyFn { // on hosts without systemd or without the DBus socket mounted, this logs a // warning and returns. func startReloadWatcher(ctx context.Context, _ Options, trigger func()) { - log := logger.L() - watcher, err := systemd.Open() if err != nil { - log.Warn("systemd reload handling disabled", "err", err) + logger.L().Warn("systemd reload handling disabled", "err", err) return } @@ -150,7 +146,7 @@ func startReloadWatcher(ctx context.Context, _ Options, trigger func()) { defer func() { closeErr := watcher.Close() if closeErr != nil { - log.Warn("close systemd watcher", "err", closeErr) + logger.L().Warn("close systemd watcher", "err", closeErr) } }() diff --git a/internal/daemon/events.go b/internal/daemon/events.go index 2eff251..1f71bfd 100644 --- a/internal/daemon/events.go +++ b/internal/daemon/events.go @@ -96,7 +96,6 @@ func listenEvents( stream client.EventsResult, cancelStream context.CancelFunc, ) { - log := logger.L() backoff := minBackoff timeout := opts.timeout() @@ -123,7 +122,7 @@ func listenEvents( return } - log.Error("could not subscribe to docker events, retrying", + logger.L().Error("could not subscribe to docker events, retrying", "err", subErr, "backoff", backoff) opts.Metrics.IncDockerReconnect() observability.SetReady(false) @@ -138,12 +137,12 @@ func listenEvents( // gap can be lost: re-list and reconcile every running // container. The coordinator's processed map skips replayed // start events the pass already covered. - log.Info("docker event stream re-subscribed; reconciling running containers") + logger.L().Info("docker event stream re-subscribed; reconciling running containers") coord.requestReenumeration() } observability.SetReady(true) - log.Debug("subscribed to docker events") + logger.L().Debug("subscribed to docker events") disconnected := consumeEvents( ctx, @@ -239,8 +238,6 @@ func consumeEvents( lastEventNano *int64, metrics *observability.Recorder, ) bool { - log := logger.L() - for { select { case <-ctx.Done(): @@ -263,7 +260,7 @@ func consumeEvents( return false } - log.Error("docker events stream error, reconnecting", + logger.L().Error("docker events stream error, reconnecting", "err", streamErr, "backoff", *backoff) metrics.IncDockerReconnect() observability.SetReady(false) @@ -274,7 +271,7 @@ func consumeEvents( case msg, ok := <-msgs: if !ok { - log.Warn("docker events channel closed, reconnecting", + logger.L().Warn("docker events channel closed, reconnecting", "backoff", *backoff) metrics.IncDockerReconnect() observability.SetReady(false) diff --git a/internal/observability/http.go b/internal/observability/http.go index cd32ce0..4c6a785 100644 --- a/internal/observability/http.go +++ b/internal/observability/http.go @@ -136,17 +136,16 @@ func newDebugServer() *http.Server { // serve runs srv on ln until ctx is done or the returned stop is called. func serve(ctx context.Context, name string, listener net.Listener, srv *http.Server) func() { - log := logger.L() served := make(chan struct{}) - log.Info(name+" server listening", "addr", listener.Addr().String()) + logger.L().Info(name+" server listening", "addr", listener.Addr().String()) go func() { defer close(served) err := srv.Serve(listener) if err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Error(name+" server error", "err", err) + logger.L().Error(name+" server error", "err", err) } }() @@ -156,7 +155,7 @@ func serve(ctx context.Context, name string, listener net.Listener, srv *http.Se err := srv.Shutdown(shutCtx) if err != nil { - log.Warn(name+" server shutdown error", "err", err) + logger.L().Warn(name+" server shutdown error", "err", err) } <-served diff --git a/internal/systemd/reload.go b/internal/systemd/reload.go index 7a5397f..444f6e6 100644 --- a/internal/systemd/reload.go +++ b/internal/systemd/reload.go @@ -160,12 +160,11 @@ func (w *Watcher) Watch(ctx context.Context, onReload func()) { // completes while it runs is absorbed into one more run after it // finishes. watch returns only after the last run has finished. func watch(ctx context.Context, sigCh <-chan *dbus.Signal, owner string, onReload func()) { - log := logger.L() runs := &coalescer{run: onReload} defer runs.wait() - log.Debug("systemd reload watcher started", "systemd", owner) + logger.L().Debug("systemd reload watcher started", "systemd", owner) for { select { @@ -174,13 +173,13 @@ func watch(ctx context.Context, sigCh <-chan *dbus.Signal, owner string, onReloa case sig, ok := <-sigCh: if !ok { - log.Warn("systemd signal channel closed; reload watcher exiting") + logger.L().Warn("systemd signal channel closed; reload watcher exiting") return } if newOwner, changed := ownerChange(sig); changed { - log.Debug("systemd bus name changed owner", "old", owner, "new", newOwner) + logger.L().Debug("systemd bus name changed owner", "old", owner, "new", newOwner) owner = newOwner continue @@ -190,7 +189,7 @@ func watch(ctx context.Context, sigCh <-chan *dbus.Signal, owner string, onReloa continue } - log.Info("systemd reload completed; re-applying device rules") + logger.L().Info("systemd reload completed; re-applying device rules") runs.trigger() } } diff --git a/internal/systemd/reload_test.go b/internal/systemd/reload_test.go index 387911a..fab44bd 100644 --- a/internal/systemd/reload_test.go +++ b/internal/systemd/reload_test.go @@ -19,11 +19,16 @@ package systemd import ( + "bytes" "context" + "log/slog" + "strings" "sync/atomic" "testing" "github.com/godbus/dbus/v5" + + "github.com/leinardi/swarm-device-access/internal/logger" ) func TestIsReloadCompleted(t *testing.T) { @@ -227,3 +232,45 @@ func TestWatch_CoalescesWhileRunning(t *testing.T) { t.Errorf("onReload ran %d times, want 2 (the running one and one trailing run)", got) } } + +// A reload of the log settings (logger.Set) reaches a watcher that is +// already running: it logs through logger.L at each site. +func TestWatch_UsesTheCurrentLogger(t *testing.T) { + prev := logger.L() + + t.Cleanup(func() { logger.Set(prev) }) + + var before, after bytes.Buffer + + logger.Set(slog.New(slog.NewTextHandler(&before, nil))) + + sigCh := make(chan *dbus.Signal) + done := make(chan struct{}) + + go func() { + defer close(done) + + watch(context.Background(), sigCh, testOwner, func() {}) + }() + + // Each send returns once the previous signal has been handled. + sigCh <- completed(testOwner, systemdObjectPath) + + sigCh <- completed(":1.99", systemdObjectPath) + + logger.Set(slog.New(slog.NewTextHandler(&after, nil))) + + sigCh <- completed(testOwner, systemdObjectPath) + + close(sigCh) + <-done + + const msg = "systemd reload completed" + if strings.Count(before.String(), msg) != 1 || strings.Count(after.String(), msg) != 1 { + t.Errorf( + "want one line per logger; before:\n%s\nafter:\n%s", + before.String(), + after.String(), + ) + } +} From 282291660278cb3b479223e2bd75a7fa1e03b051 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 12:26:43 +0200 Subject: [PATCH 31/37] fix(processor): only walk bind mounts A mount was walked whenever its Source was under /dev, whatever its type, but only a bind mount's Source is a host path. The processor now requires Type bind before treating a mount as a device mount; volumes, tmpfs and other types are ignored, and /dev is not even opened for a container without a /dev bind mount. A test covers volume, tmpfs, named pipe and untyped mounts with a /dev Source next to a real bind mount. --- docs/architecture.md | 3 +- internal/processor/processor.go | 12 ++++- internal/processor/rules_test.go | 81 +++++++++++++++++++++++++++++--- 3 files changed, 87 insertions(+), 9 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 007969a..d77c9b0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -166,7 +166,8 @@ look like for this to work. ## Device mount collection -For every bind mount whose source is `/dev` or lives under `/dev/`, the processor collects one device rule per device node: +For every bind mount (`Type: bind`) whose source is `/dev` or lives under `/dev/`, the processor collects one device rule per device node. +Other mount types (volumes, tmpfs) are never walked, whatever their `Source` says: only a bind mount's source is a host path. - **Single-file mount** (e.g. `/dev/nvidia0`): the source is one candidate. A missing or non-device source is an error, because it was mounted explicitly. diff --git a/internal/processor/processor.go b/internal/processor/processor.go index fbb9f05..937bfd2 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -28,6 +28,7 @@ import ( "time" "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/mount" "github.com/moby/moby/client" "github.com/leinardi/swarm-device-access/internal/cgroup" @@ -425,6 +426,13 @@ type containerRules struct { devMounts int } +// isDeviceMount reports whether mnt bind-mounts something from /dev. Only a +// bind mount's Source is a host path; a volume or tmpfs mount is never +// walked, whatever its Source says. +func isDeviceMount(mnt *container.MountPoint) bool { + return mnt.Type == mount.TypeBind && IsMountSource(mnt.Source) +} + // collectDevices opens /dev for this pass and collects the container's // rules. Failing to open it leaves every device unresolved. func (p *Processor) collectDevices( @@ -435,7 +443,7 @@ func (p *Processor) collectDevices( cpol policy.Container, ) containerRules { if !slices.ContainsFunc(mounts, func(mnt container.MountPoint) bool { - return IsMountSource(mnt.Source) + return isDeviceMount(&mnt) }) { return containerRules{} } @@ -485,7 +493,7 @@ func collectContainerRules( ) for _, mnt := range mounts { - if !IsMountSource(mnt.Source) { + if !isDeviceMount(&mnt) { continue } diff --git a/internal/processor/rules_test.go b/internal/processor/rules_test.go index 8969105..1126be2 100644 --- a/internal/processor/rules_test.go +++ b/internal/processor/rules_test.go @@ -24,6 +24,7 @@ import ( "io/fs" "os" "path/filepath" + "slices" "strconv" "strings" "testing" @@ -759,8 +760,8 @@ func TestCollect_IncompleteEnumeration(t *testing.T) { info := &container.InspectResponse{ State: &container.State{Running: true, Pid: 1}, Mounts: []container.MountPoint{ - {Source: "/dev/null", Destination: "/dev/null"}, - {Source: "/dev/dri", Destination: "/dev/dri"}, + {Source: "/dev/null", Destination: "/dev/null", Type: mount.TypeBind}, + {Source: "/dev/dri", Destination: "/dev/dri", Type: mount.TypeBind}, }, } @@ -864,13 +865,81 @@ func TestCollect_ExclusionWarnDoesNotAliasTheGlobalPolicy(t *testing.T) { } } +// Only a bind mount's Source is a host path under /dev. A volume or tmpfs +// mount that reports such a Source is never walked: /dev is not even +// opened for it. +func TestCollect_OnlyBindMountsAreWalked(t *testing.T) { + t.Parallel() + + dev := newFakeDevFS(t) + dev.device("sda", devSda, "sda") + + nonBind := []container.MountPoint{ + {Type: mount.TypeVolume, Source: "/dev/sda", Destination: "/data"}, + {Type: mount.TypeTmpfs, Source: "/dev/sda", Destination: "/tmp"}, + {Type: mount.TypeNamedPipe, Source: "/dev/sda", Destination: "/pipe"}, + {Source: "/dev/sda", Destination: "/untyped"}, + } + + got := collectContainerRules( + dev, + "abc", + 1, + nonBind, + policy.Global{Mode: policy.ModeAll}, + policy.Container{}, + nil, + ) + if got.devMounts != 0 || len(got.granted) != 0 || len(got.deviceErrs) != 0 { + t.Errorf("collected %+v from non-bind mounts, want nothing", got) + } + + proc := &Processor{devfs: func() (devFS, error) { + t.Error("/dev opened for a container without a /dev bind mount") + + return nil, unix.ENOSYS + }} + info := &container.InspectResponse{ + State: &container.State{Running: true, Pid: 1}, + Mounts: nonBind, + } + + desired := proc.computeDesired( + "abc", + info, + config.Runtime{Policy: policy.Global{Mode: policy.ModeAll}}, + ) + if len(desired.rules) != 0 || desired.incomplete != nil { + t.Errorf("desired = %+v, want an empty, complete set", desired) + } + + bind := slices.Concat(nonBind, []container.MountPoint{ + {Type: mount.TypeBind, Source: "/dev/sda", Destination: "/dev/sda"}, + }) + wantRules( + t, + collectContainerRules( + dev, + "abc", + 1, + bind, + policy.Global{Mode: policy.ModeAll}, + policy.Container{}, + nil, + ), + devSda, + ) +} + func TestProcessor_DevFSOpenFailureEmptiesTheSet(t *testing.T) { t.Parallel() proc := &Processor{devfs: func() (devFS, error) { return nil, unix.ENOSYS }} info := &container.InspectResponse{ - State: &container.State{Running: true, Pid: 1}, - Mounts: []container.MountPoint{{Source: "/dev/null", Destination: "/dev/null"}}, + State: &container.State{Running: true, Pid: 1}, + Mounts: []container.MountPoint{ + {Source: "/dev/null", Destination: "/dev/null", Type: mount.TypeBind}, + }, } desired := proc.computeDesired( @@ -896,8 +965,8 @@ func TestProcessor_UnresolvedCandidateEmptiesTheSet(t *testing.T) { info := &container.InspectResponse{ State: &container.State{Running: true, Pid: 1}, Mounts: []container.MountPoint{ - {Source: "/dev/null", Destination: "/dev/null"}, - {Source: "/dev/sda", Destination: "/dev/sda"}, + {Source: "/dev/null", Destination: "/dev/null", Type: mount.TypeBind}, + {Source: "/dev/sda", Destination: "/dev/sda", Type: mount.TypeBind}, }, } From 3ef48eafa848980d33d497faccec919d33094454 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 12:30:24 +0200 Subject: [PATCH 32/37] test(daemon): replace sleep-based cancellation in the event tests The consumeEvents tests in events_test.go and daemon_test.go ended by cancelling 20ms after starting, which either raced a slow runner or waited for nothing. A test expecting N applies now cancels from inside apply once the N-th call is observed (cancelAfter), never earlier. A test expecting zero applies runs the consumer in a goroutine and waits for a positive signal that the event was handled, the coordinator's new debug line event already covered by a pass; skipped, before it cancels, with a two-second wait kept only as the documented upper bound. --- internal/daemon/coordinator.go | 3 + internal/daemon/daemon_test.go | 9 +- internal/daemon/events_test.go | 186 +++++++++++++++++++++------------ 3 files changed, 129 insertions(+), 69 deletions(-) diff --git a/internal/daemon/coordinator.go b/internal/daemon/coordinator.go index d102520..fe94df7 100644 --- a/internal/daemon/coordinator.go +++ b/internal/daemon/coordinator.go @@ -357,6 +357,9 @@ func (c *coordinator) handleEvent(ctx context.Context, msg *events.Message) { c.terminateContainer(ctx, containerID, eventTime) default: if c.skipEvent(containerID, eventTime) { + logger.L().Debug("event already covered by a pass; skipped", + "id", containerID, "action", string(msg.Action)) + return } diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index 884a599..dad9201 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -538,16 +538,15 @@ func TestProcessOne_StartupFailureMatchesEventPath(t *testing.T) { msgs <- events.Message{Actor: events.Actor{ID: "bad"}, TimeNano: time.Now().UnixNano()} - go func() { - time.Sleep(20 * time.Millisecond) - cancel() - }() + // The failure is recorded before consumeEvents returns, so ending from + // inside the one expected apply loses nothing. + failOnce, _ := cancelAfter(1, cancel, errApplyFailed) consumeEvents( ctx, msgs, errs, - testCoordinatorWith(failingApply, metrics, nil), + testCoordinatorWith(failOnce, metrics, nil), &backoff, new(int64), metrics, diff --git a/internal/daemon/events_test.go b/internal/daemon/events_test.go index 864b87d..1c8bfcf 100644 --- a/internal/daemon/events_test.go +++ b/internal/daemon/events_test.go @@ -23,6 +23,7 @@ import ( "errors" "maps" "strings" + "sync" "sync/atomic" "testing" "time" @@ -255,24 +256,12 @@ func TestConsumeEvents_EventCallsApply(t *testing.T) { msgs, errs := makeChans(2, 0) backoff := minBackoff - var called atomic.Int32 - - apply := func(_ context.Context, _ string) error { - called.Add(1) - - return nil - } + apply, called := cancelAfter(2, cancel, nil) msgs <- events.Message{Actor: events.Actor{ID: "container-1"}} msgs <- events.Message{Actor: events.Actor{ID: "container-2"}} - // Cancel after a brief delay so consumeEvents exits cleanly. - go func() { - time.Sleep(20 * time.Millisecond) - cancel() - }() - consumeEvents( ctx, msgs, @@ -298,30 +287,21 @@ func TestConsumeEvents_DeduplicatesProcessedIDs(t *testing.T) { "already-seen": time.Now(), } - var called atomic.Int32 - - apply := func(_ context.Context, _ string) error { - called.Add(1) - - return nil - } + apply, called := cancelAfter(1, cancel, nil) msgs <- events.Message{Actor: events.Actor{ID: "already-seen"}} - go func() { - time.Sleep(20 * time.Millisecond) - cancel() - }() - - consumeEvents( - ctx, - msgs, - errs, - testCoordinatorWith(apply, nil, processed), - &backoff, - new(int64), - nil, - ) + consumeUntilSkipped(t, ctx, cancel, func() { + consumeEvents( + ctx, + msgs, + errs, + testCoordinatorWith(apply, nil, processed), + &backoff, + new(int64), + nil, + ) + }) if called.Load() != 0 { t.Errorf("apply called %d times for deduplicated ID, want 0", called.Load()) @@ -363,16 +343,13 @@ func TestConsumeEvents_BackoffResetsOnSuccessfulEvent(t *testing.T) { msgs <- events.Message{Actor: events.Actor{ID: "c1"}} - go func() { - time.Sleep(20 * time.Millisecond) - cancel() - }() + apply, _ := cancelAfter(1, cancel, nil) consumeEvents( ctx, msgs, errs, - testCoordinatorWith(noopApply, nil, nil), + testCoordinatorWith(apply, nil, nil), &backoff, new(int64), nil, @@ -408,30 +385,29 @@ func TestConsumeEvents_RestartWithinWindow(t *testing.T) { backoff := minBackoff processed := map[string]time.Time{"c1": recordedAt} - var called atomic.Int32 + apply, called := cancelAfter(1, cancel, nil) - apply := func(_ context.Context, _ string) error { - called.Add(1) + msgs <- events.Message{Actor: events.Actor{ID: "c1"}, TimeNano: tc.eventTime.UnixNano()} - return nil + consume := func() { + consumeEvents( + ctx, + msgs, + errs, + testCoordinatorWith(apply, nil, processed), + &backoff, + new(int64), + nil, + ) } - msgs <- events.Message{Actor: events.Actor{ID: "c1"}, TimeNano: tc.eventTime.UnixNano()} - - go func() { - time.Sleep(20 * time.Millisecond) - cancel() - }() - - consumeEvents( - ctx, - msgs, - errs, - testCoordinatorWith(apply, nil, processed), - &backoff, - new(int64), - nil, - ) + // A positive case ends from inside apply; a skipped event has + // no apply to end it, so it waits for the skip log instead. + if tc.wantApply > 0 { + consume() + } else { + consumeUntilSkipped(t, ctx, cancel, consume) + } if called.Load() != tc.wantApply { t.Errorf("apply called %d times, want %d", called.Load(), tc.wantApply) @@ -457,16 +433,13 @@ func TestConsumeEvents_TracksLastEventNano(t *testing.T) { msgs <- events.Message{Actor: events.Actor{ID: "c2"}, TimeNano: 300} - go func() { - time.Sleep(20 * time.Millisecond) - cancel() - }() + apply, _ := cancelAfter(2, cancel, nil) consumeEvents( ctx, msgs, errs, - testCoordinatorWith(noopApply, nil, nil), + testCoordinatorWith(apply, nil, nil), &backoff, &lastEventNano, nil, @@ -505,6 +478,91 @@ func TestResubscribeSince(t *testing.T) { // testCoordinator returns a coordinator for consumer tests, with processed // as its enumeration entries when non-nil. +// cancelAfter returns an apply that counts its calls and returns result; +// the n-th call cancels the context, so a test expecting n applies ends as +// soon as the last one is observed and never earlier. +func cancelAfter(n int32, cancel context.CancelFunc, result error) (applyFn, *atomic.Int32) { + var calls atomic.Int32 + + return func(_ context.Context, _ string) error { + if calls.Add(1) == n { + cancel() + } + + return result + }, &calls +} + +// skipSignalTimeout is how long a test expecting a skipped event waits for +// the skip log before giving up. It is only an upper bound: the test goes on +// as soon as the log line appears. +const skipSignalTimeout = 2 * time.Second + +// consumeUntilSkipped runs consume in a goroutine, waits for the coordinator's +// skip log (the positive signal that the event was handled), then cancels ctx +// and waits for consume to return. A test expecting zero applies cannot end +// from inside apply, and canceling before the event is handled would pass +// without checking anything. +func consumeUntilSkipped( + t *testing.T, + ctx context.Context, + cancel context.CancelFunc, + consume func(), +) { + t.Helper() + + skipped := logSignal(t, "event already covered by a pass; skipped") + done := make(chan struct{}) + + go func() { + defer close(done) + + consume() + }() + + select { + case <-skipped: + case <-ctx.Done(): + t.Error("context ended before the event was skipped") + case <-time.After(skipSignalTimeout): + t.Error("the event was not reported as skipped") + } + + cancel() + <-done +} + +// logSignal sends logger.L() to a writer that closes the returned channel +// the first time a line contains msg. The writer is safe for the concurrent +// use a running consumer makes of it. +func logSignal(t *testing.T, msg string) <-chan struct{} { + t.Helper() + + sink := &signalWriter{want: msg, seen: make(chan struct{})} + setTestLogger(t, sink) + + return sink.seen +} + +type signalWriter struct { + mu sync.Mutex + want string + seen chan struct{} + done bool +} + +func (w *signalWriter) Write(line []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + + if !w.done && strings.Contains(string(line), w.want) { + w.done = true + close(w.seen) + } + + return len(line), nil +} + func testCoordinator(processed map[string]time.Time) *coordinator { return testCoordinatorWith(noopApply, nil, processed) } From 4f0b2e78e029a1f8c3c05c0a866f425a740572c7 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 12:57:25 +0200 Subject: [PATCH 33/37] fix(systemd): keep the reload watcher subscribed without blocking startup The watcher now runs on its own goroutine, started before the startup pass, so a slow or wedged system bus never delays the pass or the event loop. Each connection attempt is bounded, a failed attempt or a lost subscription (a DBus restart) is retried with backoff, and every successful subscription requests a full pass, since a reload that completed while no watcher was subscribed, the startup pass included, was never seen. --- docs/architecture.md | 15 ++- internal/daemon/daemon.go | 61 +++++------ internal/daemon/daemon_test.go | 51 ++++++++- internal/systemd/reload.go | 178 ++++++++++++++++++++++++++++++- internal/systemd/reload_test.go | 180 ++++++++++++++++++++++++++++++++ 5 files changed, 444 insertions(+), 41 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index d77c9b0..8b9208a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -32,7 +32,8 @@ Docker daemon | │ passes: list running containers, reconcile each | | │ (startup, SIGHUP via PublishAndReconcile, systemd | | │ reload), retries with backoff, lifecycle sweep | -| ├─ startReloadWatcher (systemd DBus) ← requests a pass | +| ├─ startReloadWatcher (systemd DBus) ← own goroutine, | +| │ requests a pass per subscription and per reload | | └─ listenEvents → consumeEvents ← reconnect loop | | └─ coordinator.handleEvent | | start/unpause → reconcileContainer → processOne | @@ -122,8 +123,14 @@ not cut off a reconcile in the middle of its cgroup mutation. `org.freedesktop.systemd1.Manager.Reloading` and requests a pass when it receives the completion edge (`active=false`). Any client on the system bus can emit a signal with that name, so the watcher only counts one sent from `/org/freedesktop/systemd1` by the unique bus name that owns `org.freedesktop.systemd1` (resolved with `GetNameOwner` at startup and followed through the bus daemon's `NameOwnerChanged`). Re-applies -are coalesced: reloads that complete while one runs lead to a single trailing run. It gracefully degrades to a warning when the DBus socket is -not mounted. +are coalesced: reloads that complete while one runs lead to a single trailing run. + +The watcher runs in its own goroutine, started before the startup pass, so connecting to the bus never delays the pass or the event loop. +Each connection attempt is bounded (10 s). An attempt that fails, and a subscription that is lost later (the bus daemon restarting, for +one), is retried with a backoff of 1 s doubling to 5 min, which starts over once a subscription has held for 5 min; the first failure is logged +at `Warn`, later ones at `Debug`. Every successful +subscription requests a pass, because a reload that completed while the watcher was not subscribed, including one during the startup +pass, sent a signal nobody received. ## BPF program structure @@ -351,7 +358,7 @@ level=INFO msg="dry-run: would set device rules" id=abc rules=1 ### Daemon does not re-apply rules after `systemctl daemon-reload` -Ensure the host DBus socket is bind-mounted as `-v /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket`. Without it, the reload watcher is disabled (logged at `Warn` on startup). The container-side path must be `/var/run/dbus/system_bus_socket` — `dhi.io/static` has no `/var/run → /run` symlink. +Ensure the host DBus socket is bind-mounted as `-v /run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket`. Without it, the reload watcher cannot subscribe: the first failure is logged at `Warn` (`systemd reload handling unavailable; retrying in the background`) and it keeps retrying at `Debug`. The container-side path must be `/var/run/dbus/system_bus_socket` — `dhi.io/static` has no `/var/run → /run` symlink. ### Daemon cannot connect to Docker diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 00e968a..b8d5f8a 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -75,7 +75,7 @@ func Run(ctx context.Context, opts Options) error { func run( ctx context.Context, opts Options, - startWatcher func(context.Context, Options, func()), + startWatcher func(ctx context.Context, onReload, onSubscribed func()), ) error { since := time.Now() @@ -101,10 +101,25 @@ func run( // Waited for before Run returns, so shutdown does not cut off a // reconcile in the middle of its cgroup mutation. - var coordinator sync.WaitGroup - defer coordinator.Wait() - - coordinator.Go(func() { coord.run(ctx) }) + var background sync.WaitGroup + defer background.Wait() + + background.Go(func() { coord.run(ctx) }) + + // The reload watcher starts before the startup pass, on its own + // goroutine so a wedged system bus never delays the pass or the event + // loop. Every time it subscribes, the first time included, it requests + // a pass: a daemon-reload that completed before it was listening (for + // example during the startup pass) was not seen. + background.Go(func() { + startWatcher(ctx, + func() { + opts.Metrics.IncReloadReapply() + coord.request(opts.Proc.Cfg.Generation()) + }, + func() { coord.request(opts.Proc.Cfg.Generation()) }, + ) + }) // The startup pass reconciles every running container under the // startup config. In live mode that replaces or strips grants a @@ -112,11 +127,6 @@ func run( coord.request(opts.Proc.Cfg.Generation()) coord.awaitFirstPass(ctx) - startWatcher(ctx, opts, func() { - opts.Metrics.IncReloadReapply() - coord.request(opts.Proc.Cfg.Generation()) - }) - listenEvents(ctx, opts, coord, since, stream, cancelStream) return nil @@ -129,27 +139,12 @@ func processorApply(proc *processor.Processor) applyFn { } } -// startReloadWatcher tries to subscribe to systemd's DBus Reloading signal so -// that when daemon-reload wipes the cgroup BPF programs, trigger requests a -// pass that re-applies rules to every running container. DBus is optional — -// on hosts without systemd or without the DBus socket mounted, this logs a -// warning and returns. -func startReloadWatcher(ctx context.Context, _ Options, trigger func()) { - watcher, err := systemd.Open() - if err != nil { - logger.L().Warn("systemd reload handling disabled", "err", err) - - return - } - - go func() { - defer func() { - closeErr := watcher.Close() - if closeErr != nil { - logger.L().Warn("close systemd watcher", "err", closeErr) - } - }() - - watcher.Watch(ctx, trigger) - }() +// startReloadWatcher keeps a subscription to systemd's DBus Reloading +// signal until ctx is done (see systemd.Supervise). daemon-reload can wipe +// the cgroup BPF programs, so each completed reload requests a pass that +// re-applies rules to every running container. DBus is optional: on hosts +// without systemd or without the DBus socket mounted, this warns once and +// keeps retrying in the background. +func startReloadWatcher(ctx context.Context, onReload, onSubscribed func()) { + systemd.Supervise(ctx, onReload, onSubscribed) } diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index dad9201..c1cd1b4 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -206,7 +206,7 @@ func waitForWithin(t *testing.T, limit time.Duration, cond func() bool) { t.Fatal("condition not met before deadline") } -func noopWatcher(context.Context, Options, func()) {} +func noopWatcher(context.Context, func(), func()) {} // newTestStore returns a store holding the zero config. func newTestStore() *config.Store { @@ -269,6 +269,55 @@ func TestRun_SubscribesBeforeEnumerating(t *testing.T) { } } +// TestRun_WatcherRunsBesideStartupAndEvents checks that the reload watcher +// cannot hold up the startup pass or the event loop (it blocks here for the +// whole run, like a wedged system bus), and that each subscription it +// reports requests a pass, covering a reload it could not have seen. +func TestRun_WatcherRunsBesideStartupAndEvents(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + msgs := make(chan events.Message, 1) + docker := &fakeDocker{ + containers: []container.Summary{{ID: "listed"}}, + firstMsgs: msgs, + firstErrs: make(chan error), + } + + insp := &recordingInspector{} + opts := Options{ + Docker: docker, + Proc: &processor.Processor{Inspector: insp, Cfg: newTestStore()}, + } + + subscribed := make(chan func(), 1) + blockingWatcher := func(ctx context.Context, _, onSubscribed func()) { + subscribed <- onSubscribed + + <-ctx.Done() + } + + done := make(chan struct{}) + + go func() { + defer close(done) + + _ = run(ctx, opts, blockingWatcher) + }() + + msgs <- events.Message{Actor: events.Actor{ID: "started"}, TimeNano: time.Now().UnixNano()} + + waitFor(t, func() bool { return slices.Contains(insp.inspected(), "started") }) + + before := docker.listCalls() + + (<-subscribed)() + + waitFor(t, func() bool { return docker.listCalls() > before }) + cancel() + <-done +} + // TestListenEvents_ReconnectSinceAfterLastEvent checks that a re-subscription // starts one nanosecond after the last received event. func TestListenEvents_ReconnectSinceAfterLastEvent(t *testing.T) { diff --git a/internal/systemd/reload.go b/internal/systemd/reload.go index 444f6e6..9a3fb46 100644 --- a/internal/systemd/reload.go +++ b/internal/systemd/reload.go @@ -22,8 +22,10 @@ package systemd import ( "context" + "errors" "fmt" "sync" + "time" "github.com/godbus/dbus/v5" @@ -43,8 +45,19 @@ const ( nameOwnerChangedFullName = busDaemonName + "." + nameOwnerChangedMember signalChanBuffer = 16 + + // setupTimeout bounds one attempt to connect and subscribe: none of the + // DBus calls involved take a deadline of their own. + setupTimeout = 10 * time.Second + // minRetry and maxRetry bound the pause between subscription attempts. + minRetry = time.Second + maxRetry = 5 * time.Minute ) +// errSetupTimeout reports a subscription attempt that did not finish within +// setupTimeout. +var errSetupTimeout = errors.New("connecting to the system bus timed out") + // Watcher holds a DBus connection subscribed to systemd's Reloading signal. // systemctl daemon-reload clears cgroup BPF programs, so any container that // had device-allow rules attached loses access. Watch invokes the callback @@ -58,12 +71,171 @@ type Watcher struct { owner string } +// Supervise keeps a Watcher subscribed until ctx is done, invoking onReload +// on each completed systemd reload. It never blocks its caller's other work +// on DBus: run it on its own goroutine. Each attempt to connect and +// subscribe is bounded by setupTimeout; a failed attempt, or a connection +// that is lost later (a DBus restart), is retried with backoff up to +// maxRetry. +// +// onSubscribed runs after every successful subscription, the first one +// included: a reload that completed while no watcher was subscribed (during +// startup, or while the bus was unreachable) was not seen, so the caller +// must reconcile everything once. +func Supervise(ctx context.Context, onReload, onSubscribed func()) { + sup := supervisor{ + open: Open, + setupTimeout: setupTimeout, + minRetry: minRetry, + maxRetry: maxRetry, + } + sup.run(ctx, onReload, onSubscribed) +} + +// supervisor is Supervise with its seams: tests replace open and shorten +// the timings. +type supervisor struct { + open func() (*Watcher, error) + setupTimeout time.Duration + minRetry time.Duration + maxRetry time.Duration +} + +func (s *supervisor) run(ctx context.Context, onReload, onSubscribed func()) { + retry := s.minRetry + failures := 0 + + for { + watcher, err := s.openBounded(ctx) + if ctx.Err() != nil { + closeWatcher(watcher) + + return + } + + if err != nil { + // Hosts without systemd or without the socket mounted fail + // every attempt; say so once, then keep trying quietly. + if failures == 0 { + logger.L().Warn("systemd reload handling unavailable; retrying in the background", + "err", err, "retry_in", retry) + } else { + logger.L().Debug("systemd reload watcher still unavailable", + "err", err, "retry_in", retry) + } + + failures++ + + if !sleepCtx(ctx, retry) { + return + } + + retry = min(retry*2, s.maxRetry) + + continue + } + + if failures > 0 { + logger.L().Info("systemd reload handling available again") + } + + failures = 0 + subscribedAt := time.Now() + + // Signals are buffered from here on, so none is lost while the + // caller reconciles. + onSubscribed() + watcher.Watch(ctx, onReload) + closeWatcher(watcher) + + if ctx.Err() != nil { + return + } + + // A subscription that held for a while starts the backoff over; + // one that keeps dropping right away backs off like a failure, so + // a flapping bus does not request a pass every minRetry. + if time.Since(subscribedAt) >= s.maxRetry { + retry = s.minRetry + } + + logger.L().Warn("systemd reload watcher disconnected; resubscribing", "retry_in", retry) + + if !sleepCtx(ctx, retry) { + return + } + + retry = min(retry*2, s.maxRetry) + } +} + +// openBounded runs open, giving up after setupTimeout or when ctx is done. +// An attempt given up on keeps running in the background and closes its +// watcher if it ever succeeds, so nothing leaks a subscription. +func (s *supervisor) openBounded(ctx context.Context) (*Watcher, error) { + type attempt struct { + watcher *Watcher + err error + } + + result := make(chan attempt, 1) + + go func() { + watcher, err := s.open() + result <- attempt{watcher, err} + }() + + abandon := func() { + go func() { closeWatcher((<-result).watcher) }() + } + + timer := time.NewTimer(s.setupTimeout) + defer timer.Stop() + + select { + case got := <-result: + return got.watcher, got.err + case <-timer.C: + abandon() + + return nil, errSetupTimeout + case <-ctx.Done(): + abandon() + + return nil, fmt.Errorf("subscribe to systemd reloads: %w", ctx.Err()) + } +} + +func closeWatcher(watcher *Watcher) { + if watcher == nil { + return + } + + err := watcher.Close() + if err != nil { + logger.L().Warn("close systemd watcher", "err", err) + } +} + +// sleepCtx waits for duration and reports whether ctx is still live. +func sleepCtx(ctx context.Context, duration time.Duration) bool { + timer := time.NewTimer(duration) + defer timer.Stop() + + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + // Open connects to the system DBus, resolves the unique name that owns // org.freedesktop.systemd1, and registers signal matches for systemd's // Reloading signal (from systemd's name and object path only) and for // changes of that name's owner. Returns an error if DBus is unavailable (no -// socket bind-mount, no systemd, daemon running off-host). Callers should -// treat this as non-fatal and continue without reload handling. +// socket bind-mount, no systemd, daemon running off-host). Supervise +// retries it; the daemon runs without reload handling meanwhile. func Open() (*Watcher, error) { conn, err := dbus.ConnectSystemBus() if err != nil { @@ -173,7 +345,7 @@ func watch(ctx context.Context, sigCh <-chan *dbus.Signal, owner string, onReloa case sig, ok := <-sigCh: if !ok { - logger.L().Warn("systemd signal channel closed; reload watcher exiting") + logger.L().Debug("systemd signal channel closed") return } diff --git a/internal/systemd/reload_test.go b/internal/systemd/reload_test.go index fab44bd..0bc4c36 100644 --- a/internal/systemd/reload_test.go +++ b/internal/systemd/reload_test.go @@ -23,8 +23,10 @@ import ( "context" "log/slog" "strings" + "sync" "sync/atomic" "testing" + "time" "github.com/godbus/dbus/v5" @@ -274,3 +276,181 @@ func TestWatch_UsesTheCurrentLogger(t *testing.T) { ) } } + +// fakeOpens scripts successive Open results for a supervisor. +type fakeOpens struct { + mu sync.Mutex + steps []func() (*Watcher, error) + calls int +} + +func (f *fakeOpens) open() (*Watcher, error) { + f.mu.Lock() + step := f.steps[min(f.calls, len(f.steps)-1)] + f.calls++ + f.mu.Unlock() + + return step() +} + +func failOpen() (*Watcher, error) { return nil, errSetupTimeout } + +func openOn(sigCh chan *dbus.Signal) func() (*Watcher, error) { + return func() (*Watcher, error) { return &Watcher{sigCh: sigCh, owner: testOwner}, nil } +} + +// startSupervisor runs a supervisor with millisecond timings and returns +// channels that receive one value per onSubscribed and onReload call. It +// is stopped, and must return promptly, when the test ends. +func startSupervisor( + t *testing.T, + opens *fakeOpens, +) (subscribed, reloaded <-chan struct{}) { + t.Helper() + + subCh := make(chan struct{}, 16) + reloadCh := make(chan struct{}, 16) + + sup := &supervisor{ + open: opens.open, + setupTimeout: 50 * time.Millisecond, + minRetry: time.Millisecond, + maxRetry: 4 * time.Millisecond, + } + + ctx, cancelCtx := context.WithCancel(context.Background()) + done := make(chan struct{}) + + go func() { + defer close(done) + + sup.run(ctx, + func() { reloadCh <- struct{}{} }, + func() { subCh <- struct{}{} }, + ) + }() + + t.Cleanup(func() { + cancelCtx() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("supervisor did not return after cancel") + } + }) + + return subCh, reloadCh +} + +func receive(t *testing.T, ch <-chan struct{}, what string) { + t.Helper() + + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for %s", what) + } +} + +// A system bus that is not reachable yet is retried; the first +// subscription requests a pass and reloads are delivered from then on. +func TestSupervise_RetriesUntilSubscribed(t *testing.T) { + t.Parallel() + + sigCh := make(chan *dbus.Signal) + opens := &fakeOpens{steps: []func() (*Watcher, error){failOpen, failOpen, openOn(sigCh)}} + subscribed, reloaded := startSupervisor(t, opens) + + receive(t, subscribed, "the first subscription") + + sigCh <- completed(testOwner, systemdObjectPath) + + receive(t, reloaded, "the reload") +} + +// A lost connection (the signal channel closes, as on a DBus restart) is +// resubscribed, and the new subscription requests a pass again: reloads +// during the outage were not seen. +func TestSupervise_ResubscribesAfterDisconnect(t *testing.T) { + t.Parallel() + + first := make(chan *dbus.Signal) + second := make(chan *dbus.Signal) + opens := &fakeOpens{steps: []func() (*Watcher, error){openOn(first), openOn(second)}} + subscribed, reloaded := startSupervisor(t, opens) + + receive(t, subscribed, "the first subscription") + close(first) + receive(t, subscribed, "the resubscription") + + second <- completed(testOwner, systemdObjectPath) + + receive(t, reloaded, "the reload on the new subscription") +} + +// An attempt stuck on a wedged bus is given up after setupTimeout and the +// next one proceeds; canceling returns even while an attempt hangs. +func TestSupervise_BoundsAHungAttempt(t *testing.T) { + t.Parallel() + + hang := make(chan struct{}) + + t.Cleanup(func() { close(hang) }) + + sigCh := make(chan *dbus.Signal) + opens := &fakeOpens{steps: []func() (*Watcher, error){ + func() (*Watcher, error) { + <-hang + + return nil, errSetupTimeout + }, + openOn(sigCh), + }} + subscribed, _ := startSupervisor(t, opens) + + receive(t, subscribed, "the subscription after the hung attempt") +} + +func TestSupervise_CancelWhileAttemptHangs(t *testing.T) { + t.Parallel() + + hang := make(chan struct{}) + + t.Cleanup(func() { close(hang) }) + + entered := make(chan struct{}) + opens := &fakeOpens{steps: []func() (*Watcher, error){ + func() (*Watcher, error) { + close(entered) + <-hang + + return nil, errSetupTimeout + }, + }} + + sup := &supervisor{ + open: opens.open, + setupTimeout: time.Hour, + minRetry: time.Hour, + maxRetry: time.Hour, + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + + go func() { + defer close(done) + + sup.run(ctx, func() {}, func() { t.Error("subscribed without a watcher") }) + }() + + <-entered + cancel() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("supervisor did not return after cancel while an attempt hung") + } +} From 0eab351e26cce0789f56c2cf302c16d5c017b07d Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 13:00:32 +0200 Subject: [PATCH 34/37] fix(processor): revoke the previous cgroup when a run's cgroup changes A running container can move to another cgroup under the same run, for example when systemd inside it moves its processes into a child cgroup. The record used to be overwritten with the new cgroup, leaving the grants on the previous one, where they keep applying to every descendant. The previous cgroup now gets the empty set first; until that succeeds the new cgroup is not granted and the record keeps naming the previous one, so a retry revokes it again. --- README.md | 4 ++ docs/architecture.md | 5 +- internal/processor/applyerr_test.go | 10 +-- internal/processor/reconcile.go | 99 ++++++++++++++++++++++++---- internal/processor/reconcile_test.go | 98 +++++++++++++++++++++++++++ 5 files changed, 198 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 5c98aee..5292736 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,10 @@ that cgroup. A recycled pid therefore never directs grants to another cgroup. Once a container has exited, or when Docker cannot be asked about it, its grants are revoked in the cgroup the daemon last verified for it, and only if that directory is still the same one (same inode) and was not recreated. +A running container found in another cgroup than the one it was granted in +(systemd inside the container moving its processes into a child cgroup, for +example) loses the grants in the previous cgroup before it is granted in the +new one, since a grant on a parent cgroup applies to every cgroup below it. The daemon keeps a short history of each container's runs. A `die` or `destroy` event revokes the grants of the run it ended (the latest run that diff --git a/docs/architecture.md b/docs/architecture.md index 8b9208a..ca65163 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -69,7 +69,10 @@ cgroup is gone). Before changing a cgroup the processor pins the container's process with a pidfd, resolves the cgroup from `/proc//cgroup`, opens it as a directory handle, and re-checks that the process is alive, that Docker still reports it as the container's process with the same start time and, -for a grant, that it is in that cgroup. All mutation goes through the handle, never the path again. +for a grant, that it is in that cgroup. All mutation goes through the handle, never the path again. A run found in another cgroup than the one +it was last verified in (systemd inside the container moving its processes into a child cgroup, for one) first has the empty set applied at the +previous cgroup, since a grant on a parent applies to every descendant; until that succeeds the new cgroup is not granted and the container is +retried. On cgroup v1 a child cannot hold what its parent denies, so such a container gets no grants there. ### Coordinator diff --git a/internal/processor/applyerr_test.go b/internal/processor/applyerr_test.go index d9a3f03..5cf19ee 100644 --- a/internal/processor/applyerr_test.go +++ b/internal/processor/applyerr_test.go @@ -42,10 +42,11 @@ import ( type failingCgroup struct { cgroup.Interface - err error - calls int - identity cgroup.Identity - rules [][]cgroup.DeviceRule + err error + calls int + identity cgroup.Identity + identities []cgroup.Identity + rules [][]cgroup.DeviceRule } func (f *failingCgroup) SetDeviceRules( @@ -54,6 +55,7 @@ func (f *failingCgroup) SetDeviceRules( ) error { f.calls++ f.identity = handle.Identity() + f.identities = append(f.identities, f.identity) f.rules = append(f.rules, rules) return f.err diff --git a/internal/processor/reconcile.go b/internal/processor/reconcile.go index c12f727..a11f22c 100644 --- a/internal/processor/reconcile.go +++ b/internal/processor/reconcile.go @@ -101,13 +101,30 @@ func (p *Processor) applyPinned( return cgroupPath, err } + identity := handle.Identity() + + // The same run can move to another cgroup: a container running systemd + // moves its processes into a child cgroup, and a grant left on the + // parent keeps applying to every descendant. The grants at the recorded + // cgroup go before the record is replaced; if they cannot, the record + // keeps naming that cgroup and the container is retried. On cgroup v1 a + // child cannot be granted what its parent denies, so such a run ends + // up with no grants at all: fail closed. + moved, ok := p.lifecycleStore().lookup(containerID, state.StartedAt) + if ok && moved.HasIdentity() && moved.Identity != identity { + err = p.revokeMoved(&moved, identity) + if err != nil { + return cgroupPath, err + } + } + // Recorded after verification, so an identity resolved through a // recycled pid is never kept, and before the mutation, so a failed or // partial one can still be revoked here later. p.lifecycleStore().verify(&LifecycleRecord{ ContainerID: containerID, StartedAt: state.StartedAt, - Identity: handle.Identity(), + Identity: identity, Version: resolved.Version, Privileged: privileged, }) @@ -337,25 +354,19 @@ func cgroupGone(identity cgroup.Identity) (bool, error) { func (p *Processor) revokeAt(rec *LifecycleRecord, mayRun bool) error { log := logger.L() - handle, err := cgroup.OpenCgroup(rec.Identity.Path) - if errors.Is(err, fs.ErrNotExist) { - p.release(rec, "cgroup removed") - - return nil - } - + handle, goneWhy, err := openRecorded(rec) if err != nil { - return fmt.Errorf("revoke container %q: %w", rec.ContainerID, err) + return err } - defer closeHandle(handle) - - if handle.Identity() != rec.Identity { - p.release(rec, "cgroup recreated") + if goneWhy != "" { + p.release(rec, goneWhy) return nil } + defer closeHandle(handle) + api, err := p.cgroupAPI(rec.Version) if err != nil { return err @@ -387,6 +398,68 @@ func (p *Processor) revokeAt(rec *LifecycleRecord, mayRun bool) error { return nil } +// revokeMoved applies the empty set at the cgroup a still-running run was +// recorded in, once the run is verified in another one (current). The +// cgroup API then forgets that directory: no record names it any more. +// A missing filter means nothing attached there holds a grant. +func (p *Processor) revokeMoved(prev *LifecycleRecord, current cgroup.Identity) error { + log := logger.L() + + handle, goneWhy, err := openRecorded(prev) + if err != nil { + return err + } + + if goneWhy == "" { + defer closeHandle(handle) + + api, err := p.cgroupAPI(prev.Version) + if err != nil { + return err + } + + err = api.SetDeviceRules(handle, nil) + if err != nil && !errors.Is(err, cgroup.ErrFilterMissing) { + return fmt.Errorf( + "revoke container %q in its previous cgroup %q: %w", + prev.ContainerID, + prev.Identity.Path, + err, + ) + } + } + + log.Info("container moved to another cgroup; previous grants revoked", + "id", prev.ContainerID, "from", prev.Identity.Path, "to", current.Path) + + p.ledger.Forget(prev.Identity) + p.filterCache.Forget(prev.Identity) + + return nil +} + +// openRecorded re-opens the cgroup a record names. goneWhy is set, and +// the handle nil, when the directory is gone or was recreated at the same +// path (another inode): either way it holds none of the run's grants. +func openRecorded(rec *LifecycleRecord) (handle *cgroup.CgroupHandle, goneWhy string, err error) { + handle, err = cgroup.OpenCgroup(rec.Identity.Path) + if errors.Is(err, fs.ErrNotExist) { + return nil, "cgroup removed", nil + } + + if err != nil { + return nil, "", fmt.Errorf("revoke container %q: %w", rec.ContainerID, err) + } + + if handle.Identity() != rec.Identity { + closeHandle(handle) + + return nil, "cgroup recreated", nil + } + + return handle, "", nil +} + // release forgets a run whose cgroup is verified gone, together with what // the cgroup API remembered for that directory (the v1 ledger and the // cached v2 filters). diff --git a/internal/processor/reconcile_test.go b/internal/processor/reconcile_test.go index 03f9b7f..bf90a37 100644 --- a/internal/processor/reconcile_test.go +++ b/internal/processor/reconcile_test.go @@ -346,6 +346,104 @@ func TestReconcile_ExitAfterCgroupRemovedReleases(t *testing.T) { } } +// moveToChildCgroup moves the test container's process into a child of its +// cgroup, as systemd running inside it does, and returns the child's path. +func moveToChildCgroup(t *testing.T, env *reconcileEnv) string { + t.Helper() + + child := filepath.Join(testCgroupDir(env.proc.HostRoot), "init.scope") + + err := os.Mkdir(child, 0o755) + if err != nil { + t.Fatal(err) + } + + err = os.WriteFile( + filepath.Join(child, "cgroup.procs"), + []byte(strconv.Itoa(reconcilePid)+"\n"), + 0o600, + ) + if err != nil { + t.Fatal(err) + } + + writeProcs(t, env.proc.HostRoot) + + err = os.WriteFile( + filepath.Join(env.proc.ProcRoot, "proc", strconv.Itoa(reconcilePid), "cgroup"), + []byte("0::/docker/testcontainer/init.scope\n"), + 0o600, + ) + if err != nil { + t.Fatal(err) + } + + return child +} + +// TestReconcile_MovedRunRevokesPreviousCgroup: a run found in another +// cgroup than the one it was granted in loses the grants there before it +// is granted in the new one, since a grant on a parent cgroup applies to +// every descendant. +func TestReconcile_MovedRunRevokesPreviousCgroup(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.grantOnce(t) + + parent := env.fake.identity + child := moveToChildCgroup(t, env) + + err := env.reconcile() + if err != nil { + t.Fatal(err) + } + + if env.fake.calls != 3 || + env.fake.identities[1] != parent || len(env.fake.rules[1]) != 0 || + env.fake.identities[2].Path != child || len(env.fake.rules[2]) != 1 { + t.Fatalf( + "calls = %d, identities = %+v, rules = %v; want the empty set on %s, then the grant on %s", + env.fake.calls, + env.fake.identities, + env.fake.rules, + parent.Path, + child, + ) + } + + recs := verifiedRecords(env.proc) + if len(recs) != 1 || recs[0].Identity.Path != child { + t.Errorf("records = %+v, want one naming %s", recs, child) + } +} + +// TestReconcile_MovedRunRevokeFailureKeepsRecord: when the previous cgroup +// cannot be revoked, the new one is not granted and the record keeps +// naming the previous cgroup, so a retry revokes it again. +func TestReconcile_MovedRunRevokeFailureKeepsRecord(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.grantOnce(t) + + parent := env.fake.identity + moveToChildCgroup(t, env) + + env.fake.err = errRevokeFailed + + err := env.reconcile() + if !errors.Is(err, errRevokeFailed) { + t.Fatalf("err = %v, want the revoke failure", err) + } + + if env.fake.calls != 2 || env.fake.identities[1] != parent || len(env.fake.rules[1]) != 0 { + t.Fatalf("calls = %d, identities = %+v, rules = %v; want only the revoke attempt on %s", + env.fake.calls, env.fake.identities, env.fake.rules, parent.Path) + } + + recs := verifiedRecords(env.proc) + if len(recs) != 1 || recs[0].Identity != parent { + t.Errorf("records = %+v, want the one naming %s kept", recs, parent.Path) + } +} + // TestReconcile_InspectFailureRevokesKnownIdentity: without a pid the // container's grants are revoked where its lifecycle was verified, and the // container stays pending. With no known identity it is pending only. From d56c1eac46eba18574c0330a462df652de9dc8e4 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 13:03:12 +0200 Subject: [PATCH 35/37] test(integration): wait for the startup passes before acting A daemon now requests one pass at startup and another when its reload watcher subscribes, and the second can land after a test has started counting processed records. waitReady now also waits for the watcher's outcome and for the last of those passes to complete, so every later record comes from what the test does. The reload subtest also explains why the restarted daemon finds a filter to wrap. --- test/integration/enforce_test.go | 5 ++++- test/integration/helpers_test.go | 34 +++++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/test/integration/enforce_test.go b/test/integration/enforce_test.go index e0280b9..a5b80c4 100644 --- a/test/integration/enforce_test.go +++ b/test/integration/enforce_test.go @@ -55,7 +55,6 @@ const ( // mounted under /var/run inside the container. containerDBusSocket = "/var/run/dbus/system_bus_socket" - msgWatcherStarted = "systemd reload watcher started" msgDryRunLeftovers = "dry-run: grants left by a previous live run cannot be detected or cleaned in dry-run" msgEPERM = "Operation not permitted" @@ -243,6 +242,10 @@ func testReloadReapply( reloadUnsupported(t, fmt.Errorf("daemon-reload did not wipe the device program: %w", err)) } + // Denied, not open: the reload removed the daemon's wrapper, but systemd + // attached the unit's own device filter again, so the cgroup is not left + // without one. A restarted daemon, whose filter cache is empty, can wrap + // that filter; it never reaches the filter_missing path here. _, proc := startDaemonContainer(ctx, t, cli, image) proc.wait(ctx, t, "systemd reload watcher", withMsg(msgWatcherStarted)) diff --git a/test/integration/helpers_test.go b/test/integration/helpers_test.go index 9bf272f..b7d980c 100644 --- a/test/integration/helpers_test.go +++ b/test/integration/helpers_test.go @@ -72,10 +72,14 @@ const ( imagePullTimeout = 45 * time.Second // Messages of the daemon's log records the tests wait for. - msgReady = "subscribed to docker events" - msgProcessed = "container processed" - msgSkippedPolicy = "container skipped by policy" - msgInvalidLabels = "container skipped: invalid policy labels" + msgReady = "subscribed to docker events" + msgProcessed = "container processed" + msgSkippedPolicy = "container skipped by policy" + msgInvalidLabels = "container skipped: invalid policy labels" + msgWatcherStarted = "systemd reload watcher started" + msgWatcherUnavailable = "systemd reload handling unavailable; retrying in the background" + // msgRequestDone is logged once per reconcile request, with its epoch. + msgRequestDone = "config reload complete" testImage = "docker.io/library/busybox:1.36" ) @@ -335,11 +339,31 @@ func newDaemonProc() *daemonProc { return &daemonProc{changed: make(chan struct{}), exited: make(chan struct{})} } -// waitReady blocks until the daemon has subscribed to Docker events. +// waitReady blocks until the daemon has subscribed to Docker events, its +// reload watcher has subscribed to systemd or given up for now, and the +// passes those requested have finished. The startup pass is one request and +// a watcher subscription another, so with both the second completion is +// the last: every later "container processed" record comes from what the +// test does next, not from a pass still running. func (d *daemonProc) waitReady(ctx context.Context, t *testing.T) { t.Helper() d.waitN(ctx, t, "daemon subscription to docker events", withMsg(msgReady), 1) + + watcher := d.wait(ctx, t, "systemd reload watcher outcome", func(rec logRecord) bool { + return rec.str("msg") == msgWatcherStarted || rec.str("msg") == msgWatcherUnavailable + }) + + requests := 1 + if watcher.str("msg") == msgWatcherStarted { + requests = 2 + } + + d.wait(ctx, t, "startup passes complete", func(rec logRecord) bool { + epoch, ok := rec.num("epoch") + + return rec.str("msg") == msgRequestDone && ok && epoch == requests + }) } // withMsg matches records with the given message. From bcab2b162b0afa3ebba5fb3b831562f609e494e6 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 13:22:43 +0200 Subject: [PATCH 36/37] fix(processor): recheck the run after revoking its previous cgroup Revoking the previous cgroup is real work between the pre-mutation check and the grant, and the run can move again meanwhile. The pinned process is now verified once more after the revoke, cgroup membership included, before the new cgroup is recorded or granted. --- docs/architecture.md | 2 +- internal/processor/applyerr_test.go | 6 +++++ internal/processor/reconcile.go | 9 ++++++++ internal/processor/reconcile_test.go | 33 ++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/architecture.md b/docs/architecture.md index ca65163..f84ce12 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -72,7 +72,7 @@ directory handle, and re-checks that the process is alive, that Docker still rep for a grant, that it is in that cgroup. All mutation goes through the handle, never the path again. A run found in another cgroup than the one it was last verified in (systemd inside the container moving its processes into a child cgroup, for one) first has the empty set applied at the previous cgroup, since a grant on a parent applies to every descendant; until that succeeds the new cgroup is not granted and the container is -retried. On cgroup v1 a child cannot hold what its parent denies, so such a container gets no grants there. +retried. The run is then verified again, cgroup membership included, since it can move once more while the previous cgroup is revoked. On cgroup v1 a child cannot hold what its parent denies, so such a container gets no grants there. ### Coordinator diff --git a/internal/processor/applyerr_test.go b/internal/processor/applyerr_test.go index 5cf19ee..bdc5e91 100644 --- a/internal/processor/applyerr_test.go +++ b/internal/processor/applyerr_test.go @@ -47,6 +47,8 @@ type failingCgroup struct { identity cgroup.Identity identities []cgroup.Identity rules [][]cgroup.DeviceRule + // onSet, when set, runs inside every call, after it is recorded. + onSet func() } func (f *failingCgroup) SetDeviceRules( @@ -58,6 +60,10 @@ func (f *failingCgroup) SetDeviceRules( f.identities = append(f.identities, f.identity) f.rules = append(f.rules, rules) + if f.onSet != nil { + f.onSet() + } + return f.err } diff --git a/internal/processor/reconcile.go b/internal/processor/reconcile.go index a11f22c..e333d02 100644 --- a/internal/processor/reconcile.go +++ b/internal/processor/reconcile.go @@ -116,6 +116,15 @@ func (p *Processor) applyPinned( if err != nil { return cgroupPath, err } + + // The revoke is real work between the check above and the + // mutation below, and the run can move again meanwhile: check once + // more, membership included even for the empty set, so the record + // never names a cgroup the run was not verified in. + err = p.verifyPinned(ctx, containerID, state, pinned, handle, true) + if err != nil { + return cgroupPath, err + } } // Recorded after verification, so an identity resolved through a diff --git a/internal/processor/reconcile_test.go b/internal/processor/reconcile_test.go index bf90a37..cc4dc05 100644 --- a/internal/processor/reconcile_test.go +++ b/internal/processor/reconcile_test.go @@ -416,6 +416,39 @@ func TestReconcile_MovedRunRevokesPreviousCgroup(t *testing.T) { } } +// TestReconcile_MovedAgainDuringRevokeIsRechecked: the run leaves the new +// cgroup while the previous one is revoked, so the new one is neither +// granted nor recorded, and the container is retried. +func TestReconcile_MovedAgainDuringRevokeIsRechecked(t *testing.T) { + env := newReconcileEnv(t, policy.ModeAll, false) + env.grantOnce(t) + + parent := env.fake.identity + child := moveToChildCgroup(t, env) + + env.fake.onSet = func() { + err := os.WriteFile(filepath.Join(child, "cgroup.procs"), nil, 0o600) + if err != nil { + t.Error(err) + } + } + + err := env.reconcile() + if !errors.Is(err, errNotInCgroup) { + t.Fatalf("err = %v, want errNotInCgroup", err) + } + + if env.fake.calls != 2 || env.fake.identities[1] != parent || len(env.fake.rules[1]) != 0 { + t.Fatalf("calls = %d, identities = %+v, rules = %v; want only the revoke on %s", + env.fake.calls, env.fake.identities, env.fake.rules, parent.Path) + } + + recs := verifiedRecords(env.proc) + if len(recs) != 1 || recs[0].Identity != parent { + t.Errorf("records = %+v, want the one naming %s kept", recs, parent.Path) + } +} + // TestReconcile_MovedRunRevokeFailureKeepsRecord: when the previous cgroup // cannot be revoked, the new one is not granted and the record keeps // naming the previous cgroup, so a retry revokes it again. From 1d8a1b4f0bee2f01d2b2b3d776517ac0fc71ef51 Mon Sep 17 00:00:00 2001 From: Roberto Leinardi Date: Sun, 27 Sep 2026 13:25:41 +0200 Subject: [PATCH 37/37] fix(systemd): cancel a timed-out bus connection instead of abandoning it A setup attempt that timed out used to keep running in the background with its connection open, and on a wedged bus every retry added another. The connection is now bound to a per-attempt context that the setup timeout cancels: godbus closes it, which fails the handshake or call still waiting, and the attempt returns before the next one starts. The match rules and the owner lookup take the same context. --- docs/architecture.md | 12 +-- internal/systemd/reload.go | 142 +++++++++++++++++--------------- internal/systemd/reload_test.go | 102 ++++++++++++++++------- 3 files changed, 151 insertions(+), 105 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index f84ce12..3f778e8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -128,12 +128,12 @@ system bus can emit a signal with that name, so the watcher only counts one sent owns `org.freedesktop.systemd1` (resolved with `GetNameOwner` at startup and followed through the bus daemon's `NameOwnerChanged`). Re-applies are coalesced: reloads that complete while one runs lead to a single trailing run. -The watcher runs in its own goroutine, started before the startup pass, so connecting to the bus never delays the pass or the event loop. -Each connection attempt is bounded (10 s). An attempt that fails, and a subscription that is lost later (the bus daemon restarting, for -one), is retried with a backoff of 1 s doubling to 5 min, which starts over once a subscription has held for 5 min; the first failure is logged -at `Warn`, later ones at `Debug`. Every successful -subscription requests a pass, because a reload that completed while the watcher was not subscribed, including one during the startup -pass, sent a signal nobody received. +The watcher runs in its own goroutine, started before the startup pass, so connecting to the bus never delays the pass or the event loop. Each +connection attempt is bounded (10 s): the connection is bound to the attempt, so one that times out is closed, which fails whatever it was +waiting on, and the next attempt starts only after it has returned. An attempt that fails, and a subscription that is lost later (the bus +daemon restarting, for one), is retried with a backoff of 1 s doubling to 5 min, which starts over once a subscription has held for 5 min; the +first failure is logged at `Warn`, later ones at `Debug`. Every successful subscription requests a pass, because a reload that completed while +the watcher was not subscribed, including one during the startup pass, sent a signal nobody received. ## BPF program structure diff --git a/internal/systemd/reload.go b/internal/systemd/reload.go index 9a3fb46..89fd14b 100644 --- a/internal/systemd/reload.go +++ b/internal/systemd/reload.go @@ -95,7 +95,7 @@ func Supervise(ctx context.Context, onReload, onSubscribed func()) { // supervisor is Supervise with its seams: tests replace open and shorten // the timings. type supervisor struct { - open func() (*Watcher, error) + open func(ctx context.Context) (*Watcher, error) setupTimeout time.Duration minRetry time.Duration maxRetry time.Duration @@ -106,10 +106,16 @@ func (s *supervisor) run(ctx context.Context, onReload, onSubscribed func()) { failures := 0 for { - watcher, err := s.openBounded(ctx) - if ctx.Err() != nil { - closeWatcher(watcher) + held, err := s.attempt(ctx, onReload, func() { + if failures > 0 { + logger.L().Info("systemd reload handling available again") + } + + failures = 0 + onSubscribed() + }) + if ctx.Err() != nil { return } @@ -125,42 +131,17 @@ func (s *supervisor) run(ctx context.Context, onReload, onSubscribed func()) { } failures++ - - if !sleepCtx(ctx, retry) { - return + } else { + // A subscription that held for a while starts the backoff over; + // one that keeps dropping right away backs off like a failure, + // so a flapping bus does not request a pass every minRetry. + if held >= s.maxRetry { + retry = s.minRetry } - retry = min(retry*2, s.maxRetry) - - continue - } - - if failures > 0 { - logger.L().Info("systemd reload handling available again") + logger.L().Warn("systemd reload watcher disconnected; resubscribing", "retry_in", retry) } - failures = 0 - subscribedAt := time.Now() - - // Signals are buffered from here on, so none is lost while the - // caller reconciles. - onSubscribed() - watcher.Watch(ctx, onReload) - closeWatcher(watcher) - - if ctx.Err() != nil { - return - } - - // A subscription that held for a while starts the backoff over; - // one that keeps dropping right away backs off like a failure, so - // a flapping bus does not request a pass every minRetry. - if time.Since(subscribedAt) >= s.maxRetry { - retry = s.minRetry - } - - logger.L().Warn("systemd reload watcher disconnected; resubscribing", "retry_in", retry) - if !sleepCtx(ctx, retry) { return } @@ -169,41 +150,56 @@ func (s *supervisor) run(ctx context.Context, onReload, onSubscribed func()) { } } -// openBounded runs open, giving up after setupTimeout or when ctx is done. -// An attempt given up on keeps running in the background and closes its -// watcher if it ever succeeds, so nothing leaks a subscription. -func (s *supervisor) openBounded(ctx context.Context) (*Watcher, error) { - type attempt struct { - watcher *Watcher - err error +// attempt connects, subscribes and watches until the subscription is lost +// or ctx is done, and returns how long the subscription held; err is the +// setup failure when there was none. Everything the attempt opened is +// closed before it returns, so attempts never overlap. +func (s *supervisor) attempt( + ctx context.Context, + onReload, onSubscribed func(), +) (time.Duration, error) { + attemptCtx, cancel := context.WithCancel(ctx) + defer cancel() + + watcher, err := s.openBounded(attemptCtx, cancel) + if err != nil { + return 0, err } - result := make(chan attempt, 1) + defer closeWatcher(watcher) - go func() { - watcher, err := s.open() - result <- attempt{watcher, err} - }() + subscribedAt := time.Now() - abandon := func() { - go func() { closeWatcher((<-result).watcher) }() - } + // Signals are buffered from here on, so none is lost while the caller + // reconciles. + onSubscribed() + watcher.Watch(attemptCtx, onReload) - timer := time.NewTimer(s.setupTimeout) - defer timer.Stop() + return time.Since(subscribedAt), nil +} - select { - case got := <-result: - return got.watcher, got.err - case <-timer.C: - abandon() +// openBounded runs open with ctx canceled (through cancel) once +// setupTimeout passes. The connection is bound to ctx, so a timed-out +// attempt is closed rather than abandoned: every read and call still +// waiting on the bus fails, and open returns before the next attempt +// starts. +func (s *supervisor) openBounded(ctx context.Context, cancel context.CancelFunc) (*Watcher, error) { + timer := time.AfterFunc(s.setupTimeout, cancel) + + watcher, err := s.open(ctx) + if !timer.Stop() { + // The timer fired: the connection is closed even if open still + // got a watcher out of it. + closeWatcher(watcher) return nil, errSetupTimeout - case <-ctx.Done(): - abandon() + } - return nil, fmt.Errorf("subscribe to systemd reloads: %w", ctx.Err()) + if err != nil { + return nil, err } + + return watcher, nil } func closeWatcher(watcher *Watcher) { @@ -236,13 +232,19 @@ func sleepCtx(ctx context.Context, duration time.Duration) bool { // changes of that name's owner. Returns an error if DBus is unavailable (no // socket bind-mount, no systemd, daemon running off-host). Supervise // retries it; the daemon runs without reload handling meanwhile. -func Open() (*Watcher, error) { - conn, err := dbus.ConnectSystemBus() +// +// The connection is bound to ctx: it closes when ctx is done, which also +// fails any setup step still waiting on the bus (the authentication +// handshake, Hello, the match rules, the owner lookup). Connecting the +// socket itself takes no context, but a unix-socket connect does not wait +// for a bus that stops accepting: it fails at once. +func Open(ctx context.Context) (*Watcher, error) { + conn, err := dbus.ConnectSystemBus(dbus.WithContext(ctx)) if err != nil { return nil, fmt.Errorf("connect system bus: %w", err) } - watcher, err := subscribe(conn) + watcher, err := subscribe(ctx, conn) if err != nil { closeErr := conn.Close() if closeErr != nil { @@ -259,8 +261,9 @@ func Open() (*Watcher, error) { // resolves systemd's owner: godbus drops signals no channel is registered // for, so a NameOwnerChanged between the lookup and the registration would // otherwise be lost and leave the watcher trusting a dead name. -func subscribe(conn *dbus.Conn) (*Watcher, error) { - err := conn.AddMatchSignal( +func subscribe(ctx context.Context, conn *dbus.Conn) (*Watcher, error) { + err := conn.AddMatchSignalContext( + ctx, dbus.WithMatchSender(systemdBusName), dbus.WithMatchObjectPath(systemdObjectPath), dbus.WithMatchInterface(reloadingInterface), @@ -270,7 +273,8 @@ func subscribe(conn *dbus.Conn) (*Watcher, error) { return nil, fmt.Errorf("add match signal %s: %w", reloadingFullName, err) } - err = conn.AddMatchSignal( + err = conn.AddMatchSignalContext( + ctx, dbus.WithMatchSender(busDaemonName), dbus.WithMatchObjectPath(busDaemonPath), dbus.WithMatchInterface(busDaemonName), @@ -286,7 +290,9 @@ func subscribe(conn *dbus.Conn) (*Watcher, error) { var owner string - err = conn.BusObject().Call(busDaemonName+".GetNameOwner", 0, systemdBusName).Store(&owner) + err = conn.BusObject(). + CallWithContext(ctx, busDaemonName+".GetNameOwner", 0, systemdBusName). + Store(&owner) if err != nil { conn.RemoveSignal(sigCh) diff --git a/internal/systemd/reload_test.go b/internal/systemd/reload_test.go index 0bc4c36..3200df7 100644 --- a/internal/systemd/reload_test.go +++ b/internal/systemd/reload_test.go @@ -21,6 +21,7 @@ package systemd import ( "bytes" "context" + "fmt" "log/slog" "strings" "sync" @@ -277,26 +278,54 @@ func TestWatch_UsesTheCurrentLogger(t *testing.T) { } } -// fakeOpens scripts successive Open results for a supervisor. +// fakeOpens scripts successive Open results for a supervisor and records +// how many attempts ever ran at once. type fakeOpens struct { - mu sync.Mutex - steps []func() (*Watcher, error) - calls int + mu sync.Mutex + steps []func(ctx context.Context) (*Watcher, error) + calls int + inFlight int + maxInFlight int } -func (f *fakeOpens) open() (*Watcher, error) { +func (f *fakeOpens) open(ctx context.Context) (*Watcher, error) { f.mu.Lock() step := f.steps[min(f.calls, len(f.steps)-1)] f.calls++ + f.inFlight++ + f.maxInFlight = max(f.maxInFlight, f.inFlight) f.mu.Unlock() - return step() + defer func() { + f.mu.Lock() + f.inFlight-- + f.mu.Unlock() + }() + + return step(ctx) } -func failOpen() (*Watcher, error) { return nil, errSetupTimeout } +func (f *fakeOpens) stats() (calls, maxInFlight int) { + f.mu.Lock() + defer f.mu.Unlock() -func openOn(sigCh chan *dbus.Signal) func() (*Watcher, error) { - return func() (*Watcher, error) { return &Watcher{sigCh: sigCh, owner: testOwner}, nil } + return f.calls, f.maxInFlight +} + +func failOpen(context.Context) (*Watcher, error) { return nil, errSetupTimeout } + +func openOn(sigCh chan *dbus.Signal) func(context.Context) (*Watcher, error) { + return func(context.Context) (*Watcher, error) { + return &Watcher{sigCh: sigCh, owner: testOwner}, nil + } +} + +// hungOpen blocks like a wedged bus until its context is canceled, the +// way a real connection bound to that context fails its waiting calls. +func hungOpen(ctx context.Context) (*Watcher, error) { + <-ctx.Done() + + return nil, fmt.Errorf("hung open: %w", ctx.Err()) } // startSupervisor runs a supervisor with millisecond timings and returns @@ -359,7 +388,9 @@ func TestSupervise_RetriesUntilSubscribed(t *testing.T) { t.Parallel() sigCh := make(chan *dbus.Signal) - opens := &fakeOpens{steps: []func() (*Watcher, error){failOpen, failOpen, openOn(sigCh)}} + opens := &fakeOpens{steps: []func(context.Context) (*Watcher, error){ + failOpen, failOpen, openOn(sigCh), + }} subscribed, reloaded := startSupervisor(t, opens) receive(t, subscribed, "the first subscription") @@ -377,7 +408,9 @@ func TestSupervise_ResubscribesAfterDisconnect(t *testing.T) { first := make(chan *dbus.Signal) second := make(chan *dbus.Signal) - opens := &fakeOpens{steps: []func() (*Watcher, error){openOn(first), openOn(second)}} + opens := &fakeOpens{steps: []func(context.Context) (*Watcher, error){ + openOn(first), openOn(second), + }} subscribed, reloaded := startSupervisor(t, opens) receive(t, subscribed, "the first subscription") @@ -389,43 +422,46 @@ func TestSupervise_ResubscribesAfterDisconnect(t *testing.T) { receive(t, reloaded, "the reload on the new subscription") } -// An attempt stuck on a wedged bus is given up after setupTimeout and the -// next one proceeds; canceling returns even while an attempt hangs. -func TestSupervise_BoundsAHungAttempt(t *testing.T) { +// An attempt stuck on a wedged bus is canceled after setupTimeout, and +// returns before the next attempt starts. Even a watcher it produces once +// canceled is not used: its connection is already closed. +func TestSupervise_CancelsAHungAttempt(t *testing.T) { t.Parallel() - hang := make(chan struct{}) - - t.Cleanup(func() { close(hang) }) - + late := make(chan *dbus.Signal) sigCh := make(chan *dbus.Signal) - opens := &fakeOpens{steps: []func() (*Watcher, error){ - func() (*Watcher, error) { - <-hang + opens := &fakeOpens{steps: []func(context.Context) (*Watcher, error){ + func(ctx context.Context) (*Watcher, error) { + <-ctx.Done() - return nil, errSetupTimeout + return &Watcher{sigCh: late, owner: testOwner}, nil }, + hungOpen, openOn(sigCh), }} subscribed, _ := startSupervisor(t, opens) - receive(t, subscribed, "the subscription after the hung attempt") + receive(t, subscribed, "the subscription after the hung attempts") + + calls, maxInFlight := opens.stats() + if calls != 3 || maxInFlight != 1 { + t.Errorf( + "calls = %d, max in flight = %d; want 3 attempts, one at a time", + calls, + maxInFlight, + ) + } } func TestSupervise_CancelWhileAttemptHangs(t *testing.T) { t.Parallel() - hang := make(chan struct{}) - - t.Cleanup(func() { close(hang) }) - entered := make(chan struct{}) - opens := &fakeOpens{steps: []func() (*Watcher, error){ - func() (*Watcher, error) { + opens := &fakeOpens{steps: []func(context.Context) (*Watcher, error){ + func(ctx context.Context) (*Watcher, error) { close(entered) - <-hang - return nil, errSetupTimeout + return hungOpen(ctx) }, }} @@ -453,4 +489,8 @@ func TestSupervise_CancelWhileAttemptHangs(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("supervisor did not return after cancel while an attempt hung") } + + if calls, _ := opens.stats(); calls != 1 { + t.Errorf("calls = %d, want no attempt after cancel", calls) + } }