diff --git a/README.md b/README.md index 81f01e7..300d934 100644 --- a/README.md +++ b/README.md @@ -90,14 +90,14 @@ services: cuda-worker: image: nvidia/cuda:12.4.0-base-ubuntu22.04 command: ["nvidia-smi"] - labels: - swarm-device-access.enable: "true" - swarm-device-access.device-allow: "/dev/nvidia*" volumes: - /dev/nvidia0:/dev/nvidia0 - /dev/nvidiactl:/dev/nvidiactl - /dev/nvidia-uvm:/dev/nvidia-uvm deploy: + labels: + swarm-device-access.enable: "true" + swarm-device-access.device-allow: "/dev/nvidia*" mode: replicated replicas: 1 ``` @@ -150,6 +150,12 @@ Consumer services opt in and narrow their allowed device set with labels: | `swarm-device-access.device-allow` | Comma-separated globs | Allow only matching `/dev/...` paths. Empty means inherit. | | `swarm-device-access.device-deny` | Comma-separated globs | Deny matching `/dev/...` paths. Deny overrides allow. | +Declare these labels under `deploy.labels:` (the Swarm service spec — the +natural home for service-level metadata, alongside Traefik / Homepage / other +label-driven tooling). The daemon also reads top-level `labels:` if you need to +override a service-wide value on a single task; per-container values win on +conflict. + Global `-device-allow` and `-device-deny` define the broadest access the daemon may grant. Per-container labels can only narrow that access. Deny rules always win. diff --git a/docs/architecture.md b/docs/architecture.md index 467d294..ced44a8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -117,6 +117,11 @@ The daemon uses a two-level policy: **Per-container policy** (Docker labels): +Declare these labels under `deploy.labels:` in your Swarm stack file — that is +the service spec, the natural home alongside Traefik / Homepage / other +label-driven tooling. The daemon also reads top-level `labels:` when present; +per-container (task-level) values win on conflict. + | Label | Description | |------------------------------------|----------------------------------------------------------------| | `swarm-device-access.enable` | `true` to opt in, `false` to explicitly opt out. | diff --git a/docs/testing.md b/docs/testing.md index 62e9e71..0bbce43 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -52,7 +52,9 @@ or deployment behavior: 3. Start a consumer container with `--label swarm-device-access.enable=true` and a real `/dev/...` bind mount and confirm the daemon logs `device mount detected` - and `adding device rule`. + and `adding device rule`. (For Swarm stacks, the equivalent placement is + `deploy.labels:` in the service spec — `docker service create --label` writes + to the same location.) 4. If the host uses cgroup v2, confirm a `BPF_CGROUP_DEVICE` program is attached to the consumer cgroup with `bpftool`. diff --git a/examples/audio-passthrough/docker-compose.yaml b/examples/audio-passthrough/docker-compose.yaml index 44bd54b..7754f25 100644 --- a/examples/audio-passthrough/docker-compose.yaml +++ b/examples/audio-passthrough/docker-compose.yaml @@ -11,14 +11,14 @@ services: audio-worker: image: ubuntu:24.04 - labels: - swarm-device-access.enable: "true" - swarm-device-access.device-allow: "/dev/snd/*" volumes: # /dev/snd is a directory; the daemon walks it and applies a rule per device. - /dev/snd:/dev/snd command: [ "aplay", "-l" ] deploy: + labels: + swarm-device-access.enable: "true" + swarm-device-access.device-allow: "/dev/snd/*" mode: replicated replicas: 1 swarm-device-access: diff --git a/examples/gpu-passthrough/docker-compose.yaml b/examples/gpu-passthrough/docker-compose.yaml index 22e8ea6..d9e98ab 100644 --- a/examples/gpu-passthrough/docker-compose.yaml +++ b/examples/gpu-passthrough/docker-compose.yaml @@ -14,16 +14,16 @@ services: cuda-worker: image: nvidia/cuda:12.4.0-base-ubuntu22.04 - # Required when daemon runs in opt-in mode (the default). - labels: - swarm-device-access.enable: "true" - swarm-device-access.device-allow: "/dev/nvidia*" volumes: - /dev/nvidia0:/dev/nvidia0 - /dev/nvidiactl:/dev/nvidiactl - /dev/nvidia-uvm:/dev/nvidia-uvm command: [ "nvidia-smi" ] deploy: + labels: + # Required when daemon runs in opt-in mode (the default). + swarm-device-access.enable: "true" + swarm-device-access.device-allow: "/dev/nvidia*" mode: replicated replicas: 1 swarm-device-access: diff --git a/examples/usb-passthrough/docker-compose.yaml b/examples/usb-passthrough/docker-compose.yaml index bddc00e..0ea6af7 100644 --- a/examples/usb-passthrough/docker-compose.yaml +++ b/examples/usb-passthrough/docker-compose.yaml @@ -40,13 +40,13 @@ services: usb-worker: image: ubuntu:24.04 - labels: - swarm-device-access.enable: "true" - swarm-device-access.device-allow: "/dev/bus/usb/*" volumes: # Bind-mount the entire USB bus; the daemon will apply rules to each device. - /dev/bus/usb:/dev/bus/usb command: [ "lsusb" ] deploy: + labels: + swarm-device-access.enable: "true" + swarm-device-access.device-allow: "/dev/bus/usb/*" mode: replicated replicas: 1 diff --git a/examples/v4l2-passthrough/docker-compose.yaml b/examples/v4l2-passthrough/docker-compose.yaml index 6cabce1..78ca5c7 100644 --- a/examples/v4l2-passthrough/docker-compose.yaml +++ b/examples/v4l2-passthrough/docker-compose.yaml @@ -40,9 +40,6 @@ services: v4l2-worker: image: linuxserver/ffmpeg:latest # dclint disable-line service-image-require-explicit-tag - labels: - swarm-device-access.enable: "true" - swarm-device-access.device-allow: "/dev/video*" volumes: - /dev/video0:/dev/video0 - /tmp/output:/output @@ -56,5 +53,8 @@ services: - "5" - /output/capture.mp4 deploy: + labels: + swarm-device-access.enable: "true" + swarm-device-access.device-allow: "/dev/video*" mode: replicated replicas: 1 diff --git a/internal/policy/policy.go b/internal/policy/policy.go index 84f688d..86e01c5 100644 --- a/internal/policy/policy.go +++ b/internal/policy/policy.go @@ -20,7 +20,9 @@ package policy import ( "fmt" + "maps" "path/filepath" + "sort" "strconv" "strings" ) @@ -231,3 +233,43 @@ func matchAny(patterns []string, path string) bool { return false } + +// knownLabels is the set of recognized swarm-device-access.* label keys. +var knownLabels = map[string]struct{}{ + LabelEnable: {}, + LabelDeviceAllow: {}, + LabelDeviceDeny: {}, +} + +// MergeLabels returns a merged label map: service labels as base, container +// labels win on conflict. Nil inputs are treated as empty maps. +func MergeLabels(service, container map[string]string) map[string]string { + merged := make(map[string]string, len(service)+len(container)) + + maps.Copy(merged, service) + maps.Copy(merged, container) + + return merged +} + +// UnknownLabels returns a sorted slice of keys in labels that start with +// LabelPrefix but are not in the known label set. Returns nil when none. +func UnknownLabels(labels map[string]string) []string { + var unknown []string + + for k := range labels { + if strings.HasPrefix(k, LabelPrefix) { + if _, ok := knownLabels[k]; !ok { + unknown = append(unknown, k) + } + } + } + + if len(unknown) == 0 { + return nil + } + + sort.Strings(unknown) + + return unknown +} diff --git a/internal/policy/policy_test.go b/internal/policy/policy_test.go index b8d8ccd..2736791 100644 --- a/internal/policy/policy_test.go +++ b/internal/policy/policy_test.go @@ -328,3 +328,131 @@ func sliceEqual(a, b []string) bool { return true } + +func TestMergeLabels(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + service map[string]string + container map[string]string + want map[string]string + }{ + { + name: "both nil", + want: map[string]string{}, + }, + { + name: "nil service", + container: map[string]string{"a": "1"}, + want: map[string]string{"a": "1"}, + }, + { + name: "nil container", + service: map[string]string{"a": "1"}, + want: map[string]string{"a": "1"}, + }, + { + name: "disjoint union", + service: map[string]string{"a": "1"}, + container: map[string]string{"b": "2"}, + want: map[string]string{"a": "1", "b": "2"}, + }, + { + name: "container wins on conflict", + service: map[string]string{"a": "service"}, + container: map[string]string{"a": "container"}, + want: map[string]string{"a": "container"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got := policy.MergeLabels(tc.service, tc.container) + + if len(got) != len(tc.want) { + t.Fatalf("MergeLabels() len=%d, want %d; got=%v", len(got), len(tc.want), got) + } + + for k, wantV := range tc.want { + if got[k] != wantV { + t.Errorf("key %q: got %q, want %q", k, got[k], wantV) + } + } + }) + } +} + +func TestUnknownLabels(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + labels map[string]string + want []string + }{ + { + name: "nil map", + labels: nil, + want: nil, + }, + { + name: "empty map", + labels: map[string]string{}, + want: nil, + }, + { + name: "no swarm-device-access keys", + labels: map[string]string{"traefik.enable": "true"}, + want: nil, + }, + { + name: "only known keys", + labels: map[string]string{ + policy.LabelEnable: "true", + policy.LabelDeviceAllow: "/dev/snd/*", + policy.LabelDeviceDeny: "/dev/sda", + }, + want: nil, + }, + { + name: "mixed known and unknown", + labels: map[string]string{ + policy.LabelEnable: "true", + policy.LabelPrefix + "enabled": "true", + policy.LabelPrefix + "zzz": "x", + }, + want: []string{ + policy.LabelPrefix + "enabled", + policy.LabelPrefix + "zzz", + }, + }, + { + name: "unknown sorted deterministically", + labels: map[string]string{ + policy.LabelPrefix + "zzz": "1", + policy.LabelPrefix + "aaa": "2", + policy.LabelPrefix + "mmm": "3", + }, + want: []string{ + policy.LabelPrefix + "aaa", + policy.LabelPrefix + "mmm", + policy.LabelPrefix + "zzz", + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got := policy.UnknownLabels(tc.labels) + + if !sliceEqual(got, tc.want) { + t.Errorf("UnknownLabels() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 0a188da..4060429 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -25,6 +25,7 @@ import ( "path/filepath" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/swarm" "github.com/leinardi/swarm-device-access/internal/cgroup" "github.com/leinardi/swarm-device-access/internal/config" @@ -33,11 +34,18 @@ import ( "github.com/leinardi/swarm-device-access/internal/policy" ) -// ContainerInspector is the subset of *client.Client used by Processor. +const swarmServiceIDLabel = "com.docker.swarm.service.id" + +// DockerInspector is the subset of *client.Client used by Processor. // It exists solely to allow unit tests to inject a fake without standing up a // real Docker daemon. -type ContainerInspector interface { +type DockerInspector interface { ContainerInspect(ctx context.Context, containerID string) (container.InspectResponse, error) + ServiceInspectWithRaw( + ctx context.Context, + serviceID string, + opts swarm.ServiceInspectOptions, + ) (swarm.Service, []byte, error) } // deviceRuleKey is the deduplication key for cgroup device rules collected @@ -54,7 +62,7 @@ type deviceRuleKey struct { // host root (typically "/host"). ProcRoot is used for /proc lookups ("/" in // production, temp dir in tests). type Processor struct { - Inspector ContainerInspector + Inspector DockerInspector Cfg *config.Store Metrics *observability.Recorder HostRoot string @@ -80,14 +88,48 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er return nil } - var labels map[string]string + var containerLabels map[string]string if info.Config != nil { - labels = info.Config.Labels + containerLabels = info.Config.Labels } + var ( + serviceLabels map[string]string + svc swarm.Service + ) + + if serviceID := containerLabels[swarmServiceIDLabel]; serviceID != "" { + var svcErr error + + svc, _, svcErr = p.Inspector.ServiceInspectWithRaw( + ctx, + serviceID, + swarm.ServiceInspectOptions{}, + ) + if svcErr != nil { + log.Warn("could not inspect parent service; using container labels only", + "id", containerID, + "service_id", serviceID, + "err", svcErr, + ) + } else { + serviceLabels = svc.Spec.Labels + + for _, unknownKey := range policy.UnknownLabels(serviceLabels) { + log.Warn("unrecognized swarm-device-access label on parent service", + "id", containerID, + "service_id", serviceID, + "label", unknownKey, + ) + } + } + } + + effectiveLabels := policy.MergeLabels(serviceLabels, containerLabels) + cfg := p.Cfg.Load() - cpol, parseErr := policy.ParseContainer(labels) + cpol, parseErr := policy.ParseContainer(effectiveLabels) if parseErr != nil { log.Warn("container skipped: invalid policy labels", "id", containerID, "err", parseErr) @@ -106,6 +148,17 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er return nil } + if svc.Spec.Name != "" { + cpolContainer, _ := policy.ParseContainer(containerLabels) + if !cfg.Policy.Enabled(cpolContainer) { + log.Info("opt-in granted via service-level label", + "id", containerID, + "service_id", containerLabels[swarmServiceIDLabel], + "service_name", svc.Spec.Name, + ) + } + } + p.Metrics.RecordContainerScanned() pid := info.State.Pid diff --git a/internal/processor/processor_test.go b/internal/processor/processor_test.go index 2713f81..cc8f14c 100644 --- a/internal/processor/processor_test.go +++ b/internal/processor/processor_test.go @@ -19,24 +19,33 @@ package processor import ( + "bytes" "context" "errors" + "log/slog" + "maps" "os" "path/filepath" "strconv" + "strings" "testing" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/api/types/swarm" "github.com/leinardi/swarm-device-access/internal/config" + "github.com/leinardi/swarm-device-access/internal/logger" "github.com/leinardi/swarm-device-access/internal/policy" ) -// fakeInspector is a test double for ContainerInspector. +// fakeInspector is a test double for DockerInspector. type fakeInspector struct { - result container.InspectResponse - err error + result container.InspectResponse + err error + serviceResult swarm.Service + serviceErr error + serviceCalls int } func (f *fakeInspector) ContainerInspect( @@ -46,11 +55,21 @@ func (f *fakeInspector) ContainerInspect( return f.result, f.err } +func (f *fakeInspector) ServiceInspectWithRaw( + _ context.Context, + _ string, + _ swarm.ServiceInspectOptions, +) (swarm.Service, []byte, error) { + f.serviceCalls++ + + return f.serviceResult, nil, f.serviceErr +} + // buildProcRoot creates a minimal /proc//{cgroup,mountinfo} structure // under a temp dir so ProcessContainer can resolve the cgroup path without a // real /proc filesystem. // -//nolint:unparam // cgroupContent varies across test cases; linter sees current call sites only + func buildProcRoot( t *testing.T, pid int, @@ -445,3 +464,168 @@ func TestCollectMountRules_BadPath(t *testing.T) { t.Error("expected errors for bad path, got none") } } + +// captureLogger sets logger.L() to write to a buffer for the duration of the +// test and restores the previous logger when the test ends. +func captureLogger(t *testing.T) *bytes.Buffer { + t.Helper() + + var buf bytes.Buffer + logger.Set(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{ + Level: slog.LevelDebug, + }))) + t.Cleanup(func() { logger.Set(nil) }) + + return &buf +} + +//nolint:tparallel // subtests share the global logger via captureLogger; parallel would cause log interleaving +func TestProcessContainer_SwarmServiceLabels(t *testing.T) { + t.Parallel() + + const ( + cid = "abc123" + serviceID = "svc456" + pid = 51 + ) + + makeSwarmContainer := func(extraLabels map[string]string) container.InspectResponse { + labels := map[string]string{swarmServiceIDLabel: serviceID} + maps.Copy(labels, extraLabels) + + return container.InspectResponse{ + ContainerJSONBase: &container.ContainerJSONBase{ + State: &container.State{Pid: pid}, + }, + Config: &container.Config{Labels: labels}, + } + } + + cgroupContent := "0::/docker/testcontainer\n" + mountinfoContent := "35 22 0:29 / /sys/fs/cgroup rw,nosuid,nodev shared:11 - cgroup2 cgroup2 rw\n" //nolint:dupword // cgroup2 appears twice: fs type and superblock type in mountinfo format + + cases := []struct { + name string + containerInfo container.InspectResponse + svcResult swarm.Service + svcErr error + store *config.Store + wantServiceCall bool + wantLogMsg string + wantSkip bool + }{ + { + name: "deploy.labels-only grants opt-in", + containerInfo: makeSwarmContainer(nil), + svcResult: swarm.Service{ + Spec: swarm.ServiceSpec{ + Annotations: swarm.Annotations{ + Name: "my-service", + Labels: map[string]string{policy.LabelEnable: "true"}, + }, + }, + }, + store: newStore(policy.ModeOptIn, true), + wantServiceCall: true, + wantLogMsg: "opt-in granted via service-level label", + }, + { + name: "container labels override service", + containerInfo: makeSwarmContainer(map[string]string{ + policy.LabelEnable: "false", + }), + svcResult: swarm.Service{ + Spec: swarm.ServiceSpec{ + Annotations: swarm.Annotations{ + Name: "my-service", + Labels: map[string]string{policy.LabelEnable: "true"}, + }, + }, + }, + store: newStore(policy.ModeOptIn, true), + wantServiceCall: true, + wantSkip: true, + }, + { + name: "non-Swarm passthrough", + containerInfo: container.InspectResponse{ + ContainerJSONBase: &container.ContainerJSONBase{ + State: &container.State{Pid: pid}, + }, + Config: &container.Config{Labels: map[string]string{ + policy.LabelEnable: "true", + }}, + }, + store: newStore(policy.ModeOptIn, true), + wantServiceCall: false, + }, + { + name: "service inspect error is non-fatal", + containerInfo: makeSwarmContainer(map[string]string{ + policy.LabelEnable: "true", + }), + svcErr: errDaemonUnavail, + store: newStore(policy.ModeOptIn, true), + wantServiceCall: true, + wantLogMsg: "could not inspect parent service", + }, + { + name: "typo WARN on service label", + containerInfo: makeSwarmContainer(map[string]string{ + policy.LabelEnable: "true", + }), + svcResult: swarm.Service{ + Spec: swarm.ServiceSpec{ + Annotations: swarm.Annotations{ + Name: "my-service", + Labels: map[string]string{policy.LabelPrefix + "enabled": "true"}, + }, + }, + }, + store: newStore(policy.ModeOptIn, true), + wantServiceCall: true, + wantLogMsg: "unrecognized swarm-device-access label on parent service", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + buf := captureLogger(t) + + procRoot := buildProcRoot(t, pid, cgroupContent, mountinfoContent) + + inspector := &fakeInspector{ + result: tc.containerInfo, + serviceResult: tc.svcResult, + serviceErr: tc.svcErr, + } + + proc := &Processor{ + Inspector: inspector, + Cfg: tc.store, + HostRoot: t.TempDir(), + ProcRoot: procRoot, + } + + _ = proc.ProcessContainer(context.Background(), cid) + + logOutput := buf.String() + + if tc.wantServiceCall && inspector.serviceCalls == 0 { + t.Error("expected ServiceInspectWithRaw to be called, was not") + } + + if !tc.wantServiceCall && inspector.serviceCalls > 0 { + t.Errorf("expected no ServiceInspectWithRaw call, got %d", inspector.serviceCalls) + } + + if tc.wantLogMsg != "" && !strings.Contains(logOutput, tc.wantLogMsg) { + t.Errorf("expected log to contain %q, got:\n%s", tc.wantLogMsg, logOutput) + } + + if tc.wantSkip && !strings.Contains(logOutput, "skipped by policy") { + t.Errorf("expected skip-by-policy log, got:\n%s", logOutput) + } + }) + } +}