diff --git a/README.md b/README.md index f647334..bfc6bdb 100644 --- a/README.md +++ b/README.md @@ -94,10 +94,13 @@ services: - /dev/nvidia0:/dev/nvidia0 - /dev/nvidiactl:/dev/nvidiactl - /dev/nvidia-uvm:/dev/nvidia-uvm + # Use top-level labels: so the daemon can read them on both manager and + # worker nodes. deploy.labels: are service-level metadata only visible to + # the Docker API on manager nodes. + labels: + swarm-device-access.enable: "true" + swarm-device-access.device-allow: "/dev/nvidia*" deploy: - labels: - swarm-device-access.enable: "true" - swarm-device-access.device-allow: "/dev/nvidia*" mode: replicated replicas: 1 ``` @@ -150,11 +153,26 @@ Consumer services opt in and narrow their allowed device set with labels: | `swarm-device-access.device-allow` | Comma-separated globs | Allow only matching `/dev/...` paths. Empty means inherit. | | `swarm-device-access.device-deny` | Comma-separated globs | Deny matching `/dev/...` paths. Deny overrides allow. | -Declare these labels under `deploy.labels:` (the Swarm service spec — the -natural home for service-level metadata, alongside Traefik / Homepage / other -label-driven tooling). The daemon also reads top-level `labels:` if you need to -override a service-wide value on a single task; per-container values win on -conflict. +Declare these labels under top-level `labels:` in your service definition. +Docker copies top-level labels into each task container, so the daemon can read +them on every node — including worker nodes. + +> **Note:** Do not use `deploy.labels:` for these labels. `deploy.labels:` are +> Swarm service metadata that only the Docker Swarm API (manager nodes) can +> read. The daemon cannot see them on worker nodes and will warn you if it +> detects them on a manager. + +### Worker and Manager Nodes + +The daemon auto-detects its role at startup by querying the local Docker API. +On **worker nodes** it skips the Swarm service inspect entirely (no spurious +warnings). On **manager nodes** it additionally reads service-level metadata +for backward compatibility — but emits a warning if it finds +`swarm-device-access.*` labels under `deploy.labels:`, advising you to move +them to top-level `labels:`. + +If the node role changes (promote/demote), restart the daemon so it re-detects +its role. If you bind-mount a directory (for example `source: /dev/dri`), the `device-allow` glob is evaluated **per child node** inside that directory — diff --git a/cmd/swarm-device-access/main.go b/cmd/swarm-device-access/main.go index b9d70b2..d765216 100644 --- a/cmd/swarm-device-access/main.go +++ b/cmd/swarm-device-access/main.go @@ -111,6 +111,23 @@ func run() int { recorder := observability.NewRecorder() + isSwarmManager := false + + nodeInfo, infoErr := cli.Info(rootCtx) + if infoErr != nil { + log.Warn( + "could not query docker info; assuming worker node (service-label inspection disabled)", + "err", + infoErr, + ) + } else { + isSwarmManager = nodeInfo.Swarm.ControlAvailable + log.Info("swarm role detected", + "manager", isSwarmManager, + "local_node_state", nodeInfo.Swarm.LocalNodeState, + ) + } + // Start optional observability servers before the main loop so they are // reachable during startup enumeration. if *metricsAddr != "" { @@ -122,11 +139,12 @@ func run() int { } proc := &processor.Processor{ - Inspector: cli, - Cfg: store, - Metrics: recorder, - HostRoot: hostRootPath, - ProcRoot: "/", + Inspector: cli, + Cfg: store, + Metrics: recorder, + HostRoot: hostRootPath, + ProcRoot: "/", + IsSwarmManager: isSwarmManager, } runErr := daemon.Run(rootCtx, daemon.Options{ diff --git a/docs/architecture.md b/docs/architecture.md index ced44a8..657f995 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -117,10 +117,11 @@ The daemon uses a two-level policy: **Per-container policy** (Docker labels): -Declare these labels under `deploy.labels:` in your Swarm stack file — that is -the service spec, the natural home alongside Traefik / Homepage / other -label-driven tooling. The daemon also reads top-level `labels:` when present; -per-container (task-level) values win on conflict. +Declare these labels under top-level `labels:` in your Swarm stack file. +Docker copies top-level labels into each task container so the daemon can read +them on both manager and worker nodes. Do **not** use `deploy.labels:` — those +are Swarm service metadata only accessible via the manager API; the daemon +cannot see them on worker nodes and will warn if it finds them on a manager. | Label | Description | |------------------------------------|----------------------------------------------------------------| diff --git a/examples/audio-passthrough/docker-compose.yaml b/examples/audio-passthrough/docker-compose.yaml index 7754f25..b962b8f 100644 --- a/examples/audio-passthrough/docker-compose.yaml +++ b/examples/audio-passthrough/docker-compose.yaml @@ -15,10 +15,11 @@ services: # /dev/snd is a directory; the daemon walks it and applies a rule per device. - /dev/snd:/dev/snd command: [ "aplay", "-l" ] + # Use top-level labels: — deploy.labels: are only visible on manager nodes. + labels: + swarm-device-access.enable: "true" + swarm-device-access.device-allow: "/dev/snd/*" deploy: - labels: - swarm-device-access.enable: "true" - swarm-device-access.device-allow: "/dev/snd/*" mode: replicated replicas: 1 swarm-device-access: diff --git a/examples/gpu-passthrough/docker-compose.yaml b/examples/gpu-passthrough/docker-compose.yaml index d9e98ab..eec9d49 100644 --- a/examples/gpu-passthrough/docker-compose.yaml +++ b/examples/gpu-passthrough/docker-compose.yaml @@ -19,11 +19,12 @@ services: - /dev/nvidiactl:/dev/nvidiactl - /dev/nvidia-uvm:/dev/nvidia-uvm command: [ "nvidia-smi" ] + # Use top-level labels: — deploy.labels: are only visible on manager nodes. + # Required when daemon runs in opt-in mode (the default). + labels: + swarm-device-access.enable: "true" + swarm-device-access.device-allow: "/dev/nvidia*" deploy: - labels: - # Required when daemon runs in opt-in mode (the default). - swarm-device-access.enable: "true" - swarm-device-access.device-allow: "/dev/nvidia*" mode: replicated replicas: 1 swarm-device-access: diff --git a/examples/smartctl-exporter/docker-compose.yaml b/examples/smartctl-exporter/docker-compose.yaml new file mode 100644 index 0000000..83d7be9 --- /dev/null +++ b/examples/smartctl-exporter/docker-compose.yaml @@ -0,0 +1,70 @@ +# smartctl-exporter passthrough example for Docker Swarm +# +# Runs smartctl-exporter as a global service so it can read disk health +# metrics from /dev/sg* and /dev/nvme* on every storage node. +# The swarm-device-access daemon injects cgroup BPF device-allow rules at +# container start, replacing the privileged: true that Compose would use. +# +# Prerequisites on each node: +# - /dev/sd*, /dev/nvme*, or /dev/sg* devices present +# - /run/udev populated by the host udevd +# - swarm-device-access daemon running (global service below) +# +# Deploy: +# docker stack deploy -c docker-compose.yaml smartctl-example + +services: + smartctl-exporter: + image: prometheuscommunity/smartctl-exporter:v0.14.0 + volumes: + # Bind-mount the full /dev tree; daemon will allow each disk device. + - /dev:/dev:ro + - /run/udev:/run/udev:ro + command: + - --smartctl.interval=5m + - --smartctl.rescan=30m + - --web.listen-address=:9633 + - --smartctl.device-exclude=^/dev/(loop|md|zram) + user: "0:0" + cap_add: + - SYS_RAWIO + - SYS_ADMIN + security_opt: + - no-new-privileges=true + # Use top-level labels: — deploy.labels: are only visible on manager nodes. + labels: + swarm-device-access.enable: "true" + swarm-device-access.device-allow: "/dev/sd*,/dev/nvme*,/dev/sg*,/dev/disk/*" + deploy: + mode: global + + swarm-device-access: + image: docker:29 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: + - run + - -i + - --rm + - --name=swarm-device-access + - --privileged + - --cgroupns=host + - --pid=host + - --userns=host + - -v + - /sys:/host/sys + - -v + - /var/run/docker.sock:/var/run/docker.sock + - -v + - /dev:/dev + # Enable reload-after-daemon-reload (optional): + # - -v + # - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket + # Restrict to disk devices only (optional): + # - -device-allow=/dev/sd*,/dev/nvme*,/dev/sg*,/dev/disk/* + - ghcr.io/leinardi/swarm-device-access:latest + entrypoint: docker + deploy: + mode: global + restart_policy: + condition: any diff --git a/examples/usb-passthrough/docker-compose.yaml b/examples/usb-passthrough/docker-compose.yaml index 0ea6af7..332bf32 100644 --- a/examples/usb-passthrough/docker-compose.yaml +++ b/examples/usb-passthrough/docker-compose.yaml @@ -44,9 +44,10 @@ services: # Bind-mount the entire USB bus; the daemon will apply rules to each device. - /dev/bus/usb:/dev/bus/usb command: [ "lsusb" ] + # Use top-level labels: — deploy.labels: are only visible on manager nodes. + labels: + swarm-device-access.enable: "true" + swarm-device-access.device-allow: "/dev/bus/usb/*" deploy: - labels: - swarm-device-access.enable: "true" - swarm-device-access.device-allow: "/dev/bus/usb/*" mode: replicated replicas: 1 diff --git a/examples/v4l2-passthrough/docker-compose.yaml b/examples/v4l2-passthrough/docker-compose.yaml index 78ca5c7..ada0964 100644 --- a/examples/v4l2-passthrough/docker-compose.yaml +++ b/examples/v4l2-passthrough/docker-compose.yaml @@ -52,9 +52,10 @@ services: - -t - "5" - /output/capture.mp4 + # Use top-level labels: — deploy.labels: are only visible on manager nodes. + labels: + swarm-device-access.enable: "true" + swarm-device-access.device-allow: "/dev/video*" deploy: - labels: - swarm-device-access.enable: "true" - swarm-device-access.device-allow: "/dev/video*" mode: replicated replicas: 1 diff --git a/internal/policy/policy.go b/internal/policy/policy.go index 86e01c5..cf56250 100644 --- a/internal/policy/policy.go +++ b/internal/policy/policy.go @@ -273,3 +273,23 @@ func UnknownLabels(labels map[string]string) []string { return unknown } + +// KnownLabels returns a sorted list of swarm-device-access.* keys from labels +// that are recognized by this package (i.e. the inverse of UnknownLabels). +func KnownLabels(labels map[string]string) []string { + var known []string + + for k := range labels { + if _, ok := knownLabels[k]; ok { + known = append(known, k) + } + } + + if len(known) == 0 { + return nil + } + + sort.Strings(known) + + return known +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 4060429..b6eac30 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -62,11 +62,12 @@ type deviceRuleKey struct { // host root (typically "/host"). ProcRoot is used for /proc lookups ("/" in // production, temp dir in tests). type Processor struct { - Inspector DockerInspector - Cfg *config.Store - Metrics *observability.Recorder - HostRoot string - ProcRoot string + Inspector DockerInspector + Cfg *config.Store + Metrics *observability.Recorder + HostRoot string + ProcRoot string + IsSwarmManager bool } // ProcessContainer inspects a container and applies cgroup BPF device-allow @@ -93,37 +94,7 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er containerLabels = info.Config.Labels } - var ( - serviceLabels map[string]string - svc swarm.Service - ) - - if serviceID := containerLabels[swarmServiceIDLabel]; serviceID != "" { - var svcErr error - - svc, _, svcErr = p.Inspector.ServiceInspectWithRaw( - ctx, - serviceID, - swarm.ServiceInspectOptions{}, - ) - if svcErr != nil { - log.Warn("could not inspect parent service; using container labels only", - "id", containerID, - "service_id", serviceID, - "err", svcErr, - ) - } else { - serviceLabels = svc.Spec.Labels - - for _, unknownKey := range policy.UnknownLabels(serviceLabels) { - log.Warn("unrecognized swarm-device-access label on parent service", - "id", containerID, - "service_id", serviceID, - "label", unknownKey, - ) - } - } - } + svc, serviceLabels := p.resolveServiceLabels(ctx, containerID, containerLabels) effectiveLabels := policy.MergeLabels(serviceLabels, containerLabels) @@ -236,6 +207,69 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er return errors.Join(allErrs...) } +// resolveServiceLabels fetches parent service labels on manager nodes. On +// worker nodes (IsSwarmManager=false) it returns immediately with zero values +// so ProcessContainer can continue using container-level labels only. +func (p *Processor) resolveServiceLabels( + ctx context.Context, + containerID string, + containerLabels map[string]string, +) (svc swarm.Service, serviceLabels map[string]string) { + log := logger.L() + + if !p.IsSwarmManager { + return svc, nil + } + + serviceID := containerLabels[swarmServiceIDLabel] + if serviceID == "" { + return svc, nil + } + + var svcErr error + + svc, _, svcErr = p.Inspector.ServiceInspectWithRaw( + ctx, + serviceID, + swarm.ServiceInspectOptions{}, + ) + if svcErr != nil { + log.Warn("could not inspect parent service; using container labels only", + "id", containerID, + "service_id", serviceID, + "err", svcErr, + ) + + return swarm.Service{}, nil + } + + serviceLabels = svc.Spec.Labels + + for _, unknownKey := range policy.UnknownLabels(serviceLabels) { + log.Warn("unrecognized swarm-device-access label on parent service", + "id", containerID, + "service_id", serviceID, + "label", unknownKey, + ) + } + + for _, knownKey := range policy.KnownLabels(serviceLabels) { + log.Warn( + "swarm-device-access label set via deploy.labels on parent service; move to top-level labels: so worker nodes can read it", + "id", + containerID, + "service_id", + serviceID, + "service", + svc.Spec.Name, + "label", + knownKey, + ) + } + + return svc, serviceLabels +} + // applyRulesToCgroup logs and (unless dryRun) attaches the collected device // rules to the cgroup at cgroupPath via a single AddDeviceRules call. func (p *Processor) applyRulesToCgroup( diff --git a/internal/processor/processor_test.go b/internal/processor/processor_test.go index 9dc8e5d..0f9f144 100644 --- a/internal/processor/processor_test.go +++ b/internal/processor/processor_test.go @@ -620,12 +620,13 @@ func TestProcessContainer_SwarmServiceLabels(t *testing.T) { svcResult swarm.Service svcErr error store *config.Store + isSwarmManager bool wantServiceCall bool wantLogMsg string wantSkip bool }{ { - name: "deploy.labels-only grants opt-in", + name: "deploy.labels-only grants opt-in on manager", containerInfo: makeSwarmContainer(nil), svcResult: swarm.Service{ Spec: swarm.ServiceSpec{ @@ -636,11 +637,12 @@ func TestProcessContainer_SwarmServiceLabels(t *testing.T) { }, }, store: newStore(policy.ModeOptIn, true), + isSwarmManager: true, wantServiceCall: true, wantLogMsg: "opt-in granted via service-level label", }, { - name: "container labels override service", + name: "container labels override service on manager", containerInfo: makeSwarmContainer(map[string]string{ policy.LabelEnable: "false", }), @@ -653,6 +655,7 @@ func TestProcessContainer_SwarmServiceLabels(t *testing.T) { }, }, store: newStore(policy.ModeOptIn, true), + isSwarmManager: true, wantServiceCall: true, wantSkip: true, }, @@ -667,15 +670,17 @@ func TestProcessContainer_SwarmServiceLabels(t *testing.T) { }}, }, store: newStore(policy.ModeOptIn, true), + isSwarmManager: true, wantServiceCall: false, }, { - name: "service inspect error is non-fatal", + name: "service inspect error is non-fatal on manager", containerInfo: makeSwarmContainer(map[string]string{ policy.LabelEnable: "true", }), svcErr: errDaemonUnavail, store: newStore(policy.ModeOptIn, true), + isSwarmManager: true, wantServiceCall: true, wantLogMsg: "could not inspect parent service", }, @@ -693,9 +698,35 @@ func TestProcessContainer_SwarmServiceLabels(t *testing.T) { }, }, store: newStore(policy.ModeOptIn, true), + isSwarmManager: true, wantServiceCall: true, wantLogMsg: "unrecognized swarm-device-access label on parent service", }, + { + name: "worker node skips service inspect", + containerInfo: makeSwarmContainer(map[string]string{ + policy.LabelEnable: "true", + }), + store: newStore(policy.ModeOptIn, true), + isSwarmManager: false, + wantServiceCall: false, + }, + { + name: "manager warns when known label set via deploy.labels", + containerInfo: makeSwarmContainer(nil), + svcResult: swarm.Service{ + Spec: swarm.ServiceSpec{ + Annotations: swarm.Annotations{ + Name: "my-service", + Labels: map[string]string{policy.LabelEnable: "true"}, + }, + }, + }, + store: newStore(policy.ModeOptIn, true), + isSwarmManager: true, + wantServiceCall: true, + wantLogMsg: "swarm-device-access label set via deploy.labels on parent service", + }, } for _, tc := range cases { @@ -711,10 +742,11 @@ func TestProcessContainer_SwarmServiceLabels(t *testing.T) { } proc := &Processor{ - Inspector: inspector, - Cfg: tc.store, - HostRoot: t.TempDir(), - ProcRoot: procRoot, + Inspector: inspector, + Cfg: tc.store, + HostRoot: t.TempDir(), + ProcRoot: procRoot, + IsSwarmManager: tc.isSwarmManager, } _ = proc.ProcessContainer(context.Background(), cid)