diff --git a/Makefile b/Makefile index 314f92ca..d48837e8 100644 --- a/Makefile +++ b/Makefile @@ -101,6 +101,7 @@ OUTPUT_BIN ?= $(repo_dir)/bin DCP_DIR ?= $(home_dir)/.dcp DCP_BINARY ?= ${OUTPUT_BIN}/dcp$(bin_exe_suffix) DCPTUN_CLIENT_BINARY ?= $(OUTPUT_BIN)/dcptun_c +DCPDNS_BINARY ?= $(OUTPUT_BIN)/dcpdns_c # Locations and definitions for tool binaries GO_BIN ?= go @@ -266,11 +267,11 @@ endif ##@ Development ifeq ($(build_os),linux) -COMMON_BUILD_PREREQS := build-dcp build-dcptun-containerexe +COMMON_BUILD_PREREQS := build-dcp build-dcptun-containerexe build-dcpdns-containerexe COMPILE_PREREQS := $(COMMON_BUILD_PREREQS) else COMMON_BUILD_PREREQS := build-dcp -COMPILE_PREREQS := $(COMMON_BUILD_PREREQS) build-dcptun-containerexe +COMPILE_PREREQS := $(COMMON_BUILD_PREREQS) build-dcptun-containerexe build-dcpdns-containerexe endif # Note: Go runtime is incompatible with C/C++ stack protection feature https://github.com/golang/go/blob/master/src/runtime/cgo/cgo.go#L28 More info/rationale https://github.com/golang/go/issues/21871#issuecomment-329330371 @@ -305,6 +306,15 @@ else GOOS=linux $(GO_BIN) build -o $(DCPTUN_CLIENT_BINARY) $(BUILD_ARGS) ./cmd/dcptun endif +.PHONY: build-dcpdns-containerexe +build-dcpdns-containerexe: $(DCPDNS_BINARY) ## Builds DCP DNS forwarder binary for Linux (used in containers on the Apple container runtime) +$(DCPDNS_BINARY): $(GO_SOURCES) go.mod | $(OUTPUT_BIN) +ifeq ($(detected_OS),windows) + $$env:GOOS = "linux"; $(GO_BIN) build -o $(DCPDNS_BINARY) $(BUILD_ARGS) ./cmd/dcpdns +else + GOOS=linux $(GO_BIN) build -o $(DCPDNS_BINARY) $(BUILD_ARGS) ./cmd/dcpdns +endif + .PHONY: clean clean: | ${OUTPUT_BIN} ${TOOL_BIN} ## Deletes build output (all binaries), and all cached tool binaries. $(rm_rf) $(OUTPUT_BIN)/* @@ -318,11 +328,13 @@ lint: golangci-lint generate-grpc ## Runs the linter install: compile | $(DCP_DIR) ## Installs all binaries to their destinations $(install) $(DCP_BINARY) $(DCP_DIR) $(install) $(DCPTUN_CLIENT_BINARY) $(DCP_DIR) + $(install) $(DCPDNS_BINARY) $(DCP_DIR) .PHONY: uninstall uninstall: ## Uninstalls all binaries from their destinations $(rm_f) $(DCP_DIR)/dcp$(bin_exe_suffix) $(rm_f) $(DCP_DIR)/dcptun_c + $(rm_f) $(DCP_DIR)/dcpdns_c ifneq ($(detected_OS),windows) .PHONY: link-dcp diff --git a/cmd/dcpdns/main.go b/cmd/dcpdns/main.go new file mode 100644 index 00000000..e7990e3b --- /dev/null +++ b/cmd/dcpdns/main.go @@ -0,0 +1,101 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// dcpdns is the DNS forwarder DCP runs inside a container on the Apple container runtime's +// network to provide name resolution for container names and network aliases. +// See internal/dcpdns for details. +package main + +import ( + "context" + "flag" + "fmt" + "net" + "os" + "os/signal" + "strings" + "syscall" + + "github.com/go-logr/logr/funcr" + + "github.com/microsoft/dcp/internal/dcpdns" +) + +func main() { + listenAddr := flag.String("listen", ":53", "Address to serve DNS on") + hostsPath := flag.String("hosts", "", "Path to the hosts-format records file (required)") + upstream := flag.String("upstream", "", "Upstream resolver as host[:port]; defaults to the first nameserver in /etc/resolv.conf") + verbose := flag.Bool("v", false, "Enable verbose logging") + flag.Parse() + + log := funcr.New(func(prefix, args string) { + fmt.Fprintln(os.Stdout, prefix, args) + }, funcr.Options{Verbosity: verbosity(*verbose)}) + + if *hostsPath == "" { + log.Error(nil, "--hosts is required") + os.Exit(2) + } + + upstreamAddr := *upstream + if upstreamAddr == "" { + resolvConfUpstream, resolvErr := upstreamFromResolvConf("/etc/resolv.conf") + if resolvErr != nil { + log.Error(resolvErr, "No --upstream specified and no nameserver found in /etc/resolv.conf") + os.Exit(2) + } + upstreamAddr = resolvConfUpstream + } + if _, _, splitErr := net.SplitHostPort(upstreamAddr); splitErr != nil { + upstreamAddr = net.JoinHostPort(upstreamAddr, "53") + } + + udpConn, udpErr := net.ListenPacket("udp", *listenAddr) + if udpErr != nil { + log.Error(udpErr, "Failed to listen for UDP DNS queries", "Address", *listenAddr) + os.Exit(1) + } + + tcpListener, tcpErr := net.Listen("tcp", *listenAddr) + if tcpErr != nil { + log.Error(tcpErr, "Failed to listen for TCP DNS queries", "Address", *listenAddr) + os.Exit(1) + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + log.Info("DCP DNS forwarder starting", "Listen", *listenAddr, "Upstream", upstreamAddr, "Hosts", *hostsPath) + + server := dcpdns.NewServer(dcpdns.NewRecords(*hostsPath), upstreamAddr, log) + if serveErr := server.Serve(ctx, udpConn, tcpListener); serveErr != nil { + log.Error(serveErr, "DNS forwarder failed") + os.Exit(1) + } +} + +func verbosity(verbose bool) int { + if verbose { + return 2 + } + return 0 +} + +// upstreamFromResolvConf returns the first nameserver listed in the given resolv.conf file. +func upstreamFromResolvConf(path string) (string, error) { + content, readErr := os.ReadFile(path) + if readErr != nil { + return "", readErr + } + + for _, line := range strings.Split(string(content), "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "nameserver" { + return fields[1], nil + } + } + + return "", fmt.Errorf("no nameserver entries in %s", path) +} diff --git a/controllers/container_controller.go b/controllers/container_controller.go index 76efe6da..72307164 100644 --- a/controllers/container_controller.go +++ b/controllers/container_controller.go @@ -8,6 +8,8 @@ package controllers import ( "context" "crypto/sha256" + "encoding/base64" + "encoding/hex" "errors" "fmt" "os" @@ -16,6 +18,7 @@ import ( "reflect" "regexp" "runtime" + "sort" "strings" "sync" "time" @@ -75,7 +78,15 @@ const ( var ( containerFinalizer string = fmt.Sprintf("%s/container-reconciler", apiv1.GroupVersion.Group) - containerKind = apiv1.GroupVersion.WithKind(reflect.TypeOf(apiv1.Container{}).Name()) + + // bakedLayerModTime is a fixed, deterministic modification time applied to files baked into a + // derived image layer. Using a constant (rather than time.Now()) keeps the resulting tar bytes — + // and therefore the layer digest, which is the cache key — stable across runs when the file contents + // are unchanged, which is what lets the content-addressed derived-image cache reuse a built image. + // Certificate identity needs no special handling: the certificate bytes are part of the hashed tar, + // so rotating a certificate changes the digest and correctly misses the cache. + bakedLayerModTime = time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) + containerKind = apiv1.GroupVersion.WithKind(reflect.TypeOf(apiv1.Container{}).Name()) errInitialContainerNetworksNotReady = errors.New("initial container networks are not ready") @@ -1260,13 +1271,26 @@ func (r *ContainerReconciler) startContainerWithOrchestrator(container *apiv1.Co } // Add labels to the container for lifecycle management and other metadata - lifecycleKey, hashErr := r.addContainerCreationLabels(container, rcd, log) + _, hashErr := r.addContainerCreationLabels(container, rcd, log) if hashErr != nil { return hashErr } + // Runtimes that cannot copy files into a created-but-not-running container (e.g. the Apple + // container runtime) bake the configured create-files and PEM certificates into a derived + // image as additional layers, so the files are present in the container filesystem before + // the entrypoint runs. + var createFilesLayers []apiv1.ImageLayer + if r.orchestrator.CreateFilesRequiresRunningContainer() { + bakedLayers, bakeErr := r.bakeCreateFilesImageLayers(startupCtx, rcd, log) + if bakeErr != nil { + return bakeErr + } + createFilesLayers = bakedLayers + } + // Apply any image layers that are specified in the container spec, and get the effective image to use for container creation - effectiveImage, imageLayersErr := r.applyContainerImageLayers(startupCtx, rcd, lifecycleKey, log) + effectiveImage, imageLayersErr := r.applyContainerImageLayers(startupCtx, rcd, createFilesLayers, log) if imageLayersErr != nil { return imageLayersErr } @@ -1309,17 +1333,15 @@ func (r *ContainerReconciler) startContainerWithOrchestrator(container *apiv1.Co r.runContainerLifecycleMonitor(rcd, log) } - // Copy any general files specified in the container spec to the container's filesystem - fileModTime := time.Now() - copyFilesErr := r.copyContainerCreateFiles(startupCtx, rcd, inspected, fileModTime, log) - if copyFilesErr != nil { - return copyFilesErr - } - - // Copy any PEM certificates specified in the container spec to the container's filesystem - copyCertsErr := r.copyContainerPemCertificates(startupCtx, rcd, inspected, fileModTime, log) - if copyCertsErr != nil { - return copyCertsErr + // Copy any general files and PEM certificates specified in the container spec into the + // container's filesystem. Most runtimes copy into the created, not-yet-started container. + // Runtimes that can only write into a running container (e.g. the Apple container runtime) + // cannot do this; they instead bake the files into the image as layers at creation time + // (see bakeCreateFilesImageLayers above), so the copy here is skipped for them. + if !r.orchestrator.CreateFilesRequiresRunningContainer() { + if copyErr := r.copyContainerCreateFilesAndCertificates(startupCtx, rcd, inspected.Id, time.Now(), r.orchestrator.CreateFiles, log); copyErr != nil { + return copyErr + } } // Finish the container startup process, including any finalization steps @@ -1517,14 +1539,21 @@ func (r *ContainerReconciler) addContainerCreationLabels(container *apiv1.Contai } // applyContainerImageLayers builds a derived image when image layers are configured and returns the image to create. -func (r *ContainerReconciler) applyContainerImageLayers(ctx context.Context, rcd *runningContainerData, lifecycleKey string, log logr.Logger) (string, error) { +func (r *ContainerReconciler) applyContainerImageLayers(ctx context.Context, rcd *runningContainerData, extraLayers []apiv1.ImageLayer, log logr.Logger) (string, error) { effectiveImage := rcd.runSpec.Image - if len(rcd.runSpec.ImageLayers) == 0 { + + // Combine the layers declared in the run spec with any layers baked at creation time (e.g. Apple + // container create-files). Do not mutate rcd.runSpec.ImageLayers: the container creation block may + // retry, and mutating the spec would append the extra layers more than once. + layers := make([]apiv1.ImageLayer, 0, len(rcd.runSpec.ImageLayers)+len(extraLayers)) + layers = append(layers, rcd.runSpec.ImageLayers...) + layers = append(layers, extraLayers...) + if len(layers) == 0 { return effectiveImage, nil } - digests := make([]string, len(rcd.runSpec.ImageLayers)) - for i, layer := range rcd.runSpec.ImageLayers { + digests := make([]string, len(layers)) + for i, layer := range layers { digests[i] = layer.Digest } rcd.runSpec.Labels = append(rcd.runSpec.Labels, apiv1.ContainerLabel{ @@ -1543,24 +1572,40 @@ func (r *ContainerReconciler) applyContainerImageLayers(ctx context.Context, rcd return "", ensureErr } - // Derive a tag by replacing the base image's tag/digest with the lifecycle key. - // The image ref may be name:tag or name@sha256:digest — we need just the name part. + // Derive a deterministic, content-addressed tag for the derived image. The tag keeps the base + // image's repository name and original tag as a prefix, then appends a digest over the resolved + // base image ID plus the applied layer digests, so identical inputs map to the same tag on every + // run while staying easy to correlate with the base image (e.g. redis:7.2 -> redis:7.2-dcp-). + // The image ref may be name:tag, name@sha256:digest, or name:tag@sha256:digest. baseRepo := rcd.runSpec.Image + baseTag := "" if atidx := strings.Index(baseRepo, "@"); atidx != -1 { - baseRepo = baseRepo[:atidx] - } else if colonidx := strings.LastIndex(baseRepo, ":"); colonidx != -1 { - // Only strip at colon if it's a tag separator, not part of a port/registry. - // A tag separator colon comes after the last slash (or is the only colon). - slashIdx := strings.LastIndex(baseRepo, "/") - if colonidx > slashIdx { - baseRepo = baseRepo[:colonidx] - } + baseRepo = baseRepo[:atidx] // drop any digest portion before parsing the tag + } + // A tag separator colon comes after the last slash (otherwise the colon is part of a registry port). + if colonidx := strings.LastIndex(baseRepo, ":"); colonidx > strings.LastIndex(baseRepo, "/") { + baseTag = baseRepo[colonidx+1:] + baseRepo = baseRepo[:colonidx] } - sanitizedKey := strings.ReplaceAll(lifecycleKey, ":", "-") - derivedTag := fmt.Sprintf("%s:dcp-%s", baseRepo, sanitizedKey) + contentDigest := sha256.Sum256([]byte(baseImageInspected.Id + "\n" + strings.Join(digests, "\n"))) + derivedTagSuffix := fmt.Sprintf("dcp-%x", contentDigest) + if baseTag != "" { + derivedTagSuffix = baseTag + "-" + derivedTagSuffix + } + derivedTag := fmt.Sprintf("%s:%s", baseRepo, derivedTagSuffix) + + // Reuse a previously built derived image when one already exists for this exact content. Building + // the derived image is expensive, so skipping it when the inputs are unchanged avoids redundant + // work across runs. A miss here is expected (first run, or inputs changed) and falls through to + // building. + if existing, inspectErr := r.orchestrator.InspectImages(ctx, containers.InspectImagesOptions{Images: []string{derivedTag}}); inspectErr == nil && len(existing) > 0 { + log.V(1).Info("Reusing cached derived image for image layers", "DerivedImage", derivedTag) + return derivedTag, nil + } + applyLayersOptions := containers.ApplyImageLayersOptions{ BaseImage: *baseImageInspected, - Layers: rcd.runSpec.ImageLayers, + Layers: layers, Tag: derivedTag, } derivedImage, applyErr := r.orchestrator.ApplyImageLayers(ctx, applyLayersOptions) @@ -1645,12 +1690,18 @@ func (r *ContainerReconciler) releasePersistentContainerResourceLease( return nil } +// createFilesSink consumes a single CreateFiles operation. The default sink writes the files into a +// container via the orchestrator; an alternative sink (see bakeCreateFilesImageLayers) turns each +// operation into an image layer instead. +type createFilesSink func(ctx context.Context, options containers.CreateFilesOptions) error + // copyContainerCreateFiles copies configured file entries into the created container before it starts. func (r *ContainerReconciler) copyContainerCreateFiles( ctx context.Context, rcd *runningContainerData, - inspected *containers.InspectedContainer, + containerID string, fileModTime time.Time, + sink createFilesSink, log logr.Logger, ) error { for _, createFileRequest := range rcd.runSpec.CreateFiles { @@ -1660,7 +1711,7 @@ func (r *ContainerReconciler) copyContainerCreateFiles( } createFilesOptions := containers.CreateFilesOptions{ - Container: inspected.Id, + Container: containerID, Entries: createFileRequest.Entries, Destination: createFileRequest.Destination, DefaultOwner: createFileRequest.DefaultOwner, @@ -1669,7 +1720,7 @@ func (r *ContainerReconciler) copyContainerCreateFiles( ModTime: fileModTime, } - copyErr := r.orchestrator.CreateFiles(ctx, createFilesOptions) + copyErr := sink(ctx, createFilesOptions) if copyErr != nil { log.Error(copyErr, "Could not copy files to the container", "Destination", createFileRequest.Destination) return copyErr @@ -1681,12 +1732,64 @@ func (r *ContainerReconciler) copyContainerCreateFiles( return nil } +// copyContainerCreateFilesAndCertificates copies configured file entries and PEM certificates into the container. +func (r *ContainerReconciler) copyContainerCreateFilesAndCertificates( + ctx context.Context, + rcd *runningContainerData, + containerID string, + fileModTime time.Time, + sink createFilesSink, + log logr.Logger, +) error { + if copyFilesErr := r.copyContainerCreateFiles(ctx, rcd, containerID, fileModTime, sink, log); copyFilesErr != nil { + return copyFilesErr + } + + return r.copyContainerPemCertificates(ctx, rcd, containerID, fileModTime, sink, log) +} + +// bakeCreateFilesImageLayers converts the container's configured create-files and PEM certificates into +// image layers (tar archives). Runtimes that cannot copy files into a created-but-not-running container +// (e.g. the Apple container runtime) bake these layers into a derived image at creation time, so +// injected files — including TLS certificates an entrypoint reads at startup — are present before the +// container starts. +func (r *ContainerReconciler) bakeCreateFilesImageLayers(ctx context.Context, rcd *runningContainerData, log logr.Logger) ([]apiv1.ImageLayer, error) { + layers := []apiv1.ImageLayer{} + sink := func(_ context.Context, options containers.CreateFilesOptions) error { + tarBytes, buildErr := containers.BuildCreateFilesTar(options, log) + if buildErr != nil { + return buildErr + } + if tarBytes == nil { + return nil + } + + digest := sha256.Sum256(tarBytes) + layers = append(layers, apiv1.ImageLayer{ + Digest: hex.EncodeToString(digest[:]), + RawContents: base64.StdEncoding.EncodeToString(tarBytes), + }) + return nil + } + + // The container does not exist yet, so the container ID passed to the sink is empty; the sink + // ignores it and turns each CreateFiles operation into an image layer. A fixed, deterministic mod + // time is used so unchanged contents produce an identical layer digest on every run, allowing the + // content-addressed derived-image cache to reuse a previously built image. + if buildErr := r.copyContainerCreateFilesAndCertificates(ctx, rcd, "", bakedLayerModTime, sink, log); buildErr != nil { + return nil, buildErr + } + + return layers, nil +} + // copyContainerPemCertificates installs configured PEM certificates and optional bundle overwrites into the container. func (r *ContainerReconciler) copyContainerPemCertificates( ctx context.Context, rcd *runningContainerData, - inspected *containers.InspectedContainer, + containerID string, fileModTime time.Time, + sink createFilesSink, log logr.Logger, ) error { if rcd.runSpec.PemCertificates == nil { @@ -1734,7 +1837,7 @@ func (r *ContainerReconciler) copyContainerPemCertificates( } createFilesOptions := containers.CreateFilesOptions{ - Container: inspected.Id, + Container: containerID, Destination: rcd.runSpec.PemCertificates.Destination, Entries: []apiv1.FileSystemEntry{ { @@ -1751,7 +1854,7 @@ func (r *ContainerReconciler) copyContainerPemCertificates( ModTime: fileModTime, } - copyErr := r.orchestrator.CreateFiles(ctx, createFilesOptions) + copyErr := sink(ctx, createFilesOptions) if copyErr != nil { log.Error(copyErr, "Could not copy certificates to the container", "Destination", createFilesOptions.Destination) return copyErr @@ -1761,9 +1864,16 @@ func (r *ContainerReconciler) copyContainerPemCertificates( return path.Dir(bundlePath) }) - for bundleDir, files := range overwritePaths { + // Iterate the overwrite directories in a stable order. When these operations are baked into image + // layers, the layer sequence determines the content-addressed cache key, so a deterministic order is + // required for the derived-image cache to reuse a previously built image. + bundleDirs := maps.Keys(overwritePaths) + sort.Strings(bundleDirs) + + for _, bundleDir := range bundleDirs { + files := overwritePaths[bundleDir] bundleCreateFilesOptions := containers.CreateFilesOptions{ - Container: inspected.Id, + Container: containerID, Destination: bundleDir, Entries: slices.Map[apiv1.FileSystemEntry](files, func(file string) apiv1.FileSystemEntry { return apiv1.FileSystemEntry{ @@ -1776,7 +1886,7 @@ func (r *ContainerReconciler) copyContainerPemCertificates( ModTime: fileModTime, } - bundleCopyErr := r.orchestrator.CreateFiles(ctx, bundleCreateFilesOptions) + bundleCopyErr := sink(ctx, bundleCreateFilesOptions) if bundleCopyErr != nil { if rcd.runSpec.PemCertificates.ContinueOnError { log.Info("Could not overwrite certificate bundle in the container, continuing...", "Destination", bundleDir, "Error", bundleCopyErr.Error()) @@ -2155,8 +2265,10 @@ func (r *ContainerReconciler) ensureContainerNetworkConnections( return []*apiv1.ContainerNetwork{}, err } - if container.Spec.Networks == nil || container.Spec.Stop || !container.DeletionTimestamp.IsZero() || rcd.containerState == apiv1.ContainerStateFailedToStart { + if container.Spec.Networks == nil || containers.UsesSingleNetwork(r.orchestrator) || container.Spec.Stop || !container.DeletionTimestamp.IsZero() || rcd.containerState == apiv1.ContainerStateFailedToStart { // Delete all connections if no networks are defined, stop has been requested, or the container is being deleted. + // Single-network runtimes (e.g. Apple container) have no custom networks to connect to: every + // container is already on the default network, so no ContainerNetworkConnection objects are created. // // We also delete all connections if the container fails to start, because the container orchestrator (Docker/Podman) // may not be able to attach a "dead" container to a network. Specifically, in dead state, inspecting the container diff --git a/controllers/container_image_layer_cache_test.go b/controllers/container_image_layer_cache_test.go new file mode 100644 index 00000000..ef5a5841 --- /dev/null +++ b/controllers/container_image_layer_cache_test.go @@ -0,0 +1,241 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package controllers + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/require" + + apiv1 "github.com/microsoft/dcp/api/v1" + "github.com/microsoft/dcp/internal/containers" + "github.com/microsoft/dcp/pkg/testutil" +) + +// imageLayerCacheTestOrchestrator is a minimal ContainerOrchestrator used to exercise the +// derived-image caching behavior in applyContainerImageLayers. Only the image-inspection, +// pull, and apply-layers methods are implemented; any other method panics (nil embedded +// interface) which keeps the test honest about the code path under test. +type imageLayerCacheTestOrchestrator struct { + containers.ContainerOrchestrator + + mu sync.Mutex + images map[string]containers.InspectedImage // ref/tag -> inspected image + applyCalls []string // derived tags passed to ApplyImageLayers, in order + pullCalls int +} + +func newImageLayerCacheTestOrchestrator() *imageLayerCacheTestOrchestrator { + return &imageLayerCacheTestOrchestrator{images: map[string]containers.InspectedImage{}} +} + +func (o *imageLayerCacheTestOrchestrator) addImage(ref string, img containers.InspectedImage) { + o.mu.Lock() + defer o.mu.Unlock() + o.images[ref] = img +} + +func (o *imageLayerCacheTestOrchestrator) InspectImages(_ context.Context, options containers.InspectImagesOptions) ([]containers.InspectedImage, error) { + o.mu.Lock() + defer o.mu.Unlock() + + var result []containers.InspectedImage + var err error + for _, ref := range options.Images { + if img, ok := o.images[ref]; ok { + result = append(result, img) + } else { + err = errors.Join(err, containers.ErrNotFound) + } + } + return result, err +} + +func (o *imageLayerCacheTestOrchestrator) PullImage(_ context.Context, options containers.PullImageOptions) (string, error) { + o.mu.Lock() + defer o.mu.Unlock() + + o.pullCalls++ + img := containers.InspectedImage{Id: "pulled-" + options.Image, Tags: []string{options.Image}} + o.images[options.Image] = img + return img.Id, nil +} + +func (o *imageLayerCacheTestOrchestrator) ApplyImageLayers(_ context.Context, options containers.ApplyImageLayersOptions) (string, error) { + o.mu.Lock() + defer o.mu.Unlock() + + o.applyCalls = append(o.applyCalls, options.Tag) + // Simulate the build registering the derived image so subsequent inspections hit the cache. + o.images[options.Tag] = containers.InspectedImage{Id: "built-" + options.Tag, Tags: []string{options.Tag}} + return options.Tag, nil +} + +func newImageLayerRCD(image string, layers ...apiv1.ImageLayer) *runningContainerData { + return &runningContainerData{ + runSpec: &apiv1.ContainerSpec{ + Image: image, + ImageLayers: layers, + }, + } +} + +// Verifies that a second apply with identical base image and layers reuses the cached derived +// image instead of rebuilding it. +func TestApplyContainerImageLayers_ReusesCachedDerivedImage(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, 30*time.Second) + defer cancel() + + orch := newImageLayerCacheTestOrchestrator() + orch.addImage("redis:7.2", containers.InspectedImage{Id: "base-redis-id", Tags: []string{"redis:7.2"}}) + + r := &ContainerReconciler{orchestrator: orch} + layer := apiv1.ImageLayer{Digest: "layer-digest-1", RawContents: "dGFy"} + + firstImage, firstErr := r.applyContainerImageLayers(ctx, newImageLayerRCD("redis:7.2", layer), nil, logr.Discard()) + require.NoError(t, firstErr) + require.NotEqual(t, "redis:7.2", firstImage, "a derived image should be produced") + require.Regexp(t, `^redis:7\.2-dcp-[0-9a-f]{64}$`, firstImage, "the derived tag should preserve the original tag as a prefix") + + secondImage, secondErr := r.applyContainerImageLayers(ctx, newImageLayerRCD("redis:7.2", layer), nil, logr.Discard()) + require.NoError(t, secondErr) + + require.Equal(t, firstImage, secondImage, "identical inputs must map to the same derived tag") + require.Len(t, orch.applyCalls, 1, "the derived image should be built only once and reused on the second call") +} + +// Verifies that changing layer content produces a different derived tag and triggers a rebuild. +func TestApplyContainerImageLayers_RebuildsWhenLayerContentChanges(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, 30*time.Second) + defer cancel() + + orch := newImageLayerCacheTestOrchestrator() + orch.addImage("redis:7.2", containers.InspectedImage{Id: "base-redis-id", Tags: []string{"redis:7.2"}}) + + r := &ContainerReconciler{orchestrator: orch} + + firstImage, firstErr := r.applyContainerImageLayers( + ctx, newImageLayerRCD("redis:7.2", apiv1.ImageLayer{Digest: "layer-digest-1"}), nil, logr.Discard()) + require.NoError(t, firstErr) + + secondImage, secondErr := r.applyContainerImageLayers( + ctx, newImageLayerRCD("redis:7.2", apiv1.ImageLayer{Digest: "layer-digest-2"}), nil, logr.Discard()) + require.NoError(t, secondErr) + + require.NotEqual(t, firstImage, secondImage, "different layer content must map to different derived tags") + require.Len(t, orch.applyCalls, 2, "a content change must trigger a rebuild") +} + +// Verifies that the same layer applied on top of different base image content yields different +// derived tags, confirming the base image ID is folded into the cache key. +func TestApplyContainerImageLayers_BaseImageIdAffectsDerivedTag(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, 30*time.Second) + defer cancel() + + layer := apiv1.ImageLayer{Digest: "shared-layer-digest"} + + orchA := newImageLayerCacheTestOrchestrator() + orchA.addImage("redis:7.2", containers.InspectedImage{Id: "base-id-A", Tags: []string{"redis:7.2"}}) + rA := &ContainerReconciler{orchestrator: orchA} + imageA, errA := rA.applyContainerImageLayers(ctx, newImageLayerRCD("redis:7.2", layer), nil, logr.Discard()) + require.NoError(t, errA) + + orchB := newImageLayerCacheTestOrchestrator() + orchB.addImage("redis:7.2", containers.InspectedImage{Id: "base-id-B", Tags: []string{"redis:7.2"}}) + rB := &ContainerReconciler{orchestrator: orchB} + imageB, errB := rB.applyContainerImageLayers(ctx, newImageLayerRCD("redis:7.2", layer), nil, logr.Discard()) + require.NoError(t, errB) + + require.NotEqual(t, imageA, imageB, "a change in the base image content must change the derived tag") +} + +// Verifies that layers baked at creation time (passed as extraLayers) are folded into the cache +// key, and that the cache hits across calls when the baked content is unchanged. +func TestApplyContainerImageLayers_ExtraLayersParticipateInCacheKey(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, 30*time.Second) + defer cancel() + + orch := newImageLayerCacheTestOrchestrator() + orch.addImage("postgres:16", containers.InspectedImage{Id: "base-pg-id", Tags: []string{"postgres:16"}}) + + r := &ContainerReconciler{orchestrator: orch} + extra := []apiv1.ImageLayer{{Digest: "cert-layer-digest"}} + + firstImage, firstErr := r.applyContainerImageLayers(ctx, newImageLayerRCD("postgres:16"), extra, logr.Discard()) + require.NoError(t, firstErr) + + secondImage, secondErr := r.applyContainerImageLayers(ctx, newImageLayerRCD("postgres:16"), extra, logr.Discard()) + require.NoError(t, secondErr) + + require.Equal(t, firstImage, secondImage) + require.Len(t, orch.applyCalls, 1, "unchanged baked layers should hit the cache on the second call") + + differentExtra := []apiv1.ImageLayer{{Digest: "cert-layer-digest-rotated"}} + thirdImage, thirdErr := r.applyContainerImageLayers(ctx, newImageLayerRCD("postgres:16"), differentExtra, logr.Discard()) + require.NoError(t, thirdErr) + + require.NotEqual(t, firstImage, thirdImage, "rotated baked content must produce a new derived tag") + require.Len(t, orch.applyCalls, 2) +} + +// Verifies the derived tag format preserves the base repository and original tag as a prefix across +// the supported image reference forms (plain, registry-with-port, and digest-only). +func TestApplyContainerImageLayers_DerivedTagFormat(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, 30*time.Second) + defer cancel() + + layer := apiv1.ImageLayer{Digest: "layer-digest"} + + cases := []struct { + name string + image string + tagDigest string // digest portion to register the base image under, if any + wantPattern string + }{ + { + name: "plain repository and tag", + image: "redis:7.2", + wantPattern: `^redis:7\.2-dcp-[0-9a-f]{64}$`, + }, + { + name: "registry with port and tag", + image: "localhost:5000/redis:7.2", + wantPattern: `^localhost:5000/redis:7\.2-dcp-[0-9a-f]{64}$`, + }, + { + name: "digest-only reference has no original tag to preserve", + image: "redis@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + wantPattern: `^redis:dcp-[0-9a-f]{64}$`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + orch := newImageLayerCacheTestOrchestrator() + orch.addImage(tc.image, containers.InspectedImage{Id: "base-id-" + tc.name, Tags: []string{tc.image}}) + r := &ContainerReconciler{orchestrator: orch} + + derived, err := r.applyContainerImageLayers(ctx, newImageLayerRCD(tc.image, layer), nil, logr.Discard()) + require.NoError(t, err) + require.Regexp(t, tc.wantPattern, derived) + }) + } +} diff --git a/controllers/network_controller.go b/controllers/network_controller.go index e4be197e..ecd2edbd 100644 --- a/controllers/network_controller.go +++ b/controllers/network_controller.go @@ -321,6 +321,12 @@ func (r *NetworkReconciler) deleteNetwork(ctx context.Context, network *apiv1.Co return nil } + if containers.UsesSingleNetwork(r.orchestrator) { + // The network was mapped onto the runtime's permanent, shared default network; there is + // nothing to disconnect or remove. + return nil + } + if network.Status.ID != "" { inspectedNetwork, err := r.orchestrator.InspectNetworks(ctx, containers.InspectNetworksOptions{ Networks: []string{network.Status.ID}, @@ -440,6 +446,12 @@ func (r *NetworkReconciler) failNetworkToStart(network *apiv1.ContainerNetwork, } func (r *NetworkReconciler) ensureNetworkWithName(ctx context.Context, network *apiv1.ContainerNetwork, networkName string, log logr.Logger) objectChange { + if containers.UsesSingleNetwork(r.orchestrator) { + // The runtime exposes only the built-in default network (e.g. Apple container). Map this + // network onto it instead of creating a custom network. + return r.ensureSingleNetwork(ctx, network, log) + } + effectiveMode := network.Spec.EffectiveMode() if shouldReuseExistingNetwork(effectiveMode) { isNetworkSafeToReuse := r.harvester.IsDone() || r.harvester.TryProtectNetwork(ctx, networkName) @@ -541,6 +553,37 @@ func (r *NetworkReconciler) ensureNetworkWithName(ctx context.Context, network * return statusChanged } +// ensureSingleNetwork maps a ContainerNetwork onto the runtime's built-in default network for +// single-network runtimes (e.g. Apple container), which have no concept of custom networks. +// Containers are created directly on the default network and reach each other via their IPs, so +// there is nothing to create — we just reflect the default network's details into the status. +func (r *NetworkReconciler) ensureSingleNetwork(ctx context.Context, network *apiv1.ContainerNetwork, log logr.Logger) objectChange { + defaultNetwork, err := inspectNetwork(ctx, r.orchestrator, r.orchestrator.DefaultNetworkName()) + if err != nil { + if errors.Is(err, containers.ErrRuntimeNotHealthy) { + log.Error(err, "Could not inspect the default network as the container runtime is not healthy, retrying...") + return additionalReconciliationNeeded + } + + log.Error(err, "Could not inspect the default network") + r.existingNetworks.Store(network.NamespacedName(), network.Name, &runningNetworkState{state: apiv1.ContainerNetworkStateFailedToStart, message: err.Error()}) + network.Status.State = apiv1.ContainerNetworkStateFailedToStart + network.Status.Message = err.Error() + return statusChanged + } + + r.existingNetworks.Store(network.NamespacedName(), defaultNetwork.Id, &runningNetworkState{state: apiv1.ContainerNetworkStateRunning, id: defaultNetwork.Id}) + network.Status.ID = defaultNetwork.Id + network.Status.State = apiv1.ContainerNetworkStateRunning + network.Status.NetworkName = defaultNetwork.Name + network.Status.Driver = defaultNetwork.Driver + network.Status.IPv6 = defaultNetwork.IPv6 + network.Status.Subnets = defaultNetwork.Subnets + network.Status.Gateways = defaultNetwork.Gateways + + return statusChanged +} + func (r *NetworkReconciler) ensureConnections(ctx context.Context, network *apiv1.ContainerNetwork, networkState *runningNetworkState, log logr.Logger) objectChange { change := noChange diff --git a/internal/applecontainer/cli_orchestrator.go b/internal/applecontainer/cli_orchestrator.go new file mode 100644 index 00000000..c0cdacce --- /dev/null +++ b/internal/applecontainer/cli_orchestrator.go @@ -0,0 +1,1477 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// Package applecontainer implements a container orchestrator backed by the Apple `container` CLI +// (https://github.com/apple/container), available on macOS (Apple Silicon) only. +// +// The Apple container runtime differs from Docker/Podman in several important ways that shape +// this implementation: +// +// - Containers can only be attached to networks at creation time; there is no +// `network connect` / `network disconnect`. DCP maps all DCP-managed networks onto the +// built-in "default" network: CreateNetwork returns "default", ConnectNetwork verifies the +// container is attached, and DisconnectNetwork is a no-op. Containers on the default network +// can reach each other directly via their routable IPs. +// - There is no network alias support and the runtime's DNS serves no records for container +// names. DCP runs a small DNS forwarder in a container and points every container at it, +// so container names and network aliases resolve (see dns_forwarder.go). +// - There is no `host.docker.internal` equivalent. The host is reachable from inside a +// container via the network gateway IP, which ContainerHost() reports. +// - There is no `events` command; container and network events are synthesized by polling. +// - `inspect` commands are all-or-nothing, so multi-object inspections are issued one +// object at a time to allow partial success. +// - Exit codes of detached containers are not reported by the runtime, so InspectContainers +// always reports exit code 0. (`container start --attach` does propagate the exit code, which +// a future improvement could use by keeping an attached observer process per container.) +// - Files can only be copied into a *running* container (see +// CreateFilesRequiresRunningContainer), so files that must be present before a container +// starts are baked into a derived image by the container controller instead. +// - `--label` values cannot contain '='; such values are encoded on the way in and decoded +// when read back (see labels.go). +package applecontainer + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "regexp" + goruntime "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/go-logr/logr" + + apiv1 "github.com/microsoft/dcp/api/v1" + "github.com/microsoft/dcp/pkg/concurrency" + usvc_io "github.com/microsoft/dcp/pkg/io" + "github.com/microsoft/dcp/pkg/osutil" + "github.com/microsoft/dcp/pkg/process" + "github.com/microsoft/dcp/pkg/slices" + + "github.com/microsoft/dcp/internal/containers" + "github.com/microsoft/dcp/internal/dcpproc" + "github.com/microsoft/dcp/internal/networking" + "github.com/microsoft/dcp/internal/pubsub" + "github.com/microsoft/dcp/internal/termpty" +) + +const ( + // Name of the built-in network every container is attached to by default. + defaultNetworkName = "default" + + // Used as the container host name if the default network gateway cannot be determined. + // 192.168.65.1 is the documented default gateway for the built-in "default" network. + fallbackContainerHost = "192.168.65.1" +) + +var ( + containerNotFoundRegEx = regexp.MustCompile(`(?i)container (not found|with ID (.*) not found)`) + volumeNotFoundRegEx = regexp.MustCompile(`(?i)volume not found`) + volumeAlreadyExistsRegEx = regexp.MustCompile(`(?i)volume '(.*)' already exists`) + volumeDeleteFailedRegEx = regexp.MustCompile(`(?i)failed to delete one or more volumes`) + networkNotFoundRegEx = regexp.MustCompile(`(?i)network not found`) + networkDeleteFailedRegEx = regexp.MustCompile(`(?i)failed to delete one or more networks`) + imageNotFoundRegEx = regexp.MustCompile(`(?i)image not found`) + runtimeNotRunningRegEx = regexp.MustCompile(`(?i)(XPC connection error|connection refused|failed to connect|services (are )?not running|container system start)`) + + newContainerNotFoundErrorMatch = containers.NewCliErrorMatch(containerNotFoundRegEx, errors.Join(containers.ErrNotFound, fmt.Errorf("container not found"))) + newVolumeNotFoundErrorMatch = containers.NewCliErrorMatch(volumeNotFoundRegEx, errors.Join(containers.ErrNotFound, fmt.Errorf("volume not found"))) + volumeAlreadyExistsErrorMatch = containers.NewCliErrorMatch(volumeAlreadyExistsRegEx, errors.Join(containers.ErrAlreadyExists, fmt.Errorf("volume already exists"))) + volumeDeleteFailedErrorMatch = containers.NewCliErrorMatch(volumeDeleteFailedRegEx, errors.Join(containers.ErrNotFound, fmt.Errorf("volume could not be deleted"))) + newNetworkNotFoundErrorMatch = containers.NewCliErrorMatch(networkNotFoundRegEx, errors.Join(containers.ErrNotFound, fmt.Errorf("network not found"))) + networkDeleteFailedErrorMatch = containers.NewCliErrorMatch(networkDeleteFailedRegEx, errors.Join(containers.ErrNotFound, fmt.Errorf("network could not be deleted"))) + imageNotFoundErrorMatch = containers.NewCliErrorMatch(imageNotFoundRegEx, errors.Join(containers.ErrNotFound, fmt.Errorf("image not found"))) + runtimeNotRunningErrorMatch = containers.NewCliErrorMatch(runtimeNotRunningRegEx, errors.Join(containers.ErrRuntimeNotHealthy, fmt.Errorf("the Apple container runtime is not healthy"))) + + // We expect almost all `container` CLI invocations to finish within this time. + ordinaryCommandTimeout = 30 * time.Second + + // We allow up to a minute for diagnostic commands to finish as we'd rather wait a bit longer than miss information. + diagnosticCommandTimeout = 1 * time.Minute + + defaultBuildImageTimeout = 10 * time.Minute + defaultPullImageTimeout = 10 * time.Minute + defaultCreateContainerTimeout = 10 * time.Minute + defaultRunContainerTimeout = 10 * time.Minute + + // Cache and synchronization control for checking runtime cachedStatus + cachedStatus *containers.ContainerRuntimeStatus + // Ensure that only one goroutine is checking the status at a time + checkStatusLock = concurrency.NewContextAwareLock() + // Mutex to control read/write access to the cached status + updateStatus = &sync.RWMutex{} + backgroundStatusUpdates atomic.Int32 + + // Cached container host (default network gateway IP) + containerHostLock = &sync.Mutex{} + cachedContainerHost string +) + +type AppleContainerCliOrchestrator struct { + log logr.Logger + + // Process executor for running `container` commands + executor process.Executor + + // Event watcher for container events + containerEvtWatcher *pubsub.SubscriptionSet[containers.EventMessage] + + // Event watcher for network events + networkEvtWatcher *pubsub.SubscriptionSet[containers.EventMessage] + + // State of the DNS forwarder that provides network alias resolution (see dns_forwarder.go) + dns dnsForwarderState +} + +func NewAppleContainerCliOrchestrator(log logr.Logger, executor process.Executor) containers.ContainerOrchestrator { + aco := &AppleContainerCliOrchestrator{ + log: log, + executor: executor, + } + + aco.containerEvtWatcher = pubsub.NewSubscriptionSet(aco.doWatchContainers, context.Background()) + aco.networkEvtWatcher = pubsub.NewSubscriptionSet(aco.doWatchNetworks, context.Background()) + + return aco +} + +func (*AppleContainerCliOrchestrator) IsDefault() bool { + return false +} + +func (*AppleContainerCliOrchestrator) Name() string { + return "container" +} + +// ContainerHost returns the address at which the host machine is reachable from inside a container. +// The Apple container runtime has no built-in DNS name for the host (like host.docker.internal); +// instead the host is reachable via the gateway IP of the container network, so we report +// the gateway of the built-in default network. +func (aco *AppleContainerCliOrchestrator) ContainerHost() string { + containerHostLock.Lock() + defer containerHostLock.Unlock() + + if cachedContainerHost != "" { + return cachedContainerHost + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + cachedContainerHost = fallbackContainerHost + if network, err := aco.inspectSingleNetwork(ctx, defaultNetworkName); err == nil && len(network.Gateways) > 0 && network.Gateways[0] != "" { + cachedContainerHost = network.Gateways[0] + } + + return cachedContainerHost +} + +func (aco *AppleContainerCliOrchestrator) CheckStatus(ctx context.Context, cacheUsage containers.CachedRuntimeStatusUsage) containers.ContainerRuntimeStatus { + // A cached status is already available, return it + updateStatus.RLock() + if cachedStatus != nil && cacheUsage == containers.CachedRuntimeStatusAllowed { + defer updateStatus.RUnlock() + return *cachedStatus + } + updateStatus.RUnlock() + + if cacheUsage == containers.CachedRuntimeStatusAllowed { + // For cached results, only one goroutine should be checking the status at a time + if syncErr := checkStatusLock.Lock(ctx); syncErr != nil { + // Timed out, assume the runtime is not responsive and unavailable + return containers.ContainerRuntimeStatus{ + Installed: false, + Running: false, + Error: "Timed out while checking Apple container runtime status; the `container` CLI is not responsive.", + } + } + + defer checkStatusLock.Unlock() + } + + updateStatus.RLock() + // Check again if the status is available in the cache + if cachedStatus != nil && cacheUsage == containers.CachedRuntimeStatusAllowed { + defer updateStatus.RUnlock() + return *cachedStatus + } + updateStatus.RUnlock() + + newStatus := aco.getStatus(ctx) + + updateStatus.Lock() + // Update the cached status + cachedStatus = &newStatus + updateStatus.Unlock() + + return newStatus +} + +// Check the status of the Apple container runtime in the background until the context is canceled. +func (aco *AppleContainerCliOrchestrator) EnsureBackgroundStatusUpdates(ctx context.Context) { + if !backgroundStatusUpdates.CompareAndSwap(0, 1) { + return + } + + go func() { + timer := time.NewTimer(0) + timer.Stop() + for { + // Only one goroutine should be checking the status at a time + if checkStatusLock.TryLock() { + newStatus := aco.getStatus(ctx) + + updateStatus.Lock() + // Update the cached status + cachedStatus = &newStatus + updateStatus.Unlock() + + checkStatusLock.Unlock() + } + + // Wait for 5 seconds before checking again + timer.Reset(5 * time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + continue + } + } + }() +} + +func (aco *AppleContainerCliOrchestrator) getStatus(ctx context.Context) containers.ContainerRuntimeStatus { + // The Apple container runtime is only available on macOS (Apple Silicon). + if goruntime.GOOS != "darwin" { + return containers.ContainerRuntimeStatus{ + Installed: false, + Running: false, + Error: "The Apple container runtime is only supported on macOS.", + } + } + + // The Apple container CLI always includes the built-in default network and formats its response + // as a JSON array, so we can use this to ensure we're talking to the Apple container CLI and not + // some other binary that happens to be named "container". + _, err := aco.ListNetworks(ctx, containers.ListNetworksOptions{}) + + var invalidUnmarshalError *json.InvalidUnmarshalError + var unmarshalTypeError *json.UnmarshalTypeError + var syntaxError *json.SyntaxError + + if errors.Is(err, exec.ErrNotFound) { + // Try to get the inner error if this is an exec.ErrNotFound error + // The outer error is potentially a "DCP" error and not the actual error from exec + // which makes it harder for users to understand that the error is simply that we + // couldn't find the `container` binary on their path. + if unwrapErr := errors.Unwrap(err); errors.Is(unwrapErr, exec.ErrNotFound) { + err = unwrapErr + } + + // Couldn't find the `container` CLI, so it's not installed + return containers.ContainerRuntimeStatus{ + Installed: false, + Running: false, + Error: err.Error(), + } + } else if errors.Is(err, context.DeadlineExceeded) { + // Timed out, assume the runtime is not responsive but available + return containers.ContainerRuntimeStatus{ + Installed: true, + Running: false, + Error: "The `container` CLI timed out while checking status. Ensure the Apple container CLI is functioning correctly and try again.", + } + } else if errors.As(err, &invalidUnmarshalError) || errors.As(err, &unmarshalTypeError) || errors.As(err, &syntaxError) { + return containers.ContainerRuntimeStatus{ + Installed: false, + Running: false, + Error: "Output from the `container` CLI didn't match the expected format. The `container` CLI found on your PATH may not be a valid Apple container installation.", + } + } else if errors.Is(err, containers.ErrRuntimeNotHealthy) { + return containers.ContainerRuntimeStatus{ + Installed: true, + Running: false, + Error: "The Apple container system services are not running. Start them with 'container system start'.", + } + } else if err != nil { + // Error response from the `container` command, assume runtime isn't available + return containers.ContainerRuntimeStatus{ + Installed: true, + Running: false, + Error: err.Error(), + } + } + + // The command returned successfully, assume runtime is ready + return containers.ContainerRuntimeStatus{ + Installed: true, + Running: true, + } +} + +func (aco *AppleContainerCliOrchestrator) GetDiagnostics(ctx context.Context) (containers.ContainerDiagnostics, error) { + cmd := makeAppleContainerCommand("system", "status", "--format", "json") + outBuf, errBuf, err := aco.runBufferedCommand(ctx, "Version", cmd, nil, nil, diagnosticCommandTimeout) + if err != nil { + return containers.ContainerDiagnostics{}, errors.Join(err, normalizeCliErrors(errBuf)) + } + + var diagnostics containers.ContainerDiagnostics + if unmarshalErr := unmarshalDiagnostics(outBuf.Bytes(), &diagnostics); unmarshalErr != nil { + return containers.ContainerDiagnostics{}, unmarshalErr + } + + versionCmd := makeAppleContainerCommand("--version") + versionOut, _, versionErr := aco.runBufferedCommand(ctx, "ClientVersion", versionCmd, nil, nil, diagnosticCommandTimeout) + if versionErr == nil { + diagnostics.ClientVersion = extractVersion(versionOut.String()) + } + + return diagnostics, nil +} + +// VOLUME MANAGEMENT + +func (aco *AppleContainerCliOrchestrator) CreateVolume(ctx context.Context, options containers.CreateVolumeOptions) error { + cmd := makeAppleContainerCommand("volume", "create", options.Name) + outBuf, errBuf, err := aco.runBufferedCommand(ctx, "CreateVolume", cmd, nil, nil, ordinaryCommandTimeout) + if err != nil { + // Note: unlike Docker, the Apple container CLI returns an error if the volume already exists. + return errors.Join(err, normalizeCliErrors(errBuf, volumeAlreadyExistsErrorMatch)) + } + + return containers.ExpectCliStrings(outBuf, []string{options.Name}) +} + +// InspectVolumes returns the details of successfully inspected volumes, and a slice of all errors encountered during the inspection. +// The Apple container CLI fails the whole inspection if any of the requested volumes is missing, +// so volumes are inspected one at a time to allow partial success. +func (aco *AppleContainerCliOrchestrator) InspectVolumes(ctx context.Context, options containers.InspectVolumesOptions) ([]containers.InspectedVolume, error) { + if len(options.Volumes) == 0 { + return nil, fmt.Errorf("must specify at least one volume") + } + + var inspectedVolumes []containers.InspectedVolume + var err error + + for _, volume := range options.Volumes { + cmd := makeAppleContainerCommand("volume", "inspect", volume) + outBuf, errBuf, cmdErr := aco.runBufferedCommand(ctx, "InspectVolumes", cmd, nil, nil, ordinaryCommandTimeout) + if cmdErr != nil { + err = errors.Join(err, cmdErr, normalizeCliErrors(errBuf, newVolumeNotFoundErrorMatch)) + continue + } + + volumes, unmarshalErr := asArrayOfObjects(outBuf, unmarshalVolume) + err = errors.Join(err, unmarshalErr) + inspectedVolumes = append(inspectedVolumes, volumes...) + } + + if len(inspectedVolumes) < len(options.Volumes) { + err = errors.Join(err, errors.Join(containers.ErrIncomplete, fmt.Errorf("only %v out of %v volumes were successfully inspected", len(inspectedVolumes), len(options.Volumes)))) + } + + return inspectedVolumes, err +} + +func (aco *AppleContainerCliOrchestrator) RemoveVolumes(ctx context.Context, options containers.RemoveVolumesOptions) ([]string, error) { + if len(options.Volumes) == 0 { + return nil, fmt.Errorf("must specify at least one volume") + } + + // Note: the Apple container CLI does not support force-removal of volumes; the Force option is ignored. + args := []string{"volume", "rm"} + args = append(args, options.Volumes...) + + cmd := makeAppleContainerCommand(args...) + + outBuf, errBuf, err := aco.runBufferedCommand(ctx, "RemoveVolumes", cmd, nil, nil, ordinaryCommandTimeout) + if err != nil { + err = errors.Join(err, normalizeCliErrors(errBuf, volumeDeleteFailedErrorMatch.MaxObjects(len(options.Volumes)))) + } + + nonEmpty := slices.NonEmpty[byte](bytes.Split(outBuf.Bytes(), osutil.LF())) + removed := slices.Map[string](nonEmpty, func(bs []byte) string { return string(bs) }) + + if len(removed) < len(options.Volumes) { + err = errors.Join(err, errors.Join(containers.ErrIncomplete, fmt.Errorf("only %v out of %v volumes were successfully removed", len(removed), len(options.Volumes)))) + } + + return removed, err +} + +// IMAGE MANAGEMENT + +func (aco *AppleContainerCliOrchestrator) BuildImage(ctx context.Context, options containers.BuildImageOptions) error { + args := []string{"build"} + + if options.Dockerfile != "" { + args = append(args, "-f", options.Dockerfile) + } + + // Should base images be updated even if they are already present locally? + if options.Pull { + args = append(args, "--pull") + } + + // The Apple container CLI does not support --iidfile; if an image ID file is requested, + // the image is resolved by tag after the build finishes (see below). Make sure there is + // at least one tag to resolve the image by. + tags := options.Tags + if len(tags) == 0 && options.IidFile != "" { + tags = []string{fmt.Sprintf("dcp-build-%d", time.Now().UnixNano())} + } + + // The Apple container CLI accepts a single tag; additional tags are applied after the build. + if len(tags) > 0 { + args = append(args, "-t", tags[0]) + } + + // Apply all specified build arguments + for _, buildArg := range options.Args { + if buildArg.Value != "" { + args = append(args, "--build-arg", fmt.Sprintf("%s=%s", buildArg.Name, buildArg.Value)) + } else { + args = append(args, "--build-arg", buildArg.Name) + } + } + + // Secret values that need to be applied to the build command environment + secretEnvironment := map[string]string{} + + // Apply all specified build secrets + for _, secret := range options.Secrets { + switch secret.Type { + case apiv1.FileSecret, "": + args = append(args, "--secret", fmt.Sprintf("id=%s,src=%s", secret.ID, secret.Source)) + case apiv1.EnvSecret: + if secret.Source != "" { + args = append(args, "--secret", fmt.Sprintf("id=%s,env=%s", secret.ID, secret.Source)) + if secret.Value != "" { + secretEnvironment[secret.Source] = secret.Value + } + } else { + args = append(args, "--secret", fmt.Sprintf("id=%s,env=%s", secret.ID, secret.ID)) + if secret.Value != "" { + secretEnvironment[secret.ID] = secret.Value + } + } + } + } + + // If a build stage is given, use it + if options.Stage != "" { + args = append(args, "--target", options.Stage) + } + + // Apply any specified labels + for _, label := range options.Labels { + args = append(args, "--label", labelArg(label.Key, label.Value)) + } + + // If a target platform is specified, build for that platform + if options.Platform != "" { + args = append(args, "--platform", options.Platform) + } + + // Enable plain output mode + args = append(args, "--progress", "plain") + + // Append the build context argument + args = append(args, options.Context) + + cmd := makeAppleContainerCommand(args...) + + // Append secret environment + cmd.Env = os.Environ() + for secretName, secretValue := range secretEnvironment { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", secretName, secretValue)) + } + + // Building an image can take a long time to finish, particularly if any base images are not available locally. + // Use a much longer timeout than for other commands. + if options.Timeout == 0 { + options.Timeout = defaultBuildImageTimeout + } + _, errBuf, err := aco.runBufferedCommand(ctx, "BuildImage", cmd, options.StdOutStream, options.StdErrStream, options.Timeout) + if err != nil { + return errors.Join(err, normalizeCliErrors(errBuf)) + } + + // Apply any additional tags + for _, extraTag := range tags[min(1, len(tags)):] { + tagCmd := makeAppleContainerCommand("image", "tag", tags[0], extraTag) + if _, tagErrBuf, tagErr := aco.runBufferedCommand(ctx, "TagImage", tagCmd, nil, nil, ordinaryCommandTimeout); tagErr != nil { + return errors.Join(tagErr, normalizeCliErrors(tagErrBuf)) + } + } + + // The Apple container CLI does not support --iidfile, so resolve the image ID by tag and write the file ourselves. + if options.IidFile != "" { + inspected, inspectErr := aco.InspectImages(ctx, containers.InspectImagesOptions{Images: []string{tags[0]}}) + if inspectErr != nil || len(inspected) == 0 { + return errors.Join(inspectErr, fmt.Errorf("could not resolve the ID of the built image '%s'", tags[0])) + } + if writeErr := os.WriteFile(options.IidFile, []byte(inspected[0].Id), 0644); writeErr != nil { + return fmt.Errorf("could not write image ID file '%s': %w", options.IidFile, writeErr) + } + } + + return nil +} + +// InspectImages inspects images one at a time (the Apple container CLI fails the whole +// inspection if any of the requested images is missing). +func (aco *AppleContainerCliOrchestrator) InspectImages(ctx context.Context, options containers.InspectImagesOptions) ([]containers.InspectedImage, error) { + if len(options.Images) == 0 { + return nil, fmt.Errorf("must specify at least one image") + } + + var inspectedImages []containers.InspectedImage + var err error + + for _, image := range options.Images { + cmd := makeAppleContainerCommand("image", "inspect", image) + outBuf, errBuf, cmdErr := aco.runBufferedCommand(ctx, "InspectImages", cmd, nil, nil, ordinaryCommandTimeout) + if cmdErr != nil { + err = errors.Join(err, cmdErr, normalizeCliErrors(errBuf, imageNotFoundErrorMatch)) + continue + } + + images, unmarshalErr := asArrayOfObjects(outBuf, unmarshalImage) + err = errors.Join(err, unmarshalErr) + inspectedImages = append(inspectedImages, images...) + } + + if len(inspectedImages) < len(options.Images) { + err = errors.Join(err, errors.Join(containers.ErrIncomplete, fmt.Errorf("only %v out of %v images were successfully inspected", len(inspectedImages), len(options.Images)))) + } + + return inspectedImages, err +} + +func (aco *AppleContainerCliOrchestrator) PullImage(ctx context.Context, options containers.PullImageOptions) (string, error) { + if options.Image == "" { + return "", fmt.Errorf("must specify an image to pull") + } + + imageRef := options.Image + if options.Digest != "" { + imageRef = options.Image + "@" + options.Digest + } + + cmd := makeAppleContainerCommand("image", "pull", "--progress", "none", imageRef) + // Pulling large images can take a long time, especially if the image is not available locally and the network is slow. + if options.Timeout == 0 { + options.Timeout = defaultPullImageTimeout + } + _, errBuf, err := aco.runBufferedCommand(ctx, "PullImage", cmd, nil, nil, options.Timeout) + if err != nil { + return "", errors.Join(err, normalizeCliErrors(errBuf, imageNotFoundErrorMatch)) + } + + // The pull command does not print the image ID, so resolve it via image inspect. + inspected, inspectErr := aco.InspectImages(ctx, containers.InspectImagesOptions{Images: []string{imageRef}}) + if inspectErr != nil || len(inspected) == 0 { + // Best effort: the pull itself succeeded, so return the reference if the image ID + // cannot be resolved. + return imageRef, nil + } + + return inspected[0].Id, nil +} + +// ApplyImageLayersImpl in the containers package streams the build context to the runtime via stdin, +// which the Apple container CLI does not support. Instead, the build context (Dockerfile + layer tars) +// is materialized in a temporary directory and built from there. +func (aco *AppleContainerCliOrchestrator) ApplyImageLayers(ctx context.Context, options containers.ApplyImageLayersOptions) (string, error) { + if len(options.Layers) == 0 { + return "", fmt.Errorf("at least one image layer must be specified") + } + + timeout := options.Timeout + if timeout == 0 { + timeout = defaultBuildImageTimeout + } + + // Use a tag if available, otherwise fall back to the image ID for the FROM directive + baseImage := options.BaseImage.Id + if len(options.BaseImage.Tags) > 0 { + baseImage = options.BaseImage.Tags[0] + } + + contextDir, mkdirErr := os.MkdirTemp("", "dcp-image-layers-") + if mkdirErr != nil { + return "", fmt.Errorf("creating temporary build context directory: %w", mkdirErr) + } + defer func() { _ = os.RemoveAll(contextDir) }() + + dockerfile := fmt.Sprintf("FROM %s\n", baseImage) + for i := range options.Layers { + layer := &options.Layers[i] + layerFileName := fmt.Sprintf("layer%d.tar", i) + + if writeErr := writeLayerToFile(layer, contextDir, layerFileName); writeErr != nil { + return "", fmt.Errorf("materializing image layer %d: %w", i, writeErr) + } + + dockerfile += fmt.Sprintf("ADD %s /\n", layerFileName) + } + + if writeErr := os.WriteFile(contextDir+string(os.PathSeparator)+"Dockerfile", []byte(dockerfile), 0644); writeErr != nil { + return "", fmt.Errorf("writing Dockerfile to build context: %w", writeErr) + } + + tag := options.Tag + if tag == "" { + tag = fmt.Sprintf("dcp-layered-%d", time.Now().UnixNano()) + } + + cmd := makeAppleContainerCommand("build", "--progress", "plain", "-t", tag, contextDir) + + _, errBuf, buildErr := aco.runBufferedCommand(ctx, "ApplyImageLayers", cmd, nil, nil, timeout) + if buildErr != nil { + return "", fmt.Errorf("building derived image with image layers: %w: %s", buildErr, errBuf.String()) + } + + aco.log.V(1).Info("Built derived image with image layers", "ImageRef", tag, "LayerCount", len(options.Layers)) + + return tag, nil +} + +// CONTAINER MANAGEMENT + +func (aco *AppleContainerCliOrchestrator) applyCreateContainerOptions(args []string, options containers.CreateContainerOptions) []string { + if options.Name != "" { + args = append(args, "--name", options.Name) + } + + // The Apple container runtime supports repeating --network to attach multiple networks at + // creation time, but has no --network-alias equivalent; aliases are resolved by the DCP DNS + // forwarder instead (see dns_forwarder.go). Networks are deduplicated because DCP-managed + // networks all map onto the built-in default network (see UsesSingleNetwork), so multiple + // requested networks typically resolve to the same runtime network. + attachedNetworks := map[string]bool{} + for _, network := range options.Networks { + if network.Name == "" || attachedNetworks[network.Name] { + continue + } + attachedNetworks[network.Name] = true + args = append(args, "--network", network.Name) + } + + for _, mount := range options.VolumeMounts { + mountVal := fmt.Sprintf("type=%s", mount.Type) + if mount.Source != "" { + mountVal += fmt.Sprintf(",source=%s", mount.Source) + } + mountVal += fmt.Sprintf(",target=%s", mount.Target) + if mount.ReadOnly { + mountVal += ",readonly" + } + args = append(args, "--mount", mountVal) + } + + for _, port := range options.Ports { + hostIP := port.HostIP + if hostIP == "" { + // Bind to 127.0.0.1 for extra security, not to 0.0.0.0 (all interfaces, making it accessible from the outside) + // IPv6 is not well supported with container networking, so we assume IPv4. We'll need to revisit this logic + // if we start getting requests to support IPv6 container networking. + hostIP = networking.IPv4LocalhostDefaultAddress + } + + hostPort := port.HostPort + if hostPort == 0 { + // The Apple container CLI requires an explicit host port in the publish specification, + // so allocate a free one ourselves. + protocol := port.Protocol + if protocol == "" { + protocol = apiv1.TCP + } + freePort, portErr := networking.GetFreePort(protocol, hostIP, aco.log) + if portErr != nil { + // Let the runtime report the invalid (zero) port; there is no good way to recover here. + aco.log.Error(portErr, "Could not allocate a free host port for container port", "ContainerPort", port.ContainerPort) + } + hostPort = freePort + } + + portVal := fmt.Sprintf("%s:%d:%d", hostIP, hostPort, port.ContainerPort) + + if port.Protocol != "" { + portVal = fmt.Sprintf("%s/%s", portVal, strings.ToLower(string(port.Protocol))) + } + + args = append(args, "-p", portVal) + } + + for _, envVar := range options.Env { + eVal := fmt.Sprintf("%s=%s", envVar.Name, envVar.Value) + args = append(args, "-e", eVal) + } + + for _, envFile := range options.EnvFiles { + args = append(args, "--env-file", envFile) + } + + for _, label := range options.Labels { + args = append(args, "--label", labelArg(label.Key, label.Value)) + } + + if options.RestartPolicy != "" && options.RestartPolicy != apiv1.RestartPolicyNone { + // The Apple container runtime has no restart policy support. + aco.log.V(1).Info("Restart policies are not supported by the Apple container runtime and will be ignored", "RestartPolicy", options.RestartPolicy) + } + + // Note: the Apple container runtime has no pull policy flag; images are pulled if (and only if) + // they are not available locally, which matches the "missing" pull policy. + if options.PullPolicy == apiv1.PullPolicyAlways { + aco.log.V(1).Info("Pull policy 'always' is not supported by the Apple container runtime; the image will only be pulled if not available locally") + } + + if options.Command != "" { + args = append(args, "--entrypoint", options.Command) + } + + if len(options.Healthcheck.Command) > 0 { + // The Apple container runtime has no health check support. + aco.log.V(1).Info("Container health checks are not supported by the Apple container runtime and will be ignored") + } + + if options.Terminal != nil { + // Attach a TTY (-t) and keep STDIN open (-i) if a terminal is requested + args = append(args, "-it") + } + + args = append(args, options.RunArgs...) + + return args +} + +func (aco *AppleContainerCliOrchestrator) CreateContainer(ctx context.Context, options containers.CreateContainerOptions) (string, error) { + args := []string{"create"} + + args = aco.applyCreateContainerOptions(args, options) + + // Point the container at the DCP DNS forwarder so container names and network aliases + // resolve (see dns_forwarder.go). If the forwarder is unavailable the container keeps + // the runtime default (gateway) resolver and only alias resolution is degraded. + if dnsIP := aco.ensureDnsForwarder(ctx); dnsIP != "" { + args = append(args, "--dns", dnsIP) + } + aco.registerPendingDnsNames(options.Name, options.Networks) + + args = append(args, options.Image) + + if len(options.Args) > 0 { + args = append(args, options.Args...) + } + + cmd := makeAppleContainerCommand(args...) + + // Create container can take a long time to finish if the image is not available locally. + // Use a much longer timeout than for other commands. + if options.Timeout == 0 { + options.Timeout = defaultCreateContainerTimeout + } + + outBuf, errBuf, err := aco.runBufferedCommand(ctx, "CreateContainer", cmd, options.StdOutStream, options.StdErrStream, options.Timeout) + if err != nil { + err = errors.Join(err, normalizeCliErrors(errBuf)) + if id, err2 := asId(outBuf); err2 == nil { + // We got an ID, so the container was created, but the command failed. + return id, err + } + + return "", err + } + + return asId(outBuf) +} + +func (aco *AppleContainerCliOrchestrator) RunContainer(ctx context.Context, options containers.RunContainerOptions) (string, error) { + args := []string{"run"} + + args = aco.applyCreateContainerOptions(args, options.CreateContainerOptions) + + // See CreateContainer for the DNS forwarder handling. + if dnsIP := aco.ensureDnsForwarder(ctx); dnsIP != "" { + args = append(args, "--dns", dnsIP) + } + aco.registerPendingDnsNames(options.Name, options.Networks) + + args = append(args, "--detach") + args = append(args, options.Image) + + if len(options.Args) > 0 { + args = append(args, options.Args...) + } + + cmd := makeAppleContainerCommand(args...) + + // The run container command can take a long time to finish if the image is not available locally. + // So we use much longer timeout than for other commands. + if options.Timeout == 0 { + options.Timeout = defaultRunContainerTimeout + } + + outBuf, errBuf, err := aco.runBufferedCommand(ctx, "RunContainer", cmd, options.StdOutStream, options.StdErrStream, options.Timeout) + if err != nil { + return "", errors.Join(err, normalizeCliErrors(errBuf)) + } + + id, idErr := asId(outBuf) + if idErr == nil { + aco.publishDnsRecords(ctx, id) + } + return id, idErr +} + +func (aco *AppleContainerCliOrchestrator) ExecContainer(ctx context.Context, options containers.ExecContainerOptions) (<-chan int32, error) { + args := []string{"exec"} + + if options.WorkingDirectory != "" { + args = append(args, "--workdir", options.WorkingDirectory) + } + + for _, envVar := range options.Env { + eVal := fmt.Sprintf("%s=%s", envVar.Name, envVar.Value) + args = append(args, "-e", eVal) + } + + for _, envFile := range options.EnvFiles { + args = append(args, "--env-file", envFile) + } + + args = append(args, options.Container) + args = append(args, options.Command) + if len(options.Args) > 0 { + args = append(args, options.Args...) + } + + cmd := makeAppleContainerCommand(args...) + + cmd.Stdout = options.StdOutStream + cmd.Stderr = options.StdErrStream + + exitCh := make(chan int32) + exitHandler := func(_ process.Pid_t, exitCode int32, err error) { + // We only care about the exit code, not the error. The only scenario where we should get an error + // is if the context for an exec command is canceled during DCP shutdown, in which case that's expected. + if err != nil && !errors.Is(err, context.Canceled) { + aco.log.Error(err, "Unexpected error during container exec command", "Command", cmd.String()) + } + exitCh <- exitCode + close(exitCh) + } + + aco.log.V(1).Info("Running Apple container command", "Command", cmd.String()) + _, startWaitForProcessExit, err := aco.executor.StartProcess(ctx, cmd, process.ProcessExitHandlerFunc(exitHandler), process.CreationFlagsNone, nil) + if err != nil { + close(exitCh) + return nil, errors.Join(err, fmt.Errorf("failed to start Apple container command '%s'", "ExecContainer")) + } + startWaitForProcessExit() + + return exitCh, nil +} + +// AttachContainer is not supported: the Apple container CLI has no `attach` command. +func (aco *AppleContainerCliOrchestrator) AttachContainer(ctx context.Context, options containers.AttachContainerOptions) (*termpty.PseudoTerminalProcess, error) { + return nil, fmt.Errorf("attaching a terminal to a container is not supported by the Apple container runtime") +} + +func (aco *AppleContainerCliOrchestrator) ListContainers(ctx context.Context, options containers.ListContainersOptions) ([]containers.ListedContainer, error) { + // Note: the Apple container CLI has no server-side label filtering; filters are applied client-side. + cmd := makeAppleContainerCommand("ls", "--format", "json") + outBuf, errBuf, err := aco.runBufferedCommand(ctx, "ListContainers", cmd, nil, nil, ordinaryCommandTimeout) + if err != nil { + // If the command failed, return the error + return nil, errors.Join(err, normalizeCliErrors(errBuf)) + } + + listed, unmarshalErr := asArrayOfObjects(outBuf, unmarshalListedContainer) + if unmarshalErr != nil { + return nil, unmarshalErr + } + + return filterByLabels(listed, options.Filters.LabelFilters, func(lc containers.ListedContainer) map[string]string { return lc.Labels }), nil +} + +// InspectContainers inspects containers one at a time (the Apple container CLI fails the whole +// inspection if any of the requested containers is missing). +func (aco *AppleContainerCliOrchestrator) InspectContainers(ctx context.Context, options containers.InspectContainersOptions) ([]containers.InspectedContainer, error) { + if len(options.Containers) == 0 { + return nil, fmt.Errorf("must specify at least one container") + } + + var inspectedContainers []containers.InspectedContainer + var err error + + for _, container := range options.Containers { + cmd := makeAppleContainerCommand("inspect", container) + outBuf, errBuf, cmdErr := aco.runBufferedCommand(ctx, "InspectContainers", cmd, nil, nil, ordinaryCommandTimeout) + if cmdErr != nil { + err = errors.Join(err, cmdErr, normalizeCliErrors(errBuf, newContainerNotFoundErrorMatch)) + continue + } + + inspected, unmarshalErr := asArrayOfObjects(outBuf, unmarshalContainer) + err = errors.Join(err, unmarshalErr) + inspectedContainers = append(inspectedContainers, inspected...) + } + + if len(inspectedContainers) < len(options.Containers) { + // We're returning an incomplete set of inspected containers + err = errors.Join(err, errors.Join(containers.ErrIncomplete, fmt.Errorf("only %v out of %v containers were successfully inspected", len(inspectedContainers), len(options.Containers)))) + } + + return inspectedContainers, err +} + +// StartContainers starts containers one at a time (the Apple container CLI `start` command +// accepts a single container only). +func (aco *AppleContainerCliOrchestrator) StartContainers(ctx context.Context, options containers.StartContainersOptions) ([]string, error) { + if len(options.Containers) == 0 { + return nil, fmt.Errorf("must specify at least one container") + } + + var started []string + var err error + + for _, container := range options.Containers { + cmd := makeAppleContainerCommand("start", container) + outBuf, errBuf, cmdErr := aco.runBufferedCommand(ctx, "StartContainers", cmd, options.StdOutStream, options.StdErrStream, ordinaryCommandTimeout) + if cmdErr != nil { + err = errors.Join(err, cmdErr, normalizeCliErrors(errBuf, newContainerNotFoundErrorMatch)) + continue + } + + if id, idErr := asId(outBuf); idErr == nil { + started = append(started, id) + } else { + started = append(started, container) + } + + // The container now has an IP address; publish its DNS names to the forwarder. + aco.publishDnsRecords(ctx, container) + } + + if len(started) < len(options.Containers) { + err = errors.Join(err, errors.Join(containers.ErrIncomplete, fmt.Errorf("only %v out of %v containers were successfully started", len(started), len(options.Containers)))) + } + + return started, err +} + +func (aco *AppleContainerCliOrchestrator) StopContainers(ctx context.Context, options containers.StopContainersOptions) ([]string, error) { + if len(options.Containers) == 0 { + return nil, fmt.Errorf("must specify at least one container") + } + + args := []string{"stop"} + var timeout time.Duration = ordinaryCommandTimeout + if options.SecondsToKill > 0 { + args = append(args, "--time", fmt.Sprintf("%d", options.SecondsToKill)) + timeout = time.Duration(options.SecondsToKill)*time.Second + ordinaryCommandTimeout + } + args = append(args, options.Containers...) + + cmd := makeAppleContainerCommand(args...) + outBuf, errBuf, err := aco.runBufferedCommand(ctx, "StopContainers", cmd, nil, nil, timeout) + if err != nil { + err = errors.Join(err, normalizeCliErrors(errBuf, newContainerNotFoundErrorMatch.MaxObjects(len(options.Containers)))) + } + + nonEmpty := slices.NonEmpty[byte](bytes.Split(outBuf.Bytes(), osutil.LF())) + stopped := slices.Map[string](nonEmpty, func(bs []byte) string { return string(bs) }) + + // The stopped containers' IP addresses are no longer valid; retract their DNS records. + aco.unpublishDnsRecords(stopped) + + if len(stopped) < len(options.Containers) { + err = errors.Join(err, errors.Join(containers.ErrIncomplete, fmt.Errorf("only %v out of %v containers were successfully stopped", len(stopped), len(options.Containers)))) + } + + return stopped, err +} + +func (aco *AppleContainerCliOrchestrator) RemoveContainers(ctx context.Context, options containers.RemoveContainersOptions) ([]string, error) { + if len(options.Containers) == 0 { + return nil, fmt.Errorf("must specify at least one container") + } + + args := []string{"rm"} + if options.Force { + args = append(args, "--force") + } + args = append(args, options.Containers...) + + cmd := makeAppleContainerCommand(args...) + outBuf, errBuf, err := aco.runBufferedCommand(ctx, "RemoveContainers", cmd, nil, nil, ordinaryCommandTimeout) + if err != nil { + err = errors.Join(err, normalizeCliErrors(errBuf, newContainerNotFoundErrorMatch.MaxObjects(len(options.Containers)))) + } + + nonEmpty := slices.NonEmpty[byte](bytes.Split(outBuf.Bytes(), osutil.LF())) + removed := slices.Map[string](nonEmpty, func(bs []byte) string { return string(bs) }) + + aco.unpublishDnsRecords(removed) + + if len(removed) < len(options.Containers) { + err = errors.Join(err, errors.Join(containers.ErrIncomplete, fmt.Errorf("only %v out of %v containers were successfully removed", len(removed), len(options.Containers)))) + } + + return removed, err +} + +// CreateFilesRequiresRunningContainer reports that the Apple container runtime can only inject +// files into a running container. The container controller responds by baking configured files +// and certificates into a derived image at creation time (so they are present before the +// entrypoint runs) instead of calling CreateFiles on a created-but-not-started container. +func (aco *AppleContainerCliOrchestrator) CreateFilesRequiresRunningContainer() bool { + return true +} + +// CreateFiles copies files into a container by streaming a tar archive through +// `container exec ... tar -x`. The Apple container CLI can only copy files into a +// *running* container (and `container cp` does not preserve file ownership), so this +// requires the container to be running and the container image to include a `tar` binary. +// Files that must be present before the container starts are baked into a derived image +// by the container controller instead (see CreateFilesRequiresRunningContainer). +func (aco *AppleContainerCliOrchestrator) CreateFiles(ctx context.Context, options containers.CreateFilesOptions) error { + tarBytes, tarErr := containers.BuildCreateFilesTar(options, aco.log) + if tarErr != nil { + return tarErr + } + if tarBytes == nil { + // Can happen if all ContinueOnError items fail + return nil + } + + cmd := makeAppleContainerCommand("exec", "-i", options.Container, "tar", "-xpf", "-", "-C", "/") + cmd.Stdin = bytes.NewReader(tarBytes) + _, errBuf, err := aco.runBufferedCommand(ctx, "CreateFiles", cmd, nil, nil, ordinaryCommandTimeout) + if err != nil { + return errors.Join( + err, + normalizeCliErrors(errBuf, newContainerNotFoundErrorMatch), + fmt.Errorf("the Apple container runtime can only copy files into a running container whose image provides a 'tar' binary")) + } + + return nil +} + +func (aco *AppleContainerCliOrchestrator) WatchContainers(sink chan<- containers.EventMessage) (*pubsub.Subscription[containers.EventMessage], error) { + sub := aco.containerEvtWatcher.Subscribe(sink) + return sub, nil +} + +func (aco *AppleContainerCliOrchestrator) CaptureContainerLogs(ctx context.Context, container string, stdout usvc_io.WriteSyncerCloser, stderr usvc_io.WriteSyncerCloser, options containers.StreamContainerLogsOptions) error { + args := []string{"logs"} + if options.Follow { + args = append(args, "--follow") + } + args = append(args, container) + + cmd := makeAppleContainerCommand(args...) + + // The Apple container CLI has no --timestamps option; if timestamps are requested, + // stamp each line with the time it was received, mimicking the Docker/Podman output format. + var stdOutWriter io.Writer = stdout + var stdErrWriter io.Writer = stderr + if options.Timestamps { + stdOutWriter = newTimestampingWriter(stdout) + stdErrWriter = newTimestampingWriter(stderr) + } + + exitCh, err := aco.streamCommand(ctx, "CaptureContainerLogs", cmd, stdOutWriter, stdErrWriter, streamCommandOptionUseWatcher) + if err != nil { + return err + } + + go func() { + // Wait for the command to finish and clean up any resources + exitErr := <-exitCh + if exitErr != nil && !errors.Is(exitErr, context.Canceled) && !errors.Is(exitErr, context.DeadlineExceeded) { + aco.log.Error(exitErr, "Capturing container logs failed", "Container", container) + } + + if stdOutCloseErr := stdout.Close(); stdOutCloseErr != nil { + aco.log.Error(stdOutCloseErr, "Closing stdout log destination failed", "Container", container) + } + if stdErrCloseErr := stderr.Close(); stdErrCloseErr != nil { + aco.log.Error(stdErrCloseErr, "Closing stderr log destination failed", "Container", container) + } + }() + + return nil +} + +// NETWORK MANAGEMENT +// +// The Apple container runtime can attach containers to networks only when the container is +// created; there is no `network connect`/`network disconnect`. DCP's container and network +// controllers, however, create containers first and connect them to the requested networks +// afterwards. To bridge this gap, all DCP-managed networks are mapped onto the built-in +// "default" network that every container is attached to automatically: +// +// - CreateNetwork does not create anything and returns the "default" network ID. +// - ConnectNetwork verifies that the container is attached to the requested network. +// - DisconnectNetwork is a no-op. +// +// Containers attached to the default network get their own routable IP address and can reach +// each other directly, so the connectivity expectations of DCP-managed networks still hold. + +func (aco *AppleContainerCliOrchestrator) WatchNetworks(sink chan<- containers.EventMessage) (*pubsub.Subscription[containers.EventMessage], error) { + sub := aco.networkEvtWatcher.Subscribe(sink) + return sub, nil +} + +func (aco *AppleContainerCliOrchestrator) CreateNetwork(ctx context.Context, options containers.CreateNetworkOptions) (string, error) { + // Containers cannot be attached to networks after creation with the Apple container runtime, + // so DCP-managed networks are mapped onto the built-in default network (see above). + aco.log.V(1).Info("The Apple container runtime does not support connecting containers to networks after creation; using the built-in default network", "RequestedNetwork", options.Name) + + network, err := aco.inspectSingleNetwork(ctx, defaultNetworkName) + if err != nil { + return "", err + } + + return network.Id, nil +} + +func (aco *AppleContainerCliOrchestrator) RemoveNetworks(ctx context.Context, options containers.RemoveNetworksOptions) ([]string, error) { + if len(options.Networks) == 0 { + return nil, fmt.Errorf("must specify at least one network") + } + + var removed []string + var err error + var toRemove []string + + for _, network := range options.Networks { + if network == defaultNetworkName { + // The built-in default network cannot (and should not) be removed. + // DCP-managed networks map onto it, so report success. + removed = append(removed, network) + } else { + toRemove = append(toRemove, network) + } + } + + if len(toRemove) > 0 { + args := append([]string{"network", "rm"}, toRemove...) + cmd := makeAppleContainerCommand(args...) + outBuf, errBuf, cmdErr := aco.runBufferedCommand(ctx, "RemoveNetworks", cmd, nil, nil, ordinaryCommandTimeout) + if cmdErr != nil { + err = errors.Join(cmdErr, normalizeCliErrors(errBuf, networkDeleteFailedErrorMatch.MaxObjects(len(toRemove)), newNetworkNotFoundErrorMatch.MaxObjects(len(toRemove)))) + } + + nonEmpty := slices.NonEmpty[byte](bytes.Split(outBuf.Bytes(), osutil.LF())) + removed = append(removed, slices.Map[string](nonEmpty, func(bs []byte) string { return string(bs) })...) + } + + if len(removed) < len(options.Networks) { + err = errors.Join(err, errors.Join(containers.ErrIncomplete, fmt.Errorf("only %v out of %v networks were successfully removed", len(removed), len(options.Networks)))) + } + + return removed, err +} + +// InspectNetworks inspects networks one at a time (the Apple container CLI fails the whole +// inspection if any of the requested networks is missing). The list of attached containers +// is not part of the Apple network inspect output and is synthesized from the container list. +func (aco *AppleContainerCliOrchestrator) InspectNetworks(ctx context.Context, options containers.InspectNetworksOptions) ([]containers.InspectedNetwork, error) { + if len(options.Networks) == 0 { + return nil, fmt.Errorf("must specify at least one network") + } + + var inspected []containers.InspectedNetwork + var err error + + for _, network := range options.Networks { + net, inspectErr := aco.inspectSingleNetwork(ctx, network) + if inspectErr != nil { + err = errors.Join(err, inspectErr) + continue + } + inspected = append(inspected, *net) + } + + // Populate the attached containers for the networks we successfully inspected. + if len(inspected) > 0 { + if attachments, attachErr := aco.getNetworkAttachments(ctx); attachErr != nil { + err = errors.Join(err, attachErr) + } else { + for i := range inspected { + inspected[i].Containers = attachments[inspected[i].Name] + } + } + } + + if len(inspected) < len(options.Networks) { + err = errors.Join(err, errors.Join(containers.ErrIncomplete, fmt.Errorf("only %v out of %v networks were successfully inspected", len(inspected), len(options.Networks)))) + } + + return inspected, err +} + +func (aco *AppleContainerCliOrchestrator) inspectSingleNetwork(ctx context.Context, network string) (*containers.InspectedNetwork, error) { + cmd := makeAppleContainerCommand("network", "inspect", network) + outBuf, errBuf, err := aco.runBufferedCommand(ctx, "InspectNetworks", cmd, nil, nil, ordinaryCommandTimeout) + if err != nil { + return nil, errors.Join(err, normalizeCliErrors(errBuf, newNetworkNotFoundErrorMatch)) + } + + networks, unmarshalErr := asArrayOfObjects(outBuf, unmarshalNetwork) + if unmarshalErr != nil { + return nil, unmarshalErr + } + if len(networks) == 0 { + return nil, errors.Join(containers.ErrNotFound, fmt.Errorf("network '%s' was not found", network)) + } + + return &networks[0], nil +} + +// getNetworkAttachments returns the containers attached to each network, keyed by network name. +func (aco *AppleContainerCliOrchestrator) getNetworkAttachments(ctx context.Context) (map[string][]containers.InspectedNetworkContainer, error) { + cmd := makeAppleContainerCommand("ls", "--all", "--format", "json") + outBuf, errBuf, err := aco.runBufferedCommand(ctx, "ListContainers", cmd, nil, nil, ordinaryCommandTimeout) + if err != nil { + return nil, errors.Join(err, normalizeCliErrors(errBuf)) + } + + var listed []appleListedContainer + if unmarshalErr := json.Unmarshal(outBuf.Bytes(), &listed); unmarshalErr != nil { + return nil, errors.Join(containers.ErrUnmarshalling, unmarshalErr) + } + + attachments := map[string][]containers.InspectedNetworkContainer{} + for _, ctr := range listed { + for _, network := range ctr.Configuration.Networks { + attachments[network.Network] = append(attachments[network.Network], containers.InspectedNetworkContainer{ + Id: ctr.Id, + Name: ctr.Id, + }) + } + } + + return attachments, nil +} + +// ConnectNetwork verifies that the container is attached to the requested network. +// The Apple container runtime cannot attach a container to a network after the container +// has been created, so if the container is not attached this returns an error. +func (aco *AppleContainerCliOrchestrator) ConnectNetwork(ctx context.Context, options containers.ConnectNetworkOptions) error { + inspected, err := aco.InspectContainers(ctx, containers.InspectContainersOptions{Containers: []string{options.Container}}) + if err != nil { + return err + } + + for _, network := range inspected[0].Networks { + if network.Name == options.Network || network.Id == options.Network { + // The container is already attached to the network (the only kind of "connection" + // the Apple container runtime supports). + return errors.Join(containers.ErrAlreadyExists, fmt.Errorf("container is already attached to network '%s'", options.Network)) + } + } + + return fmt.Errorf("the Apple container runtime does not support connecting a container to a network after the container has been created") +} + +// DisconnectNetwork is a no-op: the Apple container runtime does not support detaching +// a container from a network. DCP uses disconnection only as a preparation step before +// connecting a container to its requested networks, which maps onto the built-in default +// network for this runtime (see above), so there is nothing to do here. +func (aco *AppleContainerCliOrchestrator) DisconnectNetwork(ctx context.Context, options containers.DisconnectNetworkOptions) error { + aco.log.V(1).Info("The Apple container runtime does not support disconnecting containers from networks; ignoring", "Network", options.Network, "Container", options.Container) + return nil +} + +func (aco *AppleContainerCliOrchestrator) ListNetworks(ctx context.Context, options containers.ListNetworksOptions) ([]containers.ListedNetwork, error) { + // Note: the Apple container CLI has no server-side label filtering; filters are applied client-side. + cmd := makeAppleContainerCommand("network", "ls", "--format", "json") + outBuf, errBuf, err := aco.runBufferedCommand(ctx, "ListNetworks", cmd, nil, nil, ordinaryCommandTimeout) + if err != nil { + return nil, errors.Join(err, normalizeCliErrors(errBuf)) + } + + listed, unmarshalErr := asArrayOfObjects(outBuf, unmarshalListedNetwork) + if unmarshalErr != nil { + return nil, unmarshalErr + } + + return filterByLabels(listed, options.Filters.LabelFilters, func(ln containers.ListedNetwork) map[string]string { return ln.Labels }), nil +} + +func (aco *AppleContainerCliOrchestrator) DefaultNetworkName() string { + return defaultNetworkName +} + +// UsesSingleNetwork reports that the Apple container runtime exposes only the built-in default +// network. DCP maps all networks onto it and skips the connect/disconnect machinery. See +// containers.SingleNetworkOrchestrator. +func (aco *AppleContainerCliOrchestrator) UsesSingleNetwork() bool { + return true +} + +// COMMAND EXECUTION HELPERS + +type streamCommandOption uint32 + +const ( + streamCommandOptionNone streamCommandOption = 0 + streamCommandOptionUseWatcher streamCommandOption = 1 +) + +func (aco *AppleContainerCliOrchestrator) streamCommand( + ctx context.Context, + commandName string, + cmd *exec.Cmd, + stdOutWriter io.Writer, + stdErrWriter io.Writer, + opts streamCommandOption, +) (<-chan error, error) { + cmd.Stdout = stdOutWriter + cmd.Stderr = stdErrWriter + + exitCh := make(chan error) + exitHandler := func(_ process.Pid_t, exitCode int32, err error) { + defer close(exitCh) + if err != nil { + exitCh <- err + } + + if exitCode != 0 { + exitCh <- fmt.Errorf("apple container command '%s' returned with non-zero exit code %d", commandName, exitCode) + } + } + + aco.log.V(1).Info("Running Apple container command", "Command", cmd.String()) + handle, startWaitForProcessExit, err := aco.executor.StartProcess(ctx, cmd, process.ProcessExitHandlerFunc(exitHandler), process.CreationFlagsNone, nil) + if err != nil { + close(exitCh) + return nil, errors.Join(err, fmt.Errorf("failed to start Apple container command '%s'", commandName)) + } + + if opts&streamCommandOptionUseWatcher != 0 { + dcpproc.RunProcessWatcher(aco.executor, handle, aco.log) + } + + startWaitForProcessExit() + + return exitCh, nil +} + +func (aco *AppleContainerCliOrchestrator) runBufferedCommand( + ctx context.Context, + commandName string, + cmd *exec.Cmd, + stdOutWriteCloser io.WriteCloser, + stdErrWriteCloser io.WriteCloser, + timeout time.Duration, +) (*bytes.Buffer, *bytes.Buffer, error) { + effectiveCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + outBuf := new(bytes.Buffer) + var stdOutWriter io.Writer + if stdOutWriteCloser != nil { + defer func() { _ = stdOutWriteCloser.Close() }() + stdOutWriter = io.MultiWriter(stdOutWriteCloser, outBuf) + } else { + stdOutWriter = outBuf + } + + errBuf := new(bytes.Buffer) + var stdErrWriter io.Writer + if stdErrWriteCloser != nil { + defer func() { _ = stdErrWriteCloser.Close() }() + stdErrWriter = io.MultiWriter(stdErrWriteCloser, errBuf) + } else { + stdErrWriter = errBuf + } + + exitCh, err := aco.streamCommand(effectiveCtx, commandName, cmd, stdOutWriter, stdErrWriter, streamCommandOptionNone) + if err == nil { + // If we successfully started running, wait for the command to finish + exitErr := <-exitCh + if exitErr != nil { + err = exitErr + } + } + + return outBuf, errBuf, err +} + +func makeAppleContainerCommand(args ...string) *exec.Cmd { + cmd := exec.Command("container", args...) + // exec.Command resolves cmd.Path via LookPath but leaves cmd.Args[0] as + // the original "container" string. Align Args[0] with the resolved Path so + // downstream consumers and child-process + // argv[0] observers see a consistent, fully-qualified command name. + if cmd.Path != "" { + cmd.Args[0] = cmd.Path + } + return cmd +} + +func (aco *AppleContainerCliOrchestrator) MakeCommand(args ...string) *exec.Cmd { + return makeAppleContainerCommand(args...) +} + +func (aco *AppleContainerCliOrchestrator) RunBufferedCommand(ctx context.Context, opName string, cmd *exec.Cmd, stdout io.WriteCloser, stderr io.WriteCloser, timeout time.Duration) (*bytes.Buffer, *bytes.Buffer, error) { + return aco.runBufferedCommand(ctx, opName, cmd, stdout, stderr, timeout) +} + +func normalizeCliErrors(errBuf *bytes.Buffer, errorMatches ...containers.ErrorMatch) error { + errorMatches = append(errorMatches, runtimeNotRunningErrorMatch) + return containers.NormalizeCliErrors(errBuf, errorMatches...) +} + +// writeLayerToFile materializes an image layer (either a source file reference or inline +// base64 contents) as a tar file in the build context directory, verifying the SHA256 hash +// of source files. +func writeLayerToFile(layer *apiv1.ImageLayer, contextDir string, fileName string) error { + targetPath := contextDir + string(os.PathSeparator) + fileName + + if layer.Source != "" { + data, readErr := os.ReadFile(layer.Source) + if readErr != nil { + return fmt.Errorf("reading layer source file %q: %w", layer.Source, readErr) + } + + if verifyErr := verifySha256(data, layer.SHA256); verifyErr != nil { + return fmt.Errorf("verifying layer source file %q: %w", layer.Source, verifyErr) + } + + return os.WriteFile(targetPath, data, 0644) + } + + decoded, decodeErr := base64.StdEncoding.DecodeString(layer.RawContents) + if decodeErr != nil { + return fmt.Errorf("decoding base64 rawContents for layer (%q): %w", layer.Digest, decodeErr) + } + + return os.WriteFile(targetPath, decoded, 0644) +} + +var _ containers.VolumeOrchestrator = (*AppleContainerCliOrchestrator)(nil) +var _ containers.ImageOrchestrator = (*AppleContainerCliOrchestrator)(nil) +var _ containers.ContainerOrchestrator = (*AppleContainerCliOrchestrator)(nil) +var _ containers.NetworkOrchestrator = (*AppleContainerCliOrchestrator)(nil) diff --git a/internal/applecontainer/cli_orchestrator_live_test.go b/internal/applecontainer/cli_orchestrator_live_test.go new file mode 100644 index 00000000..9c7cc130 --- /dev/null +++ b/internal/applecontainer/cli_orchestrator_live_test.go @@ -0,0 +1,347 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package applecontainer + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/go-logr/logr/testr" + "github.com/stretchr/testify/require" + + apiv1 "github.com/microsoft/dcp/api/v1" + "github.com/microsoft/dcp/internal/containers" + usvc_io "github.com/microsoft/dcp/pkg/io" + "github.com/microsoft/dcp/pkg/process" + "github.com/microsoft/dcp/pkg/testutil" +) + +const liveTestImage = "docker.io/library/alpine:latest" + +// TestLiveAppleContainerOrchestrator exercises the orchestrator against a real Apple +// `container` CLI installation. It is skipped unless DCP_TEST_APPLE_CONTAINER=1 is set +// (requires macOS with the Apple container runtime installed and running). +func TestLiveAppleContainerOrchestrator(t *testing.T) { + if os.Getenv("DCP_TEST_APPLE_CONTAINER") != "1" { + t.Skip("Set DCP_TEST_APPLE_CONTAINER=1 to run tests against a live Apple container runtime") + } + + ctx, cancel := testutil.GetTestContext(t, 3*time.Minute) + defer cancel() + + log := testr.New(t) + executor := process.NewOSExecutor(log) + defer executor.Dispose() + + aco := NewAppleContainerCliOrchestrator(log, executor) + + status := aco.CheckStatus(ctx, containers.IgnoreCachedRuntimeStatus) + require.True(t, status.IsHealthy(), "The Apple container runtime must be installed and running for live tests: %s", status.Error) + + containerName := fmt.Sprintf("dcp-live-test-%d", time.Now().UnixNano()) + volumeName := containerName + "-vol" + + // Volume lifecycle + require.NoError(t, aco.CreateVolume(ctx, containers.CreateVolumeOptions{Name: volumeName})) + defer func() { + _, _ = aco.RemoveVolumes(ctx, containers.RemoveVolumesOptions{Volumes: []string{volumeName}}) + }() + + volumes, err := aco.InspectVolumes(ctx, containers.InspectVolumesOptions{Volumes: []string{volumeName}}) + require.NoError(t, err) + require.Len(t, volumes, 1) + require.Equal(t, volumeName, volumes[0].Name) + + // Image pull + inspect + imageId, err := aco.PullImage(ctx, containers.PullImageOptions{Image: liveTestImage}) + require.NoError(t, err) + require.NotEmpty(t, imageId) + + images, err := aco.InspectImages(ctx, containers.InspectImagesOptions{Images: []string{liveTestImage}}) + require.NoError(t, err) + require.Len(t, images, 1) + require.Equal(t, imageId, images[0].Id) + + // The first container triggers DNS forwarder provisioning; make sure the forwarder + // and its records directory are cleaned up after the test (registered before the + // container cleanups below so it runs last). + defer func() { + _, _ = aco.RemoveContainers(ctx, containers.RemoveContainersOptions{Containers: []string{dnsForwarderContainerName()}, Force: true}) + _ = os.RemoveAll(dnsRecordsHostDir()) + }() + + // Run a container with a label, a published port, and a volume mount + containerId, err := aco.RunContainer(ctx, containers.RunContainerOptions{ + CreateContainerOptions: containers.CreateContainerOptions{ + Name: containerName, + Networks: []containers.CreateContainerNetworkOptions{{Name: aco.DefaultNetworkName()}}, + ContainerSpec: apiv1.ContainerSpec{ + Image: liveTestImage, + Labels: []apiv1.ContainerLabel{ + {Key: "usvc-dev.dcp/live-test", Value: containerName}, + }, + Ports: []apiv1.ContainerPort{ + {ContainerPort: 80}, + }, + VolumeMounts: []apiv1.VolumeMount{ + {Type: apiv1.NamedVolumeMount, Source: volumeName, Target: "/data"}, + }, + Command: "sleep", + Args: []string{"120"}, + }, + }, + }) + require.NoError(t, err) + require.Equal(t, containerName, containerId) + defer func() { + _, _ = aco.RemoveContainers(ctx, containers.RemoveContainersOptions{Containers: []string{containerName}, Force: true}) + }() + + // The container should be listed, and label filtering should work + listed, err := aco.ListContainers(ctx, containers.ListContainersOptions{ + Filters: containers.ListContainersFilters{ + LabelFilters: []containers.LabelFilter{{Key: "usvc-dev.dcp/live-test", Value: containerName}}, + }, + }) + require.NoError(t, err) + require.Len(t, listed, 1) + require.Equal(t, containerName, listed[0].Id) + require.Equal(t, containers.ContainerStatusRunning, listed[0].Status) + + // Inspect: status, ports (auto-allocated host port), networks, mounts + inspected, err := aco.InspectContainers(ctx, containers.InspectContainersOptions{Containers: []string{containerName}}) + require.NoError(t, err) + require.Len(t, inspected, 1) + require.Equal(t, containers.ContainerStatusRunning, inspected[0].Status) + require.Len(t, inspected[0].Ports["80/tcp"], 1) + require.NotEqual(t, "0", inspected[0].Ports["80/tcp"][0].HostPort) + require.Len(t, inspected[0].Networks, 1) + require.Equal(t, defaultNetworkName, inspected[0].Networks[0].Name) + require.NotEmpty(t, inspected[0].Networks[0].IPAddress) + require.NotEmpty(t, inspected[0].Networks[0].Gateway) + + // ConnectNetwork reports the container as already attached to the default network + connectErr := aco.ConnectNetwork(ctx, containers.ConnectNetworkOptions{Network: defaultNetworkName, Container: containerName}) + require.ErrorIs(t, connectErr, containers.ErrAlreadyExists) + + // DisconnectNetwork is a no-op + require.NoError(t, aco.DisconnectNetwork(ctx, containers.DisconnectNetworkOptions{Network: defaultNetworkName, Container: containerName, Force: true})) + + // The default network inspection reports the attached container + networks, err := aco.InspectNetworks(ctx, containers.InspectNetworksOptions{Networks: []string{defaultNetworkName}}) + require.NoError(t, err) + require.Len(t, networks, 1) + require.NotEmpty(t, networks[0].Gateways) + require.Contains(t, networks[0].Containers, containers.InspectedNetworkContainer{Id: containerName, Name: containerName}) + + // CreateNetwork maps onto the default network + networkId, err := aco.CreateNetwork(ctx, containers.CreateNetworkOptions{Name: "dcp-live-test-net"}) + require.NoError(t, err) + require.Equal(t, defaultNetworkName, networkId) + + // CreateFiles copies files into the running container via exec+tar + require.NoError(t, aco.CreateFiles(ctx, containers.CreateFilesOptions{ + Container: containerName, + ModTime: time.Now(), + Destination: "/dcp-test-files", + Entries: []apiv1.FileSystemEntry{ + {Type: apiv1.FileSystemEntryTypeFile, Name: "hello.txt", Contents: "hello from DCP"}, + }, + })) + + // Exec: verify the created file content and the exit code + var execOut bytes.Buffer + exitCh, err := aco.ExecContainer(ctx, containers.ExecContainerOptions{ + Container: containerName, + Command: "cat", + Args: []string{"/dcp-test-files/hello.txt"}, + StreamCommandOptions: containers.StreamCommandOptions{ + StdOutStream: usvc_io.NopWriteCloser(&execOut), + }, + }) + require.NoError(t, err) + select { + case exitCode := <-exitCh: + require.EqualValues(t, 0, exitCode) + case <-ctx.Done(): + t.Fatal("timed out waiting for exec to complete") + } + require.Equal(t, "hello from DCP", execOut.String()) + + // Logs: the container has no output, but the log capture must start cleanly + require.NoError(t, aco.CaptureContainerLogs(ctx, containerName, &nopWriteSyncerCloser{}, &nopWriteSyncerCloser{}, containers.StreamContainerLogsOptions{})) + + // ApplyImageLayers: bake an in-memory tar layer into a derived image via the real builder. + // This is the mechanism the container controller uses to inject create-files and certificates + // on this runtime (see CreateFilesRequiresRunningContainer), so it must work end to end. + layerTar, tarErr := containers.BuildCreateFilesTar(containers.CreateFilesOptions{ + Destination: "/dcp-baked", + Entries: []apiv1.FileSystemEntry{ + {Type: apiv1.FileSystemEntryTypeFile, Name: "baked.txt", Contents: "baked by DCP"}, + }, + ModTime: time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC), + }, log) + require.NoError(t, tarErr) + require.NotNil(t, layerTar) + + layerDigest := sha256.Sum256(layerTar) + derivedTag := fmt.Sprintf("dcp-live-test-derived:%d", time.Now().UnixNano()) + derivedImage, applyErr := aco.ApplyImageLayers(ctx, containers.ApplyImageLayersOptions{ + BaseImage: images[0], + Layers: []apiv1.ImageLayer{{ + Digest: hex.EncodeToString(layerDigest[:]), + RawContents: base64.StdEncoding.EncodeToString(layerTar), + }}, + Tag: derivedTag, + }) + require.NoError(t, applyErr) + require.Equal(t, derivedTag, derivedImage) + runner := aco.(containers.CLICommandRunner) + defer func() { + rmCmd := makeAppleContainerCommand("image", "rm", derivedTag) + _, _, _ = runner.RunBufferedCommand(ctx, "RemoveImage", rmCmd, nil, nil, ordinaryCommandTimeout) + }() + + // The derived image must be inspectable (this is what the controller's cache probe does) + // and must contain the baked file. + derivedInspected, derivedInspectErr := aco.InspectImages(ctx, containers.InspectImagesOptions{Images: []string{derivedTag}}) + require.NoError(t, derivedInspectErr) + require.Len(t, derivedInspected, 1) + + bakedContainerName := containerName + "-baked" + var bakedOut bytes.Buffer + _, bakedRunErr := aco.RunContainer(ctx, containers.RunContainerOptions{ + CreateContainerOptions: containers.CreateContainerOptions{ + Name: bakedContainerName, + ContainerSpec: apiv1.ContainerSpec{ + Image: derivedTag, + Command: "cat", + Args: []string{"/dcp-baked/baked.txt"}, + }, + }, + }) + require.NoError(t, bakedRunErr) + defer func() { + _, _ = aco.RemoveContainers(ctx, containers.RemoveContainersOptions{Containers: []string{bakedContainerName}, Force: true}) + }() + + // The baked container prints the injected file and exits; read its logs to confirm the content. + require.Eventually(t, func() bool { + logsCmd := makeAppleContainerCommand("logs", bakedContainerName) + logsOut, _, logsErr := runner.RunBufferedCommand(ctx, "Logs", logsCmd, nil, nil, ordinaryCommandTimeout) + if logsErr != nil { + return false + } + bakedOut.Reset() + bakedOut.Write(logsOut.Bytes()) + return strings.Contains(bakedOut.String(), "baked by DCP") + }, 60*time.Second, 2*time.Second, "the derived image should contain the baked file (logs: %s)", bakedOut.String()) + + // DNS forwarder: every container is pointed at the DCP DNS forwarder, so network + // aliases and container names resolve between containers. Run a second container with + // an alias and resolve it from the first one. + aliasedContainerName := containerName + "-aliased" + _, aliasedRunErr := aco.RunContainer(ctx, containers.RunContainerOptions{ + CreateContainerOptions: containers.CreateContainerOptions{ + Name: aliasedContainerName, + Networks: []containers.CreateContainerNetworkOptions{ + {Name: aco.DefaultNetworkName(), Aliases: []string{"dcp-live-alias"}}, + }, + ContainerSpec: apiv1.ContainerSpec{ + Image: liveTestImage, + Command: "sleep", + Args: []string{"60"}, + }, + }, + }) + require.NoError(t, aliasedRunErr) + defer func() { + _, _ = aco.RemoveContainers(ctx, containers.RemoveContainersOptions{Containers: []string{aliasedContainerName}, Force: true}) + }() + + aliasedInspected, aliasedInspectErr := aco.InspectContainers(ctx, containers.InspectContainersOptions{Containers: []string{aliasedContainerName}}) + require.NoError(t, aliasedInspectErr) + require.Len(t, aliasedInspected, 1) + aliasedIP := aliasedInspected[0].Networks[0].IPAddress + require.NotEmpty(t, aliasedIP) + + // Helper: run a shell command in the first container and return its output. + execInFirstContainer := func(script string) string { + var out bytes.Buffer + execExitCh, execErr := aco.ExecContainer(ctx, containers.ExecContainerOptions{ + Container: containerName, + Command: "sh", + Args: []string{"-c", script}, + StreamCommandOptions: containers.StreamCommandOptions{ + StdOutStream: usvc_io.NopWriteCloser(&out), + }, + }) + require.NoError(t, execErr) + select { + case <-execExitCh: + case <-ctx.Done(): + t.Fatal("timed out waiting for exec to complete") + } + return out.String() + } + + // The first container (also pointed at the forwarder) must resolve the alias, the + // container name, and still resolve external names through the upstream relay. + // Record propagation is eventually consistent (the forwarder observes the records + // file through a shared filesystem), hence the retries. Note: the first container was + // created before the aliased one, proving records propagate to running containers. + var dnsOut string + require.Eventually(t, func() bool { + dnsOut = execInFirstContainer( + "nslookup dcp-live-alias 2>&1 | tail -2; nslookup " + aliasedContainerName + " > /dev/null 2>&1 && echo name-ok; nslookup example.com > /dev/null 2>&1 && echo upstream-ok") + return strings.Contains(dnsOut, aliasedIP) && strings.Contains(dnsOut, "name-ok") && strings.Contains(dnsOut, "upstream-ok") + }, 30*time.Second, 2*time.Second, + "the alias should resolve to the aliased container's IP, the container name should resolve, and external names should resolve via the upstream relay (last output: %s)", &dnsOut) + + // Removing the aliased container must retract its records. + _, aliasedRemoveErr := aco.RemoveContainers(ctx, containers.RemoveContainersOptions{Containers: []string{aliasedContainerName}, Force: true}) + require.NoError(t, aliasedRemoveErr) + + var dnsGoneOut string + require.Eventually(t, func() bool { + dnsGoneOut = execInFirstContainer("nslookup dcp-live-alias > /dev/null 2>&1 || echo alias-gone") + return strings.Contains(dnsGoneOut, "alias-gone") + }, 30*time.Second, 2*time.Second, + "records of removed containers should be retracted (last output: %s)", &dnsGoneOut) + + // Stop and remove + stopped, err := aco.StopContainers(ctx, containers.StopContainersOptions{Containers: []string{containerName}, SecondsToKill: 5}) + require.NoError(t, err) + require.Equal(t, []string{containerName}, stopped) + + inspected, err = aco.InspectContainers(ctx, containers.InspectContainersOptions{Containers: []string{containerName}}) + require.NoError(t, err) + require.Len(t, inspected, 1) + require.Equal(t, containers.ContainerStatusExited, inspected[0].Status) + + removed, err := aco.RemoveContainers(ctx, containers.RemoveContainersOptions{Containers: []string{containerName}}) + require.NoError(t, err) + require.Equal(t, []string{containerName}, removed) + + // Volume cleanup must succeed now that the container is gone + removedVolumes, err := aco.RemoveVolumes(ctx, containers.RemoveVolumesOptions{Volumes: []string{volumeName}}) + require.NoError(t, err) + require.Equal(t, []string{volumeName}, removedVolumes) +} + +type nopWriteSyncerCloser struct { + strings.Builder +} + +func (n *nopWriteSyncerCloser) Sync() error { return nil } +func (n *nopWriteSyncerCloser) Close() error { return nil } diff --git a/internal/applecontainer/cli_orchestrator_test.go b/internal/applecontainer/cli_orchestrator_test.go new file mode 100644 index 00000000..a8f04828 --- /dev/null +++ b/internal/applecontainer/cli_orchestrator_test.go @@ -0,0 +1,416 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package applecontainer + +import ( + "bytes" + "strings" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/require" + + apiv1 "github.com/microsoft/dcp/api/v1" + "github.com/microsoft/dcp/internal/containers" +) + +// Captured from `container inspect` (Apple container CLI 1.0.0) for a running container +// with a published port, a named volume mount, and a read-only bind mount. +const inspectedRunningContainer = `[{"configuration":{"capAdd":[],"capDrop":[],"creationDate":"2026-06-12T15:56:39Z","dns":{"nameservers":[],"options":[],"searchDomains":[]},"id":"dcp-probe-1","image":{"descriptor":{"digest":"sha256:865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62","mediaType":"application/vnd.oci.image.index.v1+json","size":9218},"reference":"docker.io/library/alpine:latest"},"initProcess":{"arguments":["600"],"environment":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"executable":"sleep","rlimits":[],"supplementalGroups":[],"terminal":false,"user":{"id":{"gid":0,"uid":0}},"workingDirectory":"/"},"labels":{"usvc-dev.dcp/probe":"1"},"mounts":[{"destination":"/vol","options":[],"source":"/Users/tester/Library/Application Support/com.apple.container/volumes/dcp-probe-vol/volume.img","type":{"volume":{"cache":{"on":{}},"format":"ext4","name":"dcp-probe-vol","sync":{"fsync":{}}}}},{"destination":"/bindro","options":["ro"],"source":"/tmp/dcp-cptree","type":{"virtiofs":{}}}],"networks":[{"network":"default","options":{"hostname":"dcp-probe-1","mtu":1280}}],"platform":{"architecture":"arm64","os":"linux"},"publishedPorts":[{"containerPort":80,"count":1,"hostAddress":"127.0.0.1","hostPort":18099,"proto":"tcp"}],"publishedSockets":[],"readOnly":false,"resources":{"cpuOverhead":1,"cpus":4,"memoryInBytes":1073741824},"rosetta":false,"runtimeHandler":"container-runtime-linux","ssh":false,"sysctls":{},"useInit":false,"virtualization":false},"id":"dcp-probe-1","status":{"networks":[{"hostname":"dcp-probe-1","ipv4Address":"192.168.65.11/24","ipv4Gateway":"192.168.65.1","ipv6Address":"fd88:ed83:617:6ebb:f4d1:1bff:fe87:2438/64","macAddress":"f6:d1:1b:87:24:38","mtu":1280,"network":"default"}],"startedDate":"2026-06-12T15:56:40Z","state":"running"}}]` + +// A container that was created but never started: state is "stopped" and there is no startedDate. +const inspectedCreatedContainer = `[{"configuration":{"creationDate":"2026-06-12T15:30:00Z","id":"created-ctr","image":{"descriptor":{"digest":"sha256:865b"},"reference":"docker.io/library/alpine:latest"},"initProcess":{"arguments":["5"],"environment":["PATH=/bin"],"executable":"sleep","terminal":false,"workingDirectory":"/"},"labels":{},"mounts":[],"networks":[{"network":"default","options":{"hostname":"created-ctr","mtu":1280}}],"publishedPorts":[]},"id":"created-ctr","status":{"networks":[],"state":"stopped"}}]` + +// A container that ran and exited: state is "stopped" and startedDate is set. +const inspectedExitedContainer = `[{"configuration":{"creationDate":"2026-06-12T15:29:00Z","id":"exited-ctr","image":{"reference":"docker.io/library/alpine:latest"},"initProcess":{"arguments":["-c","exit 7"],"environment":[],"executable":"sh"},"labels":{},"mounts":[],"networks":[],"publishedPorts":[]},"id":"exited-ctr","status":{"networks":[],"startedDate":"2026-06-12T15:29:03Z","state":"stopped"}}]` + +// Captured from `container network inspect default`. +const inspectedDefaultNetwork = `[{"configuration":{"creationDate":"2026-06-12T12:18:16Z","labels":{"com.apple.container.resource.role":"builtin"},"mode":"nat","name":"default","options":{},"plugin":"container-network-vmnet"},"id":"default","status":{"ipv4Gateway":"192.168.65.1","ipv4Subnet":"192.168.65.0/24","ipv6Subnet":"fd88:ed83:617:6ebb::/64"}}]` + +// Captured from `container volume inspect`. +const inspectedVolume = `[{"configuration":{"creationDate":"2026-06-12T12:28:56Z","driver":"local","format":"ext4","labels":{"some-label":"some-value"},"name":"my-volume","options":{},"sizeInBytes":549755813888,"source":"/Users/tester/Library/Application Support/com.apple.container/volumes/my-volume/volume.img"},"id":"my-volume"}]` + +// Captured (abbreviated) from `container image inspect`. +const inspectedImage = `[{"configuration":{"creationDate":"2025-12-18T00:11:34Z","descriptor":{"digest":"sha256:865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62","mediaType":"application/vnd.oci.image.index.v1+json","size":9218},"name":"docker.io/library/alpine:latest"},"id":"865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62","variants":[{"config":{"architecture":"arm64","config":{"Cmd":["/bin/sh"],"Labels":{"image-label":"image-value"}},"os":"linux"}}]}]` + +func TestInspectedContainerDeserialization(t *testing.T) { + t.Parallel() + + var b bytes.Buffer + _, err := b.WriteString(inspectedRunningContainer) + require.NoError(t, err) + + inspectedContainers, err := asArrayOfObjects(&b, unmarshalContainer) + require.NoError(t, err) + require.Len(t, inspectedContainers, 1) + + ct := inspectedContainers[0] + + require.Equal(t, "dcp-probe-1", ct.Id) + require.Equal(t, "dcp-probe-1", ct.Name) + require.Equal(t, "docker.io/library/alpine:latest", ct.Image) + + expectedCreatedTime, err := time.Parse(time.RFC3339, "2026-06-12T15:56:39Z") + require.NoError(t, err) + require.Equal(t, expectedCreatedTime, ct.CreatedAt) + expectedStartedTime, err := time.Parse(time.RFC3339, "2026-06-12T15:56:40Z") + require.NoError(t, err) + require.Equal(t, expectedStartedTime, ct.StartedAt) + require.Equal(t, containers.ContainerStatusRunning, ct.Status) + require.EqualValues(t, 0, ct.ExitCode) + + require.Equal(t, containers.InspectedContainerPortMapping{ + "80/tcp": []containers.InspectedContainerHostPortConfig{{HostIp: "127.0.0.1", HostPort: "18099"}}, + }, ct.Ports) + + require.Equal(t, map[string]string{ + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + }, ct.Env) + + require.Equal(t, []string{"sleep", "600"}, ct.Args) + + require.Equal(t, []apiv1.VolumeMount{ + { + Type: apiv1.NamedVolumeMount, + Source: "dcp-probe-vol", + Target: "/vol", + }, + { + Type: apiv1.BindMount, + Source: "/tmp/dcp-cptree", + Target: "/bindro", + ReadOnly: true, + }, + }, ct.Mounts) + + require.Equal(t, []containers.InspectedContainerNetwork{ + { + Id: "default", + Name: "default", + IPAddress: "192.168.65.11", + Gateway: "192.168.65.1", + MacAddress: "f6:d1:1b:87:24:38", + Aliases: []string{"dcp-probe-1"}, + }, + }, ct.Networks) + + require.Equal(t, map[string]string{"usvc-dev.dcp/probe": "1"}, ct.Labels) +} + +func TestInspectedContainerStateMapping(t *testing.T) { + t.Parallel() + + // A created (never started) container reports as "created" + var b bytes.Buffer + _, err := b.WriteString(inspectedCreatedContainer) + require.NoError(t, err) + + inspectedContainers, err := asArrayOfObjects(&b, unmarshalContainer) + require.NoError(t, err) + require.Len(t, inspectedContainers, 1) + require.Equal(t, containers.ContainerStatusCreated, inspectedContainers[0].Status) + + // The created container is not running, so network information comes from the configuration + require.Equal(t, []containers.InspectedContainerNetwork{ + { + Id: "default", + Name: "default", + Aliases: []string{"created-ctr"}, + }, + }, inspectedContainers[0].Networks) + + // A container that ran and stopped reports as "exited" + b.Reset() + _, err = b.WriteString(inspectedExitedContainer) + require.NoError(t, err) + + inspectedContainers, err = asArrayOfObjects(&b, unmarshalContainer) + require.NoError(t, err) + require.Len(t, inspectedContainers, 1) + require.Equal(t, containers.ContainerStatusExited, inspectedContainers[0].Status) +} + +func TestListedContainerDeserializationAndFiltering(t *testing.T) { + t.Parallel() + + var b bytes.Buffer + _, err := b.WriteString(inspectedRunningContainer) + require.NoError(t, err) + + listed, err := asArrayOfObjects(&b, unmarshalListedContainer) + require.NoError(t, err) + require.Len(t, listed, 1) + + lc := listed[0] + require.Equal(t, "dcp-probe-1", lc.Id) + require.Equal(t, "dcp-probe-1", lc.Name) + require.Equal(t, "docker.io/library/alpine:latest", lc.Image) + require.Equal(t, containers.ContainerStatusRunning, lc.Status) + require.Equal(t, map[string]string{"usvc-dev.dcp/probe": "1"}, lc.Labels) + require.Equal(t, []string{"default"}, lc.Networks) + + // Label filtering happens client-side + getLabels := func(lc containers.ListedContainer) map[string]string { return lc.Labels } + + filtered := filterByLabels(listed, []containers.LabelFilter{{Key: "usvc-dev.dcp/probe", Value: "1"}}, getLabels) + require.Len(t, filtered, 1) + + filtered = filterByLabels(listed, []containers.LabelFilter{{Key: "usvc-dev.dcp/probe"}}, getLabels) + require.Len(t, filtered, 1) + + filtered = filterByLabels(listed, []containers.LabelFilter{{Key: "usvc-dev.dcp/probe", Value: "2"}}, getLabels) + require.Empty(t, filtered) + + filtered = filterByLabels(listed, []containers.LabelFilter{{Key: "no-such-label"}}, getLabels) + require.Empty(t, filtered) +} + +func TestInspectedNetworkDeserialization(t *testing.T) { + t.Parallel() + + var b bytes.Buffer + _, err := b.WriteString(inspectedDefaultNetwork) + require.NoError(t, err) + + networks, err := asArrayOfObjects(&b, unmarshalNetwork) + require.NoError(t, err) + require.Len(t, networks, 1) + + net := networks[0] + require.Equal(t, "default", net.Id) + require.Equal(t, "default", net.Name) + require.Equal(t, "container-network-vmnet", net.Driver) + require.Equal(t, map[string]string{"com.apple.container.resource.role": "builtin"}, net.Labels) + require.True(t, net.IPv6) + require.Equal(t, []string{"192.168.65.0/24", "fd88:ed83:617:6ebb::/64"}, net.Subnets) + require.Equal(t, []string{"192.168.65.1"}, net.Gateways) +} + +func TestListedNetworkDeserialization(t *testing.T) { + t.Parallel() + + var b bytes.Buffer + _, err := b.WriteString(inspectedDefaultNetwork) + require.NoError(t, err) + + networks, err := asArrayOfObjects(&b, unmarshalListedNetwork) + require.NoError(t, err) + require.Len(t, networks, 1) + + net := networks[0] + require.Equal(t, "default", net.ID) + require.Equal(t, "default", net.Name) + require.Equal(t, "container-network-vmnet", net.Driver) + require.True(t, net.IPv6) + require.Equal(t, map[string]string{"com.apple.container.resource.role": "builtin"}, net.Labels) +} + +func TestInspectedVolumeDeserialization(t *testing.T) { + t.Parallel() + + var b bytes.Buffer + _, err := b.WriteString(inspectedVolume) + require.NoError(t, err) + + volumes, err := asArrayOfObjects(&b, unmarshalVolume) + require.NoError(t, err) + require.Len(t, volumes, 1) + + vol := volumes[0] + require.Equal(t, "my-volume", vol.Name) + require.Equal(t, "local", vol.Driver) + require.Equal(t, "/Users/tester/Library/Application Support/com.apple.container/volumes/my-volume/volume.img", vol.MountPoint) + require.Equal(t, map[string]string{"some-label": "some-value"}, vol.Labels) + + expectedCreatedTime, err := time.Parse(time.RFC3339, "2026-06-12T12:28:56Z") + require.NoError(t, err) + require.Equal(t, expectedCreatedTime, vol.CreatedAt) +} + +func TestInspectedImageDeserialization(t *testing.T) { + t.Parallel() + + var b bytes.Buffer + _, err := b.WriteString(inspectedImage) + require.NoError(t, err) + + images, err := asArrayOfObjects(&b, unmarshalImage) + require.NoError(t, err) + require.Len(t, images, 1) + + img := images[0] + require.Equal(t, "865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62", img.Id) + require.Equal(t, []string{"docker.io/library/alpine:latest"}, img.Tags) + require.Equal(t, "sha256:865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62", img.Digest) + require.Equal(t, map[string]string{"image-label": "image-value"}, img.Labels) +} + +func TestAsArrayOfObjectsRejectsNonArrayOutput(t *testing.T) { + t.Parallel() + + // Docker/Podman produce JSON-lines output, which is not what the Apple container CLI produces. + // Such output must be rejected so that runtime detection can recognize a mismatched CLI. + var b bytes.Buffer + _, err := b.WriteString("{\"Id\": \"abc\"}\n{\"Id\": \"def\"}\n") + require.NoError(t, err) + + _, err = asArrayOfObjects(&b, unmarshalListedContainer) + require.Error(t, err) + require.ErrorIs(t, err, containers.ErrUnmarshalling) +} + +func TestApplyCreateContainerOptions(t *testing.T) { + t.Parallel() + + aco := &AppleContainerCliOrchestrator{log: logr.Discard()} + + options := containers.CreateContainerOptions{ + Name: "my-container", + // Multiple requested networks resolve to the same runtime network on this runtime + // (see UsesSingleNetwork) and must be deduplicated; aliases are not supported. + Networks: []containers.CreateContainerNetworkOptions{ + {Name: "default"}, + {Name: "default", Aliases: []string{"alias1"}}, + }, + ContainerSpec: apiv1.ContainerSpec{ + VolumeMounts: []apiv1.VolumeMount{ + {Type: apiv1.NamedVolumeMount, Source: "myvolume", Target: "/data"}, + {Type: apiv1.BindMount, Source: "/host/path", Target: "/container/path", ReadOnly: true}, + }, + Ports: []apiv1.ContainerPort{ + {ContainerPort: 8080, HostPort: 18080}, + {ContainerPort: 9090, HostPort: 19090, HostIP: "0.0.0.0", Protocol: apiv1.UDP}, + }, + Env: []apiv1.EnvVar{ + {Name: "FOO", Value: "bar"}, + }, + EnvFiles: []string{"/some/env/file.env"}, + Labels: []apiv1.ContainerLabel{ + {Key: "label1", Value: "value1"}, + }, + Command: "/custom/entrypoint", + RunArgs: []string{"--rosetta"}, + }, + } + + args := aco.applyCreateContainerOptions([]string{"create"}, options) + + require.Equal(t, []string{ + "create", + "--name", "my-container", + "--network", "default", + "--mount", "type=volume,source=myvolume,target=/data", + "--mount", "type=bind,source=/host/path,target=/container/path,readonly", + "-p", "127.0.0.1:18080:8080", + "-p", "0.0.0.0:19090:9090/udp", + "-e", "FOO=bar", + "--env-file", "/some/env/file.env", + "--label", "label1=value1", + "--entrypoint", "/custom/entrypoint", + "--rosetta", + }, args) +} + +func TestApplyCreateContainerOptionsOmitsUnsupportedFlags(t *testing.T) { + t.Parallel() + + aco := &AppleContainerCliOrchestrator{log: logr.Discard()} + + terminal := apiv1.TerminalSpec{} + options := containers.CreateContainerOptions{ + Name: "my-container", + Networks: []containers.CreateContainerNetworkOptions{{Name: "default", Aliases: []string{"alias1"}}}, + Healthcheck: containers.ContainerHealthcheck{ + Command: []string{"CMD", "echo", "ok"}, + }, + ContainerSpec: apiv1.ContainerSpec{ + RestartPolicy: apiv1.RestartPolicyAlways, + PullPolicy: apiv1.PullPolicyAlways, + Terminal: &terminal, + }, + } + + args := aco.applyCreateContainerOptions([]string{"create"}, options) + + // Restart policies, pull policies, health checks, and network aliases are not supported by + // the Apple container runtime and must not be emitted. A requested terminal maps to -it. + require.Equal(t, []string{ + "create", + "--name", "my-container", + "--network", "default", + "-it", + }, args) +} + +func TestMapContainerState(t *testing.T) { + t.Parallel() + + started := time.Date(2026, 6, 12, 15, 0, 0, 0, time.UTC) + + require.Equal(t, containers.ContainerStatusRunning, mapContainerState("running", started)) + require.Equal(t, containers.ContainerStatusExited, mapContainerState("stopped", started)) + require.Equal(t, containers.ContainerStatusCreated, mapContainerState("stopped", time.Time{})) + require.Equal(t, containers.ContainerStatusRemoving, mapContainerState("stopping", started)) + require.Equal(t, containers.ContainerStatus("hibernating"), mapContainerState("hibernating", started)) +} + +func TestUnmarshalDiagnostics(t *testing.T) { + t.Parallel() + + statusJson := `{"apiServerAppName":"container-apiserver","apiServerBuild":"release","apiServerCommit":"unspecified","apiServerVersion":"container-apiserver version 1.0.0 (build: release, commit: unspeci)","appRoot":"/Users/tester/Library/Application Support/com.apple.container/","installRoot":"/opt/homebrew/Cellar/container/1.0.0_1/","status":"running"}` + + var diagnostics containers.ContainerDiagnostics + err := unmarshalDiagnostics([]byte(statusJson), &diagnostics) + require.NoError(t, err) + require.Equal(t, "1.0.0", diagnostics.ServerVersion) +} + +func TestExtractVersion(t *testing.T) { + t.Parallel() + + require.Equal(t, "1.0.0", extractVersion("container CLI version 1.0.0 (build: release, commit: unspeci)")) + require.Equal(t, "2.1.3-beta", extractVersion("container-apiserver version 2.1.3-beta")) + require.Equal(t, "unparseable", extractVersion(" unparseable ")) +} + +type recordingWriter struct { + bytes.Buffer +} + +func (w *recordingWriter) Sync() error { return nil } +func (w *recordingWriter) Close() error { return nil } + +func TestTimestampingWriter(t *testing.T) { + t.Parallel() + + var inner recordingWriter + w := newTimestampingWriter(&inner) + + _, err := w.Write([]byte("first line\nsecond ")) + require.NoError(t, err) + _, err = w.Write([]byte("line\n")) + require.NoError(t, err) + + lines := strings.Split(strings.TrimSuffix(inner.String(), "\n"), "\n") + require.Len(t, lines, 2) + + for i, expectedSuffix := range []string{"first line", "second line"} { + timestamp, content, found := strings.Cut(lines[i], " ") + require.True(t, found) + _, parseErr := time.Parse(time.RFC3339Nano, timestamp) + require.NoError(t, parseErr, "line %d should start with a RFC3339Nano timestamp, got: %s", i, lines[i]) + require.Equal(t, expectedSuffix, content) + } +} + +func TestStripCidrSuffix(t *testing.T) { + t.Parallel() + + require.Equal(t, "192.168.65.11", stripCidrSuffix("192.168.65.11/24")) + require.Equal(t, "192.168.65.11", stripCidrSuffix("192.168.65.11")) + require.Equal(t, "", stripCidrSuffix("")) +} diff --git a/internal/applecontainer/dns_forwarder.go b/internal/applecontainer/dns_forwarder.go new file mode 100644 index 00000000..4a8cf7a8 --- /dev/null +++ b/internal/applecontainer/dns_forwarder.go @@ -0,0 +1,465 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package applecontainer + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/microsoft/dcp/internal/containers" + "github.com/microsoft/dcp/internal/dcppaths" + "github.com/microsoft/dcp/internal/version" + usvc_io "github.com/microsoft/dcp/pkg/io" + "github.com/microsoft/dcp/pkg/osutil" + "github.com/microsoft/dcp/pkg/process" +) + +// DNS forwarder for network aliases on the Apple container runtime +// +// The Apple container runtime has no network alias support (`--network-alias` does not +// exist) and its gateway DNS serves no records for container names, so names like +// Aspire resource aliases would not resolve between containers. To provide alias +// resolution, DCP runs a small DNS forwarder (cmd/dcpdns) in a container on the default +// network and points every container it creates at it via `--dns`: +// +// - The forwarder image is built once per DCP version from the dcpdns_c binary that +// ships alongside dcp (same pattern as the dcptun client proxy image). +// - One forwarder container runs per DCP instance. It is labeled with the standard +// creator-process labels, so the resource harvester removes it once the owning DCP +// process exits. +// - Records (container names and network aliases -> container IP) are published by +// atomically rewriting a hosts-format file in a host directory that is bind-mounted +// read-only into the forwarder; the forwarder re-reads it on change. Queries for +// unknown names are relayed to the network gateway, so regular resolution still works. +// +// If the forwarder cannot be provisioned (e.g. the image build fails), containers are +// created without `--dns` and keep gateway-only resolution: alias lookups will fail, but +// nothing else regresses. + +const ( + // Name of the dcpdns binary shipped alongside the dcp executable. + dnsBinaryName = "dcpdns_c" + + // Prefix for the DNS forwarder image name (tagged with the DCP version). + dnsImageNamePrefix = "dcpdns_developer_ms" + + // Path of the dcpdns binary inside the forwarder image. + dnsBinaryContainerPath = "/usr/local/bin/dcpdns" + + // Mount point of the records directory inside the forwarder container. + dnsRecordsContainerDir = "/dcp-dns" + + // Name of the hosts-format records file within the records directory. + dnsRecordsFileName = "hosts" + + // Label marking the forwarder container itself. + dnsForwarderMarkerLabel = "com.microsoft.developer.usvc-dev.apple-dns-forwarder" + + // Labels consumed by the DCP resource harvester to clean up containers whose creator + // process has exited. The values must match the constants used by the controllers + // package (controllers.CreatorProcessIdLabel and friends), which cannot be imported + // from here without inverting the layering between orchestrators and controllers. + creatorProcessIdLabel = "com.microsoft.developer.usvc-dev.creatorProcessId" + creatorProcessStartTimeLabel = "com.microsoft.developer.usvc-dev.creatorProcessStartTime" + persistentLabel = "com.microsoft.developer.usvc-dev.persistent" + + // How long to wait before re-attempting forwarder provisioning after a failure. + dnsForwarderRetryInterval = time.Minute +) + +// dnsForwarderState tracks the per-orchestrator DNS forwarder and its published records. +type dnsForwarderState struct { + mu sync.Mutex + + // IP address of the running forwarder; empty until successfully provisioned. + ip string + + // When the last (failed) provisioning attempt finished; zero if none failed yet. + lastFailedAttempt time.Time + + // Host path of the records file mounted into the forwarder. + recordsPath string + + // Names (container name + network aliases) to publish for each container once it + // starts, recorded at creation time. + pendingNames map[string][]string + + // Currently published records: container name -> (IP, names). + published map[string]dnsRecord +} + +type dnsRecord struct { + ip string + names []string +} + +// dnsForwarderContainerName returns the name of this DCP instance's forwarder container. +func dnsForwarderContainerName() string { + return fmt.Sprintf("dcp-dns-%d", os.Getpid()) +} + +// ensureDnsForwarder makes sure the DNS forwarder container is running and returns its IP +// address. It returns an empty string when the forwarder is unavailable, in which case +// containers are created without custom DNS (alias resolution is degraded, not fatal). +func (aco *AppleContainerCliOrchestrator) ensureDnsForwarder(ctx context.Context) string { + aco.dns.mu.Lock() + defer aco.dns.mu.Unlock() + + if aco.dns.ip != "" { + return aco.dns.ip + } + if !aco.dns.lastFailedAttempt.IsZero() && time.Since(aco.dns.lastFailedAttempt) < dnsForwarderRetryInterval { + return "" + } + + ip, provisionErr := aco.provisionDnsForwarderLocked(ctx) + if provisionErr != nil { + aco.log.Error(provisionErr, "Could not provision the DNS forwarder container; containers will resolve names via the network gateway only (network aliases will not resolve)") + aco.dns.lastFailedAttempt = time.Now() + return "" + } + + aco.log.V(1).Info("DNS forwarder is running", "Container", dnsForwarderContainerName(), "IP", ip) + aco.dns.ip = ip + return ip +} + +func (aco *AppleContainerCliOrchestrator) provisionDnsForwarderLocked(ctx context.Context) (string, error) { + // The upstream resolver for non-alias names is the default network gateway + // (the resolver containers would use if DCP did not override their DNS). + defaultNetwork, networkErr := aco.inspectSingleNetwork(ctx, defaultNetworkName) + if networkErr != nil { + return "", fmt.Errorf("inspecting the default network: %w", networkErr) + } + if len(defaultNetwork.Gateways) == 0 || defaultNetwork.Gateways[0] == "" { + return "", fmt.Errorf("the default network has no gateway to use as the upstream resolver") + } + gateway := defaultNetwork.Gateways[0] + + // Reuse a running forwarder from this DCP instance if one exists (e.g. after the + // orchestrator is recreated within the same process). + forwarderName := dnsForwarderContainerName() + if existing, inspectErr := aco.InspectContainers(ctx, containers.InspectContainersOptions{Containers: []string{forwarderName}}); inspectErr == nil && len(existing) == 1 { + if existing[0].Status == containers.ContainerStatusRunning && len(existing[0].Networks) > 0 && existing[0].Networks[0].IPAddress != "" { + if aco.dns.recordsPath == "" { + aco.dns.recordsPath = filepath.Join(dnsRecordsHostDir(), dnsRecordsFileName) + } + return existing[0].Networks[0].IPAddress, nil + } + + // Not running (or not usable): remove and recreate below. + if _, removeErr := aco.RemoveContainers(ctx, containers.RemoveContainersOptions{Containers: []string{forwarderName}, Force: true}); removeErr != nil { + return "", fmt.Errorf("removing stale DNS forwarder container: %w", removeErr) + } + } + + imageName, imageErr := aco.ensureDnsImage(ctx) + if imageErr != nil { + return "", imageErr + } + + // Set up the records directory and an initially empty records file. + recordsDir := dnsRecordsHostDir() + if mkdirErr := os.MkdirAll(recordsDir, osutil.PermissionOnlyOwnerReadWriteTraverse); mkdirErr != nil { + return "", fmt.Errorf("creating DNS records directory: %w", mkdirErr) + } + recordsPath := filepath.Join(recordsDir, dnsRecordsFileName) + if writeErr := writeFileAtomically(recordsPath, aco.renderRecordsLocked()); writeErr != nil { + return "", fmt.Errorf("writing initial DNS records file: %w", writeErr) + } + aco.dns.recordsPath = recordsPath + + // Start the forwarder, labeled so the resource harvester removes it when this DCP + // process exits. + thisProcess, processErr := process.This() + if processErr != nil { + return "", fmt.Errorf("getting current process information: %w", processErr) + } + + args := []string{ + "run", "--detach", + "--name", forwarderName, + "--network", defaultNetworkName, + "--mount", fmt.Sprintf("type=bind,source=%s,target=%s,readonly", recordsDir, dnsRecordsContainerDir), + "--label", fmt.Sprintf("%s=true", dnsForwarderMarkerLabel), + "--label", fmt.Sprintf("%s=%d", creatorProcessIdLabel, thisProcess.Pid), + "--label", fmt.Sprintf("%s=%s", creatorProcessStartTimeLabel, thisProcess.IdentityTime.Format(osutil.RFC3339MiliTimestampFormat)), + "--label", fmt.Sprintf("%s=false", persistentLabel), + imageName, + dnsBinaryContainerPath, + "--hosts", dnsRecordsContainerDir + "/" + dnsRecordsFileName, + "--upstream", gateway, + } + + cmd := makeAppleContainerCommand(args...) + outBuf, errBuf, runErr := aco.runBufferedCommand(ctx, "RunDnsForwarder", cmd, nil, nil, defaultRunContainerTimeout) + if runErr != nil { + return "", fmt.Errorf("starting the DNS forwarder container: %w", errorsJoinCli(runErr, errBuf)) + } + if _, idErr := asId(outBuf); idErr != nil { + return "", fmt.Errorf("starting the DNS forwarder container: %w", idErr) + } + + // Read back the forwarder's IP address. + inspected, inspectErr := aco.InspectContainers(ctx, containers.InspectContainersOptions{Containers: []string{forwarderName}}) + if inspectErr != nil || len(inspected) == 0 || len(inspected[0].Networks) == 0 || inspected[0].Networks[0].IPAddress == "" { + return "", fmt.Errorf("could not determine the DNS forwarder IP address: %w", inspectErr) + } + + return inspected[0].Networks[0].IPAddress, nil +} + +// ensureDnsImage makes sure the forwarder image exists, building it from the dcpdns_c +// binary that ships alongside the dcp executable if necessary. The image tag encodes the +// DCP version (and, for development builds, the binary hash), so a stale image is never +// reused after an upgrade. +func (aco *AppleContainerCliOrchestrator) ensureDnsImage(ctx context.Context) (string, error) { + binaryPath, binaryErr := dnsBinaryPath() + if binaryErr != nil { + return "", binaryErr + } + + tag := version.Version().Version + if tag == version.DevelopmentVersion { + hash, hashErr := computeFileHash(binaryPath) + if hashErr != nil { + return "", fmt.Errorf("computing dcpdns binary hash: %w", hashErr) + } + tag += "_" + hash[:12] + } + imageName := fmt.Sprintf("%s:%s", dnsImageNamePrefix, tag) + + if existing, inspectErr := aco.InspectImages(ctx, containers.InspectImagesOptions{Images: []string{imageName}}); inspectErr == nil && len(existing) > 0 { + return imageName, nil + } + + buildContext, buildContextErr := os.MkdirTemp(usvc_io.DcpTempDir(), "dcpdns-build-*") + if buildContextErr != nil { + return "", fmt.Errorf("creating DNS forwarder image build context: %w", buildContextErr) + } + defer func() { _ = os.RemoveAll(buildContext) }() + + // Docker-style builds do not follow symlinks outside the build context, so copy the binary in. + if copyErr := copyFile(binaryPath, filepath.Join(buildContext, dnsBinaryName), osutil.PermissionOnlyOwnerReadWriteExecute); copyErr != nil { + return "", fmt.Errorf("copying dcpdns binary to build context: %w", copyErr) + } + + // The dcpdns binary is a static Go executable, so the image needs nothing else. + dockerfile := fmt.Sprintf("FROM scratch\nCOPY --chmod=0755 %s %s\nCMD [\"%s\"]\n", dnsBinaryName, dnsBinaryContainerPath, dnsBinaryContainerPath) + dockerfilePath := filepath.Join(buildContext, "Dockerfile") + if writeErr := os.WriteFile(dockerfilePath, []byte(dockerfile), 0644); writeErr != nil { + return "", fmt.Errorf("writing DNS forwarder Dockerfile: %w", writeErr) + } + + buildCmd := makeAppleContainerCommand("build", "--progress", "plain", "-f", dockerfilePath, "-t", imageName, buildContext) + _, errBuf, buildErr := aco.runBufferedCommand(ctx, "BuildDnsForwarderImage", buildCmd, nil, nil, defaultBuildImageTimeout) + if buildErr != nil { + return "", fmt.Errorf("building the DNS forwarder image: %w", errorsJoinCli(buildErr, errBuf)) + } + + return imageName, nil +} + +// registerPendingDnsNames records the DNS names (container name + network aliases) that +// should be published for a container once it is running. Called at creation time, when +// the aliases are known. +func (aco *AppleContainerCliOrchestrator) registerPendingDnsNames(containerName string, networks []containers.CreateContainerNetworkOptions) { + if containerName == "" { + return + } + + names := []string{containerName} + seen := map[string]bool{containerName: true} + for _, network := range networks { + for _, alias := range network.Aliases { + if alias != "" && !seen[alias] { + seen[alias] = true + names = append(names, alias) + } + } + } + + aco.dns.mu.Lock() + defer aco.dns.mu.Unlock() + if aco.dns.pendingNames == nil { + aco.dns.pendingNames = map[string][]string{} + } + aco.dns.pendingNames[containerName] = names +} + +// publishDnsRecords resolves the container's IP address and publishes its DNS names to +// the forwarder's records file. Called after a container has been started. Failures are +// logged, not returned: name resolution is best-effort and must not fail container starts. +func (aco *AppleContainerCliOrchestrator) publishDnsRecords(ctx context.Context, containerName string) { + aco.dns.mu.Lock() + forwarderRunning := aco.dns.ip != "" + names := aco.dns.pendingNames[containerName] + aco.dns.mu.Unlock() + + if !forwarderRunning || containerName == dnsForwarderContainerName() { + return + } + if len(names) == 0 { + names = []string{containerName} + } + + inspected, inspectErr := aco.InspectContainers(ctx, containers.InspectContainersOptions{Containers: []string{containerName}}) + if inspectErr != nil || len(inspected) == 0 || len(inspected[0].Networks) == 0 || inspected[0].Networks[0].IPAddress == "" { + aco.log.V(1).Info("Could not determine container IP address for DNS records", "Container", containerName) + return + } + + aco.dns.mu.Lock() + defer aco.dns.mu.Unlock() + if aco.dns.published == nil { + aco.dns.published = map[string]dnsRecord{} + } + aco.dns.published[containerName] = dnsRecord{ip: inspected[0].Networks[0].IPAddress, names: names} + aco.writeRecordsLocked() +} + +// unpublishDnsRecords removes the DNS records for the given containers (called when +// containers are stopped or removed; their IP addresses are no longer valid). +func (aco *AppleContainerCliOrchestrator) unpublishDnsRecords(containerNames []string) { + aco.dns.mu.Lock() + defer aco.dns.mu.Unlock() + + changed := false + for _, name := range containerNames { + if _, wasPublished := aco.dns.published[name]; wasPublished { + delete(aco.dns.published, name) + changed = true + } + delete(aco.dns.pendingNames, name) + } + + if changed { + aco.writeRecordsLocked() + } +} + +func (aco *AppleContainerCliOrchestrator) writeRecordsLocked() { + if aco.dns.recordsPath == "" { + return + } + if writeErr := writeFileAtomically(aco.dns.recordsPath, aco.renderRecordsLocked()); writeErr != nil { + aco.log.Error(writeErr, "Could not update the DNS records file", "Path", aco.dns.recordsPath) + } +} + +func (aco *AppleContainerCliOrchestrator) renderRecordsLocked() []byte { + var builder strings.Builder + builder.WriteString("# DNS records published by DCP for the Apple container runtime. Do not edit.\n") + + containerNames := make([]string, 0, len(aco.dns.published)) + for name := range aco.dns.published { + containerNames = append(containerNames, name) + } + sort.Strings(containerNames) + + for _, containerName := range containerNames { + record := aco.dns.published[containerName] + builder.WriteString(record.ip) + for _, name := range record.names { + builder.WriteString(" ") + builder.WriteString(name) + } + builder.WriteString("\n") + } + + return []byte(builder.String()) +} + +func dnsRecordsHostDir() string { + return filepath.Join(usvc_io.DcpTempDir(), fmt.Sprintf("apple-dns-%d", os.Getpid())) +} + +// dnsBinaryPath locates the dcpdns_c binary: next to the running dcp executable in +// installed builds, or under the build output directory in development/test trees. +func dnsBinaryPath() (string, error) { + dcpDir, dcpDirErr := dcppaths.GetDcpDir() + if dcpDirErr == nil { + binaryPath := filepath.Join(dcpDir, dnsBinaryName) + if info, statErr := os.Stat(binaryPath); statErr == nil && info.Mode().IsRegular() { + return binaryPath, nil + } + } + + // Fallback: probe for the binary in the repository build output (used primarily for testing) + rootFolder, rootFindErr := osutil.FindRootFor(osutil.FileTarget, dcppaths.BuildOutputDir, dnsBinaryName) + if rootFindErr != nil { + return "", fmt.Errorf("dcpdns binary not found next to the running binary and could not be located via filesystem probing: %w", rootFindErr) + } + + binaryPath := filepath.Join(rootFolder, dcppaths.BuildOutputDir, dnsBinaryName) + info, statErr := os.Stat(binaryPath) + if statErr != nil || !info.Mode().IsRegular() { + return "", fmt.Errorf("dcpdns binary not found at %s", binaryPath) + } + + return binaryPath, nil +} + +func computeFileHash(path string) (string, error) { + file, openErr := os.Open(path) + if openErr != nil { + return "", openErr + } + defer file.Close() + + hasher := sha256.New() + if _, copyErr := io.Copy(hasher, file); copyErr != nil { + return "", copyErr + } + + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func copyFile(sourcePath string, destinationPath string, permissions os.FileMode) error { + source, openErr := os.Open(sourcePath) + if openErr != nil { + return openErr + } + defer source.Close() + + destination, createErr := os.OpenFile(destinationPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, permissions) + if createErr != nil { + return createErr + } + + if _, copyErr := io.Copy(destination, source); copyErr != nil { + _ = destination.Close() + return copyErr + } + + return destination.Close() +} + +// errorsJoinCli combines a command error with the normalized CLI error output. +func errorsJoinCli(err error, errBuf *bytes.Buffer) error { + return errors.Join(err, normalizeCliErrors(errBuf)) +} + +// writeFileAtomically writes content to a temporary file and renames it over the target, +// so the forwarder never observes a partially written records file. +func writeFileAtomically(path string, content []byte) error { + tempPath := path + ".tmp" + if writeErr := os.WriteFile(tempPath, content, 0644); writeErr != nil { + return writeErr + } + return os.Rename(tempPath, path) +} diff --git a/internal/applecontainer/dns_forwarder_test.go b/internal/applecontainer/dns_forwarder_test.go new file mode 100644 index 00000000..8ff9d465 --- /dev/null +++ b/internal/applecontainer/dns_forwarder_test.go @@ -0,0 +1,44 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package applecontainer + +import ( + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/require" + + "github.com/microsoft/dcp/internal/containers" +) + +func TestDnsRecordsRendering(t *testing.T) { + t.Parallel() + + aco := &AppleContainerCliOrchestrator{log: logr.Discard()} + + // Aliases are recorded at creation time, deduplicated, with the container name first. + aco.registerPendingDnsNames("myredis-abc123", []containers.CreateContainerNetworkOptions{ + {Name: "default", Aliases: []string{"myredis", "cache"}}, + {Name: "default", Aliases: []string{"myredis"}}, + }) + require.Equal(t, []string{"myredis-abc123", "myredis", "cache"}, aco.dns.pendingNames["myredis-abc123"]) + + aco.dns.published = map[string]dnsRecord{ + "myredis-abc123": {ip: "192.168.64.7", names: aco.dns.pendingNames["myredis-abc123"]}, + "api-def456": {ip: "192.168.64.9", names: []string{"api-def456", "api"}}, + } + + rendered := string(aco.renderRecordsLocked()) + require.Contains(t, rendered, "192.168.64.7 myredis-abc123 myredis cache\n") + require.Contains(t, rendered, "192.168.64.9 api-def456 api\n") + + // Retracting a container's records removes only its entries. + aco.unpublishDnsRecords([]string{"myredis-abc123", "never-published"}) + rendered = string(aco.renderRecordsLocked()) + require.NotContains(t, rendered, "myredis") + require.Contains(t, rendered, "192.168.64.9 api-def456 api\n") + require.NotContains(t, aco.dns.pendingNames, "myredis-abc123") +} diff --git a/internal/applecontainer/json_types.go b/internal/applecontainer/json_types.go new file mode 100644 index 00000000..0ca1d41b --- /dev/null +++ b/internal/applecontainer/json_types.go @@ -0,0 +1,562 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package applecontainer + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "strings" + "sync" + "time" + + apiv1 "github.com/microsoft/dcp/api/v1" + "github.com/microsoft/dcp/internal/containers" + "github.com/microsoft/dcp/pkg/slices" +) + +// The types below correspond to data returned by `container ... --format json` commands +// of the Apple container CLI (validated against version 1.0.0). The JSON schema is +// Apple-specific and unrelated to the Docker API schema; definitions only include the +// data DCP cares about. + +type appleListedContainer struct { + Id string `json:"id"` + Configuration appleContainerConfiguration `json:"configuration"` + Status appleContainerStatus `json:"status"` +} + +type appleContainerConfiguration struct { + Id string `json:"id"` + CreationDate time.Time `json:"creationDate"` + Image appleContainerImage `json:"image"` + InitProcess appleContainerInitProcess `json:"initProcess"` + Labels map[string]string `json:"labels"` + Mounts []appleContainerMount `json:"mounts"` + Networks []appleContainerNetworkConfig `json:"networks"` + PublishedPorts []appleContainerPublishedPort `json:"publishedPorts"` +} + +type appleContainerImage struct { + Reference string `json:"reference"` + Descriptor appleImageDescriptor `json:"descriptor"` +} + +type appleImageDescriptor struct { + Digest string `json:"digest"` +} + +type appleContainerInitProcess struct { + Arguments []string `json:"arguments"` + Environment []string `json:"environment"` + Executable string `json:"executable"` + WorkingDirectory string `json:"workingDirectory"` +} + +type appleContainerMount struct { + Destination string `json:"destination"` + Source string `json:"source"` + Options []string `json:"options"` + Type appleContainerMountType `json:"type"` +} + +// The mount type is encoded as a single-key object, e.g. {"virtiofs":{}}, {"tmpfs":{}} +// or {"volume":{"name":"...", ...}}. +type appleContainerMountType struct { + Virtiofs *struct{} `json:"virtiofs,omitempty"` + Tmpfs *struct{} `json:"tmpfs,omitempty"` + Volume *appleContainerMountVolume `json:"volume,omitempty"` +} + +type appleContainerMountVolume struct { + Name string `json:"name"` +} + +// Note: network option values are not uniformly typed (e.g. "hostname" is a string +// but "mtu" is a number), hence the "any" value type. +type appleContainerNetworkConfig struct { + Network string `json:"network"` + Options map[string]any `json:"options"` +} + +type appleContainerPublishedPort struct { + ContainerPort int32 `json:"containerPort"` + HostAddress string `json:"hostAddress"` + HostPort int32 `json:"hostPort"` + Proto string `json:"proto"` +} + +type appleContainerStatus struct { + State string `json:"state"` + StartedDate time.Time `json:"startedDate"` + Networks []appleContainerNetworkStatus `json:"networks"` +} + +type appleContainerNetworkStatus struct { + Network string `json:"network"` + Hostname string `json:"hostname"` + Ipv4Address string `json:"ipv4Address"` + Ipv4Gateway string `json:"ipv4Gateway"` + MacAddress string `json:"macAddress"` +} + +type appleInspectedVolume struct { + Id string `json:"id"` + Configuration appleVolumeConfiguration `json:"configuration"` +} + +type appleVolumeConfiguration struct { + CreationDate time.Time `json:"creationDate"` + Driver string `json:"driver"` + Labels map[string]string `json:"labels"` + Name string `json:"name"` + Source string `json:"source"` +} + +type appleInspectedImage struct { + Id string `json:"id"` + Configuration appleImageConfiguration `json:"configuration"` + Variants []appleImageVariant `json:"variants"` +} + +type appleImageConfiguration struct { + Name string `json:"name"` + Descriptor appleImageDescriptor `json:"descriptor"` +} + +type appleImageVariant struct { + Config appleImageVariantConfig `json:"config"` +} + +type appleImageVariantConfig struct { + Config appleImageVariantInnerConfig `json:"config"` +} + +type appleImageVariantInnerConfig struct { + Labels map[string]string `json:"Labels"` +} + +type appleInspectedNetwork struct { + Id string `json:"id"` + Configuration appleNetworkConfiguration `json:"configuration"` + Status appleNetworkStatus `json:"status"` +} + +type appleNetworkConfiguration struct { + CreationDate time.Time `json:"creationDate"` + Labels map[string]string `json:"labels"` + Mode string `json:"mode"` + Name string `json:"name"` + Plugin string `json:"plugin"` +} + +type appleNetworkStatus struct { + Ipv4Gateway string `json:"ipv4Gateway"` + Ipv4Subnet string `json:"ipv4Subnet"` + Ipv6Subnet string `json:"ipv6Subnet"` +} + +type appleSystemStatus struct { + ApiServerVersion string `json:"apiServerVersion"` + Status string `json:"status"` +} + +// mapContainerState converts an Apple container state into the Docker-compatible +// container status DCP uses. The Apple runtime only reports "running"/"stopped" +// (plus a transient "stopping"); a stopped container that has never been started +// is equivalent to Docker's "created" state. +func mapContainerState(state string, startedDate time.Time) containers.ContainerStatus { + switch state { + case "running": + return containers.ContainerStatusRunning + case "stopping": + return containers.ContainerStatusRemoving + case "stopped": + if startedDate.IsZero() { + return containers.ContainerStatusCreated + } + return containers.ContainerStatusExited + default: + return containers.ContainerStatus(state) + } +} + +// asArrayOfObjects parses command output that contains a single JSON array of objects +// (the format used by `container ... --format json` and `container ... inspect` commands). +func asArrayOfObjects[T any](b *bytes.Buffer, unmarshalFn func(json.RawMessage, *T) error) ([]T, error) { + if b == nil { + return nil, fmt.Errorf("the Apple container command timed out without returning any data") + } + + var rawItems []json.RawMessage + if err := json.Unmarshal(b.Bytes(), &rawItems); err != nil { + return nil, errors.Join(containers.ErrUnmarshalling, err) + } + + retval := []T{} + var err error + for _, raw := range rawItems { + var obj T + if unmarshalErr := unmarshalFn(raw, &obj); unmarshalErr != nil { + err = errors.Join(err, errors.Join(containers.ErrUnmarshalling, unmarshalErr)) + } else { + retval = append(retval, obj) + } + } + + return retval, err +} + +func asId(b *bytes.Buffer) (string, error) { + if b == nil { + return "", fmt.Errorf("the Apple container command timed out without returning object identifier") + } + + chunks := slices.NonEmpty[byte](slices.Map[[]byte, []byte](bytes.Split(b.Bytes(), []byte("\n")), bytes.TrimSpace)) + if len(chunks) != 1 { + return "", fmt.Errorf("command output does not contain a single identifier (it is '%s')", b.String()) + } + return string(chunks[0]), nil +} + +// filterByLabels applies DCP label filters client-side (the Apple container CLI has no +// server-side filtering support). +func filterByLabels[T any](items []T, filters []containers.LabelFilter, getLabels func(T) map[string]string) []T { + if len(filters) == 0 { + return items + } + + filtered := []T{} + for _, item := range items { + labels := getLabels(item) + match := true + for _, filter := range filters { + value, found := labels[filter.Key] + if !found || (filter.Value != "" && value != filter.Value) { + match = false + break + } + } + if match { + filtered = append(filtered, item) + } + } + + return filtered +} + +func unmarshalDiagnostics(data []byte, info *containers.ContainerDiagnostics) error { + if data == nil { + return fmt.Errorf("the Apple container command timed out without returning version data") + } + + var status appleSystemStatus + if err := json.Unmarshal(data, &status); err != nil { + return err + } + + info.ServerVersion = extractVersion(status.ApiServerVersion) + + return nil +} + +var versionRegEx = regexp.MustCompile(`version\s+([0-9]\S*)`) + +// extractVersion extracts a semantic version from Apple container version strings like +// "container CLI version 1.0.0 (build: release, commit: abcdef)". +func extractVersion(versionString string) string { + if matches := versionRegEx.FindStringSubmatch(versionString); len(matches) > 1 { + return matches[1] + } + return strings.TrimSpace(versionString) +} + +func verifySha256(data []byte, expectedHash string) error { + hash := sha256.Sum256(data) + actualHashHex := hex.EncodeToString(hash[:]) + + expected := strings.TrimSpace(expectedHash) + if strings.HasPrefix(strings.ToLower(expected), "sha256:") { + expected = expected[7:] + } + if !strings.EqualFold(actualHashHex, expected) { + return fmt.Errorf("SHA256 mismatch: expected %s, got %s", expectedHash, actualHashHex) + } + + return nil +} + +func unmarshalVolume(data json.RawMessage, vol *containers.InspectedVolume) error { + var aiv appleInspectedVolume + if err := json.Unmarshal(data, &aiv); err != nil { + return err + } + + name := aiv.Configuration.Name + if name == "" { + name = aiv.Id + } + + vol.Name = name + vol.Driver = aiv.Configuration.Driver + vol.MountPoint = aiv.Configuration.Source + vol.Labels = aiv.Configuration.Labels + vol.CreatedAt = aiv.Configuration.CreationDate + + return nil +} + +func unmarshalImage(data json.RawMessage, ic *containers.InspectedImage) error { + var aii appleInspectedImage + if err := json.Unmarshal(data, &aii); err != nil { + return err + } + + ic.Id = aii.Id + ic.Tags = []string{aii.Configuration.Name} + ic.Digest = aii.Configuration.Descriptor.Digest + for _, variant := range aii.Variants { + if len(variant.Config.Config.Labels) > 0 { + ic.Labels = variant.Config.Config.Labels + break + } + } + + return nil +} + +func unmarshalListedContainer(data json.RawMessage, lc *containers.ListedContainer) error { + var alc appleListedContainer + if err := json.Unmarshal(data, &alc); err != nil { + return err + } + + lc.Id = alc.Id + lc.Name = alc.Id + lc.Image = alc.Configuration.Image.Reference + lc.Status = mapContainerState(alc.Status.State, alc.Status.StartedDate) + lc.Labels = decodeLabels(alc.Configuration.Labels) + lc.Networks = slices.Map[string](alc.Configuration.Networks, func(n appleContainerNetworkConfig) string { return n.Network }) + + return nil +} + +func unmarshalContainer(data json.RawMessage, ic *containers.InspectedContainer) error { + var aic appleListedContainer + if err := json.Unmarshal(data, &aic); err != nil { + return err + } + + ic.Id = aic.Id + ic.Name = aic.Id + ic.Image = aic.Configuration.Image.Reference + ic.CreatedAt = aic.Configuration.CreationDate + ic.StartedAt = aic.Status.StartedDate + ic.Status = mapContainerState(aic.Status.State, aic.Status.StartedDate) + // Note: the Apple container runtime does not report exit codes (or exit errors) of + // detached containers, so ExitCode is always 0 and Error/FinishedAt remain empty. + + ic.Mounts = make([]apiv1.VolumeMount, 0, len(aic.Configuration.Mounts)) + for _, mount := range aic.Configuration.Mounts { + readOnly := false + for _, option := range mount.Options { + if option == "ro" || option == "readonly" { + readOnly = true + break + } + } + + switch { + case mount.Type.Volume != nil: + ic.Mounts = append(ic.Mounts, apiv1.VolumeMount{ + Type: apiv1.NamedVolumeMount, + Source: mount.Type.Volume.Name, + Target: mount.Destination, + ReadOnly: readOnly, + }) + case mount.Type.Virtiofs != nil: + ic.Mounts = append(ic.Mounts, apiv1.VolumeMount{ + Type: apiv1.BindMount, + Source: strings.TrimSuffix(mount.Source, "/"), + Target: mount.Destination, + ReadOnly: readOnly, + }) + default: + // Other mount types (e.g. tmpfs) have no DCP equivalent + continue + } + } + + ic.Ports = make(containers.InspectedContainerPortMapping) + for _, port := range aic.Configuration.PublishedPorts { + proto := port.Proto + if proto == "" { + proto = "tcp" + } + key := fmt.Sprintf("%d/%s", port.ContainerPort, strings.ToLower(proto)) + ic.Ports[key] = append(ic.Ports[key], containers.InspectedContainerHostPortConfig{ + HostIp: port.HostAddress, + HostPort: fmt.Sprintf("%d", port.HostPort), + }) + } + + ic.Env = make(map[string]string) + for _, envVar := range aic.Configuration.InitProcess.Environment { + parts := strings.SplitN(envVar, "=", 2) + if len(parts) > 1 { + ic.Env[parts[0]] = parts[1] + } else if len(parts) == 1 { + ic.Env[parts[0]] = "" + } + } + + if aic.Configuration.InitProcess.Executable != "" { + ic.Args = append(ic.Args, aic.Configuration.InitProcess.Executable) + } + ic.Args = append(ic.Args, aic.Configuration.InitProcess.Arguments...) + + // For a running container the network status includes addressing information. + // For a stopped (or not yet started) container the status network list is empty, + // so fall back to the configured networks. + if len(aic.Status.Networks) > 0 { + for _, network := range aic.Status.Networks { + ic.Networks = append(ic.Networks, containers.InspectedContainerNetwork{ + Id: network.Network, + Name: network.Network, + IPAddress: stripCidrSuffix(network.Ipv4Address), + Gateway: network.Ipv4Gateway, + MacAddress: network.MacAddress, + Aliases: nonEmptyStrings(network.Hostname), + }) + } + } else { + for _, network := range aic.Configuration.Networks { + hostname, _ := network.Options["hostname"].(string) + ic.Networks = append(ic.Networks, containers.InspectedContainerNetwork{ + Id: network.Network, + Name: network.Network, + Aliases: nonEmptyStrings(hostname), + }) + } + } + + ic.Labels = decodeLabels(aic.Configuration.Labels) + + return nil +} + +func unmarshalNetwork(data json.RawMessage, net *containers.InspectedNetwork) error { + var ain appleInspectedNetwork + if err := json.Unmarshal(data, &ain); err != nil { + return err + } + + name := ain.Configuration.Name + if name == "" { + name = ain.Id + } + + net.Id = ain.Id + net.Name = name + net.CreatedAt = ain.Configuration.CreationDate + net.Driver = ain.Configuration.Plugin + net.Labels = ain.Configuration.Labels + net.IPv6 = ain.Status.Ipv6Subnet != "" + if ain.Status.Ipv4Subnet != "" { + net.Subnets = append(net.Subnets, ain.Status.Ipv4Subnet) + net.Gateways = append(net.Gateways, ain.Status.Ipv4Gateway) + } + if ain.Status.Ipv6Subnet != "" { + net.Subnets = append(net.Subnets, ain.Status.Ipv6Subnet) + } + + return nil +} + +func unmarshalListedNetwork(data json.RawMessage, net *containers.ListedNetwork) error { + var ain appleInspectedNetwork + if err := json.Unmarshal(data, &ain); err != nil { + return err + } + + name := ain.Configuration.Name + if name == "" { + name = ain.Id + } + + net.Driver = ain.Configuration.Plugin + net.ID = ain.Id + net.IPv6 = ain.Status.Ipv6Subnet != "" + net.Internal = false + net.Labels = ain.Configuration.Labels + net.Name = name + + return nil +} + +func stripCidrSuffix(address string) string { + if idx := strings.Index(address, "/"); idx >= 0 { + return address[:idx] + } + return address +} + +func nonEmptyStrings(values ...string) []string { + return slices.NonEmpty[byte](values) +} + +// timestampingWriter prefixes every line written through it with an RFC3339Nano timestamp, +// mimicking the output of `docker logs --timestamps` (which the Apple container CLI does +// not support). +type timestampingWriter struct { + inner io.Writer + lock sync.Mutex + atLineStart bool +} + +func newTimestampingWriter(inner io.Writer) *timestampingWriter { + return ×tampingWriter{inner: inner, atLineStart: true} +} + +func (w *timestampingWriter) Write(p []byte) (int, error) { + w.lock.Lock() + defer w.lock.Unlock() + + written := 0 + for len(p) > 0 { + if w.atLineStart { + timestamp := time.Now().UTC().Format(time.RFC3339Nano) + if _, err := w.inner.Write([]byte(timestamp + " ")); err != nil { + return written, err + } + w.atLineStart = false + } + + lineEnd := bytes.IndexByte(p, '\n') + var chunk []byte + if lineEnd >= 0 { + chunk = p[:lineEnd+1] + w.atLineStart = true + } else { + chunk = p + } + + n, err := w.inner.Write(chunk) + written += n + if err != nil { + return written, err + } + + p = p[len(chunk):] + } + + return written, nil +} diff --git a/internal/applecontainer/labels.go b/internal/applecontainer/labels.go new file mode 100644 index 00000000..7ab11db5 --- /dev/null +++ b/internal/applecontainer/labels.go @@ -0,0 +1,63 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package applecontainer + +import ( + "encoding/base64" + "strings" +) + +// The Apple `container` CLI rejects `--label` values that contain '=' with +// "invalid label format ...", unlike Docker/OCI which permit arbitrary label values. +// DCP stamps labels whose values contain '=' — notably the mounts-tracking label, whose +// value looks like "type=volume,src=" (read back in the container controller to clean +// up mounts). To preserve such values we base64-encode (RawURL, which emits no '=' padding) +// any label value that contains '=', mark it with a sentinel prefix on the way in, and +// decode it again when we read labels back from `container ls`/`container inspect`. Values +// without '=' are passed through unchanged so labels stay human-readable in the common case. +const appleEncodedLabelPrefix = "dcp-b64." + +// encodeLabelValue encodes a label value for the Apple container CLI if it would otherwise +// be rejected (i.e. it contains '='). The returned value never contains '='. +func encodeLabelValue(value string) string { + if !strings.Contains(value, "=") { + return value + } + return appleEncodedLabelPrefix + base64.RawURLEncoding.EncodeToString([]byte(value)) +} + +// decodeLabelValue reverses encodeLabelValue. Values without the sentinel prefix (including +// labels not set by DCP) are returned unchanged. +func decodeLabelValue(value string) string { + encoded, found := strings.CutPrefix(value, appleEncodedLabelPrefix) + if !found { + return value + } + decoded, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil { + // Not something we encoded after all; leave it untouched. + return value + } + return string(decoded) +} + +// labelArg renders a "key=value" argument for the Apple container CLI's `--label` flag, +// encoding the value if necessary. +func labelArg(key, value string) string { + return key + "=" + encodeLabelValue(value) +} + +// decodeLabels returns a copy of labels with any encoded values decoded back to plaintext. +func decodeLabels(labels map[string]string) map[string]string { + if labels == nil { + return nil + } + decoded := make(map[string]string, len(labels)) + for key, value := range labels { + decoded[key] = decodeLabelValue(value) + } + return decoded +} diff --git a/internal/applecontainer/labels_test.go b/internal/applecontainer/labels_test.go new file mode 100644 index 00000000..91b1d2b3 --- /dev/null +++ b/internal/applecontainer/labels_test.go @@ -0,0 +1,66 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package applecontainer + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEncodeLabelValueLeavesPlainValuesUntouched(t *testing.T) { + for _, value := range []string{"", "dev", "6379/tcp", "a,b,c", "host.docker.internal", "line1\nline2"} { + require.Equal(t, value, encodeLabelValue(value), "values without '=' should be passed through unchanged") + } +} + +func TestEncodeLabelValueEncodesValuesContainingEquals(t *testing.T) { + encoded := encodeLabelValue("type=volume,src=redis-data") + require.NotContains(t, encoded, "=", "the encoded value must not contain '=' (the Apple CLI rejects it)") + require.True(t, strings.HasPrefix(encoded, appleEncodedLabelPrefix)) +} + +func TestLabelValueRoundTrips(t *testing.T) { + for _, value := range []string{ + "type=volume,src=redis.apphost-12ca8fe773-redis-data", + "type=volume,src=a\ntype=volume,src=b", + "plain-value", + "k=v=w", + "", + } { + require.Equal(t, value, decodeLabelValue(encodeLabelValue(value))) + } +} + +func TestDecodeLabelValueIgnoresUnencodedValues(t *testing.T) { + // A value that was never encoded (no sentinel prefix) must be returned unchanged, even if it + // happens to contain characters used by the encoding. + require.Equal(t, "some/plain.value-123", decodeLabelValue("some/plain.value-123")) +} + +func TestDecodeLabelsDecodesOnlyEncodedEntries(t *testing.T) { + in := map[string]string{ + "com.microsoft.developer.usvc-dev.mountsLabel": encodeLabelValue("type=volume,src=data"), + "com.microsoft.developer.usvc-dev.build": "dev", + "image.label.from.elsewhere": "unrelated=value-we-did-not-set", + } + + out := decodeLabels(in) + + require.Equal(t, "type=volume,src=data", out["com.microsoft.developer.usvc-dev.mountsLabel"]) + require.Equal(t, "dev", out["com.microsoft.developer.usvc-dev.build"]) + // Not encoded by us -> left untouched. + require.Equal(t, "unrelated=value-we-did-not-set", out["image.label.from.elsewhere"]) +} + +func TestLabelArgNeverContainsEqualsInValue(t *testing.T) { + arg := labelArg("com.microsoft.developer.usvc-dev.mountsLabel", "type=volume,src=x") + key, value, found := strings.Cut(arg, "=") + require.True(t, found) + require.Equal(t, "com.microsoft.developer.usvc-dev.mountsLabel", key) + require.NotContains(t, value, "=") +} diff --git a/internal/applecontainer/network_capability_test.go b/internal/applecontainer/network_capability_test.go new file mode 100644 index 00000000..a4ffc920 --- /dev/null +++ b/internal/applecontainer/network_capability_test.go @@ -0,0 +1,28 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package applecontainer + +import ( + "testing" + + "github.com/go-logr/logr/testr" + "github.com/stretchr/testify/require" + + "github.com/microsoft/dcp/internal/containers" + "github.com/microsoft/dcp/pkg/process" +) + +func TestAppleContainerUsesSingleNetwork(t *testing.T) { + log := testr.New(t) + executor := process.NewOSExecutor(log) + defer executor.Dispose() + + orchestrator := NewAppleContainerCliOrchestrator(log, executor) + + // The Apple runtime exposes only the built-in default network, so the controllers must take + // the single-network path (skip custom-network create/connect/detach). + require.True(t, containers.UsesSingleNetwork(orchestrator), "the Apple orchestrator must report UsesSingleNetwork() == true") +} diff --git a/internal/applecontainer/watcher.go b/internal/applecontainer/watcher.go new file mode 100644 index 00000000..7d027422 --- /dev/null +++ b/internal/applecontainer/watcher.go @@ -0,0 +1,167 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package applecontainer + +import ( + "context" + "encoding/json" + "time" + + "github.com/microsoft/dcp/internal/containers" + "github.com/microsoft/dcp/internal/pubsub" +) + +// The Apple container CLI has no `events` command, so container and network events +// are synthesized by polling the runtime and diffing consecutive snapshots. +// Events are delivered on a best-effort basis (same as for Docker/Podman, where the +// `events` stream is not restarted if it fails). + +const ( + containerPollInterval = 2 * time.Second + networkPollInterval = 5 * time.Second +) + +func (aco *AppleContainerCliOrchestrator) doWatchContainers(watcherCtx context.Context, ss *pubsub.SubscriptionSet[containers.EventMessage]) { + knownStates, _ := aco.snapshotContainerStates(watcherCtx) + + timer := time.NewTicker(containerPollInterval) + defer timer.Stop() + + for { + select { + case <-watcherCtx.Done(): + return + case <-timer.C: + } + + newStates, err := aco.snapshotContainerStates(watcherCtx) + if err != nil { + if watcherCtx.Err() == nil { + aco.log.V(1).Info("Polling for container events failed", "Error", err.Error()) + } + continue + } + + for id, state := range newStates { + oldState, known := knownStates[id] + switch { + case !known: + ss.Notify(containerEvent(containers.EventActionCreate, id)) + if state == "running" { + ss.Notify(containerEvent(containers.EventActionStart, id)) + } + case oldState != "running" && state == "running": + ss.Notify(containerEvent(containers.EventActionStart, id)) + case oldState == "running" && state != "running": + ss.Notify(containerEvent(containers.EventActionDie, id)) + } + } + + for id := range knownStates { + if _, stillExists := newStates[id]; !stillExists { + ss.Notify(containerEvent(containers.EventActionDestroy, id)) + } + } + + knownStates = newStates + } +} + +func (aco *AppleContainerCliOrchestrator) doWatchNetworks(watcherCtx context.Context, ss *pubsub.SubscriptionSet[containers.EventMessage]) { + knownNetworks, _ := aco.snapshotNetworks(watcherCtx) + + timer := time.NewTicker(networkPollInterval) + defer timer.Stop() + + for { + select { + case <-watcherCtx.Done(): + return + case <-timer.C: + } + + newNetworks, err := aco.snapshotNetworks(watcherCtx) + if err != nil { + if watcherCtx.Err() == nil { + aco.log.V(1).Info("Polling for network events failed", "Error", err.Error()) + } + continue + } + + for id := range newNetworks { + if _, known := knownNetworks[id]; !known { + ss.Notify(networkEvent(containers.EventActionCreate, id)) + } + } + + for id := range knownNetworks { + if _, stillExists := newNetworks[id]; !stillExists { + ss.Notify(networkEvent(containers.EventActionDestroy, id)) + } + } + + knownNetworks = newNetworks + } +} + +// snapshotContainerStates returns the current state (e.g. "running", "stopped") of all +// containers known to the runtime, keyed by container ID. +func (aco *AppleContainerCliOrchestrator) snapshotContainerStates(ctx context.Context) (map[string]string, error) { + cmd := makeAppleContainerCommand("ls", "--all", "--format", "json") + outBuf, errBuf, err := aco.runBufferedCommand(ctx, "WatchContainers", cmd, nil, nil, ordinaryCommandTimeout) + if err != nil { + return nil, normalizeCliErrors(errBuf) + } + + var listed []appleListedContainer + if unmarshalErr := json.Unmarshal(outBuf.Bytes(), &listed); unmarshalErr != nil { + return nil, unmarshalErr + } + + states := make(map[string]string, len(listed)) + for _, ctr := range listed { + states[ctr.Id] = ctr.Status.State + } + + return states, nil +} + +// snapshotNetworks returns the set of network IDs known to the runtime. +func (aco *AppleContainerCliOrchestrator) snapshotNetworks(ctx context.Context) (map[string]struct{}, error) { + cmd := makeAppleContainerCommand("network", "ls", "--format", "json") + outBuf, errBuf, err := aco.runBufferedCommand(ctx, "WatchNetworks", cmd, nil, nil, ordinaryCommandTimeout) + if err != nil { + return nil, normalizeCliErrors(errBuf) + } + + var listed []appleInspectedNetwork + if unmarshalErr := json.Unmarshal(outBuf.Bytes(), &listed); unmarshalErr != nil { + return nil, unmarshalErr + } + + networks := make(map[string]struct{}, len(listed)) + for _, network := range listed { + networks[network.Id] = struct{}{} + } + + return networks, nil +} + +func containerEvent(action containers.EventAction, containerId string) containers.EventMessage { + return containers.EventMessage{ + Source: containers.EventSourceContainer, + Action: action, + Actor: containers.EventActor{ID: containerId}, + } +} + +func networkEvent(action containers.EventAction, networkId string) containers.EventMessage { + return containers.EventMessage{ + Source: containers.EventSourceNetwork, + Action: action, + Actor: containers.EventActor{ID: networkId}, + } +} diff --git a/internal/applecontainer/watcher_test.go b/internal/applecontainer/watcher_test.go new file mode 100644 index 00000000..8fa1804d --- /dev/null +++ b/internal/applecontainer/watcher_test.go @@ -0,0 +1,131 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package applecontainer + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/go-logr/logr/testr" + "github.com/stretchr/testify/require" + + ct "github.com/microsoft/dcp/internal/containers" + internal_testutil "github.com/microsoft/dcp/internal/testutil" + "github.com/microsoft/dcp/pkg/testutil" +) + +func listedContainerJson(id string, state string) string { + startedDate := "" + if state != "created" { + startedDate = `"startedDate":"2026-06-12T15:00:00Z",` + } + return fmt.Sprintf( + `{"configuration":{"id":"%[1]s","image":{"reference":"docker.io/library/alpine:latest"},"labels":{},"networks":[{"network":"default","options":{"hostname":"%[1]s"}}]},"id":"%[1]s","status":{"networks":[],%[2]s"state":"%[3]s"}}`, + id, startedDate, state) +} + +// installContainerListAutoExecution makes every `container ls --all --format json` invocation +// respond with the current value of the provided snapshot function. +func installContainerListAutoExecution(pe *internal_testutil.TestProcessExecutor, snapshot func() string) { + pe.InstallAutoExecution(internal_testutil.AutoExecution{ + Condition: internal_testutil.ProcessSearchCriteria{ + Command: []string{"container", "ls", "--all"}, + }, + RunCommand: func(p *internal_testutil.ProcessExecution) int32 { + _, err := p.Cmd.Stdout.Write([]byte(snapshot())) + if err != nil { + return 1 + } + return 0 + }, + }) +} + +func waitForEvent(ctx context.Context, evtC <-chan ct.EventMessage) (ct.EventMessage, error) { + select { + case <-ctx.Done(): + return ct.EventMessage{}, fmt.Errorf("timed out waiting for an event") + case evt, ok := <-evtC: + if !ok { + return ct.EventMessage{}, fmt.Errorf("the event channel was closed") + } + return evt, nil + } +} + +func TestSynthesizesContainerEventsFromPolling(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, 60*time.Second) + defer cancel() + pe := internal_testutil.NewTestProcessExecutor(ctx) + defer func() { + require.NoError(t, pe.Close()) + }() + + var lock sync.Mutex + snapshot := "[]" + firstPollDone := false + setSnapshot := func(s string) { + lock.Lock() + defer lock.Unlock() + snapshot = s + } + getSnapshot := func() string { + lock.Lock() + defer lock.Unlock() + if !firstPollDone { + // The watcher primes its state from the first poll without emitting events, + // so the first poll must observe an empty container list regardless of when + // the test changes the snapshot. + firstPollDone = true + return "[]" + } + return snapshot + } + + installContainerListAutoExecution(pe, getSnapshot) + + aco := NewAppleContainerCliOrchestrator(testr.New(t), pe) + + evtC := make(chan ct.EventMessage, 10) + sub, err := aco.WatchContainers(evtC) + require.NoError(t, err) + defer sub.Cancel() + + // A new running container appears: expect "create" followed by "start" + setSnapshot("[" + listedContainerJson("ctr-1", "running") + "]") + + evt, err := waitForEvent(ctx, evtC) + require.NoError(t, err) + require.Equal(t, ct.EventSourceContainer, evt.Source) + require.Equal(t, ct.EventActionCreate, evt.Action) + require.Equal(t, "ctr-1", evt.Actor.ID) + + evt, err = waitForEvent(ctx, evtC) + require.NoError(t, err) + require.Equal(t, ct.EventActionStart, evt.Action) + require.Equal(t, "ctr-1", evt.Actor.ID) + + // The container stops: expect "die" + setSnapshot("[" + listedContainerJson("ctr-1", "stopped") + "]") + + evt, err = waitForEvent(ctx, evtC) + require.NoError(t, err) + require.Equal(t, ct.EventActionDie, evt.Action) + require.Equal(t, "ctr-1", evt.Actor.ID) + + // The container is removed: expect "destroy" + setSnapshot("[]") + + evt, err = waitForEvent(ctx, evtC) + require.NoError(t, err) + require.Equal(t, ct.EventActionDestroy, evt.Action) + require.Equal(t, "ctr-1", evt.Actor.ID) +} diff --git a/internal/containers/container_orchestrator.go b/internal/containers/container_orchestrator.go index 3eb4803b..dd1fd978 100644 --- a/internal/containers/container_orchestrator.go +++ b/internal/containers/container_orchestrator.go @@ -442,6 +442,13 @@ type CreateFilesOptions struct { type CreateFiles interface { // Create files/folders in the container based on the provided structure CreateFiles(ctx context.Context, options CreateFilesOptions) error + + // CreateFilesRequiresRunningContainer reports whether CreateFiles can only write into a running + // container. Runtimes that inject files by executing a command inside the container (e.g. the + // Apple container runtime via `exec ... tar`) return true, so the controller bakes the files into + // a derived image instead of copying them into the created container. Runtimes with a + // copy-into-created-container primitive (Docker/Podman `cp`) return false. + CreateFilesRequiresRunningContainer() bool } // ApplyImageLayers command types diff --git a/internal/containers/containers_common.go b/internal/containers/containers_common.go index 5d8f6f45..cc1df668 100644 --- a/internal/containers/containers_common.go +++ b/internal/containers/containers_common.go @@ -52,6 +52,64 @@ func (o StreamContainerLogsOptions) Apply(args []string) []string { return args } +// BuildCreateFilesTar builds an in-memory tar archive (rooted at "/") containing the entries described +// by options. It returns nil (with no error) when no content was produced, which can happen if every +// entry is marked ContinueOnError and all of them fail. The archive can be extracted into a container +// filesystem or used as an image layer. +func BuildCreateFilesTar(options CreateFilesOptions, log logr.Logger) ([]byte, error) { + tarWriter := usvc_io.NewTarWriter() + + certificateHashes := []string{} + for _, item := range options.Entries { + switch item.Type { + case apiv1.FileSystemEntryTypeDir: + if addDirectoryErr := AddDirectoryToTar(tarWriter, options.Destination, options.DefaultOwner, options.DefaultGroup, options.Umask, item, options.ModTime, log); addDirectoryErr != nil { + return nil, addDirectoryErr + } + case apiv1.FileSystemEntryTypeSymlink: + if addSymlinkErr := AddSymlinkToTar(tarWriter, options.Destination, options.DefaultOwner, options.DefaultGroup, options.Umask, item, options.ModTime, log); addSymlinkErr != nil { + if item.ContinueOnError { + log.Error(addSymlinkErr, "Failed to add symlink to tar archive, continuing", "SymLink", item) + } else { + return nil, addSymlinkErr + } + } + case apiv1.FileSystemEntryTypeOpenSSL: + hash, addCertErr := AddCertificateToTar(tarWriter, options.Destination, options.DefaultOwner, options.DefaultGroup, options.Umask, item, options.ModTime, certificateHashes, log) + if addCertErr != nil { + if item.ContinueOnError { + log.Error(addCertErr, "Failed to add a certificate to the tar file, but continueOnError is set", "Certificate", item) + } else { + return nil, addCertErr + } + } + + // Keep track of the certificate hashes we've added to this directory so that we can deal with the possibility of collisions + certificateHashes = append(certificateHashes, hash) + default: + if addFileErr := AddFileToTar(tarWriter, options.Destination, options.DefaultOwner, options.DefaultGroup, options.Umask, item, options.ModTime, log); addFileErr != nil { + if item.ContinueOnError { + log.Error(addFileErr, "Failed to add a file to the tar file, but continueOnError is set", "File", item) + } else { + return nil, addFileErr + } + } + } + } + + if tarWriter.Empty() { + // Can happen if all ContinueOnError items fail + return nil, nil + } + + buffer, bufferErr := tarWriter.Buffer() + if bufferErr != nil { + return nil, bufferErr + } + + return buffer.Bytes(), nil +} + func AddCertificateToTar(tarWriter *usvc_io.TarWriter, basePath string, owner int32, group int32, umask fs.FileMode, certificate apiv1.FileSystemEntry, modTime time.Time, hashes []string, log logr.Logger) (string, error) { if certificate.Type != apiv1.FileSystemEntryTypeOpenSSL { return "", fmt.Errorf("item is not a certificate") diff --git a/internal/containers/containers_common_test.go b/internal/containers/containers_common_test.go new file mode 100644 index 00000000..698bed55 --- /dev/null +++ b/internal/containers/containers_common_test.go @@ -0,0 +1,111 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package containers + +import ( + "archive/tar" + "bytes" + "io" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + apiv1 "github.com/microsoft/dcp/api/v1" + "github.com/microsoft/dcp/pkg/osutil" +) + +// readTarEntries reads a tar archive and returns a map of entry name to its string contents. +func readTarEntries(t *testing.T, data []byte) map[string]string { + t.Helper() + entries := map[string]string{} + reader := tar.NewReader(bytes.NewReader(data)) + for { + header, err := reader.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + + var contents bytes.Buffer + if header.Typeflag == tar.TypeReg { + _, copyErr := io.Copy(&contents, reader) + require.NoError(t, copyErr) + } else if header.Typeflag == tar.TypeSymlink { + contents.WriteString(header.Linkname) + } + entries[header.Name] = contents.String() + } + return entries +} + +func TestBuildCreateFilesTar_FilesAndSymlink(t *testing.T) { + options := CreateFilesOptions{ + Container: "ignored", + Destination: "/etc/app", + Umask: osutil.DefaultUmaskBitmask, + Entries: []apiv1.FileSystemEntry{ + { + Name: "config.txt", + Contents: "hello", + }, + { + Name: "link", + Type: apiv1.FileSystemEntryTypeSymlink, + Target: "./config.txt", + }, + }, + } + + tarBytes, err := BuildCreateFilesTar(options, logr.Discard()) + require.NoError(t, err) + require.NotNil(t, tarBytes) + + entries := readTarEntries(t, tarBytes) + assert.Equal(t, "hello", entries["/etc/app/config.txt"]) + assert.Equal(t, "./config.txt", entries["/etc/app/link"]) +} + +func TestBuildCreateFilesTar_EmptyReturnsNil(t *testing.T) { + options := CreateFilesOptions{ + Container: "ignored", + Destination: "/etc/app", + Umask: osutil.DefaultUmaskBitmask, + Entries: []apiv1.FileSystemEntry{}, + } + + tarBytes, err := BuildCreateFilesTar(options, logr.Discard()) + require.NoError(t, err) + assert.Nil(t, tarBytes) +} + +func TestBuildCreateFilesTar_Directory(t *testing.T) { + options := CreateFilesOptions{ + Container: "ignored", + Destination: "/etc/app", + Umask: osutil.DefaultUmaskBitmask, + Entries: []apiv1.FileSystemEntry{ + { + Name: "certs", + Type: apiv1.FileSystemEntryTypeDir, + Entries: []apiv1.FileSystemEntry{ + { + Name: "bundle.pem", + Contents: "cert-bytes", + }, + }, + }, + }, + } + + tarBytes, err := BuildCreateFilesTar(options, logr.Discard()) + require.NoError(t, err) + require.NotNil(t, tarBytes) + + entries := readTarEntries(t, tarBytes) + assert.Equal(t, "cert-bytes", entries["/etc/app/certs/bundle.pem"]) +} diff --git a/internal/containers/flags/container_runtime.go b/internal/containers/flags/container_runtime.go index 13a5d5e7..82d98f68 100644 --- a/internal/containers/flags/container_runtime.go +++ b/internal/containers/flags/container_runtime.go @@ -18,13 +18,14 @@ const RuntimeFlagName = "container-runtime" type RuntimeFlagValue string const ( - UnknownRuntime RuntimeFlagValue = "" - DockerRuntime RuntimeFlagValue = "docker" - PodmanRuntime RuntimeFlagValue = "podman" + UnknownRuntime RuntimeFlagValue = "" + DockerRuntime RuntimeFlagValue = "docker" + PodmanRuntime RuntimeFlagValue = "podman" + AppleContainerRuntime RuntimeFlagValue = "container" ) var ( - supportedRuntimeNames = []string{"docker", "podman"} + supportedRuntimeNames = []string{"docker", "podman", "container"} runtime = UnknownRuntime ) diff --git a/internal/containers/network_orchestrator.go b/internal/containers/network_orchestrator.go index aa4ddeda..40af13d9 100644 --- a/internal/containers/network_orchestrator.go +++ b/internal/containers/network_orchestrator.go @@ -187,3 +187,22 @@ type NetworkOrchestrator interface { RuntimeStatusChecker } + +// SingleNetworkOrchestrator is an optional capability implemented by orchestrators that expose +// only a single, built-in network (the default network) and cannot create custom networks or +// connect/disconnect containers after creation — for example the Apple `container` runtime. +// +// For such runtimes, all DCP-managed networks are mapped onto the default network: containers are +// created directly on it and reach each other via their routable IPs, and the controllers skip +// the create/connect/detach/disconnect machinery that is meaningless (and impossible) there. +type SingleNetworkOrchestrator interface { + UsesSingleNetwork() bool +} + +// UsesSingleNetwork reports whether the orchestrator exposes only the built-in default network +// (see SingleNetworkOrchestrator). It is false for orchestrators that do not implement the +// capability (Docker, Podman). +func UsesSingleNetwork(o any) bool { + sno, ok := o.(SingleNetworkOrchestrator) + return ok && sno.UsesSingleNetwork() +} diff --git a/internal/containers/runtimes/runtime.go b/internal/containers/runtimes/runtime.go index 23bdc26c..1b2409a8 100644 --- a/internal/containers/runtimes/runtime.go +++ b/internal/containers/runtimes/runtime.go @@ -10,6 +10,7 @@ import ( "fmt" "github.com/go-logr/logr" + "github.com/microsoft/dcp/internal/applecontainer" "github.com/microsoft/dcp/internal/containers" "github.com/microsoft/dcp/internal/containers/flags" "github.com/microsoft/dcp/internal/docker" @@ -22,8 +23,9 @@ type ContainerOrchestratorFactory func(log logr.Logger, executor process.Executo var ( errNoRuntimeFound = fmt.Errorf("no container runtime was found") supportedRuntimes = map[flags.RuntimeFlagValue]ContainerOrchestratorFactory{ - flags.DockerRuntime: docker.NewDockerCliOrchestrator, - flags.PodmanRuntime: podman.NewPodmanCliOrchestrator, + flags.DockerRuntime: docker.NewDockerCliOrchestrator, + flags.PodmanRuntime: podman.NewPodmanCliOrchestrator, + flags.AppleContainerRuntime: applecontainer.NewAppleContainerCliOrchestrator, } ) diff --git a/internal/dcp/commands/info.go b/internal/dcp/commands/info.go index 2cca7522..dae380d8 100644 --- a/internal/dcp/commands/info.go +++ b/internal/dcp/commands/info.go @@ -51,7 +51,7 @@ func NewInfoCommand(log logr.Logger) (*cobra.Command, error) { } type containerRuntime struct { - // Name of the container runtime (i.e. docker, podman) + // Name of the container runtime (i.e. docker, podman, container) Runtime string `json:"runtime"` // Default hostname within a container for accessing the host machine network diff --git a/internal/dcpdns/dcpdns_test.go b/internal/dcpdns/dcpdns_test.go new file mode 100644 index 00000000..e4452927 --- /dev/null +++ b/internal/dcpdns/dcpdns_test.go @@ -0,0 +1,209 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package dcpdns + +import ( + "context" + "encoding/binary" + "net" + "net/netip" + "os" + "path/filepath" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/require" + + "github.com/microsoft/dcp/pkg/testutil" +) + +// buildQuery constructs a DNS query message for the given name and type. +func buildQuery(t *testing.T, id uint16, name string, qtype uint16) []byte { + t.Helper() + + msg := make([]byte, 12) + binary.BigEndian.PutUint16(msg[0:2], id) + binary.BigEndian.PutUint16(msg[2:4], flagRecursionDesired) + binary.BigEndian.PutUint16(msg[4:6], 1) // QDCOUNT + + for start := 0; start < len(name); { + end := start + for end < len(name) && name[end] != '.' { + end++ + } + require.Greater(t, end, start, "empty label in %q", name) + msg = append(msg, byte(end-start)) + msg = append(msg, name[start:end]...) + start = end + 1 + } + msg = append(msg, 0) + + var typeAndClass [4]byte + binary.BigEndian.PutUint16(typeAndClass[0:2], qtype) + binary.BigEndian.PutUint16(typeAndClass[2:4], ClassINET) + return append(msg, typeAndClass[:]...) +} + +func writeRecordsFile(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "hosts") + require.NoError(t, os.WriteFile(path, []byte(content), 0644)) + return path +} + +func TestParseQuestion(t *testing.T) { + t.Parallel() + + query := buildQuery(t, 0x1234, "MyRedis.Example", TypeA) + question, err := ParseQuestion(query) + require.NoError(t, err) + require.EqualValues(t, 0x1234, question.ID) + require.Equal(t, "myredis.example", question.Name, "names should be lower-cased") + require.EqualValues(t, TypeA, question.Type) + require.EqualValues(t, ClassINET, question.Class) + require.Equal(t, len(query), question.EndOffset) + require.True(t, question.RecursionDesired) +} + +func TestParseQuestionRejectsResponsesAndGarbage(t *testing.T) { + t.Parallel() + + _, err := ParseQuestion([]byte{0x00, 0x01, 0x02}) + require.Error(t, err, "truncated message should be rejected") + + response := buildQuery(t, 1, "example.com", TypeA) + binary.BigEndian.PutUint16(response[2:4], flagResponse) + _, err = ParseQuestion(response) + require.Error(t, err, "a response message should be rejected") +} + +func TestBuildResponseAnswersARecords(t *testing.T) { + t.Parallel() + + query := buildQuery(t, 42, "myredis", TypeA) + question, err := ParseQuestion(query) + require.NoError(t, err) + + addr := netip.MustParseAddr("192.168.64.7") + response := BuildResponse(query, question, []netip.Addr{addr}) + + require.EqualValues(t, 42, binary.BigEndian.Uint16(response[0:2]), "response ID must match the query") + flags := binary.BigEndian.Uint16(response[2:4]) + require.NotZero(t, flags&flagResponse, "QR bit must be set") + require.NotZero(t, flags&flagAuthoritative, "AA bit must be set") + require.Zero(t, flags&0x000F, "RCODE must be NOERROR") + require.EqualValues(t, 1, binary.BigEndian.Uint16(response[6:8]), "one answer expected") + + // The answer's RDATA is the last 4 bytes + require.Equal(t, addr.As4(), [4]byte(response[len(response)-4:])) +} + +func TestBuildResponseEmptyAnswerForUnservedType(t *testing.T) { + t.Parallel() + + query := buildQuery(t, 7, "myredis", TypeAAAA) + question, err := ParseQuestion(query) + require.NoError(t, err) + + response := BuildResponse(query, question, nil) + flags := binary.BigEndian.Uint16(response[2:4]) + require.Zero(t, flags&0x000F, "RCODE must be NOERROR (not NXDOMAIN) so clients fall back to A") + require.EqualValues(t, 0, binary.BigEndian.Uint16(response[6:8]), "no answers expected") +} + +func TestRecordsLookupAndReload(t *testing.T) { + t.Parallel() + + path := writeRecordsFile(t, "192.168.64.5 myredis myredis.dcp # a comment\nbogus line\n") + records := NewRecords(path) + + addrs, found := records.Lookup("MyRedis.") + require.True(t, found) + require.Equal(t, []netip.Addr{netip.MustParseAddr("192.168.64.5")}, addrs) + + _, found = records.Lookup("unknown") + require.False(t, found) + + // Rewrite the file with a future mod time and verify the new content is served + require.NoError(t, os.WriteFile(path, []byte("192.168.64.9 postgres\n"), 0644)) + future := time.Now().Add(2 * time.Second) + require.NoError(t, os.Chtimes(path, future, future)) + + _, found = records.Lookup("myredis") + require.False(t, found, "removed records should disappear after reload") + addrs, found = records.Lookup("postgres") + require.True(t, found) + require.Equal(t, []netip.Addr{netip.MustParseAddr("192.168.64.9")}, addrs) +} + +// TestServerEndToEnd runs the forwarder on loopback with a fake upstream resolver and +// verifies both the authoritative and the relay paths over UDP. +func TestServerEndToEnd(t *testing.T) { + t.Parallel() + + ctx, cancel := testutil.GetTestContext(t, 30*time.Second) + defer cancel() + + // Fake upstream: replies to any query with a fixed A record response + upstream, upstreamErr := net.ListenPacket("udp", "127.0.0.1:0") + require.NoError(t, upstreamErr) + defer upstream.Close() + upstreamAnswer := netip.MustParseAddr("203.0.113.99") + go func() { + buffer := make([]byte, maxUDPMessageSize) + for { + length, client, readErr := upstream.ReadFrom(buffer) + if readErr != nil { + return + } + query := make([]byte, length) + copy(query, buffer[:length]) + if question, parseErr := ParseQuestion(query); parseErr == nil { + _, _ = upstream.WriteTo(BuildResponse(query, question, []netip.Addr{upstreamAnswer}), client) + } + } + }() + + recordsPath := writeRecordsFile(t, "192.168.64.5 myredis\n") + + serverConn, listenErr := net.ListenPacket("udp", "127.0.0.1:0") + require.NoError(t, listenErr) + + server := NewServer(NewRecords(recordsPath), upstream.LocalAddr().String(), logr.Discard()) + serveCtx, stopServer := context.WithCancel(ctx) + defer stopServer() + go func() { _ = server.Serve(serveCtx, serverConn, nil) }() + + resolve := func(name string, qtype uint16) []byte { + conn, dialErr := net.Dial("udp", serverConn.LocalAddr().String()) + require.NoError(t, dialErr) + defer conn.Close() + require.NoError(t, conn.SetDeadline(time.Now().Add(10*time.Second))) + + _, writeErr := conn.Write(buildQuery(t, 99, name, qtype)) + require.NoError(t, writeErr) + + response := make([]byte, maxUDPMessageSize) + length, readErr := conn.Read(response) + require.NoError(t, readErr) + return response[:length] + } + + // Known name: answered authoritatively from the records file + response := resolve("myredis", TypeA) + require.EqualValues(t, 1, binary.BigEndian.Uint16(response[6:8])) + require.Equal(t, netip.MustParseAddr("192.168.64.5").As4(), [4]byte(response[len(response)-4:])) + + // Known name, AAAA: empty NOERROR (not relayed upstream) + response = resolve("myredis", TypeAAAA) + require.EqualValues(t, 0, binary.BigEndian.Uint16(response[6:8])) + + // Unknown name: relayed to the upstream resolver + response = resolve("example.com", TypeA) + require.EqualValues(t, 1, binary.BigEndian.Uint16(response[6:8])) + require.Equal(t, upstreamAnswer.As4(), [4]byte(response[len(response)-4:])) +} diff --git a/internal/dcpdns/message.go b/internal/dcpdns/message.go new file mode 100644 index 00000000..87008444 --- /dev/null +++ b/internal/dcpdns/message.go @@ -0,0 +1,158 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package dcpdns + +import ( + "encoding/binary" + "fmt" + "net/netip" + "strings" +) + +// Minimal DNS wire-format handling (RFC 1035) for the DCP DNS forwarder. +// +// The forwarder only needs to understand enough of the protocol to answer A-record +// queries for names it knows about; everything else is relayed verbatim to the +// upstream resolver, so no general-purpose DNS library is required. + +const ( + dnsHeaderLength = 12 + + // DNS TYPE and CLASS values (RFC 1035, section 3.2) + TypeA = 1 + TypeAAAA = 28 + ClassINET = 1 + + // Response TTL for records served by the forwarder. Container IPs can change when + // containers are recreated, so responses should not be cached for long. + answerTTLSeconds = 10 + + // Flag bits within the second header pair (RFC 1035, section 4.1.1) + flagResponse = 1 << 15 // QR + flagAuthoritative = 1 << 10 // AA + flagRecursionDesired = 1 << 8 // RD + flagRecursionAvailable = 1 << 7 // RA +) + +// Question describes the first question of a DNS query message. +type Question struct { + // Query ID from the message header. + ID uint16 + + // The queried name, lower-cased, without the trailing dot. + Name string + + // The query TYPE (e.g. TypeA). + Type uint16 + + // The query CLASS (usually ClassINET). + Class uint16 + + // Offset of the first byte past the question section entry within the message. + // Used to copy the question section verbatim into a response. + EndOffset int + + // Whether the query requested recursion. + RecursionDesired bool +} + +// ParseQuestion extracts the first question from a DNS query message. +// It returns an error for messages that are not plain queries with at least one question. +func ParseQuestion(msg []byte) (Question, error) { + if len(msg) < dnsHeaderLength { + return Question{}, fmt.Errorf("message too short: %d bytes", len(msg)) + } + + flags := binary.BigEndian.Uint16(msg[2:4]) + if flags&flagResponse != 0 { + return Question{}, fmt.Errorf("message is a response, not a query") + } + + questionCount := binary.BigEndian.Uint16(msg[4:6]) + if questionCount == 0 { + return Question{}, fmt.Errorf("query contains no questions") + } + + var labels []string + offset := dnsHeaderLength + for { + if offset >= len(msg) { + return Question{}, fmt.Errorf("question name extends past end of message") + } + + labelLength := int(msg[offset]) + if labelLength&0xC0 != 0 { + // Compression pointers are not legal in the question name of a query. + return Question{}, fmt.Errorf("unexpected compression pointer in question name") + } + offset++ + + if labelLength == 0 { + break + } + if offset+labelLength > len(msg) { + return Question{}, fmt.Errorf("question label extends past end of message") + } + + labels = append(labels, strings.ToLower(string(msg[offset:offset+labelLength]))) + offset += labelLength + } + + if offset+4 > len(msg) { + return Question{}, fmt.Errorf("question type/class extends past end of message") + } + + question := Question{ + ID: binary.BigEndian.Uint16(msg[0:2]), + Name: strings.Join(labels, "."), + Type: binary.BigEndian.Uint16(msg[offset : offset+2]), + Class: binary.BigEndian.Uint16(msg[offset+2 : offset+4]), + EndOffset: offset + 4, + RecursionDesired: flags&flagRecursionDesired != 0, + } + + return question, nil +} + +// BuildResponse builds an authoritative response to the given query for the provided +// IPv4 addresses. An empty address list produces a NOERROR response with no answers, +// which is the correct authoritative reply for a known name queried with a type the +// forwarder has no records for (e.g. AAAA): the client then falls back to other types +// instead of treating the name as nonexistent. +func BuildResponse(query []byte, question Question, addrs []netip.Addr) []byte { + response := make([]byte, 0, question.EndOffset+len(addrs)*16) + + // Header + var header [dnsHeaderLength]byte + binary.BigEndian.PutUint16(header[0:2], question.ID) + flags := uint16(flagResponse | flagAuthoritative | flagRecursionAvailable) + if question.RecursionDesired { + flags |= flagRecursionDesired + } + binary.BigEndian.PutUint16(header[2:4], flags) + binary.BigEndian.PutUint16(header[4:6], 1) // QDCOUNT + binary.BigEndian.PutUint16(header[6:8], uint16(len(addrs))) // ANCOUNT + response = append(response, header[:]...) + + // Question section, copied verbatim from the query + response = append(response, query[dnsHeaderLength:question.EndOffset]...) + + // Answer records: a compression pointer to the question name, followed by an A record + for _, addr := range addrs { + ip4 := addr.As4() + var rr [16]byte + rr[0] = 0xC0 + rr[1] = dnsHeaderLength // pointer to the question name + binary.BigEndian.PutUint16(rr[2:4], TypeA) + binary.BigEndian.PutUint16(rr[4:6], ClassINET) + binary.BigEndian.PutUint32(rr[6:10], answerTTLSeconds) + binary.BigEndian.PutUint16(rr[10:12], 4) // RDLENGTH + copy(rr[12:16], ip4[:]) + response = append(response, rr[:]...) + } + + return response +} diff --git a/internal/dcpdns/records.go b/internal/dcpdns/records.go new file mode 100644 index 00000000..af553b91 --- /dev/null +++ b/internal/dcpdns/records.go @@ -0,0 +1,98 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package dcpdns + +import ( + "net/netip" + "os" + "strings" + "sync" + "time" +) + +// Records serves name -> IPv4 lookups backed by a hosts-format file +// (" [ ...]" per line, '#' comments). The file is re-read when its +// modification time changes, so the publisher (DCP) can update records by atomically +// rewriting the file — no control channel between DCP and the forwarder is needed. +type Records struct { + path string + + mu sync.Mutex + loadedAt time.Time // mod time of the file content currently loaded + checkedAt time.Time // when the file was last (re)read + byName map[string][]netip.Addr +} + +// How long loaded records may be served before the file is unconditionally re-read. +// The records file is updated on the host and observed through a shared filesystem +// (virtiofs) whose attribute caching can delay modification-time changes, so a pure +// mod-time check could serve stale records indefinitely. +const recordsMaxAge = 1 * time.Second + +func NewRecords(path string) *Records { + return &Records{path: path} +} + +// Lookup returns the IPv4 addresses for the given name (case-insensitive, with or +// without a trailing dot), or false if the name is not known. +func (r *Records) Lookup(name string) ([]netip.Addr, bool) { + name = strings.ToLower(strings.TrimSuffix(name, ".")) + + r.mu.Lock() + defer r.mu.Unlock() + + r.refreshLocked() + + addrs, found := r.byName[name] + return addrs, found +} + +func (r *Records) refreshLocked() { + info, statErr := os.Stat(r.path) + if statErr != nil { + // Treat a missing or unreadable file as "no records"; it may reappear later. + r.byName = nil + r.loadedAt = time.Time{} + return + } + + if r.byName != nil && !info.ModTime().After(r.loadedAt) && time.Since(r.checkedAt) < recordsMaxAge { + return // Cached content is fresh enough + } + + content, readErr := os.ReadFile(r.path) + if readErr != nil { + r.byName = nil + r.loadedAt = time.Time{} + return + } + r.checkedAt = time.Now() + + byName := map[string][]netip.Addr{} + for _, line := range strings.Split(string(content), "\n") { + if commentStart := strings.IndexByte(line, '#'); commentStart >= 0 { + line = line[:commentStart] + } + + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + + addr, parseErr := netip.ParseAddr(fields[0]) + if parseErr != nil || !addr.Is4() { + continue // Only IPv4 records are served + } + + for _, name := range fields[1:] { + name = strings.ToLower(strings.TrimSuffix(name, ".")) + byName[name] = append(byName[name], addr) + } + } + + r.byName = byName + r.loadedAt = info.ModTime() +} diff --git a/internal/dcpdns/server.go b/internal/dcpdns/server.go new file mode 100644 index 00000000..dda7d33e --- /dev/null +++ b/internal/dcpdns/server.go @@ -0,0 +1,228 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// Package dcpdns implements the small DNS forwarder DCP runs inside a container on the +// Apple container runtime's network. The Apple runtime has no network alias support and +// its gateway DNS serves no records for container names, so DCP points every container it +// creates at this forwarder (via `--dns`). The forwarder answers A-record queries for +// names DCP publishes (container names, network aliases, tunnel proxy aliases) from a +// hosts-format records file, and relays every other query verbatim to the upstream +// resolver (the container network gateway), so regular name resolution keeps working. +package dcpdns + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "net/netip" + "time" + + "github.com/go-logr/logr" +) + +const ( + // Maximum DNS message size the forwarder accepts over UDP. Larger answers from + // upstream are truncated by the upstream resolver itself (TC bit), prompting + // clients to retry over TCP. + maxUDPMessageSize = 4096 + + // How long to wait for the upstream resolver. + upstreamTimeout = 5 * time.Second +) + +type Server struct { + records *Records + upstream string // host:port of the upstream resolver + log logr.Logger +} + +func NewServer(records *Records, upstream string, log logr.Logger) *Server { + return &Server{ + records: records, + upstream: upstream, + log: log, + } +} + +// Serve answers DNS queries on the given UDP and TCP listeners until the context is +// canceled. Either listener may be nil. +func (s *Server) Serve(ctx context.Context, udpConn net.PacketConn, tcpListener net.Listener) error { + go func() { + <-ctx.Done() + if udpConn != nil { + _ = udpConn.Close() + } + if tcpListener != nil { + _ = tcpListener.Close() + } + }() + + errCh := make(chan error, 2) + + if udpConn != nil { + go func() { errCh <- s.serveUDP(ctx, udpConn) }() + } + if tcpListener != nil { + go func() { errCh <- s.serveTCP(ctx, tcpListener) }() + } + + err := <-errCh + if ctx.Err() != nil { + return nil // Shutting down + } + return err +} + +func (s *Server) serveUDP(ctx context.Context, conn net.PacketConn) error { + buffer := make([]byte, maxUDPMessageSize) + for { + length, client, readErr := conn.ReadFrom(buffer) + if readErr != nil { + if ctx.Err() != nil { + return nil + } + return fmt.Errorf("reading UDP query: %w", readErr) + } + + query := make([]byte, length) + copy(query, buffer[:length]) + + go func() { + response := s.handleQuery(query, s.relayUDP) + if response != nil { + if _, writeErr := conn.WriteTo(response, client); writeErr != nil && ctx.Err() == nil { + s.log.V(1).Info("Failed to write DNS response", "Client", client.String(), "Error", writeErr.Error()) + } + } + }() + } +} + +func (s *Server) serveTCP(ctx context.Context, listener net.Listener) error { + for { + conn, acceptErr := listener.Accept() + if acceptErr != nil { + if ctx.Err() != nil { + return nil + } + return fmt.Errorf("accepting TCP connection: %w", acceptErr) + } + + go func() { + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(upstreamTimeout)) + + query, readErr := readTCPMessage(conn) + if readErr != nil { + return + } + + response := s.handleQuery(query, s.relayTCP) + if response != nil { + _ = writeTCPMessage(conn, response) + } + }() + } +} + +// handleQuery answers the query authoritatively when its name is known, and relays it to +// the upstream resolver otherwise. It returns nil if no response should be sent. +func (s *Server) handleQuery(query []byte, relay func([]byte) ([]byte, error)) []byte { + question, parseErr := ParseQuestion(query) + if parseErr != nil { + // Not a query the forwarder understands; let the upstream resolver decide. + response, relayErr := relay(query) + if relayErr != nil { + return nil + } + return response + } + + if addrs, known := s.records.Lookup(question.Name); known { + // The forwarder is authoritative for this name. Non-A queries (e.g. AAAA) get an + // empty NOERROR answer so clients fall back to the A records instead of receiving + // an upstream NXDOMAIN for a name that does exist here. + var answers []netip.Addr + if question.Type == TypeA && question.Class == ClassINET { + answers = addrs + } + return BuildResponse(query, question, answers) + } + + response, relayErr := relay(query) + if relayErr != nil { + s.log.V(1).Info("Failed to relay DNS query upstream", "Name", question.Name, "Upstream", s.upstream, "Error", relayErr.Error()) + return nil + } + return response +} + +func (s *Server) relayUDP(query []byte) ([]byte, error) { + conn, dialErr := net.DialTimeout("udp", s.upstream, upstreamTimeout) + if dialErr != nil { + return nil, dialErr + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(upstreamTimeout)) + + if _, writeErr := conn.Write(query); writeErr != nil { + return nil, writeErr + } + + response := make([]byte, maxUDPMessageSize) + length, readErr := conn.Read(response) + if readErr != nil { + return nil, readErr + } + + return response[:length], nil +} + +func (s *Server) relayTCP(query []byte) ([]byte, error) { + conn, dialErr := net.DialTimeout("tcp", s.upstream, upstreamTimeout) + if dialErr != nil { + return nil, dialErr + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(upstreamTimeout)) + + if writeErr := writeTCPMessage(conn, query); writeErr != nil { + return nil, writeErr + } + + return readTCPMessage(conn) +} + +// readTCPMessage reads a DNS message in TCP framing (2-byte length prefix, RFC 1035 4.2.2). +func readTCPMessage(conn net.Conn) ([]byte, error) { + var lengthPrefix [2]byte + if _, readErr := io.ReadFull(conn, lengthPrefix[:]); readErr != nil { + return nil, readErr + } + + length := binary.BigEndian.Uint16(lengthPrefix[:]) + if length == 0 { + return nil, errors.New("zero-length DNS message") + } + + message := make([]byte, length) + if _, readErr := io.ReadFull(conn, message); readErr != nil { + return nil, readErr + } + + return message, nil +} + +func writeTCPMessage(conn net.Conn, message []byte) error { + framed := make([]byte, 2+len(message)) + binary.BigEndian.PutUint16(framed[0:2], uint16(len(message))) + copy(framed[2:], message) + + _, writeErr := conn.Write(framed) + return writeErr +} diff --git a/internal/docker/cli_orchestrator.go b/internal/docker/cli_orchestrator.go index 37488d86..003fc64a 100644 --- a/internal/docker/cli_orchestrator.go +++ b/internal/docker/cli_orchestrator.go @@ -962,6 +962,10 @@ func (dco *DockerCliOrchestrator) RemoveContainers(ctx context.Context, options return removed, err } +func (dco *DockerCliOrchestrator) CreateFilesRequiresRunningContainer() bool { + return false +} + func (dco *DockerCliOrchestrator) CreateFiles(ctx context.Context, options containers.CreateFilesOptions) error { args := []string{"container", "cp"} diff --git a/internal/podman/cli_orchestrator.go b/internal/podman/cli_orchestrator.go index aaec7579..3ac119d8 100644 --- a/internal/podman/cli_orchestrator.go +++ b/internal/podman/cli_orchestrator.go @@ -827,6 +827,10 @@ func (pco *PodmanCliOrchestrator) RemoveContainers(ctx context.Context, options return removed, err } +func (pco *PodmanCliOrchestrator) CreateFilesRequiresRunningContainer() bool { + return false +} + func (pco *PodmanCliOrchestrator) CreateFiles(ctx context.Context, options containers.CreateFilesOptions) error { args := []string{"container", "cp"} diff --git a/internal/testutil/ctrlutil/test_container_orchestrator.go b/internal/testutil/ctrlutil/test_container_orchestrator.go index 1314434d..e8528c7e 100644 --- a/internal/testutil/ctrlutil/test_container_orchestrator.go +++ b/internal/testutil/ctrlutil/test_container_orchestrator.go @@ -2099,6 +2099,10 @@ func (to *TestContainerOrchestrator) InspectContainers(ctx context.Context, opti return result, err } +func (to *TestContainerOrchestrator) CreateFilesRequiresRunningContainer() bool { + return false +} + func (to *TestContainerOrchestrator) CreateFiles(ctx context.Context, options containers.CreateFilesOptions) error { to.mutex.Lock() defer to.mutex.Unlock()