bootc: populate separate /var filesystems in bootc disk images

pkg/bootc/bootc.go

@@ -28,6 +28,10 @@ type Info struct {
 	// Is the container using a unified kernel?
 	UnifiedKernel bool
 
+	// Does the container's bootc initialize filesystems mounted at /var
+	// (and below) during "bootc install to-filesystem"?
+	InstallVarMounts bool
+
 	// What bootloader should be passed?
 	Bootloader *string
 }

pkg/bootc/resolver.go

@@ -235,11 +235,12 @@ func (c *Container) ResolveInfo(variant string) (*Info, error) {
 	bootcInfo.DefaultRootFs = bootcInstallConfig.Filesystem.Root.Type
 	bootcInfo.Bootloader = bootcInstallConfig.Bootloader
 
-	unifiedKernel, err := c.UnifiedKernel()
+	inspect, err := c.inspect()
 	if err != nil {
 		return nil, err
 	}
-	bootcInfo.UnifiedKernel = unifiedKernel
+	bootcInfo.UnifiedKernel = inspect.Kernel.Unified
+	bootcInfo.InstallVarMounts = inspect.installVarMounts()
 
 	size, err := getContainerSize(c.ref, c.storeOpts)
 	if err != nil {
@@ -254,10 +255,17 @@ func (c *Container) ResolveInfo(variant string) (*Info, error) {
 // used for build containers where we don't need all the information and trying
 // to get it might break things.
 func (c *Container) ResolveBuildInfo() (*Info, error) {
+	// The build container runs "bootc install", so its bootc determines
+	// which filesystems can be mounted for the installation.
+	installVarMounts, err := c.InstallVarMounts()
+	if err != nil {
+		return nil, err
+	}
 	return &Info{
-		Imgref:  c.ref,
-		ImageID: c.id,
-		Arch:    c.Arch(),
+		Imgref:           c.ref,
+		ImageID:          c.id,
+		Arch:             c.Arch(),
+		InstallVarMounts: installVarMounts,
 	}, nil
 }
 
@@ -382,8 +390,35 @@ func (c *Container) InitrdModules(kver string) ([]string, error) {
 	return strings.Split(strings.TrimRight(string(output), "\n"), "\n"), nil
 }
 
-// UnifiedKernel finds out if the kernel inside the bootc container is unified
-func (c *Container) UnifiedKernel() (bool, error) {
+// InstallFeatureVarMounts is advertised by "bootc container inspect" when
+// "bootc install to-filesystem" initializes filesystems mounted at /var (and
+// below) from the image. Older versions leave such filesystems empty, so they
+// hide the image's /var content, and versions before 1.12 reject them.
+const InstallFeatureVarMounts = "initialize-var-mounts"
+
+// containerInspect is the subset of "bootc container inspect --json" used here.
+type containerInspect struct {
+	Kernel struct {
+		Unified bool `json:"unified"`
+	} `json:"kernel"`
+	InstallFeatures []string `json:"install-features"`
+}
+
+func (i *containerInspect) installVarMounts() bool {
+	return slices.Contains(i.InstallFeatures, InstallFeatureVarMounts)
+}
+
+func parseContainerInspect(data []byte) (*containerInspect, error) {
+	var inspect containerInspect
+	if err := json.Unmarshal(data, &inspect); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal bootc inspect : %w", err)
+	}
+	return &inspect, nil
+}
+
+// inspect runs "bootc container inspect" in the container. The result is
+// empty for bootc versions without that command.
+func (c *Container) inspect() (*containerInspect, error) {
 	args := []string{"exec"}
 	args = append(args, c.storeOpts...)
 	args = append(args, c.id, "bootc", "container", "inspect", "--json")
@@ -393,25 +428,31 @@ func (c *Container) UnifiedKernel() (bool, error) {
 	if err != nil {
 		if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 2 {
 			// NOTE: the 'bootc container inspect' was added in version 1.12.0, which was not in RHEL-10.1.
-			// Treat exit value '2' as a non-error, since it means the command is not available and return false.
-			return false, nil
+			// Treat exit value '2' as a non-error, since it means the command is not available.
+			return &containerInspect{}, nil
 		}
-		return false, fmt.Errorf("failed to run bootc container inspect: %w, output:\n%s", err, output)
+		return nil, fmt.Errorf("failed to run bootc container inspect: %w, output:\n%s", err, output)
 	}
+	return parseContainerInspect(output)
+}
 
-	var bootcInspect struct {
-		Kargs  []string `json:"kargs"`
-		Kernel struct {
-			Version string `json:"version"`
-			Unified bool   `json:"unified"`
-		} `json:"kernel"`
+// UnifiedKernel finds out if the kernel inside the bootc container is unified
+func (c *Container) UnifiedKernel() (bool, error) {
+	inspect, err := c.inspect()
+	if err != nil {
+		return false, err
 	}
+	return inspect.Kernel.Unified, nil
+}
 
-	if err := json.Unmarshal(output, &bootcInspect); err != nil {
-		return false, fmt.Errorf("failed to unmarshal bootc inspect : %w", err)
+// InstallVarMounts finds out if the bootc inside the container initializes
+// filesystems mounted at /var (and below) during "bootc install to-filesystem".
+func (c *Container) InstallVarMounts() (bool, error) {
+	inspect, err := c.inspect()
+	if err != nil {
+		return false, err
 	}
-
-	return bootcInspect.Kernel.Unified, nil
+	return inspect.installVarMounts(), nil
 }
 
 func findImageIdFor(cntId, ref string, extraOpts []string) (string, error) {

pkg/bootc/resolver_test.go

@@ -259,6 +259,38 @@ func TestUnifiedKernelHappy(t *testing.T) {
 	}
 }
 
+func TestInstallVarMounts(t *testing.T) {
+	for _, tc := range []struct {
+		Name     string
+		In       string
+		ExitCode int
+		Out      bool
+		ErrMsg   string
+	}{
+		{Name: "supported", In: `{"install-features": ["initialize-var-mounts"]}`, Out: true},
+		{Name: "other features", In: `{"install-features": ["something-else"]}`, Out: false},
+		{Name: "no features", In: `{"kernel": {"unified": false}}`, Out: false},
+		{Name: "unknown command", In: "", ExitCode: 2, Out: false},
+		{Name: "error", In: "", ExitCode: 1, ErrMsg: "failed to run bootc container inspect"},
+		{Name: "bad json", In: "{", ErrMsg: "failed to unmarshal bootc inspect"},
+	} {
+		t.Run(tc.Name, func(t *testing.T) {
+			makeFakePodman(t, fmt.Sprintf(`#!/bin/sh
+				echo '%s'
+				exit %d
+				`, tc.In, tc.ExitCode))
+			cnt := bootc.Container{}
+			varMounts, err := cnt.InstallVarMounts()
+			if tc.ErrMsg != "" {
+				assert.ErrorContains(t, err, tc.ErrMsg)
+			} else {
+				assert.NoError(t, err)
+			}
+			assert.Equal(t, tc.Out, varMounts)
+		})
+	}
+}
+
 func TestBootloaderHappy(t *testing.T) {
 	for _, tc := range []struct {
 		In  string

pkg/distro/defs/bootc.go

@@ -26,6 +26,8 @@ type BootcDistro struct {
 	buildSourceInfo *osinfo.Info
 	unifiedKernel   bool
 	bootloader      *string
+	// set from the build container, which runs "bootc install"
+	installVarMounts bool
 
 	id            distro.ID
 	defaultFs     string
@@ -129,17 +131,18 @@ func NewBootcWithLoader(loader *Loader, name string, cinfo *bootc.Info) (*BootcD
 	d := &BootcDistro{
 		// the ID is technically not allowed by the ID parser, as it doesn't
 		// contain a version, but we will relax this requirement later
-		id:              *id,
-		imgref:          cinfo.Imgref,
-		imageID:         cinfo.ImageID,
-		buildImgref:     cinfo.Imgref, // default to using the same image for build for now
-		sourceInfo:      osInfo,
-		buildSourceInfo: osInfo,
-		defaultFs:       cinfo.DefaultRootFs,
-		releasever:      osInfo.OSRelease.VersionID,
-		rootfsMinSize:   cinfo.Size * containerSizeToDiskSizeMultiplier,
-		bootloader:      cinfo.Bootloader,
-		unifiedKernel:   cinfo.UnifiedKernel,
+		id:               *id,
+		imgref:           cinfo.Imgref,
+		imageID:          cinfo.ImageID,
+		buildImgref:      cinfo.Imgref, // default to using the same image for build for now
+		sourceInfo:       osInfo,
+		buildSourceInfo:  osInfo,
+		defaultFs:        cinfo.DefaultRootFs,
+		releasever:       osInfo.OSRelease.VersionID,
+		rootfsMinSize:    cinfo.Size * containerSizeToDiskSizeMultiplier,
+		bootloader:       cinfo.Bootloader,
+		unifiedKernel:    cinfo.UnifiedKernel,
+		installVarMounts: cinfo.InstallVarMounts,
 	}
 
 	// load image types from bootc-generic-1
@@ -298,6 +301,7 @@ func (d *BootcDistro) SetBuildContainer(cinfo *bootc.Info) error {
 
 	d.buildImgref = cinfo.Imgref
 	d.buildImageID = cinfo.ImageID
+	d.installVarMounts = cinfo.InstallVarMounts
 	d.buildSourceInfo = cinfo.OSInfo
 
 	return nil

pkg/distro/defs/bootc_imagetype.go

@@ -261,6 +261,7 @@ func (t *bootcImageType) manifestForDisk(bp *blueprint.Blueprint, options distro
 
 	img.Bootloader = bd.bootloader
 	img.UnifiedKernel = bd.unifiedKernel
+	img.InstallVarMounts = bd.installVarMounts
 
 	img.OSCustomizations.Subscription = options.Subscription
 	img.OSCustomizations.Users = users.UsersFromBP(customizations.GetUsers())

pkg/distro/defs/bootc_test.go

@@ -459,11 +459,12 @@ func TestSetBuildContainer(t *testing.T) {
 	// base bootc container info to initialise the distro before setting the
 	// build container info
 	baseBootcInfo := &bootc.Info{
-		Imgref:        "example.com/containers/distro-bootc:version12",
-		ImageID:       "acf88e518194fac963a1b2e2e4110e38a4ce5fb3fceddd624fae8997d4566930",
-		Arch:          "aarch64",
-		DefaultRootFs: "xfs",
-		Size:          100 * datasizes.MiB,
+		Imgref:           "example.com/containers/distro-bootc:version12",
+		ImageID:          "acf88e518194fac963a1b2e2e4110e38a4ce5fb3fceddd624fae8997d4566930",
+		Arch:             "aarch64",
+		DefaultRootFs:    "xfs",
+		Size:             100 * datasizes.MiB,
+		InstallVarMounts: true,
 		OSInfo: &osinfo.Info{
 			OSRelease: osinfo.OSRelease{
 				ID:        "whatever",
@@ -477,6 +478,8 @@ func TestSetBuildContainer(t *testing.T) {
 		expectedImgref  string
 		expectedImageID string
 		expectedError   string
+		// the build container runs bootc install, not the base container
+		expectedInstallVarMounts bool
 	}
 
 	testCases := map[string]testCase{
@@ -494,6 +497,16 @@ func TestSetBuildContainer(t *testing.T) {
 			expectedImageID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
 		},
 
+		"var-mounts": {
+			buildInfo: &bootc.Info{
+				Imgref:           "example.com/containers/distro-bootc:build43",
+				Arch:             "arm64",
+				InstallVarMounts: true,
+			},
+			expectedImgref:           "example.com/containers/distro-bootc:build43",
+			expectedInstallVarMounts: true,
+		},
+
 		"noimgref": {
 			buildInfo: &bootc.Info{
 				ImageID: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
@@ -541,6 +554,7 @@ func TestSetBuildContainer(t *testing.T) {
 
 			require.Equal(tc.expectedImgref, bd.buildImgref)
 			require.Equal(tc.expectedImageID, bd.buildImageID)
+			require.Equal(tc.expectedInstallVarMounts, bd.installVarMounts)
 		})
 	}
 }

pkg/image/bootc_disk.go

@@ -23,6 +23,8 @@ type BootcDiskImage struct {
 
 	Bootloader    *string
 	UnifiedKernel bool
+	// The installer's bootc initializes filesystems mounted at /var
+	InstallVarMounts bool
 
 	Partitions []PartitionConfig
 
@@ -114,6 +116,7 @@ func (img *BootcDiskImage) InstantiateManifestFromContainers(m *manifest.Manifes
 	}
 	rawImage.Bootloader = img.Bootloader
 	rawImage.UnifiedKernel = img.UnifiedKernel
+	rawImage.InstallVarMounts = img.InstallVarMounts
 	rawImage.PartitionTable = img.PartitionTable
 	rawImage.OSCustomizations = img.OSCustomizations
 	rawImage.DiskCustomizations = img.DiskCustomizations

pkg/manifest/raw_bootc.go

@@ -38,6 +38,8 @@ type RawBootcImage struct {
 
 	UnifiedKernel bool
 	Bootloader    *string
+	// The installer's bootc initializes filesystems mounted at /var
+	InstallVarMounts bool
 
 	// customizations go here because there is no intermediate
 	// tree, with `bootc install to-filesystem` we can only work
@@ -186,7 +188,7 @@ func (p *RawBootcImage) serialize() (osbuild.Pipeline, error) {
 	if err != nil {
 		return osbuild.Pipeline{}, err
 	}
-	st, err := osbuild.NewBootcInstallToFilesystemStage(opts, inputs, devices, mounts, p.platform)
+	st, err := osbuild.NewBootcInstallToFilesystemStage(opts, inputs, devices, mounts, p.platform, p.InstallVarMounts)
 	if err != nil {
 		return osbuild.Pipeline{}, err
 	}
@@ -204,8 +206,7 @@ func (p *RawBootcImage) serialize() (osbuild.Pipeline, error) {
 			return osbuild.Pipeline{}, fmt.Errorf("gen devices stage failed %w", err)
 		}
 
-		mounts = append(mounts, *osbuild.NewOSTreeDeploymentMountDefault("ostree.deployment", osbuild.OSTreeMountSourceMount))
-		mounts = append(mounts, *osbuild.NewBindMount("bind-ostree-deployment-to-tree", "mount://", "tree://"))
+		mounts = deploymentMounts(mounts)
 
 		postStages := []*osbuild.Stage{}
 
@@ -385,7 +386,15 @@ func (p *RawBootcImage) serialize() (osbuild.Pipeline, error) {
 		// then per the selinux policy
 		if p.OSCustomizations.SELinux != "" {
 			if len(postStages) > 0 {
-				for _, changedFile := range []string{"/etc", "/var"} {
+				// setfiles does not descend into other mounts, so relabel
+				// filesystems mounted below /var separately.
+				changedFiles := []string{"/etc", "/var"}
+				for _, m := range mounts {
+					if osbuild.IsVarMountpoint(m.Target) {
+						changedFiles = append(changedFiles, m.Target)
+					}
+				}
+				for _, changedFile := range changedFiles {
 					opts := &osbuild.SELinuxStageOptions{
 						Target:       "tree://" + changedFile,
 						FileContexts: fmt.Sprintf("etc/selinux/%s/contexts/files/file_contexts", p.OSCustomizations.SELinux),
@@ -403,6 +412,25 @@ func (p *RawBootcImage) serialize() (osbuild.Pipeline, error) {
 	return pipeline, nil
 }
 
+// deploymentMounts returns the mounts presenting the ostree deployment as the
+// tree. The deployment's /var is a bind mount of the stateroot's var, which
+// hides filesystems mounted on the physical root's /var. Mount those on top of
+// the deployment instead, so that customizations written to /var (e.g. home
+// directories) end up on the filesystems that are mounted at boot.
+func deploymentMounts(diskMounts []osbuild.Mount) []osbuild.Mount {
+	var mounts, varMounts []osbuild.Mount
+	for _, m := range diskMounts {
+		if osbuild.IsVarMountpoint(m.Target) {
+			varMounts = append(varMounts, m)
+		} else {
+			mounts = append(mounts, m)
+		}
+	}
+	mounts = append(mounts, *osbuild.NewOSTreeDeploymentMountDefault("ostree.deployment", osbuild.OSTreeMountSourceMount))
+	mounts = append(mounts, varMounts...)
+	return append(mounts, *osbuild.NewBindMount("bind-ostree-deployment-to-tree", "mount://", "tree://"))
+}
+
 // genMountpointSELinuxStages creates stages that label the root and /boot
 // mount point directories with proper SELinux contexts before bootc install.
 //