Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 20 additions & 5 deletions src/components/bigfiles.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,21 @@ pub struct BigfilesRepo {
components: IndexMap<String, u64>,
/// Mapping from path to ComponentId.
path_to_component: HashMap<Utf8PathBuf, ComponentId>,
/// Cap for component mtime clamps.
mtime_clamp: u64,
}

impl BigfilesRepo {
/// Load bigfiles repo by scanning for files >= MIN_SIZE.
///
/// Returns None if no qualifying files are found. Hardlinked files (same
/// inode, nlink > 1) are grouped into the same component.
/// inode, nlink > 1) are grouped into the same component. Component mtime
/// clamps are set to the file's mtime so that ancestor dirs are clamped to
/// a reproducible value, capped at `mtime_clamp`.
// TODO: the upfront scan logic here (inode table, path_to_component map)
// could be deferred to weak_claims_for_path time since it receives
// &FileInfo with size/inode/nlink.
pub fn load(files: &FileMap) -> Option<Self> {
pub fn load(files: &FileMap, mtime_clamp: u64) -> Option<Self> {
let mut components: IndexMap<String, u64> = IndexMap::new();
let mut path_to_component: HashMap<Utf8PathBuf, ComponentId> = HashMap::new();

Expand Down Expand Up @@ -107,6 +111,7 @@ impl BigfilesRepo {
Some(Self {
components,
path_to_component,
mtime_clamp,
})
}
}
Expand Down Expand Up @@ -134,12 +139,22 @@ impl ComponentsRepo for BigfilesRepo {
}

fn component_info(&self, id: ComponentId) -> ComponentInfo<'_> {
let (name, &mtime_clamp) = self
let (name, &mtime) = self
.components
.get_index(id.0)
// SAFETY: the ids we're given come from the IndexMap itself
// when we inserted the element, so it must be valid.
.expect("invalid ComponentId");
// There's no canonical mtime for bigfiles (unlike e.g. the RPM build time): the file's
// mtime is the only signal, but it may be entirely arbitrary, as there's no way to know if
// the user set it explicitly for reproducibility. So we always take the min() with the
// default clamp (which comes from SOURCE_DATE_EPOCH or defaults to $now):
// - with an explicit epoch, arbitrary file mtimes (e.g. "state" files like the rpmdb) get
// clamped down, restoring reproducibility;
// - without one, the clamp defaults to $now; file mtimes are necessarily at or before $now
// (the rootfs predates chunking), so explicitly-set mtimes win, and arbitrary ones float
// but were never reproducible to begin with.
let mtime_clamp = mtime.min(self.mtime_clamp);
ComponentInfo {
name,
mtime_clamp,
Expand Down Expand Up @@ -221,7 +236,7 @@ mod tests {
let rootfs = Dir::open_ambient_dir(tmp.path(), ambient_authority()).unwrap();
let files = crate::scan::Scanner::new(&rootfs).scan().unwrap();

let repo = BigfilesRepo::load(&files).unwrap();
let repo = BigfilesRepo::load(&files, 0).unwrap();

// small file should not be claimed
let claims = repo
Expand Down Expand Up @@ -269,7 +284,7 @@ mod tests {
create_sparse_file(rootfs, "b/foobar", 4 * 1024 * 1024);
});

let repo = BigfilesRepo::load(&files).unwrap();
let repo = BigfilesRepo::load(&files, 0).unwrap();

// First one uses filename, second uses full path
assert_component(&repo, &rootfs, "/a/foobar", "foobar");
Expand Down
20 changes: 15 additions & 5 deletions src/components/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ pub struct ComponentsRepos {
pub struct Component {
/// The maximum mtime for files in this component during the build phase.
/// File mtimes will be clamped to this value.
///
/// The clamp is what makes layers reproducible: package repos use a
/// canonical mtime (e.g. the RPM build time), which anchors the whole
/// layer regardless of when it was written. Repos without a canonical
/// mtime (bigfiles, xattr) fall back to the most recent file mtime,
/// capped by the default clamp.
pub mtime_clamp: u64,
/// Probability that the component doesn't change over STABILITY_PERIOD_DAYS.
/// Used by the packing algorithm.
Expand Down Expand Up @@ -112,11 +118,15 @@ impl ComponentsRepos {
/// The `files` map is the set of paths in the rootfs. This avoids the xattr
/// repo having to walk the rootfs again. The `default_mtime_clamp` will be
/// used as the mtime clamp for components that don't have a reproducible
/// clamp (e.g. xattr-claimed files, unclaimed files).
/// clamp (e.g. unclaimed files), as the stability anchor for package
/// repos, and to cap the mtime clamps of repos without canonical mtimes
/// (e.g. big files, xattr-claimed files).
pub fn load(rootfs: &Dir, files: &FileMap, default_mtime_clamp: u64) -> Result<Self> {
let mut repos: Vec<Box<dyn ComponentsRepo>> = Vec::new();

if let Some(repo) = xattr::XattrRepo::load(files).context("loading xattrs")? {
if let Some(repo) =
xattr::XattrRepo::load(files, default_mtime_clamp).context("loading xattrs")?
{
tracing::info!(repo = "xattr", "loaded repo");
repos.push(Box::new(repo));
}
Expand All @@ -135,7 +145,7 @@ impl ComponentsRepos {
repos.push(Box::new(repo));
}

if let Some(repo) = bigfiles::BigfilesRepo::load(files) {
if let Some(repo) = bigfiles::BigfilesRepo::load(files, default_mtime_clamp) {
tracing::info!(repo = "bigfiles", "loaded repo");
repos.push(Box::new(repo));
}
Expand Down Expand Up @@ -410,7 +420,7 @@ mod tests {

let files = crate::scan::Scanner::new(&rootfs).scan().unwrap();

let xattr_repo = xattr::XattrRepo::load(&files).unwrap().unwrap();
let xattr_repo = xattr::XattrRepo::load(&files, 0).unwrap().unwrap();
let packages = rpm_qa::load_from_str(RPM_FIXTURE).unwrap();
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
Expand Down Expand Up @@ -478,7 +488,7 @@ mod tests {

let files = crate::scan::Scanner::new(&rootfs).scan().unwrap();

let xattr_repo = xattr::XattrRepo::load(&files).unwrap().unwrap();
let xattr_repo = xattr::XattrRepo::load(&files, 0).unwrap().unwrap();
let repos: Vec<Box<dyn ComponentsRepo>> = vec![Box::new(xattr_repo)];
let loaded = ComponentsRepos {
repos,
Expand Down
26 changes: 18 additions & 8 deletions src/components/xattr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,19 @@ pub struct XattrRepo {
path_to_component: HashMap<Utf8PathBuf, ComponentId>,
/// Per-component stability, indexed by ComponentId.
component_stability: Vec<f64>,
/// Cap for component mtime clamps.
mtime_clamp: u64,
}

impl XattrRepo {
/// Load xattr repo by scanning rootfs for user.component xattrs.
/// Pre-computes directory inheritance for all paths in `files`.
/// Uses cached xattrs from FileInfo rather than reading from disk.
pub fn load(files: &FileMap) -> Result<Option<Self>> {
///
/// Component mtime clamps are set to the most recent file mtime in the
/// component, so that ancestor dirs are clamped to a reproducible value,
/// capped at `mtime_clamp`.
pub fn load(files: &FileMap, mtime_clamp: u64) -> Result<Option<Self>> {
let mut components: IndexMap<String, u64> = IndexMap::new();
let mut path_to_component: HashMap<Utf8PathBuf, ComponentId> = HashMap::new();
// Track raw intervals during scanning to detect conflicts
Expand Down Expand Up @@ -155,6 +161,7 @@ impl XattrRepo {
components,
path_to_component,
component_stability,
mtime_clamp,
}))
}
}
Expand Down Expand Up @@ -234,12 +241,15 @@ impl ComponentsRepo for XattrRepo {
}

fn component_info(&self, id: ComponentId) -> ComponentInfo<'_> {
let (name, &mtime_clamp) = self
let (name, &mtime) = self
.components
.get_index(id.0)
// SAFETY: the ids we're given come from the IndexMap itself
// when we inserted the element, so it must be valid.
.expect("invalid ComponentId");
// Same reasoning as bigfiles: no canonical mtime for xattr components
// and file mtimes may be arbitrary, so cap with the default clamp.
let mtime_clamp = mtime.min(self.mtime_clamp);
ComponentInfo {
name,
mtime_clamp,
Expand Down Expand Up @@ -315,7 +325,7 @@ mod tests {
// File without xattr outside of directory - should not be claimed
rootfs.write("noattr", "content").unwrap();
});
let repo = XattrRepo::load(&files).unwrap().unwrap();
let repo = XattrRepo::load(&files, 0).unwrap().unwrap();

// /mydir and /mydir/normal should be dircomponent
assert_component(&repo, "/mydir", FileType::Directory, "dircomponent");
Expand Down Expand Up @@ -345,7 +355,7 @@ mod tests {
set_component(rootfs, "a/b/c/d", "compD");
set_component(rootfs, "x", "compX");
});
let repo = XattrRepo::load(&files).unwrap().unwrap();
let repo = XattrRepo::load(&files, 0).unwrap().unwrap();

assert_component(&repo, "/a", FileType::Directory, "compA");
assert_component(&repo, "/a/other", FileType::File, "compA"); // inherits from /a
Expand All @@ -366,7 +376,7 @@ mod tests {
// Create a symlink inside the directory - it should inherit from parent
rootfs.symlink("../somewhere", "mydir/link").unwrap();
});
let repo = XattrRepo::load(&files).unwrap().unwrap();
let repo = XattrRepo::load(&files, 0).unwrap().unwrap();

// Both should be claimed by mycomp
assert_component(&repo, "/mydir", FileType::Directory, "mycomp");
Expand Down Expand Up @@ -402,7 +412,7 @@ mod tests {
rootfs.write("mydir/file", "content").unwrap();
set_update_interval(rootfs, "mydir/file", "60");
});
let result = XattrRepo::load(&files);
let result = XattrRepo::load(&files, 0);
assert!(result.is_err());
let msg = format!("{:#}", result.err().unwrap());
assert!(msg.contains("conflicting"), "unexpected error: {msg}");
Expand All @@ -417,7 +427,7 @@ mod tests {
rootfs.write("mydir/file", "content").unwrap();
set_update_interval(rootfs, "mydir/file", "30");
});
let repo = XattrRepo::load(&files).unwrap().unwrap();
let repo = XattrRepo::load(&files, 0).unwrap().unwrap();

let claims = repo.strong_claims_for_path(Utf8Path::new("/mydir"), &fi(FileType::Directory));
let expected = interval_to_stability(30);
Expand All @@ -437,7 +447,7 @@ mod tests {
set_component(rootfs, "app2", "comp2");
rootfs.write("app2/file", "content").unwrap();
});
let repo = XattrRepo::load(&files).unwrap().unwrap();
let repo = XattrRepo::load(&files, 0).unwrap().unwrap();

// comp1 should have yearly stability
let claims = repo.strong_claims_for_path(Utf8Path::new("/app1"), &fi(FileType::Directory));
Expand Down
13 changes: 13 additions & 0 deletions tests/e2e/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,19 @@ assert_path_exists() {
fi
}

# Assert that a path in an image has the given mtime (epoch seconds).
assert_path_mtime() {
local image="${1}"; shift
local path="${1}"; shift
local expected="${1}"; shift
local actual
actual=$(podman run --rm "${image}" stat -c '%Y' "${path}")
if [[ "${actual}" != "${expected}" ]]; then
echo "ERROR: mtime of ${path} in ${image} is ${actual}, expected ${expected}" >&2
return 1
fi
}

# Assert that a path does not exist in an image.
assert_path_not_exists() {
local image="${1}"; shift
Expand Down
16 changes: 13 additions & 3 deletions tests/e2e/test-buildtime.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,16 @@ podman pull "${BASE_IMAGE}"
cat > Containerfile.rootfs <<EOF
FROM ${BASE_IMAGE}
RUN dnf install -y jq acl attr && \
dnf clean all && \
dd if=/dev/urandom of=/usr/bin/chungus bs=1M count=20
dnf clean all
EOF
buildah build -t "${ROOTFS_IMAGE}" -f Containerfile.rootfs .

# build chunked image using FROM oci-archive: trick (note the use of oci-archive
# instead of oci is intended here to make sure we have coverage for this now
# that the Containerfile.splitter approach has moved to oci)
# `chungus` gets a canonical mtime earlier than SOURCE_DATE_EPOCH (its mtime
# should win); `chungus2` gets a wall-clock mtime later than SOURCE_DATE_EPOCH
# (its mtime should be clamped down)
cat > Containerfile <<EOF
FROM ${ROOTFS_IMAGE} AS builder
# create a test binary with various xattr types
Expand All @@ -45,6 +47,8 @@ RUN cp /usr/bin/true /usr/bin/test-xattrs && \
setfacl -m u:nobody:r /usr/bin/test-xattrs && \
setfattr -n user.testkey1 -v testvalue1 /usr/bin/test-xattrs && \
setfattr -n user.testkey2 -v testvalue2 /usr/bin/test-xattrs && \
truncate -s 20M /usr/bin/chungus && \
cp /usr/bin/chungus /usr/bin/chungus2 && \
touch -d @0 /usr/bin/chungus

FROM ${CHUNKAH_IMG:?} AS chunkah
Expand All @@ -62,11 +66,17 @@ buildah_build -t "${CHUNKED_IMAGE}" -f Containerfile .
podman run --rm "${CHUNKED_IMAGE}" jq --version

# check for expected components
assert_has_components "${CHUNKED_IMAGE}" "rpm/filesystem" "rpm/setup" "rpm/glibc" "rpm/jq" "bigfiles/chungus"
assert_has_components "${CHUNKED_IMAGE}" "rpm/filesystem" "rpm/setup" "rpm/glibc" "rpm/jq" "bigfiles/chungus" "bigfiles/chungus2"

# verify we got exactly 64 layers (the default)
assert_layer_count "${CHUNKED_IMAGE}" 64

# verify mtime clamping: `chungus` keeps its canonical mtime (0) since it's
# below SOURCE_DATE_EPOCH, while `chungus2` (wall-clock mtime) is clamped to
# SOURCE_DATE_EPOCH
assert_path_mtime "${CHUNKED_IMAGE}" /usr/bin/chungus 0
assert_path_mtime "${CHUNKED_IMAGE}" /usr/bin/chungus2 1700000000

# verify that security.capability xattrs are preserved
caps=$(podman run --rm "${CHUNKED_IMAGE}" getcap /usr/bin/test-xattrs)
[[ "${caps}" == *"cap_net_raw=ep"* ]]
Expand Down
Loading