Skip to content

Commit

Permalink
Merge pull request #134 from ariel-miculas/support_symlinks_in_ocidir
Browse files Browse the repository at this point in the history 
Support a shared cache for the OCI blobs, used in the LXC OCI template
  • Loading branch information
@hallyn
hallyn authored Oct 15, 2024
2 parents 79fefdd + 37b83f0 commit 40d0371
Show file tree
Hide file tree
Showing 4 changed files with 43 additions and 81 deletions.
7 changes: 3 additions & 4 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion puzzlefs-lib/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ os_pipe = "1.1.2"
tempfile = "3.10"
openat = "0.1.21"
zstd-seekable = "0.1.23"
ocidir = "0.3.0"
ocidir = {git="https://github.com/containers/ocidir-rs"}
cap-std = "3.2.0"


Expand Down
32 changes: 17 additions & 15 deletions puzzlefs-lib/src/builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -502,14 +502,12 @@ pub fn enable_fs_verity(oci: Image, tag: &str, manifest_root_hash: &str) -> Resu
.find_manifest_with_tag(tag)?
.ok_or_else(|| WireFormatError::MissingManifest(tag.to_string(), Backtrace::capture()))?;
let config_digest = manifest.config().digest().digest();
let config_digest_path = oci.blob_path().join(config_digest);
enable_verity_for_file(&oci.0.dir.open(config_digest_path)?)?;
let config_digest_path = Image::blob_path().join(config_digest);
enable_verity_for_file(&oci.0.dir().open(config_digest_path)?)?;

for (content_addressed_file, verity_hash) in rootfs.get_verity_data()? {
let file_path = oci
.blob_path()
.join(Digest::new(&content_addressed_file).to_string());
let fd = oci.0.dir.open(&file_path)?;
let file_path = Image::blob_path().join(Digest::new(&content_addressed_file).to_string());
let fd = oci.0.dir().open(&file_path)?;
if let Err(e) = fsverity_enable(
fd.as_raw_fd(),
FS_VERITY_BLOCK_SIZE_DEFAULT,
Expand Down Expand Up @@ -564,8 +562,8 @@ pub mod tests {

let md = image
.0
.dir
.symlink_metadata(image.blob_path().join(FILE_DIGEST))
.dir()
.symlink_metadata(Image::blob_path().join(FILE_DIGEST))
.unwrap();
assert!(md.is_file());

Expand Down Expand Up @@ -603,14 +601,15 @@ pub mod tests {
chunks[0].len,
decompressor.get_uncompressed_length().unwrap()
);
Ok(())
} else {
panic!("bad inode mode: {:?}", inodes[1].mode);
}
};
image.0.fsck()?;
Ok::<(), anyhow::Error>(())
}

#[test]
fn test_delta_generation() {
fn test_delta_generation() -> anyhow::Result<()> {
let dir = tempdir().unwrap();
let image = Image::new(dir.path()).unwrap();
let tag = "test";
Expand All @@ -623,6 +622,7 @@ pub mod tests {
delta_dir.join("SekienAkashita.jpg"),
)
.unwrap();
image.0.fsck()?;

let new_tag = "test2";
let (_desc, image) =
Expand All @@ -631,6 +631,7 @@ pub mod tests {
assert_eq!(delta.metadatas.len(), 2);

let image = Image::new(dir.path()).unwrap();
image.0.fsck()?;
let mut pfs = PuzzleFS::open(image, new_tag, None).unwrap();
assert_eq!(pfs.max_inode().unwrap(), 3);
let mut walker = WalkPuzzleFS::walk(&mut pfs).unwrap();
Expand All @@ -651,6 +652,7 @@ pub mod tests {
assert_eq!(foo_dir.inode.dir_entries().unwrap().len(), 0);

assert!(walker.next().is_none());
Ok(())
}

fn do_vecs_match<T: PartialEq>(a: &[T], b: &[T]) -> bool {
Expand All @@ -662,8 +664,8 @@ pub mod tests {
matching == a.len()
}

fn get_image_blobs(image: &Image) -> Vec<OsString> {
WalkDir::new(image.blob_path())
fn get_image_blobs() -> Vec<OsString> {
WalkDir::new(Image::blob_path())
.contents_first(false)
.follow_links(false)
.same_file_system(true)
Expand All @@ -685,7 +687,7 @@ pub mod tests {

for (i, image) in images.iter().enumerate() {
build_test_fs(path, image, "test").unwrap();
let ents = get_image_blobs(image);
let ents = get_image_blobs();
sha_suite.push(ents);

if i != 0 && !do_vecs_match(&sha_suite[i - 1], &sha_suite[i]) {
Expand All @@ -708,7 +710,7 @@ pub mod tests {

for (i, image) in images.iter().enumerate() {
build_test_fs(&path[i], image, "test").unwrap();
let ents = get_image_blobs(image);
let ents = get_image_blobs();
sha_suite.push(ents);

if i != 0 && !do_vecs_match(&sha_suite[i - 1], &sha_suite[i]) {
Expand Down
83 changes: 22 additions & 61 deletions puzzlefs-lib/src/oci.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ use std::any::Any;
use std::backtrace::Backtrace;
use std::fs;
use std::io;
use std::io::Write;
use std::io::{Read, Seek};
use std::path::{Path, PathBuf};

Expand All @@ -17,37 +16,37 @@ pub use crate::format::Digest;
use crate::oci::media_types::{PuzzleFSMediaType, PUZZLEFS_ROOTFS, VERITY_ROOT_HASH_ANNOTATION};
use ocidir::oci_spec::image;
pub use ocidir::oci_spec::image::Descriptor;
use ocidir::oci_spec::image::{
DescriptorBuilder, ImageIndex, ImageManifest, ImageManifestBuilder, MediaType, Sha256Digest,
};
use ocidir::oci_spec::OciSpecError;
use ocidir::oci_spec::image::{ImageIndex, ImageManifest, MediaType};
use ocidir::OciDir;
use std::collections::HashMap;
use std::str::FromStr;

use std::io::Cursor;

pub mod media_types;
const OCI_TAG_ANNOTATION: &str = "org.opencontainers.image.ref.name";

pub struct Image(pub OciDir);

impl Image {
pub fn new(oci_dir: &Path) -> Result<Self> {
fs::create_dir_all(oci_dir)?;
let d = cap_std::fs::Dir::open_ambient_dir(oci_dir, cap_std::ambient_authority())?;
let oci_dir = OciDir::ensure(&d)?;
let oci_dir = OciDir::ensure(d)?;

Ok(Self(oci_dir))
}

pub fn open(oci_dir: &Path) -> Result<Self> {
let d = cap_std::fs::Dir::open_ambient_dir(oci_dir, cap_std::ambient_authority())?;
let oci_dir = OciDir::open(&d)?;
let blobs_dir = cap_std::fs::Dir::open_ambient_dir(
oci_dir.join(Self::blob_path()),
cap_std::ambient_authority(),
)?;
let oci_dir = OciDir::open_with_external_blobs(d, blobs_dir)?;
Ok(Self(oci_dir))
}

pub fn blob_path(&self) -> PathBuf {
pub fn blob_path() -> PathBuf {
// TODO: use BLOBDIR constant from ocidir after making it public
PathBuf::from("blobs/sha256")
}
Expand All @@ -72,6 +71,7 @@ impl Image {
let uncompressed_size = io::copy(&mut <&[u8]>::clone(&buf), &mut compressed)?;
compressed.end()?;
let compressed_size = compressed_data.get_ref().len() as u64;
let final_size = std::cmp::min(compressed_size, uncompressed_size);

// store the uncompressed blob if the compressed version has bigger size
let final_data = if compressed_blob && compressed_size >= uncompressed_size {
Expand All @@ -90,7 +90,7 @@ impl Image {
let fs_verity_digest = get_fs_verity_digest(&compressed_data.get_ref()[..])?;
let mut descriptor = Descriptor::new(
MediaType::Other(media_type_with_extension),
uncompressed_size,
final_size,
image::Digest::from_str(&digest_string)?,
);
// We need to store the PuzzleFS Rootfs verity digest as an annotation (obviously we cannot
Expand All @@ -103,12 +103,12 @@ impl Image {
);
descriptor.set_annotations(Some(annotations));
}
let path = self.blob_path().join(descriptor.digest().digest());
let path = Self::blob_path().join(descriptor.digest().digest());

// avoid replacing the data blob so we don't drop fsverity data
if self.0.dir.exists(&path) {
if self.0.dir().exists(&path) {
let mut hasher = Sha256::new();
let mut file = self.0.dir.open(&path)?;
let mut file = self.0.dir().open(&path)?;
io::copy(&mut file, &mut hasher)?;
let existing_digest = hasher.finalize();
if existing_digest != digest {
Expand All @@ -120,7 +120,7 @@ impl Image {
.into());
}
} else {
self.0.dir.write(&path, final_data)?;
self.0.dir().write(&path, final_data)?;
}

// Let's make the PuzzleFS image rootfs the first layer so it's easy to find
Expand All @@ -136,7 +136,7 @@ impl Image {
}

fn open_raw_blob(&self, digest: &str, verity: Option<&[u8]>) -> io::Result<cap_std::fs::File> {
let file = self.0.dir.open(self.blob_path().join(digest))?;
let file = self.0.blobs_dir().open(digest)?;
if let Some(verity) = verity {
check_fs_verity(&file, verity).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
}
Expand Down Expand Up @@ -201,21 +201,10 @@ impl Image {
Ok(file)
}

// TODO: export this function from ocidr / find another way to avoid code duplication
fn descriptor_is_tagged(d: &Descriptor, tag: &str) -> bool {
d.annotations()
.as_ref()
.and_then(|annos| annos.get(OCI_TAG_ANNOTATION))
.filter(|tagval| tagval.as_str() == tag)
.is_some()
}

pub fn get_image_manifest_fd(&self, tag: &str) -> Result<cap_std::fs::File> {
let index = self.get_index()?;
let image_manifest = index
.manifests()
.iter()
.find(|desc| Self::descriptor_is_tagged(desc, tag))
let image_manifest = self
.0
.find_manifest_descriptor_with_tag(tag)?
.ok_or_else(|| {
WireFormatError::MissingManifest(tag.to_string(), Backtrace::capture())
})?;
Expand Down Expand Up @@ -270,39 +259,11 @@ impl Image {
}

pub fn get_index(&self) -> Result<ImageIndex> {
Ok(self
.0
.read_index()?
.ok_or_else(|| OciSpecError::Other("missing OCI index".to_string()))?)
Ok(self.0.read_index()?)
}

pub fn get_empty_manifest(&self) -> Result<ImageManifest> {
// see https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidance-for-an-empty-descriptor
let config = DescriptorBuilder::default()
.media_type(MediaType::EmptyJSON)
.size(2_u32)
.digest(Sha256Digest::from_str(
"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
)?)
.data("e30=")
.build()?;

if !self.0.dir.exists(
self.blob_path()
.join("44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"),
) {
let mut blob = self.0.create_blob()?;
blob.write_all("{}".as_bytes())?;
// TODO: blob.complete_verified_as(&config)? once https://github.com/containers/ocidir-rs/pull/18 is merged
blob.complete()?;
}

let image_manifest = ImageManifestBuilder::default()
.schema_version(2_u32)
.config(config)
.layers(Vec::new())
.build()?;
Ok(image_manifest)
Ok(self.0.new_empty_manifest()?.build()?)
}
}

Expand Down Expand Up @@ -330,8 +291,8 @@ mod tests {

let md = image
.0
.dir
.symlink_metadata(image.blob_path().join(DIGEST))?;
.dir()
.symlink_metadata(Image::blob_path().join(DIGEST))?;
assert!(md.is_file());
Ok(())
}
Expand Down

0 comments on commit 40d0371

Please sign in to comment.