-
Notifications
You must be signed in to change notification settings - Fork 141
WIP: Use our container-storage for host images if the refspec exists #1601
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Draft
jmarrero
wants to merge
1
commit into
bootc-dev:main
Choose a base branch
from
jmarrero:unified-storage-wip
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Draft
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,7 +6,7 @@ use std::collections::HashSet; | |
use std::io::{BufRead, Write}; | ||
|
||
use anyhow::Ok; | ||
use anyhow::{anyhow, Context, Result}; | ||
use anyhow::{Context, Result, anyhow}; | ||
use cap_std::fs::{Dir, MetadataExt}; | ||
use cap_std_ext::cap_std; | ||
use cap_std_ext::dirext::CapStdExtDirExt; | ||
|
@@ -380,6 +380,112 @@ pub(crate) async fn prepare_for_pull( | |
Ok(PreparedPullResult::Ready(Box::new(prepared_image))) | ||
} | ||
|
||
/// Unified approach: Use bootc's CStorage to pull the image, then prepare from containers-storage. | ||
/// This reuses the same infrastructure as LBIs. | ||
pub(crate) async fn prepare_for_pull_unified( | ||
repo: &ostree::Repo, | ||
imgref: &ImageReference, | ||
target_imgref: Option<&OstreeImageReference>, | ||
store: &Storage, | ||
) -> Result<PreparedPullResult> { | ||
// Get or initialize the bootc container storage (same as used for LBIs) | ||
let imgstore = store.get_ensure_imgstore()?; | ||
|
||
let image_ref_str = format!("{imgref:#}"); | ||
|
||
// Log the original transport being used for the pull | ||
tracing::info!( | ||
"Unified pull: pulling from transport '{}' to bootc storage", | ||
&imgref.transport | ||
); | ||
|
||
// Pull the image to bootc storage using the same method as LBIs | ||
imgstore | ||
.pull(&image_ref_str, crate::podstorage::PullMode::Always) | ||
.await?; | ||
|
||
// Now create a containers-storage reference to read from bootc storage | ||
tracing::info!("Unified pull: now importing from containers-storage transport"); | ||
let containers_storage_imgref = ImageReference { | ||
transport: "containers-storage".to_string(), | ||
image: imgref.image.clone(), | ||
signature: imgref.signature.clone(), | ||
}; | ||
let ostree_imgref = OstreeImageReference::from(containers_storage_imgref); | ||
|
||
// Use the standard preparation flow but reading from containers-storage | ||
let mut imp = new_importer(repo, &ostree_imgref).await?; | ||
if let Some(target) = target_imgref { | ||
imp.set_target(target); | ||
} | ||
let prep = match imp.prepare().await? { | ||
PrepareResult::AlreadyPresent(c) => { | ||
println!("No changes in {imgref:#} => {}", c.manifest_digest); | ||
return Ok(PreparedPullResult::AlreadyPresent(Box::new((*c).into()))); | ||
} | ||
PrepareResult::Ready(p) => p, | ||
}; | ||
check_bootc_label(&prep.config); | ||
if let Some(warning) = prep.deprecated_warning() { | ||
ostree_ext::cli::print_deprecated_warning(warning).await; | ||
} | ||
ostree_ext::cli::print_layer_status(&prep); | ||
let layers_to_fetch = prep.layers_to_fetch().collect::<Result<Vec<_>>>()?; | ||
|
||
// Log that we're importing a new image from containers-storage | ||
const PULLING_NEW_IMAGE_ID: &str = "6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f1a0"; | ||
tracing::info!( | ||
message_id = PULLING_NEW_IMAGE_ID, | ||
bootc.image.reference = &imgref.image, | ||
bootc.image.transport = "containers-storage", | ||
bootc.original_transport = &imgref.transport, | ||
bootc.status = "importing_from_storage", | ||
"Importing image from bootc storage: {}", | ||
ostree_imgref | ||
); | ||
|
||
let prepared_image = PreparedImportMeta { | ||
imp, | ||
n_layers_to_fetch: layers_to_fetch.len(), | ||
layers_total: prep.all_layers().count(), | ||
bytes_to_fetch: layers_to_fetch.iter().map(|(l, _)| l.layer.size()).sum(), | ||
bytes_total: prep.all_layers().map(|l| l.layer.size()).sum(), | ||
digest: prep.manifest_digest.clone(), | ||
prep, | ||
}; | ||
|
||
Ok(PreparedPullResult::Ready(Box::new(prepared_image))) | ||
} | ||
|
||
/// Unified pull: Use podman to pull to containers-storage, then read from there | ||
pub(crate) async fn pull_unified( | ||
repo: &ostree::Repo, | ||
imgref: &ImageReference, | ||
target_imgref: Option<&OstreeImageReference>, | ||
quiet: bool, | ||
prog: ProgressWriter, | ||
store: &Storage, | ||
) -> Result<Box<ImageState>> { | ||
match prepare_for_pull_unified(repo, imgref, target_imgref, store).await? { | ||
PreparedPullResult::AlreadyPresent(existing) => { | ||
// Log that the image was already present (Debug level since it's not actionable) | ||
const IMAGE_ALREADY_PRESENT_ID: &str = "5c4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9"; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Note that if we go this route, we should also not log to the journal in the ostree pull path because otherwise we're double logging. |
||
tracing::debug!( | ||
message_id = IMAGE_ALREADY_PRESENT_ID, | ||
bootc.image.reference = &imgref.image, | ||
bootc.image.transport = &imgref.transport, | ||
bootc.status = "already_present", | ||
"Image already present: {}", | ||
imgref | ||
); | ||
Ok(existing) | ||
} | ||
PreparedPullResult::Ready(prepared_image_meta) => { | ||
pull_from_prepared(imgref, quiet, prog, *prepared_image_meta).await | ||
} | ||
} | ||
} | ||
|
||
#[context("Pulling")] | ||
pub(crate) async fn pull_from_prepared( | ||
imgref: &ImageReference, | ||
|
@@ -429,18 +535,21 @@ pub(crate) async fn pull_from_prepared( | |
let imgref_canonicalized = imgref.clone().canonicalize()?; | ||
tracing::debug!("Canonicalized image reference: {imgref_canonicalized:#}"); | ||
|
||
// Log successful import completion | ||
const IMPORT_COMPLETE_JOURNAL_ID: &str = "4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9a8"; | ||
|
||
tracing::info!( | ||
message_id = IMPORT_COMPLETE_JOURNAL_ID, | ||
bootc.image.reference = &imgref.image, | ||
bootc.image.transport = &imgref.transport, | ||
bootc.manifest_digest = import.manifest_digest.as_ref(), | ||
bootc.ostree_commit = &import.merge_commit, | ||
"Successfully imported image: {}", | ||
imgref | ||
); | ||
// Log successful import completion (skip if using unified storage to avoid double logging) | ||
let is_unified_path = imgref.transport == "containers-storage"; | ||
if !is_unified_path { | ||
const IMPORT_COMPLETE_JOURNAL_ID: &str = "4d3e2f1a0b9c8d7e6f5a4b3c2d1e0f9a8"; | ||
|
||
tracing::info!( | ||
message_id = IMPORT_COMPLETE_JOURNAL_ID, | ||
bootc.image.reference = &imgref.image, | ||
bootc.image.transport = &imgref.transport, | ||
bootc.manifest_digest = import.manifest_digest.as_ref(), | ||
bootc.ostree_commit = &import.merge_commit, | ||
"Successfully imported image: {}", | ||
imgref | ||
); | ||
} | ||
|
||
if let Some(msg) = | ||
ostree_container::store::image_filtered_content_warning(&import.filtered_files) | ||
|
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This relates to #1599 (comment)
Basically how about changing this code to take a
Storage
which would hold the persistent flag, and thencrate::deploy::pull
would itself query that flag and change its behavior.That would also implicitly then fix the install path to behave the same.