Skip to content

Commit 7925931

Browse files
authored
feat: Use reflink_or_copy for export as well (#137)
## Description Use reflink_or_copy when exporing, not just when importing. ## Breaking Changes None ## Notes & open questions Note: we lose progress, but for modern file systems it will be instant, so 🤷 . If you are a poor person having to live with Fat32, you can always use TryReference. ## Change checklist - [ ] Self-review. - [ ] Documentation updates following the [style guide](https://rust-lang.github.io/rfcs/1574-more-api-documentation-conventions.html#appendix-a-full-conventions-text), if relevant. - [ ] Tests if relevant. - [ ] All breaking changes documented.
1 parent ee3e710 commit 7925931

File tree

3 files changed

+58
-14
lines changed

3 files changed

+58
-14
lines changed

Cargo.lock

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/store/fs.rs

Lines changed: 49 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ use entity_manager::{EntityManagerState, SpawnArg};
9393
use entry_state::{DataLocation, OutboardLocation};
9494
use gc::run_gc;
9595
use import::{ImportEntry, ImportSource};
96-
use irpc::channel::mpsc;
96+
use irpc::{channel::mpsc, RpcMessage};
9797
use meta::list_blobs;
9898
use n0_future::{future::yield_now, io};
9999
use nested_enum_utils::enum_conversions;
@@ -1263,9 +1263,12 @@ async fn export_path_impl(
12631263
}
12641264
MemOrFile::File((source_path, size)) => match mode {
12651265
ExportMode::Copy => {
1266-
let source = fs::File::open(&source_path)?;
1267-
let mut target = fs::File::create(&target)?;
1268-
copy_with_progress(&source, size, &mut target, tx).await?
1266+
let res = reflink_or_copy_with_progress(&source_path, &target, size, tx).await?;
1267+
trace!(
1268+
"exported {} to {}, {res:?}",
1269+
source_path.display(),
1270+
target.display()
1271+
);
12691272
}
12701273
ExportMode::TryReference => {
12711274
match std::fs::rename(&source_path, &target) {
@@ -1295,11 +1298,50 @@ async fn export_path_impl(
12951298
Ok(())
12961299
}
12971300

1298-
async fn copy_with_progress(
1301+
trait CopyProgress: RpcMessage {
1302+
fn from_offset(offset: u64) -> Self;
1303+
}
1304+
1305+
impl CopyProgress for ExportProgressItem {
1306+
fn from_offset(offset: u64) -> Self {
1307+
ExportProgressItem::CopyProgress(offset)
1308+
}
1309+
}
1310+
1311+
impl CopyProgress for AddProgressItem {
1312+
fn from_offset(offset: u64) -> Self {
1313+
AddProgressItem::CopyProgress(offset)
1314+
}
1315+
}
1316+
1317+
#[derive(Debug)]
1318+
enum CopyResult {
1319+
Reflinked,
1320+
Copied,
1321+
}
1322+
1323+
async fn reflink_or_copy_with_progress(
1324+
from: impl AsRef<Path>,
1325+
to: impl AsRef<Path>,
1326+
size: u64,
1327+
tx: &mut mpsc::Sender<impl CopyProgress>,
1328+
) -> io::Result<CopyResult> {
1329+
let from = from.as_ref();
1330+
let to = to.as_ref();
1331+
if reflink_copy::reflink(from, to).is_ok() {
1332+
return Ok(CopyResult::Reflinked);
1333+
}
1334+
let source = fs::File::open(from)?;
1335+
let mut target = fs::File::create(to)?;
1336+
copy_with_progress(source, size, &mut target, tx).await?;
1337+
Ok(CopyResult::Copied)
1338+
}
1339+
1340+
async fn copy_with_progress<T: CopyProgress>(
12991341
file: impl ReadAt,
13001342
size: u64,
13011343
target: &mut impl Write,
1302-
tx: &mut mpsc::Sender<ExportProgressItem>,
1344+
tx: &mut mpsc::Sender<T>,
13031345
) -> io::Result<()> {
13041346
let mut offset = 0;
13051347
let mut buf = vec![0u8; 1024 * 1024];
@@ -1308,7 +1350,7 @@ async fn copy_with_progress(
13081350
let buf: &mut [u8] = &mut buf[..remaining];
13091351
file.read_exact_at(offset, buf)?;
13101352
target.write_all(buf)?;
1311-
tx.try_send(ExportProgressItem::CopyProgress(offset))
1353+
tx.try_send(T::from_offset(offset))
13121354
.await
13131355
.map_err(|_e| io::Error::other(""))?;
13141356
yield_now().await;

src/store/fs/import.rs

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ use crate::{
4343
},
4444
},
4545
store::{
46+
fs::reflink_or_copy_with_progress,
4647
util::{MemOrFile, DD},
4748
IROH_BLOCK_SIZE,
4849
},
@@ -491,11 +492,12 @@ async fn import_path_impl(
491492
let temp_path = options.path.temp_file_name();
492493
// todo: if reflink works, we don't need progress.
493494
// But if it does not, it might take a while and we won't get progress.
494-
if reflink_copy::reflink_or_copy(&path, &temp_path)?.is_none() {
495-
trace!("reflinked {} to {}", path.display(), temp_path.display());
496-
} else {
497-
trace!("copied {} to {}", path.display(), temp_path.display());
498-
}
495+
let res = reflink_or_copy_with_progress(&path, &temp_path, size, tx).await?;
496+
trace!(
497+
"imported {} to {}, {res:?}",
498+
path.display(),
499+
temp_path.display()
500+
);
499501
// copy from path to temp_path
500502
let file = OpenOptions::new().read(true).open(&temp_path)?;
501503
tx.send(AddProgressItem::CopyDone)

0 commit comments

Comments
 (0)