From 52f86d262d6d65bc9e33559620457c99f9a945ee Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Sat, 3 Nov 2018 15:27:29 +1100 Subject: [PATCH 01/27] Start of testing for fs.rs. --- src/Cargo.lock | 1 + src/libstd/Cargo.toml | 1 + src/libstd/tests/lib.rs | 2 + src/libstd/tests/sys/mod.rs | 2 + src/libstd/tests/sys/unix/fs.rs | 67 ++++++++++++++++++++++++++++++++ src/libstd/tests/sys/unix/mod.rs | 2 + 6 files changed, 75 insertions(+) create mode 100644 src/libstd/tests/lib.rs create mode 100644 src/libstd/tests/sys/mod.rs create mode 100644 src/libstd/tests/sys/unix/fs.rs create mode 100644 src/libstd/tests/sys/unix/mod.rs diff --git a/src/Cargo.lock b/src/Cargo.lock index 32304c81182f1..d23764caddcff 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -2679,6 +2679,7 @@ dependencies = [ "rustc_lsan 0.0.0", "rustc_msan 0.0.0", "rustc_tsan 0.0.0", + "tempfile 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "unwind 0.0.0", ] diff --git a/src/libstd/Cargo.toml b/src/libstd/Cargo.toml index 2b1d515c83b75..554aa7b65d20e 100644 --- a/src/libstd/Cargo.toml +++ b/src/libstd/Cargo.toml @@ -24,6 +24,7 @@ unwind = { path = "../libunwind" } [dev-dependencies] rand = "0.5" +tempfile = "3" [target.x86_64-apple-darwin.dependencies] rustc_asan = { path = "../librustc_asan" } diff --git a/src/libstd/tests/lib.rs b/src/libstd/tests/lib.rs new file mode 100644 index 0000000000000..e1b64941f1847 --- /dev/null +++ b/src/libstd/tests/lib.rs @@ -0,0 +1,2 @@ + +mod sys; diff --git a/src/libstd/tests/sys/mod.rs b/src/libstd/tests/sys/mod.rs new file mode 100644 index 0000000000000..e454fbfd10884 --- /dev/null +++ b/src/libstd/tests/sys/mod.rs @@ -0,0 +1,2 @@ + +mod unix; diff --git a/src/libstd/tests/sys/unix/fs.rs b/src/libstd/tests/sys/unix/fs.rs new file mode 100644 index 0000000000000..17cf3989e4b28 --- /dev/null +++ b/src/libstd/tests/sys/unix/fs.rs @@ -0,0 +1,67 @@ + +use std::fs::OpenOptions; +use std::io::{Seek, SeekFrom, Read, Write, Result}; +use std::path::PathBuf; + + +#[cfg(all(test, any(target_os = "linux", target_os = "android")))] +mod test_linux { + use super::*; + use std::process::{Command, Output}; + + fn create_sparse(file: &PathBuf, head: u64, tail: u64) -> Result { + let data = "c00lc0d3"; + let len = 4096u64 * 4096 + data.len() as u64 + tail; + + let out = Command::new("truncate") + .args(&["-s", len.to_string().as_str(), + file.to_str().unwrap()]) + .output()?; + assert!(out.status.success()); + + let mut fd = OpenOptions::new() + .write(true) + .append(false) + .open(&file)?; + + fd.seek(SeekFrom::Start(head))?; + write!(fd, "{}", data); + + fd.seek(SeekFrom::Start(1024*4096))?; + write!(fd, "{}", data); + + fd.seek(SeekFrom::Start(4096*4096))?; + write!(fd, "{}", data); + + Ok(len as u64) + } + + fn quickstat(file: &PathBuf) -> Result<(i32, i32, i32)> { + let out = Command::new("stat") + .args(&["--format", "%s %b %B", + file.to_str().unwrap()]) + .output()?; + assert!(out.status.success()); + + let stdout = String::from_utf8(out.stdout).unwrap(); + let stats = stdout + .split_whitespace() + .map(|s| s.parse::().unwrap()) + .collect::>(); + let (size, blocks, blksize) = (stats[0], stats[1], stats[2]); + + Ok((size, blocks, blksize)) + } + + fn probably_sparse(file: &PathBuf) -> Result { + let (size, blocks, blksize) = quickstat(file)?; + + Ok(blocks < size / blksize) + } + + + #[test] + fn test_tests() { + assert!(true); + } +} diff --git a/src/libstd/tests/sys/unix/mod.rs b/src/libstd/tests/sys/unix/mod.rs new file mode 100644 index 0000000000000..ac21831582800 --- /dev/null +++ b/src/libstd/tests/sys/unix/mod.rs @@ -0,0 +1,2 @@ + +mod fs; From 2e7931b4b5ba6a636e17b6e38c23d13f5c675e32 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Sat, 3 Nov 2018 15:57:43 +1100 Subject: [PATCH 02/27] Add failing test of sparse copy. --- src/libstd/tests/sys/unix/fs.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/libstd/tests/sys/unix/fs.rs b/src/libstd/tests/sys/unix/fs.rs index 17cf3989e4b28..9390f5993e746 100644 --- a/src/libstd/tests/sys/unix/fs.rs +++ b/src/libstd/tests/sys/unix/fs.rs @@ -1,8 +1,9 @@ -use std::fs::OpenOptions; +use std::fs::{copy, OpenOptions}; use std::io::{Seek, SeekFrom, Read, Write, Result}; use std::path::PathBuf; - +extern crate tempfile; +use self::tempfile::tempdir; #[cfg(all(test, any(target_os = "linux", target_os = "android")))] mod test_linux { @@ -64,4 +65,22 @@ mod test_linux { fn test_tests() { assert!(true); } + + #[test] + fn test_sparse() { + let dir = tempdir().unwrap(); + let from = dir.path().join("sparse.bin"); + let to = dir.path().join("target.bin"); + + let slen = create_sparse(&from, 0, 0).unwrap(); + assert_eq!(slen, from.metadata().unwrap().len()); + assert!(probably_sparse(&from).unwrap()); + + let written = copy(&from, &to).unwrap(); + assert_eq!(slen, written); + assert!(probably_sparse(&to).unwrap()); + + } + + } From 822760a3ef19023cda39b8ab2d55de1c80719cc9 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Sat, 3 Nov 2018 16:18:26 +1100 Subject: [PATCH 03/27] Add some more test cases. --- src/libstd/tests/sys/unix/fs.rs | 68 ++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/src/libstd/tests/sys/unix/fs.rs b/src/libstd/tests/sys/unix/fs.rs index 9390f5993e746..28161687d51c9 100644 --- a/src/libstd/tests/sys/unix/fs.rs +++ b/src/libstd/tests/sys/unix/fs.rs @@ -1,5 +1,5 @@ -use std::fs::{copy, OpenOptions}; +use std::fs::{copy, read, OpenOptions}; use std::io::{Seek, SeekFrom, Read, Write, Result}; use std::path::PathBuf; extern crate tempfile; @@ -80,7 +80,73 @@ mod test_linux { assert_eq!(slen, written); assert!(probably_sparse(&to).unwrap()); + let from_data = read(&from).unwrap(); + let to_data = read(&to).unwrap(); + assert_eq!(from_data, to_data); } + #[test] + fn test_sparse_leading_gap() { + let dir = tempdir().unwrap(); + let from = dir.path().join("sparse.bin"); + let to = dir.path().join("target.bin"); + + let slen = create_sparse(&from, 1024, 0).unwrap(); + assert_eq!(slen, from.metadata().unwrap().len()); + assert!(probably_sparse(&from).unwrap()); + + let written = copy(&from, &to).unwrap(); + assert_eq!(slen, written); + assert!(probably_sparse(&to).unwrap()); + + assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); + + let from_data = read(&from).unwrap(); + let to_data = read(&to).unwrap(); + assert_eq!(from_data, to_data); + } + + #[test] + fn test_sparse_trailng_gap() { + let dir = tempdir().unwrap(); + let from = dir.path().join("sparse.bin"); + let to = dir.path().join("target.bin"); + + let slen = create_sparse(&from, 1024, 1024).unwrap(); + assert_eq!(slen, from.metadata().unwrap().len()); + assert!(probably_sparse(&from).unwrap()); + + let written = copy(&from, &to).unwrap(); + assert_eq!(slen, written); + assert!(probably_sparse(&to).unwrap()); + assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); + + let from_data = read(&from).unwrap(); + let to_data = read(&to).unwrap(); + assert_eq!(from_data, to_data); + } + + #[test] + fn test_empty_sparse() { + let dir = tempdir().unwrap(); + let from = dir.path().join("sparse.bin"); + let to = dir.path().join("target.bin"); + + let out = Command::new("/usr/bin/truncate") + .args(&["-s", "1M", from.to_str().unwrap()]) + .output().unwrap(); + assert!(out.status.success()); + assert_eq!(from.metadata().unwrap().len(), 1024*1024); + + let written = copy(&from, &to).unwrap(); + assert_eq!(to.metadata().unwrap().len(), 1024*1024); + + assert!(probably_sparse(&to).unwrap()); + assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); + + let from_data = read(&from).unwrap(); + let to_data = read(&to).unwrap(); + assert_eq!(from_data, to_data); + } } From 00e8db014b70aff24f75902ceb7fa57367f1d16c Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Sun, 4 Nov 2018 19:06:02 +1100 Subject: [PATCH 04/27] Minor test cleanups. --- src/libstd/tests/sys/unix/fs.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libstd/tests/sys/unix/fs.rs b/src/libstd/tests/sys/unix/fs.rs index 28161687d51c9..54b80fb63932c 100644 --- a/src/libstd/tests/sys/unix/fs.rs +++ b/src/libstd/tests/sys/unix/fs.rs @@ -1,6 +1,6 @@ use std::fs::{copy, read, OpenOptions}; -use std::io::{Seek, SeekFrom, Read, Write, Result}; +use std::io::{SeekFrom, Result}; use std::path::PathBuf; extern crate tempfile; use self::tempfile::tempdir; @@ -8,7 +8,8 @@ use self::tempfile::tempdir; #[cfg(all(test, any(target_os = "linux", target_os = "android")))] mod test_linux { use super::*; - use std::process::{Command, Output}; + use std::io::{Seek, Write}; + use std::process::Command; fn create_sparse(file: &PathBuf, head: u64, tail: u64) -> Result { let data = "c00lc0d3"; @@ -138,7 +139,7 @@ mod test_linux { assert!(out.status.success()); assert_eq!(from.metadata().unwrap().len(), 1024*1024); - let written = copy(&from, &to).unwrap(); + let _written = copy(&from, &to).unwrap(); assert_eq!(to.metadata().unwrap().len(), 1024*1024); assert!(probably_sparse(&to).unwrap()); From e838cbd81b7214644bd6ed4d3d4977783764b486 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 5 Nov 2018 08:02:48 +1100 Subject: [PATCH 05/27] Add sparse detection routine. --- src/libstd/sys/unix/fs.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libstd/sys/unix/fs.rs b/src/libstd/sys/unix/fs.rs index add06aec11b64..b337ab4ba460e 100644 --- a/src/libstd/sys/unix/fs.rs +++ b/src/libstd/sys/unix/fs.rs @@ -879,6 +879,12 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { ) } + fn is_sparse(fd: &File) -> io::Result { + let mut stat: libc::stat = unsafe { mem::uninitialized() }; + cvt(unsafe { libc::fstat(fd.as_raw_fd(), &mut stat) })?; + Ok(stat.st_blocks < stat.st_size / stat.st_blksize) + } + if !from.is_file() { return Err(Error::new(ErrorKind::InvalidInput, "the source path is not an existing regular file")) @@ -890,6 +896,7 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { let metadata = reader.metadata()?; (metadata.permissions(), metadata.size()) }; + let _sparse = is_sparse(&reader)?; let has_copy_file_range = HAS_COPY_FILE_RANGE.load(Ordering::Relaxed); let mut written = 0u64; From abc69fed8d1a4ff2fc306ed51f807627e7ab2ea3 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 6 Nov 2018 12:31:49 +1100 Subject: [PATCH 06/27] Add test of current simple file copy before refactoring. --- src/libstd/tests/sys/unix/fs.rs | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/libstd/tests/sys/unix/fs.rs b/src/libstd/tests/sys/unix/fs.rs index 54b80fb63932c..8bf254c7261c1 100644 --- a/src/libstd/tests/sys/unix/fs.rs +++ b/src/libstd/tests/sys/unix/fs.rs @@ -1,5 +1,5 @@ -use std::fs::{copy, read, OpenOptions}; +use std::fs::{copy, read, File, OpenOptions}; use std::io::{SeekFrom, Result}; use std::path::PathBuf; extern crate tempfile; @@ -27,13 +27,13 @@ mod test_linux { .open(&file)?; fd.seek(SeekFrom::Start(head))?; - write!(fd, "{}", data); + write!(fd, "{}", data).unwrap(); fd.seek(SeekFrom::Start(1024*4096))?; - write!(fd, "{}", data); + write!(fd, "{}", data).unwrap(); fd.seek(SeekFrom::Start(4096*4096))?; - write!(fd, "{}", data); + write!(fd, "{}", data).unwrap(); Ok(len as u64) } @@ -64,7 +64,22 @@ mod test_linux { #[test] fn test_tests() { - assert!(true); + let dir = tempdir().unwrap(); + let from = dir.path().join("source.txt"); + let to = dir.path().join("dest.txt"); + let text = "This is a test file."; + + { + let file = File::create(&from).unwrap(); + write!(&file, "{}", text).unwrap(); + } + + let written = copy(&from, &to).unwrap(); + assert_eq!(text.len() as u64, written); + + let from_data = read(&from).unwrap(); + let to_data = read(&to).unwrap(); + assert_eq!(from_data, to_data); } #[test] From b5eef917d38f32f8780a4331ddbdcdcf7af6b4ed Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 6 Nov 2018 13:06:31 +1100 Subject: [PATCH 07/27] Disable (deliberatly) non-working tests for now. --- src/libstd/tests/sys/unix/fs.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstd/tests/sys/unix/fs.rs b/src/libstd/tests/sys/unix/fs.rs index 8bf254c7261c1..4080603d8cc47 100644 --- a/src/libstd/tests/sys/unix/fs.rs +++ b/src/libstd/tests/sys/unix/fs.rs @@ -63,7 +63,7 @@ mod test_linux { #[test] - fn test_tests() { + fn test_simple_copy() { let dir = tempdir().unwrap(); let from = dir.path().join("source.txt"); let to = dir.path().join("dest.txt"); @@ -82,7 +82,7 @@ mod test_linux { assert_eq!(from_data, to_data); } - #[test] + //#[test] fn test_sparse() { let dir = tempdir().unwrap(); let from = dir.path().join("sparse.bin"); @@ -101,7 +101,7 @@ mod test_linux { assert_eq!(from_data, to_data); } - #[test] + //#[test] fn test_sparse_leading_gap() { let dir = tempdir().unwrap(); let from = dir.path().join("sparse.bin"); @@ -122,7 +122,7 @@ mod test_linux { assert_eq!(from_data, to_data); } - #[test] + //#[test] fn test_sparse_trailng_gap() { let dir = tempdir().unwrap(); let from = dir.path().join("sparse.bin"); @@ -142,7 +142,7 @@ mod test_linux { assert_eq!(from_data, to_data); } - #[test] + //#[test] fn test_empty_sparse() { let dir = tempdir().unwrap(); let from = dir.path().join("sparse.bin"); From c0d936a2f23d66662a34ac105db1b577841748f6 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 6 Nov 2018 13:34:53 +1100 Subject: [PATCH 08/27] Refactor copy_file_range() call to enable copying of chunks. --- src/libstd/sys/unix/fs.rs | 53 ++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/src/libstd/sys/unix/fs.rs b/src/libstd/sys/unix/fs.rs index b337ab4ba460e..a759bf749b1de 100644 --- a/src/libstd/sys/unix/fs.rs +++ b/src/libstd/sys/unix/fs.rs @@ -879,6 +879,37 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { ) } + // Version of copy_file_range(2) that copies the give range to the + // same place in the target file. If off is None then use nul to + // tell copy_file_range() track the file offset. See the manpage + // for details. + fn copy_file_chunk(reader: &File, writer: &File, off: Option, bytes_to_copy: usize) -> io::Result { + let mut off_val = off.unwrap_or(0); + let copy_result = unsafe { + let off_ptr = if off.is_some() { + &mut off_val as *mut i64 + } else { + ptr::null_mut() + }; + cvt(copy_file_range(reader.as_raw_fd(), + off_ptr, + writer.as_raw_fd(), + off_ptr, + bytes_to_copy, + 0) + ) + }; + if let Err(ref copy_err) = copy_result { + match copy_err.raw_os_error() { + Some(libc::ENOSYS) | Some(libc::EPERM) => { + HAS_COPY_FILE_RANGE.store(false, Ordering::Relaxed); + } + _ => {} + } + } + copy_result + } + fn is_sparse(fd: &File) -> io::Result { let mut stat: libc::stat = unsafe { mem::uninitialized() }; cvt(unsafe { libc::fstat(fd.as_raw_fd(), &mut stat) })?; @@ -903,26 +934,8 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { while written < len { let copy_result = if has_copy_file_range { let bytes_to_copy = cmp::min(len - written, usize::max_value() as u64) as usize; - let copy_result = unsafe { - // We actually don't have to adjust the offsets, - // because copy_file_range adjusts the file offset automatically - cvt(copy_file_range(reader.as_raw_fd(), - ptr::null_mut(), - writer.as_raw_fd(), - ptr::null_mut(), - bytes_to_copy, - 0) - ) - }; - if let Err(ref copy_err) = copy_result { - match copy_err.raw_os_error() { - Some(libc::ENOSYS) | Some(libc::EPERM) => { - HAS_COPY_FILE_RANGE.store(false, Ordering::Relaxed); - } - _ => {} - } - } - copy_result + copy_file_chunk(&reader, &writer, None, bytes_to_copy) + } else { Err(io::Error::from_raw_os_error(libc::ENOSYS)) }; From e6b19ec49a5c81f01fcaac63f9a5f809ba06ee2c Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 7 Nov 2018 10:58:59 +1100 Subject: [PATCH 09/27] Break Linux copy function out to own mod for sparse refactoring. --- src/libstd/sys/unix/fs.rs | 116 +------------------- src/libstd/sys/unix/fs_linux.rs | 184 ++++++++++++++++++++++++++++++++ src/libstd/sys/unix/mod.rs | 2 + 3 files changed, 189 insertions(+), 113 deletions(-) create mode 100644 src/libstd/sys/unix/fs_linux.rs diff --git a/src/libstd/sys/unix/fs.rs b/src/libstd/sys/unix/fs.rs index a759bf749b1de..5663268a6e617 100644 --- a/src/libstd/sys/unix/fs.rs +++ b/src/libstd/sys/unix/fs.rs @@ -12,7 +12,7 @@ use os::unix::prelude::*; use ffi::{CString, CStr, OsString, OsStr}; use fmt; -use io::{self, Error, ErrorKind, SeekFrom}; +use io::{self, Error, SeekFrom}; use libc::{self, c_int, mode_t}; use mem; use path::{Path, PathBuf}; @@ -850,116 +850,6 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { Ok(ret) } -#[cfg(any(target_os = "linux", target_os = "android"))] -pub fn copy(from: &Path, to: &Path) -> io::Result { - use cmp; - use fs::File; - use sync::atomic::{AtomicBool, Ordering}; - - // Kernel prior to 4.5 don't have copy_file_range - // We store the availability in a global to avoid unnecessary syscalls - static HAS_COPY_FILE_RANGE: AtomicBool = AtomicBool::new(true); - - unsafe fn copy_file_range( - fd_in: libc::c_int, - off_in: *mut libc::loff_t, - fd_out: libc::c_int, - off_out: *mut libc::loff_t, - len: libc::size_t, - flags: libc::c_uint, - ) -> libc::c_long { - libc::syscall( - libc::SYS_copy_file_range, - fd_in, - off_in, - fd_out, - off_out, - len, - flags, - ) - } - - // Version of copy_file_range(2) that copies the give range to the - // same place in the target file. If off is None then use nul to - // tell copy_file_range() track the file offset. See the manpage - // for details. - fn copy_file_chunk(reader: &File, writer: &File, off: Option, bytes_to_copy: usize) -> io::Result { - let mut off_val = off.unwrap_or(0); - let copy_result = unsafe { - let off_ptr = if off.is_some() { - &mut off_val as *mut i64 - } else { - ptr::null_mut() - }; - cvt(copy_file_range(reader.as_raw_fd(), - off_ptr, - writer.as_raw_fd(), - off_ptr, - bytes_to_copy, - 0) - ) - }; - if let Err(ref copy_err) = copy_result { - match copy_err.raw_os_error() { - Some(libc::ENOSYS) | Some(libc::EPERM) => { - HAS_COPY_FILE_RANGE.store(false, Ordering::Relaxed); - } - _ => {} - } - } - copy_result - } - - fn is_sparse(fd: &File) -> io::Result { - let mut stat: libc::stat = unsafe { mem::uninitialized() }; - cvt(unsafe { libc::fstat(fd.as_raw_fd(), &mut stat) })?; - Ok(stat.st_blocks < stat.st_size / stat.st_blksize) - } - - if !from.is_file() { - return Err(Error::new(ErrorKind::InvalidInput, - "the source path is not an existing regular file")) - } - let mut reader = File::open(from)?; - let mut writer = File::create(to)?; - let (perm, len) = { - let metadata = reader.metadata()?; - (metadata.permissions(), metadata.size()) - }; - let _sparse = is_sparse(&reader)?; - - let has_copy_file_range = HAS_COPY_FILE_RANGE.load(Ordering::Relaxed); - let mut written = 0u64; - while written < len { - let copy_result = if has_copy_file_range { - let bytes_to_copy = cmp::min(len - written, usize::max_value() as u64) as usize; - copy_file_chunk(&reader, &writer, None, bytes_to_copy) - - } else { - Err(io::Error::from_raw_os_error(libc::ENOSYS)) - }; - match copy_result { - Ok(ret) => written += ret as u64, - Err(err) => { - match err.raw_os_error() { - Some(os_err) if os_err == libc::ENOSYS - || os_err == libc::EXDEV - || os_err == libc::EPERM => { - // Try fallback io::copy if either: - // - Kernel version is < 4.5 (ENOSYS) - // - Files are mounted on different fs (EXDEV) - // - copy_file_range is disallowed, for example by seccomp (EPERM) - assert_eq!(written, 0); - let ret = io::copy(&mut reader, &mut writer)?; - writer.set_permissions(perm)?; - return Ok(ret) - }, - _ => return Err(err), - } - } - } - } - writer.set_permissions(perm)?; - Ok(written) -} +#[cfg(any(target_os = "linux", target_os = "android"))] +pub use super::fs_linux::copy; diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs new file mode 100644 index 0000000000000..6b3b7e6a3f1ab --- /dev/null +++ b/src/libstd/sys/unix/fs_linux.rs @@ -0,0 +1,184 @@ + +use io::{self, Error, ErrorKind}; +use libc; +use mem; +use path::Path; +use ptr; +use sys::cvt; +use cmp; +use fs::File; +use sync::atomic::{AtomicBool, Ordering}; +use super::ext::fs::MetadataExt; +use super::ext::io::AsRawFd; + +// /// Version of copy_file_range that defers offset-management to the +// /// syscall. see copy_file_range(2) for details. +// pub fn copy_file_bytes(infd: &File, outfd: &File, bytes: u64) -> Result { +// let r = unsafe { +// ffi::copy_file_range( +// infd.as_raw_fd(), +// null_mut(), +// outfd.as_raw_fd(), +// null_mut(), +// bytes as usize, +// 0, +// ) as i64 +// }; +// result_or_errno(r, r as u64) +// } + + +// /// Copy len bytes from whereever the descriptor cursors are set. +// fn copy_range(infd: &File, outfd: &File, len: u64) -> io::Result { +// let mut written = 0u64; +// while written < len { +// let bytes_to_copy = cmp::min(len - written, updates.batch_size); +// let result = copy_file_bytes(&infd, &outfd, bytes_to_copy)?; +// written += result; +// updates.update(Ok(result))?; +// } + +// Ok(written) +// } + +// fn next_sparse_segments(fd: &File, pos: u64) -> io::Result<(u64, u64)> { +// let next_data = match lseek(fd, pos as i64, Wence::Data)? { +// SeekOff::Offset(off) => off, +// SeekOff::EOF => fd.metadata()?.len() +// }; +// let next_hole = match lseek(fd, next_data as i64, Wence::Hole)? { +// SeekOff::Offset(off) => off, +// SeekOff::EOF => fd.metadata()?.len() +// }; + +// Ok((next_data, next_hole)) +// } + +// fn copy_sparse(infd: &File, outfd: &File) -> io::Result { +// let len = infd.metadata()?.len(); +// allocate_file(&outfd, len)?; + +// let mut pos = 0; + +// while pos < len { +// let (next_data, next_hole) = next_sparse_segments(infd, pos)?; +// lseek(infd, next_data as i64, Wence::Set)?; // FIXME: EOF (but shouldn't happen) +// lseek(outfd, next_data as i64, Wence::Set)?; + +// let _written = copy_range(infd, outfd, next_hole - next_data, updates)?; +// pos = next_hole; +// } + +// Ok(len) +// } + + +pub fn copy(from: &Path, to: &Path) -> io::Result { + + // Kernel prior to 4.5 don't have copy_file_range + // We store the availability in a global to avoid unnecessary syscalls + static HAS_COPY_FILE_RANGE: AtomicBool = AtomicBool::new(true); + + unsafe fn copy_file_range( + fd_in: libc::c_int, + off_in: *mut libc::loff_t, + fd_out: libc::c_int, + off_out: *mut libc::loff_t, + len: libc::size_t, + flags: libc::c_uint, + ) -> libc::c_long { + libc::syscall( + libc::SYS_copy_file_range, + fd_in, + off_in, + fd_out, + off_out, + len, + flags, + ) + } + + // Version of copy_file_range(2) that copies the give range to the + // same place in the target file. If off is None then use nul to + // tell copy_file_range() track the file offset. See the manpage + // for details. + fn copy_file_chunk(reader: &File, writer: &File, off: Option, bytes_to_copy: usize) -> io::Result { + let mut off_val = off.unwrap_or(0); + let copy_result = unsafe { + let off_ptr = if off.is_some() { + &mut off_val as *mut i64 + } else { + ptr::null_mut() + }; + cvt(copy_file_range(reader.as_raw_fd(), + off_ptr, + writer.as_raw_fd(), + off_ptr, + bytes_to_copy, + 0) + ) + }; + if let Err(ref copy_err) = copy_result { + match copy_err.raw_os_error() { + Some(libc::ENOSYS) | Some(libc::EPERM) => { + HAS_COPY_FILE_RANGE.store(false, Ordering::Relaxed); + } + _ => {} + } + } + copy_result + } + + fn is_sparse(fd: &File) -> io::Result { + let mut stat: libc::stat = unsafe { mem::uninitialized() }; + cvt(unsafe { libc::fstat(fd.as_raw_fd(), &mut stat) })?; + Ok(stat.st_blocks < stat.st_size / stat.st_blksize) + } + + if !from.is_file() { + return Err(Error::new(ErrorKind::InvalidInput, + "the source path is not an existing regular file")) + } + + let mut reader = File::open(from)?; + let mut writer = File::create(to)?; + let (perm, len) = { + let metadata = reader.metadata()?; + (metadata.permissions(), metadata.size()) + }; + let _sparse = is_sparse(&reader)?; + + let has_copy_file_range = HAS_COPY_FILE_RANGE.load(Ordering::Relaxed); + let mut written = 0u64; + while written < len { + let copy_result = if has_copy_file_range { + let bytes_to_copy = cmp::min(len - written, usize::max_value() as u64) as usize; + copy_file_chunk(&reader, &writer, None, bytes_to_copy) + + } else { + Err(io::Error::from_raw_os_error(libc::ENOSYS)) + }; + match copy_result { + Ok(ret) => written += ret as u64, + Err(err) => { + match err.raw_os_error() { + Some(os_err) if os_err == libc::ENOSYS + || os_err == libc::EXDEV + || os_err == libc::EPERM => { + // Try fallback io::copy if either: + // - Kernel version is < 4.5 (ENOSYS) + // - Files are mounted on different fs (EXDEV) + // - copy_file_range is disallowed, for example by seccomp (EPERM) + assert_eq!(written, 0); + let ret = io::copy(&mut reader, &mut writer)?; + writer.set_permissions(perm)?; + return Ok(ret) + }, + _ => return Err(err), + } + } + } + } + writer.set_permissions(perm)?; + Ok(written) +} diff --git a/src/libstd/sys/unix/mod.rs b/src/libstd/sys/unix/mod.rs index e8101bd0bc964..d1b7f0f68bc07 100644 --- a/src/libstd/sys/unix/mod.rs +++ b/src/libstd/sys/unix/mod.rs @@ -48,6 +48,8 @@ pub mod ext; pub mod fast_thread_local; pub mod fd; pub mod fs; +#[cfg(any(target_os = "linux", target_os = "android"))] +pub mod fs_linux; pub mod memchr; pub mod mutex; #[cfg(not(target_os = "l4re"))] From d3564510016e827ede5862d89a4d34652bc7c840 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 7 Nov 2018 11:01:38 +1100 Subject: [PATCH 10/27] Move some internal fns up to root. --- src/libstd/sys/unix/fs_linux.rs | 172 +++++++++++--------------------- 1 file changed, 56 insertions(+), 116 deletions(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index 6b3b7e6a3f1ab..f038e267da6ce 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -11,129 +11,69 @@ use sync::atomic::{AtomicBool, Ordering}; use super::ext::fs::MetadataExt; use super::ext::io::AsRawFd; -// /// Version of copy_file_range that defers offset-management to the -// /// syscall. see copy_file_range(2) for details. -// pub fn copy_file_bytes(infd: &File, outfd: &File, bytes: u64) -> Result { -// let r = unsafe { -// ffi::copy_file_range( -// infd.as_raw_fd(), -// null_mut(), -// outfd.as_raw_fd(), -// null_mut(), -// bytes as usize, -// 0, -// ) as i64 -// }; -// result_or_errno(r, r as u64) -// } +// Kernel prior to 4.5 don't have copy_file_range +// We store the availability in a global to avoid unnecessary syscalls +static HAS_COPY_FILE_RANGE: AtomicBool = AtomicBool::new(true); + +unsafe fn copy_file_range( + fd_in: libc::c_int, + off_in: *mut libc::loff_t, + fd_out: libc::c_int, + off_out: *mut libc::loff_t, + len: libc::size_t, + flags: libc::c_uint, +) -> libc::c_long { + libc::syscall( + libc::SYS_copy_file_range, + fd_in, + off_in, + fd_out, + off_out, + len, + flags, + ) +} -// /// Copy len bytes from whereever the descriptor cursors are set. -// fn copy_range(infd: &File, outfd: &File, len: u64) -> io::Result { -// let mut written = 0u64; -// while written < len { -// let bytes_to_copy = cmp::min(len - written, updates.batch_size); -// let result = copy_file_bytes(&infd, &outfd, bytes_to_copy)?; -// written += result; -// updates.update(Ok(result))?; -// } - -// Ok(written) -// } - -// fn next_sparse_segments(fd: &File, pos: u64) -> io::Result<(u64, u64)> { -// let next_data = match lseek(fd, pos as i64, Wence::Data)? { -// SeekOff::Offset(off) => off, -// SeekOff::EOF => fd.metadata()?.len() -// }; -// let next_hole = match lseek(fd, next_data as i64, Wence::Hole)? { -// SeekOff::Offset(off) => off, -// SeekOff::EOF => fd.metadata()?.len() -// }; - -// Ok((next_data, next_hole)) -// } - -// fn copy_sparse(infd: &File, outfd: &File) -> io::Result { -// let len = infd.metadata()?.len(); -// allocate_file(&outfd, len)?; - -// let mut pos = 0; - -// while pos < len { -// let (next_data, next_hole) = next_sparse_segments(infd, pos)?; -// lseek(infd, next_data as i64, Wence::Set)?; // FIXME: EOF (but shouldn't happen) -// lseek(outfd, next_data as i64, Wence::Set)?; - -// let _written = copy_range(infd, outfd, next_hole - next_data, updates)?; -// pos = next_hole; -// } - -// Ok(len) -// } - - -pub fn copy(from: &Path, to: &Path) -> io::Result { - - // Kernel prior to 4.5 don't have copy_file_range - // We store the availability in a global to avoid unnecessary syscalls - static HAS_COPY_FILE_RANGE: AtomicBool = AtomicBool::new(true); - - unsafe fn copy_file_range( - fd_in: libc::c_int, - off_in: *mut libc::loff_t, - fd_out: libc::c_int, - off_out: *mut libc::loff_t, - len: libc::size_t, - flags: libc::c_uint, - ) -> libc::c_long { - libc::syscall( - libc::SYS_copy_file_range, - fd_in, - off_in, - fd_out, - off_out, - len, - flags, - ) - } - - // Version of copy_file_range(2) that copies the give range to the - // same place in the target file. If off is None then use nul to - // tell copy_file_range() track the file offset. See the manpage - // for details. - fn copy_file_chunk(reader: &File, writer: &File, off: Option, bytes_to_copy: usize) -> io::Result { - let mut off_val = off.unwrap_or(0); - let copy_result = unsafe { - let off_ptr = if off.is_some() { - &mut off_val as *mut i64 - } else { - ptr::null_mut() - }; - cvt(copy_file_range(reader.as_raw_fd(), - off_ptr, - writer.as_raw_fd(), - off_ptr, - bytes_to_copy, - 0) - ) +// Version of copy_file_range(2) that copies the give range to the +// same place in the target file. If off is None then use nul to +// tell copy_file_range() track the file offset. See the manpage +// for details. +fn copy_file_chunk(reader: &File, writer: &File, off: Option, bytes_to_copy: usize) -> io::Result { + let mut off_val = off.unwrap_or(0); + let copy_result = unsafe { + let off_ptr = if off.is_some() { + &mut off_val as *mut i64 + } else { + ptr::null_mut() }; - if let Err(ref copy_err) = copy_result { - match copy_err.raw_os_error() { - Some(libc::ENOSYS) | Some(libc::EPERM) => { - HAS_COPY_FILE_RANGE.store(false, Ordering::Relaxed); - } - _ => {} + cvt(copy_file_range(reader.as_raw_fd(), + off_ptr, + writer.as_raw_fd(), + off_ptr, + bytes_to_copy, + 0) + ) + }; + if let Err(ref copy_err) = copy_result { + match copy_err.raw_os_error() { + Some(libc::ENOSYS) | Some(libc::EPERM) => { + HAS_COPY_FILE_RANGE.store(false, Ordering::Relaxed); } + _ => {} } - copy_result } + copy_result +} - fn is_sparse(fd: &File) -> io::Result { - let mut stat: libc::stat = unsafe { mem::uninitialized() }; - cvt(unsafe { libc::fstat(fd.as_raw_fd(), &mut stat) })?; - Ok(stat.st_blocks < stat.st_size / stat.st_blksize) - } +fn is_sparse(fd: &File) -> io::Result { + let mut stat: libc::stat = unsafe { mem::uninitialized() }; + cvt(unsafe { libc::fstat(fd.as_raw_fd(), &mut stat) })?; + Ok(stat.st_blocks < stat.st_size / stat.st_blksize) +} + + +pub fn copy(from: &Path, to: &Path) -> io::Result { if !from.is_file() { return Err(Error::new(ErrorKind::InvalidInput, From 69cb12418cddefcb1fddf26eaf02a6be81eea08b Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 7 Nov 2018 14:43:57 +1100 Subject: [PATCH 11/27] Add functions to handle sparse files, with tests. Not used yet. --- src/libstd/sys/unix/fs_linux.rs | 322 +++++++++++++++++++++++++++++++- 1 file changed, 321 insertions(+), 1 deletion(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index f038e267da6ce..1dd7c3ffcca1b 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -4,7 +4,7 @@ use libc; use mem; use path::Path; use ptr; -use sys::cvt; +use sys::{cvt, cvt_r}; use cmp; use fs::File; use sync::atomic::{AtomicBool, Ordering}; @@ -12,6 +12,7 @@ use super::ext::fs::MetadataExt; use super::ext::io::AsRawFd; + // Kernel prior to 4.5 don't have copy_file_range // We store the availability in a global to avoid unnecessary syscalls static HAS_COPY_FILE_RANGE: AtomicBool = AtomicBool::new(true); @@ -35,6 +36,109 @@ unsafe fn copy_file_range( ) } +/// Corresponds to lseek(2) `wence`. This exists in std, but doesn't support sparse-files. +#[allow(dead_code)] +pub enum Wence { + Set = libc::SEEK_SET as isize, + Cur = libc::SEEK_CUR as isize, + End = libc::SEEK_END as isize, + Data = libc::SEEK_DATA as isize, + Hole = libc::SEEK_HOLE as isize, +} + +#[derive(PartialEq, Debug)] +pub enum SeekOff { + Offset(u64), + EOF +} + +pub fn lseek(fd: &File, off: i64, wence: Wence) -> io::Result { + let r = unsafe { + libc::lseek64( + fd.as_raw_fd(), + off, + wence as libc::c_int + ) + }; + + if r == -1 { + let err = io::Error::last_os_error(); + match err.raw_os_error() { + Some(errno) if errno == libc::ENXIO => { + Ok(SeekOff::EOF) + } + _ => Err(err.into()) + } + + } else { + Ok(SeekOff::Offset(r as u64)) + } + +} + +pub fn allocate_file(fd: &File, len: u64) -> io::Result<()> { + cvt_r(|| unsafe {libc::ftruncate64(fd.as_raw_fd(), len as i64)})?; + Ok(()) +} + +/// Version of copy_file_range that defers offset-management to the +/// syscall. see copy_file_range(2) for details. +pub fn copy_file_bytes(infd: &File, outfd: &File, bytes: u64) -> io::Result { + let r = cvt(unsafe { + copy_file_range( + infd.as_raw_fd(), + ptr::null_mut(), + outfd.as_raw_fd(), + ptr::null_mut(), + bytes as usize, + 0, + ) as i64 + })?; + Ok(r as u64) +} + +/// Copy len bytes from whereever the descriptor cursors are set. +fn copy_range(infd: &File, outfd: &File, len: u64) -> io::Result { + let mut written = 0u64; + while written < len { + let result = copy_file_bytes(&infd, &outfd, len - written)?; + written += result; + } + Ok(written) +} + +fn next_sparse_segments(fd: &File, pos: u64) -> io::Result<(u64, u64)> { + let next_data = match lseek(fd, pos as i64, Wence::Data)? { + SeekOff::Offset(off) => off, + SeekOff::EOF => fd.metadata()?.len() + }; + let next_hole = match lseek(fd, next_data as i64, Wence::Hole)? { + SeekOff::Offset(off) => off, + SeekOff::EOF => fd.metadata()?.len() + }; + + Ok((next_data, next_hole)) +} + +fn copy_sparse(infd: &File, outfd: &File) -> io::Result { + let len = infd.metadata()?.len(); + allocate_file(&outfd, len)?; + + let mut pos = 0; + + while pos < len { + let (next_data, next_hole) = next_sparse_segments(infd, pos)?; + lseek(infd, next_data as i64, Wence::Set)?; // FIXME: EOF (but shouldn't happen) + lseek(outfd, next_data as i64, Wence::Set)?; + + let _written = copy_range(infd, outfd, next_hole - next_data)?; + pos = next_hole; + } + + Ok(len) +} + + // Version of copy_file_range(2) that copies the give range to the // same place in the target file. If off is None then use nul to // tell copy_file_range() track the file offset. See the manpage @@ -122,3 +226,219 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { writer.set_permissions(perm)?; Ok(written) } + + +#[cfg(test)] +mod tests { + use super::*; + extern crate tempfile; + use self::tempfile::{tempdir, TempDir}; + use fs::{read, OpenOptions}; + use io::{Seek, SeekFrom, Write}; + + fn create_sparse(file: &String) { + let fd = File::create(file).unwrap(); + cvt(unsafe {libc::ftruncate64(fd.as_raw_fd(), 1024*1024)}).unwrap(); + } + + fn tmps(dir: &TempDir) -> (String, String) { + let file = dir.path().join("sparse.bin"); + let from = dir.path().join("from.txt"); + (file.to_str().unwrap().to_string(), + from.to_str().unwrap().to_string()) + } + + + #[test] + fn test_sparse_detection() { + assert!(!is_sparse(&File::open("Cargo.toml").unwrap()).unwrap()); + + let dir = tempdir().unwrap(); + let (file, _) = tmps(&dir); + create_sparse(&file); + + { + let fd = File::open(&file).unwrap(); + assert!(is_sparse(&fd).unwrap()); + } + { + let mut fd = File::open(&file).unwrap(); + write!(fd, "{}", "test"); + } + { + let fd = File::open(&file).unwrap(); + assert!(is_sparse(&fd).unwrap()); + } + } + + #[test] + fn test_copy_range_sparse() { + let dir = tempdir().unwrap(); + let (file, from) = tmps(&dir); + let data = "test data"; + + { + let mut fd = File::create(&from).unwrap(); + write!(fd, "{}", data); + } + + create_sparse(&file); + + { + let infd = File::open(&from).unwrap(); + let outfd: File = OpenOptions::new() + .write(true) + .append(false) + .open(&file).unwrap(); + copy_file_bytes(&infd, &outfd, data.len() as u64).unwrap(); + } + + assert!(is_sparse(&File::open(&file).unwrap()).unwrap()); + } + + #[test] + fn test_sparse_copy_middle() { + let dir = tempdir().unwrap(); + let (file, from) = tmps(&dir); + let data = "test data"; + + { + let mut fd = File::create(&from).unwrap(); + write!(fd, "{}", data); + } + + create_sparse(&file); + + let offset: usize = 512*1024; + { + let infd = File::open(&from).unwrap(); + let outfd: File = OpenOptions::new() + .write(true) + .append(false) + .open(&file).unwrap(); + let mut offdat: i64 = 512*1024; + let offptr = &mut offdat as *mut i64; + cvt( + unsafe { + copy_file_range( + infd.as_raw_fd(), + ptr::null_mut(), + outfd.as_raw_fd(), + offptr, + data.len(), + 0) as i64 + }).unwrap(); + } + + assert!(is_sparse(&File::open(&file).unwrap()).unwrap()); + + let bytes = read(&file).unwrap(); + assert!(bytes.len() == 1024*1024); + assert!(bytes[offset] == b't'); + assert!(bytes[offset+1] == b'e'); + assert!(bytes[offset+2] == b's'); + assert!(bytes[offset+3] == b't'); + assert!(bytes[offset+data.len()] == 0); + } + + #[test] + fn test_lseek_data() { + let dir = tempdir().unwrap(); + let (file, from) = tmps(&dir); + let data = "test data"; + let offset = 512*1024; + + { + let mut fd = File::create(&from).unwrap(); + write!(fd, "{}", data); + } + + create_sparse(&file); + + { + let infd = File::open(&from).unwrap(); + let outfd: File = OpenOptions::new() + .write(true) + .append(false) + .open(&file).unwrap(); + cvt( + unsafe { + copy_file_range( + infd.as_raw_fd(), + ptr::null_mut(), + outfd.as_raw_fd(), + &mut (offset as i64) as *mut i64, + data.len(), + 0) as i64 + }).unwrap(); + } + + assert!(is_sparse(&File::open(&file).unwrap()).unwrap()); + + let off = lseek(&File::open(&file).unwrap(), 0, Wence::Data).unwrap(); + assert_eq!(off, SeekOff::Offset(offset)); + } + + #[test] + fn test_sparse_rust_seek() { + let dir = tempdir().unwrap(); + let (file, _) = tmps(&dir); + + let data = "c00lc0d3"; + + { + let mut fd = File::create(&file).unwrap(); + write!(fd, "{}", data); + + fd.seek(SeekFrom::Start(1024*4096)).unwrap(); + write!(fd, "{}", data); + + fd.seek(SeekFrom::Start(4096*4096 - data.len() as u64)).unwrap(); + write!(fd, "{}", data); + } + + assert!(is_sparse(&File::open(&file).unwrap()).unwrap()); + + let bytes = read(&file).unwrap(); + assert!(bytes.len() == 4096*4096); + + let offset = 1024 * 4096; + assert!(bytes[offset] == b'c'); + assert!(bytes[offset+1] == b'0'); + assert!(bytes[offset+2] == b'0'); + assert!(bytes[offset+3] == b'l'); + assert!(bytes[offset+data.len()] == 0); + } + + + #[test] + fn test_lseek_no_data() { + let dir = tempdir().unwrap(); + let (file, _) = tmps(&dir); + create_sparse(&file); + + assert!(is_sparse(&File::open(&file).unwrap()).unwrap()); + + let fd = File::open(&file).unwrap(); + let off = lseek(&fd, 0, Wence::Data).unwrap(); + assert!(off == SeekOff::EOF); + } + + #[test] + fn test_allocate_file_is_sparse() { + let dir = tempdir().unwrap(); + let (file, _) = tmps(&dir); + let len = 32 * 1024 * 1024; + + { + let fd = File::create(&file).unwrap(); + allocate_file(&fd, len).unwrap(); + } + + { + let fd = File::open(&file).unwrap(); + assert_eq!(len, fd.metadata().unwrap().len()); + assert!(is_sparse(&fd).unwrap()); + } + } +} From 091eec70fc413d973834a6ff1dfd9be1a8ea09de Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 8 Nov 2018 14:53:37 +1100 Subject: [PATCH 12/27] Add tests with sparse data. --- src/libstd/sys/unix/fs_linux.rs | 48 +++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index 1dd7c3ffcca1b..24ce32ee85eaf 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -128,7 +128,7 @@ fn copy_sparse(infd: &File, outfd: &File) -> io::Result { while pos < len { let (next_data, next_hole) = next_sparse_segments(infd, pos)?; - lseek(infd, next_data as i64, Wence::Set)?; // FIXME: EOF (but shouldn't happen) + lseek(infd, next_data as i64, Wence::Set)?; lseek(outfd, next_data as i64, Wence::Set)?; let _written = copy_range(infd, outfd, next_hole - next_data)?; @@ -241,6 +241,33 @@ mod tests { cvt(unsafe {libc::ftruncate64(fd.as_raw_fd(), 1024*1024)}).unwrap(); } + fn create_sparse_with_data(file: &String, head: u64, tail: u64) -> u64 { + let data = "c00lc0d3"; + let len = 4096u64 * 4096 + data.len() as u64 + tail; + + { + let fd = File::create(file).unwrap(); + cvt(unsafe {libc::ftruncate64(fd.as_raw_fd(), len as i64)}).unwrap(); + } + + let mut fd = OpenOptions::new() + .write(true) + .append(false) + .open(&file).unwrap(); + + fd.seek(SeekFrom::Start(head)).unwrap(); + write!(fd, "{}", data); + + fd.seek(SeekFrom::Start(1024*4096)).unwrap(); + write!(fd, "{}", data); + + fd.seek(SeekFrom::Start(4096*4096)).unwrap(); + write!(fd, "{}", data); + + len + } + + fn tmps(dir: &TempDir) -> (String, String) { let file = dir.path().join("sparse.bin"); let from = dir.path().join("from.txt"); @@ -255,7 +282,7 @@ mod tests { let dir = tempdir().unwrap(); let (file, _) = tmps(&dir); - create_sparse(&file); + create_sparse_with_data(&file, 0, 0); { let fd = File::open(&file).unwrap(); @@ -384,30 +411,17 @@ mod tests { let dir = tempdir().unwrap(); let (file, _) = tmps(&dir); - let data = "c00lc0d3"; - - { - let mut fd = File::create(&file).unwrap(); - write!(fd, "{}", data); - - fd.seek(SeekFrom::Start(1024*4096)).unwrap(); - write!(fd, "{}", data); - - fd.seek(SeekFrom::Start(4096*4096 - data.len() as u64)).unwrap(); - write!(fd, "{}", data); - } - + let len = create_sparse_with_data(&file, 0, 10) as usize; assert!(is_sparse(&File::open(&file).unwrap()).unwrap()); let bytes = read(&file).unwrap(); - assert!(bytes.len() == 4096*4096); + assert!(bytes.len() == len); let offset = 1024 * 4096; assert!(bytes[offset] == b'c'); assert!(bytes[offset+1] == b'0'); assert!(bytes[offset+2] == b'0'); assert!(bytes[offset+3] == b'l'); - assert!(bytes[offset+data.len()] == 0); } From be75c6fd4805b45336e711b3642adb76b6169802 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 9 Nov 2018 09:27:36 +1100 Subject: [PATCH 13/27] Interim checkin; start moving kernel/uspace branch to lower-level fn. --- src/libstd/sys/unix/fs_linux.rs | 77 +++++++++++++++++---------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index 24ce32ee85eaf..f1d7ffa1bb0ba 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -1,5 +1,5 @@ -use io::{self, Error, ErrorKind}; +use io::{self, Error, ErrorKind, Read, Write}; use libc; use mem; use path::Path; @@ -38,7 +38,7 @@ unsafe fn copy_file_range( /// Corresponds to lseek(2) `wence`. This exists in std, but doesn't support sparse-files. #[allow(dead_code)] -pub enum Wence { +enum Wence { Set = libc::SEEK_SET as isize, Cur = libc::SEEK_CUR as isize, End = libc::SEEK_END as isize, @@ -47,12 +47,12 @@ pub enum Wence { } #[derive(PartialEq, Debug)] -pub enum SeekOff { +enum SeekOff { Offset(u64), EOF } -pub fn lseek(fd: &File, off: i64, wence: Wence) -> io::Result { +fn lseek(fd: &File, off: i64, wence: Wence) -> io::Result { let r = unsafe { libc::lseek64( fd.as_raw_fd(), @@ -76,11 +76,43 @@ pub fn lseek(fd: &File, off: i64, wence: Wence) -> io::Result { } -pub fn allocate_file(fd: &File, len: u64) -> io::Result<()> { +fn allocate_file(fd: &File, len: u64) -> io::Result<()> { cvt_r(|| unsafe {libc::ftruncate64(fd.as_raw_fd(), len as i64)})?; Ok(()) } + +// Version of copy_file_range(2) that copies the give range to the +// same place in the target file. If off is None then use nul to +// tell copy_file_range() track the file offset. See the manpage +// for details. +fn copy_bytes_kernel(reader: &File, writer: &File, nbytes: usize) -> io::Result { + let copy_result = unsafe { + cvt(copy_file_range(reader.as_raw_fd(), + ptr::null_mut(), + writer.as_raw_fd(), + ptr::null_mut(), + nbytes, + 0) + ) + } + .map(|v| v as u64); + + if let Err(ref copy_err) = copy_result { + match copy_err.raw_os_error() { + Some(libc::ENOSYS) | Some(libc::EPERM) => { + HAS_COPY_FILE_RANGE.store(false, Ordering::Relaxed); + } + _ => {} + } + } + copy_result +} + +fn copy_bytes(reader: &File, writer: &File, nbytes: usize) -> io::Result { + copy_bytes_kernel(reader, writer, nbytes) +} + /// Version of copy_file_range that defers offset-management to the /// syscall. see copy_file_range(2) for details. pub fn copy_file_bytes(infd: &File, outfd: &File, bytes: u64) -> io::Result { @@ -97,6 +129,8 @@ pub fn copy_file_bytes(infd: &File, outfd: &File, bytes: u64) -> io::Result Ok(r as u64) } + + /// Copy len bytes from whereever the descriptor cursors are set. fn copy_range(infd: &File, outfd: &File, len: u64) -> io::Result { let mut written = 0u64; @@ -139,37 +173,6 @@ fn copy_sparse(infd: &File, outfd: &File) -> io::Result { } -// Version of copy_file_range(2) that copies the give range to the -// same place in the target file. If off is None then use nul to -// tell copy_file_range() track the file offset. See the manpage -// for details. -fn copy_file_chunk(reader: &File, writer: &File, off: Option, bytes_to_copy: usize) -> io::Result { - let mut off_val = off.unwrap_or(0); - let copy_result = unsafe { - let off_ptr = if off.is_some() { - &mut off_val as *mut i64 - } else { - ptr::null_mut() - }; - cvt(copy_file_range(reader.as_raw_fd(), - off_ptr, - writer.as_raw_fd(), - off_ptr, - bytes_to_copy, - 0) - ) - }; - if let Err(ref copy_err) = copy_result { - match copy_err.raw_os_error() { - Some(libc::ENOSYS) | Some(libc::EPERM) => { - HAS_COPY_FILE_RANGE.store(false, Ordering::Relaxed); - } - _ => {} - } - } - copy_result -} - fn is_sparse(fd: &File) -> io::Result { let mut stat: libc::stat = unsafe { mem::uninitialized() }; cvt(unsafe { libc::fstat(fd.as_raw_fd(), &mut stat) })?; @@ -197,7 +200,7 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { while written < len { let copy_result = if has_copy_file_range { let bytes_to_copy = cmp::min(len - written, usize::max_value() as u64) as usize; - copy_file_chunk(&reader, &writer, None, bytes_to_copy) + copy_bytes(&reader, &writer, bytes_to_copy) } else { Err(io::Error::from_raw_os_error(libc::ENOSYS)) From dfc92e5963d32ee273b877bfc83a22ce5c7fc104 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 9 Nov 2018 13:36:34 +1100 Subject: [PATCH 14/27] Move flag of copy_file_range to thread-local, and move the decision to its own fn. --- src/libstd/sys/unix/fs_linux.rs | 70 ++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index f1d7ffa1bb0ba..531262efc48b1 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -1,4 +1,5 @@ +use cell::RefCell; use io::{self, Error, ErrorKind, Read, Write}; use libc; use mem; @@ -13,10 +14,6 @@ use super::ext::io::AsRawFd; -// Kernel prior to 4.5 don't have copy_file_range -// We store the availability in a global to avoid unnecessary syscalls -static HAS_COPY_FILE_RANGE: AtomicBool = AtomicBool::new(true); - unsafe fn copy_file_range( fd_in: libc::c_int, off_in: *mut libc::loff_t, @@ -87,7 +84,7 @@ fn allocate_file(fd: &File, len: u64) -> io::Result<()> { // tell copy_file_range() track the file offset. See the manpage // for details. fn copy_bytes_kernel(reader: &File, writer: &File, nbytes: usize) -> io::Result { - let copy_result = unsafe { + unsafe { cvt(copy_file_range(reader.as_raw_fd(), ptr::null_mut(), writer.as_raw_fd(), @@ -96,21 +93,40 @@ fn copy_bytes_kernel(reader: &File, writer: &File, nbytes: usize) -> io::Result< 0) ) } - .map(|v| v as u64); + .map(|v| v as u64) +} - if let Err(ref copy_err) = copy_result { - match copy_err.raw_os_error() { - Some(libc::ENOSYS) | Some(libc::EPERM) => { - HAS_COPY_FILE_RANGE.store(false, Ordering::Relaxed); - } - _ => {} - } - } - copy_result + +// Kernel prior to 4.5 don't have copy_file_range We store the +// availability in a thread-local flag to avoid unnecessary syscalls +thread_local! { + static HAS_COPY_FILE_RANGE: RefCell = RefCell::new(true); } -fn copy_bytes(reader: &File, writer: &File, nbytes: usize) -> io::Result { - copy_bytes_kernel(reader, writer, nbytes) +fn copy_bytes(mut reader: &File, mut writer: &File, nbytes: usize) -> io::Result { + HAS_COPY_FILE_RANGE.with(|cfr| { + loop { + if !*cfr.borrow() { + return io::copy(&mut reader, &mut writer); + + } else { + let result = copy_bytes_kernel(reader, writer, nbytes); + + if let Err(ref err) = result { + match err.raw_os_error() { + Some(libc::ENOSYS) | Some(libc::EPERM) => { + // Flag as missing and retry. + *cfr.borrow_mut() = false; + continue; + } + _ => {} + + } + } + return result; + } + } + }) } /// Version of copy_file_range that defers offset-management to the @@ -195,27 +211,17 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { }; let _sparse = is_sparse(&reader)?; - let has_copy_file_range = HAS_COPY_FILE_RANGE.load(Ordering::Relaxed); let mut written = 0u64; while written < len { - let copy_result = if has_copy_file_range { - let bytes_to_copy = cmp::min(len - written, usize::max_value() as u64) as usize; - copy_bytes(&reader, &writer, bytes_to_copy) + let bytes_to_copy = cmp::min(len - written, usize::max_value() as u64) as usize; + let copy_result = copy_bytes_kernel(&mut reader, &mut writer, bytes_to_copy); - } else { - Err(io::Error::from_raw_os_error(libc::ENOSYS)) - }; match copy_result { - Ok(ret) => written += ret as u64, + Ok(ret) => written += ret, Err(err) => { match err.raw_os_error() { - Some(os_err) if os_err == libc::ENOSYS - || os_err == libc::EXDEV - || os_err == libc::EPERM => { - // Try fallback io::copy if either: - // - Kernel version is < 4.5 (ENOSYS) - // - Files are mounted on different fs (EXDEV) - // - copy_file_range is disallowed, for example by seccomp (EPERM) + Some(os_err) if os_err == libc::EXDEV => { + // Files are mounted on different fs (EXDEV); try fallback io::copy. assert_eq!(written, 0); let ret = io::copy(&mut reader, &mut writer)?; writer.set_permissions(perm)?; From 84313fbe3fa1fbe0375d5392539d3441c3133edd Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 9 Nov 2018 14:01:22 +1100 Subject: [PATCH 15/27] Add override of kernel vs userspace copy for cross-device copy. --- src/libstd/sys/unix/fs_linux.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index 531262efc48b1..dc48509697bee 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -103,10 +103,10 @@ thread_local! { static HAS_COPY_FILE_RANGE: RefCell = RefCell::new(true); } -fn copy_bytes(mut reader: &File, mut writer: &File, nbytes: usize) -> io::Result { +fn copy_bytes(mut reader: &File, mut writer: &File, uspace: bool, nbytes: usize) -> io::Result { HAS_COPY_FILE_RANGE.with(|cfr| { loop { - if !*cfr.borrow() { + if uspace || !*cfr.borrow() { return io::copy(&mut reader, &mut writer); } else { @@ -211,21 +211,21 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { }; let _sparse = is_sparse(&reader)?; + let mut userspace = false; let mut written = 0u64; while written < len { let bytes_to_copy = cmp::min(len - written, usize::max_value() as u64) as usize; - let copy_result = copy_bytes_kernel(&mut reader, &mut writer, bytes_to_copy); + let copy_result = copy_bytes(&mut reader, &mut writer, userspace, bytes_to_copy); match copy_result { Ok(ret) => written += ret, Err(err) => { match err.raw_os_error() { Some(os_err) if os_err == libc::EXDEV => { - // Files are mounted on different fs (EXDEV); try fallback io::copy. + // Files are mounted on different fs (EXDEV); + // flag as retry with userspace copy. assert_eq!(written, 0); - let ret = io::copy(&mut reader, &mut writer)?; - writer.set_permissions(perm)?; - return Ok(ret) + userspace = true; }, _ => return Err(err), } From 16c4010ae405666a1b0f5bc23bfddcd88696efe0 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Sat, 10 Nov 2018 15:22:49 +1100 Subject: [PATCH 16/27] Minor cleanup of tests. --- src/libstd/sys/unix/fs_linux.rs | 80 ++++++++++++++++----------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index dc48509697bee..1b173920189a6 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -278,10 +278,10 @@ mod tests { fn tmps(dir: &TempDir) -> (String, String) { - let file = dir.path().join("sparse.bin"); - let from = dir.path().join("from.txt"); - (file.to_str().unwrap().to_string(), - from.to_str().unwrap().to_string()) + let sparse = dir.path().join("sparse.bin"); + let other = dir.path().join("other.txt"); + (sparse.to_str().unwrap().to_string(), + other.to_str().unwrap().to_string()) } @@ -290,19 +290,19 @@ mod tests { assert!(!is_sparse(&File::open("Cargo.toml").unwrap()).unwrap()); let dir = tempdir().unwrap(); - let (file, _) = tmps(&dir); - create_sparse_with_data(&file, 0, 0); + let (sparse, _) = tmps(&dir); + create_sparse_with_data(&sparse, 0, 0); { - let fd = File::open(&file).unwrap(); + let fd = File::open(&sparse).unwrap(); assert!(is_sparse(&fd).unwrap()); } { - let mut fd = File::open(&file).unwrap(); + let mut fd = File::open(&sparse).unwrap(); write!(fd, "{}", "test"); } { - let fd = File::open(&file).unwrap(); + let fd = File::open(&sparse).unwrap(); assert!(is_sparse(&fd).unwrap()); } } @@ -310,48 +310,48 @@ mod tests { #[test] fn test_copy_range_sparse() { let dir = tempdir().unwrap(); - let (file, from) = tmps(&dir); + let (sparse, other) = tmps(&dir); let data = "test data"; { - let mut fd = File::create(&from).unwrap(); + let mut fd = File::create(&other).unwrap(); write!(fd, "{}", data); } - create_sparse(&file); + create_sparse(&sparse); { - let infd = File::open(&from).unwrap(); + let infd = File::open(&other).unwrap(); let outfd: File = OpenOptions::new() .write(true) .append(false) - .open(&file).unwrap(); + .open(&sparse).unwrap(); copy_file_bytes(&infd, &outfd, data.len() as u64).unwrap(); } - assert!(is_sparse(&File::open(&file).unwrap()).unwrap()); + assert!(is_sparse(&File::open(&sparse).unwrap()).unwrap()); } #[test] fn test_sparse_copy_middle() { let dir = tempdir().unwrap(); - let (file, from) = tmps(&dir); + let (sparse, other) = tmps(&dir); let data = "test data"; { - let mut fd = File::create(&from).unwrap(); + let mut fd = File::create(&other).unwrap(); write!(fd, "{}", data); } - create_sparse(&file); + create_sparse(&sparse); let offset: usize = 512*1024; { - let infd = File::open(&from).unwrap(); + let infd = File::open(&other).unwrap(); let outfd: File = OpenOptions::new() .write(true) .append(false) - .open(&file).unwrap(); + .open(&sparse).unwrap(); let mut offdat: i64 = 512*1024; let offptr = &mut offdat as *mut i64; cvt( @@ -366,9 +366,9 @@ mod tests { }).unwrap(); } - assert!(is_sparse(&File::open(&file).unwrap()).unwrap()); + assert!(is_sparse(&File::open(&sparse).unwrap()).unwrap()); - let bytes = read(&file).unwrap(); + let bytes = read(&sparse).unwrap(); assert!(bytes.len() == 1024*1024); assert!(bytes[offset] == b't'); assert!(bytes[offset+1] == b'e'); @@ -380,23 +380,23 @@ mod tests { #[test] fn test_lseek_data() { let dir = tempdir().unwrap(); - let (file, from) = tmps(&dir); + let (sparse, other) = tmps(&dir); let data = "test data"; let offset = 512*1024; { - let mut fd = File::create(&from).unwrap(); + let mut fd = File::create(&other).unwrap(); write!(fd, "{}", data); } - create_sparse(&file); + create_sparse(&sparse); { - let infd = File::open(&from).unwrap(); + let infd = File::open(&other).unwrap(); let outfd: File = OpenOptions::new() .write(true) .append(false) - .open(&file).unwrap(); + .open(&sparse).unwrap(); cvt( unsafe { copy_file_range( @@ -409,21 +409,21 @@ mod tests { }).unwrap(); } - assert!(is_sparse(&File::open(&file).unwrap()).unwrap()); + assert!(is_sparse(&File::open(&sparse).unwrap()).unwrap()); - let off = lseek(&File::open(&file).unwrap(), 0, Wence::Data).unwrap(); + let off = lseek(&File::open(&sparse).unwrap(), 0, Wence::Data).unwrap(); assert_eq!(off, SeekOff::Offset(offset)); } #[test] fn test_sparse_rust_seek() { let dir = tempdir().unwrap(); - let (file, _) = tmps(&dir); + let (sparse, _) = tmps(&dir); - let len = create_sparse_with_data(&file, 0, 10) as usize; - assert!(is_sparse(&File::open(&file).unwrap()).unwrap()); + let len = create_sparse_with_data(&sparse, 0, 10) as usize; + assert!(is_sparse(&File::open(&sparse).unwrap()).unwrap()); - let bytes = read(&file).unwrap(); + let bytes = read(&sparse).unwrap(); assert!(bytes.len() == len); let offset = 1024 * 4096; @@ -437,12 +437,12 @@ mod tests { #[test] fn test_lseek_no_data() { let dir = tempdir().unwrap(); - let (file, _) = tmps(&dir); - create_sparse(&file); + let (sparse, _) = tmps(&dir); + create_sparse(&sparse); - assert!(is_sparse(&File::open(&file).unwrap()).unwrap()); + assert!(is_sparse(&File::open(&sparse).unwrap()).unwrap()); - let fd = File::open(&file).unwrap(); + let fd = File::open(&sparse).unwrap(); let off = lseek(&fd, 0, Wence::Data).unwrap(); assert!(off == SeekOff::EOF); } @@ -450,16 +450,16 @@ mod tests { #[test] fn test_allocate_file_is_sparse() { let dir = tempdir().unwrap(); - let (file, _) = tmps(&dir); + let (sparse, _) = tmps(&dir); let len = 32 * 1024 * 1024; { - let fd = File::create(&file).unwrap(); + let fd = File::create(&sparse).unwrap(); allocate_file(&fd, len).unwrap(); } { - let fd = File::open(&file).unwrap(); + let fd = File::open(&sparse).unwrap(); assert_eq!(len, fd.metadata().unwrap().len()); assert!(is_sparse(&fd).unwrap()); } From 054990fecad1e8376add4ea725f2a00d39c0f29d Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Sun, 11 Nov 2018 14:06:39 +1100 Subject: [PATCH 17/27] Initial cut of user-space copy range. --- src/libstd/sys/unix/fs_linux.rs | 79 +++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 3 deletions(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index 1b173920189a6..dfc57a7d2e883 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -13,7 +13,6 @@ use super::ext::fs::MetadataExt; use super::ext::io::AsRawFd; - unsafe fn copy_file_range( fd_in: libc::c_int, off_in: *mut libc::loff_t, @@ -96,6 +95,31 @@ fn copy_bytes_kernel(reader: &File, writer: &File, nbytes: usize) -> io::Result< .map(|v| v as u64) } +// Slightly modified version of io::copy() that only copies a set amount of bytes. +fn copy_bytes_uspace(mut reader: &File, mut writer: &File, nbytes: usize) -> io::Result { + let mut buf = unsafe { + // Assume 4k blocks on disk. + let mut buf: [u8; 4 * 1024] = mem::uninitialized(); + reader.initializer().initialize(&mut buf); + buf + }; + + let mut written = 0; + while written < nbytes { + let left = nbytes - written; + let len = match reader.read(&mut buf[..left]) { + Ok(0) => return Err(Error::new(ErrorKind::InvalidData, + "Source file ended prematurely.")), + Ok(len) => len, + Err(ref e) if e.kind() == ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + }; + writer.write_all(&buf[..len])?; + written += len; + } + Ok(written as u64) +} + // Kernel prior to 4.5 don't have copy_file_range We store the // availability in a thread-local flag to avoid unnecessary syscalls @@ -245,9 +269,13 @@ mod tests { use fs::{read, OpenOptions}; use io::{Seek, SeekFrom, Write}; - fn create_sparse(file: &String) { + fn create_sparse_len(file: &String, len: i64) { let fd = File::create(file).unwrap(); - cvt(unsafe {libc::ftruncate64(fd.as_raw_fd(), 1024*1024)}).unwrap(); + cvt(unsafe {libc::ftruncate64(fd.as_raw_fd(), len)}).unwrap(); + } + + fn create_sparse(file: &String) { + create_sparse_len(file, 1024*1024); } fn create_sparse_with_data(file: &String, head: u64, tail: u64) -> u64 { @@ -464,4 +492,49 @@ mod tests { assert!(is_sparse(&fd).unwrap()); } } + + + #[test] + fn test_copy_bytes_uspace() { + let dir = tempdir().unwrap(); + let (sparse, other) = tmps(&dir); + let data = "test data"; + let offset = 32; + + { + let mut fd = File::create(&other).unwrap(); + write!(fd, "{}", data); + } + + create_sparse_len(&sparse, 128); + create_sparse_len(&other, 128); + + { + let mut fd: File = OpenOptions::new() + .write(true) + .append(false) + .open(&sparse).unwrap(); + fd.seek(SeekFrom::Start(offset)).unwrap(); + write!(fd, "{}", data); + } + + { + let mut infd = File::open(&sparse).unwrap(); + let mut outfd: File = OpenOptions::new() + .write(true) + .append(false) + .open(&other).unwrap(); + infd.seek(SeekFrom::Start(offset)).unwrap(); + outfd.seek(SeekFrom::Start(offset)).unwrap(); + + let written = copy_bytes_uspace(&infd, &outfd, data.len()).unwrap(); + assert_eq!(written, data.len() as u64); + } + + { + let from_data = read(&sparse).unwrap(); + let to_data = read(&other).unwrap(); + assert_eq!(from_data, to_data); + } + } } From 95460c85688d6829da652b4f585fc635c4780ea0 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 12 Nov 2018 09:43:32 +1100 Subject: [PATCH 18/27] Do cross-mount detection up-front, and replace current copy impl with sparse aware parts. --- src/libstd/sys/unix/fs_linux.rs | 122 ++++++++++++++------------------ 1 file changed, 54 insertions(+), 68 deletions(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index dfc57a7d2e883..af84bf1831578 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -6,10 +6,7 @@ use mem; use path::Path; use ptr; use sys::{cvt, cvt_r}; -use cmp; use fs::File; -use sync::atomic::{AtomicBool, Ordering}; -use super::ext::fs::MetadataExt; use super::ext::io::AsRawFd; @@ -122,24 +119,24 @@ fn copy_bytes_uspace(mut reader: &File, mut writer: &File, nbytes: usize) -> io: // Kernel prior to 4.5 don't have copy_file_range We store the -// availability in a thread-local flag to avoid unnecessary syscalls +// availability in a thread-local flag to avoid unnecessary syscalls. thread_local! { static HAS_COPY_FILE_RANGE: RefCell = RefCell::new(true); } -fn copy_bytes(mut reader: &File, mut writer: &File, uspace: bool, nbytes: usize) -> io::Result { +fn copy_bytes(reader: &File, writer: &File, uspace: bool, nbytes: u64) -> io::Result { HAS_COPY_FILE_RANGE.with(|cfr| { loop { if uspace || !*cfr.borrow() { - return io::copy(&mut reader, &mut writer); + return copy_bytes_uspace(reader, writer, nbytes as usize) } else { - let result = copy_bytes_kernel(reader, writer, nbytes); + let result = copy_bytes_kernel(reader, writer, nbytes as usize); if let Err(ref err) = result { match err.raw_os_error() { Some(libc::ENOSYS) | Some(libc::EPERM) => { - // Flag as missing and retry. + // Flag as unavailable and retry. *cfr.borrow_mut() = false; continue; } @@ -153,29 +150,12 @@ fn copy_bytes(mut reader: &File, mut writer: &File, uspace: bool, nbytes: usize) }) } -/// Version of copy_file_range that defers offset-management to the -/// syscall. see copy_file_range(2) for details. -pub fn copy_file_bytes(infd: &File, outfd: &File, bytes: u64) -> io::Result { - let r = cvt(unsafe { - copy_file_range( - infd.as_raw_fd(), - ptr::null_mut(), - outfd.as_raw_fd(), - ptr::null_mut(), - bytes as usize, - 0, - ) as i64 - })?; - Ok(r as u64) -} - - /// Copy len bytes from whereever the descriptor cursors are set. -fn copy_range(infd: &File, outfd: &File, len: u64) -> io::Result { - let mut written = 0u64; +fn copy_range(infd: &File, outfd: &File, uspace: bool, len: u64) -> io::Result { + let mut written = 0; while written < len { - let result = copy_file_bytes(&infd, &outfd, len - written)?; + let result = copy_bytes(&infd, &outfd, uspace, len - written)?; written += result; } Ok(written) @@ -194,7 +174,7 @@ fn next_sparse_segments(fd: &File, pos: u64) -> io::Result<(u64, u64)> { Ok((next_data, next_hole)) } -fn copy_sparse(infd: &File, outfd: &File) -> io::Result { +fn copy_sparse(infd: &File, outfd: &File, uspace: bool) -> io::Result { let len = infd.metadata()?.len(); allocate_file(&outfd, len)?; @@ -205,7 +185,7 @@ fn copy_sparse(infd: &File, outfd: &File) -> io::Result { lseek(infd, next_data as i64, Wence::Set)?; lseek(outfd, next_data as i64, Wence::Set)?; - let _written = copy_range(infd, outfd, next_hole - next_data)?; + let _written = copy_range(infd, outfd, uspace, next_hole - next_data)?; pos = next_hole; } @@ -213,54 +193,46 @@ fn copy_sparse(infd: &File, outfd: &File) -> io::Result { } -fn is_sparse(fd: &File) -> io::Result { +fn stat(fd: &File) -> io::Result { let mut stat: libc::stat = unsafe { mem::uninitialized() }; cvt(unsafe { libc::fstat(fd.as_raw_fd(), &mut stat) })?; - Ok(stat.st_blocks < stat.st_size / stat.st_blksize) + Ok(stat) } +fn copy_parms(infd: &File, outfd: &File) -> io::Result<(bool, bool)> { + let in_stat = stat(infd)?; + let out_stat = stat(outfd)?; + let is_sparse = in_stat.st_blocks < in_stat.st_size / in_stat.st_blksize; + let is_xmount = in_stat.st_dev != out_stat.st_dev; + Ok((is_sparse, is_xmount)) +} -pub fn copy(from: &Path, to: &Path) -> io::Result { +pub fn copy(from: &Path, to: &Path) -> io::Result { if !from.is_file() { return Err(Error::new(ErrorKind::InvalidInput, "the source path is not an existing regular file")) } - let mut reader = File::open(from)?; - let mut writer = File::create(to)?; - let (perm, len) = { - let metadata = reader.metadata()?; - (metadata.permissions(), metadata.size()) + let infd = File::open(from)?; + let outfd = File::create(to)?; + let (is_sparse, is_xmount) = copy_parms(&infd, &outfd)?; + let uspace = is_xmount; + + let total = if is_sparse { + copy_sparse(&infd, &outfd, uspace)? + + } else { + let len = infd.metadata()?.len(); + copy_range(&infd, &outfd, uspace, len)? }; - let _sparse = is_sparse(&reader)?; - let mut userspace = false; - let mut written = 0u64; - while written < len { - let bytes_to_copy = cmp::min(len - written, usize::max_value() as u64) as usize; - let copy_result = copy_bytes(&mut reader, &mut writer, userspace, bytes_to_copy); - - match copy_result { - Ok(ret) => written += ret, - Err(err) => { - match err.raw_os_error() { - Some(os_err) if os_err == libc::EXDEV => { - // Files are mounted on different fs (EXDEV); - // flag as retry with userspace copy. - assert_eq!(written, 0); - userspace = true; - }, - _ => return Err(err), - } - } - } - } - writer.set_permissions(perm)?; - Ok(written) + outfd.set_permissions(infd.metadata()?.permissions())?; + Ok(total) } + #[cfg(test)] mod tests { use super::*; @@ -313,6 +285,11 @@ mod tests { } + fn is_sparse(fd: &File) -> io::Result { + let stat = stat(fd)?; + Ok(stat.st_blocks < stat.st_size / stat.st_blksize) + } + #[test] fn test_sparse_detection() { assert!(!is_sparse(&File::open("Cargo.toml").unwrap()).unwrap()); @@ -335,8 +312,7 @@ mod tests { } } - #[test] - fn test_copy_range_sparse() { + fn test_copy_range(uspace: bool) { let dir = tempdir().unwrap(); let (sparse, other) = tmps(&dir); let data = "test data"; @@ -354,12 +330,22 @@ mod tests { .write(true) .append(false) .open(&sparse).unwrap(); - copy_file_bytes(&infd, &outfd, data.len() as u64).unwrap(); + copy_range(&infd, &outfd, uspace, data.len() as u64).unwrap(); } assert!(is_sparse(&File::open(&sparse).unwrap()).unwrap()); } + #[test] + fn test_copy_range_sparse_kernel() { + test_copy_range(false); + } + + #[test] + fn test_copy_range_sparse_uspace() { + test_copy_range(true); + } + #[test] fn test_sparse_copy_middle() { let dir = tempdir().unwrap(); @@ -373,7 +359,7 @@ mod tests { create_sparse(&sparse); - let offset: usize = 512*1024; + let offset = 512*1024; { let infd = File::open(&other).unwrap(); let outfd: File = OpenOptions::new() @@ -448,11 +434,11 @@ mod tests { let dir = tempdir().unwrap(); let (sparse, _) = tmps(&dir); - let len = create_sparse_with_data(&sparse, 0, 10) as usize; + let len = create_sparse_with_data(&sparse, 0, 10); assert!(is_sparse(&File::open(&sparse).unwrap()).unwrap()); let bytes = read(&sparse).unwrap(); - assert!(bytes.len() == len); + assert!(bytes.len() == len as usize); let offset = 1024 * 4096; assert!(bytes[offset] == b'c'); From b6f6f97ef58a9e034a85eb6ea864ad2e53056a05 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 12 Nov 2018 09:47:31 +1100 Subject: [PATCH 19/27] Enable disabled tests. --- src/libstd/sys/unix/fs_linux.rs | 1 - src/libstd/tests/sys/unix/fs.rs | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index af84bf1831578..3900a6b483ef6 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -232,7 +232,6 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { } - #[cfg(test)] mod tests { use super::*; diff --git a/src/libstd/tests/sys/unix/fs.rs b/src/libstd/tests/sys/unix/fs.rs index 4080603d8cc47..58b90663fbc9b 100644 --- a/src/libstd/tests/sys/unix/fs.rs +++ b/src/libstd/tests/sys/unix/fs.rs @@ -82,7 +82,7 @@ mod test_linux { assert_eq!(from_data, to_data); } - //#[test] + #[test] fn test_sparse() { let dir = tempdir().unwrap(); let from = dir.path().join("sparse.bin"); @@ -101,7 +101,7 @@ mod test_linux { assert_eq!(from_data, to_data); } - //#[test] + #[test] fn test_sparse_leading_gap() { let dir = tempdir().unwrap(); let from = dir.path().join("sparse.bin"); @@ -122,7 +122,7 @@ mod test_linux { assert_eq!(from_data, to_data); } - //#[test] + #[test] fn test_sparse_trailng_gap() { let dir = tempdir().unwrap(); let from = dir.path().join("sparse.bin"); @@ -142,7 +142,7 @@ mod test_linux { assert_eq!(from_data, to_data); } - //#[test] + #[test] fn test_empty_sparse() { let dir = tempdir().unwrap(); let from = dir.path().join("sparse.bin"); From 5b847461444da8765777f4561e1e57279684390b Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Mon, 12 Nov 2018 10:25:34 +1100 Subject: [PATCH 20/27] Port in external test module as we now have a local one, and use std-provided testing tmpdir impl. --- src/Cargo.lock | 1 - src/libstd/Cargo.toml | 1 - src/libstd/sys/unix/fs_linux.rs | 226 ++++++++++++++++++++++--------- src/libstd/tests/lib.rs | 2 - src/libstd/tests/sys/mod.rs | 2 - src/libstd/tests/sys/unix/fs.rs | 168 ----------------------- src/libstd/tests/sys/unix/mod.rs | 2 - 7 files changed, 160 insertions(+), 242 deletions(-) delete mode 100644 src/libstd/tests/lib.rs delete mode 100644 src/libstd/tests/sys/mod.rs delete mode 100644 src/libstd/tests/sys/unix/fs.rs delete mode 100644 src/libstd/tests/sys/unix/mod.rs diff --git a/src/Cargo.lock b/src/Cargo.lock index d23764caddcff..32304c81182f1 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -2679,7 +2679,6 @@ dependencies = [ "rustc_lsan 0.0.0", "rustc_msan 0.0.0", "rustc_tsan 0.0.0", - "tempfile 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "unwind 0.0.0", ] diff --git a/src/libstd/Cargo.toml b/src/libstd/Cargo.toml index 554aa7b65d20e..2b1d515c83b75 100644 --- a/src/libstd/Cargo.toml +++ b/src/libstd/Cargo.toml @@ -24,7 +24,6 @@ unwind = { path = "../libunwind" } [dev-dependencies] rand = "0.5" -tempfile = "3" [target.x86_64-apple-darwin.dependencies] rustc_asan = { path = "../librustc_asan" } diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index 3900a6b483ef6..c19124755a664 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -235,21 +235,17 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { #[cfg(test)] mod tests { use super::*; - extern crate tempfile; - use self::tempfile::{tempdir, TempDir}; + use sys_common::io::test::{TempDir, tmpdir}; use fs::{read, OpenOptions}; use io::{Seek, SeekFrom, Write}; + use path::PathBuf; - fn create_sparse_len(file: &String, len: i64) { + fn create_sparse(file: &PathBuf, len: i64) { let fd = File::create(file).unwrap(); cvt(unsafe {libc::ftruncate64(fd.as_raw_fd(), len)}).unwrap(); } - fn create_sparse(file: &String) { - create_sparse_len(file, 1024*1024); - } - - fn create_sparse_with_data(file: &String, head: u64, tail: u64) -> u64 { + fn create_sparse_with_data(file: &PathBuf, head: u64, tail: u64) -> u64 { let data = "c00lc0d3"; let len = 4096u64 * 4096 + data.len() as u64 + tail; @@ -276,11 +272,10 @@ mod tests { } - fn tmps(dir: &TempDir) -> (String, String) { - let sparse = dir.path().join("sparse.bin"); - let other = dir.path().join("other.txt"); - (sparse.to_str().unwrap().to_string(), - other.to_str().unwrap().to_string()) + fn tmps(dir: &TempDir) -> (PathBuf, PathBuf) { + let from = dir.path().join("from.bin"); + let to = dir.path().join("to.bin"); + (from, to) } @@ -289,50 +284,55 @@ mod tests { Ok(stat.st_blocks < stat.st_size / stat.st_blksize) } + fn is_fsparse(file: &PathBuf) -> io::Result { + let fd = File::open(file)?; + is_sparse(&fd) + } + #[test] fn test_sparse_detection() { assert!(!is_sparse(&File::open("Cargo.toml").unwrap()).unwrap()); - let dir = tempdir().unwrap(); - let (sparse, _) = tmps(&dir); - create_sparse_with_data(&sparse, 0, 0); + let dir = tmpdir(); + let (from, _) = tmps(&dir); + create_sparse_with_data(&from, 0, 0); { - let fd = File::open(&sparse).unwrap(); + let fd = File::open(&from).unwrap(); assert!(is_sparse(&fd).unwrap()); } { - let mut fd = File::open(&sparse).unwrap(); + let mut fd = File::open(&from).unwrap(); write!(fd, "{}", "test"); } { - let fd = File::open(&sparse).unwrap(); + let fd = File::open(&from).unwrap(); assert!(is_sparse(&fd).unwrap()); } } fn test_copy_range(uspace: bool) { - let dir = tempdir().unwrap(); - let (sparse, other) = tmps(&dir); + let dir = tmpdir(); + let (from, to) = tmps(&dir); let data = "test data"; { - let mut fd = File::create(&other).unwrap(); + let mut fd = File::create(&to).unwrap(); write!(fd, "{}", data); } - create_sparse(&sparse); + create_sparse(&from, 1024*1024); { - let infd = File::open(&other).unwrap(); + let infd = File::open(&to).unwrap(); let outfd: File = OpenOptions::new() .write(true) .append(false) - .open(&sparse).unwrap(); + .open(&from).unwrap(); copy_range(&infd, &outfd, uspace, data.len() as u64).unwrap(); } - assert!(is_sparse(&File::open(&sparse).unwrap()).unwrap()); + assert!(is_sparse(&File::open(&from).unwrap()).unwrap()); } #[test] @@ -347,24 +347,24 @@ mod tests { #[test] fn test_sparse_copy_middle() { - let dir = tempdir().unwrap(); - let (sparse, other) = tmps(&dir); + let dir = tmpdir(); + let (from, to) = tmps(&dir); let data = "test data"; { - let mut fd = File::create(&other).unwrap(); + let mut fd = File::create(&to).unwrap(); write!(fd, "{}", data); } - create_sparse(&sparse); + create_sparse(&from, 1024*1024); let offset = 512*1024; { - let infd = File::open(&other).unwrap(); + let infd = File::open(&to).unwrap(); let outfd: File = OpenOptions::new() .write(true) .append(false) - .open(&sparse).unwrap(); + .open(&from).unwrap(); let mut offdat: i64 = 512*1024; let offptr = &mut offdat as *mut i64; cvt( @@ -379,9 +379,9 @@ mod tests { }).unwrap(); } - assert!(is_sparse(&File::open(&sparse).unwrap()).unwrap()); + assert!(is_sparse(&File::open(&from).unwrap()).unwrap()); - let bytes = read(&sparse).unwrap(); + let bytes = read(&from).unwrap(); assert!(bytes.len() == 1024*1024); assert!(bytes[offset] == b't'); assert!(bytes[offset+1] == b'e'); @@ -392,24 +392,24 @@ mod tests { #[test] fn test_lseek_data() { - let dir = tempdir().unwrap(); - let (sparse, other) = tmps(&dir); + let dir = tmpdir(); + let (from, to) = tmps(&dir); let data = "test data"; let offset = 512*1024; { - let mut fd = File::create(&other).unwrap(); + let mut fd = File::create(&to).unwrap(); write!(fd, "{}", data); } - create_sparse(&sparse); + create_sparse(&from, 1024*1024); { - let infd = File::open(&other).unwrap(); + let infd = File::open(&to).unwrap(); let outfd: File = OpenOptions::new() .write(true) .append(false) - .open(&sparse).unwrap(); + .open(&from).unwrap(); cvt( unsafe { copy_file_range( @@ -422,21 +422,21 @@ mod tests { }).unwrap(); } - assert!(is_sparse(&File::open(&sparse).unwrap()).unwrap()); + assert!(is_sparse(&File::open(&from).unwrap()).unwrap()); - let off = lseek(&File::open(&sparse).unwrap(), 0, Wence::Data).unwrap(); + let off = lseek(&File::open(&from).unwrap(), 0, Wence::Data).unwrap(); assert_eq!(off, SeekOff::Offset(offset)); } #[test] fn test_sparse_rust_seek() { - let dir = tempdir().unwrap(); - let (sparse, _) = tmps(&dir); + let dir = tmpdir(); + let (from, _) = tmps(&dir); - let len = create_sparse_with_data(&sparse, 0, 10); - assert!(is_sparse(&File::open(&sparse).unwrap()).unwrap()); + let len = create_sparse_with_data(&from, 0, 10); + assert!(is_sparse(&File::open(&from).unwrap()).unwrap()); - let bytes = read(&sparse).unwrap(); + let bytes = read(&from).unwrap(); assert!(bytes.len() == len as usize); let offset = 1024 * 4096; @@ -449,30 +449,30 @@ mod tests { #[test] fn test_lseek_no_data() { - let dir = tempdir().unwrap(); - let (sparse, _) = tmps(&dir); - create_sparse(&sparse); + let dir = tmpdir(); + let (from, _) = tmps(&dir); + create_sparse(&from, 1024*1024); - assert!(is_sparse(&File::open(&sparse).unwrap()).unwrap()); + assert!(is_sparse(&File::open(&from).unwrap()).unwrap()); - let fd = File::open(&sparse).unwrap(); + let fd = File::open(&from).unwrap(); let off = lseek(&fd, 0, Wence::Data).unwrap(); assert!(off == SeekOff::EOF); } #[test] fn test_allocate_file_is_sparse() { - let dir = tempdir().unwrap(); - let (sparse, _) = tmps(&dir); + let dir = tmpdir(); + let (from, _) = tmps(&dir); let len = 32 * 1024 * 1024; { - let fd = File::create(&sparse).unwrap(); + let fd = File::create(&from).unwrap(); allocate_file(&fd, len).unwrap(); } { - let fd = File::open(&sparse).unwrap(); + let fd = File::open(&from).unwrap(); assert_eq!(len, fd.metadata().unwrap().len()); assert!(is_sparse(&fd).unwrap()); } @@ -481,34 +481,34 @@ mod tests { #[test] fn test_copy_bytes_uspace() { - let dir = tempdir().unwrap(); - let (sparse, other) = tmps(&dir); + let dir = tmpdir(); + let (from, to) = tmps(&dir); let data = "test data"; let offset = 32; { - let mut fd = File::create(&other).unwrap(); + let mut fd = File::create(&to).unwrap(); write!(fd, "{}", data); } - create_sparse_len(&sparse, 128); - create_sparse_len(&other, 128); + create_sparse(&from, 128); + create_sparse(&to, 128); { let mut fd: File = OpenOptions::new() .write(true) .append(false) - .open(&sparse).unwrap(); + .open(&from).unwrap(); fd.seek(SeekFrom::Start(offset)).unwrap(); write!(fd, "{}", data); } { - let mut infd = File::open(&sparse).unwrap(); + let mut infd = File::open(&from).unwrap(); let mut outfd: File = OpenOptions::new() .write(true) .append(false) - .open(&other).unwrap(); + .open(&to).unwrap(); infd.seek(SeekFrom::Start(offset)).unwrap(); outfd.seek(SeekFrom::Start(offset)).unwrap(); @@ -517,9 +517,103 @@ mod tests { } { - let from_data = read(&sparse).unwrap(); - let to_data = read(&other).unwrap(); + let from_data = read(&from).unwrap(); + let to_data = read(&to).unwrap(); assert_eq!(from_data, to_data); } } + + + + + #[test] + fn test_simple_copy() { + let dir = tmpdir(); + let (from, to) = tmps(&dir); + let text = "This is a test file."; + + { + let file = File::create(&from).unwrap(); + write!(&file, "{}", text).unwrap(); + } + + let written = copy(&from, &to).unwrap(); + assert_eq!(text.len() as u64, written); + + let from_data = read(&from).unwrap(); + let to_data = read(&to).unwrap(); + assert_eq!(from_data, to_data); + } + + #[test] + fn test_sparse() { + let dir = tmpdir(); + let (from, to) = tmps(&dir); + + let slen = create_sparse_with_data(&from, 0, 0); + assert_eq!(slen, from.metadata().unwrap().len()); + assert!(is_fsparse(&from).unwrap()); + + let written = copy(&from, &to).unwrap(); + assert_eq!(slen, written); + assert!(is_fsparse(&to).unwrap()); + + let from_data = read(&from).unwrap(); + let to_data = read(&to).unwrap(); + assert_eq!(from_data, to_data); + } + + #[test] + fn test_sparse_leading_gap() { + let dir = tmpdir(); + let (from, to) = tmps(&dir); + + let slen = create_sparse_with_data(&from, 1024, 0); + assert_eq!(slen, from.metadata().unwrap().len()); + assert!(is_fsparse(&from).unwrap()); + + let written = copy(&from, &to).unwrap(); + assert_eq!(slen, written); + assert!(is_fsparse(&to).unwrap()); + + let from_data = read(&from).unwrap(); + let to_data = read(&to).unwrap(); + assert_eq!(from_data, to_data); + } + + #[test] + fn test_sparse_trailng_gap() { + let dir = tmpdir(); + let (from, to) = tmps(&dir); + + let slen = create_sparse_with_data(&from, 1024, 1024); + assert_eq!(slen, from.metadata().unwrap().len()); + assert!(is_fsparse(&from).unwrap()); + + let written = copy(&from, &to).unwrap(); + assert_eq!(slen, written); + assert!(is_fsparse(&to).unwrap()); + + let from_data = read(&from).unwrap(); + let to_data = read(&to).unwrap(); + assert_eq!(from_data, to_data); + } + + #[test] + fn test_empty_sparse() { + let dir = tmpdir(); + let (from, to) = tmps(&dir); + + create_sparse(&from, 1024*1024); + assert_eq!(from.metadata().unwrap().len(), 1024*1024); + + let _written = copy(&from, &to).unwrap(); + assert_eq!(to.metadata().unwrap().len(), 1024*1024); + + assert!(is_fsparse(&to).unwrap()); + + let from_data = read(&from).unwrap(); + let to_data = read(&to).unwrap(); + assert_eq!(from_data, to_data); + } } diff --git a/src/libstd/tests/lib.rs b/src/libstd/tests/lib.rs deleted file mode 100644 index e1b64941f1847..0000000000000 --- a/src/libstd/tests/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ - -mod sys; diff --git a/src/libstd/tests/sys/mod.rs b/src/libstd/tests/sys/mod.rs deleted file mode 100644 index e454fbfd10884..0000000000000 --- a/src/libstd/tests/sys/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ - -mod unix; diff --git a/src/libstd/tests/sys/unix/fs.rs b/src/libstd/tests/sys/unix/fs.rs deleted file mode 100644 index 58b90663fbc9b..0000000000000 --- a/src/libstd/tests/sys/unix/fs.rs +++ /dev/null @@ -1,168 +0,0 @@ - -use std::fs::{copy, read, File, OpenOptions}; -use std::io::{SeekFrom, Result}; -use std::path::PathBuf; -extern crate tempfile; -use self::tempfile::tempdir; - -#[cfg(all(test, any(target_os = "linux", target_os = "android")))] -mod test_linux { - use super::*; - use std::io::{Seek, Write}; - use std::process::Command; - - fn create_sparse(file: &PathBuf, head: u64, tail: u64) -> Result { - let data = "c00lc0d3"; - let len = 4096u64 * 4096 + data.len() as u64 + tail; - - let out = Command::new("truncate") - .args(&["-s", len.to_string().as_str(), - file.to_str().unwrap()]) - .output()?; - assert!(out.status.success()); - - let mut fd = OpenOptions::new() - .write(true) - .append(false) - .open(&file)?; - - fd.seek(SeekFrom::Start(head))?; - write!(fd, "{}", data).unwrap(); - - fd.seek(SeekFrom::Start(1024*4096))?; - write!(fd, "{}", data).unwrap(); - - fd.seek(SeekFrom::Start(4096*4096))?; - write!(fd, "{}", data).unwrap(); - - Ok(len as u64) - } - - fn quickstat(file: &PathBuf) -> Result<(i32, i32, i32)> { - let out = Command::new("stat") - .args(&["--format", "%s %b %B", - file.to_str().unwrap()]) - .output()?; - assert!(out.status.success()); - - let stdout = String::from_utf8(out.stdout).unwrap(); - let stats = stdout - .split_whitespace() - .map(|s| s.parse::().unwrap()) - .collect::>(); - let (size, blocks, blksize) = (stats[0], stats[1], stats[2]); - - Ok((size, blocks, blksize)) - } - - fn probably_sparse(file: &PathBuf) -> Result { - let (size, blocks, blksize) = quickstat(file)?; - - Ok(blocks < size / blksize) - } - - - #[test] - fn test_simple_copy() { - let dir = tempdir().unwrap(); - let from = dir.path().join("source.txt"); - let to = dir.path().join("dest.txt"); - let text = "This is a test file."; - - { - let file = File::create(&from).unwrap(); - write!(&file, "{}", text).unwrap(); - } - - let written = copy(&from, &to).unwrap(); - assert_eq!(text.len() as u64, written); - - let from_data = read(&from).unwrap(); - let to_data = read(&to).unwrap(); - assert_eq!(from_data, to_data); - } - - #[test] - fn test_sparse() { - let dir = tempdir().unwrap(); - let from = dir.path().join("sparse.bin"); - let to = dir.path().join("target.bin"); - - let slen = create_sparse(&from, 0, 0).unwrap(); - assert_eq!(slen, from.metadata().unwrap().len()); - assert!(probably_sparse(&from).unwrap()); - - let written = copy(&from, &to).unwrap(); - assert_eq!(slen, written); - assert!(probably_sparse(&to).unwrap()); - - let from_data = read(&from).unwrap(); - let to_data = read(&to).unwrap(); - assert_eq!(from_data, to_data); - } - - #[test] - fn test_sparse_leading_gap() { - let dir = tempdir().unwrap(); - let from = dir.path().join("sparse.bin"); - let to = dir.path().join("target.bin"); - - let slen = create_sparse(&from, 1024, 0).unwrap(); - assert_eq!(slen, from.metadata().unwrap().len()); - assert!(probably_sparse(&from).unwrap()); - - let written = copy(&from, &to).unwrap(); - assert_eq!(slen, written); - assert!(probably_sparse(&to).unwrap()); - - assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); - - let from_data = read(&from).unwrap(); - let to_data = read(&to).unwrap(); - assert_eq!(from_data, to_data); - } - - #[test] - fn test_sparse_trailng_gap() { - let dir = tempdir().unwrap(); - let from = dir.path().join("sparse.bin"); - let to = dir.path().join("target.bin"); - - let slen = create_sparse(&from, 1024, 1024).unwrap(); - assert_eq!(slen, from.metadata().unwrap().len()); - assert!(probably_sparse(&from).unwrap()); - - let written = copy(&from, &to).unwrap(); - assert_eq!(slen, written); - assert!(probably_sparse(&to).unwrap()); - assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); - - let from_data = read(&from).unwrap(); - let to_data = read(&to).unwrap(); - assert_eq!(from_data, to_data); - } - - #[test] - fn test_empty_sparse() { - let dir = tempdir().unwrap(); - let from = dir.path().join("sparse.bin"); - let to = dir.path().join("target.bin"); - - let out = Command::new("/usr/bin/truncate") - .args(&["-s", "1M", from.to_str().unwrap()]) - .output().unwrap(); - assert!(out.status.success()); - assert_eq!(from.metadata().unwrap().len(), 1024*1024); - - let _written = copy(&from, &to).unwrap(); - assert_eq!(to.metadata().unwrap().len(), 1024*1024); - - assert!(probably_sparse(&to).unwrap()); - assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); - - let from_data = read(&from).unwrap(); - let to_data = read(&to).unwrap(); - assert_eq!(from_data, to_data); - } - -} diff --git a/src/libstd/tests/sys/unix/mod.rs b/src/libstd/tests/sys/unix/mod.rs deleted file mode 100644 index ac21831582800..0000000000000 --- a/src/libstd/tests/sys/unix/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ - -mod fs; From ce17903eccdf8bf5200b355b9554dc79001281d0 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 13 Nov 2018 09:34:18 +1100 Subject: [PATCH 21/27] Test and fix for userspace copies larger than read blocks. --- src/libstd/sys/unix/fs_linux.rs | 48 +++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index c19124755a664..ad600cf3d9a2b 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -1,5 +1,6 @@ use cell::RefCell; +use cmp; use io::{self, Error, ErrorKind, Read, Write}; use libc; use mem; @@ -94,17 +95,17 @@ fn copy_bytes_kernel(reader: &File, writer: &File, nbytes: usize) -> io::Result< // Slightly modified version of io::copy() that only copies a set amount of bytes. fn copy_bytes_uspace(mut reader: &File, mut writer: &File, nbytes: usize) -> io::Result { + const BLKSIZE: usize = 4 * 1024; // Assume 4k blocks on disk. let mut buf = unsafe { - // Assume 4k blocks on disk. - let mut buf: [u8; 4 * 1024] = mem::uninitialized(); + let mut buf: [u8; BLKSIZE] = mem::uninitialized(); reader.initializer().initialize(&mut buf); buf }; let mut written = 0; while written < nbytes { - let left = nbytes - written; - let len = match reader.read(&mut buf[..left]) { + let next = cmp::min(nbytes - written, BLKSIZE); + let len = match reader.read(&mut buf[..next]) { Ok(0) => return Err(Error::new(ErrorKind::InvalidData, "Source file ended prematurely.")), Ok(len) => len, @@ -235,14 +236,15 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { #[cfg(test)] mod tests { use super::*; + use iter; use sys_common::io::test::{TempDir, tmpdir}; use fs::{read, OpenOptions}; use io::{Seek, SeekFrom, Write}; use path::PathBuf; - fn create_sparse(file: &PathBuf, len: i64) { + fn create_sparse(file: &PathBuf, len: u64) { let fd = File::create(file).unwrap(); - cvt(unsafe {libc::ftruncate64(fd.as_raw_fd(), len)}).unwrap(); + cvt(unsafe {libc::ftruncate64(fd.as_raw_fd(), len as i64)}).unwrap(); } fn create_sparse_with_data(file: &PathBuf, head: u64, tail: u64) -> u64 { @@ -321,7 +323,7 @@ mod tests { write!(fd, "{}", data); } - create_sparse(&from, 1024*1024); + create_sparse_with_data(&from, 0, 0); { let infd = File::open(&to).unwrap(); @@ -480,7 +482,7 @@ mod tests { #[test] - fn test_copy_bytes_uspace() { + fn test_copy_bytes_uspace_small() { let dir = tmpdir(); let (from, to) = tmps(&dir); let data = "test data"; @@ -523,6 +525,36 @@ mod tests { } } + #[test] + fn test_copy_bytes_uspace_large() { + let dir = tmpdir(); + let (from, to) = tmps(&dir); + let size = 128*1024; + let data = iter::repeat("X").take(size).collect::(); + + { + let mut fd: File = File::create(&from).unwrap(); + write!(fd, "{}", data).unwrap(); + } + + { + let infd = File::open(&from).unwrap(); + let outfd = File::create(&to).unwrap(); + let written = copy_bytes_uspace(&infd, &outfd, size).unwrap(); + + assert_eq!(written, size as u64); + } + + assert_eq!(from.metadata().unwrap().len(), + to.metadata().unwrap().len()); + + { + let from_data = read(&from).unwrap(); + let to_data = read(&to).unwrap(); + assert_eq!(from_data, to_data); + } + } + From 9e52d35a917f23e296b1c1b82ff1688bbe7c2007 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 13 Nov 2018 13:12:20 +1100 Subject: [PATCH 22/27] Use MetadataExt rather than direct stat() call. --- src/libstd/sys/unix/fs_linux.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index ad600cf3d9a2b..358ef9cdf412c 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -194,17 +194,11 @@ fn copy_sparse(infd: &File, outfd: &File, uspace: bool) -> io::Result { } -fn stat(fd: &File) -> io::Result { - let mut stat: libc::stat = unsafe { mem::uninitialized() }; - cvt(unsafe { libc::fstat(fd.as_raw_fd(), &mut stat) })?; - Ok(stat) -} - fn copy_parms(infd: &File, outfd: &File) -> io::Result<(bool, bool)> { - let in_stat = stat(infd)?; - let out_stat = stat(outfd)?; - let is_sparse = in_stat.st_blocks < in_stat.st_size / in_stat.st_blksize; - let is_xmount = in_stat.st_dev != out_stat.st_dev; + let in_stat = infd.metadata()?; + let out_stat = outfd.metadata()?; + let is_sparse = in_stat.st_blocks() < in_stat.st_size() / in_stat.st_blksize(); + let is_xmount = in_stat.st_dev() != out_stat.st_dev(); Ok((is_sparse, is_xmount)) } From b9338365563dcbc6dace7d775f83266808b383bf Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 13 Nov 2018 13:12:31 +1100 Subject: [PATCH 23/27] Minor test and fmt cleanups. --- src/libstd/sys/unix/fs_linux.rs | 41 +++++++++++++++++---------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index 358ef9cdf412c..440bb16f81444 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -1,15 +1,15 @@ use cell::RefCell; use cmp; +use fs::File; use io::{self, Error, ErrorKind, Read, Write}; use libc; use mem; +use os::linux::fs::MetadataExt; use path::Path; use ptr; -use sys::{cvt, cvt_r}; -use fs::File; use super::ext::io::AsRawFd; - +use sys::{cvt, cvt_r}; unsafe fn copy_file_range( fd_in: libc::c_int, @@ -76,10 +76,8 @@ fn allocate_file(fd: &File, len: u64) -> io::Result<()> { } -// Version of copy_file_range(2) that copies the give range to the -// same place in the target file. If off is None then use nul to -// tell copy_file_range() track the file offset. See the manpage -// for details. +// Wrapper for copy_file_range(2) that defers file offset tracking to +// the underlying call. See the manpage for details. fn copy_bytes_kernel(reader: &File, writer: &File, nbytes: usize) -> io::Result { unsafe { cvt(copy_file_range(reader.as_raw_fd(), @@ -119,7 +117,7 @@ fn copy_bytes_uspace(mut reader: &File, mut writer: &File, nbytes: usize) -> io: } -// Kernel prior to 4.5 don't have copy_file_range We store the +// Kernels prior to 4.5 don't have copy_file_range,so we store the // availability in a thread-local flag to avoid unnecessary syscalls. thread_local! { static HAS_COPY_FILE_RANGE: RefCell = RefCell::new(true); @@ -256,13 +254,13 @@ mod tests { .open(&file).unwrap(); fd.seek(SeekFrom::Start(head)).unwrap(); - write!(fd, "{}", data); + write!(fd, "{}", data).unwrap(); fd.seek(SeekFrom::Start(1024*4096)).unwrap(); - write!(fd, "{}", data); + write!(fd, "{}", data).unwrap(); fd.seek(SeekFrom::Start(4096*4096)).unwrap(); - write!(fd, "{}", data); + write!(fd, "{}", data).unwrap(); len } @@ -276,8 +274,8 @@ mod tests { fn is_sparse(fd: &File) -> io::Result { - let stat = stat(fd)?; - Ok(stat.st_blocks < stat.st_size / stat.st_blksize) + let stat = fd.metadata()?; + Ok(stat.st_blocks() < stat.st_size() / stat.st_blksize()) } fn is_fsparse(file: &PathBuf) -> io::Result { @@ -298,8 +296,11 @@ mod tests { assert!(is_sparse(&fd).unwrap()); } { - let mut fd = File::open(&from).unwrap(); - write!(fd, "{}", "test"); + let mut fd = OpenOptions::new() + .write(true) + .append(true) + .open(&from).unwrap(); + write!(fd, "{}", "test").unwrap(); } { let fd = File::open(&from).unwrap(); @@ -314,7 +315,7 @@ mod tests { { let mut fd = File::create(&to).unwrap(); - write!(fd, "{}", data); + write!(fd, "{}", data).unwrap(); } create_sparse_with_data(&from, 0, 0); @@ -349,7 +350,7 @@ mod tests { { let mut fd = File::create(&to).unwrap(); - write!(fd, "{}", data); + write!(fd, "{}", data).unwrap(); } create_sparse(&from, 1024*1024); @@ -395,7 +396,7 @@ mod tests { { let mut fd = File::create(&to).unwrap(); - write!(fd, "{}", data); + write!(fd, "{}", data).unwrap(); } create_sparse(&from, 1024*1024); @@ -484,7 +485,7 @@ mod tests { { let mut fd = File::create(&to).unwrap(); - write!(fd, "{}", data); + write!(fd, "{}", data).unwrap(); } create_sparse(&from, 128); @@ -496,7 +497,7 @@ mod tests { .append(false) .open(&from).unwrap(); fd.seek(SeekFrom::Start(offset)).unwrap(); - write!(fd, "{}", data); + write!(fd, "{}", data).unwrap(); } { From 92283c8112c8f6df6519155f02a6b22a782817e4 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 14 Nov 2018 12:32:21 +1100 Subject: [PATCH 24/27] Travis runs under 3.x kernel, so we need to disable any tests that rely on copy_file_range(). --- src/libstd/sys/unix/fs_linux.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index 440bb16f81444..9eee7c6df6010 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -229,11 +229,28 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { mod tests { use super::*; use iter; + use ffi::CStr; use sys_common::io::test::{TempDir, tmpdir}; use fs::{read, OpenOptions}; use io::{Seek, SeekFrom, Write}; use path::PathBuf; + fn supported_kernel() -> bool { + let mut uname = unsafe { mem::zeroed() }; + cvt(unsafe { libc::uname(&mut uname) }).unwrap(); + + let release = + unsafe { CStr::from_ptr(uname.release.as_ptr()) } + .to_str().unwrap(); + let s = release + .split(|c| c == '.' || c == '-') + .collect::>(); + + // Kernel >= 4.5 + s[0].parse::().unwrap() >= 4 && + s[1].parse::().unwrap() >= 5 + } + fn create_sparse(file: &PathBuf, len: u64) { let fd = File::create(file).unwrap(); cvt(unsafe {libc::ftruncate64(fd.as_raw_fd(), len as i64)}).unwrap(); @@ -344,6 +361,10 @@ mod tests { #[test] fn test_sparse_copy_middle() { + if !supported_kernel() { + return; + } + let dir = tmpdir(); let (from, to) = tmps(&dir); let data = "test data"; @@ -389,6 +410,10 @@ mod tests { #[test] fn test_lseek_data() { + if !supported_kernel() { + return; + } + let dir = tmpdir(); let (from, to) = tmps(&dir); let data = "test data"; From a437a7abd92ad25cc8c31f726a64f039c4d7f4bb Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 15 Nov 2018 09:03:20 +1100 Subject: [PATCH 25/27] Add notes about platform-specific handling in copy(). --- src/libstd/fs.rs | 18 ++++++++++++++---- src/libstd/io/util.rs | 5 +++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/libstd/fs.rs b/src/libstd/fs.rs index f4703dec187b8..1fc2fd3ccfa79 100644 --- a/src/libstd/fs.rs +++ b/src/libstd/fs.rs @@ -1569,20 +1569,30 @@ pub fn rename, Q: AsRef>(from: P, to: Q) -> io::Result<()> /// If you’re wanting to copy the contents of one file to another and you’re /// working with [`File`]s, see the [`io::copy`] function. /// -/// [`io::copy`]: ../io/fn.copy.html -/// [`File`]: ./struct.File.html -/// /// # Platform-specific behavior /// +/// Note that, these behaviours [may change in the future][changes]. +/// +/// ## Unix-like +/// /// This function currently corresponds to the `open` function in Unix /// with `O_RDONLY` for `from` and `O_WRONLY`, `O_CREAT`, and `O_TRUNC` for `to`. /// `O_CLOEXEC` is set for returned file descriptors. +/// +/// On Linux this call will attempt to preserve [sparse-files][sparse] +/// if the target filesystem supports them. If this is not the desired +/// behaviour use [`io::copy`]. +/// +/// ## Windows +/// /// On Windows, this function currently corresponds to `CopyFileEx`. Alternate /// NTFS streams are copied but only the size of the main stream is returned by /// this function. -/// Note that, this [may change in the future][changes]. /// /// [changes]: ../io/index.html#platform-specific-behavior +/// [sparse]: https://en.wikipedia.org/wiki/Sparse_file +/// [`io::copy`]: ../io/fn.copy.html +/// [`File`]: ./struct.File.html /// /// # Errors /// diff --git a/src/libstd/io/util.rs b/src/libstd/io/util.rs index 12995d0868345..a2df69d2081ce 100644 --- a/src/libstd/io/util.rs +++ b/src/libstd/io/util.rs @@ -23,8 +23,9 @@ use mem; /// On success, the total number of bytes that were copied from /// `reader` to `writer` is returned. /// -/// If you’re wanting to copy the contents of one file to another and you’re -/// working with filesystem paths, see the [`fs::copy`] function. +/// If you’re copying the contents of one file to another and you’re +/// working with filesystem paths, see the [`fs::copy`] function, +/// which handles permissions and platform-specific optimisations. /// /// [`fs::copy`]: ../fs/fn.copy.html /// From a4358744d501813c216686a3a34cf462bfd72ecd Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 20 Nov 2018 11:00:27 +1100 Subject: [PATCH 26/27] Minimise calls to .metadata(). --- src/libstd/sys/unix/fs_linux.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index 9eee7c6df6010..51dfae64cbb25 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -1,7 +1,7 @@ use cell::RefCell; use cmp; -use fs::File; +use fs::{File, Metadata}; use io::{self, Error, ErrorKind, Read, Write}; use libc; use mem; @@ -160,27 +160,26 @@ fn copy_range(infd: &File, outfd: &File, uspace: bool, len: u64) -> io::Result io::Result<(u64, u64)> { +fn next_sparse_segments(fd: &File, pos: u64, len: u64) -> io::Result<(u64, u64)> { let next_data = match lseek(fd, pos as i64, Wence::Data)? { SeekOff::Offset(off) => off, - SeekOff::EOF => fd.metadata()?.len() + SeekOff::EOF => len }; let next_hole = match lseek(fd, next_data as i64, Wence::Hole)? { SeekOff::Offset(off) => off, - SeekOff::EOF => fd.metadata()?.len() + SeekOff::EOF => len }; Ok((next_data, next_hole)) } -fn copy_sparse(infd: &File, outfd: &File, uspace: bool) -> io::Result { - let len = infd.metadata()?.len(); +fn copy_sparse(infd: &File, outfd: &File, uspace: bool, len: u64) -> io::Result { allocate_file(&outfd, len)?; let mut pos = 0; while pos < len { - let (next_data, next_hole) = next_sparse_segments(infd, pos)?; + let (next_data, next_hole) = next_sparse_segments(infd, pos, len)?; lseek(infd, next_data as i64, Wence::Set)?; lseek(outfd, next_data as i64, Wence::Set)?; @@ -192,11 +191,9 @@ fn copy_sparse(infd: &File, outfd: &File, uspace: bool) -> io::Result { } -fn copy_parms(infd: &File, outfd: &File) -> io::Result<(bool, bool)> { - let in_stat = infd.metadata()?; - let out_stat = outfd.metadata()?; - let is_sparse = in_stat.st_blocks() < in_stat.st_size() / in_stat.st_blksize(); - let is_xmount = in_stat.st_dev() != out_stat.st_dev(); +fn copy_parms(in_meta: &Metadata, out_meta: &Metadata) -> io::Result<(bool, bool)> { + let is_sparse = in_meta.st_blocks() < in_meta.st_size() / in_meta.st_blksize(); + let is_xmount = in_meta.st_dev() != out_meta.st_dev(); Ok((is_sparse, is_xmount)) } @@ -209,18 +206,21 @@ pub fn copy(from: &Path, to: &Path) -> io::Result { let infd = File::open(from)?; let outfd = File::create(to)?; - let (is_sparse, is_xmount) = copy_parms(&infd, &outfd)?; + let in_meta = infd.metadata()?; + let out_meta = outfd.metadata()?; + + let (is_sparse, is_xmount) = copy_parms(&in_meta, &out_meta)?; let uspace = is_xmount; + let len = in_meta.len(); let total = if is_sparse { - copy_sparse(&infd, &outfd, uspace)? + copy_sparse(&infd, &outfd, uspace, len)? } else { - let len = infd.metadata()?.len(); copy_range(&infd, &outfd, uspace, len)? }; - outfd.set_permissions(infd.metadata()?.permissions())?; + outfd.set_permissions(in_meta.permissions())?; Ok(total) } From f1e7e35764de80a6da3183132bb657b51f53e954 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 20 Nov 2018 15:20:21 +1100 Subject: [PATCH 27/27] Add copyright header. --- src/libstd/sys/unix/fs_linux.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libstd/sys/unix/fs_linux.rs b/src/libstd/sys/unix/fs_linux.rs index 51dfae64cbb25..659ee4678ee17 100644 --- a/src/libstd/sys/unix/fs_linux.rs +++ b/src/libstd/sys/unix/fs_linux.rs @@ -1,3 +1,12 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. use cell::RefCell; use cmp;