1//! This module contains specializations that can offload `io::copy()` operations on file descriptor
2//! containing types (`File`, `TcpStream`, etc.) to more efficient syscalls than `read(2)` and `write(2)`.
3//!
4//! Specialization is only applied to wholly std-owned types so that user code can't observe
5//! that the `Read` and `Write` traits are not used.
6//!
7//! Since a copy operation involves a reader and writer side where each can consist of different types
8//! and also involve generic wrappers (e.g. `Take`, `BufReader`) it is not practical to specialize
9//! a single method on all possible combinations.
10//!
11//! Instead readers and writers are handled separately by the `CopyRead` and `CopyWrite` specialization
12//! traits and then specialized on by the `Copier::copy` method.
13//!
14//! `Copier` uses the specialization traits to unpack the underlying file descriptors and
15//! additional prerequisites and constraints imposed by the wrapper types.
16//!
17//! Once it has obtained all necessary pieces and brought any wrapper types into a state where they
18//! can be safely bypassed it will attempt to use the `copy_file_range(2)`,
19//! `sendfile(2)` or `splice(2)` syscalls to move data directly between file descriptors.
20//! Since those syscalls have requirements that cannot be fully checked in advance it attempts
21//! to use them one after another (guided by hints) to figure out which one works and
22//! falls back to the generic read-write copy loop if none of them does.
23//! Once a working syscall is found for a pair of file descriptors it will be called in a loop
24//! until the copy operation is completed.
25//!
26//! Advantages of using these syscalls:
27//!
28//! * fewer context switches since reads and writes are coalesced into a single syscall
29//! and more bytes are transferred per syscall. This translates to higher throughput
30//! and fewer CPU cycles, at least for sufficiently large transfers to amortize the initial probing.
31//! * `copy_file_range` creates reflink copies on CoW filesystems, thus moving less data and
32//! consuming less disk space
33//! * `sendfile` and `splice` can perform zero-copy IO under some circumstances while
34//! a naive copy loop would move every byte through the CPU.
35//!
36//! Drawbacks:
37//!
38//! * copy operations smaller than the default buffer size can under some circumstances, especially
39//! on older kernels, incur more syscalls than the naive approach would. As mentioned above
40//! the syscall selection is guided by hints to minimize this possibility but they are not perfect.
41//! * optimizations only apply to std types. If a user adds a custom wrapper type, e.g. to report
42//! progress, they can hit a performance cliff.
43//! * complexity
44
45#[cfg(not(any(all(target_os = "linux", target_env = "gnu"), target_os = "hurd")))]
46use libc::sendfile as sendfile64;
47#[cfg(any(all(target_os = "linux", target_env = "gnu"), target_os = "hurd"))]
48use libc::sendfile64;
49use libc::{EBADF, EINVAL, ENOSYS, EOPNOTSUPP, EOVERFLOW, EPERM, EXDEV};
50
51use super::CopyState;
52use crate::cmp::min;
53use crate::fs::{File, Metadata};
54use crate::io::{
55 BufRead, BufReader, BufWriter, Error, PipeReader, PipeWriter, Read, Result, StderrLock,
56 StdinLock, StdoutLock, Take, Write,
57};
58use crate::mem::ManuallyDrop;
59use crate::net::TcpStream;
60use crate::os::unix::fs::FileTypeExt;
61use crate::os::unix::io::{AsRawFd, FromRawFd, RawFd};
62use crate::os::unix::net::UnixStream;
63use crate::process::{ChildStderr, ChildStdin, ChildStdout};
64use crate::ptr;
65use crate::sync::atomic::{Atomic, AtomicBool, AtomicU8, Ordering};
66use crate::sys::cvt;
67use crate::sys::fs::CachedFileMetadata;
68use crate::sys::weak::syscall;
69
70#[cfg(test)]
71mod tests;
72
73pub fn kernel_copy<R: Read + ?Sized, W: Write + ?Sized>(
74 read: &mut R,
75 write: &mut W,
76) -> Result<CopyState> {
77 let copier: Copier<'_, '_, R, W> = Copier { read, write };
78 SpecCopy::copy(self:copier)
79}
80
81/// This type represents either the inferred `FileType` of a `RawFd` based on the source
82/// type from which it was extracted or the actual metadata
83///
84/// The methods on this type only provide hints, due to `AsRawFd` and `FromRawFd` the inferred
85/// type may be wrong.
86enum FdMeta {
87 Metadata(Metadata),
88 Socket,
89 Pipe,
90 /// We don't have any metadata because the stat syscall failed
91 NoneObtained,
92}
93
94#[derive(PartialEq)]
95enum FdHandle {
96 Input,
97 Output,
98}
99
100impl FdMeta {
101 fn maybe_fifo(&self) -> bool {
102 match self {
103 FdMeta::Metadata(meta) => meta.file_type().is_fifo(),
104 FdMeta::Socket => false,
105 FdMeta::Pipe => true,
106 FdMeta::NoneObtained => true,
107 }
108 }
109
110 fn potential_sendfile_source(&self) -> bool {
111 match self {
112 // procfs erroneously shows 0 length on non-empty readable files.
113 // and if a file is truly empty then a `read` syscall will determine that and skip the write syscall
114 // thus there would be benefit from attempting sendfile
115 FdMeta::Metadata(meta)
116 if meta.file_type().is_file() && meta.len() > 0
117 || meta.file_type().is_block_device() =>
118 {
119 true
120 }
121 _ => false,
122 }
123 }
124
125 fn copy_file_range_candidate(&self, f: FdHandle) -> bool {
126 match self {
127 // copy_file_range will fail on empty procfs files. `read` can determine whether EOF has been reached
128 // without extra cost and skip the write, thus there is no benefit in attempting copy_file_range
129 FdMeta::Metadata(meta) if f == FdHandle::Input && meta.is_file() && meta.len() > 0 => {
130 true
131 }
132 FdMeta::Metadata(meta) if f == FdHandle::Output && meta.is_file() => true,
133 _ => false,
134 }
135 }
136}
137
138/// Returns true either if changes made to the source after a sendfile/splice call won't become
139/// visible in the sink or the source has explicitly opted into such behavior (e.g. by splicing
140/// a file into a pipe, the pipe being the source in this case).
141///
142/// This will prevent File -> Pipe and File -> Socket splicing/sendfile optimizations to uphold
143/// the Read/Write API semantics of io::copy.
144///
145/// Note: This is not 100% airtight, the caller can use the RawFd conversion methods to turn a
146/// regular file into a TcpSocket which will be treated as a socket here without checking.
147fn safe_kernel_copy(source: &FdMeta, sink: &FdMeta) -> bool {
148 match (source, sink) {
149 // Data arriving from a socket is safe because the sender can't modify the socket buffer.
150 // Data arriving from a pipe is safe(-ish) because either the sender *copied*
151 // the bytes into the pipe OR explicitly performed an operation that enables zero-copy,
152 // thus promising not to modify the data later.
153 (FdMeta::Socket, _) => true,
154 (FdMeta::Pipe, _) => true,
155 (FdMeta::Metadata(meta: &Metadata), _)
156 if meta.file_type().is_fifo() || meta.file_type().is_socket() =>
157 {
158 true
159 }
160 // Data going into non-pipes/non-sockets is safe because the "later changes may become visible" issue
161 // only happens for pages sitting in send buffers or pipes.
162 (_, FdMeta::Metadata(meta: &Metadata))
163 if !meta.file_type().is_fifo() && !meta.file_type().is_socket() =>
164 {
165 true
166 }
167 _ => false,
168 }
169}
170
171struct CopyParams(FdMeta, Option<RawFd>);
172
173struct Copier<'a, 'b, R: Read + ?Sized, W: Write + ?Sized> {
174 read: &'a mut R,
175 write: &'b mut W,
176}
177
178trait SpecCopy {
179 fn copy(self) -> Result<CopyState>;
180}
181
182impl<R: Read + ?Sized, W: Write + ?Sized> SpecCopy for Copier<'_, '_, R, W> {
183 default fn copy(self) -> Result<CopyState> {
184 Ok(CopyState::Fallback(0))
185 }
186}
187
188impl<R: CopyRead, W: CopyWrite> SpecCopy for Copier<'_, '_, R, W> {
189 fn copy(self) -> Result<CopyState> {
190 let (reader, writer) = (self.read, self.write);
191 let r_cfg = reader.properties();
192 let w_cfg = writer.properties();
193
194 // before direct operations on file descriptors ensure that all source and sink buffers are empty
195 let mut flush = || -> Result<u64> {
196 let bytes = reader.drain_to(writer, u64::MAX)?;
197 // BufWriter buffered bytes have already been accounted for in earlier write() calls
198 writer.flush()?;
199 Ok(bytes)
200 };
201
202 let mut written = 0u64;
203
204 if let (CopyParams(input_meta, Some(readfd)), CopyParams(output_meta, Some(writefd))) =
205 (r_cfg, w_cfg)
206 {
207 written += flush()?;
208 let max_write = reader.min_limit();
209
210 if input_meta.copy_file_range_candidate(FdHandle::Input)
211 && output_meta.copy_file_range_candidate(FdHandle::Output)
212 {
213 let result = copy_regular_files(readfd, writefd, max_write);
214 result.update_take(reader);
215
216 match result {
217 CopyResult::Ended(bytes_copied) => {
218 return Ok(CopyState::Ended(bytes_copied + written));
219 }
220 CopyResult::Error(e, _) => return Err(e),
221 CopyResult::Fallback(bytes) => written += bytes,
222 }
223 }
224
225 // on modern kernels sendfile can copy from any mmapable type (some but not all regular files and block devices)
226 // to any writable file descriptor. On older kernels the writer side can only be a socket.
227 // So we just try and fallback if needed.
228 // If current file offsets + write sizes overflow it may also fail, we do not try to fix that and instead
229 // fall back to the generic copy loop.
230 if input_meta.potential_sendfile_source() && safe_kernel_copy(&input_meta, &output_meta)
231 {
232 let result = sendfile_splice(SpliceMode::Sendfile, readfd, writefd, max_write);
233 result.update_take(reader);
234
235 match result {
236 CopyResult::Ended(bytes_copied) => {
237 return Ok(CopyState::Ended(bytes_copied + written));
238 }
239 CopyResult::Error(e, _) => return Err(e),
240 CopyResult::Fallback(bytes) => written += bytes,
241 }
242 }
243
244 if (input_meta.maybe_fifo() || output_meta.maybe_fifo())
245 && safe_kernel_copy(&input_meta, &output_meta)
246 {
247 let result = sendfile_splice(SpliceMode::Splice, readfd, writefd, max_write);
248 result.update_take(reader);
249
250 match result {
251 CopyResult::Ended(bytes_copied) => {
252 return Ok(CopyState::Ended(bytes_copied + written));
253 }
254 CopyResult::Error(e, _) => return Err(e),
255 CopyResult::Fallback(0) => { /* use the fallback below */ }
256 CopyResult::Fallback(_) => {
257 unreachable!("splice should not return > 0 bytes on the fallback path")
258 }
259 }
260 }
261 }
262
263 // fallback if none of the more specialized syscalls wants to work with these file descriptors
264 Ok(CopyState::Fallback(written))
265 }
266}
267
268#[rustc_specialization_trait]
269trait CopyRead: Read {
270 /// Implementations that contain buffers (i.e. `BufReader`) must transfer data from their internal
271 /// buffers into `writer` until either the buffers are emptied or `limit` bytes have been
272 /// transferred, whichever occurs sooner.
273 /// If nested buffers are present the outer buffers must be drained first.
274 ///
275 /// This is necessary to directly bypass the wrapper types while preserving the data order
276 /// when operating directly on the underlying file descriptors.
277 fn drain_to<W: Write>(&mut self, _writer: &mut W, _limit: u64) -> Result<u64> {
278 Ok(0)
279 }
280
281 /// Updates `Take` wrappers to remove the number of bytes copied.
282 fn taken(&mut self, _bytes: u64) {}
283
284 /// The minimum of the limit of all `Take<_>` wrappers, `u64::MAX` otherwise.
285 /// This method does not account for data `BufReader` buffers and would underreport
286 /// the limit of a `Take<BufReader<Take<_>>>` type. Thus its result is only valid
287 /// after draining the buffers via `drain_to`.
288 fn min_limit(&self) -> u64 {
289 u64::MAX
290 }
291
292 /// Extracts the file descriptor and hints/metadata, delegating through wrappers if necessary.
293 fn properties(&self) -> CopyParams;
294}
295
296#[rustc_specialization_trait]
297trait CopyWrite: Write {
298 /// Extracts the file descriptor and hints/metadata, delegating through wrappers if necessary.
299 fn properties(&self) -> CopyParams;
300}
301
302impl<T> CopyRead for &mut T
303where
304 T: CopyRead,
305{
306 fn drain_to<W: Write>(&mut self, writer: &mut W, limit: u64) -> Result<u64> {
307 (**self).drain_to(writer, limit)
308 }
309
310 fn taken(&mut self, bytes: u64) {
311 (**self).taken(bytes);
312 }
313
314 fn min_limit(&self) -> u64 {
315 (**self).min_limit()
316 }
317
318 fn properties(&self) -> CopyParams {
319 (**self).properties()
320 }
321}
322
323impl<T> CopyWrite for &mut T
324where
325 T: CopyWrite,
326{
327 fn properties(&self) -> CopyParams {
328 (**self).properties()
329 }
330}
331
332impl CopyRead for File {
333 fn properties(&self) -> CopyParams {
334 CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
335 }
336}
337
338impl CopyRead for &File {
339 fn properties(&self) -> CopyParams {
340 CopyParams(fd_to_meta(*self), Some(self.as_raw_fd()))
341 }
342}
343
344impl CopyWrite for File {
345 fn properties(&self) -> CopyParams {
346 CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
347 }
348}
349
350impl CopyWrite for &File {
351 fn properties(&self) -> CopyParams {
352 CopyParams(fd_to_meta(*self), Some(self.as_raw_fd()))
353 }
354}
355
356impl CopyRead for TcpStream {
357 fn properties(&self) -> CopyParams {
358 // avoid the stat syscall since we can be fairly sure it's a socket
359 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
360 }
361}
362
363impl CopyRead for &TcpStream {
364 fn properties(&self) -> CopyParams {
365 // avoid the stat syscall since we can be fairly sure it's a socket
366 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
367 }
368}
369
370impl CopyWrite for TcpStream {
371 fn properties(&self) -> CopyParams {
372 // avoid the stat syscall since we can be fairly sure it's a socket
373 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
374 }
375}
376
377impl CopyWrite for &TcpStream {
378 fn properties(&self) -> CopyParams {
379 // avoid the stat syscall since we can be fairly sure it's a socket
380 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
381 }
382}
383
384impl CopyRead for UnixStream {
385 fn properties(&self) -> CopyParams {
386 // avoid the stat syscall since we can be fairly sure it's a socket
387 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
388 }
389}
390
391impl CopyRead for &UnixStream {
392 fn properties(&self) -> CopyParams {
393 // avoid the stat syscall since we can be fairly sure it's a socket
394 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
395 }
396}
397
398impl CopyWrite for UnixStream {
399 fn properties(&self) -> CopyParams {
400 // avoid the stat syscall since we can be fairly sure it's a socket
401 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
402 }
403}
404
405impl CopyWrite for &UnixStream {
406 fn properties(&self) -> CopyParams {
407 // avoid the stat syscall since we can be fairly sure it's a socket
408 CopyParams(FdMeta::Socket, Some(self.as_raw_fd()))
409 }
410}
411
412impl CopyRead for PipeReader {
413 fn properties(&self) -> CopyParams {
414 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
415 }
416}
417
418impl CopyRead for &PipeReader {
419 fn properties(&self) -> CopyParams {
420 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
421 }
422}
423
424impl CopyWrite for PipeWriter {
425 fn properties(&self) -> CopyParams {
426 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
427 }
428}
429
430impl CopyWrite for &PipeWriter {
431 fn properties(&self) -> CopyParams {
432 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
433 }
434}
435
436impl CopyWrite for ChildStdin {
437 fn properties(&self) -> CopyParams {
438 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
439 }
440}
441
442impl CopyRead for ChildStdout {
443 fn properties(&self) -> CopyParams {
444 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
445 }
446}
447
448impl CopyRead for ChildStderr {
449 fn properties(&self) -> CopyParams {
450 CopyParams(FdMeta::Pipe, Some(self.as_raw_fd()))
451 }
452}
453
454impl CopyRead for StdinLock<'_> {
455 fn drain_to<W: Write>(&mut self, writer: &mut W, outer_limit: u64) -> Result<u64> {
456 let buf_reader: &mut BufReader = self.as_mut_buf();
457 let buf: &[u8] = buf_reader.buffer();
458 let buf: &[u8] = &buf[0..min(v1:buf.len(), v2:outer_limit.try_into().unwrap_or(default:usize::MAX))];
459 let bytes_drained: usize = buf.len();
460 writer.write_all(buf)?;
461 buf_reader.consume(amount:bytes_drained);
462
463 Ok(bytes_drained as u64)
464 }
465
466 fn properties(&self) -> CopyParams {
467 CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
468 }
469}
470
471impl CopyWrite for StdoutLock<'_> {
472 fn properties(&self) -> CopyParams {
473 CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
474 }
475}
476
477impl CopyWrite for StderrLock<'_> {
478 fn properties(&self) -> CopyParams {
479 CopyParams(fd_to_meta(self), Some(self.as_raw_fd()))
480 }
481}
482
483impl<T: CopyRead> CopyRead for Take<T> {
484 fn drain_to<W: Write>(&mut self, writer: &mut W, outer_limit: u64) -> Result<u64> {
485 let local_limit: u64 = self.limit();
486 let combined_limit: u64 = min(v1:outer_limit, v2:local_limit);
487 let bytes_drained: u64 = self.get_mut().drain_to(writer, combined_limit)?;
488 // update limit since read() was bypassed
489 self.set_limit(local_limit - bytes_drained);
490
491 Ok(bytes_drained)
492 }
493
494 fn taken(&mut self, bytes: u64) {
495 self.set_limit(self.limit() - bytes);
496 self.get_mut().taken(bytes);
497 }
498
499 fn min_limit(&self) -> u64 {
500 min(v1:Take::limit(self), self.get_ref().min_limit())
501 }
502
503 fn properties(&self) -> CopyParams {
504 self.get_ref().properties()
505 }
506}
507
508impl<T: ?Sized + CopyRead> CopyRead for BufReader<T> {
509 fn drain_to<W: Write>(&mut self, writer: &mut W, outer_limit: u64) -> Result<u64> {
510 let buf = self.buffer();
511 let buf = &buf[0..min(buf.len(), outer_limit.try_into().unwrap_or(usize::MAX))];
512 let bytes = buf.len();
513 writer.write_all(buf)?;
514 self.consume(bytes);
515
516 let remaining = outer_limit - bytes as u64;
517
518 // in case of nested bufreaders we also need to drain the ones closer to the source
519 let inner_bytes = self.get_mut().drain_to(writer, remaining)?;
520
521 Ok(bytes as u64 + inner_bytes)
522 }
523
524 fn taken(&mut self, bytes: u64) {
525 self.get_mut().taken(bytes);
526 }
527
528 fn min_limit(&self) -> u64 {
529 self.get_ref().min_limit()
530 }
531
532 fn properties(&self) -> CopyParams {
533 self.get_ref().properties()
534 }
535}
536
537impl<T: ?Sized + CopyWrite> CopyWrite for BufWriter<T> {
538 fn properties(&self) -> CopyParams {
539 self.get_ref().properties()
540 }
541}
542
543impl CopyRead for CachedFileMetadata {
544 fn properties(&self) -> CopyParams {
545 CopyParams(FdMeta::Metadata(self.1.clone()), Some(self.0.as_raw_fd()))
546 }
547}
548
549impl CopyWrite for CachedFileMetadata {
550 fn properties(&self) -> CopyParams {
551 CopyParams(FdMeta::Metadata(self.1.clone()), Some(self.0.as_raw_fd()))
552 }
553}
554
555fn fd_to_meta<T: AsRawFd>(fd: &T) -> FdMeta {
556 let fd: i32 = fd.as_raw_fd();
557 let file: ManuallyDrop<File> = ManuallyDrop::new(unsafe { File::from_raw_fd(fd) });
558 match file.metadata() {
559 Ok(meta: Metadata) => FdMeta::Metadata(meta),
560 Err(_) => FdMeta::NoneObtained,
561 }
562}
563
564enum CopyResult {
565 Ended(u64),
566 Error(Error, u64),
567 Fallback(u64),
568}
569
570impl CopyResult {
571 fn update_take(&self, reader: &mut impl CopyRead) {
572 match *self {
573 CopyResult::Fallback(bytes: u64)
574 | CopyResult::Ended(bytes: u64)
575 | CopyResult::Error(_, bytes: u64) => reader.taken(bytes),
576 }
577 }
578}
579
580/// Invalid file descriptor.
581///
582/// Valid file descriptors are guaranteed to be positive numbers (see `open()` manpage)
583/// while negative values are used to indicate errors.
584/// Thus -1 will never be overlap with a valid open file.
585const INVALID_FD: RawFd = -1;
586
587/// Linux-specific implementation that will attempt to use copy_file_range for copy offloading.
588/// As the name says, it only works on regular files.
589///
590/// Callers must handle fallback to a generic copy loop.
591/// `Fallback` may indicate non-zero number of bytes already written
592/// if one of the files' cursor +`max_len` would exceed u64::MAX (`EOVERFLOW`).
593fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> CopyResult {
594 use crate::cmp;
595
596 const NOT_PROBED: u8 = 0;
597 const UNAVAILABLE: u8 = 1;
598 const AVAILABLE: u8 = 2;
599
600 // Kernel prior to 4.5 don't have copy_file_range
601 // We store the availability in a global to avoid unnecessary syscalls
602 static HAS_COPY_FILE_RANGE: Atomic<u8> = AtomicU8::new(NOT_PROBED);
603
604 let mut have_probed = match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
605 NOT_PROBED => false,
606 UNAVAILABLE => return CopyResult::Fallback(0),
607 _ => true,
608 };
609
610 syscall!(
611 fn copy_file_range(
612 fd_in: libc::c_int,
613 off_in: *mut libc::loff_t,
614 fd_out: libc::c_int,
615 off_out: *mut libc::loff_t,
616 len: libc::size_t,
617 flags: libc::c_uint,
618 ) -> libc::ssize_t;
619 );
620
621 fn probe_copy_file_range_support() -> u8 {
622 // In some cases, we cannot determine availability from the first
623 // `copy_file_range` call. In this case, we probe with an invalid file
624 // descriptor so that the results are easily interpretable.
625 match unsafe {
626 cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0))
627 .map_err(|e| e.raw_os_error())
628 } {
629 Err(Some(EPERM | ENOSYS)) => UNAVAILABLE,
630 Err(Some(EBADF)) => AVAILABLE,
631 Ok(_) => panic!("unexpected copy_file_range probe success"),
632 // Treat other errors as the syscall
633 // being unavailable.
634 Err(_) => UNAVAILABLE,
635 }
636 }
637
638 let mut written = 0u64;
639 while written < max_len {
640 let bytes_to_copy = cmp::min(max_len - written, usize::MAX as u64);
641 // cap to 1GB chunks in case u64::MAX is passed as max_len and the file has a non-zero seek position
642 // this allows us to copy large chunks without hitting EOVERFLOW,
643 // unless someone sets a file offset close to u64::MAX - 1GB, in which case a fallback would be required
644 let bytes_to_copy = cmp::min(bytes_to_copy as usize, 0x4000_0000usize);
645 let copy_result = unsafe {
646 // We actually don't have to adjust the offsets,
647 // because copy_file_range adjusts the file offset automatically
648 cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0))
649 };
650
651 if !have_probed && copy_result.is_ok() {
652 have_probed = true;
653 HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
654 }
655
656 match copy_result {
657 Ok(0) if written == 0 => {
658 // fallback to work around several kernel bugs where copy_file_range will fail to
659 // copy any bytes and return 0 instead of an error if
660 // - reading virtual files from the proc filesystem which appear to have 0 size
661 // but are not empty. noted in coreutils to affect kernels at least up to 5.6.19.
662 // - copying from an overlay filesystem in docker. reported to occur on fedora 32.
663 return CopyResult::Fallback(0);
664 }
665 Ok(0) => return CopyResult::Ended(written), // reached EOF
666 Ok(ret) => written += ret as u64,
667 Err(err) => {
668 return match err.raw_os_error() {
669 // when file offset + max_length > u64::MAX
670 Some(EOVERFLOW) => CopyResult::Fallback(written),
671 Some(raw_os_error @ (ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF))
672 if written == 0 =>
673 {
674 if !have_probed {
675 let available = if matches!(raw_os_error, ENOSYS | EOPNOTSUPP | EPERM) {
676 // EPERM can indicate seccomp filters or an
677 // immutable file. To distinguish these
678 // cases we probe with invalid file
679 // descriptors which should result in EBADF
680 // if the syscall is supported and EPERM or
681 // ENOSYS if it's not available.
682 //
683 // For EOPNOTSUPP, see below. In the case of
684 // ENOSYS, we try to cover for faulty FUSE
685 // drivers.
686 probe_copy_file_range_support()
687 } else {
688 AVAILABLE
689 };
690 HAS_COPY_FILE_RANGE.store(available, Ordering::Relaxed);
691 }
692
693 // Try fallback io::copy if either:
694 // - Kernel version is < 4.5 (ENOSYS¹)
695 // - Files are mounted on different fs (EXDEV)
696 // - copy_file_range is broken in various ways on RHEL/CentOS 7 (EOPNOTSUPP)
697 // - copy_file_range file is immutable or syscall is blocked by seccomp¹ (EPERM)
698 // - copy_file_range cannot be used with pipes or device nodes (EINVAL)
699 // - the writer fd was opened with O_APPEND (EBADF²)
700 // and no bytes were written successfully yet. (All these errnos should
701 // not be returned if something was already written, but they happen in
702 // the wild, see #91152.)
703 //
704 // ¹ these cases should be detected by the initial probe but we handle them here
705 // anyway in case syscall interception changes during runtime
706 // ² actually invalid file descriptors would cause this too, but in that case
707 // the fallback code path is expected to encounter the same error again
708 CopyResult::Fallback(0)
709 }
710 _ => CopyResult::Error(err, written),
711 };
712 }
713 }
714 }
715 CopyResult::Ended(written)
716}
717
718#[derive(PartialEq)]
719enum SpliceMode {
720 Sendfile,
721 Splice,
722}
723
724/// performs splice or sendfile between file descriptors
725/// Does _not_ fall back to a generic copy loop.
726fn sendfile_splice(mode: SpliceMode, reader: RawFd, writer: RawFd, len: u64) -> CopyResult {
727 static HAS_SENDFILE: Atomic<bool> = AtomicBool::new(true);
728 static HAS_SPLICE: Atomic<bool> = AtomicBool::new(true);
729
730 // Android builds use feature level 14, but the libc wrapper for splice is
731 // gated on feature level 21+, so we have to invoke the syscall directly.
732 #[cfg(target_os = "android")]
733 syscall!(
734 fn splice(
735 srcfd: libc::c_int,
736 src_offset: *const i64,
737 dstfd: libc::c_int,
738 dst_offset: *const i64,
739 len: libc::size_t,
740 flags: libc::c_int,
741 ) -> libc::ssize_t;
742 );
743
744 #[cfg(target_os = "linux")]
745 use libc::splice;
746
747 match mode {
748 SpliceMode::Sendfile if !HAS_SENDFILE.load(Ordering::Relaxed) => {
749 return CopyResult::Fallback(0);
750 }
751 SpliceMode::Splice if !HAS_SPLICE.load(Ordering::Relaxed) => {
752 return CopyResult::Fallback(0);
753 }
754 _ => (),
755 }
756
757 let mut written = 0u64;
758 while written < len {
759 // according to its manpage that's the maximum size sendfile() will copy per invocation
760 let chunk_size = crate::cmp::min(len - written, 0x7ffff000_u64) as usize;
761
762 let result = match mode {
763 SpliceMode::Sendfile => {
764 cvt(unsafe { sendfile64(writer, reader, ptr::null_mut(), chunk_size) })
765 }
766 SpliceMode::Splice => cvt(unsafe {
767 splice(reader, ptr::null_mut(), writer, ptr::null_mut(), chunk_size, 0)
768 }),
769 };
770
771 match result {
772 Ok(0) => break, // EOF
773 Ok(ret) => written += ret as u64,
774 Err(err) => {
775 return match err.raw_os_error() {
776 Some(ENOSYS | EPERM) => {
777 // syscall not supported (ENOSYS)
778 // syscall is disallowed, e.g. by seccomp (EPERM)
779 match mode {
780 SpliceMode::Sendfile => HAS_SENDFILE.store(false, Ordering::Relaxed),
781 SpliceMode::Splice => HAS_SPLICE.store(false, Ordering::Relaxed),
782 }
783 assert_eq!(written, 0);
784 CopyResult::Fallback(0)
785 }
786 Some(EINVAL) => {
787 // splice/sendfile do not support this particular file descriptor (EINVAL)
788 assert_eq!(written, 0);
789 CopyResult::Fallback(0)
790 }
791 Some(os_err) if mode == SpliceMode::Sendfile && os_err == EOVERFLOW => {
792 CopyResult::Fallback(written)
793 }
794 _ => CopyResult::Error(err, written),
795 };
796 }
797 }
798 }
799 CopyResult::Ended(written)
800}
801