1use core::iter::FromIterator;
2use core::ops::{Deref, RangeBounds};
3use core::{cmp, fmt, hash, mem, ptr, slice, usize};
4
5use alloc::{
6 alloc::{dealloc, Layout},
7 borrow::Borrow,
8 boxed::Box,
9 string::String,
10 vec::Vec,
11};
12
13use crate::buf::IntoIter;
14#[allow(unused)]
15use crate::loom::sync::atomic::AtomicMut;
16use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize, Ordering};
17use crate::Buf;
18
19/// A cheaply cloneable and sliceable chunk of contiguous memory.
20///
21/// `Bytes` is an efficient container for storing and operating on contiguous
22/// slices of memory. It is intended for use primarily in networking code, but
23/// could have applications elsewhere as well.
24///
25/// `Bytes` values facilitate zero-copy network programming by allowing multiple
26/// `Bytes` objects to point to the same underlying memory.
27///
28/// `Bytes` does not have a single implementation. It is an interface, whose
29/// exact behavior is implemented through dynamic dispatch in several underlying
30/// implementations of `Bytes`.
31///
32/// All `Bytes` implementations must fulfill the following requirements:
33/// - They are cheaply cloneable and thereby shareable between an unlimited amount
34/// of components, for example by modifying a reference count.
35/// - Instances can be sliced to refer to a subset of the original buffer.
36///
37/// ```
38/// use bytes::Bytes;
39///
40/// let mut mem = Bytes::from("Hello world");
41/// let a = mem.slice(0..5);
42///
43/// assert_eq!(a, "Hello");
44///
45/// let b = mem.split_to(6);
46///
47/// assert_eq!(mem, "world");
48/// assert_eq!(b, "Hello ");
49/// ```
50///
51/// # Memory layout
52///
53/// The `Bytes` struct itself is fairly small, limited to 4 `usize` fields used
54/// to track information about which segment of the underlying memory the
55/// `Bytes` handle has access to.
56///
57/// `Bytes` keeps both a pointer to the shared state containing the full memory
58/// slice and a pointer to the start of the region visible by the handle.
59/// `Bytes` also tracks the length of its view into the memory.
60///
61/// # Sharing
62///
63/// `Bytes` contains a vtable, which allows implementations of `Bytes` to define
64/// how sharing/cloning is implemented in detail.
65/// When `Bytes::clone()` is called, `Bytes` will call the vtable function for
66/// cloning the backing storage in order to share it behind between multiple
67/// `Bytes` instances.
68///
69/// For `Bytes` implementations which refer to constant memory (e.g. created
70/// via `Bytes::from_static()`) the cloning implementation will be a no-op.
71///
72/// For `Bytes` implementations which point to a reference counted shared storage
73/// (e.g. an `Arc<[u8]>`), sharing will be implemented by increasing the
74/// reference count.
75///
76/// Due to this mechanism, multiple `Bytes` instances may point to the same
77/// shared memory region.
78/// Each `Bytes` instance can point to different sections within that
79/// memory region, and `Bytes` instances may or may not have overlapping views
80/// into the memory.
81///
82/// The following diagram visualizes a scenario where 2 `Bytes` instances make
83/// use of an `Arc`-based backing storage, and provide access to different views:
84///
85/// ```text
86///
87/// Arc ptrs ┌─────────┐
88/// ________________________ / │ Bytes 2 │
89/// / └─────────┘
90/// / ┌───────────┐ | |
91/// |_________/ │ Bytes 1 │ | |
92/// | └───────────┘ | |
93/// | | | ___/ data | tail
94/// | data | tail |/ |
95/// v v v v
96/// ┌─────┬─────┬───────────┬───────────────┬─────┐
97/// │ Arc │ │ │ │ │
98/// └─────┴─────┴───────────┴───────────────┴─────┘
99/// ```
100pub struct Bytes {
101 ptr: *const u8,
102 len: usize,
103 // inlined "trait object"
104 data: AtomicPtr<()>,
105 vtable: &'static Vtable,
106}
107
108pub(crate) struct Vtable {
109 /// fn(data, ptr, len)
110 pub clone: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Bytes,
111 /// fn(data, ptr, len)
112 ///
113 /// takes `Bytes` to value
114 pub to_vec: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Vec<u8>,
115 /// fn(data, ptr, len)
116 pub drop: unsafe fn(&mut AtomicPtr<()>, *const u8, usize),
117}
118
119impl Bytes {
120 /// Creates a new empty `Bytes`.
121 ///
122 /// This will not allocate and the returned `Bytes` handle will be empty.
123 ///
124 /// # Examples
125 ///
126 /// ```
127 /// use bytes::Bytes;
128 ///
129 /// let b = Bytes::new();
130 /// assert_eq!(&b[..], b"");
131 /// ```
132 #[inline]
133 #[cfg(not(all(loom, test)))]
134 pub const fn new() -> Self {
135 // Make it a named const to work around
136 // "unsizing casts are not allowed in const fn"
137 const EMPTY: &[u8] = &[];
138 Bytes::from_static(EMPTY)
139 }
140
141 #[cfg(all(loom, test))]
142 pub fn new() -> Self {
143 const EMPTY: &[u8] = &[];
144 Bytes::from_static(EMPTY)
145 }
146
147 /// Creates a new `Bytes` from a static slice.
148 ///
149 /// The returned `Bytes` will point directly to the static slice. There is
150 /// no allocating or copying.
151 ///
152 /// # Examples
153 ///
154 /// ```
155 /// use bytes::Bytes;
156 ///
157 /// let b = Bytes::from_static(b"hello");
158 /// assert_eq!(&b[..], b"hello");
159 /// ```
160 #[inline]
161 #[cfg(not(all(loom, test)))]
162 pub const fn from_static(bytes: &'static [u8]) -> Self {
163 Bytes {
164 ptr: bytes.as_ptr(),
165 len: bytes.len(),
166 data: AtomicPtr::new(ptr::null_mut()),
167 vtable: &STATIC_VTABLE,
168 }
169 }
170
171 #[cfg(all(loom, test))]
172 pub fn from_static(bytes: &'static [u8]) -> Self {
173 Bytes {
174 ptr: bytes.as_ptr(),
175 len: bytes.len(),
176 data: AtomicPtr::new(ptr::null_mut()),
177 vtable: &STATIC_VTABLE,
178 }
179 }
180
181 /// Returns the number of bytes contained in this `Bytes`.
182 ///
183 /// # Examples
184 ///
185 /// ```
186 /// use bytes::Bytes;
187 ///
188 /// let b = Bytes::from(&b"hello"[..]);
189 /// assert_eq!(b.len(), 5);
190 /// ```
191 #[inline]
192 pub const fn len(&self) -> usize {
193 self.len
194 }
195
196 /// Returns true if the `Bytes` has a length of 0.
197 ///
198 /// # Examples
199 ///
200 /// ```
201 /// use bytes::Bytes;
202 ///
203 /// let b = Bytes::new();
204 /// assert!(b.is_empty());
205 /// ```
206 #[inline]
207 pub const fn is_empty(&self) -> bool {
208 self.len == 0
209 }
210
211 /// Creates `Bytes` instance from slice, by copying it.
212 pub fn copy_from_slice(data: &[u8]) -> Self {
213 data.to_vec().into()
214 }
215
216 /// Returns a slice of self for the provided range.
217 ///
218 /// This will increment the reference count for the underlying memory and
219 /// return a new `Bytes` handle set to the slice.
220 ///
221 /// This operation is `O(1)`.
222 ///
223 /// # Examples
224 ///
225 /// ```
226 /// use bytes::Bytes;
227 ///
228 /// let a = Bytes::from(&b"hello world"[..]);
229 /// let b = a.slice(2..5);
230 ///
231 /// assert_eq!(&b[..], b"llo");
232 /// ```
233 ///
234 /// # Panics
235 ///
236 /// Requires that `begin <= end` and `end <= self.len()`, otherwise slicing
237 /// will panic.
238 pub fn slice(&self, range: impl RangeBounds<usize>) -> Self {
239 use core::ops::Bound;
240
241 let len = self.len();
242
243 let begin = match range.start_bound() {
244 Bound::Included(&n) => n,
245 Bound::Excluded(&n) => n + 1,
246 Bound::Unbounded => 0,
247 };
248
249 let end = match range.end_bound() {
250 Bound::Included(&n) => n.checked_add(1).expect("out of range"),
251 Bound::Excluded(&n) => n,
252 Bound::Unbounded => len,
253 };
254
255 assert!(
256 begin <= end,
257 "range start must not be greater than end: {:?} <= {:?}",
258 begin,
259 end,
260 );
261 assert!(
262 end <= len,
263 "range end out of bounds: {:?} <= {:?}",
264 end,
265 len,
266 );
267
268 if end == begin {
269 return Bytes::new();
270 }
271
272 let mut ret = self.clone();
273
274 ret.len = end - begin;
275 ret.ptr = unsafe { ret.ptr.add(begin) };
276
277 ret
278 }
279
280 /// Returns a slice of self that is equivalent to the given `subset`.
281 ///
282 /// When processing a `Bytes` buffer with other tools, one often gets a
283 /// `&[u8]` which is in fact a slice of the `Bytes`, i.e. a subset of it.
284 /// This function turns that `&[u8]` into another `Bytes`, as if one had
285 /// called `self.slice()` with the offsets that correspond to `subset`.
286 ///
287 /// This operation is `O(1)`.
288 ///
289 /// # Examples
290 ///
291 /// ```
292 /// use bytes::Bytes;
293 ///
294 /// let bytes = Bytes::from(&b"012345678"[..]);
295 /// let as_slice = bytes.as_ref();
296 /// let subset = &as_slice[2..6];
297 /// let subslice = bytes.slice_ref(&subset);
298 /// assert_eq!(&subslice[..], b"2345");
299 /// ```
300 ///
301 /// # Panics
302 ///
303 /// Requires that the given `sub` slice is in fact contained within the
304 /// `Bytes` buffer; otherwise this function will panic.
305 pub fn slice_ref(&self, subset: &[u8]) -> Self {
306 // Empty slice and empty Bytes may have their pointers reset
307 // so explicitly allow empty slice to be a subslice of any slice.
308 if subset.is_empty() {
309 return Bytes::new();
310 }
311
312 let bytes_p = self.as_ptr() as usize;
313 let bytes_len = self.len();
314
315 let sub_p = subset.as_ptr() as usize;
316 let sub_len = subset.len();
317
318 assert!(
319 sub_p >= bytes_p,
320 "subset pointer ({:p}) is smaller than self pointer ({:p})",
321 subset.as_ptr(),
322 self.as_ptr(),
323 );
324 assert!(
325 sub_p + sub_len <= bytes_p + bytes_len,
326 "subset is out of bounds: self = ({:p}, {}), subset = ({:p}, {})",
327 self.as_ptr(),
328 bytes_len,
329 subset.as_ptr(),
330 sub_len,
331 );
332
333 let sub_offset = sub_p - bytes_p;
334
335 self.slice(sub_offset..(sub_offset + sub_len))
336 }
337
338 /// Splits the bytes into two at the given index.
339 ///
340 /// Afterwards `self` contains elements `[0, at)`, and the returned `Bytes`
341 /// contains elements `[at, len)`.
342 ///
343 /// This is an `O(1)` operation that just increases the reference count and
344 /// sets a few indices.
345 ///
346 /// # Examples
347 ///
348 /// ```
349 /// use bytes::Bytes;
350 ///
351 /// let mut a = Bytes::from(&b"hello world"[..]);
352 /// let b = a.split_off(5);
353 ///
354 /// assert_eq!(&a[..], b"hello");
355 /// assert_eq!(&b[..], b" world");
356 /// ```
357 ///
358 /// # Panics
359 ///
360 /// Panics if `at > len`.
361 #[must_use = "consider Bytes::truncate if you don't need the other half"]
362 pub fn split_off(&mut self, at: usize) -> Self {
363 assert!(
364 at <= self.len(),
365 "split_off out of bounds: {:?} <= {:?}",
366 at,
367 self.len(),
368 );
369
370 if at == self.len() {
371 return Bytes::new();
372 }
373
374 if at == 0 {
375 return mem::replace(self, Bytes::new());
376 }
377
378 let mut ret = self.clone();
379
380 self.len = at;
381
382 unsafe { ret.inc_start(at) };
383
384 ret
385 }
386
387 /// Splits the bytes into two at the given index.
388 ///
389 /// Afterwards `self` contains elements `[at, len)`, and the returned
390 /// `Bytes` contains elements `[0, at)`.
391 ///
392 /// This is an `O(1)` operation that just increases the reference count and
393 /// sets a few indices.
394 ///
395 /// # Examples
396 ///
397 /// ```
398 /// use bytes::Bytes;
399 ///
400 /// let mut a = Bytes::from(&b"hello world"[..]);
401 /// let b = a.split_to(5);
402 ///
403 /// assert_eq!(&a[..], b" world");
404 /// assert_eq!(&b[..], b"hello");
405 /// ```
406 ///
407 /// # Panics
408 ///
409 /// Panics if `at > len`.
410 #[must_use = "consider Bytes::advance if you don't need the other half"]
411 pub fn split_to(&mut self, at: usize) -> Self {
412 assert!(
413 at <= self.len(),
414 "split_to out of bounds: {:?} <= {:?}",
415 at,
416 self.len(),
417 );
418
419 if at == self.len() {
420 return mem::replace(self, Bytes::new());
421 }
422
423 if at == 0 {
424 return Bytes::new();
425 }
426
427 let mut ret = self.clone();
428
429 unsafe { self.inc_start(at) };
430
431 ret.len = at;
432 ret
433 }
434
435 /// Shortens the buffer, keeping the first `len` bytes and dropping the
436 /// rest.
437 ///
438 /// If `len` is greater than the buffer's current length, this has no
439 /// effect.
440 ///
441 /// The [`split_off`] method can emulate `truncate`, but this causes the
442 /// excess bytes to be returned instead of dropped.
443 ///
444 /// # Examples
445 ///
446 /// ```
447 /// use bytes::Bytes;
448 ///
449 /// let mut buf = Bytes::from(&b"hello world"[..]);
450 /// buf.truncate(5);
451 /// assert_eq!(buf, b"hello"[..]);
452 /// ```
453 ///
454 /// [`split_off`]: #method.split_off
455 #[inline]
456 pub fn truncate(&mut self, len: usize) {
457 if len < self.len {
458 // The Vec "promotable" vtables do not store the capacity,
459 // so we cannot truncate while using this repr. We *have* to
460 // promote using `split_off` so the capacity can be stored.
461 if self.vtable as *const Vtable == &PROMOTABLE_EVEN_VTABLE
462 || self.vtable as *const Vtable == &PROMOTABLE_ODD_VTABLE
463 {
464 drop(self.split_off(len));
465 } else {
466 self.len = len;
467 }
468 }
469 }
470
471 /// Clears the buffer, removing all data.
472 ///
473 /// # Examples
474 ///
475 /// ```
476 /// use bytes::Bytes;
477 ///
478 /// let mut buf = Bytes::from(&b"hello world"[..]);
479 /// buf.clear();
480 /// assert!(buf.is_empty());
481 /// ```
482 #[inline]
483 pub fn clear(&mut self) {
484 self.truncate(0);
485 }
486
487 #[inline]
488 pub(crate) unsafe fn with_vtable(
489 ptr: *const u8,
490 len: usize,
491 data: AtomicPtr<()>,
492 vtable: &'static Vtable,
493 ) -> Bytes {
494 Bytes {
495 ptr,
496 len,
497 data,
498 vtable,
499 }
500 }
501
502 // private
503
504 #[inline]
505 fn as_slice(&self) -> &[u8] {
506 unsafe { slice::from_raw_parts(self.ptr, self.len) }
507 }
508
509 #[inline]
510 unsafe fn inc_start(&mut self, by: usize) {
511 // should already be asserted, but debug assert for tests
512 debug_assert!(self.len >= by, "internal: inc_start out of bounds");
513 self.len -= by;
514 self.ptr = self.ptr.add(by);
515 }
516}
517
518// Vtable must enforce this behavior
519unsafe impl Send for Bytes {}
520unsafe impl Sync for Bytes {}
521
522impl Drop for Bytes {
523 #[inline]
524 fn drop(&mut self) {
525 unsafe { (self.vtable.drop)(&mut self.data, self.ptr, self.len) }
526 }
527}
528
529impl Clone for Bytes {
530 #[inline]
531 fn clone(&self) -> Bytes {
532 unsafe { (self.vtable.clone)(&self.data, self.ptr, self.len) }
533 }
534}
535
536impl Buf for Bytes {
537 #[inline]
538 fn remaining(&self) -> usize {
539 self.len()
540 }
541
542 #[inline]
543 fn chunk(&self) -> &[u8] {
544 self.as_slice()
545 }
546
547 #[inline]
548 fn advance(&mut self, cnt: usize) {
549 assert!(
550 cnt <= self.len(),
551 "cannot advance past `remaining`: {:?} <= {:?}",
552 cnt,
553 self.len(),
554 );
555
556 unsafe {
557 self.inc_start(cnt);
558 }
559 }
560
561 fn copy_to_bytes(&mut self, len: usize) -> crate::Bytes {
562 if len == self.remaining() {
563 core::mem::replace(self, Bytes::new())
564 } else {
565 let ret = self.slice(..len);
566 self.advance(len);
567 ret
568 }
569 }
570}
571
572impl Deref for Bytes {
573 type Target = [u8];
574
575 #[inline]
576 fn deref(&self) -> &[u8] {
577 self.as_slice()
578 }
579}
580
581impl AsRef<[u8]> for Bytes {
582 #[inline]
583 fn as_ref(&self) -> &[u8] {
584 self.as_slice()
585 }
586}
587
588impl hash::Hash for Bytes {
589 fn hash<H>(&self, state: &mut H)
590 where
591 H: hash::Hasher,
592 {
593 self.as_slice().hash(state);
594 }
595}
596
597impl Borrow<[u8]> for Bytes {
598 fn borrow(&self) -> &[u8] {
599 self.as_slice()
600 }
601}
602
603impl IntoIterator for Bytes {
604 type Item = u8;
605 type IntoIter = IntoIter<Bytes>;
606
607 fn into_iter(self) -> Self::IntoIter {
608 IntoIter::new(self)
609 }
610}
611
612impl<'a> IntoIterator for &'a Bytes {
613 type Item = &'a u8;
614 type IntoIter = core::slice::Iter<'a, u8>;
615
616 fn into_iter(self) -> Self::IntoIter {
617 self.as_slice().iter()
618 }
619}
620
621impl FromIterator<u8> for Bytes {
622 fn from_iter<T: IntoIterator<Item = u8>>(into_iter: T) -> Self {
623 Vec::from_iter(into_iter).into()
624 }
625}
626
627// impl Eq
628
629impl PartialEq for Bytes {
630 fn eq(&self, other: &Bytes) -> bool {
631 self.as_slice() == other.as_slice()
632 }
633}
634
635impl PartialOrd for Bytes {
636 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
637 self.as_slice().partial_cmp(other.as_slice())
638 }
639}
640
641impl Ord for Bytes {
642 fn cmp(&self, other: &Bytes) -> cmp::Ordering {
643 self.as_slice().cmp(other.as_slice())
644 }
645}
646
647impl Eq for Bytes {}
648
649impl PartialEq<[u8]> for Bytes {
650 fn eq(&self, other: &[u8]) -> bool {
651 self.as_slice() == other
652 }
653}
654
655impl PartialOrd<[u8]> for Bytes {
656 fn partial_cmp(&self, other: &[u8]) -> Option<cmp::Ordering> {
657 self.as_slice().partial_cmp(other)
658 }
659}
660
661impl PartialEq<Bytes> for [u8] {
662 fn eq(&self, other: &Bytes) -> bool {
663 *other == *self
664 }
665}
666
667impl PartialOrd<Bytes> for [u8] {
668 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
669 <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
670 }
671}
672
673impl PartialEq<str> for Bytes {
674 fn eq(&self, other: &str) -> bool {
675 self.as_slice() == other.as_bytes()
676 }
677}
678
679impl PartialOrd<str> for Bytes {
680 fn partial_cmp(&self, other: &str) -> Option<cmp::Ordering> {
681 self.as_slice().partial_cmp(other.as_bytes())
682 }
683}
684
685impl PartialEq<Bytes> for str {
686 fn eq(&self, other: &Bytes) -> bool {
687 *other == *self
688 }
689}
690
691impl PartialOrd<Bytes> for str {
692 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
693 <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
694 }
695}
696
697impl PartialEq<Vec<u8>> for Bytes {
698 fn eq(&self, other: &Vec<u8>) -> bool {
699 *self == other[..]
700 }
701}
702
703impl PartialOrd<Vec<u8>> for Bytes {
704 fn partial_cmp(&self, other: &Vec<u8>) -> Option<cmp::Ordering> {
705 self.as_slice().partial_cmp(&other[..])
706 }
707}
708
709impl PartialEq<Bytes> for Vec<u8> {
710 fn eq(&self, other: &Bytes) -> bool {
711 *other == *self
712 }
713}
714
715impl PartialOrd<Bytes> for Vec<u8> {
716 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
717 <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
718 }
719}
720
721impl PartialEq<String> for Bytes {
722 fn eq(&self, other: &String) -> bool {
723 *self == other[..]
724 }
725}
726
727impl PartialOrd<String> for Bytes {
728 fn partial_cmp(&self, other: &String) -> Option<cmp::Ordering> {
729 self.as_slice().partial_cmp(other.as_bytes())
730 }
731}
732
733impl PartialEq<Bytes> for String {
734 fn eq(&self, other: &Bytes) -> bool {
735 *other == *self
736 }
737}
738
739impl PartialOrd<Bytes> for String {
740 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
741 <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
742 }
743}
744
745impl PartialEq<Bytes> for &[u8] {
746 fn eq(&self, other: &Bytes) -> bool {
747 *other == *self
748 }
749}
750
751impl PartialOrd<Bytes> for &[u8] {
752 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
753 <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
754 }
755}
756
757impl PartialEq<Bytes> for &str {
758 fn eq(&self, other: &Bytes) -> bool {
759 *other == *self
760 }
761}
762
763impl PartialOrd<Bytes> for &str {
764 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
765 <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
766 }
767}
768
769impl<'a, T: ?Sized> PartialEq<&'a T> for Bytes
770where
771 Bytes: PartialEq<T>,
772{
773 fn eq(&self, other: &&'a T) -> bool {
774 *self == **other
775 }
776}
777
778impl<'a, T: ?Sized> PartialOrd<&'a T> for Bytes
779where
780 Bytes: PartialOrd<T>,
781{
782 fn partial_cmp(&self, other: &&'a T) -> Option<cmp::Ordering> {
783 self.partial_cmp(&**other)
784 }
785}
786
787// impl From
788
789impl Default for Bytes {
790 #[inline]
791 fn default() -> Bytes {
792 Bytes::new()
793 }
794}
795
796impl From<&'static [u8]> for Bytes {
797 fn from(slice: &'static [u8]) -> Bytes {
798 Bytes::from_static(slice)
799 }
800}
801
802impl From<&'static str> for Bytes {
803 fn from(slice: &'static str) -> Bytes {
804 Bytes::from_static(slice.as_bytes())
805 }
806}
807
808impl From<Vec<u8>> for Bytes {
809 fn from(vec: Vec<u8>) -> Bytes {
810 let mut vec = vec;
811 let ptr = vec.as_mut_ptr();
812 let len = vec.len();
813 let cap = vec.capacity();
814
815 // Avoid an extra allocation if possible.
816 if len == cap {
817 return Bytes::from(vec.into_boxed_slice());
818 }
819
820 let shared = Box::new(Shared {
821 buf: ptr,
822 cap,
823 ref_cnt: AtomicUsize::new(1),
824 });
825 mem::forget(vec);
826
827 let shared = Box::into_raw(shared);
828 // The pointer should be aligned, so this assert should
829 // always succeed.
830 debug_assert!(
831 0 == (shared as usize & KIND_MASK),
832 "internal: Box<Shared> should have an aligned pointer",
833 );
834 Bytes {
835 ptr,
836 len,
837 data: AtomicPtr::new(shared as _),
838 vtable: &SHARED_VTABLE,
839 }
840 }
841}
842
843impl From<Box<[u8]>> for Bytes {
844 fn from(slice: Box<[u8]>) -> Bytes {
845 // Box<[u8]> doesn't contain a heap allocation for empty slices,
846 // so the pointer isn't aligned enough for the KIND_VEC stashing to
847 // work.
848 if slice.is_empty() {
849 return Bytes::new();
850 }
851
852 let len = slice.len();
853 let ptr = Box::into_raw(slice) as *mut u8;
854
855 if ptr as usize & 0x1 == 0 {
856 let data = ptr_map(ptr, |addr| addr | KIND_VEC);
857 Bytes {
858 ptr,
859 len,
860 data: AtomicPtr::new(data.cast()),
861 vtable: &PROMOTABLE_EVEN_VTABLE,
862 }
863 } else {
864 Bytes {
865 ptr,
866 len,
867 data: AtomicPtr::new(ptr.cast()),
868 vtable: &PROMOTABLE_ODD_VTABLE,
869 }
870 }
871 }
872}
873
874impl From<String> for Bytes {
875 fn from(s: String) -> Bytes {
876 Bytes::from(s.into_bytes())
877 }
878}
879
880impl From<Bytes> for Vec<u8> {
881 fn from(bytes: Bytes) -> Vec<u8> {
882 let bytes = mem::ManuallyDrop::new(bytes);
883 unsafe { (bytes.vtable.to_vec)(&bytes.data, bytes.ptr, bytes.len) }
884 }
885}
886
887// ===== impl Vtable =====
888
889impl fmt::Debug for Vtable {
890 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
891 f.debug_struct("Vtable")
892 .field("clone", &(self.clone as *const ()))
893 .field("drop", &(self.drop as *const ()))
894 .finish()
895 }
896}
897
898// ===== impl StaticVtable =====
899
900const STATIC_VTABLE: Vtable = Vtable {
901 clone: static_clone,
902 to_vec: static_to_vec,
903 drop: static_drop,
904};
905
906unsafe fn static_clone(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
907 let slice = slice::from_raw_parts(ptr, len);
908 Bytes::from_static(slice)
909}
910
911unsafe fn static_to_vec(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
912 let slice = slice::from_raw_parts(ptr, len);
913 slice.to_vec()
914}
915
916unsafe fn static_drop(_: &mut AtomicPtr<()>, _: *const u8, _: usize) {
917 // nothing to drop for &'static [u8]
918}
919
920// ===== impl PromotableVtable =====
921
922static PROMOTABLE_EVEN_VTABLE: Vtable = Vtable {
923 clone: promotable_even_clone,
924 to_vec: promotable_even_to_vec,
925 drop: promotable_even_drop,
926};
927
928static PROMOTABLE_ODD_VTABLE: Vtable = Vtable {
929 clone: promotable_odd_clone,
930 to_vec: promotable_odd_to_vec,
931 drop: promotable_odd_drop,
932};
933
934unsafe fn promotable_even_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
935 let shared = data.load(Ordering::Acquire);
936 let kind = shared as usize & KIND_MASK;
937
938 if kind == KIND_ARC {
939 shallow_clone_arc(shared.cast(), ptr, len)
940 } else {
941 debug_assert_eq!(kind, KIND_VEC);
942 let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK);
943 shallow_clone_vec(data, shared, buf, ptr, len)
944 }
945}
946
947unsafe fn promotable_to_vec(
948 data: &AtomicPtr<()>,
949 ptr: *const u8,
950 len: usize,
951 f: fn(*mut ()) -> *mut u8,
952) -> Vec<u8> {
953 let shared = data.load(Ordering::Acquire);
954 let kind = shared as usize & KIND_MASK;
955
956 if kind == KIND_ARC {
957 shared_to_vec_impl(shared.cast(), ptr, len)
958 } else {
959 // If Bytes holds a Vec, then the offset must be 0.
960 debug_assert_eq!(kind, KIND_VEC);
961
962 let buf = f(shared);
963
964 let cap = (ptr as usize - buf as usize) + len;
965
966 // Copy back buffer
967 ptr::copy(ptr, buf, len);
968
969 Vec::from_raw_parts(buf, len, cap)
970 }
971}
972
973unsafe fn promotable_even_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
974 promotable_to_vec(data, ptr, len, |shared| {
975 ptr_map(shared.cast(), |addr| addr & !KIND_MASK)
976 })
977}
978
979unsafe fn promotable_even_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) {
980 data.with_mut(|shared| {
981 let shared = *shared;
982 let kind = shared as usize & KIND_MASK;
983
984 if kind == KIND_ARC {
985 release_shared(shared.cast());
986 } else {
987 debug_assert_eq!(kind, KIND_VEC);
988 let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK);
989 free_boxed_slice(buf, ptr, len);
990 }
991 });
992}
993
994unsafe fn promotable_odd_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
995 let shared = data.load(Ordering::Acquire);
996 let kind = shared as usize & KIND_MASK;
997
998 if kind == KIND_ARC {
999 shallow_clone_arc(shared as _, ptr, len)
1000 } else {
1001 debug_assert_eq!(kind, KIND_VEC);
1002 shallow_clone_vec(data, shared, shared.cast(), ptr, len)
1003 }
1004}
1005
1006unsafe fn promotable_odd_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
1007 promotable_to_vec(data, ptr, len, |shared| shared.cast())
1008}
1009
1010unsafe fn promotable_odd_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) {
1011 data.with_mut(|shared| {
1012 let shared = *shared;
1013 let kind = shared as usize & KIND_MASK;
1014
1015 if kind == KIND_ARC {
1016 release_shared(shared.cast());
1017 } else {
1018 debug_assert_eq!(kind, KIND_VEC);
1019
1020 free_boxed_slice(shared.cast(), ptr, len);
1021 }
1022 });
1023}
1024
1025unsafe fn free_boxed_slice(buf: *mut u8, offset: *const u8, len: usize) {
1026 let cap = (offset as usize - buf as usize) + len;
1027 dealloc(buf, Layout::from_size_align(cap, 1).unwrap())
1028}
1029
1030// ===== impl SharedVtable =====
1031
1032struct Shared {
1033 // Holds arguments to dealloc upon Drop, but otherwise doesn't use them
1034 buf: *mut u8,
1035 cap: usize,
1036 ref_cnt: AtomicUsize,
1037}
1038
1039impl Drop for Shared {
1040 fn drop(&mut self) {
1041 unsafe { dealloc(self.buf, Layout::from_size_align(self.cap, 1).unwrap()) }
1042 }
1043}
1044
1045// Assert that the alignment of `Shared` is divisible by 2.
1046// This is a necessary invariant since we depend on allocating `Shared` a
1047// shared object to implicitly carry the `KIND_ARC` flag in its pointer.
1048// This flag is set when the LSB is 0.
1049const _: [(); 0 - mem::align_of::<Shared>() % 2] = []; // Assert that the alignment of `Shared` is divisible by 2.
1050
1051static SHARED_VTABLE: Vtable = Vtable {
1052 clone: shared_clone,
1053 to_vec: shared_to_vec,
1054 drop: shared_drop,
1055};
1056
1057const KIND_ARC: usize = 0b0;
1058const KIND_VEC: usize = 0b1;
1059const KIND_MASK: usize = 0b1;
1060
1061unsafe fn shared_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
1062 let shared = data.load(Ordering::Relaxed);
1063 shallow_clone_arc(shared as _, ptr, len)
1064}
1065
1066unsafe fn shared_to_vec_impl(shared: *mut Shared, ptr: *const u8, len: usize) -> Vec<u8> {
1067 // Check that the ref_cnt is 1 (unique).
1068 //
1069 // If it is unique, then it is set to 0 with AcqRel fence for the same
1070 // reason in release_shared.
1071 //
1072 // Otherwise, we take the other branch and call release_shared.
1073 if (*shared)
1074 .ref_cnt
1075 .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Relaxed)
1076 .is_ok()
1077 {
1078 let buf = (*shared).buf;
1079 let cap = (*shared).cap;
1080
1081 // Deallocate Shared
1082 drop(Box::from_raw(shared as *mut mem::ManuallyDrop<Shared>));
1083
1084 // Copy back buffer
1085 ptr::copy(ptr, buf, len);
1086
1087 Vec::from_raw_parts(buf, len, cap)
1088 } else {
1089 let v = slice::from_raw_parts(ptr, len).to_vec();
1090 release_shared(shared);
1091 v
1092 }
1093}
1094
1095unsafe fn shared_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
1096 shared_to_vec_impl(data.load(Ordering::Relaxed).cast(), ptr, len)
1097}
1098
1099unsafe fn shared_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) {
1100 data.with_mut(|shared| {
1101 release_shared(shared.cast());
1102 });
1103}
1104
1105unsafe fn shallow_clone_arc(shared: *mut Shared, ptr: *const u8, len: usize) -> Bytes {
1106 let old_size = (*shared).ref_cnt.fetch_add(1, Ordering::Relaxed);
1107
1108 if old_size > usize::MAX >> 1 {
1109 crate::abort();
1110 }
1111
1112 Bytes {
1113 ptr,
1114 len,
1115 data: AtomicPtr::new(shared as _),
1116 vtable: &SHARED_VTABLE,
1117 }
1118}
1119
1120#[cold]
1121unsafe fn shallow_clone_vec(
1122 atom: &AtomicPtr<()>,
1123 ptr: *const (),
1124 buf: *mut u8,
1125 offset: *const u8,
1126 len: usize,
1127) -> Bytes {
1128 // If the buffer is still tracked in a `Vec<u8>`. It is time to
1129 // promote the vec to an `Arc`. This could potentially be called
1130 // concurrently, so some care must be taken.
1131
1132 // First, allocate a new `Shared` instance containing the
1133 // `Vec` fields. It's important to note that `ptr`, `len`,
1134 // and `cap` cannot be mutated without having `&mut self`.
1135 // This means that these fields will not be concurrently
1136 // updated and since the buffer hasn't been promoted to an
1137 // `Arc`, those three fields still are the components of the
1138 // vector.
1139 let shared = Box::new(Shared {
1140 buf,
1141 cap: (offset as usize - buf as usize) + len,
1142 // Initialize refcount to 2. One for this reference, and one
1143 // for the new clone that will be returned from
1144 // `shallow_clone`.
1145 ref_cnt: AtomicUsize::new(2),
1146 });
1147
1148 let shared = Box::into_raw(shared);
1149
1150 // The pointer should be aligned, so this assert should
1151 // always succeed.
1152 debug_assert!(
1153 0 == (shared as usize & KIND_MASK),
1154 "internal: Box<Shared> should have an aligned pointer",
1155 );
1156
1157 // Try compare & swapping the pointer into the `arc` field.
1158 // `Release` is used synchronize with other threads that
1159 // will load the `arc` field.
1160 //
1161 // If the `compare_exchange` fails, then the thread lost the
1162 // race to promote the buffer to shared. The `Acquire`
1163 // ordering will synchronize with the `compare_exchange`
1164 // that happened in the other thread and the `Shared`
1165 // pointed to by `actual` will be visible.
1166 match atom.compare_exchange(ptr as _, shared as _, Ordering::AcqRel, Ordering::Acquire) {
1167 Ok(actual) => {
1168 debug_assert!(actual as usize == ptr as usize);
1169 // The upgrade was successful, the new handle can be
1170 // returned.
1171 Bytes {
1172 ptr: offset,
1173 len,
1174 data: AtomicPtr::new(shared as _),
1175 vtable: &SHARED_VTABLE,
1176 }
1177 }
1178 Err(actual) => {
1179 // The upgrade failed, a concurrent clone happened. Release
1180 // the allocation that was made in this thread, it will not
1181 // be needed.
1182 let shared = Box::from_raw(shared);
1183 mem::forget(*shared);
1184
1185 // Buffer already promoted to shared storage, so increment ref
1186 // count.
1187 shallow_clone_arc(actual as _, offset, len)
1188 }
1189 }
1190}
1191
1192unsafe fn release_shared(ptr: *mut Shared) {
1193 // `Shared` storage... follow the drop steps from Arc.
1194 if (*ptr).ref_cnt.fetch_sub(1, Ordering::Release) != 1 {
1195 return;
1196 }
1197
1198 // This fence is needed to prevent reordering of use of the data and
1199 // deletion of the data. Because it is marked `Release`, the decreasing
1200 // of the reference count synchronizes with this `Acquire` fence. This
1201 // means that use of the data happens before decreasing the reference
1202 // count, which happens before this fence, which happens before the
1203 // deletion of the data.
1204 //
1205 // As explained in the [Boost documentation][1],
1206 //
1207 // > It is important to enforce any possible access to the object in one
1208 // > thread (through an existing reference) to *happen before* deleting
1209 // > the object in a different thread. This is achieved by a "release"
1210 // > operation after dropping a reference (any access to the object
1211 // > through this reference must obviously happened before), and an
1212 // > "acquire" operation before deleting the object.
1213 //
1214 // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)
1215 //
1216 // Thread sanitizer does not support atomic fences. Use an atomic load
1217 // instead.
1218 (*ptr).ref_cnt.load(Ordering::Acquire);
1219
1220 // Drop the data
1221 drop(Box::from_raw(ptr));
1222}
1223
1224// Ideally we would always use this version of `ptr_map` since it is strict
1225// provenance compatible, but it results in worse codegen. We will however still
1226// use it on miri because it gives better diagnostics for people who test bytes
1227// code with miri.
1228//
1229// See https://github.com/tokio-rs/bytes/pull/545 for more info.
1230#[cfg(miri)]
1231fn ptr_map<F>(ptr: *mut u8, f: F) -> *mut u8
1232where
1233 F: FnOnce(usize) -> usize,
1234{
1235 let old_addr = ptr as usize;
1236 let new_addr = f(old_addr);
1237 let diff = new_addr.wrapping_sub(old_addr);
1238 ptr.wrapping_add(diff)
1239}
1240
1241#[cfg(not(miri))]
1242fn ptr_map<F>(ptr: *mut u8, f: F) -> *mut u8
1243where
1244 F: FnOnce(usize) -> usize,
1245{
1246 let old_addr = ptr as usize;
1247 let new_addr = f(old_addr);
1248 new_addr as *mut u8
1249}
1250
1251// compile-fails
1252
1253/// ```compile_fail
1254/// use bytes::Bytes;
1255/// #[deny(unused_must_use)]
1256/// {
1257/// let mut b1 = Bytes::from("hello world");
1258/// b1.split_to(6);
1259/// }
1260/// ```
1261fn _split_to_must_use() {}
1262
1263/// ```compile_fail
1264/// use bytes::Bytes;
1265/// #[deny(unused_must_use)]
1266/// {
1267/// let mut b1 = Bytes::from("hello world");
1268/// b1.split_off(6);
1269/// }
1270/// ```
1271fn _split_off_must_use() {}
1272
1273// fuzz tests
1274#[cfg(all(test, loom))]
1275mod fuzz {
1276 use loom::sync::Arc;
1277 use loom::thread;
1278
1279 use super::Bytes;
1280 #[test]
1281 fn bytes_cloning_vec() {
1282 loom::model(|| {
1283 let a = Bytes::from(b"abcdefgh".to_vec());
1284 let addr = a.as_ptr() as usize;
1285
1286 // test the Bytes::clone is Sync by putting it in an Arc
1287 let a1 = Arc::new(a);
1288 let a2 = a1.clone();
1289
1290 let t1 = thread::spawn(move || {
1291 let b: Bytes = (*a1).clone();
1292 assert_eq!(b.as_ptr() as usize, addr);
1293 });
1294
1295 let t2 = thread::spawn(move || {
1296 let b: Bytes = (*a2).clone();
1297 assert_eq!(b.as_ptr() as usize, addr);
1298 });
1299
1300 t1.join().unwrap();
1301 t2.join().unwrap();
1302 });
1303 }
1304}
1305