1 use core::iter::FromIterator;
2 use core::ops::{Deref, RangeBounds};
3 use core::{cmp, fmt, hash, mem, ptr, slice, usize};
4
5 use alloc::{borrow::Borrow, boxed::Box, string::String, vec::Vec};
6
7 use crate::buf::IntoIter;
8 #[allow(unused)]
9 use crate::loom::sync::atomic::AtomicMut;
10 use crate::loom::sync::atomic::{self, AtomicPtr, AtomicUsize, Ordering};
11 use crate::Buf;
12
13 /// A cheaply cloneable and sliceable chunk of contiguous memory.
14 ///
15 /// `Bytes` is an efficient container for storing and operating on contiguous
16 /// slices of memory. It is intended for use primarily in networking code, but
17 /// could have applications elsewhere as well.
18 ///
19 /// `Bytes` values facilitate zero-copy network programming by allowing multiple
20 /// `Bytes` objects to point to the same underlying memory.
21 ///
22 /// `Bytes` does not have a single implementation. It is an interface, whose
23 /// exact behavior is implemented through dynamic dispatch in several underlying
24 /// implementations of `Bytes`.
25 ///
26 /// All `Bytes` implementations must fulfill the following requirements:
27 /// - They are cheaply cloneable and thereby shareable between an unlimited amount
28 /// of components, for example by modifying a reference count.
29 /// - Instances can be sliced to refer to a subset of the the original buffer.
30 ///
31 /// ```
32 /// use bytes::Bytes;
33 ///
34 /// let mut mem = Bytes::from("Hello world");
35 /// let a = mem.slice(0..5);
36 ///
37 /// assert_eq!(a, "Hello");
38 ///
39 /// let b = mem.split_to(6);
40 ///
41 /// assert_eq!(mem, "world");
42 /// assert_eq!(b, "Hello ");
43 /// ```
44 ///
45 /// # Memory layout
46 ///
47 /// The `Bytes` struct itself is fairly small, limited to 4 `usize` fields used
48 /// to track information about which segment of the underlying memory the
49 /// `Bytes` handle has access to.
50 ///
51 /// `Bytes` keeps both a pointer to the shared state containing the full memory
52 /// slice and a pointer to the start of the region visible by the handle.
53 /// `Bytes` also tracks the length of its view into the memory.
54 ///
55 /// # Sharing
56 ///
57 /// `Bytes` contains a vtable, which allows implementations of `Bytes` to define
58 /// how sharing/cloneing is implemented in detail.
59 /// When `Bytes::clone()` is called, `Bytes` will call the vtable function for
60 /// cloning the backing storage in order to share it behind between multiple
61 /// `Bytes` instances.
62 ///
63 /// For `Bytes` implementations which refer to constant memory (e.g. created
64 /// via `Bytes::from_static()`) the cloning implementation will be a no-op.
65 ///
66 /// For `Bytes` implementations which point to a reference counted shared storage
67 /// (e.g. an `Arc<[u8]>`), sharing will be implemented by increasing the
68 /// the reference count.
69 ///
70 /// Due to this mechanism, multiple `Bytes` instances may point to the same
71 /// shared memory region.
72 /// Each `Bytes` instance can point to different sections within that
73 /// memory region, and `Bytes` instances may or may not have overlapping views
74 /// into the memory.
75 ///
76 /// The following diagram visualizes a scenario where 2 `Bytes` instances make
77 /// use of an `Arc`-based backing storage, and provide access to different views:
78 ///
79 /// ```text
80 ///
81 /// Arc ptrs +---------+
82 /// ________________________ / | Bytes 2 |
83 /// / +---------+
84 /// / +-----------+ | |
85 /// |_________/ | Bytes 1 | | |
86 /// | +-----------+ | |
87 /// | | | ___/ data | tail
88 /// | data | tail |/ |
89 /// v v v v
90 /// +-----+---------------------------------+-----+
91 /// | Arc | | | | |
92 /// +-----+---------------------------------+-----+
93 /// ```
94 pub struct Bytes {
95 ptr: *const u8,
96 len: usize,
97 // inlined "trait object"
98 data: AtomicPtr<()>,
99 vtable: &'static Vtable,
100 }
101
102 pub(crate) struct Vtable {
103 /// fn(data, ptr, len)
104 pub clone: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Bytes,
105 /// fn(data, ptr, len)
106 pub drop: unsafe fn(&mut AtomicPtr<()>, *const u8, usize),
107 }
108
109 impl Bytes {
110 /// Creates a new empty `Bytes`.
111 ///
112 /// This will not allocate and the returned `Bytes` handle will be empty.
113 ///
114 /// # Examples
115 ///
116 /// ```
117 /// use bytes::Bytes;
118 ///
119 /// let b = Bytes::new();
120 /// assert_eq!(&b[..], b"");
121 /// ```
122 #[inline]
123 #[cfg(not(all(loom, test)))]
new() -> Bytes124 pub const fn new() -> Bytes {
125 // Make it a named const to work around
126 // "unsizing casts are not allowed in const fn"
127 const EMPTY: &[u8] = &[];
128 Bytes::from_static(EMPTY)
129 }
130
131 #[cfg(all(loom, test))]
new() -> Bytes132 pub fn new() -> Bytes {
133 const EMPTY: &[u8] = &[];
134 Bytes::from_static(EMPTY)
135 }
136
137 /// Creates a new `Bytes` from a static slice.
138 ///
139 /// The returned `Bytes` will point directly to the static slice. There is
140 /// no allocating or copying.
141 ///
142 /// # Examples
143 ///
144 /// ```
145 /// use bytes::Bytes;
146 ///
147 /// let b = Bytes::from_static(b"hello");
148 /// assert_eq!(&b[..], b"hello");
149 /// ```
150 #[inline]
151 #[cfg(not(all(loom, test)))]
from_static(bytes: &'static [u8]) -> Bytes152 pub const fn from_static(bytes: &'static [u8]) -> Bytes {
153 Bytes {
154 ptr: bytes.as_ptr(),
155 len: bytes.len(),
156 data: AtomicPtr::new(ptr::null_mut()),
157 vtable: &STATIC_VTABLE,
158 }
159 }
160
161 #[cfg(all(loom, test))]
from_static(bytes: &'static [u8]) -> Bytes162 pub fn from_static(bytes: &'static [u8]) -> Bytes {
163 Bytes {
164 ptr: bytes.as_ptr(),
165 len: bytes.len(),
166 data: AtomicPtr::new(ptr::null_mut()),
167 vtable: &STATIC_VTABLE,
168 }
169 }
170
171 /// Returns the number of bytes contained in this `Bytes`.
172 ///
173 /// # Examples
174 ///
175 /// ```
176 /// use bytes::Bytes;
177 ///
178 /// let b = Bytes::from(&b"hello"[..]);
179 /// assert_eq!(b.len(), 5);
180 /// ```
181 #[inline]
len(&self) -> usize182 pub fn len(&self) -> usize {
183 self.len
184 }
185
186 /// Returns true if the `Bytes` has a length of 0.
187 ///
188 /// # Examples
189 ///
190 /// ```
191 /// use bytes::Bytes;
192 ///
193 /// let b = Bytes::new();
194 /// assert!(b.is_empty());
195 /// ```
196 #[inline]
is_empty(&self) -> bool197 pub fn is_empty(&self) -> bool {
198 self.len == 0
199 }
200
201 /// Creates `Bytes` instance from slice, by copying it.
copy_from_slice(data: &[u8]) -> Self202 pub fn copy_from_slice(data: &[u8]) -> Self {
203 data.to_vec().into()
204 }
205
206 /// Returns a slice of self for the provided range.
207 ///
208 /// This will increment the reference count for the underlying memory and
209 /// return a new `Bytes` handle set to the slice.
210 ///
211 /// This operation is `O(1)`.
212 ///
213 /// # Examples
214 ///
215 /// ```
216 /// use bytes::Bytes;
217 ///
218 /// let a = Bytes::from(&b"hello world"[..]);
219 /// let b = a.slice(2..5);
220 ///
221 /// assert_eq!(&b[..], b"llo");
222 /// ```
223 ///
224 /// # Panics
225 ///
226 /// Requires that `begin <= end` and `end <= self.len()`, otherwise slicing
227 /// will panic.
slice(&self, range: impl RangeBounds<usize>) -> Bytes228 pub fn slice(&self, range: impl RangeBounds<usize>) -> Bytes {
229 use core::ops::Bound;
230
231 let len = self.len();
232
233 let begin = match range.start_bound() {
234 Bound::Included(&n) => n,
235 Bound::Excluded(&n) => n + 1,
236 Bound::Unbounded => 0,
237 };
238
239 let end = match range.end_bound() {
240 Bound::Included(&n) => n.checked_add(1).expect("out of range"),
241 Bound::Excluded(&n) => n,
242 Bound::Unbounded => len,
243 };
244
245 assert!(
246 begin <= end,
247 "range start must not be greater than end: {:?} <= {:?}",
248 begin,
249 end,
250 );
251 assert!(
252 end <= len,
253 "range end out of bounds: {:?} <= {:?}",
254 end,
255 len,
256 );
257
258 if end == begin {
259 return Bytes::new();
260 }
261
262 let mut ret = self.clone();
263
264 ret.len = end - begin;
265 ret.ptr = unsafe { ret.ptr.offset(begin as isize) };
266
267 ret
268 }
269
270 /// Returns a slice of self that is equivalent to the given `subset`.
271 ///
272 /// When processing a `Bytes` buffer with other tools, one often gets a
273 /// `&[u8]` which is in fact a slice of the `Bytes`, i.e. a subset of it.
274 /// This function turns that `&[u8]` into another `Bytes`, as if one had
275 /// called `self.slice()` with the offsets that correspond to `subset`.
276 ///
277 /// This operation is `O(1)`.
278 ///
279 /// # Examples
280 ///
281 /// ```
282 /// use bytes::Bytes;
283 ///
284 /// let bytes = Bytes::from(&b"012345678"[..]);
285 /// let as_slice = bytes.as_ref();
286 /// let subset = &as_slice[2..6];
287 /// let subslice = bytes.slice_ref(&subset);
288 /// assert_eq!(&subslice[..], b"2345");
289 /// ```
290 ///
291 /// # Panics
292 ///
293 /// Requires that the given `sub` slice is in fact contained within the
294 /// `Bytes` buffer; otherwise this function will panic.
slice_ref(&self, subset: &[u8]) -> Bytes295 pub fn slice_ref(&self, subset: &[u8]) -> Bytes {
296 // Empty slice and empty Bytes may have their pointers reset
297 // so explicitly allow empty slice to be a subslice of any slice.
298 if subset.is_empty() {
299 return Bytes::new();
300 }
301
302 let bytes_p = self.as_ptr() as usize;
303 let bytes_len = self.len();
304
305 let sub_p = subset.as_ptr() as usize;
306 let sub_len = subset.len();
307
308 assert!(
309 sub_p >= bytes_p,
310 "subset pointer ({:p}) is smaller than self pointer ({:p})",
311 sub_p as *const u8,
312 bytes_p as *const u8,
313 );
314 assert!(
315 sub_p + sub_len <= bytes_p + bytes_len,
316 "subset is out of bounds: self = ({:p}, {}), subset = ({:p}, {})",
317 bytes_p as *const u8,
318 bytes_len,
319 sub_p as *const u8,
320 sub_len,
321 );
322
323 let sub_offset = sub_p - bytes_p;
324
325 self.slice(sub_offset..(sub_offset + sub_len))
326 }
327
328 /// Splits the bytes into two at the given index.
329 ///
330 /// Afterwards `self` contains elements `[0, at)`, and the returned `Bytes`
331 /// contains elements `[at, len)`.
332 ///
333 /// This is an `O(1)` operation that just increases the reference count and
334 /// sets a few indices.
335 ///
336 /// # Examples
337 ///
338 /// ```
339 /// use bytes::Bytes;
340 ///
341 /// let mut a = Bytes::from(&b"hello world"[..]);
342 /// let b = a.split_off(5);
343 ///
344 /// assert_eq!(&a[..], b"hello");
345 /// assert_eq!(&b[..], b" world");
346 /// ```
347 ///
348 /// # Panics
349 ///
350 /// Panics if `at > len`.
351 #[must_use = "consider Bytes::truncate if you don't need the other half"]
split_off(&mut self, at: usize) -> Bytes352 pub fn split_off(&mut self, at: usize) -> Bytes {
353 assert!(
354 at <= self.len(),
355 "split_off out of bounds: {:?} <= {:?}",
356 at,
357 self.len(),
358 );
359
360 if at == self.len() {
361 return Bytes::new();
362 }
363
364 if at == 0 {
365 return mem::replace(self, Bytes::new());
366 }
367
368 let mut ret = self.clone();
369
370 self.len = at;
371
372 unsafe { ret.inc_start(at) };
373
374 ret
375 }
376
377 /// Splits the bytes into two at the given index.
378 ///
379 /// Afterwards `self` contains elements `[at, len)`, and the returned
380 /// `Bytes` contains elements `[0, at)`.
381 ///
382 /// This is an `O(1)` operation that just increases the reference count and
383 /// sets a few indices.
384 ///
385 /// # Examples
386 ///
387 /// ```
388 /// use bytes::Bytes;
389 ///
390 /// let mut a = Bytes::from(&b"hello world"[..]);
391 /// let b = a.split_to(5);
392 ///
393 /// assert_eq!(&a[..], b" world");
394 /// assert_eq!(&b[..], b"hello");
395 /// ```
396 ///
397 /// # Panics
398 ///
399 /// Panics if `at > len`.
400 #[must_use = "consider Bytes::advance if you don't need the other half"]
split_to(&mut self, at: usize) -> Bytes401 pub fn split_to(&mut self, at: usize) -> Bytes {
402 assert!(
403 at <= self.len(),
404 "split_to out of bounds: {:?} <= {:?}",
405 at,
406 self.len(),
407 );
408
409 if at == self.len() {
410 return mem::replace(self, Bytes::new());
411 }
412
413 if at == 0 {
414 return Bytes::new();
415 }
416
417 let mut ret = self.clone();
418
419 unsafe { self.inc_start(at) };
420
421 ret.len = at;
422 ret
423 }
424
425 /// Shortens the buffer, keeping the first `len` bytes and dropping the
426 /// rest.
427 ///
428 /// If `len` is greater than the buffer's current length, this has no
429 /// effect.
430 ///
431 /// The [`split_off`] method can emulate `truncate`, but this causes the
432 /// excess bytes to be returned instead of dropped.
433 ///
434 /// # Examples
435 ///
436 /// ```
437 /// use bytes::Bytes;
438 ///
439 /// let mut buf = Bytes::from(&b"hello world"[..]);
440 /// buf.truncate(5);
441 /// assert_eq!(buf, b"hello"[..]);
442 /// ```
443 ///
444 /// [`split_off`]: #method.split_off
445 #[inline]
truncate(&mut self, len: usize)446 pub fn truncate(&mut self, len: usize) {
447 if len < self.len {
448 // The Vec "promotable" vtables do not store the capacity,
449 // so we cannot truncate while using this repr. We *have* to
450 // promote using `split_off` so the capacity can be stored.
451 if self.vtable as *const Vtable == &PROMOTABLE_EVEN_VTABLE
452 || self.vtable as *const Vtable == &PROMOTABLE_ODD_VTABLE
453 {
454 drop(self.split_off(len));
455 } else {
456 self.len = len;
457 }
458 }
459 }
460
461 /// Clears the buffer, removing all data.
462 ///
463 /// # Examples
464 ///
465 /// ```
466 /// use bytes::Bytes;
467 ///
468 /// let mut buf = Bytes::from(&b"hello world"[..]);
469 /// buf.clear();
470 /// assert!(buf.is_empty());
471 /// ```
472 #[inline]
clear(&mut self)473 pub fn clear(&mut self) {
474 self.truncate(0);
475 }
476
477 #[inline]
with_vtable( ptr: *const u8, len: usize, data: AtomicPtr<()>, vtable: &'static Vtable, ) -> Bytes478 pub(crate) unsafe fn with_vtable(
479 ptr: *const u8,
480 len: usize,
481 data: AtomicPtr<()>,
482 vtable: &'static Vtable,
483 ) -> Bytes {
484 Bytes {
485 ptr,
486 len,
487 data,
488 vtable,
489 }
490 }
491
492 // private
493
494 #[inline]
as_slice(&self) -> &[u8]495 fn as_slice(&self) -> &[u8] {
496 unsafe { slice::from_raw_parts(self.ptr, self.len) }
497 }
498
499 #[inline]
inc_start(&mut self, by: usize)500 unsafe fn inc_start(&mut self, by: usize) {
501 // should already be asserted, but debug assert for tests
502 debug_assert!(self.len >= by, "internal: inc_start out of bounds");
503 self.len -= by;
504 self.ptr = self.ptr.offset(by as isize);
505 }
506 }
507
508 // Vtable must enforce this behavior
509 unsafe impl Send for Bytes {}
510 unsafe impl Sync for Bytes {}
511
512 impl Drop for Bytes {
513 #[inline]
drop(&mut self)514 fn drop(&mut self) {
515 unsafe { (self.vtable.drop)(&mut self.data, self.ptr, self.len) }
516 }
517 }
518
519 impl Clone for Bytes {
520 #[inline]
clone(&self) -> Bytes521 fn clone(&self) -> Bytes {
522 unsafe { (self.vtable.clone)(&self.data, self.ptr, self.len) }
523 }
524 }
525
526 impl Buf for Bytes {
527 #[inline]
remaining(&self) -> usize528 fn remaining(&self) -> usize {
529 self.len()
530 }
531
532 #[inline]
chunk(&self) -> &[u8]533 fn chunk(&self) -> &[u8] {
534 self.as_slice()
535 }
536
537 #[inline]
advance(&mut self, cnt: usize)538 fn advance(&mut self, cnt: usize) {
539 assert!(
540 cnt <= self.len(),
541 "cannot advance past `remaining`: {:?} <= {:?}",
542 cnt,
543 self.len(),
544 );
545
546 unsafe {
547 self.inc_start(cnt);
548 }
549 }
550
copy_to_bytes(&mut self, len: usize) -> crate::Bytes551 fn copy_to_bytes(&mut self, len: usize) -> crate::Bytes {
552 if len == self.remaining() {
553 core::mem::replace(self, Bytes::new())
554 } else {
555 let ret = self.slice(..len);
556 self.advance(len);
557 ret
558 }
559 }
560 }
561
562 impl Deref for Bytes {
563 type Target = [u8];
564
565 #[inline]
deref(&self) -> &[u8]566 fn deref(&self) -> &[u8] {
567 self.as_slice()
568 }
569 }
570
571 impl AsRef<[u8]> for Bytes {
572 #[inline]
as_ref(&self) -> &[u8]573 fn as_ref(&self) -> &[u8] {
574 self.as_slice()
575 }
576 }
577
578 impl hash::Hash for Bytes {
hash<H>(&self, state: &mut H) where H: hash::Hasher,579 fn hash<H>(&self, state: &mut H)
580 where
581 H: hash::Hasher,
582 {
583 self.as_slice().hash(state);
584 }
585 }
586
587 impl Borrow<[u8]> for Bytes {
borrow(&self) -> &[u8]588 fn borrow(&self) -> &[u8] {
589 self.as_slice()
590 }
591 }
592
593 impl IntoIterator for Bytes {
594 type Item = u8;
595 type IntoIter = IntoIter<Bytes>;
596
into_iter(self) -> Self::IntoIter597 fn into_iter(self) -> Self::IntoIter {
598 IntoIter::new(self)
599 }
600 }
601
602 impl<'a> IntoIterator for &'a Bytes {
603 type Item = &'a u8;
604 type IntoIter = core::slice::Iter<'a, u8>;
605
into_iter(self) -> Self::IntoIter606 fn into_iter(self) -> Self::IntoIter {
607 self.as_slice().into_iter()
608 }
609 }
610
611 impl FromIterator<u8> for Bytes {
from_iter<T: IntoIterator<Item = u8>>(into_iter: T) -> Self612 fn from_iter<T: IntoIterator<Item = u8>>(into_iter: T) -> Self {
613 Vec::from_iter(into_iter).into()
614 }
615 }
616
617 // impl Eq
618
619 impl PartialEq for Bytes {
eq(&self, other: &Bytes) -> bool620 fn eq(&self, other: &Bytes) -> bool {
621 self.as_slice() == other.as_slice()
622 }
623 }
624
625 impl PartialOrd for Bytes {
partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering>626 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
627 self.as_slice().partial_cmp(other.as_slice())
628 }
629 }
630
631 impl Ord for Bytes {
cmp(&self, other: &Bytes) -> cmp::Ordering632 fn cmp(&self, other: &Bytes) -> cmp::Ordering {
633 self.as_slice().cmp(other.as_slice())
634 }
635 }
636
637 impl Eq for Bytes {}
638
639 impl PartialEq<[u8]> for Bytes {
eq(&self, other: &[u8]) -> bool640 fn eq(&self, other: &[u8]) -> bool {
641 self.as_slice() == other
642 }
643 }
644
645 impl PartialOrd<[u8]> for Bytes {
partial_cmp(&self, other: &[u8]) -> Option<cmp::Ordering>646 fn partial_cmp(&self, other: &[u8]) -> Option<cmp::Ordering> {
647 self.as_slice().partial_cmp(other)
648 }
649 }
650
651 impl PartialEq<Bytes> for [u8] {
eq(&self, other: &Bytes) -> bool652 fn eq(&self, other: &Bytes) -> bool {
653 *other == *self
654 }
655 }
656
657 impl PartialOrd<Bytes> for [u8] {
partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering>658 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
659 <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
660 }
661 }
662
663 impl PartialEq<str> for Bytes {
eq(&self, other: &str) -> bool664 fn eq(&self, other: &str) -> bool {
665 self.as_slice() == other.as_bytes()
666 }
667 }
668
669 impl PartialOrd<str> for Bytes {
partial_cmp(&self, other: &str) -> Option<cmp::Ordering>670 fn partial_cmp(&self, other: &str) -> Option<cmp::Ordering> {
671 self.as_slice().partial_cmp(other.as_bytes())
672 }
673 }
674
675 impl PartialEq<Bytes> for str {
eq(&self, other: &Bytes) -> bool676 fn eq(&self, other: &Bytes) -> bool {
677 *other == *self
678 }
679 }
680
681 impl PartialOrd<Bytes> for str {
partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering>682 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
683 <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
684 }
685 }
686
687 impl PartialEq<Vec<u8>> for Bytes {
eq(&self, other: &Vec<u8>) -> bool688 fn eq(&self, other: &Vec<u8>) -> bool {
689 *self == &other[..]
690 }
691 }
692
693 impl PartialOrd<Vec<u8>> for Bytes {
partial_cmp(&self, other: &Vec<u8>) -> Option<cmp::Ordering>694 fn partial_cmp(&self, other: &Vec<u8>) -> Option<cmp::Ordering> {
695 self.as_slice().partial_cmp(&other[..])
696 }
697 }
698
699 impl PartialEq<Bytes> for Vec<u8> {
eq(&self, other: &Bytes) -> bool700 fn eq(&self, other: &Bytes) -> bool {
701 *other == *self
702 }
703 }
704
705 impl PartialOrd<Bytes> for Vec<u8> {
partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering>706 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
707 <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
708 }
709 }
710
711 impl PartialEq<String> for Bytes {
eq(&self, other: &String) -> bool712 fn eq(&self, other: &String) -> bool {
713 *self == &other[..]
714 }
715 }
716
717 impl PartialOrd<String> for Bytes {
partial_cmp(&self, other: &String) -> Option<cmp::Ordering>718 fn partial_cmp(&self, other: &String) -> Option<cmp::Ordering> {
719 self.as_slice().partial_cmp(other.as_bytes())
720 }
721 }
722
723 impl PartialEq<Bytes> for String {
eq(&self, other: &Bytes) -> bool724 fn eq(&self, other: &Bytes) -> bool {
725 *other == *self
726 }
727 }
728
729 impl PartialOrd<Bytes> for String {
partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering>730 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
731 <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
732 }
733 }
734
735 impl PartialEq<Bytes> for &[u8] {
eq(&self, other: &Bytes) -> bool736 fn eq(&self, other: &Bytes) -> bool {
737 *other == *self
738 }
739 }
740
741 impl PartialOrd<Bytes> for &[u8] {
partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering>742 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
743 <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
744 }
745 }
746
747 impl PartialEq<Bytes> for &str {
eq(&self, other: &Bytes) -> bool748 fn eq(&self, other: &Bytes) -> bool {
749 *other == *self
750 }
751 }
752
753 impl PartialOrd<Bytes> for &str {
partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering>754 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
755 <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
756 }
757 }
758
759 impl<'a, T: ?Sized> PartialEq<&'a T> for Bytes
760 where
761 Bytes: PartialEq<T>,
762 {
eq(&self, other: &&'a T) -> bool763 fn eq(&self, other: &&'a T) -> bool {
764 *self == **other
765 }
766 }
767
768 impl<'a, T: ?Sized> PartialOrd<&'a T> for Bytes
769 where
770 Bytes: PartialOrd<T>,
771 {
partial_cmp(&self, other: &&'a T) -> Option<cmp::Ordering>772 fn partial_cmp(&self, other: &&'a T) -> Option<cmp::Ordering> {
773 self.partial_cmp(&**other)
774 }
775 }
776
777 // impl From
778
779 impl Default for Bytes {
780 #[inline]
default() -> Bytes781 fn default() -> Bytes {
782 Bytes::new()
783 }
784 }
785
786 impl From<&'static [u8]> for Bytes {
from(slice: &'static [u8]) -> Bytes787 fn from(slice: &'static [u8]) -> Bytes {
788 Bytes::from_static(slice)
789 }
790 }
791
792 impl From<&'static str> for Bytes {
from(slice: &'static str) -> Bytes793 fn from(slice: &'static str) -> Bytes {
794 Bytes::from_static(slice.as_bytes())
795 }
796 }
797
798 impl From<Vec<u8>> for Bytes {
from(vec: Vec<u8>) -> Bytes799 fn from(vec: Vec<u8>) -> Bytes {
800 let slice = vec.into_boxed_slice();
801 slice.into()
802 }
803 }
804
805 impl From<Box<[u8]>> for Bytes {
from(slice: Box<[u8]>) -> Bytes806 fn from(slice: Box<[u8]>) -> Bytes {
807 // Box<[u8]> doesn't contain a heap allocation for empty slices,
808 // so the pointer isn't aligned enough for the KIND_VEC stashing to
809 // work.
810 if slice.is_empty() {
811 return Bytes::new();
812 }
813
814 let len = slice.len();
815 let ptr = Box::into_raw(slice) as *mut u8;
816
817 if ptr as usize & 0x1 == 0 {
818 let data = ptr as usize | KIND_VEC;
819 Bytes {
820 ptr,
821 len,
822 data: AtomicPtr::new(data as *mut _),
823 vtable: &PROMOTABLE_EVEN_VTABLE,
824 }
825 } else {
826 Bytes {
827 ptr,
828 len,
829 data: AtomicPtr::new(ptr as *mut _),
830 vtable: &PROMOTABLE_ODD_VTABLE,
831 }
832 }
833 }
834 }
835
836 impl From<String> for Bytes {
from(s: String) -> Bytes837 fn from(s: String) -> Bytes {
838 Bytes::from(s.into_bytes())
839 }
840 }
841
842 // ===== impl Vtable =====
843
844 impl fmt::Debug for Vtable {
fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result845 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
846 f.debug_struct("Vtable")
847 .field("clone", &(self.clone as *const ()))
848 .field("drop", &(self.drop as *const ()))
849 .finish()
850 }
851 }
852
853 // ===== impl StaticVtable =====
854
855 const STATIC_VTABLE: Vtable = Vtable {
856 clone: static_clone,
857 drop: static_drop,
858 };
859
static_clone(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes860 unsafe fn static_clone(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
861 let slice = slice::from_raw_parts(ptr, len);
862 Bytes::from_static(slice)
863 }
864
static_drop(_: &mut AtomicPtr<()>, _: *const u8, _: usize)865 unsafe fn static_drop(_: &mut AtomicPtr<()>, _: *const u8, _: usize) {
866 // nothing to drop for &'static [u8]
867 }
868
869 // ===== impl PromotableVtable =====
870
871 static PROMOTABLE_EVEN_VTABLE: Vtable = Vtable {
872 clone: promotable_even_clone,
873 drop: promotable_even_drop,
874 };
875
876 static PROMOTABLE_ODD_VTABLE: Vtable = Vtable {
877 clone: promotable_odd_clone,
878 drop: promotable_odd_drop,
879 };
880
promotable_even_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes881 unsafe fn promotable_even_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
882 let shared = data.load(Ordering::Acquire);
883 let kind = shared as usize & KIND_MASK;
884
885 if kind == KIND_ARC {
886 shallow_clone_arc(shared as _, ptr, len)
887 } else {
888 debug_assert_eq!(kind, KIND_VEC);
889 let buf = (shared as usize & !KIND_MASK) as *mut u8;
890 shallow_clone_vec(data, shared, buf, ptr, len)
891 }
892 }
893
promotable_even_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize)894 unsafe fn promotable_even_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) {
895 data.with_mut(|shared| {
896 let shared = *shared;
897 let kind = shared as usize & KIND_MASK;
898
899 if kind == KIND_ARC {
900 release_shared(shared as *mut Shared);
901 } else {
902 debug_assert_eq!(kind, KIND_VEC);
903 let buf = (shared as usize & !KIND_MASK) as *mut u8;
904 drop(rebuild_boxed_slice(buf, ptr, len));
905 }
906 });
907 }
908
promotable_odd_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes909 unsafe fn promotable_odd_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
910 let shared = data.load(Ordering::Acquire);
911 let kind = shared as usize & KIND_MASK;
912
913 if kind == KIND_ARC {
914 shallow_clone_arc(shared as _, ptr, len)
915 } else {
916 debug_assert_eq!(kind, KIND_VEC);
917 shallow_clone_vec(data, shared, shared as *mut u8, ptr, len)
918 }
919 }
920
promotable_odd_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize)921 unsafe fn promotable_odd_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) {
922 data.with_mut(|shared| {
923 let shared = *shared;
924 let kind = shared as usize & KIND_MASK;
925
926 if kind == KIND_ARC {
927 release_shared(shared as *mut Shared);
928 } else {
929 debug_assert_eq!(kind, KIND_VEC);
930
931 drop(rebuild_boxed_slice(shared as *mut u8, ptr, len));
932 }
933 });
934 }
935
rebuild_boxed_slice(buf: *mut u8, offset: *const u8, len: usize) -> Box<[u8]>936 unsafe fn rebuild_boxed_slice(buf: *mut u8, offset: *const u8, len: usize) -> Box<[u8]> {
937 let cap = (offset as usize - buf as usize) + len;
938 Box::from_raw(slice::from_raw_parts_mut(buf, cap))
939 }
940
941 // ===== impl SharedVtable =====
942
943 struct Shared {
944 // holds vec for drop, but otherwise doesnt access it
945 _vec: Vec<u8>,
946 ref_cnt: AtomicUsize,
947 }
948
949 // Assert that the alignment of `Shared` is divisible by 2.
950 // This is a necessary invariant since we depend on allocating `Shared` a
951 // shared object to implicitly carry the `KIND_ARC` flag in its pointer.
952 // This flag is set when the LSB is 0.
953 const _: [(); 0 - mem::align_of::<Shared>() % 2] = []; // Assert that the alignment of `Shared` is divisible by 2.
954
955 static SHARED_VTABLE: Vtable = Vtable {
956 clone: shared_clone,
957 drop: shared_drop,
958 };
959
960 const KIND_ARC: usize = 0b0;
961 const KIND_VEC: usize = 0b1;
962 const KIND_MASK: usize = 0b1;
963
shared_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes964 unsafe fn shared_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
965 let shared = data.load(Ordering::Relaxed);
966 shallow_clone_arc(shared as _, ptr, len)
967 }
968
shared_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize)969 unsafe fn shared_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) {
970 data.with_mut(|shared| {
971 release_shared(*shared as *mut Shared);
972 });
973 }
974
shallow_clone_arc(shared: *mut Shared, ptr: *const u8, len: usize) -> Bytes975 unsafe fn shallow_clone_arc(shared: *mut Shared, ptr: *const u8, len: usize) -> Bytes {
976 let old_size = (*shared).ref_cnt.fetch_add(1, Ordering::Relaxed);
977
978 if old_size > usize::MAX >> 1 {
979 crate::abort();
980 }
981
982 Bytes {
983 ptr,
984 len,
985 data: AtomicPtr::new(shared as _),
986 vtable: &SHARED_VTABLE,
987 }
988 }
989
990 #[cold]
shallow_clone_vec( atom: &AtomicPtr<()>, ptr: *const (), buf: *mut u8, offset: *const u8, len: usize, ) -> Bytes991 unsafe fn shallow_clone_vec(
992 atom: &AtomicPtr<()>,
993 ptr: *const (),
994 buf: *mut u8,
995 offset: *const u8,
996 len: usize,
997 ) -> Bytes {
998 // If the buffer is still tracked in a `Vec<u8>`. It is time to
999 // promote the vec to an `Arc`. This could potentially be called
1000 // concurrently, so some care must be taken.
1001
1002 // First, allocate a new `Shared` instance containing the
1003 // `Vec` fields. It's important to note that `ptr`, `len`,
1004 // and `cap` cannot be mutated without having `&mut self`.
1005 // This means that these fields will not be concurrently
1006 // updated and since the buffer hasn't been promoted to an
1007 // `Arc`, those three fields still are the components of the
1008 // vector.
1009 let vec = rebuild_boxed_slice(buf, offset, len).into_vec();
1010 let shared = Box::new(Shared {
1011 _vec: vec,
1012 // Initialize refcount to 2. One for this reference, and one
1013 // for the new clone that will be returned from
1014 // `shallow_clone`.
1015 ref_cnt: AtomicUsize::new(2),
1016 });
1017
1018 let shared = Box::into_raw(shared);
1019
1020 // The pointer should be aligned, so this assert should
1021 // always succeed.
1022 debug_assert!(
1023 0 == (shared as usize & KIND_MASK),
1024 "internal: Box<Shared> should have an aligned pointer",
1025 );
1026
1027 // Try compare & swapping the pointer into the `arc` field.
1028 // `Release` is used synchronize with other threads that
1029 // will load the `arc` field.
1030 //
1031 // If the `compare_exchange` fails, then the thread lost the
1032 // race to promote the buffer to shared. The `Acquire`
1033 // ordering will synchronize with the `compare_exchange`
1034 // that happened in the other thread and the `Shared`
1035 // pointed to by `actual` will be visible.
1036 match atom.compare_exchange(ptr as _, shared as _, Ordering::AcqRel, Ordering::Acquire) {
1037 Ok(actual) => {
1038 debug_assert!(actual as usize == ptr as usize);
1039 // The upgrade was successful, the new handle can be
1040 // returned.
1041 Bytes {
1042 ptr: offset,
1043 len,
1044 data: AtomicPtr::new(shared as _),
1045 vtable: &SHARED_VTABLE,
1046 }
1047 }
1048 Err(actual) => {
1049 // The upgrade failed, a concurrent clone happened. Release
1050 // the allocation that was made in this thread, it will not
1051 // be needed.
1052 let shared = Box::from_raw(shared);
1053 mem::forget(*shared);
1054
1055 // Buffer already promoted to shared storage, so increment ref
1056 // count.
1057 shallow_clone_arc(actual as _, offset, len)
1058 }
1059 }
1060 }
1061
release_shared(ptr: *mut Shared)1062 unsafe fn release_shared(ptr: *mut Shared) {
1063 // `Shared` storage... follow the drop steps from Arc.
1064 if (*ptr).ref_cnt.fetch_sub(1, Ordering::Release) != 1 {
1065 return;
1066 }
1067
1068 // This fence is needed to prevent reordering of use of the data and
1069 // deletion of the data. Because it is marked `Release`, the decreasing
1070 // of the reference count synchronizes with this `Acquire` fence. This
1071 // means that use of the data happens before decreasing the reference
1072 // count, which happens before this fence, which happens before the
1073 // deletion of the data.
1074 //
1075 // As explained in the [Boost documentation][1],
1076 //
1077 // > It is important to enforce any possible access to the object in one
1078 // > thread (through an existing reference) to *happen before* deleting
1079 // > the object in a different thread. This is achieved by a "release"
1080 // > operation after dropping a reference (any access to the object
1081 // > through this reference must obviously happened before), and an
1082 // > "acquire" operation before deleting the object.
1083 //
1084 // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)
1085 atomic::fence(Ordering::Acquire);
1086
1087 // Drop the data
1088 Box::from_raw(ptr);
1089 }
1090
1091 // compile-fails
1092
1093 /// ```compile_fail
1094 /// use bytes::Bytes;
1095 /// #[deny(unused_must_use)]
1096 /// {
1097 /// let mut b1 = Bytes::from("hello world");
1098 /// b1.split_to(6);
1099 /// }
1100 /// ```
_split_to_must_use()1101 fn _split_to_must_use() {}
1102
1103 /// ```compile_fail
1104 /// use bytes::Bytes;
1105 /// #[deny(unused_must_use)]
1106 /// {
1107 /// let mut b1 = Bytes::from("hello world");
1108 /// b1.split_off(6);
1109 /// }
1110 /// ```
_split_off_must_use()1111 fn _split_off_must_use() {}
1112
1113 // fuzz tests
1114 #[cfg(all(test, loom))]
1115 mod fuzz {
1116 use loom::sync::Arc;
1117 use loom::thread;
1118
1119 use super::Bytes;
1120 #[test]
bytes_cloning_vec()1121 fn bytes_cloning_vec() {
1122 loom::model(|| {
1123 let a = Bytes::from(b"abcdefgh".to_vec());
1124 let addr = a.as_ptr() as usize;
1125
1126 // test the Bytes::clone is Sync by putting it in an Arc
1127 let a1 = Arc::new(a);
1128 let a2 = a1.clone();
1129
1130 let t1 = thread::spawn(move || {
1131 let b: Bytes = (*a1).clone();
1132 assert_eq!(b.as_ptr() as usize, addr);
1133 });
1134
1135 let t2 = thread::spawn(move || {
1136 let b: Bytes = (*a2).clone();
1137 assert_eq!(b.as_ptr() as usize, addr);
1138 });
1139
1140 t1.join().unwrap();
1141 t2.join().unwrap();
1142 });
1143 }
1144 }
1145