-
Notifications
You must be signed in to change notification settings - Fork 145
/
lib.rs
2314 lines (2058 loc) · 67.9 KB
/
lib.rs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! Small vectors in various sizes. These store a certain number of elements inline, and fall back
//! to the heap for larger allocations. This can be a useful optimization for improving cache
//! locality and reducing allocator traffic for workloads that fit within the inline buffer.
//!
//! ## no_std support
//!
//! By default, `smallvec` depends on `libstd`. However, it can be configured to use the unstable
//! `liballoc` API instead, for use on platforms that have `liballoc` but not `libstd`. This
//! configuration is currently unstable and is not guaranteed to work on all versions of Rust.
//!
//! To depend on `smallvec` without `libstd`, use `default-features = false` in the `smallvec`
//! section of Cargo.toml to disable its `"std"` feature.
//!
//! ## `union` feature
//!
//! When the `union` feature is enabled `smallvec` will track its state (inline or spilled)
//! without the use of an enum tag, reducing the size of the `smallvec` by one machine word.
//! This means that there is potentially no space overhead compared to `Vec`.
//! Note that `smallvec` can still be larger than `Vec` if the inline buffer is larger than two
//! machine words.
//!
//! To use this feature add `features = ["union"]` in the `smallvec` section of Cargo.toml.
//! Note that this feature requires a nightly compiler (for now).
#![cfg_attr(not(feature = "std"), no_std)]
#![cfg_attr(not(feature = "std"), feature(alloc))]
#![cfg_attr(feature = "union", feature(untagged_unions))]
#![cfg_attr(feature = "specialization", feature(specialization))]
#![cfg_attr(feature = "may_dangle", feature(dropck_eyepatch))]
#![deny(missing_docs)]
#[cfg(not(feature = "std"))]
#[macro_use]
extern crate alloc;
#[cfg(not(feature = "std"))]
use alloc::vec::Vec;
#[cfg(feature = "serde")]
extern crate serde;
#[cfg(not(feature = "std"))]
mod std {
pub use core::*;
}
use std::borrow::{Borrow, BorrowMut};
use std::cmp;
use std::fmt;
use std::hash::{Hash, Hasher};
use std::iter::{IntoIterator, FromIterator, repeat};
use std::mem;
use std::mem::ManuallyDrop;
use std::ops;
use std::ptr;
use std::slice;
#[cfg(feature = "std")]
use std::io;
#[cfg(feature = "serde")]
use serde::ser::{Serialize, Serializer, SerializeSeq};
#[cfg(feature = "serde")]
use serde::de::{Deserialize, Deserializer, SeqAccess, Visitor};
#[cfg(feature = "serde")]
use std::marker::PhantomData;
/// Creates a [`SmallVec`] containing the arguments.
///
/// `smallvec!` allows `SmallVec`s to be defined with the same syntax as array expressions.
/// There are two forms of this macro:
///
/// - Create a [`SmallVec`] containing a given list of elements:
///
/// ```
/// # #[macro_use] extern crate smallvec;
/// # use smallvec::SmallVec;
/// # fn main() {
/// let v: SmallVec<[_; 128]> = smallvec![1, 2, 3];
/// assert_eq!(v[0], 1);
/// assert_eq!(v[1], 2);
/// assert_eq!(v[2], 3);
/// # }
/// ```
///
/// - Create a [`SmallVec`] from a given element and size:
///
/// ```
/// # #[macro_use] extern crate smallvec;
/// # use smallvec::SmallVec;
/// # fn main() {
/// let v: SmallVec<[_; 0x8000]> = smallvec![1; 3];
/// assert_eq!(v, SmallVec::from_buf([1, 1, 1]));
/// # }
/// ```
///
/// Note that unlike array expressions this syntax supports all elements
/// which implement [`Clone`] and the number of elements doesn't have to be
/// a constant.
///
/// This will use `clone` to duplicate an expression, so one should be careful
/// using this with types having a nonstandard `Clone` implementation. For
/// example, `smallvec![Rc::new(1); 5]` will create a vector of five references
/// to the same boxed integer value, not five references pointing to independently
/// boxed integers.
#[macro_export]
macro_rules! smallvec {
// count helper: transform any expression into 1
(@one $x:expr) => (1usize);
($elem:expr; $n:expr) => ({
$crate::SmallVec::from_elem($elem, $n)
});
($($x:expr),*$(,)*) => ({
let count = 0usize $(+ smallvec!(@one $x))*;
let mut vec = $crate::SmallVec::new();
if count <= vec.inline_size() {
$(vec.push($x);)*
vec
} else {
$crate::SmallVec::from_vec(vec![$($x,)*])
}
});
}
/// Hint to the optimizer that any code path which calls this function is
/// statically unreachable and can be removed.
///
/// Equivalent to `std::hint::unreachable_unchecked` but works in older versions of Rust.
#[inline]
pub unsafe fn unreachable() -> ! {
enum Void {}
let x: &Void = mem::transmute(1usize);
match *x {}
}
/// `panic!()` in debug builds, optimization hint in release.
#[cfg(not(feature = "union"))]
macro_rules! debug_unreachable {
() => { debug_unreachable!("entered unreachable code") };
($e:expr) => {
if cfg!(not(debug_assertions)) {
unreachable();
} else {
panic!($e);
}
}
}
/// Common operations implemented by both `Vec` and `SmallVec`.
///
/// This can be used to write generic code that works with both `Vec` and `SmallVec`.
///
/// ## Example
///
/// ```rust
/// use smallvec::{VecLike, SmallVec};
///
/// fn initialize<V: VecLike<u8>>(v: &mut V) {
/// for i in 0..5 {
/// v.push(i);
/// }
/// }
///
/// let mut vec = Vec::new();
/// initialize(&mut vec);
///
/// let mut small_vec = SmallVec::<[u8; 8]>::new();
/// initialize(&mut small_vec);
/// ```
#[deprecated(note = "Use `Extend` and `Deref<[T]>` instead")]
pub trait VecLike<T>:
ops::Index<usize, Output=T> +
ops::IndexMut<usize> +
ops::Index<ops::Range<usize>, Output=[T]> +
ops::IndexMut<ops::Range<usize>> +
ops::Index<ops::RangeFrom<usize>, Output=[T]> +
ops::IndexMut<ops::RangeFrom<usize>> +
ops::Index<ops::RangeTo<usize>, Output=[T]> +
ops::IndexMut<ops::RangeTo<usize>> +
ops::Index<ops::RangeFull, Output=[T]> +
ops::IndexMut<ops::RangeFull> +
ops::DerefMut<Target = [T]> +
Extend<T> {
/// Append an element to the vector.
fn push(&mut self, value: T);
}
#[allow(deprecated)]
impl<T> VecLike<T> for Vec<T> {
#[inline]
fn push(&mut self, value: T) {
Vec::push(self, value);
}
}
/// Trait to be implemented by a collection that can be extended from a slice
///
/// ## Example
///
/// ```rust
/// use smallvec::{ExtendFromSlice, SmallVec};
///
/// fn initialize<V: ExtendFromSlice<u8>>(v: &mut V) {
/// v.extend_from_slice(b"Test!");
/// }
///
/// let mut vec = Vec::new();
/// initialize(&mut vec);
/// assert_eq!(&vec, b"Test!");
///
/// let mut small_vec = SmallVec::<[u8; 8]>::new();
/// initialize(&mut small_vec);
/// assert_eq!(&small_vec as &[_], b"Test!");
/// ```
pub trait ExtendFromSlice<T> {
/// Extends a collection from a slice of its element type
fn extend_from_slice(&mut self, other: &[T]);
}
impl<T: Clone> ExtendFromSlice<T> for Vec<T> {
fn extend_from_slice(&mut self, other: &[T]) {
Vec::extend_from_slice(self, other)
}
}
unsafe fn deallocate<T>(ptr: *mut T, capacity: usize) {
let _vec: Vec<T> = Vec::from_raw_parts(ptr, 0, capacity);
// Let it drop.
}
/// An iterator that removes the items from a `SmallVec` and yields them by value.
///
/// Returned from [`SmallVec::drain`][1].
///
/// [1]: struct.SmallVec.html#method.drain
pub struct Drain<'a, T: 'a> {
iter: slice::IterMut<'a,T>,
}
impl<'a, T: 'a> Iterator for Drain<'a,T> {
type Item = T;
#[inline]
fn next(&mut self) -> Option<T> {
self.iter.next().map(|reference| unsafe { ptr::read(reference) })
}
#[inline]
fn size_hint(&self) -> (usize, Option<usize>) {
self.iter.size_hint()
}
}
impl<'a, T: 'a> DoubleEndedIterator for Drain<'a, T> {
#[inline]
fn next_back(&mut self) -> Option<T> {
self.iter.next_back().map(|reference| unsafe { ptr::read(reference) })
}
}
impl<'a, T> ExactSizeIterator for Drain<'a, T> { }
impl<'a, T: 'a> Drop for Drain<'a,T> {
fn drop(&mut self) {
// Destroy the remaining elements.
for _ in self.by_ref() {}
}
}
#[cfg(feature = "union")]
union SmallVecData<A: Array> {
inline: ManuallyDrop<A>,
heap: (*mut A::Item, usize),
}
#[cfg(feature = "union")]
impl<A: Array> SmallVecData<A> {
#[inline]
unsafe fn inline(&self) -> &A {
&self.inline
}
#[inline]
unsafe fn inline_mut(&mut self) -> &mut A {
&mut self.inline
}
#[inline]
fn from_inline(inline: A) -> SmallVecData<A> {
SmallVecData { inline: ManuallyDrop::new(inline) }
}
#[inline]
unsafe fn into_inline(self) -> A { ManuallyDrop::into_inner(self.inline) }
#[inline]
unsafe fn heap(&self) -> (*mut A::Item, usize) {
self.heap
}
#[inline]
unsafe fn heap_mut(&mut self) -> &mut (*mut A::Item, usize) {
&mut self.heap
}
#[inline]
fn from_heap(ptr: *mut A::Item, len: usize) -> SmallVecData<A> {
SmallVecData { heap: (ptr, len) }
}
}
#[cfg(not(feature = "union"))]
enum SmallVecData<A: Array> {
Inline(ManuallyDrop<A>),
Heap((*mut A::Item, usize)),
}
#[cfg(not(feature = "union"))]
impl<A: Array> SmallVecData<A> {
#[inline]
unsafe fn inline(&self) -> &A {
match *self {
SmallVecData::Inline(ref a) => a,
_ => debug_unreachable!(),
}
}
#[inline]
unsafe fn inline_mut(&mut self) -> &mut A {
match *self {
SmallVecData::Inline(ref mut a) => a,
_ => debug_unreachable!(),
}
}
#[inline]
fn from_inline(inline: A) -> SmallVecData<A> {
SmallVecData::Inline(ManuallyDrop::new(inline))
}
#[inline]
unsafe fn into_inline(self) -> A {
match self {
SmallVecData::Inline(a) => ManuallyDrop::into_inner(a),
_ => debug_unreachable!(),
}
}
#[inline]
unsafe fn heap(&self) -> (*mut A::Item, usize) {
match *self {
SmallVecData::Heap(data) => data,
_ => debug_unreachable!(),
}
}
#[inline]
unsafe fn heap_mut(&mut self) -> &mut (*mut A::Item, usize) {
match *self {
SmallVecData::Heap(ref mut data) => data,
_ => debug_unreachable!(),
}
}
#[inline]
fn from_heap(ptr: *mut A::Item, len: usize) -> SmallVecData<A> {
SmallVecData::Heap((ptr, len))
}
}
unsafe impl<A: Array + Send> Send for SmallVecData<A> {}
unsafe impl<A: Array + Sync> Sync for SmallVecData<A> {}
/// A `Vec`-like container that can store a small number of elements inline.
///
/// `SmallVec` acts like a vector, but can store a limited amount of data inline within the
/// `SmallVec` struct rather than in a separate allocation. If the data exceeds this limit, the
/// `SmallVec` will "spill" its data onto the heap, allocating a new buffer to hold it.
///
/// The amount of data that a `SmallVec` can store inline depends on its backing store. The backing
/// store can be any type that implements the `Array` trait; usually it is a small fixed-sized
/// array. For example a `SmallVec<[u64; 8]>` can hold up to eight 64-bit integers inline.
///
/// ## Example
///
/// ```rust
/// use smallvec::SmallVec;
/// let mut v = SmallVec::<[u8; 4]>::new(); // initialize an empty vector
///
/// // The vector can hold up to 4 items without spilling onto the heap.
/// v.extend(0..4);
/// assert_eq!(v.len(), 4);
/// assert!(!v.spilled());
///
/// // Pushing another element will force the buffer to spill:
/// v.push(4);
/// assert_eq!(v.len(), 5);
/// assert!(v.spilled());
/// ```
pub struct SmallVec<A: Array> {
// The capacity field is used to determine which of the storage variants is active:
// If capacity <= A::size() then the inline variant is used and capacity holds the current length of the vector (number of elements actually in use).
// If capacity > A::size() then the heap variant is used and capacity holds the size of the memory allocation.
capacity: usize,
data: SmallVecData<A>,
}
impl<A: Array> SmallVec<A> {
/// Construct an empty vector
#[inline]
pub fn new() -> SmallVec<A> {
unsafe {
SmallVec {
capacity: 0,
data: SmallVecData::from_inline(mem::uninitialized()),
}
}
}
/// Construct an empty vector with enough capacity pre-allocated to store at least `n`
/// elements.
///
/// Will create a heap allocation only if `n` is larger than the inline capacity.
///
/// ```
/// # use smallvec::SmallVec;
///
/// let v: SmallVec<[u8; 3]> = SmallVec::with_capacity(100);
///
/// assert!(v.is_empty());
/// assert!(v.capacity() >= 100);
/// ```
#[inline]
pub fn with_capacity(n: usize) -> Self {
let mut v = SmallVec::new();
v.reserve_exact(n);
v
}
/// Construct a new `SmallVec` from a `Vec<A::Item>`.
///
/// Elements will be copied to the inline buffer if vec.capacity() <= A::size().
///
/// ```rust
/// use smallvec::SmallVec;
///
/// let vec = vec![1, 2, 3, 4, 5];
/// let small_vec: SmallVec<[_; 3]> = SmallVec::from_vec(vec);
///
/// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]);
/// ```
#[inline]
pub fn from_vec(mut vec: Vec<A::Item>) -> SmallVec<A> {
if vec.capacity() <= A::size() {
unsafe {
let mut data = SmallVecData::<A>::from_inline(mem::uninitialized());
let len = vec.len();
vec.set_len(0);
ptr::copy_nonoverlapping(vec.as_ptr(), data.inline_mut().ptr_mut(), len);
SmallVec {
capacity: len,
data,
}
}
} else {
let (ptr, cap, len) = (vec.as_mut_ptr(), vec.capacity(), vec.len());
mem::forget(vec);
SmallVec {
capacity: cap,
data: SmallVecData::from_heap(ptr, len),
}
}
}
/// Constructs a new `SmallVec` on the stack from an `A` without
/// copying elements.
///
/// ```rust
/// use smallvec::SmallVec;
///
/// let buf = [1, 2, 3, 4, 5];
/// let small_vec: SmallVec<_> = SmallVec::from_buf(buf);
///
/// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]);
/// ```
#[inline]
pub fn from_buf(buf: A) -> SmallVec<A> {
SmallVec {
capacity: A::size(),
data: SmallVecData::from_inline(buf),
}
}
/// Constructs a new `SmallVec` on the stack from an `A` without
/// copying elements. Also sets the length, which must be less or
/// equal to the size of `buf`.
///
/// ```rust
/// use smallvec::SmallVec;
///
/// let buf = [1, 2, 3, 4, 5, 0, 0, 0];
/// let small_vec: SmallVec<_> = SmallVec::from_buf_and_len(buf, 5);
///
/// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]);
/// ```
#[inline]
pub fn from_buf_and_len(buf: A, len: usize) -> SmallVec<A> {
assert!(len <= A::size());
unsafe { SmallVec::from_buf_and_len_unchecked(buf, len) }
}
/// Constructs a new `SmallVec` on the stack from an `A` without
/// copying elements. Also sets the length. The user is responsible
/// for ensuring that `len <= A::size()`.
///
/// ```rust
/// use smallvec::SmallVec;
///
/// let buf = [1, 2, 3, 4, 5, 0, 0, 0];
/// let small_vec: SmallVec<_> = unsafe {
/// SmallVec::from_buf_and_len_unchecked(buf, 5)
/// };
///
/// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]);
/// ```
#[inline]
pub unsafe fn from_buf_and_len_unchecked(buf: A, len: usize) -> SmallVec<A> {
SmallVec {
capacity: len,
data: SmallVecData::from_inline(buf),
}
}
/// Sets the length of a vector.
///
/// This will explicitly set the size of the vector, without actually
/// modifying its buffers, so it is up to the caller to ensure that the
/// vector is actually the specified size.
pub unsafe fn set_len(&mut self, new_len: usize) {
let (_, len_ptr, _) = self.triple_mut();
*len_ptr = new_len;
}
/// The maximum number of elements this vector can hold inline
#[inline]
pub fn inline_size(&self) -> usize {
A::size()
}
/// The number of elements stored in the vector
#[inline]
pub fn len(&self) -> usize {
self.triple().1
}
/// Returns `true` if the vector is empty
#[inline]
pub fn is_empty(&self) -> bool {
self.len() == 0
}
/// The number of items the vector can hold without reallocating
#[inline]
pub fn capacity(&self) -> usize {
self.triple().2
}
/// Returns a tuple with (data ptr, len, capacity)
/// Useful to get all SmallVec properties with a single check of the current storage variant.
#[inline]
fn triple(&self) -> (*const A::Item, usize, usize) {
unsafe {
if self.spilled() {
let (ptr, len) = self.data.heap();
(ptr, len, self.capacity)
} else {
(self.data.inline().ptr(), self.capacity, A::size())
}
}
}
/// Returns a tuple with (data ptr, len ptr, capacity)
#[inline]
fn triple_mut(&mut self) -> (*mut A::Item, &mut usize, usize) {
unsafe {
if self.spilled() {
let &mut (ptr, ref mut len_ptr) = self.data.heap_mut();
(ptr, len_ptr, self.capacity)
} else {
(self.data.inline_mut().ptr_mut(), &mut self.capacity, A::size())
}
}
}
/// Returns `true` if the data has spilled into a separate heap-allocated buffer.
#[inline]
pub fn spilled(&self) -> bool {
self.capacity > A::size()
}
/// Empty the vector and return an iterator over its former contents.
pub fn drain(&mut self) -> Drain<A::Item> {
unsafe {
let ptr = self.as_mut_ptr();
let current_len = self.len();
self.set_len(0);
let slice = slice::from_raw_parts_mut(ptr, current_len);
Drain {
iter: slice.iter_mut(),
}
}
}
/// Append an item to the vector.
#[inline]
pub fn push(&mut self, value: A::Item) {
unsafe {
let (_, &mut len, cap) = self.triple_mut();
if len == cap {
self.reserve(1);
}
let (ptr, len_ptr, _) = self.triple_mut();
*len_ptr = len + 1;
ptr::write(ptr.offset(len as isize), value);
}
}
/// Remove an item from the end of the vector and return it, or None if empty.
#[inline]
pub fn pop(&mut self) -> Option<A::Item> {
unsafe {
let (ptr, len_ptr, _) = self.triple_mut();
if *len_ptr == 0 {
return None;
}
let last_index = *len_ptr - 1;
*len_ptr = last_index;
Some(ptr::read(ptr.offset(last_index as isize)))
}
}
/// Re-allocate to set the capacity to `max(new_cap, inline_size())`.
///
/// Panics if `new_cap` is less than the vector's length.
pub fn grow(&mut self, new_cap: usize) {
unsafe {
let (ptr, &mut len, cap) = self.triple_mut();
let unspilled = !self.spilled();
assert!(new_cap >= len);
if new_cap <= self.inline_size() {
if unspilled {
return;
}
self.data = SmallVecData::from_inline(mem::uninitialized());
ptr::copy_nonoverlapping(ptr, self.data.inline_mut().ptr_mut(), len);
} else if new_cap != cap {
let mut vec = Vec::with_capacity(new_cap);
let new_alloc = vec.as_mut_ptr();
mem::forget(vec);
ptr::copy_nonoverlapping(ptr, new_alloc, len);
self.data = SmallVecData::from_heap(new_alloc, len);
self.capacity = new_cap;
if unspilled {
return;
}
}
deallocate(ptr, cap);
}
}
/// Reserve capacity for `additional` more elements to be inserted.
///
/// May reserve more space to avoid frequent reallocations.
///
/// If the new capacity would overflow `usize` then it will be set to `usize::max_value()`
/// instead. (This means that inserting `additional` new elements is not guaranteed to be
/// possible after calling this function.)
#[inline]
pub fn reserve(&mut self, additional: usize) {
// prefer triple_mut() even if triple() would work
// so that the optimizer removes duplicated calls to it
// from callers like insert()
let (_, &mut len, cap) = self.triple_mut();
if cap - len < additional {
let new_cap = len.checked_add(additional).
and_then(usize::checked_next_power_of_two).
unwrap_or(usize::max_value());
self.grow(new_cap);
}
}
/// Reserve the minimum capacity for `additional` more elements to be inserted.
///
/// Panics if the new capacity overflows `usize`.
pub fn reserve_exact(&mut self, additional: usize) {
let (_, &mut len, cap) = self.triple_mut();
if cap - len < additional {
match len.checked_add(additional) {
Some(cap) => self.grow(cap),
None => panic!("reserve_exact overflow"),
}
}
}
/// Shrink the capacity of the vector as much as possible.
///
/// When possible, this will move data from an external heap buffer to the vector's inline
/// storage.
pub fn shrink_to_fit(&mut self) {
if !self.spilled() {
return;
}
let len = self.len();
if self.inline_size() >= len {
unsafe {
let (ptr, len) = self.data.heap();
self.data = SmallVecData::from_inline(mem::uninitialized());
ptr::copy_nonoverlapping(ptr, self.data.inline_mut().ptr_mut(), len);
deallocate(ptr, self.capacity);
self.capacity = len;
}
} else if self.capacity() > len {
self.grow(len);
}
}
/// Shorten the vector, keeping the first `len` elements and dropping the rest.
///
/// If `len` is greater than or equal to the vector's current length, this has no
/// effect.
///
/// This does not re-allocate. If you want the vector's capacity to shrink, call
/// `shrink_to_fit` after truncating.
pub fn truncate(&mut self, len: usize) {
unsafe {
let (ptr, len_ptr, _) = self.triple_mut();
while len < *len_ptr {
let last_index = *len_ptr - 1;
*len_ptr = last_index;
ptr::drop_in_place(ptr.offset(last_index as isize));
}
}
}
/// Extracts a slice containing the entire vector.
///
/// Equivalent to `&s[..]`.
pub fn as_slice(&self) -> &[A::Item] {
self
}
/// Extracts a mutable slice of the entire vector.
///
/// Equivalent to `&mut s[..]`.
pub fn as_mut_slice(&mut self) -> &mut [A::Item] {
self
}
/// Remove the element at position `index`, replacing it with the last element.
///
/// This does not preserve ordering, but is O(1).
///
/// Panics if `index` is out of bounds.
#[inline]
pub fn swap_remove(&mut self, index: usize) -> A::Item {
let len = self.len();
self.swap(len - 1, index);
self.pop().unwrap_or_else(|| unsafe { unreachable() })
}
/// Remove all elements from the vector.
#[inline]
pub fn clear(&mut self) {
self.truncate(0);
}
/// Remove and return the element at position `index`, shifting all elements after it to the
/// left.
///
/// Panics if `index` is out of bounds.
pub fn remove(&mut self, index: usize) -> A::Item {
unsafe {
let (mut ptr, len_ptr, _) = self.triple_mut();
let len = *len_ptr;
assert!(index < len);
*len_ptr = len - 1;
ptr = ptr.offset(index as isize);
let item = ptr::read(ptr);
ptr::copy(ptr.offset(1), ptr, len - index - 1);
item
}
}
/// Insert an element at position `index`, shifting all elements after it to the right.
///
/// Panics if `index` is out of bounds.
pub fn insert(&mut self, index: usize, element: A::Item) {
self.reserve(1);
unsafe {
let (mut ptr, len_ptr, _) = self.triple_mut();
let len = *len_ptr;
assert!(index <= len);
*len_ptr = len + 1;
ptr = ptr.offset(index as isize);
ptr::copy(ptr, ptr.offset(1), len - index);
ptr::write(ptr, element);
}
}
/// Insert multiple elements at position `index`, shifting all following elements toward the
/// back.
pub fn insert_many<I: IntoIterator<Item=A::Item>>(&mut self, index: usize, iterable: I) {
let iter = iterable.into_iter();
if index == self.len() {
return self.extend(iter);
}
let (lower_size_bound, _) = iter.size_hint();
assert!(lower_size_bound <= std::isize::MAX as usize); // Ensure offset is indexable
assert!(index + lower_size_bound >= index); // Protect against overflow
self.reserve(lower_size_bound);
unsafe {
let old_len = self.len();
assert!(index <= old_len);
let mut ptr = self.as_mut_ptr().offset(index as isize);
// Move the trailing elements.
ptr::copy(ptr, ptr.offset(lower_size_bound as isize), old_len - index);
// In case the iterator panics, don't double-drop the items we just copied above.
self.set_len(index);
let mut num_added = 0;
for element in iter {
let mut cur = ptr.offset(num_added as isize);
if num_added >= lower_size_bound {
// Iterator provided more elements than the hint. Move trailing items again.
self.reserve(1);
ptr = self.as_mut_ptr().offset(index as isize);
cur = ptr.offset(num_added as isize);
ptr::copy(cur, cur.offset(1), old_len - index);
}
ptr::write(cur, element);
num_added += 1;
}
if num_added < lower_size_bound {
// Iterator provided fewer elements than the hint
ptr::copy(ptr.offset(lower_size_bound as isize), ptr.offset(num_added as isize), old_len - index);
}
self.set_len(old_len + num_added);
}
}
/// Convert a SmallVec to a Vec, without reallocating if the SmallVec has already spilled onto
/// the heap.
pub fn into_vec(self) -> Vec<A::Item> {
if self.spilled() {
unsafe {
let (ptr, len) = self.data.heap();
let v = Vec::from_raw_parts(ptr, len, self.capacity);
mem::forget(self);
v
}
} else {
self.into_iter().collect()
}
}
/// Convert the SmallVec into an `A` if possible. Otherwise return `Err(Self)`.
///
/// This method returns `Err(Self)` if the SmallVec is too short (and the `A` contains uninitialized elements),
/// or if the SmallVec is too long (and all the elements were spilled to the heap).
pub fn into_inner(self) -> Result<A, Self> {
if self.spilled() || self.len() != A::size() {
Err(self)
} else {
unsafe {
let data = ptr::read(&self.data);
mem::forget(self);
Ok(data.into_inline())
}
}
}
/// Retains only the elements specified by the predicate.
///
/// In other words, remove all elements `e` such that `f(&e)` returns `false`.
/// This method operates in place and preserves the order of the retained
/// elements.
pub fn retain<F: FnMut(&mut A::Item) -> bool>(&mut self, mut f: F) {
let mut del = 0;
let len = self.len();
for i in 0..len {
if !f(&mut self[i]) {
del += 1;
} else if del > 0 {
self.swap(i - del, i);
}
}
self.truncate(len - del);
}
/// Removes consecutive duplicate elements.
pub fn dedup(&mut self) where A::Item: PartialEq<A::Item> {
self.dedup_by(|a, b| a == b);
}
/// Removes consecutive duplicate elements using the given equality relation.
pub fn dedup_by<F>(&mut self, mut same_bucket: F)
where F: FnMut(&mut A::Item, &mut A::Item) -> bool
{
// See the implementation of Vec::dedup_by in the
// standard library for an explanation of this algorithm.
let len = self.len();
if len <= 1 {
return;
}
let ptr = self.as_mut_ptr();
let mut w: usize = 1;
unsafe {
for r in 1..len {
let p_r = ptr.offset(r as isize);
let p_wm1 = ptr.offset((w - 1) as isize);
if !same_bucket(&mut *p_r, &mut *p_wm1) {
if r != w {
let p_w = p_wm1.offset(1);
mem::swap(&mut *p_r, &mut *p_w);
}
w += 1;
}
}
}
self.truncate(w);
}
/// Removes consecutive elements that map to the same key.
pub fn dedup_by_key<F, K>(&mut self, mut key: F)
where F: FnMut(&mut A::Item) -> K,
K: PartialEq<K>
{
self.dedup_by(|a, b| key(a) == key(b));
}
/// Creates a `SmallVec` directly from the raw components of another
/// `SmallVec`.
///
/// # Safety
///
/// This is highly unsafe, due to the number of invariants that aren't
/// checked:
///
/// * `ptr` needs to have been previously allocated via `SmallVec` for its
/// spilled storage (at least, it's highly likely to be incorrect if it
/// wasn't).
/// * `ptr`'s `A::Item` type needs to be the same size and alignment that
/// it was allocated with
/// * `length` needs to be less than or equal to `capacity`.
/// * `capacity` needs to be the capacity that the pointer was allocated
/// with.
///
/// Violating these may cause problems like corrupting the allocator's
/// internal data structures.
///
/// Additionally, `capacity` must be greater than the amount of inline
/// storage `A` has; that is, the new `SmallVec` must need to spill over
/// into heap allocated storage. This condition is asserted against.
///
/// The ownership of `ptr` is effectively transferred to the
/// `SmallVec` which may then deallocate, reallocate or change the
/// contents of memory pointed to by the pointer at will. Ensure
/// that nothing else uses the pointer after calling this
/// function.
///
/// # Examples
///
/// ```
/// # #[macro_use] extern crate smallvec;
/// # use smallvec::SmallVec;
/// use std::mem;
/// use std::ptr;
///
/// fn main() {
/// let mut v: SmallVec<[_; 1]> = smallvec![1, 2, 3];
///
/// // Pull out the important parts of `v`.
/// let p = v.as_mut_ptr();
/// let len = v.len();
/// let cap = v.capacity();
/// let spilled = v.spilled();
///
/// unsafe {
/// // Forget all about `v`. The heap allocation that stored the
/// // three values won't be deallocated.