Line data Source code
1 : #pragma once
2 :
3 : #include "MemoryResource.h"
4 :
5 : #include <algorithm>
6 : #include <iterator>
7 : #include <limits>
8 : #include <stdexcept>
9 : #include <cstring>
10 : #include <functional>
11 : #include <atomic>
12 :
13 : #include <thrust/copy.h>
14 : #include <thrust/fill.h>
15 : #include <thrust/universal_vector.h>
16 : #include <thrust/execution_policy.h>
17 :
18 : namespace elsa::mr
19 : {
20 : namespace type_tags
21 : {
22 : /* - usual type handling as expected */
23 : struct complex {
24 : };
25 :
26 : /* - use memory-transfer functions to process copy/move/set of larger ranges
27 : * - single values or non-continuous iterators may still use assigning/construction
28 : * - default-initialization will invoke the default constructor once
29 : * - no destruction performed */
30 : struct trivial {
31 : };
32 :
33 : /* - additionally to trivial, no default construction is ever performed
34 : * - other constructors may still be called */
35 : struct uninitialized : public trivial {
36 : };
37 : } // namespace type_tags
38 :
39 : namespace detail
40 : {
41 : template <class Type>
42 : using is_random_iterator =
43 : std::is_same<typename std::iterator_traits<Type>::iterator_category,
44 : std::random_access_iterator_tag>;
45 :
46 : /* check if Type defines a non-static member-function
47 : * 'base' and extract its return value */
48 : template <class Type>
49 : struct has_base_member {
50 : struct error_type {
51 : };
52 :
53 : template <class Tp, class RTp = decltype(std::declval<Tp>().base()),
54 : class _enable = std::invoke_result_t<decltype(&Tp::base), Tp>>
55 : static RTp test(int);
56 : template <class Tp>
57 : static error_type test(...);
58 :
59 : using type = decltype(test<Type>(0));
60 : static constexpr bool has = !std::is_same<type, error_type>::value;
61 : };
62 :
63 : /* check if Type defines a non-static member-function
64 : * 'get' and extract its return value */
65 : template <class Type>
66 : struct has_get_member {
67 : struct error_type {
68 : };
69 :
70 : template <class Tp, class RTp = decltype(std::declval<Tp>().get()),
71 : class _enable = std::invoke_result_t<decltype(&Tp::get), Tp>>
72 : static RTp test(int);
73 : template <class Tp>
74 : static error_type test(...);
75 :
76 : using type = decltype(test<Type>(0));
77 : static constexpr bool has = !std::is_same<type, error_type>::value;
78 : };
79 :
80 : template <class Type>
81 : using actual_pointer = std::is_pointer<std::remove_reference_t<Type>>;
82 :
83 : template <class Type>
84 : using base_returns_pointer =
85 : actual_pointer<std::conditional_t<has_base_member<Type>::has,
86 : typename has_base_member<Type>::type, void>>;
87 :
88 : template <class Type>
89 : using get_returns_pointer =
90 : actual_pointer<std::conditional_t<has_get_member<Type>::has,
91 : typename has_get_member<Type>::type, void>>;
92 :
93 : template <class Tag>
94 : constexpr bool is_trivial = std::is_base_of<type_tags::trivial, Tag>::value;
95 :
96 : template <class Tag>
97 : constexpr bool is_uninitialized = std::is_base_of<type_tags::uninitialized, Tag>::value;
98 :
99 : template <class Tag>
100 : constexpr bool is_complex = std::is_base_of<type_tags::complex, Tag>::value;
101 :
102 : /* type-wrapper for the pointer to be casted to to prevent native
103 : * opeartions on the type to be called (as the contigous-vector already takes care of them)
104 : */
105 : template <class Type>
106 : struct type_wrapper {
107 : uint8_t payload[sizeof(Type)];
108 : };
109 :
110 : template <class Type>
111 : void fillMemory(Type* ptr, const void* src, size_t count)
112 35 : {
113 35 : using Tp = type_wrapper<Type>;
114 :
115 35 : auto src_ = reinterpret_cast<const Tp*>(src);
116 35 : auto dst_ = thrust::universal_ptr<Tp>(reinterpret_cast<Tp*>(ptr));
117 :
118 35 : thrust::fill(thrust::device, dst_, dst_ + count, *src_);
119 35 : }
120 : template <class Type>
121 : void copyMemory(void* ptr, const void* src, size_t count, bool src_universal)
122 24756 : {
123 24756 : using Tp = type_wrapper<Type>;
124 :
125 24756 : auto src_ = reinterpret_cast<const Tp*>(src);
126 24756 : auto dst_ = reinterpret_cast<Tp*>(ptr);
127 :
128 24756 : if (src_universal) {
129 24103 : auto usrc = thrust::universal_ptr<const Tp>(src_);
130 24103 : auto udst = thrust::universal_ptr<Tp>(dst_);
131 24103 : thrust::copy(thrust::device, usrc, usrc + count, udst);
132 24103 : } else
133 653 : thrust::copy(thrust::host, src_, src_ + count, dst_);
134 24756 : }
135 : template <class Type>
136 : void moveMemory(void* ptr, const void* src, size_t count)
137 12 : {
138 12 : std::memmove(ptr, src, count * sizeof(Type));
139 12 : }
140 :
141 : /*
142 : * A container is a reference-counted managing unit of 'capacity' number of types,
143 : * allocated with the corresponding resource and managed with the given type-tags.
144 : * - The first 'size' number of values are at all times considered constructed.
145 : * - Size will always be less or equal to capacity.
146 : * - A capacity of zero implies no allocation and therefore a null-pointer (vice versa).
147 : * - A pointer of zero is the null-container.
148 : *
149 : * Additionally the container can also manage external alloctions. For
150 : * this, it will call the cleanup function whenever the container is not used anymore,
151 : * and resizing is disabled.
152 : */
153 : template <class Type, class TypeTag>
154 : struct ContContainer {
155 : public:
156 : using self_type = ContContainer<Type, TypeTag>;
157 : using value_type = Type;
158 : using size_type = size_t;
159 : using difference_type = ptrdiff_t;
160 :
161 : using raw_pointer = Type*;
162 : using const_raw_pointer = const Type*;
163 : using reference = Type&;
164 : using const_reference = const Type&;
165 :
166 : public:
167 : mr::MemoryResource resource;
168 : raw_pointer pointer = nullptr;
169 : size_type size = 0;
170 : size_type capacity = 0;
171 : std::function<void()> release;
172 :
173 : public:
174 : ContContainer()
175 : {
176 : static_assert(
177 : is_trivial<TypeTag> || is_uninitialized<TypeTag> || is_complex<TypeTag>,
178 : "unknown type-tag encountered (only complex, trivial, uninitialized known)");
179 : }
180 :
181 : ContContainer(const mr::MemoryResource& r, raw_pointer p, size_type s, size_type c)
182 217106 : {
183 217106 : resource = r;
184 217106 : pointer = p;
185 217106 : size = s;
186 217106 : capacity = c;
187 217106 : }
188 :
189 : ContContainer(const mr::MemoryResource& r, raw_pointer p, size_type s,
190 : std::function<void()>&& cleanup)
191 15 : {
192 15 : resource = r;
193 15 : pointer = p;
194 15 : size = s;
195 15 : capacity = s;
196 15 : release = std::move(cleanup);
197 15 : }
198 :
199 : ContContainer(const self_type&) = delete;
200 :
201 : ContContainer(self_type&& c) = delete;
202 :
203 : ContContainer& operator=(const self_type&) = delete;
204 :
205 : ContContainer& operator=(self_type&& c) = delete;
206 :
207 : ~ContContainer()
208 217121 : {
209 217121 : if (release) {
210 15 : release();
211 15 : return;
212 15 : }
213 217106 : if (pointer == nullptr)
214 142256 : return;
215 74850 : destruct_until(0);
216 74850 : resource->deallocate(pointer, capacity * sizeof(value_type), alignof(value_type));
217 74850 : }
218 :
219 : public:
220 92017 : raw_pointer end_ptr() const { return pointer + size; }
221 :
222 : void destruct_until(size_type count) noexcept
223 82364 : {
224 82364 : if constexpr (is_trivial<TypeTag>)
225 81849 : size = count;
226 515 : else {
227 11687 : while (size > count)
228 11172 : pointer[--size].~value_type();
229 515 : }
230 82364 : }
231 :
232 : /*
233 : * The container will at most have capacity 'count' after the call
234 : * -> count must be smaller or equal to 'size'
235 : */
236 : std::shared_ptr<self_type> reduce(size_type count, bool single_ref)
237 13748 : {
238 : /* set the recouce-argument, which will force a new allocation
239 : * even if the capacity would be large enough */
240 13748 : if (capacity > count)
241 12568 : return reserve(count, count, single_ref, resource);
242 1180 : return nullptr;
243 1180 : }
244 :
245 : /*
246 : * count: how many to at least require
247 : * move: if realloc, how many to move-construct over (must be less than size)
248 : * mr: if not zero, force-relloc to the new memory resource
249 : * (even if its the same resource)
250 : * returns null-container or new container if relocation occurred, in
251 : * which case the new container will have a size of 'copy'
252 : */
253 : std::shared_ptr<self_type> reserve(size_type count, size_type move, bool single_ref,
254 : const mr::MemoryResource& mr)
255 94853 : {
256 94853 : if (count <= capacity && !mr)
257 7546 : return nullptr;
258 87307 : mr::MemoryResource new_mr = bool(mr) ? mr : resource;
259 :
260 : /* check if this is a noninitial reserving for a growing size in which
261 : * case the capacity should not just satisfy the request but by some form
262 : * of equation to prevent to many reallocations */
263 87307 : size_type new_cap = std::max<size_type>(count, move);
264 87307 : if (new_cap > capacity && capacity > 0) {
265 199 : size_type min_cap = new_cap;
266 199 : new_cap = capacity;
267 :
268 199 : if (capacity >= 4) {
269 484 : while (new_cap < min_cap)
270 310 : new_cap += (new_cap / 2);
271 174 : } else {
272 76 : while (new_cap < min_cap)
273 51 : new_cap += new_cap;
274 25 : }
275 199 : }
276 :
277 : /* check if an empty container has been requested */
278 87307 : if (new_cap == 0)
279 12457 : return std::make_shared<self_type>(new_mr, nullptr, 0, 0);
280 :
281 : /* check if a relocation could suffice (not considered a container-change)
282 : * (can only occur, if this container has only one reference and is self-owned) */
283 74850 : if (pointer != nullptr && new_mr == resource && !bool(release) && single_ref
284 74850 : && resource->tryResize(pointer, capacity * sizeof(value_type),
285 296 : alignof(value_type), new_cap * sizeof(value_type))) {
286 0 : capacity = new_cap;
287 0 : return nullptr;
288 0 : }
289 :
290 : /* perform the new allocation */
291 74850 : raw_pointer new_ptr = static_cast<raw_pointer>(
292 74850 : new_mr->allocate(new_cap * sizeof(value_type), alignof(value_type)));
293 74850 : std::shared_ptr<self_type> next =
294 74850 : std::make_shared<self_type>(new_mr, new_ptr, 0, new_cap);
295 :
296 : /* either move-initialize the data or perform the memory move */
297 74850 : if constexpr (!is_trivial<TypeTag>) {
298 74545 : for (size_type i = 0; i < move; ++i) {
299 2023 : new (new_ptr + i) value_type(std::move(pointer[i]));
300 2023 : ++next->size;
301 2023 : }
302 74545 : } else if (move > 0) {
303 164 : copyMemory<value_type>(new_ptr, pointer, move, true);
304 164 : next->size = move;
305 164 : }
306 74850 : return next;
307 74850 : }
308 :
309 : /*
310 : * move [where; end] by diff within the container
311 : * (where + diff) must be less or equal to size
312 : */
313 : void move_tail(size_type where, difference_type diff)
314 20 : {
315 : /* check if this move can be delegated to the mr */
316 20 : if constexpr (is_trivial<TypeTag>) {
317 12 : moveMemory<value_type>(pointer + (where + diff), pointer + where,
318 12 : (size - where));
319 12 : if (diff >= 0)
320 3 : size = std::max<size_type>(size, (size - where) + diff);
321 9 : else
322 9 : size += diff;
323 12 : return;
324 12 : }
325 :
326 : /* check if the entire content needs to be moved down (i.e. size decrease) */
327 8 : if (diff <= 0) {
328 : /* reminder: diff is negative */
329 117 : while (where < size) {
330 109 : pointer[where + diff] = std::move(pointer[where]);
331 109 : ++where;
332 109 : }
333 8 : destruct_until(size + diff);
334 8 : return;
335 8 : }
336 :
337 : /* first move the upper 'new' part in order to ensure the entire buffer is
338 : * constructed and an exception would therefore not leave an invalid state */
339 0 : size_type end = size, next = size - diff;
340 0 : while (next < end) {
341 0 : new (pointer + size) value_type(std::move(pointer[next]));
342 0 : ++size;
343 0 : ++next;
344 0 : }
345 :
346 : /* move the remaining part up into the already, but moved out slots */
347 0 : end = where + diff;
348 0 : while (--next >= end)
349 0 : pointer[next] = std::move(pointer[next - diff]);
350 0 : }
351 :
352 : /* copy a range into a given range */
353 : template <class ItType>
354 : void insert_range(size_type off, ItType ibegin, size_type count, bool is_universal)
355 36390 : {
356 36390 : size_type end = off + count;
357 :
358 : /* check if the iterator has a 'base' function to get the raw pointer */
359 36390 : if constexpr (detail::base_returns_pointer<ItType>::value
360 36390 : && detail::is_random_iterator<ItType>::value)
361 548 : insert_range(off, ibegin.base(), count, is_universal);
362 :
363 : /* Check if the iterator has a 'get' function to get the raw pointer.
364 : * Not checking for random access iterator, because we are checking for a
365 : * thrust pointer type here, which is not tagged as random access iterator.
366 : */
367 35842 : else if constexpr (detail::get_returns_pointer<ItType>::value)
368 7608 : insert_range(off, ibegin.get(), count, is_universal);
369 :
370 : /* check if the iterator is a pointer of the right type
371 : * and can be reduced to a memory operation */
372 28234 : else if constexpr (detail::actual_pointer<ItType>::value
373 28234 : && std::is_same<std::decay_t<decltype(*ibegin)>,
374 0 : std::decay_t<value_type>>::value
375 28234 : && is_trivial<TypeTag>) {
376 3642 : copyMemory<value_type>(pointer + off, ibegin, count, is_universal);
377 3642 : size = std::max<size_type>(size, end);
378 3642 : }
379 :
380 : /* insert the range by using the iterators */
381 3642 : else {
382 3642 : static_cast<void>(is_universal);
383 :
384 3642 : size_type transition = std::min<size_type>(size, end);
385 :
386 : /* copy-assign the currently existing and to-be-overwritten part */
387 3883 : while (off < transition) {
388 241 : pointer[off] = *ibegin;
389 241 : ++off;
390 241 : ++ibegin;
391 241 : }
392 :
393 : /* copy-construct the remaining part */
394 2230428 : while (size < end) {
395 2226786 : new (pointer + size) value_type(*ibegin);
396 2226786 : ++size;
397 2226786 : ++ibegin;
398 2226786 : }
399 3642 : }
400 36390 : }
401 :
402 : /* write same value (or default-construct/reset) into a given range */
403 : void set_range(size_type off, const_raw_pointer value, size_type count)
404 53681 : {
405 53681 : size_type end = off + count;
406 53681 : size_type transition = std::min<size_type>(size, end);
407 :
408 : /* set/update the range with the default constructor/value */
409 53681 : if (value == nullptr) {
410 : /* check if the setting can be entirely skipped or delegated to the mr */
411 53646 : if constexpr (is_uninitialized<TypeTag>)
412 53624 : size = std::max<size_type>(size, end);
413 22 : else if constexpr (is_trivial<TypeTag>) {
414 : /* construct the one temporary value (only constructor is required,
415 : * destructor is not necessary nor expected from this type-tag) */
416 11 : uint8_t tmp[sizeof(value_type)] = {0};
417 11 : new (tmp) value_type();
418 :
419 11 : fillMemory<value_type>(pointer + off, tmp, count);
420 11 : size = std::max<size_type>(size, end);
421 :
422 11 : } else {
423 61 : while (off < transition)
424 50 : pointer[off++] = std::move(value_type());
425 :
426 270 : while (size < end) {
427 259 : new (pointer + size) value_type();
428 259 : ++size;
429 259 : }
430 11 : }
431 53646 : }
432 :
433 : /* set/update the range with the given value */
434 35 : else if constexpr (is_trivial<TypeTag>) {
435 11 : fillMemory<value_type>(pointer + off, value, count);
436 11 : size = std::max<size_type>(size, end);
437 11 : }
438 :
439 11 : else {
440 11 : while (off < transition)
441 0 : pointer[off++] = *value;
442 :
443 491 : while (size < end) {
444 480 : new (pointer + size) value_type(*value);
445 480 : ++size;
446 480 : }
447 11 : }
448 53681 : }
449 : };
450 : } // namespace detail
451 :
452 : /*
453 : * Return the state of the container and ensure its contents will
454 : * not be released until the returned release-function has been called.
455 : */
456 : template <class Type>
457 : struct NativeContainer {
458 : std::function<void()> release;
459 : Type* raw_pointer = nullptr;
460 : size_t size = 0;
461 : };
462 :
463 : /// @brief ContiguousVector behaves like an stl-vector (std::vector) with the difference of
464 : /// configuring its type behavior and allocations. Otherwise it uses copy/move
465 : /// semantics where possible/applicable.
466 : ///
467 : /// Exceptions thrown by the type in the
468 : /// default-constructor
469 : /// move-constructor
470 : /// copy-constructor
471 : /// move-assignment operator
472 : /// copy-assignment operator
473 : /// or any iterators passed into this objects will leave the container
474 : /// in a valid state (i.e. proper cleanup is ensured) but a modified state.
475 : /// - For all other exceptions, atomic transactional behavior is guaranteed by the
476 : /// container.
477 : ///
478 : /// PointerType and IteratorType must both be constructable from a raw Type-pointer.
479 : ///
480 : /// In case of a trivial type_tag, the container will actively look for pointers as
481 : /// parameters
482 : /// to switch over to memory operations compared to iterator operations.
483 : /// It consideres the following iterators as pointers:
484 : /// - it is a pointer
485 : /// - it is a random access iterator, which has a 'base' member function, which returns
486 : /// a
487 : /// pointer
488 : /// - it is a random access iterator, which has a 'get' member function, which returns a
489 : /// pointer In all other scenarios, simple loops over iterators will occur, compared to the
490 : /// potentially faster memory operations.
491 : ///
492 : /// Each ContiguousVector is associated with a mr::MemoryResource (polymorphic
493 : /// allocators). All allocations and memory-operations are performed on the currently used
494 : /// resource. Changes to the current resource associated with this container:
495 : /// - copy-construct (optionally inherit the incoming resource)
496 : /// - move-construct (inherit the incoming resource)
497 : /// - other-constructors (parameter-resource or mr::defaultResource)
498 : /// - swap_resource calls
499 : /// - swap_content calls
500 : /// - swap() calls (optional parameter-resource else unchanged)
501 : /// - assign()/assign_default() calls (optional parameter-resource else unchanged)
502 : /// @tparam Type type of data stored in the object
503 : /// @tparam TypeTag handling of the underlying type
504 : /// @tparam PointerType type used to pass pointer to data (must be constructable from raw
505 : /// Type-pointer).
506 : /// @tparam IteratorType type used to pass iterator to data (must be constructable from raw
507 : /// Type-pointer).
508 : template <class Type, class TypeTag, template <class> class PointerType,
509 : template <class> class IteratorType>
510 : class ContiguousVector
511 : {
512 : public:
513 : using self_type = ContiguousVector<Type, TypeTag, PointerType, IteratorType>;
514 : using container_type = detail::ContContainer<Type, TypeTag>;
515 : using value_type = typename container_type::value_type;
516 : using size_type = typename container_type::size_type;
517 : using difference_type = typename container_type::difference_type;
518 :
519 : using raw_pointer = typename container_type::raw_pointer;
520 : using const_raw_pointer = typename container_type::const_raw_pointer;
521 : using reference = typename container_type::reference;
522 : using const_reference = typename container_type::const_reference;
523 :
524 : using pointer = PointerType<Type>;
525 : using const_pointer = PointerType<const Type>;
526 : using iterator = IteratorType<Type>;
527 : using const_iterator = IteratorType<const Type>;
528 : using reverse_iterator = std::reverse_iterator<iterator>;
529 : using const_reverse_iterator = std::reverse_iterator<const_iterator>;
530 :
531 : private:
532 : template <class ItType>
533 : static constexpr bool is_universal_impl =
534 : std::is_same<ItType, iterator>::value || std::is_same<ItType, const_iterator>::value
535 : || std::is_same<ItType, pointer>::value || std::is_same<ItType, const_pointer>::value;
536 : template <class ItType>
537 : static constexpr bool is_universal = is_universal_impl<std::decay_t<ItType>>;
538 :
539 : private:
540 : std::shared_ptr<container_type> _self;
541 :
542 : private:
543 : /* reserves the given amount, updates the current container, and returns the old container
544 : */
545 : std::shared_ptr<container_type>
546 : _reserve(size_type count, size_type move,
547 : const mr::MemoryResource& mr = mr::MemoryResource())
548 82285 : {
549 82285 : std::shared_ptr<container_type> _tmp = _self->reserve(count, move, _self.unique(), mr);
550 :
551 : /* check if a new container was allocated and swap them */
552 82285 : if (_tmp == nullptr)
553 7546 : return nullptr;
554 74739 : std::swap(_self, _tmp);
555 :
556 : /* return the old container, if it is non-empty */
557 74739 : if (_tmp->pointer == nullptr)
558 74509 : return nullptr;
559 230 : return _tmp;
560 230 : }
561 :
562 : /* reduce the current container to the given amount */
563 : void _reduce(size_type count)
564 13748 : {
565 13748 : std::shared_ptr<container_type> _tmp = _self->reduce(count, _self.unique());
566 13748 : if (_tmp == nullptr)
567 1180 : return;
568 12568 : _self = _tmp;
569 12568 : }
570 :
571 : public:
572 : /// @brief Use defaultResource as initial resource
573 : ContiguousVector()
574 447 : {
575 447 : _self = std::make_shared<container_type>(mr::defaultResource(), nullptr, 0, 0);
576 447 : }
577 :
578 : /// @brief Use defaultResource as initial resource
579 : explicit ContiguousVector(size_type count)
580 53619 : {
581 53619 : _self = std::make_shared<container_type>(mr::defaultResource(), nullptr, 0, 0);
582 :
583 53619 : _reserve(count, 0);
584 53619 : _self->set_range(0, nullptr, count);
585 53619 : }
586 :
587 : /// @brief Use defaultResource as initial resource
588 : explicit ContiguousVector(size_type count, const_reference init)
589 8 : {
590 8 : _self = std::make_shared<container_type>(mr::defaultResource(), nullptr, 0, 0);
591 :
592 8 : _reserve(count, 0);
593 8 : _self->set_range(0, &init, count);
594 8 : }
595 :
596 : /// @brief Use defaultResource as initial resource
597 : template <class ItType>
598 : ContiguousVector(ItType ibegin, ItType iend)
599 4274 : {
600 4274 : _self = std::make_shared<container_type>(mr::defaultResource(), nullptr, 0, 0);
601 :
602 4274 : size_type count = std::distance(ibegin, iend);
603 4274 : _reserve(count, 0);
604 4274 : _self->insert_range(0, ibegin, count, is_universal<ItType>);
605 4274 : }
606 :
607 : /// @brief Use defaultResource as initial resource
608 : ContiguousVector(std::initializer_list<value_type> l)
609 : {
610 : _self = std::make_shared<container_type>(mr::defaultResource(), nullptr, 0, 0);
611 :
612 : _reserve(l.size(), 0);
613 : _self->insert_range(0, l.begin(), l.size(), false);
614 : }
615 :
616 : /// @brief If mr is invalid, mr::defaultResource will be used, else mr will be used
617 : explicit ContiguousVector(const mr::MemoryResource& mr)
618 45 : {
619 45 : _self =
620 45 : std::make_shared<container_type>(!mr ? mr::defaultResource() : mr, nullptr, 0, 0);
621 45 : }
622 :
623 : /// @brief If mr is invalid, mr::defaultResource will be used, else mr will be used
624 : explicit ContiguousVector(size_type count, const mr::MemoryResource& mr)
625 10 : {
626 10 : _self =
627 10 : std::make_shared<container_type>(!mr ? mr::defaultResource() : mr, nullptr, 0, 0);
628 :
629 10 : _reserve(count, 0);
630 10 : _self->set_range(0, nullptr, count);
631 10 : }
632 :
633 : /// @brief If mr is invalid, mr::defaultResource will be used, else mr will be used
634 : explicit ContiguousVector(size_type count, const_reference init,
635 : const mr::MemoryResource& mr)
636 10 : {
637 10 : _self =
638 10 : std::make_shared<container_type>(!mr ? mr::defaultResource() : mr, nullptr, 0, 0);
639 :
640 10 : _reserve(count, 0);
641 10 : _self->set_range(0, &init, count);
642 10 : }
643 :
644 : /// @brief If mr is invalid, mr::defaultResource will be used, else mr will be used
645 : template <class ItType>
646 : explicit ContiguousVector(ItType ibegin, ItType iend, const mr::MemoryResource& mr)
647 385 : {
648 385 : _self =
649 385 : std::make_shared<container_type>(!mr ? mr::defaultResource() : mr, nullptr, 0, 0);
650 :
651 385 : size_type count = std::distance(ibegin, iend);
652 385 : _reserve(count, 0);
653 385 : _self->insert_range(0, ibegin, count, is_universal<ItType>);
654 385 : }
655 :
656 : /// @brief If mr is invalid, mr::defaultResource will be used, else mr will be used
657 : explicit ContiguousVector(std::initializer_list<value_type> l, const mr::MemoryResource& mr)
658 10 : {
659 10 : _self =
660 10 : std::make_shared<container_type>(!mr ? mr::defaultResource() : mr, nullptr, 0, 0);
661 :
662 10 : _reserve(l.size(), 0);
663 10 : _self->insert_range(0, l.begin(), l.size(), false);
664 10 : }
665 :
666 : /// @brief Use s::resource as initial resource
667 : ContiguousVector(const self_type& s)
668 16228 : {
669 16228 : _self = std::make_shared<container_type>(s._self->resource, nullptr, 0, 0);
670 :
671 16228 : _reserve(s._self->size, 0);
672 16228 : _self->insert_range(0, s._self->pointer, s._self->size, true);
673 16228 : }
674 :
675 : /// @brief If mr is invalid, s::resource will be used, else mr will be used
676 : explicit ContiguousVector(const self_type& s, const mr::MemoryResource& mr)
677 10 : {
678 10 : _self = std::make_shared<container_type>(!mr ? s._self->resource : mr, nullptr, 0, 0);
679 :
680 10 : _reserve(s._self->size, 0);
681 10 : _self->insert_range(0, s._self->pointer, s._self->size, true);
682 10 : }
683 :
684 : /// @brief Use s::resource as initial resource
685 : ContiguousVector(self_type&& s) noexcept
686 54753 : {
687 54753 : _self = std::make_shared<container_type>(s._self->resource, nullptr, 0, 0);
688 54753 : std::swap(_self, s._self);
689 54753 : }
690 :
691 129799 : ~ContiguousVector() = default;
692 :
693 : public:
694 : /// @brief Retrieve a copy of the underlying native container.
695 : /// The underlying memory will not be released until the release function has been
696 : /// invoked. Any relocations/releases of this ContiguousVector will allocate a new
697 : /// container, but leave the old container alive.
698 : NativeContainer<Type> lock_native() const
699 21 : {
700 : /* setup the release function to be a functor, which has a shared
701 : * pointer and upon invocation it releases the shared pointer */
702 21 : struct CleanupFunctor {
703 21 : std::shared_ptr<container_type> _object;
704 21 : CleanupFunctor(const std::shared_ptr<container_type>& in) : _object(in) {}
705 21 : void operator()() { _object.reset(); }
706 21 : };
707 :
708 21 : NativeContainer<Type> _out;
709 21 : _out.raw_pointer = _self->pointer;
710 21 : _out.size = _self->size;
711 21 : _out.release = CleanupFunctor(_self);
712 21 : return _out;
713 21 : }
714 :
715 : /// @brief Setup the internal structure to point to the raw_pointer and use the
716 : /// size/capacity = size. The ContiguousVector will call destruct whenever the
717 : /// last reference to the resource has been released. The current resource will be
718 : /// stored and any relocation/release will allocate a new container with the old
719 : /// resource, thereby calling destruct on this object.
720 : void from_extern(Type* raw_pointer, size_t size, std::function<void()> destruct)
721 15 : {
722 15 : _self = std::make_shared<container_type>(_self->resource, raw_pointer, size,
723 15 : std::move(destruct));
724 15 : }
725 :
726 145 : mr::MemoryResource resource() const { return _self->resource; }
727 :
728 : /// @brief If mr is invalid, mr::defaultResource will be used, else mr will be used
729 : void swap_resource(const mr::MemoryResource& mr)
730 5 : {
731 5 : mr::MemoryResource actual = bool(mr) ? mr : mr::defaultResource();
732 5 : if (actual == _self->resource)
733 0 : return;
734 5 : _reserve(0, _self->size, actual);
735 5 : }
736 :
737 : /// @brief o::resourse will be used
738 5 : void swap_content(self_type& o) { std::swap(_self, o._self); }
739 :
740 : /// @brief Current resource will be kept
741 : self_type& operator=(const self_type& s)
742 10 : {
743 10 : if (&s == this)
744 0 : return *this;
745 10 : assign(s);
746 10 : return *this;
747 10 : }
748 :
749 : /// @brief Current resource will be kept
750 : self_type& operator=(self_type&& s)
751 13617 : {
752 13617 : if (&s == this)
753 0 : return *this;
754 13617 : if (_self->resource == s._self->resource)
755 13612 : std::swap(_self, s._self);
756 5 : else
757 5 : assign(s);
758 13617 : s._reduce(0);
759 13617 : return *this;
760 13617 : }
761 :
762 : /// @brief Current resource will be kept
763 : self_type& operator=(std::initializer_list<value_type> l)
764 5 : {
765 5 : assign(l);
766 5 : return *this;
767 5 : }
768 :
769 : /// @brief If mr is invalid, current resource will be used, else mr will be used
770 : void assign_default(size_type count, const mr::MemoryResource& mr = mr::MemoryResource())
771 10 : {
772 10 : _reserve(count, 0, mr);
773 10 : _self->set_range(0, nullptr, count);
774 10 : _self->destruct_until(count);
775 10 : }
776 :
777 : /// @brief If mr is invalid, current resource will be used, else mr will be used
778 : void assign(size_type count, const_reference init,
779 : const mr::MemoryResource& mr = mr::MemoryResource())
780 10 : {
781 10 : _reserve(count, 0, mr);
782 10 : _self->set_range(0, &init, count);
783 10 : _self->destruct_until(count);
784 10 : }
785 :
786 : /// @brief If mr is invalid, current resource will be used, else mr will be used
787 : template <class ItType>
788 : void assign(ItType ibegin, ItType iend, const mr::MemoryResource& mr = mr::MemoryResource())
789 7040 : {
790 7040 : size_type count = std::distance(ibegin, iend);
791 7040 : _reserve(count, 0, mr);
792 7040 : _self->insert_range(0, ibegin, count, is_universal<ItType>);
793 7040 : _self->destruct_until(count);
794 7040 : }
795 :
796 : /// @brief If mr is invalid, current resource will be used, else mr will be used
797 : void assign(std::initializer_list<value_type> l,
798 : const mr::MemoryResource& mr = mr::MemoryResource())
799 15 : {
800 15 : assign(l.begin(), l.end(), mr);
801 15 : }
802 :
803 : /// @brief If mr is invalid, current resource will be used, else mr will be used
804 : void assign(const self_type& s, const mr::MemoryResource& mr = mr::MemoryResource())
805 30 : {
806 30 : _reserve(s._self->size, 0, mr);
807 30 : _self->insert_range(0, s._self->pointer, s._self->size, true);
808 30 : _self->destruct_until(s._self->size);
809 30 : }
810 :
811 : /// @brief If mr is invalid, current resource will be used, else mr will be used
812 : void swap(self_type& o, const mr::MemoryResource& mr = mr::MemoryResource())
813 15 : {
814 15 : mr::MemoryResource actual = bool(mr) ? mr : _self->resource;
815 :
816 15 : if (_self->resource == actual && o._self->resource == actual)
817 5 : std::swap(_self, o._self);
818 10 : else {
819 10 : std::shared_ptr<container_type> _old = _reserve(o._self->size, 0, actual);
820 :
821 10 : _self->insert_range(0, o._self->pointer, o._self->size, true);
822 :
823 10 : o._reserve(_old == nullptr ? 0 : _old->size, 0);
824 10 : if (_old != nullptr)
825 10 : o._self->insert_range(0, _old->pointer, _old->size, true);
826 10 : o._self->destruct_until(_old == nullptr ? 0 : _old->size);
827 10 : }
828 15 : }
829 :
830 : reference at(size_type i)
831 545 : {
832 545 : if (i >= _self->size)
833 133 : throw std::out_of_range("Index into ContiguousVector is out of range");
834 412 : return _self->pointer[i];
835 412 : }
836 :
837 : const_reference at(size_type i) const
838 417 : {
839 417 : if (i >= _self->size)
840 97 : throw std::out_of_range("Index into ContiguousVector is out of range");
841 320 : return _self->pointer[i];
842 320 : }
843 :
844 28776206 : reference operator[](size_type i) { return _self->pointer[i]; }
845 :
846 19802959 : const_reference operator[](size_type i) const { return _self->pointer[i]; }
847 :
848 5 : reference front() { return *_self->pointer; }
849 :
850 5 : const_reference front() const { return *_self->pointer; }
851 :
852 5 : reference back() { return *(_self->end_ptr() - 1); }
853 :
854 5 : const_reference back() const { return *(_self->end_ptr() - 1); }
855 :
856 17229 : pointer data() { return pointer(_self->pointer); }
857 :
858 1810 : const_pointer data() const { return const_pointer(_self->pointer); }
859 :
860 144734 : iterator begin() { return iterator(_self->pointer); }
861 :
862 8170 : const_iterator begin() const { return const_iterator(_self->pointer); }
863 :
864 67949 : const_iterator cbegin() const { return const_iterator(_self->pointer); }
865 :
866 48252 : iterator end() { return iterator(_self->end_ptr()); }
867 :
868 8165 : const_iterator end() const { return const_iterator(_self->end_ptr()); }
869 :
870 35108 : const_iterator cend() const { return const_iterator(_self->end_ptr()); }
871 :
872 15 : reverse_iterator rbegin() { return reverse_iterator(end()); }
873 :
874 15 : const_reverse_iterator rbegin() const { return const_reverse_iterator(cend()); }
875 :
876 15 : const_reverse_iterator crbegin() const { return const_reverse_iterator(cend()); }
877 :
878 10 : reverse_iterator rend() { return reverse_iterator(begin()); }
879 :
880 10 : const_reverse_iterator rend() const { return const_reverse_iterator(cbegin()); }
881 :
882 10 : const_reverse_iterator crend() const { return const_reverse_iterator(cbegin()); }
883 :
884 49112222 : size_type size() const { return _self->size; }
885 :
886 370 : bool empty() const { return size() == 0; }
887 :
888 5 : size_type max_size() const { return std::numeric_limits<difference_type>::max(); }
889 :
890 35 : void reserve(size_type cap) { _reserve(cap, _self->size); }
891 :
892 761 : size_type capacity() const { return _self->capacity; }
893 :
894 131 : void shrink_to_fit() { _reduce(_self->size); }
895 :
896 55 : void clear() noexcept { _self->destruct_until(0); }
897 :
898 : iterator insert_default(const_iterator i, size_type count)
899 5 : {
900 5 : difference_type off = i - const_iterator(_self->pointer);
901 5 : size_type pre = static_cast<size_type>(off);
902 5 : size_type post = _self->size - static_cast<size_type>(off);
903 :
904 5 : std::shared_ptr<container_type> _old = _reserve(_self->size + count, pre);
905 :
906 : /* check if a new buffer has been constructed */
907 5 : if (_old != nullptr) {
908 5 : _self->set_range(pre, nullptr, count);
909 5 : _self->insert_range(_self->size, _old->pointer + pre, post, true);
910 5 : }
911 :
912 : /* check if the current content will overlap itself upon moving */
913 0 : else if (count < post) {
914 0 : _self->move_tail(pre, count);
915 0 : _self->set_range(pre, nullptr, count);
916 0 : }
917 :
918 0 : else {
919 : /* construct the new part in order for the tail to be appended */
920 0 : _self->set_range(_self->size, nullptr, count - post);
921 0 : _self->insert_range(_self->size, _self->pointer + pre, post, true);
922 0 : _self->set_range(pre, nullptr, post);
923 0 : }
924 5 : return iterator(_self->pointer + off);
925 5 : }
926 :
927 5 : iterator insert(const_iterator i, const_reference value) { return emplace(i, value); }
928 :
929 : iterator insert(const_iterator i, value_type&& value)
930 5 : {
931 5 : return emplace(i, std::move(value));
932 5 : }
933 :
934 : iterator insert(const_iterator i, size_type count, const_reference value)
935 5 : {
936 5 : difference_type off = i - const_iterator(_self->pointer);
937 5 : size_type pre = static_cast<size_type>(off);
938 5 : size_type post = _self->size - static_cast<size_type>(off);
939 :
940 5 : std::shared_ptr<container_type> _old = _reserve(_self->size + count, pre);
941 :
942 : /* check if a new buffer has been constructed */
943 5 : if (_old != nullptr) {
944 5 : _self->set_range(pre, &value, count);
945 5 : _self->insert_range(_self->size, _old->pointer + pre, post, true);
946 5 : }
947 :
948 : /* check if the current content will overlap itself upon moving */
949 0 : else if (count < post) {
950 0 : _self->move_tail(pre, count);
951 0 : _self->set_range(pre, &value, count);
952 0 : }
953 :
954 0 : else {
955 : /* construct the new part in order for the tail to be appended */
956 0 : _self->set_range(_self->size, &value, count - post);
957 0 : _self->insert_range(_self->size, _self->pointer + pre, post, true);
958 0 : _self->set_range(pre, &value, post);
959 0 : }
960 5 : return iterator(_self->pointer + off);
961 5 : }
962 :
963 : template <class ItType>
964 : iterator insert(const_iterator i, ItType ibegin, ItType iend)
965 85 : {
966 85 : difference_type off = i - const_iterator(_self->pointer);
967 85 : size_type pre = static_cast<size_type>(off);
968 85 : size_type post = _self->size - static_cast<size_type>(off);
969 85 : size_type count = static_cast<size_type>(std::distance(ibegin, iend));
970 :
971 85 : std::shared_ptr<container_type> _old = _reserve(_self->size + count, pre);
972 :
973 : /* check if a new buffer has been constructed */
974 85 : if (_old != nullptr) {
975 43 : _self->insert_range(pre, ibegin, count, is_universal<ItType>);
976 43 : _self->insert_range(_self->size, _old->pointer + pre, post, true);
977 43 : }
978 :
979 : /* check if the current content will overlap itself upon moving */
980 42 : else if (count < post) {
981 0 : _self->move_tail(pre, count);
982 0 : _self->insert_range(pre, ibegin, count, is_universal<ItType>);
983 0 : }
984 :
985 42 : else {
986 : /* construct the new part in order for the tail to be appended */
987 42 : _self->insert_range(_self->size, std::next(ibegin, post), count - post,
988 42 : is_universal<ItType>);
989 42 : _self->insert_range(_self->size, _self->pointer + pre, post, true);
990 42 : _self->insert_range(pre, ibegin, post, is_universal<ItType>);
991 42 : }
992 85 : return iterator(_self->pointer + off);
993 85 : }
994 :
995 : iterator insert(const_iterator i, std::initializer_list<value_type> l)
996 5 : {
997 5 : return insert(i, l.begin(), l.end());
998 5 : }
999 :
1000 : template <class... Args>
1001 : iterator emplace(const_iterator i, Args&&... args)
1002 15 : {
1003 15 : size_type off = static_cast<size_type>(i - const_iterator(_self->pointer));
1004 15 : std::shared_ptr<container_type> _old = _reserve(_self->size + 1, off);
1005 :
1006 15 : if (_old != nullptr) {
1007 15 : new (_self->end_ptr()) value_type(std::forward<Args>(args)...);
1008 15 : _self->insert_range(++_self->size, _old->pointer + off, _old->size - off, true);
1009 15 : } else if (off >= _self->size) {
1010 0 : new (_self->end_ptr()) value_type(std::forward<Args>(args)...);
1011 0 : ++_self->size;
1012 0 : } else {
1013 0 : _self->move_tail(off, 1);
1014 0 : _self->pointer[off] = std::move(value_type(std::forward<Args>(args)...));
1015 0 : }
1016 15 : return iterator(_self->pointer + off);
1017 15 : }
1018 :
1019 5 : iterator erase(iterator i) { return erase(i, std::next(i)); }
1020 :
1021 5 : iterator erase(const_iterator i) { return erase(i, std::next(i)); }
1022 :
1023 : iterator erase(iterator ibegin, iterator iend)
1024 10 : {
1025 10 : return erase(const_iterator(ibegin), const_iterator(iend));
1026 10 : }
1027 :
1028 : iterator erase(const_iterator ibegin, const_iterator iend)
1029 20 : {
1030 20 : difference_type offset = ibegin - const_iterator(_self->pointer);
1031 20 : difference_type count = iend - ibegin;
1032 20 : _self->move_tail(iend - const_iterator(_self->pointer), -count);
1033 20 : return iterator(_self->pointer + offset);
1034 20 : }
1035 :
1036 : template <class... Args>
1037 : reference emplace_back(Args&&... args)
1038 467 : {
1039 467 : _reserve(_self->size + 1, _self->size);
1040 467 : new (_self->end_ptr()) value_type(std::forward<Args>(args)...);
1041 467 : return _self->pointer[_self->size++];
1042 467 : }
1043 :
1044 359 : void push_back(const_reference value) { emplace_back(value); }
1045 :
1046 103 : void push_back(value_type&& value) { emplace_back(std::move(value)); }
1047 :
1048 15 : void pop_back() { _self->destruct_until(_self->size - 1); }
1049 :
1050 : void resize(size_type count)
1051 335 : {
1052 335 : if (count <= _self->size) {
1053 333 : _self->destruct_until(count);
1054 333 : return;
1055 333 : }
1056 2 : _reserve(count, _self->size);
1057 2 : _self->set_range(_self->size, nullptr, count - _self->size);
1058 2 : }
1059 :
1060 : void resize(size_type count, const_reference value)
1061 5 : {
1062 5 : if (count <= _self->size) {
1063 3 : _self->destruct_until(count);
1064 3 : return;
1065 3 : }
1066 2 : _reserve(count, _self->size);
1067 2 : _self->set_range(_self->size, &value, count - _self->size);
1068 2 : }
1069 : };
1070 : } // namespace elsa::mr
|
