91 Vc_ALWAYS_INLINE Vc_PURE
operator V()
const {
return value(); }
94 Vc_ALWAYS_INLINE enable_if_mutable<T, MemoryVector &> operator=(
const T &x) {
97 v.store(&m_data[0], Flags());
101 Vc_ALL_BINARY(Vc_MEM_OPERATOR_EQ);
102 Vc_ALL_ARITHMETICS(Vc_MEM_OPERATOR_EQ);
104 Vc_ALWAYS_INLINE EntryType &operator[](
size_t i) {
return m_data[i]; }
105 Vc_ALWAYS_INLINE
const EntryType &operator[](
size_t i)
const {
return m_data[i]; }
108template<
typename _V,
typename Flags>
class MemoryVectorIterator
110 typedef typename std::remove_cv<_V>::type V;
112 template<
typename T,
typename R>
using enable_if_mutable =
113 typename std::enable_if<std::is_same<T, T>::value && !std::is_const<_V>::value, R>::type;
115 using iterator_traits = std::iterator_traits<MemoryVector<_V, Flags> *>;
117 MemoryVector<_V, Flags> *d;
119 typedef typename iterator_traits::difference_type difference_type;
120 typedef typename iterator_traits::value_type value_type;
121 typedef typename iterator_traits::pointer pointer;
122 typedef typename iterator_traits::reference reference;
123 typedef typename iterator_traits::iterator_category iterator_category;
125 constexpr MemoryVectorIterator(MemoryVector<_V, Flags> *dd) : d(dd) {}
126 constexpr MemoryVectorIterator(
const MemoryVectorIterator &) =
default;
127 constexpr MemoryVectorIterator(MemoryVectorIterator &&) =
default;
128 Vc_ALWAYS_INLINE MemoryVectorIterator &operator=(
const MemoryVectorIterator &) =
default;
130 Vc_ALWAYS_INLINE
void *orderBy()
const {
return d; }
132 Vc_ALWAYS_INLINE difference_type operator-(
const MemoryVectorIterator &rhs)
const {
return d - rhs.d; }
133 Vc_ALWAYS_INLINE reference operator[](
size_t i)
const {
return d[i]; }
134 Vc_ALWAYS_INLINE reference operator*()
const {
return *d; }
135 Vc_ALWAYS_INLINE pointer operator->()
const {
return d; }
136 Vc_ALWAYS_INLINE MemoryVectorIterator &operator++() { ++d;
return *
this; }
137 Vc_ALWAYS_INLINE MemoryVectorIterator operator++(
int) { MemoryVectorIterator r(*
this); ++d;
return r; }
138 Vc_ALWAYS_INLINE MemoryVectorIterator &operator--() { --d;
return *
this; }
139 Vc_ALWAYS_INLINE MemoryVectorIterator operator--(
int) { MemoryVectorIterator r(*
this); --d;
return r; }
140 Vc_ALWAYS_INLINE MemoryVectorIterator &operator+=(
size_t n) { d += n;
return *
this; }
141 Vc_ALWAYS_INLINE MemoryVectorIterator &operator-=(
size_t n) { d -= n;
return *
this; }
142 Vc_ALWAYS_INLINE MemoryVectorIterator operator+(
size_t n)
const {
return MemoryVectorIterator(d + n); }
143 Vc_ALWAYS_INLINE MemoryVectorIterator operator-(
size_t n)
const {
return MemoryVectorIterator(d - n); }
146template<
typename V,
typename FlagsL,
typename FlagsR>
147Vc_ALWAYS_INLINE
bool operator==(
const MemoryVectorIterator<V, FlagsL> &l,
const MemoryVectorIterator<V, FlagsR> &r)
149 return l.orderBy() == r.orderBy();
151template<
typename V,
typename FlagsL,
typename FlagsR>
152Vc_ALWAYS_INLINE
bool operator!=(
const MemoryVectorIterator<V, FlagsL> &l,
const MemoryVectorIterator<V, FlagsR> &r)
154 return l.orderBy() != r.orderBy();
156template<
typename V,
typename FlagsL,
typename FlagsR>
157Vc_ALWAYS_INLINE
bool operator>=(
const MemoryVectorIterator<V, FlagsL> &l,
const MemoryVectorIterator<V, FlagsR> &r)
159 return l.orderBy() >= r.orderBy();
161template<
typename V,
typename FlagsL,
typename FlagsR>
162Vc_ALWAYS_INLINE
bool operator<=(
const MemoryVectorIterator<V, FlagsL> &l,
const MemoryVectorIterator<V, FlagsR> &r)
164 return l.orderBy() <= r.orderBy();
166template<
typename V,
typename FlagsL,
typename FlagsR>
167Vc_ALWAYS_INLINE
bool operator> (
const MemoryVectorIterator<V, FlagsL> &l,
const MemoryVectorIterator<V, FlagsR> &r)
169 return l.orderBy() > r.orderBy();
171template<
typename V,
typename FlagsL,
typename FlagsR>
172Vc_ALWAYS_INLINE
bool operator< (
const MemoryVectorIterator<V, FlagsL> &l,
const MemoryVectorIterator<V, FlagsR> &r)
174 return l.orderBy() < r.orderBy();
177#undef Vc_MEM_OPERATOR_EQ
179#define Vc_VPH_OPERATOR(op) \
180 template <typename V1, typename Flags1, typename V2, typename Flags2> \
181 decltype(std::declval<V1>() op std::declval<V2>()) operator op( \
182 const MemoryVector<V1, Flags1> &x, const MemoryVector<V2, Flags2> &y) \
184 return x.value() op y.value(); \
186Vc_ALL_ARITHMETICS(Vc_VPH_OPERATOR);
187Vc_ALL_BINARY (Vc_VPH_OPERATOR);
188Vc_ALL_COMPARES (Vc_VPH_OPERATOR);
189#undef Vc_VPH_OPERATOR
191template<
typename V,
typename Parent,
typename Flags = Prefetch<>>
class MemoryRange
198 MemoryRange(Parent *p,
size_t firstIndex,
size_t lastIndex)
199 : m_parent(p), m_first(firstIndex), m_last(lastIndex)
202 MemoryVectorIterator<V, Flags> begin()
const {
return &m_parent->vector(m_first , Flags()); }
203 MemoryVectorIterator<V, Flags> end()
const {
return &m_parent->vector(m_last + 1, Flags()); }
205template<
typename V,
typename Parent,
int Dimension,
typename RowMemory>
class MemoryDimensionBase;
206template<
typename V,
typename Parent,
typename RowMemory>
class MemoryDimensionBase<V, Parent, 1, RowMemory>
209 Parent *p() {
return static_cast<Parent *
>(
this); }
210 const Parent *p()
const {
return static_cast<const Parent *
>(
this); }
215 typedef typename V::EntryType EntryType;
220 Vc_ALWAYS_INLINE Vc_PURE EntryType *entries() {
return &p()->m_mem[0]; }
222 Vc_ALWAYS_INLINE Vc_PURE
const EntryType *entries()
const {
return &p()->m_mem[0]; }
227 Vc_ALWAYS_INLINE Vc_PURE EntryType &scalar(
size_t i) {
return entries()[i]; }
229 Vc_ALWAYS_INLINE Vc_PURE
const EntryType scalar(
size_t i)
const {
return entries()[i]; }
236 Vc_ALWAYS_INLINE Vc_PURE
operator EntryType*() {
return entries(); }
238 Vc_ALWAYS_INLINE Vc_PURE
operator const EntryType*()
const {
return entries(); }
242 template <
typename T,
243 typename std::enable_if<
244 std::is_same<typename std::remove_const<T>::type, EntryType *>::value ||
245 std::is_same<typename std::remove_const<T>::type,
void *>::value,
247 Vc_ALWAYS_INLINE Vc_PURE
operator T()
251 template <
typename T,
252 typename std::enable_if<std::is_same<T, const EntryType *>::value ||
253 std::is_same<T, const void *>::value,
255 Vc_ALWAYS_INLINE Vc_PURE
operator T()
const
264 template<
typename Flags>
265 Vc_ALWAYS_INLINE MemoryRange<V, Parent, Flags> range(
size_t firstIndex,
size_t lastIndex, Flags) {
266 return MemoryRange<V, Parent, Flags>(p(), firstIndex, lastIndex);
268 Vc_ALWAYS_INLINE MemoryRange<V, Parent> range(
size_t firstIndex,
size_t lastIndex) {
269 return MemoryRange<V, Parent>(p(), firstIndex, lastIndex);
271 template<
typename Flags>
272 Vc_ALWAYS_INLINE MemoryRange<const V, Parent, Flags> range(
size_t firstIndex,
size_t lastIndex, Flags)
const {
273 return MemoryRange<const V, Parent, Flags>(p(), firstIndex, lastIndex);
275 Vc_ALWAYS_INLINE MemoryRange<const V, Parent> range(
size_t firstIndex,
size_t lastIndex)
const {
276 return MemoryRange<const V, Parent>(p(), firstIndex, lastIndex);
282 Vc_ALWAYS_INLINE EntryType &operator[](
size_t i) {
return entries()[i]; }
284 Vc_ALWAYS_INLINE
const EntryType &operator[](
size_t i)
const {
return entries()[i]; }
297 template<
typename IndexT> Vc_ALWAYS_INLINE Vc_PURE V operator[](Vector<IndexT> i)
const
299 return V(entries(), i);
302template<
typename V,
typename Parent,
typename RowMemory>
class MemoryDimensionBase<V, Parent, 2, RowMemory>
305 Parent *p() {
return static_cast<Parent *
>(
this); }
306 const Parent *p()
const {
return static_cast<const Parent *
>(
this); }
311 typedef typename V::EntryType EntryType;
313 static constexpr size_t rowCount() {
return Parent::RowCount; }
318 Vc_ALWAYS_INLINE Vc_PURE EntryType *entries(
size_t x = 0) {
return &p()->m_mem[x][0]; }
320 Vc_ALWAYS_INLINE Vc_PURE
const EntryType *entries(
size_t x = 0)
const {
return &p()->m_mem[x][0]; }
325 Vc_ALWAYS_INLINE Vc_PURE EntryType &scalar(
size_t i,
size_t j) {
return entries(i)[j]; }
327 Vc_ALWAYS_INLINE Vc_PURE
const EntryType scalar(
size_t i,
size_t j)
const {
return entries(i)[j]; }
332 Vc_ALWAYS_INLINE Vc_PURE RowMemory &operator[](
size_t i) {
333 return p()->m_mem[i];
336 Vc_ALWAYS_INLINE Vc_PURE
const RowMemory &operator[](
size_t i)
const {
337 return p()->m_mem[i];
345 Vc_ALWAYS_INLINE Vc_PURE
size_t rowsCount()
const {
return p()->rowsCount(); }
359template<
typename V,
typename Parent,
int Dimension,
typename RowMemory>
class MemoryBase :
public MemoryDimensionBase<V, Parent, Dimension, RowMemory>
361 static_assert((V::size() *
sizeof(
typename V::EntryType)) % V::MemoryAlignment == 0,
362 "Vc::Memory can only be used for data-parallel types storing a number "
363 "of values that's a multiple of the memory alignment.");
366 Parent *p() {
return static_cast<Parent *
>(
this); }
367 const Parent *p()
const {
return static_cast<const Parent *
>(
this); }
369 template <
class Flags>
370 using vector_reference = MayAlias<MemoryVector<V, Flags>> &;
371 template <
class Flags>
372 using const_vector_reference =
const MayAlias<MemoryVector<const V, Flags>> &;
384 Vc_ALWAYS_INLINE Vc_PURE
size_t entriesCount()
const {
return p()->entriesCount(); }
389 Vc_ALWAYS_INLINE Vc_PURE
size_t vectorsCount()
const {
return p()->vectorsCount(); }
391 using MemoryDimensionBase<V, Parent, Dimension, RowMemory>::entries;
392 using MemoryDimensionBase<V, Parent, Dimension, RowMemory>::scalar;
397 template<
typename Flags = AlignedTag>
398 Vc_ALWAYS_INLINE MemoryVectorIterator< V, Flags>
begin(Flags flags = Flags()) {
return &
firstVector(flags); }
400 template<
typename Flags = AlignedTag>
401 Vc_ALWAYS_INLINE MemoryVectorIterator<const V, Flags>
begin(Flags flags = Flags())
const {
return &
firstVector(flags); }
406 template<
typename Flags = AlignedTag>
407 Vc_ALWAYS_INLINE MemoryVectorIterator< V, Flags>
end(Flags flags = Flags()) {
return &
lastVector(flags) + 1; }
409 template<
typename Flags = AlignedTag>
410 Vc_ALWAYS_INLINE MemoryVectorIterator<const V, Flags>
end(Flags flags = Flags())
const {
return &
lastVector(flags) + 1; }
432 template <
typename Flags = AlignedTag>
433 Vc_ALWAYS_INLINE Vc_PURE
434 typename std::enable_if<!std::is_convertible<Flags, int>::value,
435 vector_reference<Flags>>::type
438 return *aliasing_cast<MemoryVector<V, Flags>>(&entries()[i * V::Size]);
446 template <
typename Flags = AlignedTag>
447 Vc_ALWAYS_INLINE Vc_PURE
448 typename std::enable_if<!std::is_convertible<Flags, int>::value,
449 const_vector_reference<Flags>>::type
452 return *aliasing_cast<MemoryVector<const V, Flags>>(&entries()[i * V::Size]);
474 template <
typename Flags = UnalignedTag>
475 Vc_ALWAYS_INLINE Vc_PURE vector_reference<Flags>
vectorAt(
size_t i,
476 Flags flags = Flags())
478 return *aliasing_cast<MemoryVector<V, Flags>>(&entries()[i]);
491 template <
typename Flags = UnalignedTag>
492 Vc_ALWAYS_INLINE Vc_PURE const_vector_reference<Flags>
vectorAt(
493 size_t i, Flags flags = Flags())
const
495 return *aliasing_cast<MemoryVector<const V, Flags>>(&entries()[i]);
525 template <
typename ShiftT,
typename Flags = decltype(Unaligned)>
526 Vc_ALWAYS_INLINE Vc_PURE
typename std::enable_if<
527 std::is_convertible<ShiftT, int>::value,
528 vector_reference<decltype(std::declval<Flags>() |
Unaligned)>>::type
529 vector(
size_t i, ShiftT shift, Flags = Flags())
531 return *aliasing_cast<
533 &entries()[i * V::Size + shift]);
536 template <
typename ShiftT,
typename Flags = decltype(Unaligned)>
537 Vc_ALWAYS_INLINE Vc_PURE
typename std::enable_if<
538 std::is_convertible<ShiftT, int>::value,
539 const_vector_reference<decltype(std::declval<Flags>() |
Unaligned)>>::type
540 vector(
size_t i, ShiftT shift, Flags = Flags())
const
542 return *aliasing_cast<
544 &entries()[i * V::Size + shift]);
552 template <
typename Flags = AlignedTag>
553 Vc_ALWAYS_INLINE Vc_PURE vector_reference<Flags>
firstVector(Flags f = Flags())
558 template <
typename Flags = AlignedTag>
559 Vc_ALWAYS_INLINE Vc_PURE const_vector_reference<Flags>
firstVector(
560 Flags f = Flags())
const
570 template <
typename Flags = AlignedTag>
571 Vc_ALWAYS_INLINE Vc_PURE vector_reference<Flags>
lastVector(Flags f = Flags())
576 template <
typename Flags = AlignedTag>
577 Vc_ALWAYS_INLINE Vc_PURE const_vector_reference<Flags>
lastVector(
578 Flags f = Flags())
const
583 Vc_ALWAYS_INLINE Vc_PURE V gather(
const unsigned char *indexes)
const {
return V(entries(),
typename V::IndexType(indexes,
Vc::Unaligned)); }
584 Vc_ALWAYS_INLINE Vc_PURE V gather(
const unsigned short *indexes)
const {
return V(entries(),
typename V::IndexType(indexes,
Vc::Unaligned)); }
585 Vc_ALWAYS_INLINE Vc_PURE V gather(
const unsigned int *indexes)
const {
return V(entries(),
typename V::IndexType(indexes,
Vc::Unaligned)); }
586 Vc_ALWAYS_INLINE Vc_PURE V gather(
const unsigned long *indexes)
const {
return V(entries(),
typename V::IndexType(indexes,
Vc::Unaligned)); }
604 vector(i) = std::forward<U>(x);
611 template<
typename P2,
typename RM>
617 return static_cast<Parent &
>(*this);
623 template<
typename P2,
typename RM>
629 return static_cast<Parent &
>(*this);
635 template<
typename P2,
typename RM>
641 return static_cast<Parent &
>(*this);
647 template<
typename P2,
typename RM>
653 return static_cast<Parent &
>(*this);
664 return static_cast<Parent &
>(*this);
675 return static_cast<Parent &
>(*this);
686 return static_cast<Parent &
>(*this);
697 return static_cast<Parent &
>(*this);
703 template<
typename P2,
typename RM>
717 template<
typename P2,
typename RM>
731 template<
typename P2,
typename RM>
745 template<
typename P2,
typename RM>
759 template<
typename P2,
typename RM>
773 template<
typename P2,
typename RM>
793inline void copyVectors(MemoryBase<V, ParentL, Dimension, RowMemoryL> &dst,
794 const MemoryBase<V, ParentR, Dimension, RowMemoryR> &src)
796 const size_t vectorsCount = dst.vectorsCount();
798 for (; i < vectorsCount; i += 4) {
799 const V tmp3 = src.vector(i - 3);
800 const V tmp2 = src.vector(i - 2);
801 const V tmp1 = src.vector(i - 1);
802 const V tmp0 = src.vector(i - 0);
803 dst.vector(i - 3) = tmp3;
804 dst.vector(i - 2) = tmp2;
805 dst.vector(i - 1) = tmp1;
806 dst.vector(i - 0) = tmp0;
808 for (i -= 3; i < vectorsCount; ++i) {
809 dst.vector(i) = src.vector(i);