1
// This file is part of Eigen, a lightweight C++ template library
4
// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
6
// Eigen is free software; you can redistribute it and/or
7
// modify it under the terms of the GNU Lesser General Public
8
// License as published by the Free Software Foundation; either
9
// version 3 of the License, or (at your option) any later version.
11
// Alternatively, you can redistribute it and/or
12
// modify it under the terms of the GNU General Public License as
13
// published by the Free Software Foundation; either version 2 of
14
// the License, or (at your option) any later version.
16
// Eigen is distributed in the hope that it will be useful, but WITHOUT ANY
17
// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18
// FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License or the
19
// GNU General Public License for more details.
21
// You should have received a copy of the GNU Lesser General Public
22
// License and a copy of the GNU General Public License along with
23
// Eigen. If not, see <http://www.gnu.org/licenses/>.
25
#ifndef EIGEN_PACKET_MATH_SSE_H
26
#define EIGEN_PACKET_MATH_SSE_H
30
#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
31
#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
34
#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
35
#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS (2*sizeof(void*))
38
typedef __m128 Packet4f;
39
typedef __m128i Packet4i;
40
typedef __m128d Packet2d;
42
template<> struct is_arithmetic<__m128> { enum { value = true }; };
43
template<> struct is_arithmetic<__m128i> { enum { value = true }; };
44
template<> struct is_arithmetic<__m128d> { enum { value = true }; };
46
#define vec4f_swizzle1(v,p,q,r,s) \
47
(_mm_castsi128_ps(_mm_shuffle_epi32( _mm_castps_si128(v), ((s)<<6|(r)<<4|(q)<<2|(p)))))
49
#define vec4i_swizzle1(v,p,q,r,s) \
50
(_mm_shuffle_epi32( v, ((s)<<6|(r)<<4|(q)<<2|(p))))
52
#define vec2d_swizzle1(v,p,q) \
53
(_mm_castsi128_pd(_mm_shuffle_epi32( _mm_castpd_si128(v), ((q*2+1)<<6|(q*2)<<4|(p*2+1)<<2|(p*2)))))
55
#define vec4f_swizzle2(a,b,p,q,r,s) \
56
(_mm_shuffle_ps( (a), (b), ((s)<<6|(r)<<4|(q)<<2|(p))))
58
#define vec4i_swizzle2(a,b,p,q,r,s) \
59
(_mm_castps_si128( (_mm_shuffle_ps( _mm_castsi128_ps(a), _mm_castsi128_ps(b), ((s)<<6|(r)<<4|(q)<<2|(p))))))
61
#define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
62
const Packet4f p4f_##NAME = pset1<Packet4f>(X)
64
#define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
65
const Packet4f p4f_##NAME = _mm_castsi128_ps(pset1<Packet4i>(X))
67
#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
68
const Packet4i p4i_##NAME = pset1<Packet4i>(X)
71
template<> struct packet_traits<float> : default_packet_traits
73
typedef Packet4f type;
80
HasSin = EIGEN_FAST_MATH,
81
HasCos = EIGEN_FAST_MATH,
87
template<> struct packet_traits<double> : default_packet_traits
89
typedef Packet2d type;
98
template<> struct packet_traits<int> : default_packet_traits
100
typedef Packet4i type;
102
// FIXME check the Has*
109
template<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4}; };
110
template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2}; };
111
template<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4}; };
113
template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { return _mm_set1_ps(from); }
114
template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return _mm_set1_pd(from); }
115
template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) { return _mm_set1_epi32(from); }
117
template<> EIGEN_STRONG_INLINE Packet4f plset<float>(const float& a) { return _mm_add_ps(pset1<Packet4f>(a), _mm_set_ps(3,2,1,0)); }
118
template<> EIGEN_STRONG_INLINE Packet2d plset<double>(const double& a) { return _mm_add_pd(pset1<Packet2d>(a),_mm_set_pd(1,0)); }
119
template<> EIGEN_STRONG_INLINE Packet4i plset<int>(const int& a) { return _mm_add_epi32(pset1<Packet4i>(a),_mm_set_epi32(3,2,1,0)); }
121
template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_add_ps(a,b); }
122
template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_add_pd(a,b); }
123
template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_add_epi32(a,b); }
125
template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_sub_ps(a,b); }
126
template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_sub_pd(a,b); }
127
template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_sub_epi32(a,b); }
129
template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a)
131
const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x80000000,0x80000000,0x80000000));
132
return _mm_xor_ps(a,mask);
134
template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a)
136
const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0,0x80000000,0x0,0x80000000));
137
return _mm_xor_pd(a,mask);
139
template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a)
141
return psub(_mm_setr_epi32(0,0,0,0), a);
144
template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_mul_ps(a,b); }
145
template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_mul_pd(a,b); }
146
template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b)
148
#ifdef EIGEN_VECTORIZE_SSE4_1
149
return _mm_mullo_epi32(a,b);
151
// this version is slightly faster than 4 scalar products
152
return vec4i_swizzle1(
155
_mm_mul_epu32(vec4i_swizzle1(a,1,0,3,2),
156
vec4i_swizzle1(b,1,0,3,2)),
162
template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_div_ps(a,b); }
163
template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_div_pd(a,b); }
164
template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& /*a*/, const Packet4i& /*b*/)
165
{ eigen_assert(false && "packet integer division are not supported by SSE");
166
return pset1<Packet4i>(0);
169
// for some weird raisons, it has to be overloaded for packet of integers
170
template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return padd(pmul(a,b), c); }
172
template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_min_ps(a,b); }
173
template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_min_pd(a,b); }
174
template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b)
176
// after some bench, this version *is* faster than a scalar implementation
177
Packet4i mask = _mm_cmplt_epi32(a,b);
178
return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b));
181
template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_max_ps(a,b); }
182
template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_max_pd(a,b); }
183
template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b)
185
// after some bench, this version *is* faster than a scalar implementation
186
Packet4i mask = _mm_cmpgt_epi32(a,b);
187
return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b));
190
template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_and_ps(a,b); }
191
template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_and_pd(a,b); }
192
template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_and_si128(a,b); }
194
template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_or_ps(a,b); }
195
template<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_or_pd(a,b); }
196
template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_or_si128(a,b); }
198
template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_xor_ps(a,b); }
199
template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_xor_pd(a,b); }
200
template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_xor_si128(a,b); }
202
template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_andnot_ps(a,b); }
203
template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_andnot_pd(a,b); }
204
template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_andnot_si128(a,b); }
206
template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ps(from); }
207
template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_pd(from); }
208
template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast<const Packet4i*>(from)); }
210
#if defined(_MSC_VER)
211
template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) {
212
EIGEN_DEBUG_UNALIGNED_LOAD
214
// NOTE Some version of MSVC10 generates bad code when using _mm_loadu_ps
215
// (i.e., it does not generate an unaligned load!!
216
// TODO On most architectures this version should also be faster than a single _mm_loadu_ps
217
// so we could also enable it for MSVC08 but first we have to make this later does not generate crap when doing so...
218
__m128 res = _mm_loadl_pi(_mm_set1_ps(0.0f), (const __m64*)(from));
219
res = _mm_loadh_pi(res, (const __m64*)(from+2));
222
return _mm_loadu_ps(from);
225
template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_pd(from); }
226
template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_si128(reinterpret_cast<const Packet4i*>(from)); }
228
// Fast unaligned loads. Note that here we cannot directly use intrinsics: this would
229
// require pointer casting to incompatible pointer types and leads to invalid code
230
// because of the strict aliasing rule. The "dummy" stuff are required to enforce
231
// a correct instruction dependency.
232
// TODO: do the same for MSVC (ICC is compatible)
233
// NOTE: with the code below, MSVC's compiler crashes!
235
#if defined(__GNUC__) && defined(__i386__)
236
// bug 195: gcc/i386 emits weird x87 fldl/fstpl instructions for _mm_load_sd
237
#define EIGEN_AVOID_CUSTOM_UNALIGNED_LOADS 1
238
#elif defined(__clang__)
239
// bug 201: Segfaults in __mm_loadh_pd with clang 2.8
240
#define EIGEN_AVOID_CUSTOM_UNALIGNED_LOADS 1
242
#define EIGEN_AVOID_CUSTOM_UNALIGNED_LOADS 0
245
template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from)
247
EIGEN_DEBUG_UNALIGNED_LOAD
248
#if EIGEN_AVOID_CUSTOM_UNALIGNED_LOADS
249
return _mm_loadu_ps(from);
252
res = _mm_load_sd((const double*)(from)) ;
253
res = _mm_loadh_pd(res, (const double*)(from+2)) ;
254
return _mm_castpd_ps(res);
257
template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from)
259
EIGEN_DEBUG_UNALIGNED_LOAD
260
#if EIGEN_AVOID_CUSTOM_UNALIGNED_LOADS
261
return _mm_loadu_pd(from);
264
res = _mm_load_sd(from) ;
265
res = _mm_loadh_pd(res,from+1);
269
template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from)
271
EIGEN_DEBUG_UNALIGNED_LOAD
272
#if EIGEN_AVOID_CUSTOM_UNALIGNED_LOADS
273
return _mm_loadu_si128(reinterpret_cast<const Packet4i*>(from));
276
res = _mm_load_sd((const double*)(from)) ;
277
res = _mm_loadh_pd(res, (const double*)(from+2)) ;
278
return _mm_castpd_si128(res);
283
template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from)
285
return vec4f_swizzle1(_mm_castpd_ps(_mm_load_sd((const double*)from)), 0, 0, 1, 1);
287
template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from)
288
{ return pset1<Packet2d>(from[0]); }
289
template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from)
292
tmp = _mm_loadl_epi64(reinterpret_cast<const Packet4i*>(from));
293
return vec4i_swizzle1(tmp, 0, 0, 1, 1);
296
template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_ps(to, from); }
297
template<> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_pd(to, from); }
298
template<> EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<Packet4i*>(to), from); }
300
template<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) {
301
EIGEN_DEBUG_UNALIGNED_STORE
302
_mm_storel_pd((to), from);
303
_mm_storeh_pd((to+1), from);
305
template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, _mm_castps_pd(from)); }
306
template<> EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, _mm_castsi128_pd(from)); }
308
// some compilers might be tempted to perform multiple moves instead of using a vector path.
309
template<> EIGEN_STRONG_INLINE void pstore1<Packet4f>(float* to, const float& a)
311
Packet4f pa = _mm_set_ss(a);
312
pstore(to, vec4f_swizzle1(pa,0,0,0,0));
314
// some compilers might be tempted to perform multiple moves instead of using a vector path.
315
template<> EIGEN_STRONG_INLINE void pstore1<Packet2d>(double* to, const double& a)
317
Packet2d pa = _mm_set_sd(a);
318
pstore(to, vec2d_swizzle1(pa,0,0));
321
template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
322
template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
323
template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
325
#if defined(_MSC_VER) && defined(_WIN64) && !defined(__INTEL_COMPILER)
326
// The temporary variable fixes an internal compilation error in vs <= 2008 and a wrong-result bug in vs 2010
327
// Direct of the struct members fixed bug #62.
328
template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { return a.m128_f32[0]; }
329
template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return a.m128d_f64[0]; }
330
template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; }
331
#elif defined(_MSC_VER) && !defined(__INTEL_COMPILER)
332
// The temporary variable fixes an internal compilation error in vs <= 2008 and a wrong-result bug in vs 2010
333
template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float x = _mm_cvtss_f32(a); return x; }
334
template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { double x = _mm_cvtsd_f64(a); return x; }
335
template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; }
337
template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { return _mm_cvtss_f32(a); }
338
template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return _mm_cvtsd_f64(a); }
339
template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { return _mm_cvtsi128_si32(a); }
342
template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a)
343
{ return _mm_shuffle_ps(a,a,0x1B); }
344
template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a)
345
{ return _mm_shuffle_pd(a,a,0x1); }
346
template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a)
347
{ return _mm_shuffle_epi32(a,0x1B); }
350
template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a)
352
const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF));
353
return _mm_and_ps(a,mask);
355
template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a)
357
const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0xFFFFFFFF,0x7FFFFFFF,0xFFFFFFFF,0x7FFFFFFF));
358
return _mm_and_pd(a,mask);
360
template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a)
362
#ifdef EIGEN_VECTORIZE_SSSE3
363
return _mm_abs_epi32(a);
365
Packet4i aux = _mm_srai_epi32(a,31);
366
return _mm_sub_epi32(_mm_xor_si128(a,aux),aux);
370
EIGEN_STRONG_INLINE void punpackp(Packet4f* vecs)
372
vecs[1] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x55));
373
vecs[2] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xAA));
374
vecs[3] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xFF));
375
vecs[0] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x00));
378
#ifdef EIGEN_VECTORIZE_SSE3
379
// TODO implement SSE2 versions as well as integer versions
380
template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
382
return _mm_hadd_ps(_mm_hadd_ps(vecs[0], vecs[1]),_mm_hadd_ps(vecs[2], vecs[3]));
384
template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
386
return _mm_hadd_pd(vecs[0], vecs[1]);
389
// EIGEN_STRONG_INLINE Packet4i preduxp(const Packet4i* vecs)
391
// return _mm_hadd_epi32(_mm_hadd_epi32(vecs[0], vecs[1]),_mm_hadd_epi32(vecs[2], vecs[3]));
394
template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
396
Packet4f tmp0 = _mm_hadd_ps(a,a);
397
return pfirst(_mm_hadd_ps(tmp0, tmp0));
400
template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) { return pfirst(_mm_hadd_pd(a, a)); }
403
// EIGEN_STRONG_INLINE float predux(const Packet4i& a)
405
// Packet4i tmp0 = _mm_hadd_epi32(a,a);
406
// return pfirst(_mm_hadd_epi32(tmp0, tmp0));
410
template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
412
Packet4f tmp = _mm_add_ps(a, _mm_movehl_ps(a,a));
413
return pfirst(_mm_add_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
415
template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a)
417
return pfirst(_mm_add_sd(a, _mm_unpackhi_pd(a,a)));
420
template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
422
Packet4f tmp0, tmp1, tmp2;
423
tmp0 = _mm_unpacklo_ps(vecs[0], vecs[1]);
424
tmp1 = _mm_unpackhi_ps(vecs[0], vecs[1]);
425
tmp2 = _mm_unpackhi_ps(vecs[2], vecs[3]);
426
tmp0 = _mm_add_ps(tmp0, tmp1);
427
tmp1 = _mm_unpacklo_ps(vecs[2], vecs[3]);
428
tmp1 = _mm_add_ps(tmp1, tmp2);
429
tmp2 = _mm_movehl_ps(tmp1, tmp0);
430
tmp0 = _mm_movelh_ps(tmp0, tmp1);
431
return _mm_add_ps(tmp0, tmp2);
434
template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
436
return _mm_add_pd(_mm_unpacklo_pd(vecs[0], vecs[1]), _mm_unpackhi_pd(vecs[0], vecs[1]));
440
template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a)
442
Packet4i tmp = _mm_add_epi32(a, _mm_unpackhi_epi64(a,a));
443
return pfirst(tmp) + pfirst(_mm_shuffle_epi32(tmp, 1));
446
template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs)
448
Packet4i tmp0, tmp1, tmp2;
449
tmp0 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
450
tmp1 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
451
tmp2 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
452
tmp0 = _mm_add_epi32(tmp0, tmp1);
453
tmp1 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
454
tmp1 = _mm_add_epi32(tmp1, tmp2);
455
tmp2 = _mm_unpacklo_epi64(tmp0, tmp1);
456
tmp0 = _mm_unpackhi_epi64(tmp0, tmp1);
457
return _mm_add_epi32(tmp0, tmp2);
460
// Other reduction functions:
463
template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a)
465
Packet4f tmp = _mm_mul_ps(a, _mm_movehl_ps(a,a));
466
return pfirst(_mm_mul_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
468
template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a)
470
return pfirst(_mm_mul_sd(a, _mm_unpackhi_pd(a,a)));
472
template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a)
474
// after some experiments, it is seems this is the fastest way to implement it
475
// for GCC (eg., reusing pmul is very slow !)
476
// TODO try to call _mm_mul_epu32 directly
477
EIGEN_ALIGN16 int aux[4];
479
return (aux[0] * aux[1]) * (aux[2] * aux[3]);;
483
template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a)
485
Packet4f tmp = _mm_min_ps(a, _mm_movehl_ps(a,a));
486
return pfirst(_mm_min_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
488
template<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a)
490
return pfirst(_mm_min_sd(a, _mm_unpackhi_pd(a,a)));
492
template<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a)
494
// after some experiments, it is seems this is the fastest way to implement it
495
// for GCC (eg., it does not like using std::min after the pstore !!)
496
EIGEN_ALIGN16 int aux[4];
498
register int aux0 = aux[0]<aux[1] ? aux[0] : aux[1];
499
register int aux2 = aux[2]<aux[3] ? aux[2] : aux[3];
500
return aux0<aux2 ? aux0 : aux2;
504
template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a)
506
Packet4f tmp = _mm_max_ps(a, _mm_movehl_ps(a,a));
507
return pfirst(_mm_max_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
509
template<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a)
511
return pfirst(_mm_max_sd(a, _mm_unpackhi_pd(a,a)));
513
template<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a)
515
// after some experiments, it is seems this is the fastest way to implement it
516
// for GCC (eg., it does not like using std::min after the pstore !!)
517
EIGEN_ALIGN16 int aux[4];
519
register int aux0 = aux[0]>aux[1] ? aux[0] : aux[1];
520
register int aux2 = aux[2]>aux[3] ? aux[2] : aux[3];
521
return aux0>aux2 ? aux0 : aux2;
524
#if (defined __GNUC__)
525
// template <> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c)
528
// asm("mulps %[a], %[b] \n\taddps %[c], %[b]" : [b] "+x" (res) : [a] "x" (a), [c] "x" (c));
531
// EIGEN_STRONG_INLINE Packet4i _mm_alignr_epi8(const Packet4i& a, const Packet4i& b, const int i)
534
// asm("palignr %[i], %[a], %[b] " : [b] "+x" (res) : [a] "x" (a), [i] "i" (i));
539
#ifdef EIGEN_VECTORIZE_SSSE3
542
struct palign_impl<Offset,Packet4f>
544
EIGEN_STRONG_INLINE static void run(Packet4f& first, const Packet4f& second)
547
first = _mm_castsi128_ps(_mm_alignr_epi8(_mm_castps_si128(second), _mm_castps_si128(first), Offset*4));
552
struct palign_impl<Offset,Packet4i>
554
EIGEN_STRONG_INLINE static void run(Packet4i& first, const Packet4i& second)
557
first = _mm_alignr_epi8(second,first, Offset*4);
562
struct palign_impl<Offset,Packet2d>
564
EIGEN_STRONG_INLINE static void run(Packet2d& first, const Packet2d& second)
567
first = _mm_castsi128_pd(_mm_alignr_epi8(_mm_castpd_si128(second), _mm_castpd_si128(first), 8));
573
struct palign_impl<Offset,Packet4f>
575
EIGEN_STRONG_INLINE static void run(Packet4f& first, const Packet4f& second)
579
first = _mm_move_ss(first,second);
580
first = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(first),0x39));
584
first = _mm_movehl_ps(first,first);
585
first = _mm_movelh_ps(first,second);
589
first = _mm_move_ss(first,second);
590
first = _mm_shuffle_ps(first,second,0x93);
596
struct palign_impl<Offset,Packet4i>
598
EIGEN_STRONG_INLINE static void run(Packet4i& first, const Packet4i& second)
602
first = _mm_castps_si128(_mm_move_ss(_mm_castsi128_ps(first),_mm_castsi128_ps(second)));
603
first = _mm_shuffle_epi32(first,0x39);
607
first = _mm_castps_si128(_mm_movehl_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(first)));
608
first = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(second)));
612
first = _mm_castps_si128(_mm_move_ss(_mm_castsi128_ps(first),_mm_castsi128_ps(second)));
613
first = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(second),0x93));
619
struct palign_impl<Offset,Packet2d>
621
EIGEN_STRONG_INLINE static void run(Packet2d& first, const Packet2d& second)
625
first = _mm_castps_pd(_mm_movehl_ps(_mm_castpd_ps(first),_mm_castpd_ps(first)));
626
first = _mm_castps_pd(_mm_movelh_ps(_mm_castpd_ps(first),_mm_castpd_ps(second)));
632
} // end namespace internal
634
#endif // EIGEN_PACKET_MATH_SSE_H