1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
|
#define V_SIMD 256
#define V_SIMD_F64 1
/***************************
* Data Type
***************************/
typedef __m256 v_f32;
typedef __m256d v_f64;
#define v_nlanes_f32 8
#define v_nlanes_f64 4
/***************************
* Arithmetic
***************************/
#define v_add_f32 _mm256_add_ps
#define v_add_f64 _mm256_add_pd
#define v_sub_f32 _mm256_sub_ps
#define v_sub_f64 _mm256_sub_pd
#define v_mul_f32 _mm256_mul_ps
#define v_mul_f64 _mm256_mul_pd
#ifdef HAVE_FMA3
// multiply and add, a*b + c
#define v_muladd_f32 _mm256_fmadd_ps
#define v_muladd_f64 _mm256_fmadd_pd
// multiply and subtract, a*b - c
#define v_mulsub_f32 _mm256_fmsub_ps
#define v_mulsub_f64 _mm256_fmsub_pd
#else
// multiply and add, a*b + c
BLAS_FINLINE v_f32 v_muladd_f32(v_f32 a, v_f32 b, v_f32 c)
{ return v_add_f32(v_mul_f32(a, b), c); }
BLAS_FINLINE v_f64 v_muladd_f64(v_f64 a, v_f64 b, v_f64 c)
{ return v_add_f64(v_mul_f64(a, b), c); }
// multiply and subtract, a*b - c
BLAS_FINLINE v_f32 v_mulsub_f32(v_f32 a, v_f32 b, v_f32 c)
{ return v_sub_f32(v_mul_f32(a, b), c); }
BLAS_FINLINE v_f64 v_mulsub_f64(v_f64 a, v_f64 b, v_f64 c)
{ return v_sub_f64(v_mul_f64(a, b), c); }
#endif // !HAVE_FMA3
// Horizontal add: Calculates the sum of all vector elements.
BLAS_FINLINE float v_sum_f32(__m256 a)
{
__m256 sum_halves = _mm256_hadd_ps(a, a);
sum_halves = _mm256_hadd_ps(sum_halves, sum_halves);
__m128 lo = _mm256_castps256_ps128(sum_halves);
__m128 hi = _mm256_extractf128_ps(sum_halves, 1);
__m128 sum = _mm_add_ps(lo, hi);
return _mm_cvtss_f32(sum);
}
BLAS_FINLINE double v_sum_f64(__m256d a)
{
__m256d sum_halves = _mm256_hadd_pd(a, a);
__m128d lo = _mm256_castpd256_pd128(sum_halves);
__m128d hi = _mm256_extractf128_pd(sum_halves, 1);
__m128d sum = _mm_add_pd(lo, hi);
return _mm_cvtsd_f64(sum);
}
/***************************
* memory
***************************/
// unaligned load
#define v_loadu_f32 _mm256_loadu_ps
#define v_loadu_f64 _mm256_loadu_pd
#define v_storeu_f32 _mm256_storeu_ps
#define v_storeu_f64 _mm256_storeu_pd
#define v_setall_f32(VAL) _mm256_set1_ps(VAL)
#define v_setall_f64(VAL) _mm256_set1_pd(VAL)
#define v_zero_f32 _mm256_setzero_ps
#define v_zero_f64 _mm256_setzero_pd
|
