1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
|
#ifndef GEMV_PARAM_H
#define GEMV_PARAM_H
#ifdef movsd
#undef movsd
#endif
#undef movapd
#define movapd movaps
#ifdef ATHLON
#define ALIGNED_ACCESS
#define MOVUPS_A movaps
#define MOVUPS_XL movaps
#define MOVUPS_XS movaps
#define MOVUPS_YL movaps
#define MOVUPS_YS movaps
#define PREFETCH prefetcht0
#define PREFETCHSIZE 64 * 3
#endif
#ifdef PENTIUM4
#define ALIGNED_ACCESS
#define MOVUPS_A movaps
#define MOVUPS_XL movaps
#define MOVUPS_XS movaps
#define MOVUPS_YL movaps
#define MOVUPS_YS movaps
#define PREFETCH prefetcht0
#define PREFETCHSIZE 64 * 2
#endif
#ifdef CORE2
#define ALIGNED_ACCESS
#define MOVUPS_A movaps
#define MOVUPS_XL movaps
#define MOVUPS_XS movaps
#define MOVUPS_YL movaps
#define MOVUPS_YS movaps
#define PREFETCH prefetcht0
#define PREFETCHSIZE 64 * 4
#endif
#ifdef PENRYN
#define ALIGNED_ACCESS
#define MOVUPS_A movaps
#define MOVUPS_XL movaps
#define MOVUPS_XS movaps
#define MOVUPS_YL movaps
#define MOVUPS_YS movaps
#define PREFETCH prefetcht0
#define PREFETCHSIZE 64 * 4
#endif
#ifdef NEHALEM
#define MOVUPS_A movups
#define MOVUPS_XL movups
#define MOVUPS_XS movups
#define MOVUPS_YL movups
#define MOVUPS_YS movups
#define PREFETCH prefetcht0
#define PREFETCHW prefetcht0
#define PREFETCHSIZE 64 * 3
#endif
#ifdef SANDYBRIDGE
#define MOVUPS_A movups
#define MOVUPS_XL movups
#define MOVUPS_XS movups
#define MOVUPS_YL movups
#define MOVUPS_YS movups
#define PREFETCH prefetcht0
#define PREFETCHW prefetcht0
#define PREFETCHSIZE 64 * 3
#endif
#ifdef OPTERON
#define PREFETCH prefetch
#define PREFETCHW prefetchw
#ifndef COMPLEX
#define PREFETCHSIZE 64 * 1
#else
#define PREFETCHSIZE 64 * 1
#endif
#define movsd movlps
#endif
#if defined(BARCELONA) || defined(SHANGHAI) || defined(BOBCAT) || defined(BARCELONA_OPTIMIZATION)
#define ALIGNED_ACCESS
#define MOVUPS_A movaps
#define MOVUPS_XL movaps
#define MOVUPS_XS movaps
#define MOVUPS_YL movaps
#define MOVUPS_YS movaps
#define PREFETCH prefetch
#define PREFETCHW prefetchw
#ifndef COMPLEX
#define PREFETCHSIZE 64 * 2
#else
#define PREFETCHSIZE 64 * 4
#endif
#endif
#ifdef NANO
#define ALIGNED_ACCESS
#define MOVUPS_A movaps
#define MOVUPS_XL movaps
#define MOVUPS_XS movaps
#define MOVUPS_YL movaps
#define MOVUPS_YS movaps
#define PREFETCH prefetcht0
#ifndef COMPLEX
#define PREFETCHSIZE 64 * 1
#else
#define PREFETCHSIZE 64 * 2
#endif
#endif
#ifndef PREOFFSET
#ifdef L1_DATA_LINESIZE
#define PREOFFSET (L1_DATA_LINESIZE >> 1)
#else
#define PREOFFSET 32
#endif
#endif
#ifndef GEMV_UNROLL
#define GEMV_UNROLL 4
#endif
#ifndef ZGEMV_UNROLL
#define ZGEMV_UNROLL 4
#endif
/* #define COPY_FORCE */ /* Always copy X or Y to the buffer */
/* #define NOCOPY_UNALIGNED */ /* Not copy if X or Y is not aligned */
#ifdef MOVUPS_A
#define MOVUPS_A1(OFF, ADDR, REGS) MOVUPS_A OFF(ADDR), REGS
#define MOVUPS_A2(OFF, ADDR, BASE, SCALE, REGS) MOVUPS_A OFF(ADDR, BASE, SCALE), REGS
#else
#define MOVUPS_A1(OFF, ADDR, REGS) movsd OFF(ADDR), REGS; movhps OFF + 8(ADDR), REGS
#define MOVUPS_A2(OFF, ADDR, BASE, SCALE, REGS) movsd OFF(ADDR, BASE, SCALE), REGS; movhps OFF + 8(ADDR, BASE, SCALE), REGS
#endif
#define MOVRPS_A1(OFF, ADDR, REGS) movsd OFF + 8(ADDR), REGS; movhps OFF(ADDR), REGS
#define MOVRPS_A2(OFF, ADDR, BASE, SCALE, REGS) movsd OFF + 8(ADDR, BASE, SCALE), REGS; movhps OFF(ADDR, BASE, SCALE), REGS
#ifdef MOVUPS_XL
#define MOVUPS_XL1(OFF, ADDR, REGS) MOVUPS_XL OFF(ADDR), REGS
#else
#define MOVUPS_XL1(OFF, ADDR, REGS) movsd OFF(ADDR), REGS; movhps OFF + 8(ADDR), REGS
#endif
#ifdef MOVUPS_XS
#define MOVUPS_XS1(OFF, ADDR, REGS) MOVUPS_XS REGS, OFF(ADDR)
#else
#define MOVUPS_XS1(OFF, ADDR, REGS) movsd REGS, OFF(ADDR); movhps REGS, OFF + 8(ADDR)
#endif
#ifdef MOVUPS_YL
#define MOVUPS_YL1(OFF, ADDR, REGS) MOVUPS_YL OFF(ADDR), REGS
#else
#define MOVUPS_YL1(OFF, ADDR, REGS) movsd OFF(ADDR), REGS; movhps OFF + 8(ADDR), REGS
#endif
#ifdef MOVUPS_YS
#define MOVUPS_YS1(OFF, ADDR, REGS) MOVUPS_YS REGS, OFF(ADDR)
#else
#define MOVUPS_YS1(OFF, ADDR, REGS) movsd REGS, OFF(ADDR); movhps REGS, OFF + 8(ADDR)
#endif
#endif
|
