1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
|
#ifndef atlas_tlvl3_H
#define atlas_tlvl3_H
#include "atlas_threads.h"
#ifdef TYPE
#include "atlas_lvl3.h"
#endif
#ifndef ATL_XOVER_L3
#ifdef TREAL
#define ATL_XOVER_L3 8 /* number of NBxNB blocks */
#else
#define ATL_XOVER_L3 4
#endif
#endif
#ifndef ATL_TGEMM_XOVER
#define ATL_TGEMM_XOVER ATL_XOVER_L3
#endif
#ifndef ATL_TGEMM_ADDP
#define ATL_TGEMM_ADDP 1
#endif
/*
* Number of blocks per proc for GEMM to divide M only
*/
#ifndef ATL_TMMMINMBLKS
#define ATL_TMMMINMBLKS 4
#endif
#ifndef ATL_TGEMM_THRESH_MF
#define ATL_TGEMM_THRESH_MF ((((2.0*(ATL_TGEMM_XOVER))*MB)*NB)*KB)
#endif
/*
* This is the minimal number of flops each thread requires once THRESH
* is exceeded
*/
#ifndef ATL_TGEMM_PERTHR_MF
#define ATL_TGEMM_PERTHR_MF ((((2.0*ATL_TGEMM_ADDP)*MB)*NB)*KB)
#endif
/*
* For debugging, can define ATL_SERIAL_COMBINE, and then it any required
* workspaces of C will be allocated before beginning parallel operations,
* and all required combined will happen after parallel operations are
* done.
*/
// #define ATL_SERIAL_COMBINE
#ifdef ATL_SERIAL_COMBINE
typedef struct ATL_CombNode ATL_combnode_t;
struct ATL_CombNode
{
ATL_INT M, N, ldw, ldd;
void *W, *D; /* Work and Destination */
ATL_combnode_t *next;
};
#endif
/*
* The array Cinfp holds C partitioning information. This array holds a
* list of pointers to nodes whose data I have not been able to combine
* with my native C partition. The first nCw entries contain the pointers
* to the MMNODE of allocated C workspaces that I have not been able to
* combine. If my node has C in workspace, I am the first entry in this array.
* Sometimes, a child thread has been combined with me that owned a piece of
* the original C. These values do not need to be combined (they were written
* to the original C), but we need to combine the range of "owned" workspaces
* so that we know when it is legal for a parent node to add into the space.
* The final nCp entries of Cinfp entries of Cinfp hold these original pieces
* that need to be combined to create larger owned partitions (starting from
* the end of the array). If the C ptr is NULL, that means that entry has
* been subsumed into a new entry.
*/
typedef struct ATL_TMMNode ATL_TMMNODE_t;
struct ATL_TMMNode
{
ATL_TMMNODE_t *Cinfp[ATL_NTHREADS];
void (*gemmK)(ATL_CINT, ATL_CINT, ATL_CINT, const void*, const void *,
ATL_CINT, const void*, ATL_CINT, const void*, void*, ATL_CINT);
const void *A, *B;
void *C, *Cw;
void *alpha, *beta;
void *zero, *one;
ATL_INT ldcw, M, N, K, lda, ldb, ldc;
int mb, nb, kb;
int eltsz, eltsh; /* element size, and shift (eg. log_2(eltsz)) */
int rank; /* the rank of my thread ([0,P-1]) */
int nCw; /* # of workspace entries in 1st nCw elts of Cinfp array */
int nCp; /* # of orig. C pieces last nCp elts of Cinfp */
int ownC; /* do I own my piece of C, or only wrkspace? */
};
/*
* This data structure used for dynamically scheduled rank-K update
* It is needed only by routines that are typed, and thus define TYPE
*/
#ifdef TYPE
typedef struct
{
void *aNcnt; /* count on col-panels of C */
void *aMcnt; /* count row-panels of A */
void **aMcnts; /* P-len array of counts on row-blks of C */
void **Mlocks; /* mutexes protecting init of aMcnts */
int *Js; /* current C col for each node */
int Sync0; /* 0: no sync at end; else thr 0 waits til all done */
volatile int *chkin; /* ATL_NTHREAD-len checkin array */
TYPE **Bws; /* preallocated thread copy areas */
TYPE *Aw; /* workspace for common A */
const TYPE *A, *B; /* original input matrices */
TYPE *C; /* original output matrix */
#ifdef TREAL
TYPE alpha;
TYPE beta;
#else
const TYPE *alpha;
const TYPE *beta;
#endif
ATL_INT nKb, kr, kr8;
ATL_INT nMb, mr, nNb, nr;
ATL_INT M, N, K, lda, ldb, ldc;
enum ATLAS_TRANS TA, TB;
} ATL_TGEMM_RKK_t;
#endif
/*
* This data structure is used when we split K for SYRK
*/
typedef struct ATL_SyrkK ATL_TSYRK_K_t;
struct ATL_SyrkK
{
ATL_TSYRK_K_t *Cinfp[ATL_NTHREADS];
void (*gemmT)(const enum ATLAS_TRANS, const enum ATLAS_TRANS,
ATL_CINT, ATL_CINT, ATL_CINT, const void *,
const void *, ATL_CINT, const void *, ATL_CINT,
const void *, void *, ATL_CINT);
void (*tvsyrk)(const enum ATLAS_UPLO, const enum ATLAS_TRANS, ATL_CINT,
ATL_CINT, const void*, const void*, ATL_CINT, const void*,
void*, ATL_CINT);
const void *A;
void *C, *Cw;
void *DoComb;
ATL_LAUNCHSTRUCT_t *lp;
const void *alpha, *beta;
const void *zero, *one;
ATL_INT ldcw, N, K, nb, lda, ldc;
int eltsh, rank, nCw;
enum ATLAS_UPLO Uplo;
enum ATLAS_TRANS Trans, TB;
};
/*
* This data structure used when we divide N only, and NTHREAD is a power of 2
*/
typedef struct
{
void (*gemmK)(ATL_CINT, ATL_CINT, ATL_CINT, const void*, const void *,
ATL_CINT, const void*, ATL_CINT, const void*, void*, ATL_CINT);
void (*tvsyrk)(const enum ATLAS_UPLO, const enum ATLAS_TRANS, ATL_CINT,
ATL_CINT, const void*, const void*, ATL_CINT, const void*,
void*, ATL_CINT);
void *T; /* Triangular matrix to do SYRK into*/
void *C; /* rect matrix to do GEMM into */
const void *A0; /* input matrix for syrk, */
const void *A; /* 1st input matrix for GEMM */
const void *B; /* 2nd input matrix for GEMM */
const void *alpha, *beta;
ATL_INT M; /* size of SYRK and 1st dim of GEMM */
ATL_INT N; /* size of 2nd dim of N */
ATL_INT K; /* K of original problem */
ATL_INT lda, ldc;
int nb, eltsh; /* shift to do equivalant of *sizeof */
enum ATLAS_UPLO Uplo;
enum ATLAS_TRANS TA, TB;
} ATL_TSYRK_M_t;
typedef struct
{
const void *A, *alpha;
void *B;
ATL_INT M, N, lda, ldb;
enum ATLAS_SIDE side;
enum ATLAS_UPLO uplo;
enum ATLAS_TRANS TA;
enum ATLAS_DIAG diag;
} ATL_TTRSM_t;
typedef struct
{
const void *A, *B, *alpha, *beta;
void *C;
ATL_INT M, N, lda, ldb, ldc, nb;
enum ATLAS_SIDE side;
enum ATLAS_UPLO uplo;
} ATL_TSYMM_t;
typedef struct
{
const void *alpha, *alpha2, *beta, *one, *zero;
void (*tvgemm)(const enum ATLAS_TRANS, const enum ATLAS_TRANS,
ATL_CINT, ATL_CINT, ATL_CINT, const void *,
const void *, ATL_CINT, const void *, ATL_CINT,
const void *, void *, ATL_CINT);
void (*tvApAt)(const enum ATLAS_UPLO, ATL_CINT, const void *, ATL_CINT,
const void *, void *, ATL_CINT);
ATL_INT K, lda, ldb, ldc;
int nb, eltsh;
enum ATLAS_UPLO Uplo;
enum ATLAS_TRANS trans, TA, TB, /* trans for syr2k, TA,TB are for GEMM */
TA2, TB2; /* transpose of TA,TB */
} ATL_SYR2K_t;
/*
* =============================================================================
* Function prototypes
* =============================================================================
*/
void ATL_EnforceNonPwr2LO(ATL_TMMNODE_t *ptmms, const int P);
int Mjoin(PATL,threadMM)(const enum ATLAS_TRANS TA, const enum ATLAS_TRANS TB,
size_t M, size_t N, size_t K);
void ATL_tvsyr2k_rec(ATL_SYR2K_t *syp, ATL_CINT Nblks, ATL_CINT nr,
const void *A, const void *B, void *C);
#ifdef TYPE
void Mjoin(PATL,tsyrk)
(const enum ATLAS_UPLO Uplo, const enum ATLAS_TRANS Trans, ATL_CINT N,
ATL_CINT K, const SCALAR alpha, const TYPE *A, ATL_CINT lda,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
#ifdef TCPLX
void Mjoin(PATL,therk)
(const enum ATLAS_UPLO Uplo, const enum ATLAS_TRANS Trans, ATL_CINT N,
ATL_CINT K, const TYPE alpha, const TYPE *A, ATL_CINT lda,
const TYPE beta, TYPE *C, ATL_CINT ldc);
#endif
void Mjoin(PATL,tsymm)
(const enum ATLAS_SIDE Side, const enum ATLAS_UPLO Uplo,
ATL_CINT M, ATL_CINT N, const SCALAR alpha, const TYPE *A, ATL_CINT lda,
const TYPE *B, ATL_CINT ldb, const SCALAR beta, TYPE *C, ATL_CINT ldc);
#ifdef TCPLX
void Mjoin(PATL,themm)
(const enum ATLAS_SIDE Side, const enum ATLAS_UPLO Uplo,
ATL_CINT M, ATL_CINT N, const SCALAR alpha, const TYPE *A, ATL_CINT lda,
const TYPE *B, ATL_CINT ldb, const SCALAR beta, TYPE *C, ATL_CINT ldc);
#endif
void Mjoin(PATL,tsyr2k)
(const enum ATLAS_UPLO Uplo, const enum ATLAS_TRANS Trans,
ATL_CINT N, ATL_CINT K, const SCALAR alpha, const TYPE *A, ATL_CINT lda,
const TYPE *B, ATL_CINT ldb, const SCALAR beta, TYPE *C, ATL_CINT ldc);
void Mjoin(PATL,tsyr2k)
(const enum ATLAS_UPLO Uplo, const enum ATLAS_TRANS Trans,
ATL_CINT N, ATL_CINT K, const SCALAR alpha, const TYPE *A, ATL_CINT lda,
const TYPE *B, ATL_CINT ldb, const SCALAR beta, TYPE *C, ATL_CINT ldc);
#ifdef TCPLX
void Mjoin(PATL,ther2k)
(const enum ATLAS_UPLO Uplo, const enum ATLAS_TRANS Trans,
ATL_CINT N, ATL_CINT K, const SCALAR alpha, const TYPE *A, ATL_CINT lda,
const TYPE *B, ATL_CINT ldb, const TYPE beta, TYPE *C, ATL_CINT ldc);
#endif
void Mjoin(PATL,ttrsm)(const enum ATLAS_SIDE side, const enum ATLAS_UPLO uplo,
const enum ATLAS_TRANS TA, const enum ATLAS_DIAG diag,
ATL_CINT M, ATL_CINT N, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, TYPE *B, ATL_CINT ldb);
void Mjoin(PATL,ttrmm)(const enum ATLAS_SIDE side, const enum ATLAS_UPLO uplo,
const enum ATLAS_TRANS TA, const enum ATLAS_DIAG diag,
ATL_CINT M, ATL_CINT N, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, TYPE *B, ATL_CINT ldb);
void Mjoin(PATL,tgemm)(const enum ATLAS_TRANS TA, const enum ATLAS_TRANS TB,
ATL_CINT M, ATL_CINT N, ATL_CINT K, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, const TYPE *B, ATL_CINT ldb,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
void Mjoin(PATL,tvgemm)(const enum ATLAS_TRANS TA, const enum ATLAS_TRANS TB,
ATL_CINT M, ATL_CINT N, ATL_CINT K, const void *alpha,
const void *A, ATL_CINT lda, const void *B,ATL_CINT ldb,
const void *beta, void *C, ATL_CINT ldc);
int Mjoin(PATL,tgemm_bigMN_Kp)
(const enum ATLAS_TRANS TA, const enum ATLAS_TRANS TB,
ATL_CINT M, ATL_CINT N, ATL_CINT K, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, const TYPE *B, ATL_CINT ldb,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
int Mjoin(PATL,tgemm_rkK)
(const enum ATLAS_TRANS TA, const enum ATLAS_TRANS TB,
ATL_CINT M, ATL_CINT N, ATL_CINT K, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, const TYPE *B, ATL_CINT ldb,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
int Mjoin(PATL,tgemm_rec)
(const enum ATLAS_TRANS TA, const enum ATLAS_TRANS TB,
ATL_CINT M, ATL_CINT N, ATL_CINT K, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, const TYPE *B, ATL_CINT ldb,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
#ifdef TCPLX
void Mjoin(PATL,tgemmCC)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, const TYPE *B, ATL_CINT ldb,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
void Mjoin(PATL,tsvgemmCC)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const void* alpha,
const void *A, ATL_CINT lda, const void *B, ATL_CINT ldb,
const void *beta, void *C, ATL_CINT ldc);
#endif /* end ifdef TCPLX */
#ifdef TCPLX
void Mjoin(PATL,tgemmCN)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, const TYPE *B, ATL_CINT ldb,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
void Mjoin(PATL,tsvgemmCN)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const void* alpha,
const void *A, ATL_CINT lda, const void *B, ATL_CINT ldb,
const void *beta, void *C, ATL_CINT ldc);
#endif /* end ifdef TCPLX */
#ifdef TCPLX
void Mjoin(PATL,tgemmCT)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, const TYPE *B, ATL_CINT ldb,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
void Mjoin(PATL,tsvgemmCT)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const void* alpha,
const void *A, ATL_CINT lda, const void *B, ATL_CINT ldb,
const void *beta, void *C, ATL_CINT ldc);
#endif /* end ifdef TCPLX */
#ifdef TCPLX
void Mjoin(PATL,tgemmNC)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, const TYPE *B, ATL_CINT ldb,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
void Mjoin(PATL,tsvgemmNC)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const void* alpha,
const void *A, ATL_CINT lda, const void *B, ATL_CINT ldb,
const void *beta, void *C, ATL_CINT ldc);
#endif /* end ifdef TCPLX */
void Mjoin(PATL,tgemmNN)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, const TYPE *B, ATL_CINT ldb,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
void Mjoin(PATL,tsvgemmNN)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const void* alpha,
const void *A, ATL_CINT lda, const void *B, ATL_CINT ldb,
const void *beta, void *C, ATL_CINT ldc);
void Mjoin(PATL,tgemmNT)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, const TYPE *B, ATL_CINT ldb,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
void Mjoin(PATL,tsvgemmNT)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const void* alpha,
const void *A, ATL_CINT lda, const void *B, ATL_CINT ldb,
const void *beta, void *C, ATL_CINT ldc);
#ifdef TCPLX
void Mjoin(PATL,tgemmTC)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, const TYPE *B, ATL_CINT ldb,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
void Mjoin(PATL,tsvgemmTC)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const void* alpha,
const void *A, ATL_CINT lda, const void *B, ATL_CINT ldb,
const void *beta, void *C, ATL_CINT ldc);
#endif /* end ifdef TCPLX */
void Mjoin(PATL,tgemmTN)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, const TYPE *B, ATL_CINT ldb,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
void Mjoin(PATL,tsvgemmTN)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const void* alpha,
const void *A, ATL_CINT lda, const void *B, ATL_CINT ldb,
const void *beta, void *C, ATL_CINT ldc);
void Mjoin(PATL,tgemmTT)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const SCALAR alpha,
const TYPE *A, ATL_CINT lda, const TYPE *B, ATL_CINT ldb,
const SCALAR beta, TYPE *C, ATL_CINT ldc);
void Mjoin(PATL,tsvgemmTT)
(ATL_CINT M, ATL_CINT N, ATL_CINT K, const void* alpha,
const void *A, ATL_CINT lda, const void *B, ATL_CINT ldb,
const void *beta, void *C, ATL_CINT ldc);
#endif /* end ifdef TYPE */
#endif
|
