1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
|
// RUN: mlir-opt %s -test-linalg-elementwise-fusion-patterns=fuse-with-reshape-by-collapsing -split-input-file | FileCheck %s
// RUN: mlir-opt %s -test-linalg-elementwise-fusion-patterns=fuse-with-reshape-by-collapsing-control -split-input-file | FileCheck %s --check-prefix=CONTROL
// Static problem sizes. Checks all aspects of fusion by collapsing. Rest of the
// tests only check a subset of conditions.
#map0 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d0, d1, d2, d3, d4, d5, d6, d7)>
#map1 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d0, d1, d2)>
#map2 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d3, d4, d5, d6)>
#map3 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d0, d1, d2, d3, d4, d5, d6, d7)>
func.func @fuse_by_collapsing(%arg0 : tensor<2x12x5x336x9xi32>,
%arg1 : tensor<2x3x4xi32>, %arg2 : tensor<5x6x7x8xi32>) -> tensor<2x3x4x5x6x7x8x9xi32> {
%expand = tensor.expand_shape %arg0 [[0], [1, 2], [3], [4, 5, 6], [7]]
: tensor<2x12x5x336x9xi32> into tensor<2x3x4x5x6x7x8x9xi32>
%init = tensor.empty() : tensor<2x3x4x5x6x7x8x9xi32>
%generic = linalg.generic {
indexing_maps = [#map0, #map1, #map2, #map3],
iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel", "parallel", "parallel", "parallel"]}
ins(%expand, %arg1, %arg2 : tensor<2x3x4x5x6x7x8x9xi32>, tensor<2x3x4xi32>, tensor<5x6x7x8xi32>)
outs(%init : tensor<2x3x4x5x6x7x8x9xi32>) {
^bb0(%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32):
%t0 = arith.addi %b0, %b1 : i32
%t1 = arith.addi %t0, %b2 : i32
linalg.yield %t1 : i32
} -> tensor<2x3x4x5x6x7x8x9xi32>
return %generic : tensor<2x3x4x5x6x7x8x9xi32>
}
// CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>
// CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1)>
// CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d2, d3)>
// CHECK: func @fuse_by_collapsing(
// CHECK-SAME: %[[ARG0:.+]]: tensor<2x12x5x336x9xi32>
// CHECK-SAME: %[[ARG1:.+]]: tensor<2x3x4xi32>
// CHECK-SAME: %[[ARG2:.+]]: tensor<5x6x7x8xi32>
// CHECK-DAG: %[[INIT:.+]] = tensor.empty()
// CHECK-DAG: %[[ARG1_RESHAPE:.+]] = tensor.collapse_shape %[[ARG1]] {{\[}}[0], [1, 2]{{\]}}
// CHECK-DAG: %[[ARG2_RESHAPE:.+]] = tensor.collapse_shape %[[ARG2]] {{\[}}[0], [1, 2, 3]{{\]}}
// CHECK-DAG: %[[INIT_RESHAPE:.+]] = tensor.collapse_shape %[[INIT]] {{\[}}[0], [1, 2], [3], [4, 5, 6], [7]{{\]}}
// CHECK: %[[COLLAPSED_OP:.+]] = linalg.generic
// CHECK-SAME: indexing_maps = [#[[MAP0]], #[[MAP1]], #[[MAP2]], #[[MAP0]]]
// CHECK-SAME: iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel"]
// CHECK-SAME: ins(%[[ARG0]], %[[ARG1_RESHAPE]], %[[ARG2_RESHAPE]] :
// CHECK-SAME: outs(%[[INIT_RESHAPE]] :
// CHECK: %[[RESULT_RESHAPE:.+]] = tensor.expand_shape %[[COLLAPSED_OP]] {{\[}}[0], [1, 2], [3], [4, 5, 6], [7]{{\]}}
// CHECK: return %[[RESULT_RESHAPE]]
// CONTROL: func @fuse_by_collapsing(
// CONTROL-SAME: %[[ARG0:.+]]: tensor<2x12x5x336x9xi32>
// CONTROL-SAME: %[[ARG1:.+]]: tensor<2x3x4xi32>
// CONTROL-SAME: %[[ARG2:.+]]: tensor<5x6x7x8xi32>
// CONTROL: %[[EXPAND:.+]] = tensor.expand_shape %[[ARG0]]
// CONTROL: %[[GENERIC:.+]] = linalg.generic
// CONTROL-SAME: ins(%[[EXPAND]],
// CONTROL: return %[[GENERIC]]
// -----
#map0 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d0, d1, d2, d3, d4, d5, d6, d7)>
#map1 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d0, d1, d2)>
#map2 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d3, d4, d5, d6)>
#map3 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d0, d1, d2, d3, d4, d5, d6, d7)>
func.func @fuse_by_collapsing_indexing_op(%arg0 : tensor<2x12x5x336x9xi32>,
%arg1 : tensor<2x3x4xi32>, %arg2 : tensor<5x6x7x8xi32>) -> tensor<2x3x4x5x6x7x8x9xi32> {
%expand = tensor.expand_shape %arg0 [[0], [1, 2], [3], [4, 5, 6], [7]]
: tensor<2x12x5x336x9xi32> into tensor<2x3x4x5x6x7x8x9xi32>
%init = tensor.empty() : tensor<2x3x4x5x6x7x8x9xi32>
%generic = linalg.generic {
indexing_maps = [#map0, #map1, #map2, #map3],
iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel", "parallel", "parallel", "parallel"]}
ins(%expand, %arg1, %arg2 : tensor<2x3x4x5x6x7x8x9xi32>, tensor<2x3x4xi32>, tensor<5x6x7x8xi32>)
outs(%init : tensor<2x3x4x5x6x7x8x9xi32>) {
^bb0(%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32):
%iv0 = linalg.index 0: index
%iv1 = linalg.index 1: index
%t0 = arith.addi %iv0, %iv1 : index
%iv2 = linalg.index 2 : index
%t1 = arith.addi %t0, %iv2 : index
%iv3 = linalg.index 3 : index
%t2 = arith.addi %t1, %iv3 : index
%iv4 = linalg.index 4 : index
%t3 = arith.addi %t2, %iv4 : index
%iv5 = linalg.index 5 : index
%t4 = arith.addi %t3, %iv5 : index
%iv6 = linalg.index 6 : index
%t5 = arith.addi %t4, %iv6 : index
%iv7 = linalg.index 7 : index
%t6 = arith.addi %t5, %iv7 : index
%yield = arith.index_cast %t6 : index to i32
linalg.yield %yield : i32
} -> tensor<2x3x4x5x6x7x8x9xi32>
return %generic : tensor<2x3x4x5x6x7x8x9xi32>
}
// CHECK-LABEL: func @fuse_by_collapsing_indexing_op(
// CHECK-DAG: %[[C4:.+]] = arith.constant 4 : index
// CHECK-DAG: %[[C8:.+]] = arith.constant 8 : index
// CHECK-DAG: %[[C7:.+]] = arith.constant 7 : index
// CHECK: %[[IV0:.+]] = linalg.index 0
// CHECK: %[[IV1:.+]] = linalg.index 1
// CHECK: %[[REM_IV1:.+]] = arith.remui %[[IV1]], %[[C4]]
// CHECK: %[[DIV_IV1:.+]] = arith.divui %[[IV1]], %[[C4]]
// CHECK: %[[IV2:.+]] = linalg.index 2
// CHECK: %[[IV3:.+]] = linalg.index 3
// CHECK: %[[REM1_IV3:.+]] = arith.remui %[[IV3]], %[[C8]]
// CHECK: %[[DIV1_IV3:.+]] = arith.divui %[[IV3]], %[[C8]]
// CHECK: %[[REM2_IV3:.+]] = arith.remui %[[DIV1_IV3]], %[[C7]]
// CHECK: %[[DIV2_IV3:.+]] = arith.divui %[[DIV1_IV3]], %[[C7]]
// CHECK: %[[IV4:.+]] = linalg.index 4
// CHECK: %[[T0:.+]] = arith.addi %[[IV0]], %[[DIV_IV1]]
// CHECK: %[[T1:.+]] = arith.addi %[[T0]], %[[REM_IV1]]
// CHECK: %[[T2:.+]] = arith.addi %[[T1]], %[[IV2]]
// CHECK: %[[T3:.+]] = arith.addi %[[T2]], %[[DIV2_IV3]]
// CHECK: %[[T4:.+]] = arith.addi %[[T3]], %[[REM2_IV3]]
// CHECK: %[[T5:.+]] = arith.addi %[[T4]], %[[REM1_IV3]]
// CHECK: %[[T6:.+]] = arith.addi %[[T5]], %[[IV4]]
// CHECK: %[[YIELD:.+]] = arith.index_cast %[[T6]]
// CHECK: linalg.yield %[[YIELD]]
// -----
#map0 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d7, d5, d6, d0, d1, d2, d3, d4)>
#map1 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d5, d6, d0)>
#map2 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d4, d1, d2, d3)>
#map3 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d0, d1, d2, d3, d4, d5, d6, d7)>
func.func @fuse_by_collapsing_change_reshape_order(%arg0 : tensor<9x56x2x60x6xi32>,
%arg1 : tensor<7x8x2xi32>, %arg2 : tensor<6x3x4x5xi32>) -> tensor<2x3x4x5x6x7x8x9xi32> {
%expand = tensor.expand_shape %arg0 [[0], [1, 2], [3], [4, 5, 6], [7]]
: tensor<9x56x2x60x6xi32> into tensor<9x7x8x2x3x4x5x6xi32>
%init = tensor.empty() : tensor<2x3x4x5x6x7x8x9xi32>
%generic = linalg.generic {
indexing_maps = [#map0, #map1, #map2, #map3],
iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel", "parallel", "parallel", "parallel"]}
ins(%expand, %arg1, %arg2 : tensor<9x7x8x2x3x4x5x6xi32>, tensor<7x8x2xi32>, tensor<6x3x4x5xi32>)
outs(%init : tensor<2x3x4x5x6x7x8x9xi32>) {
^bb0(%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32):
%t0 = arith.addi %b0, %b1 : i32
%t1 = arith.addi %t0, %b2 : i32
linalg.yield %t1 : i32
} -> tensor<2x3x4x5x6x7x8x9xi32>
return %generic : tensor<2x3x4x5x6x7x8x9xi32>
}
// CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d4, d3, d0, d1, d2)>
// CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d3, d0)>
// CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d2, d1)>
// CHECK-DAG: #[[MAP3:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>
// CHECK: func @fuse_by_collapsing_change_reshape_order(
// CHECK-SAME: %[[ARG0:.+]]: tensor<9x56x2x60x6xi32>
// CHECK-SAME: %[[ARG1:.+]]: tensor<7x8x2xi32>
// CHECK-SAME: %[[ARG2:.+]]: tensor<6x3x4x5xi32>
// CHECK-DAG: %[[INIT:.+]] = tensor.empty()
// CHECK-DAG: %[[ARG1_RESHAPE:.+]] = tensor.collapse_shape %[[ARG1]] {{\[}}[0, 1], [2]{{\]}}
// CHECK-DAG: %[[ARG2_RESHAPE:.+]] = tensor.collapse_shape %[[ARG2]] {{\[}}[0], [1, 2, 3]{{\]}}
// CHECK-DAG: %[[INIT_RESHAPE:.+]] = tensor.collapse_shape %[[INIT]] {{\[}}[0], [1, 2, 3], [4], [5, 6], [7]{{\]}}
// CHECK: %[[COLLAPSED_OP:.+]] = linalg.generic
// CHECK-SAME: indexing_maps = [#[[MAP0]], #[[MAP1]], #[[MAP2]], #[[MAP3]]]
// CHECK-SAME: iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel"]
// CHECK-SAME: ins(%[[ARG0]], %[[ARG1_RESHAPE]], %[[ARG2_RESHAPE]] :
// CHECK-SAME: outs(%[[INIT_RESHAPE]] :
// CHECK: %[[RESULT_RESHAPE:.+]] = tensor.expand_shape %[[COLLAPSED_OP]] {{\[}}[0], [1, 2, 3], [4], [5, 6], [7]{{\]}}
// CHECK: return %[[RESULT_RESHAPE]]
// -----
// Dynamic case. Only checks things not covered by `fuse_by_collapsing` test above.
#map0 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d7, d5, d6, d0, d1, d2, d3, d4)>
#map1 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d5, d6, d0)>
#map2 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d4, d1, d2, d3)>
#map3 = affine_map<(d0, d1, d2, d3, d4, d5, d6, d7) -> (d0, d1, d2, d3, d4, d5, d6, d7)>
func.func @fuse_by_collapsing_dynamic(%arg0 : tensor<?x?x?x?x?xi32>,
%arg1 : tensor<?x?x?xi32>, %arg2 : tensor<?x?x?x?xi32>) -> tensor<?x3x?x5x?x7x?x?xi32> {
%c0 = arith.constant 0 : index
%c1 = arith.constant 1 : index
%c2 = arith.constant 2 : index
%expand = tensor.expand_shape %arg0 [[0], [1, 2], [3], [4, 5, 6], [7]]
: tensor<?x?x?x?x?xi32> into tensor<?x7x?x?x3x?x5x?xi32>
%d0 = tensor.dim %arg1, %c2 : tensor<?x?x?xi32>
%d2 = tensor.dim %arg2, %c2 : tensor<?x?x?x?xi32>
%d4 = tensor.dim %arg2, %c0 : tensor<?x?x?x?xi32>
%d6 = tensor.dim %arg1, %c1 : tensor<?x?x?xi32>
%d7 = tensor.dim %arg0, %c0 : tensor<?x?x?x?x?xi32>
%init = tensor.empty(%d0, %d2, %d4, %d6, %d7) : tensor<?x3x?x5x?x7x?x?xi32>
%generic = linalg.generic {
indexing_maps = [#map0, #map1, #map2, #map3],
iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel", "parallel", "parallel", "parallel"]}
ins(%expand, %arg1, %arg2 : tensor<?x7x?x?x3x?x5x?xi32>, tensor<?x?x?xi32>, tensor<?x?x?x?xi32>)
outs(%init : tensor<?x3x?x5x?x7x?x?xi32>) {
^bb0(%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32):
%iv0 = linalg.index 0: index
%iv1 = linalg.index 1: index
%t0 = arith.addi %iv0, %iv1 : index
%iv2 = linalg.index 2 : index
%t1 = arith.addi %t0, %iv2 : index
%iv3 = linalg.index 3 : index
%t2 = arith.addi %t1, %iv3 : index
%iv4 = linalg.index 4 : index
%t3 = arith.addi %t2, %iv4 : index
%iv5 = linalg.index 5 : index
%t4 = arith.addi %t3, %iv5 : index
%iv6 = linalg.index 6 : index
%t5 = arith.addi %t4, %iv6 : index
%iv7 = linalg.index 7 : index
%t6 = arith.addi %t5, %iv7 : index
%yield = arith.index_cast %t6 : index to i32
linalg.yield %yield : i32
} -> tensor<?x3x?x5x?x7x?x?xi32>
return %generic : tensor<?x3x?x5x?x7x?x?xi32>
}
// CHECK: func @fuse_by_collapsing_dynamic(
// CHECK-SAME: %[[ARG0:.+]]: tensor<?x?x?x?x?xi32>
// CHECK-DAG: %[[C2:.+]] = arith.constant 2 : index
// CHECK-DAG: %[[C5:.+]] = arith.constant 5 : index
// CHECK: %[[EXPAND:.+]] = tensor.expand_shape %[[ARG0]]
// CHECK-DAG: %[[D0:.+]] = tensor.dim %[[EXPAND]], %[[C2]]
// CHECK-DAG: %[[D1:.+]] = tensor.dim %[[EXPAND]], %[[C5]]
// CHECK: linalg.generic
// CHECK: %[[IV0:.+]] = linalg.index 1
// CHECK: %[[REM1_IV0:.+]] = arith.remui %[[IV0]], %[[C5]]
// CHECK: %[[DIV1_IV0:.+]] = arith.divui %[[IV0]], %[[C5]]
// CHECK: %[[REM2_IV0:.+]] = arith.remui %[[DIV1_IV0]], %[[D1]]
// CHECK: %[[DIV2_IV0:.+]] = arith.divui %[[DIV1_IV0]], %[[D1]]
// CHECK: %[[IV1:.+]] = linalg.index 3
// CHECK: %[[REM1_IV1:.+]] = arith.remui %[[IV1]], %[[D0]]
// CHECK: %[[DIV1_IV1:.+]] = arith.divui %[[IV1]], %[[D0]]
// -----
#map0 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d3)>
func.func @fuse_reductions(%arg0 : tensor<2x?x5xf32>, %arg1 : tensor<2x5xf32>) -> tensor<2x5xf32> {
%0 = tensor.expand_shape %arg0 [[0], [1, 2], [3]] : tensor<2x?x5xf32> into tensor<2x6x?x5xf32>
%1 = linalg.generic {
indexing_maps = [#map0, #map1],
iterator_types = ["parallel", "reduction", "reduction", "parallel"]}
ins(%0 : tensor<2x6x?x5xf32>) outs(%arg1 : tensor<2x5xf32>) {
^bb0(%b0 : f32, %b1 : f32):
%2 = arith.addf %b0, %b1 : f32
linalg.yield %2 : f32
} -> tensor<2x5xf32>
return %1 : tensor<2x5xf32>
}
// CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
// CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)>
// CHECK: func @fuse_reductions(
// CHECK-SAME: %[[ARG0:.+]]: tensor<2x?x5xf32>
// CHECK-SAME: %[[ARG1:.+]]: tensor<2x5xf32>) -> tensor<2x5xf32>
// CHECK: %[[GENERIC:.+]] = linalg.generic
// CHECK-SAME: indexing_maps = [#[[MAP0]], #[[MAP1]]]
// CHECK-SAME: iterator_types = ["parallel", "reduction", "parallel"]
// CHECK-SAME: ins(%[[ARG0]] : tensor<2x?x5xf32>)
// CHECK-SAME: outs(%[[ARG1]] : tensor<2x5xf32>)
// -----
// Test no fusion because the folded dimensions are not all preserved.
#map0 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d1)>
func.func @no_fuse_unpreserved_folding(%arg0 : tensor<2x12x5xf32>, %arg1 : tensor<2x3xf32>) -> tensor<2x3x4x5xf32> {
%0 = tensor.expand_shape %arg0 [[0], [1, 2], [3]] : tensor<2x12x5xf32> into tensor<2x3x4x5xf32>
%init = tensor.empty(): tensor<2x3x4x5xf32>
%1 = linalg.generic {
indexing_maps = [#map0, #map1, #map0],
iterator_types = ["parallel", "parallel", "parallel", "parallel"]}
ins(%0, %arg1 : tensor<2x3x4x5xf32>, tensor<2x3xf32>) outs(%init : tensor<2x3x4x5xf32>) {
^bb0(%b0 : f32, %b1 : f32, %b2 : f32):
%2 = arith.addf %b0, %b1 : f32
linalg.yield %2 : f32
} -> tensor<2x3x4x5xf32>
return %1 : tensor<2x3x4x5xf32>
}
// CHECK: func @no_fuse_unpreserved_folding
// CHECK-SAME: %[[ARG0:.+]]: tensor<2x12x5xf32>
// CHECK-SAME: %[[ARG1:.+]]: tensor<2x3xf32>
// CHECK: %[[RESHAPE:.+]] = tensor.expand_shape %[[ARG0]]
// CHECK: %[[GENERIC:.+]] = linalg.generic
// CHECK-SAME: ins(%[[RESHAPE]], %[[ARG1]] :
// CHECK: return %[[GENERIC]]
// -----
// Test no fusion because the folded dimensions are not all preserved.
#map0 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
#map1 = affine_map<(d0, d1, d2, d3) -> (d0)>
#map2 = affine_map<(d0, d1, d2, d3) -> (d0, d2, d1, d3)>
func.func @no_fuse_unpreserved_folding_transpose(%arg0 : tensor<2x12x5xf32>, %arg1 : tensor<2xf32>) -> tensor<2x4x3x5xf32> {
%0 = tensor.expand_shape %arg0 [[0], [1, 2], [3]] : tensor<2x12x5xf32> into tensor<2x3x4x5xf32>
%init = tensor.empty() : tensor<2x4x3x5xf32>
%1 = linalg.generic {
indexing_maps = [#map0, #map1, #map2],
iterator_types = ["parallel", "parallel", "parallel", "parallel"]}
ins(%0, %arg1 : tensor<2x3x4x5xf32>, tensor<2xf32>) outs(%init : tensor<2x4x3x5xf32>) {
^bb0(%b0 : f32, %b1 : f32, %b2 : f32):
%2 = arith.addf %b0, %b1 : f32
linalg.yield %2 : f32
} -> tensor<2x4x3x5xf32>
return %1 : tensor<2x4x3x5xf32>
}
// CHECK: func @no_fuse_unpreserved_folding_transpose
// CHECK-SAME: %[[ARG0:.+]]: tensor<2x12x5xf32>
// CHECK-SAME: %[[ARG1:.+]]: tensor<2xf32>
// CHECK: %[[RESHAPE:.+]] = tensor.expand_shape %[[ARG0]]
// CHECK: %[[GENERIC:.+]] = linalg.generic
// CHECK-SAME: ins(%[[RESHAPE]], %[[ARG1]] :
// CHECK: return %[[GENERIC]]
// -----
// Test no fusion because the iterator types of folded dims are not preserved.
#map0 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d1)>
#map2 = affine_map<(d0, d1, d2, d3) -> (d0, d3)>
func.func @no_fuse_mismatched_iterator_types(%arg0 : tensor<2x12x5xf32>, %arg1 : tensor<2x3xf32>) -> tensor<2x5xf32> {
%0 = tensor.expand_shape %arg0 [[0], [1, 2], [3]] : tensor<2x12x5xf32> into tensor<2x3x4x5xf32>
%init = tensor.empty() : tensor<2x5xf32>
%1 = linalg.generic {
indexing_maps = [#map0, #map1, #map2],
iterator_types = ["parallel", "reduction", "parallel", "parallel"]}
ins(%0, %arg1 : tensor<2x3x4x5xf32>, tensor<2x3xf32>) outs(%init : tensor<2x5xf32>) {
^bb0(%b0 : f32, %b1 : f32, %b2 : f32):
%2 = arith.addf %b0, %b1 : f32
linalg.yield %2 : f32
} -> tensor<2x5xf32>
return %1 : tensor<2x5xf32>
}
// CHECK: func @no_fuse_mismatched_iterator_types
// CHECK-SAME: %[[ARG0:.+]]: tensor<2x12x5xf32>
// CHECK-SAME: %[[ARG1:.+]]: tensor<2x3xf32>
// CHECK: %[[RESHAPE:.+]] = tensor.expand_shape %[[ARG0]]
// CHECK: %[[GENERIC:.+]] = linalg.generic
// CHECK-SAME: ins(%[[RESHAPE]], %[[ARG1]] :
// CHECK: return %[[GENERIC]]
// -----
// Test control of fusion using control function
// Test no fusion because the folded dimensions are not all preserved.
#map0 = affine_map<(d0, d1, d2, d3) -> (d0, d1)>
#map1 = affine_map<(d0, d1, d2, d3) -> (d2, d3)>
#map2 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
func.func @control_fusion(%arg0 : tensor<6xf32>, %arg1 : tensor<20xf32>) -> tensor<2x3x4x5xf32> {
%0 = tensor.expand_shape %arg0 [[0, 1]] : tensor<6xf32> into tensor<2x3xf32>
%1 = tensor.expand_shape %arg1 [[0, 1]] : tensor<20xf32> into tensor<4x5xf32>
%init = tensor.empty() : tensor<2x3x4x5xf32>
%2 = linalg.generic {
indexing_maps = [#map0, #map1, #map2],
iterator_types = ["parallel", "parallel", "parallel", "parallel"]}
ins(%0, %1 : tensor<2x3xf32>, tensor<4x5xf32>) outs(%init : tensor<2x3x4x5xf32>) {
^bb0(%b0 : f32, %b1 : f32, %b2 : f32):
%3 = arith.addf %b0, %b1 : f32
linalg.yield %3 : f32
} -> tensor<2x3x4x5xf32>
return %2 : tensor<2x3x4x5xf32>
}
// CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0, d1) -> (d0)>
// CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1) -> (d1)>
// CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0, d1) -> (d0, d1)>
// CHECK: func @control_fusion(
// CHECK-SAME: %[[ARG0:.+]]: tensor<6xf32>
// CHECK-SAME: %[[ARG1:.+]]: tensor<20xf32>
// CHECK: %[[GENERIC:.+]] = linalg.generic
// CHECK-SAME: indexing_maps = [#[[MAP0]], #[[MAP1]], #[[MAP2]]]
// CHECK-SAME: iterator_types = ["parallel", "parallel"]
// CHECK-SAME: ins(%[[ARG0]], %[[ARG1]] :
// CHECK-SAME: outs(%{{.+}}: tensor<6x20xf32>)
// CHECK: %[[RESHAPE1:.+]] = tensor.expand_shape %[[GENERIC]] {{\[}}[0], [1, 2]{{\]}}
// CHECK: %[[RESHAPE2:.+]] = tensor.expand_shape %[[RESHAPE1]] {{\[}}[0, 1], [2], [3]{{\]}}
// CHECK: return %[[RESHAPE2]]
// CONTROL-DAG: #[[MAP0:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)>
// CONTROL-DAG: #[[MAP1:.+]] = affine_map<(d0, d1, d2) -> (d2)>
// CONTROL-DAG: #[[MAP3:.+]] = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
// CONTROL: func @control_fusion(
// CONTROL-SAME: %[[ARG0:.+]]: tensor<6xf32>
// CONTROL-SAME: %[[ARG1:.+]]: tensor<20xf32>
// CONTROL: %[[EXPAND:.+]] = tensor.expand_shape %[[ARG0]]
// CONTROL: %[[INIT:.+]] = tensor.empty()
// CONTROL: %[[INIT_RESHAPE:.+]] = tensor.collapse_shape %[[INIT]] {{\[}}[0], [1], [2, 3]{{\]}}
// CONTROL: %[[GENERIC:.+]] = linalg.generic
// CONTROL-SAME: ins(%[[EXPAND]], %[[ARG1]] :
// CONTROL-SAME: outs(%[[INIT_RESHAPE]] :
// CONTROL: %[[RESULT:.+]] = tensor.expand_shape %[[GENERIC]] {{\[}}[0], [1], [2, 3]{{\]}}
// -----
// Corner case that isnt handled currently.
#map = affine_map<(d0) -> (d0)>
func.func @zero_D_test(%arg0: tensor<f32>) -> tensor<1xf32> {
%0 = tensor.expand_shape %arg0 [] : tensor<f32> into tensor<1xf32>
%init = tensor.empty() : tensor<1xf32>
%1 = linalg.generic {
indexing_maps = [#map, #map],
iterator_types = ["parallel"]}
ins(%0: tensor<1xf32>) outs(%init : tensor<1xf32>) {
^bb0(%b0 : f32, %b1 : f32):
linalg.yield %b0: f32
} -> tensor<1xf32>
return %1 : tensor<1xf32>
}
// CHECK: func @zero_D_test
// CHECK-SAME: %[[ARG0:.+]]: tensor<f32>
// CHECK: %[[EXPAND:.+]] = tensor.expand_shape %[[ARG0]]
// CHECK: %[[GENERIC:.+]] = linalg.generic
// CHECK-SAME: ins(%[[EXPAND]] :
// CHECK: return %[[GENERIC]]
// -----
#map0 = affine_map<(d0, d1, d2, d3) -> (d1, d0, d2, d3)>
#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
func.func @fuse_only_one_reassociation(%arg0 : tensor<?x?xf32>, %arg1 : tensor<4x?x?x8xf32>) -> tensor<4x?x?x8xf32> {
%0 = tensor.expand_shape %arg0 [[0, 1], [2, 3]] : tensor<?x?xf32> into tensor<?x4x?x8xf32>
%1 = linalg.generic {
indexing_maps = [#map0, #map1, #map1],
iterator_types = ["parallel", "parallel", "parallel", "parallel"]}
ins(%0, %arg1 : tensor<?x4x?x8xf32>, tensor<4x?x?x8xf32>)
outs(%arg1 : tensor<4x?x?x8xf32>) {
^bb0(%b0: f32, %b1 : f32, %b2 : f32):
%2 = arith.addf %b0, %b1 : f32
linalg.yield %2 : f32
} -> tensor<4x?x?x8xf32>
return %1 : tensor<4x?x?x8xf32>
}
// CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0, d1, d2) -> (d1, d0, d2)>
// CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
// CHECK: func @fuse_only_one_reassociation(
// CHECK-SAME: %[[ARG0:.+]]: tensor<?x?xf32>
// CHECK-SAME: %[[ARG1:.+]]: tensor<4x?x?x8xf32>
// CHECK-DAG: %[[EXPAND_ARG0:.+]] = tensor.expand_shape %[[ARG0]] {{\[}}[0, 1], [2, 3]{{\]}}
// CHECK-DAG: %[[COLLAPSE_ARG0:.+]] = tensor.collapse_shape %[[EXPAND_ARG0]] {{\[}}[0], [1], [2, 3]{{\]}}
// CHECK-DAG: %[[COLLAPSE_ARG1_0:.+]] = tensor.collapse_shape %[[ARG1]] {{\[}}[0], [1], [2, 3]{{\]}}
// CHECK-DAG: %[[COLLAPSE_ARG1_1:.+]] = tensor.collapse_shape %[[ARG1]] {{\[}}[0], [1], [2, 3]{{\]}}
// CHECK: %[[GENERIC:.+]] = linalg.generic
// CHECK-SAME: indexing_maps = [#[[MAP0]], #[[MAP1]], #[[MAP1]]]
// CHECK-SAME: iterator_types = ["parallel", "parallel", "parallel"]
// CHECK-SAME: ins(%[[COLLAPSE_ARG0]], %[[COLLAPSE_ARG1_0]] :
// CHECK-SAME: outs(%[[COLLAPSE_ARG1_1]] :
// CHECK: %[[EXPAND_GENERIC:.+]] = tensor.expand_shape %[[GENERIC]] {{\[}}[0], [1], [2, 3]{{\]}}
// CHECK: return %[[EXPAND_GENERIC]]
// -----
#map0 = affine_map<(d0, d1, d2, d3) -> (d0, d2, d3, d1)>
#map1 = affine_map<(d0, d1, d2, d3) -> (d3, d1, d0, d2)>
func.func @fold_non_consecutive_dims(%arg0 : tensor<?x?xi32>) -> tensor<?x8x?x4xi32> {
%c0 = arith.constant 0 : index
%c2 = arith.constant 2 : index
%0 = tensor.expand_shape %arg0 [[0, 1], [2, 3]] : tensor<?x?xi32> into tensor<?x4x?x8xi32>
%d0 = tensor.dim %0, %c0 : tensor<?x4x?x8xi32>
%d1 = tensor.dim %0, %c2 : tensor<?x4x?x8xi32>
%init = tensor.empty(%d1, %d0) : tensor<?x8x?x4xi32>
%1 = linalg.generic {
indexing_maps = [#map0, #map1],
iterator_types = ["parallel", "parallel", "parallel", "parallel"]}
ins(%0 : tensor<?x4x?x8xi32>) outs(%init : tensor<?x8x?x4xi32>) {
^bb0(%b0 : i32, %b1 : i32):
%2 = linalg.index 0 : index
%3 = linalg.index 1 : index
%4 = linalg.index 2 : index
%5 = linalg.index 3 : index
%6 = arith.addi %2, %3 : index
%7 = arith.addi %6, %4 : index
%8 = arith.addi %7, %5 : index
%9 = arith.index_cast %8 : index to i32
linalg.yield %9: i32
} -> tensor<?x8x?x4xi32>
return %1 : tensor<?x8x?x4xi32>
}
// CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0, d1) -> (d0, d1)>
// CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1) -> (d1, d0)>
// CHECK: func @fold_non_consecutive_dims(
// CHECK-SAME: %[[ARG0:.+]]: tensor<?x?xi32>)
// CHECK-DAG: %[[C4:.+]] = arith.constant 4 : index
// CHECK-DAG: %[[C8:.+]] = arith.constant 8 : index
// CHECK: %[[INIT:.+]] = tensor.empty
// CHECK: %[[COLLAPSE_INIT:.+]] = tensor.collapse_shape %[[INIT]] {{\[}}[0, 1], [2, 3]{{\]}}
// CHECK: %[[GENERIC:.+]] = linalg.generic
// CHECK-SAME: indexing_maps = [#[[MAP0]], #[[MAP1]]]
// CHECK-SAME: iterator_types = ["parallel", "parallel"]
// CHECK-SAME: ins(%[[ARG0]] :
// CHECK-SAME: outs(%[[COLLAPSE_INIT]] :
// CHECK-NEXT: ^bb{{[0-9]}}
// CHECK: %[[ID0:.+]] = linalg.index 0
// CHECK-DAG: %[[T0:.+]] = arith.remui %[[ID0]], %[[C4]]
// CHECK-DAG: %[[T1:.+]] = arith.divui %[[ID0]], %[[C4]]
// CHECK: %[[ID1:.+]] = linalg.index 1
// CHECK-DAG: %[[T2:.+]] = arith.remui %[[ID1]], %[[C8]]
// CHECK-DAG: %[[T3:.+]] = arith.divui %[[ID1]], %[[C8]]
// CHECK-DAG: %[[T4:.+]] = arith.addi %[[T1]], %[[T2]]
// CHECK-DAG: %[[T5:.+]] = arith.addi %[[T4]], %[[T0]]
// CHECK-DAG: %[[T6:.+]] = arith.addi %[[T5]], %[[T3]]
// CHECK-DAG: %[[T7:.+]] = arith.index_cast %[[T6]]
// CHECK: linalg.yield %[[T7]]
// CHECK: %[[EXPAND_GENERIC:.+]] = tensor.expand_shape %[[GENERIC]] {{\[}}[0, 1], [2, 3]{{\]}}
// CHECK: return %[[EXPAND_GENERIC]]
// -----
// None of the folded iteration space dims are contiguous reduction dimensions.
// So no change in the code.
#map0 = affine_map<(d0, d1, d2, d3) -> (d0, d2, d3, d1)>
#map1 = affine_map<(d0, d1, d2, d3) -> ()>
func.func @no_fold_non_consecutive_reduction_dims(%arg0 : tensor<?x?xi32>) -> tensor<i32> {
%c0 = arith.constant 0 : index
%c2 = arith.constant 2 : index
%0 = tensor.expand_shape %arg0 [[0, 1], [2, 3]] : tensor<?x?xi32> into tensor<?x4x?x8xi32>
%init = tensor.empty() : tensor<i32>
%1 = linalg.generic {
indexing_maps = [#map0, #map1],
iterator_types = ["reduction", "reduction", "reduction", "reduction"]}
ins(%0 : tensor<?x4x?x8xi32>) outs(%init : tensor<i32>) {
^bb0(%b0 : i32, %b1 : i32):
%2 = linalg.index 0 : index
%3 = linalg.index 1 : index
%4 = linalg.index 2 : index
%5 = linalg.index 3 : index
%6 = arith.addi %2, %3 : index
%7 = arith.addi %6, %4 : index
%8 = arith.addi %7, %5 : index
%9 = arith.index_cast %8 : index to i32
linalg.yield %9: i32
} -> tensor<i32>
return %1 : tensor<i32>
}
// CHECK: func @no_fold_non_consecutive_reduction_dims(
// CHECK-SAME: %[[ARG0:.+]]: tensor<?x?xi32>)
// CHECK: %[[EXPAND_ARG0:.+]] = tensor.expand_shape %[[ARG0]] {{\[}}[0, 1], [2, 3]{{\]}}
// CHECK: %[[GENERIC:.+]] = linalg.generic
// CHECK-SAME: ins(%[[EXPAND_ARG0]] :
// CHECK: return %[[GENERIC]]
|
