1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -passes=instcombine -S | FileCheck %s
; Fold zeroing of inactive lanes into the load's passthrough parameter.
define <4 x float> @masked_load_and_zero_inactive_1(ptr %ptr, <4 x i1> %mask) {
; CHECK-LABEL: @masked_load_and_zero_inactive_1(
; CHECK-NEXT: [[LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[PTR:%.*]], i32 4, <4 x i1> [[MASK:%.*]], <4 x float> zeroinitializer)
; CHECK-NEXT: ret <4 x float> [[LOAD]]
;
%load = call <4 x float> @llvm.masked.load.v4f32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x float> undef)
%masked = select <4 x i1> %mask, <4 x float> %load, <4 x float> zeroinitializer
ret <4 x float> %masked
}
; As above but reuse the load's existing passthrough.
define <4 x i32> @masked_load_and_zero_inactive_2(ptr %ptr, <4 x i1> %mask) {
; CHECK-LABEL: @masked_load_and_zero_inactive_2(
; CHECK-NEXT: [[LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[PTR:%.*]], i32 4, <4 x i1> [[MASK:%.*]], <4 x i32> zeroinitializer)
; CHECK-NEXT: ret <4 x i32> [[LOAD]]
;
%load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x i32> zeroinitializer)
%masked = select <4 x i1> %mask, <4 x i32> %load, <4 x i32> zeroinitializer
ret <4 x i32> %masked
}
; No transform when the load's passthrough cannot be reused or altered.
define <4 x i32> @masked_load_and_zero_inactive_3(ptr %ptr, <4 x i1> %mask, <4 x i32> %passthrough) {
; CHECK-LABEL: @masked_load_and_zero_inactive_3(
; CHECK-NEXT: [[LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[PTR:%.*]], i32 4, <4 x i1> [[MASK:%.*]], <4 x i32> [[PASSTHROUGH:%.*]])
; CHECK-NEXT: [[MASKED:%.*]] = select <4 x i1> [[MASK]], <4 x i32> [[LOAD]], <4 x i32> zeroinitializer
; CHECK-NEXT: ret <4 x i32> [[MASKED]]
;
%load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x i32> %passthrough)
%masked = select <4 x i1> %mask, <4 x i32> %load, <4 x i32> zeroinitializer
ret <4 x i32> %masked
}
; Remove redundant select when its mask doesn't overlap with the load mask.
define <4 x i32> @masked_load_and_zero_inactive_4(ptr %ptr, <4 x i1> %inv_mask) {
; CHECK-LABEL: @masked_load_and_zero_inactive_4(
; CHECK-NEXT: [[MASK:%.*]] = xor <4 x i1> [[INV_MASK:%.*]], <i1 true, i1 true, i1 true, i1 true>
; CHECK-NEXT: [[LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[PTR:%.*]], i32 4, <4 x i1> [[MASK]], <4 x i32> zeroinitializer)
; CHECK-NEXT: ret <4 x i32> [[LOAD]]
;
%mask = xor <4 x i1> %inv_mask, <i1 true, i1 true, i1 true, i1 true>
%load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x i32> undef)
%masked = select <4 x i1> %inv_mask, <4 x i32> zeroinitializer, <4 x i32> %load
ret <4 x i32> %masked
}
; As above but reuse the load's existing passthrough.
define <4 x i32> @masked_load_and_zero_inactive_5(ptr %ptr, <4 x i1> %inv_mask) {
; CHECK-LABEL: @masked_load_and_zero_inactive_5(
; CHECK-NEXT: [[MASK:%.*]] = xor <4 x i1> [[INV_MASK:%.*]], <i1 true, i1 true, i1 true, i1 true>
; CHECK-NEXT: [[LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[PTR:%.*]], i32 4, <4 x i1> [[MASK]], <4 x i32> zeroinitializer)
; CHECK-NEXT: ret <4 x i32> [[LOAD]]
;
%mask = xor <4 x i1> %inv_mask, <i1 true, i1 true, i1 true, i1 true>
%load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x i32> zeroinitializer)
%masked = select <4 x i1> %inv_mask, <4 x i32> zeroinitializer, <4 x i32> %load
ret <4 x i32> %masked
}
; No transform when the load's passthrough cannot be reused or altered.
define <4 x i32> @masked_load_and_zero_inactive_6(ptr %ptr, <4 x i1> %inv_mask, <4 x i32> %passthrough) {
; CHECK-LABEL: @masked_load_and_zero_inactive_6(
; CHECK-NEXT: [[MASK:%.*]] = xor <4 x i1> [[INV_MASK:%.*]], <i1 true, i1 true, i1 true, i1 true>
; CHECK-NEXT: [[LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[PTR:%.*]], i32 4, <4 x i1> [[MASK]], <4 x i32> [[PASSTHROUGH:%.*]])
; CHECK-NEXT: [[MASKED:%.*]] = select <4 x i1> [[INV_MASK]], <4 x i32> zeroinitializer, <4 x i32> [[LOAD]]
; CHECK-NEXT: ret <4 x i32> [[MASKED]]
;
%mask = xor <4 x i1> %inv_mask, <i1 true, i1 true, i1 true, i1 true>
%load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x i32> %passthrough)
%masked = select <4 x i1> %inv_mask, <4 x i32> zeroinitializer, <4 x i32> %load
ret <4 x i32> %masked
}
; No transform when select and load masks have no relation.
define <4 x i32> @masked_load_and_zero_inactive_7(ptr %ptr, <4 x i1> %mask1, <4 x i1> %mask2) {
; CHECK-LABEL: @masked_load_and_zero_inactive_7(
; CHECK-NEXT: [[LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[PTR:%.*]], i32 4, <4 x i1> [[MASK1:%.*]], <4 x i32> zeroinitializer)
; CHECK-NEXT: [[MASKED:%.*]] = select <4 x i1> [[MASK2:%.*]], <4 x i32> zeroinitializer, <4 x i32> [[LOAD]]
; CHECK-NEXT: ret <4 x i32> [[MASKED]]
;
%load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %ptr, i32 4, <4 x i1> %mask1, <4 x i32> zeroinitializer)
%masked = select <4 x i1> %mask2, <4 x i32> zeroinitializer, <4 x i32> %load
ret <4 x i32> %masked
}
; A more complex case where we can prove the select mask is a subset of the
; load's inactive lanes and thus the load's passthrough takes effect.
define <4 x float> @masked_load_and_zero_inactive_8(ptr %ptr, <4 x i1> %inv_mask, <4 x i1> %cond) {
; CHECK-LABEL: @masked_load_and_zero_inactive_8(
; CHECK-NEXT: [[MASK:%.*]] = xor <4 x i1> [[INV_MASK:%.*]], <i1 true, i1 true, i1 true, i1 true>
; CHECK-NEXT: [[PG:%.*]] = and <4 x i1> [[MASK]], [[COND:%.*]]
; CHECK-NEXT: [[LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[PTR:%.*]], i32 4, <4 x i1> [[PG]], <4 x float> zeroinitializer)
; CHECK-NEXT: ret <4 x float> [[LOAD]]
;
%mask = xor <4 x i1> %inv_mask, <i1 true, i1 true, i1 true, i1 true>
%pg = and <4 x i1> %mask, %cond
%load = call <4 x float> @llvm.masked.load.v4f32.p0(ptr %ptr, i32 4, <4 x i1> %pg, <4 x float> undef)
%masked = select <4 x i1> %inv_mask, <4 x float> zeroinitializer, <4 x float> %load
ret <4 x float> %masked
}
define <8 x float> @masked_load_and_scalar_select_cond(ptr %ptr, <8 x i1> %mask, i1 %cond) {
; CHECK-LABEL: @masked_load_and_scalar_select_cond(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr [[PTR:%.*]], i32 32, <8 x i1> [[MASK:%.*]], <8 x float> undef)
; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[COND:%.*]], <8 x float> zeroinitializer, <8 x float> [[TMP0]]
; CHECK-NEXT: ret <8 x float> [[TMP1]]
;
entry:
%0 = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %ptr, i32 32, <8 x i1> %mask, <8 x float> undef)
%1 = select i1 %cond, <8 x float> zeroinitializer, <8 x float> %0
ret <8 x float> %1
}
declare <8 x float> @llvm.masked.load.v8f32.p0(ptr, i32 immarg, <8 x i1>, <8 x float>)
declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32 immarg, <4 x i1>, <4 x i32>)
declare <4 x float> @llvm.masked.load.v4f32.p0(ptr, i32 immarg, <4 x i1>, <4 x float>)
|
