1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
; RUN: llc -mtriple=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
@lds = addrspace(3) global [512 x float] undef, align 4
; GCN-LABEL: @simple_write2st64_one_val_f32_0_1
; CI-DAG: s_mov_b32 m0
; GFX9-NOT: m0
; GCN-DAG: {{buffer|global}}_load_dword [[VAL:v[0-9]+]]
; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
; GCN: ds_write2st64_b32 [[VPTR]], [[VAL]], [[VAL]] offset1:1
; GCN: s_endpgm
define amdgpu_kernel void @simple_write2st64_one_val_f32_0_1(ptr addrspace(1) %C, ptr addrspace(1) %in) #0 {
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep = getelementptr float, ptr addrspace(1) %in, i32 %x.i
%val = load float, ptr addrspace(1) %in.gep, align 4
%arrayidx0 = getelementptr inbounds [512 x float], ptr addrspace(3) @lds, i32 0, i32 %x.i
store float %val, ptr addrspace(3) %arrayidx0, align 4
%add.x = add nsw i32 %x.i, 64
%arrayidx1 = getelementptr inbounds [512 x float], ptr addrspace(3) @lds, i32 0, i32 %add.x
store float %val, ptr addrspace(3) %arrayidx1, align 4
ret void
}
; GCN-LABEL: @simple_write2st64_two_val_f32_2_5
; CI-DAG: s_mov_b32 m0
; GFX9-NOT: m0
; CI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
; CI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} glc{{$}}
; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} offset:4 glc{{$}}
; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
; GCN: ds_write2st64_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset0:2 offset1:5
; GCN: s_endpgm
define amdgpu_kernel void @simple_write2st64_two_val_f32_2_5(ptr addrspace(1) %C, ptr addrspace(1) %in) #0 {
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %x.i
%in.gep.1 = getelementptr float, ptr addrspace(1) %in.gep.0, i32 1
%val0 = load volatile float, ptr addrspace(1) %in.gep.0, align 4
%val1 = load volatile float, ptr addrspace(1) %in.gep.1, align 4
%add.x.0 = add nsw i32 %x.i, 128
%arrayidx0 = getelementptr inbounds [512 x float], ptr addrspace(3) @lds, i32 0, i32 %add.x.0
store float %val0, ptr addrspace(3) %arrayidx0, align 4
%add.x.1 = add nsw i32 %x.i, 320
%arrayidx1 = getelementptr inbounds [512 x float], ptr addrspace(3) @lds, i32 0, i32 %add.x.1
store float %val1, ptr addrspace(3) %arrayidx1, align 4
ret void
}
; GCN-LABEL: @simple_write2st64_two_val_max_offset_f32
; CI-DAG: s_mov_b32 m0
; GFX9-NOT: m0
; CI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
; CI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} glc{{$}}
; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} offset:4
; GCN-DAG: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 2, v{{[0-9]+}}
; GCN-DAG: v_add_{{i|u}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}s{{[0-9]+}}, [[SHL]]
; GCN: ds_write2st64_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:255
; GCN: s_endpgm
define amdgpu_kernel void @simple_write2st64_two_val_max_offset_f32(ptr addrspace(1) %C, ptr addrspace(1) %in, ptr addrspace(3) %lds) #0 {
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %x.i
%in.gep.1 = getelementptr float, ptr addrspace(1) %in.gep.0, i32 1
%val0 = load volatile float, ptr addrspace(1) %in.gep.0, align 4
%val1 = load volatile float, ptr addrspace(1) %in.gep.1, align 4
%arrayidx0 = getelementptr inbounds float, ptr addrspace(3) %lds, i32 %x.i
store float %val0, ptr addrspace(3) %arrayidx0, align 4
%add.x = add nsw i32 %x.i, 16320
%arrayidx1 = getelementptr inbounds float, ptr addrspace(3) %lds, i32 %add.x
store float %val1, ptr addrspace(3) %arrayidx1, align 4
ret void
}
; GCN-LABEL: @simple_write2st64_two_val_max_offset_f64
; CI-DAG: s_mov_b32 m0
; GFX9-NOT: m0
; CI-DAG: buffer_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
; CI-DAG: buffer_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8
; GFX9-DAG: global_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} glc{{$}}
; GFX9-DAG: global_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} offset:8
; GCN-DAG: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 3, v{{[0-9]+}}
; GCN-DAG: v_add_{{i|u}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}s{{[0-9]+}}, [[SHL]]
; GCN: ds_write2st64_b64 [[VPTR]], [[VAL0]], [[VAL1]] offset0:4 offset1:127
; GCN: s_endpgm
define amdgpu_kernel void @simple_write2st64_two_val_max_offset_f64(ptr addrspace(1) %C, ptr addrspace(1) %in, ptr addrspace(3) %lds) #0 {
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %x.i
%in.gep.1 = getelementptr double, ptr addrspace(1) %in.gep.0, i32 1
%val0 = load volatile double, ptr addrspace(1) %in.gep.0, align 8
%val1 = load volatile double, ptr addrspace(1) %in.gep.1, align 8
%add.x.0 = add nsw i32 %x.i, 256
%arrayidx0 = getelementptr inbounds double, ptr addrspace(3) %lds, i32 %add.x.0
store double %val0, ptr addrspace(3) %arrayidx0, align 8
%add.x.1 = add nsw i32 %x.i, 8128
%arrayidx1 = getelementptr inbounds double, ptr addrspace(3) %lds, i32 %add.x.1
store double %val1, ptr addrspace(3) %arrayidx1, align 8
ret void
}
; GCN-LABEL: @byte_size_only_divisible_64_write2st64_f64
; CI-DAG: s_mov_b32 m0
; GFX9-NOT: m0
; GCN-NOT: ds_write2st64_b64
; GCN: ds_write2_b64 {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset1:8
; GCN: s_endpgm
define amdgpu_kernel void @byte_size_only_divisible_64_write2st64_f64(ptr addrspace(1) %C, ptr addrspace(1) %in, ptr addrspace(3) %lds) #0 {
%x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
%in.gep = getelementptr double, ptr addrspace(1) %in, i32 %x.i
%val = load double, ptr addrspace(1) %in.gep, align 8
%arrayidx0 = getelementptr inbounds double, ptr addrspace(3) %lds, i32 %x.i
store double %val, ptr addrspace(3) %arrayidx0, align 8
%add.x = add nsw i32 %x.i, 8
%arrayidx1 = getelementptr inbounds double, ptr addrspace(3) %lds, i32 %add.x
store double %val, ptr addrspace(3) %arrayidx1, align 8
ret void
}
declare i32 @llvm.amdgcn.workitem.id.x() #1
declare i32 @llvm.amdgcn.workitem.id.y() #1
attributes #0 = { nounwind }
attributes #1 = { nounwind readnone }
attributes #2 = { convergent nounwind }
|
