1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
|
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2020-2022 Loongson Technology Corporation Limited
*/
#include <linux/export.h>
#include <asm/alternative-asm.h>
#include <asm/asm.h>
#include <asm/asmmacro.h>
#include <asm/asm-extable.h>
#include <asm/cpu.h>
#include <asm/regdef.h>
#include <asm/unwind_hints.h>
SYM_FUNC_START(__copy_user)
/*
* Some CPUs support hardware unaligned access
*/
ALTERNATIVE "b __copy_user_generic", \
"b __copy_user_fast", CPU_FEATURE_UAL
SYM_FUNC_END(__copy_user)
EXPORT_SYMBOL(__copy_user)
/*
* unsigned long __copy_user_generic(void *to, const void *from, size_t n)
*
* a0: to
* a1: from
* a2: n
*/
SYM_FUNC_START(__copy_user_generic)
beqz a2, 3f
1: ld.b t0, a1, 0
2: st.b t0, a0, 0
addi.d a0, a0, 1
addi.d a1, a1, 1
addi.d a2, a2, -1
bgtz a2, 1b
3: move a0, a2
jr ra
_asm_extable 1b, 3b
_asm_extable 2b, 3b
SYM_FUNC_END(__copy_user_generic)
/*
* unsigned long __copy_user_fast(void *to, const void *from, unsigned long n)
*
* a0: to
* a1: from
* a2: n
*/
SYM_FUNC_START(__copy_user_fast)
sltui t0, a2, 9
bnez t0, .Lsmall
0: ld.d t0, a1, 0
1: st.d t0, a0, 0
add.d a3, a1, a2
add.d a2, a0, a2
/* align up destination address */
andi t1, a0, 7
sub.d t0, zero, t1
addi.d t0, t0, 8
add.d a1, a1, t0
add.d a0, a0, t0
addi.d a4, a3, -64
bgeu a1, a4, .Llt64
/* copy 64 bytes at a time */
.Lloop64:
2: ld.d t0, a1, 0
3: ld.d t1, a1, 8
4: ld.d t2, a1, 16
5: ld.d t3, a1, 24
6: ld.d t4, a1, 32
7: ld.d t5, a1, 40
8: ld.d t6, a1, 48
9: ld.d t7, a1, 56
10: st.d t0, a0, 0
11: st.d t1, a0, 8
12: st.d t2, a0, 16
13: st.d t3, a0, 24
14: st.d t4, a0, 32
15: st.d t5, a0, 40
16: st.d t6, a0, 48
17: st.d t7, a0, 56
addi.d a1, a1, 64
addi.d a0, a0, 64
bltu a1, a4, .Lloop64
/* copy the remaining bytes */
.Llt64:
addi.d a4, a3, -32
bgeu a1, a4, .Llt32
18: ld.d t0, a1, 0
19: ld.d t1, a1, 8
20: ld.d t2, a1, 16
21: ld.d t3, a1, 24
22: st.d t0, a0, 0
23: st.d t1, a0, 8
24: st.d t2, a0, 16
25: st.d t3, a0, 24
addi.d a1, a1, 32
addi.d a0, a0, 32
.Llt32:
addi.d a4, a3, -16
bgeu a1, a4, .Llt16
26: ld.d t0, a1, 0
27: ld.d t1, a1, 8
28: st.d t0, a0, 0
29: st.d t1, a0, 8
addi.d a1, a1, 16
addi.d a0, a0, 16
.Llt16:
addi.d a4, a3, -8
bgeu a1, a4, .Llt8
30: ld.d t0, a1, 0
31: st.d t0, a0, 0
addi.d a1, a1, 8
addi.d a0, a0, 8
.Llt8:
32: ld.d t0, a3, -8
33: st.d t0, a2, -8
/* return */
move a0, zero
jr ra
.align 5
.Lsmall:
pcaddi t0, 8
slli.d a3, a2, 5
add.d t0, t0, a3
jr t0
.align 5
move a0, zero
jr ra
.align 5
34: ld.b t0, a1, 0
35: st.b t0, a0, 0
move a0, zero
jr ra
.align 5
36: ld.h t0, a1, 0
37: st.h t0, a0, 0
move a0, zero
jr ra
.align 5
38: ld.h t0, a1, 0
39: ld.b t1, a1, 2
40: st.h t0, a0, 0
41: st.b t1, a0, 2
move a0, zero
jr ra
.align 5
42: ld.w t0, a1, 0
43: st.w t0, a0, 0
move a0, zero
jr ra
.align 5
44: ld.w t0, a1, 0
45: ld.b t1, a1, 4
46: st.w t0, a0, 0
47: st.b t1, a0, 4
move a0, zero
jr ra
.align 5
48: ld.w t0, a1, 0
49: ld.h t1, a1, 4
50: st.w t0, a0, 0
51: st.h t1, a0, 4
move a0, zero
jr ra
.align 5
52: ld.w t0, a1, 0
53: ld.w t1, a1, 3
54: st.w t0, a0, 0
55: st.w t1, a0, 3
move a0, zero
jr ra
.align 5
56: ld.d t0, a1, 0
57: st.d t0, a0, 0
move a0, zero
jr ra
/* fixup and ex_table */
.Llarge_fixup:
sub.d a2, a2, a0
.Lsmall_fixup:
58: ld.b t0, a1, 0
59: st.b t0, a0, 0
addi.d a0, a0, 1
addi.d a1, a1, 1
addi.d a2, a2, -1
bgt a2, zero, 58b
.Lexit:
move a0, a2
jr ra
_asm_extable 0b, .Lsmall_fixup
_asm_extable 1b, .Lsmall_fixup
_asm_extable 2b, .Llarge_fixup
_asm_extable 3b, .Llarge_fixup
_asm_extable 4b, .Llarge_fixup
_asm_extable 5b, .Llarge_fixup
_asm_extable 6b, .Llarge_fixup
_asm_extable 7b, .Llarge_fixup
_asm_extable 8b, .Llarge_fixup
_asm_extable 9b, .Llarge_fixup
_asm_extable 10b, .Llarge_fixup
_asm_extable 11b, .Llarge_fixup
_asm_extable 12b, .Llarge_fixup
_asm_extable 13b, .Llarge_fixup
_asm_extable 14b, .Llarge_fixup
_asm_extable 15b, .Llarge_fixup
_asm_extable 16b, .Llarge_fixup
_asm_extable 17b, .Llarge_fixup
_asm_extable 18b, .Llarge_fixup
_asm_extable 19b, .Llarge_fixup
_asm_extable 20b, .Llarge_fixup
_asm_extable 21b, .Llarge_fixup
_asm_extable 22b, .Llarge_fixup
_asm_extable 23b, .Llarge_fixup
_asm_extable 24b, .Llarge_fixup
_asm_extable 25b, .Llarge_fixup
_asm_extable 26b, .Llarge_fixup
_asm_extable 27b, .Llarge_fixup
_asm_extable 28b, .Llarge_fixup
_asm_extable 29b, .Llarge_fixup
_asm_extable 30b, .Llarge_fixup
_asm_extable 31b, .Llarge_fixup
_asm_extable 32b, .Llarge_fixup
_asm_extable 33b, .Llarge_fixup
_asm_extable 34b, .Lexit
_asm_extable 35b, .Lexit
_asm_extable 36b, .Lsmall_fixup
_asm_extable 37b, .Lsmall_fixup
_asm_extable 38b, .Lsmall_fixup
_asm_extable 39b, .Lsmall_fixup
_asm_extable 40b, .Lsmall_fixup
_asm_extable 41b, .Lsmall_fixup
_asm_extable 42b, .Lsmall_fixup
_asm_extable 43b, .Lsmall_fixup
_asm_extable 44b, .Lsmall_fixup
_asm_extable 45b, .Lsmall_fixup
_asm_extable 46b, .Lsmall_fixup
_asm_extable 47b, .Lsmall_fixup
_asm_extable 48b, .Lsmall_fixup
_asm_extable 49b, .Lsmall_fixup
_asm_extable 50b, .Lsmall_fixup
_asm_extable 51b, .Lsmall_fixup
_asm_extable 52b, .Lsmall_fixup
_asm_extable 53b, .Lsmall_fixup
_asm_extable 54b, .Lsmall_fixup
_asm_extable 55b, .Lsmall_fixup
_asm_extable 56b, .Lsmall_fixup
_asm_extable 57b, .Lsmall_fixup
_asm_extable 58b, .Lexit
_asm_extable 59b, .Lexit
SYM_FUNC_END(__copy_user_fast)
STACK_FRAME_NON_STANDARD __copy_user_fast
|
