File: galois_amd64.s

package info (click to toggle)
golang-github-klauspost-reedsolomon 1.9.9-1
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, sid
  • size: 1,248 kB
  • sloc: asm: 17,464; makefile: 8
file content (368 lines) | stat: -rw-r--r-- 10,233 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
//+build !noasm !appengine !gccgo

// Copyright 2015, Klaus Post, see LICENSE for details.

// Based on http://www.snia.org/sites/default/files2/SDC2013/presentations/NewThinking/EthanMiller_Screaming_Fast_Galois_Field%20Arithmetic_SIMD%20Instructions.pdf
// and http://jerasure.org/jerasure/gf-complete/tree/master

// func galMulSSSE3Xor(low, high, in, out []byte)
TEXT ·galMulSSSE3Xor(SB), 7, $0
	MOVQ   low+0(FP), SI     // SI: &low
	MOVQ   high+24(FP), DX   // DX: &high
	MOVOU  (SI), X6          // X6 low
	MOVOU  (DX), X7          // X7: high
	MOVQ   $15, BX           // BX: low mask
	MOVQ   BX, X8
	PXOR   X5, X5
	MOVQ   in+48(FP), SI     // R11: &in
	MOVQ   in_len+56(FP), R9 // R9: len(in)
	MOVQ   out+72(FP), DX    // DX: &out
	PSHUFB X5, X8            // X8: lomask (unpacked)
	SHRQ   $4, R9            // len(in) / 16
	MOVQ   SI, AX
	MOVQ   DX, BX
	ANDQ   $15, AX
	ANDQ   $15, BX
	CMPQ   R9, $0
	JEQ    done_xor
	ORQ    AX, BX
	CMPQ   BX, $0
	JNZ    loopback_xor

loopback_xor_aligned:
	MOVOA  (SI), X0             // in[x]
	MOVOA  (DX), X4             // out[x]
	MOVOA  X0, X1               // in[x]
	MOVOA  X6, X2               // low copy
	MOVOA  X7, X3               // high copy
	PSRLQ  $4, X1               // X1: high input
	PAND   X8, X0               // X0: low input
	PAND   X8, X1               // X0: high input
	PSHUFB X0, X2               // X2: mul low part
	PSHUFB X1, X3               // X3: mul high part
	PXOR   X2, X3               // X3: Result
	PXOR   X4, X3               // X3: Result xor existing out
	MOVOA  X3, (DX)             // Store
	ADDQ   $16, SI              // in+=16
	ADDQ   $16, DX              // out+=16
	SUBQ   $1, R9
	JNZ    loopback_xor_aligned
	JMP    done_xor

loopback_xor:
	MOVOU  (SI), X0     // in[x]
	MOVOU  (DX), X4     // out[x]
	MOVOU  X0, X1       // in[x]
	MOVOU  X6, X2       // low copy
	MOVOU  X7, X3       // high copy
	PSRLQ  $4, X1       // X1: high input
	PAND   X8, X0       // X0: low input
	PAND   X8, X1       // X0: high input
	PSHUFB X0, X2       // X2: mul low part
	PSHUFB X1, X3       // X3: mul high part
	PXOR   X2, X3       // X3: Result
	PXOR   X4, X3       // X3: Result xor existing out
	MOVOU  X3, (DX)     // Store
	ADDQ   $16, SI      // in+=16
	ADDQ   $16, DX      // out+=16
	SUBQ   $1, R9
	JNZ    loopback_xor

done_xor:
	RET

// func galMulSSSE3(low, high, in, out []byte)
TEXT ·galMulSSSE3(SB), 7, $0
	MOVQ   low+0(FP), SI     // SI: &low
	MOVQ   high+24(FP), DX   // DX: &high
	MOVOU  (SI), X6          // X6 low
	MOVOU  (DX), X7          // X7: high
	MOVQ   $15, BX           // BX: low mask
	MOVQ   BX, X8
	PXOR   X5, X5
	MOVQ   in+48(FP), SI     // R11: &in
	MOVQ   in_len+56(FP), R9 // R9: len(in)
	MOVQ   out+72(FP), DX    // DX: &out
	PSHUFB X5, X8            // X8: lomask (unpacked)
	MOVQ   SI, AX
	MOVQ   DX, BX
	SHRQ   $4, R9            // len(in) / 16
	ANDQ   $15, AX
	ANDQ   $15, BX
	CMPQ   R9, $0
	JEQ    done
	ORQ    AX, BX
	CMPQ   BX, $0
	JNZ    loopback

loopback_aligned:
	MOVOA  (SI), X0         // in[x]
	MOVOA  X0, X1           // in[x]
	MOVOA  X6, X2           // low copy
	MOVOA  X7, X3           // high copy
	PSRLQ  $4, X1           // X1: high input
	PAND   X8, X0           // X0: low input
	PAND   X8, X1           // X0: high input
	PSHUFB X0, X2           // X2: mul low part
	PSHUFB X1, X3           // X3: mul high part
	PXOR   X2, X3           // X3: Result
	MOVOA  X3, (DX)         // Store
	ADDQ   $16, SI          // in+=16
	ADDQ   $16, DX          // out+=16
	SUBQ   $1, R9
	JNZ    loopback_aligned
	JMP    done

loopback:
	MOVOU  (SI), X0 // in[x]
	MOVOU  X0, X1   // in[x]
	MOVOA  X6, X2   // low copy
	MOVOA  X7, X3   // high copy
	PSRLQ  $4, X1   // X1: high input
	PAND   X8, X0   // X0: low input
	PAND   X8, X1   // X0: high input
	PSHUFB X0, X2   // X2: mul low part
	PSHUFB X1, X3   // X3: mul high part
	PXOR   X2, X3   // X3: Result
	MOVOU  X3, (DX) // Store
	ADDQ   $16, SI  // in+=16
	ADDQ   $16, DX  // out+=16
	SUBQ   $1, R9
	JNZ    loopback

done:
	RET

// func galMulAVX2Xor(low, high, in, out []byte)
TEXT ·galMulAVX2Xor(SB), 7, $0
	MOVQ  low+0(FP), SI     // SI: &low
	MOVQ  high+24(FP), DX   // DX: &high
	MOVQ  $15, BX           // BX: low mask
	MOVQ  BX, X5
	MOVOU (SI), X6          // X6: low
	MOVOU (DX), X7          // X7: high
	MOVQ  in_len+56(FP), R9 // R9: len(in)

	VINSERTI128  $1, X6, Y6, Y6 // low
	VINSERTI128  $1, X7, Y7, Y7 // high
	VPBROADCASTB X5, Y8         // Y8: lomask (unpacked)

	SHRQ  $5, R9         // len(in) / 32
	MOVQ  out+72(FP), DX // DX: &out
	MOVQ  in+48(FP), SI  // SI: &in
	TESTQ R9, R9
	JZ    done_xor_avx2

loopback_xor_avx2:
	VMOVDQU (SI), Y0
	VMOVDQU (DX), Y4
	VPSRLQ  $4, Y0, Y1 // Y1: high input
	VPAND   Y8, Y0, Y0 // Y0: low input
	VPAND   Y8, Y1, Y1 // Y1: high input
	VPSHUFB Y0, Y6, Y2 // Y2: mul low part
	VPSHUFB Y1, Y7, Y3 // Y3: mul high part
	VPXOR   Y3, Y2, Y3 // Y3: Result
	VPXOR   Y4, Y3, Y4 // Y4: Result
	VMOVDQU Y4, (DX)

	ADDQ $32, SI           // in+=32
	ADDQ $32, DX           // out+=32
	SUBQ $1, R9
	JNZ  loopback_xor_avx2

done_xor_avx2:
	VZEROUPPER
	RET

// func galMulAVX2(low, high, in, out []byte)
TEXT ·galMulAVX2(SB), 7, $0
	MOVQ  low+0(FP), SI     // SI: &low
	MOVQ  high+24(FP), DX   // DX: &high
	MOVQ  $15, BX           // BX: low mask
	MOVQ  BX, X5
	MOVOU (SI), X6          // X6: low
	MOVOU (DX), X7          // X7: high
	MOVQ  in_len+56(FP), R9 // R9: len(in)

	VINSERTI128  $1, X6, Y6, Y6 // low
	VINSERTI128  $1, X7, Y7, Y7 // high
	VPBROADCASTB X5, Y8         // Y8: lomask (unpacked)

	SHRQ  $5, R9         // len(in) / 32
	MOVQ  out+72(FP), DX // DX: &out
	MOVQ  in+48(FP), SI  // SI: &in
	TESTQ R9, R9
	JZ    done_avx2

loopback_avx2:
	VMOVDQU (SI), Y0
	VPSRLQ  $4, Y0, Y1 // Y1: high input
	VPAND   Y8, Y0, Y0 // Y0: low input
	VPAND   Y8, Y1, Y1 // Y1: high input
	VPSHUFB Y0, Y6, Y2 // Y2: mul low part
	VPSHUFB Y1, Y7, Y3 // Y3: mul high part
	VPXOR   Y3, Y2, Y4 // Y4: Result
	VMOVDQU Y4, (DX)

	ADDQ $32, SI       // in+=32
	ADDQ $32, DX       // out+=32
	SUBQ $1, R9
	JNZ  loopback_avx2

done_avx2:
	VZEROUPPER
	RET

// func sSE2XorSlice(in, out []byte)
TEXT ·sSE2XorSlice(SB), 7, $0
	MOVQ in+0(FP), SI     // SI: &in
	MOVQ in_len+8(FP), R9 // R9: len(in)
	MOVQ out+24(FP), DX   // DX: &out
	SHRQ $4, R9           // len(in) / 16
	CMPQ R9, $0
	JEQ  done_xor_sse2

loopback_xor_sse2:
	MOVOU (SI), X0          // in[x]
	MOVOU (DX), X1          // out[x]
	PXOR  X0, X1
	MOVOU X1, (DX)
	ADDQ  $16, SI           // in+=16
	ADDQ  $16, DX           // out+=16
	SUBQ  $1, R9
	JNZ   loopback_xor_sse2

done_xor_sse2:
	RET

// func galMulAVX2Xor_64(low, high, in, out []byte)
TEXT ·galMulAVX2Xor_64(SB), 7, $0
	MOVQ  low+0(FP), SI     // SI: &low
	MOVQ  high+24(FP), DX   // DX: &high
	MOVQ  $15, BX           // BX: low mask
	MOVQ  BX, X5
	MOVOU (SI), X6          // X6: low
	MOVOU (DX), X7          // X7: high
	MOVQ  in_len+56(FP), R9 // R9: len(in)

	VINSERTI128  $1, X6, Y6, Y6 // low
	VINSERTI128  $1, X7, Y7, Y7 // high
	VPBROADCASTB X5, Y8         // Y8: lomask (unpacked)

	SHRQ  $6, R9           // len(in) / 64
	MOVQ  out+72(FP), DX   // DX: &out
	MOVQ  in+48(FP), SI    // SI: &in
	TESTQ R9, R9
	JZ    done_xor_avx2_64

loopback_xor_avx2_64:
	VMOVDQU (SI), Y0
	VMOVDQU 32(SI), Y10
	VMOVDQU (DX), Y4
	VMOVDQU 32(DX), Y14
	VPSRLQ  $4, Y0, Y1    // Y1: high input
	VPSRLQ  $4, Y10, Y11  // Y11: high input 2
	VPAND   Y8, Y0, Y0    // Y0: low input
	VPAND   Y8, Y10, Y10  // Y10: low input 2
	VPAND   Y8, Y1, Y1    // Y11: high input
	VPAND   Y8, Y11, Y11  // Y11: high input 2
	VPSHUFB Y0, Y6, Y2    // Y2: mul low part
	VPSHUFB Y10, Y6, Y12  // Y12: mul low part 2
	VPSHUFB Y1, Y7, Y3    // Y3: mul high part
	VPSHUFB Y11, Y7, Y13  // Y13: mul high part 2
	VPXOR   Y3, Y2, Y3    // Y3: Result
	VPXOR   Y13, Y12, Y13 // Y13: Result 2
	VPXOR   Y4, Y3, Y4    // Y4: Result
	VPXOR   Y14, Y13, Y14 // Y4: Result 2
	VMOVDQU Y4, (DX)
	VMOVDQU Y14, 32(DX)

	ADDQ $64, SI              // in+=64
	ADDQ $64, DX              // out+=64
	SUBQ $1, R9
	JNZ  loopback_xor_avx2_64

done_xor_avx2_64:
	VZEROUPPER
	RET

// func galMulAVX2_64(low, high, in, out []byte)
TEXT ·galMulAVX2_64(SB), 7, $0
	MOVQ  low+0(FP), SI     // SI: &low
	MOVQ  high+24(FP), DX   // DX: &high
	MOVQ  $15, BX           // BX: low mask
	MOVQ  BX, X5
	MOVOU (SI), X6          // X6: low
	MOVOU (DX), X7          // X7: high
	MOVQ  in_len+56(FP), R9 // R9: len(in)

	VINSERTI128  $1, X6, Y6, Y6 // low
	VINSERTI128  $1, X7, Y7, Y7 // high
	VPBROADCASTB X5, Y8         // Y8: lomask (unpacked)

	SHRQ  $6, R9         // len(in) / 64
	MOVQ  out+72(FP), DX // DX: &out
	MOVQ  in+48(FP), SI  // SI: &in
	TESTQ R9, R9
	JZ    done_avx2_64

loopback_avx2_64:
	VMOVDQU (SI), Y0
	VMOVDQU 32(SI), Y10
	VPSRLQ  $4, Y0, Y1    // Y1: high input
	VPSRLQ  $4, Y10, Y11  // Y11: high input 2
	VPAND   Y8, Y0, Y0    // Y0: low input
	VPAND   Y8, Y10, Y10  // Y10: low input
	VPAND   Y8, Y1, Y1    // Y1: high input
	VPAND   Y8, Y11, Y11  // Y11: high input 2
	VPSHUFB Y0, Y6, Y2    // Y2: mul low part
	VPSHUFB Y10, Y6, Y12  // Y12: mul low part 2
	VPSHUFB Y1, Y7, Y3    // Y3: mul high part
	VPSHUFB Y11, Y7, Y13  // Y13: mul high part 2
	VPXOR   Y3, Y2, Y4    // Y4: Result
	VPXOR   Y13, Y12, Y14 // Y14: Result 2
	VMOVDQU Y4, (DX)
	VMOVDQU Y14, 32(DX)

	ADDQ $64, SI          // in+=64
	ADDQ $64, DX          // out+=64
	SUBQ $1, R9
	JNZ  loopback_avx2_64

done_avx2_64:
	VZEROUPPER
	RET

// func sSE2XorSlice_64(in, out []byte)
TEXT ·sSE2XorSlice_64(SB), 7, $0
	MOVQ in+0(FP), SI     // SI: &in
	MOVQ in_len+8(FP), R9 // R9: len(in)
	MOVQ out+24(FP), DX   // DX: &out
	SHRQ $6, R9           // len(in) / 64
	CMPQ R9, $0
	JEQ  done_xor_sse2_64

loopback_xor_sse2_64:
	MOVOU (SI), X0             // in[x]
	MOVOU 16(SI), X2           // in[x]
	MOVOU 32(SI), X4           // in[x]
	MOVOU 48(SI), X6           // in[x]
	MOVOU (DX), X1             // out[x]
	MOVOU 16(DX), X3           // out[x]
	MOVOU 32(DX), X5           // out[x]
	MOVOU 48(DX), X7           // out[x]
	PXOR  X0, X1
	PXOR  X2, X3
	PXOR  X4, X5
	PXOR  X6, X7
	MOVOU X1, (DX)
	MOVOU X3, 16(DX)
	MOVOU X5, 32(DX)
	MOVOU X7, 48(DX)
	ADDQ  $64, SI              // in+=64
	ADDQ  $64, DX              // out+=64
	SUBQ  $1, R9
	JNZ   loopback_xor_sse2_64

done_xor_sse2_64:
	RET