1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
|
/*
* Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
* (C) 2020 Vladimir Sadovnikov <sadko4u@gmail.com>
*
* This file is part of lsp-dsp-lib
* Created on: 31 мар. 2020 г.
*
* lsp-dsp-lib is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* lsp-dsp-lib is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
*/
#ifndef PRIVATE_DSP_ARCH_X86_SSE_FASTCONV_APPLY_H_
#define PRIVATE_DSP_ARCH_X86_SSE_FASTCONV_APPLY_H_
#ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
#error "This header should not be included directly"
#endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */
namespace lsp
{
namespace sse
{
static inline void fastconv_apply_internal(float *tmp, const float *c1, const float *c2, size_t rank)
{
size_t items = size_t(1) << (rank + 1);
ARCH_X86_ASM
(
__ASM_EMIT("1:")
// Load data
__ASM_EMIT("movups 0x00(%[c1]), %%xmm0") /* xmm0 = r0 r1 r2 r3 */
__ASM_EMIT("movups 0x10(%[c1]), %%xmm2") /* xmm1 = i0 i1 i2 i3 */
__ASM_EMIT("movups 0x20(%[c1]), %%xmm4")
__ASM_EMIT("movups 0x30(%[c1]), %%xmm6")
// Do complex multiplication
// xmm0 = r0 r1 r2 r3
// xmm2 = i0 i1 i2 i3
// xmm4 = r4 r5 r6 r7
// xmm6 = i4 i5 i6 i7
__ASM_EMIT("movups 0x00(%[c2]), %%xmm1") /* xmm1 = rc0 rc1 rc2 rc3 */
__ASM_EMIT("movups 0x10(%[c2]), %%xmm3") /* xmm3 = ic0 ic1 ic2 ic3 */
__ASM_EMIT("movaps %%xmm1, %%xmm5") /* xmm5 = rc0 rc1 rc2 rc3 */
__ASM_EMIT("movaps %%xmm3, %%xmm7") /* xmm7 = ic0 ic1 ic2 ic3 */
__ASM_EMIT("mulps %%xmm0, %%xmm1") /* xmm1 = rc0*r0 rc1*r1 rc2*r2 rc3*r3 */
__ASM_EMIT("mulps %%xmm2, %%xmm7") /* xmm7 = ic0*i0 ic1*i1 ic2*i2 ic3*i3 */
__ASM_EMIT("mulps %%xmm0, %%xmm3") /* xmm3 = ic0*r0 ic1*r1 ic2*r2 ic3*r3 */
__ASM_EMIT("mulps %%xmm2, %%xmm5") /* xmm5 = rc0*i0 rc1*i1 rc2*i2 rc3*i3 */
__ASM_EMIT("subps %%xmm7, %%xmm1") /* xmm1 = rc0*r0-ic0*i0 rc1*r1-ic1*i1 rc2*r2-ic2*i2 rc3*r3-ic3*i3 */
__ASM_EMIT("addps %%xmm5, %%xmm3") /* xmm3 = ic0*r0+rc0*i0 ic1*r1+rc1*i1 ic2*r2+rc2*i2 ic3*r3+rc3*i3 */
__ASM_EMIT("movups 0x20(%[c2]), %%xmm0") /* xmm0 = rc4 rc5 rc6 rc7 */
__ASM_EMIT("movups 0x30(%[c2]), %%xmm2") /* xmm2 = ic4 ic5 ic6 ic7 */
__ASM_EMIT("movaps %%xmm0, %%xmm5") /* xmm5 = rc4 rc5 rc6 rc7 */
__ASM_EMIT("movaps %%xmm2, %%xmm7") /* xmm7 = ic4 ic5 ic6 ic7 */
__ASM_EMIT("mulps %%xmm4, %%xmm5") /* xmm5 = rc4*r4 rc5*r5 rc6*r6 rc7*r7 */
__ASM_EMIT("mulps %%xmm6, %%xmm2") /* xmm2 = ic4*i4 ic5*i5 ic6*i6 ic7*i7 */
__ASM_EMIT("mulps %%xmm4, %%xmm7") /* xmm7 = ic4*r4 ic5*r5 ic6*r6 ic7*r7 */
__ASM_EMIT("mulps %%xmm6, %%xmm0") /* xmm0 = rc4*i4 rc5*i5 rc6*i6 rc7*i7 */
__ASM_EMIT("subps %%xmm2, %%xmm5") /* xmm5 = rc4*r4-ic4*i4 rc5*r5-ic5*i5 rc6*r6-ic6*i6 rc7*r7-ic7*i7 */
__ASM_EMIT("addps %%xmm0, %%xmm7") /* xmm7 = ic4*r4+rc4*i4 ic5*r5+rc5*i5 ic6*r6+rc6*i6 ic7*r7+rc7*i7 */
// Do x4 reverse butterflies
// xmm1 = r0 r1 r2 r3
// xmm3 = i0 i1 i2 i3
// xmm5 = r4 r5 r6 r7
// xmm7 = i4 i5 i6 i7
__ASM_EMIT("movaps %%xmm1, %%xmm0") /* xmm0 = r0 r1 r2 r3 */
__ASM_EMIT("movaps %%xmm5, %%xmm4")
__ASM_EMIT("shufps $0x88, %%xmm3, %%xmm0") /* xmm0 = r0 r2 i0 i2 */
__ASM_EMIT("shufps $0x88, %%xmm7, %%xmm4")
__ASM_EMIT("shufps $0xdd, %%xmm3, %%xmm1") /* xmm1 = r1 r3 i1 i3 */
__ASM_EMIT("shufps $0xdd, %%xmm7, %%xmm5")
__ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0 r2 i0 i2 */
__ASM_EMIT("movaps %%xmm4, %%xmm6")
__ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0+r1 r2+r3 i0+i1 i2+i3 = r0k r2k i0k i2k */
__ASM_EMIT("addps %%xmm5, %%xmm4")
__ASM_EMIT("subps %%xmm1, %%xmm2") /* xmm2 = r0-r1 r2-r3 i0-i1 i2-i3 = r1k r3k i1k i3k */
__ASM_EMIT("subps %%xmm5, %%xmm6")
__ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = r0k r2k i0k i2k */
__ASM_EMIT("movaps %%xmm4, %%xmm5")
__ASM_EMIT("shufps $0x88, %%xmm2, %%xmm0") /* xmm0 = r0k i0k r1k i1k */
__ASM_EMIT("shufps $0x88, %%xmm6, %%xmm4")
__ASM_EMIT("shufps $0x7d, %%xmm2, %%xmm1") /* xmm1 = r2k i2k i3k r3k */
__ASM_EMIT("shufps $0x7d, %%xmm6, %%xmm5")
__ASM_EMIT("movaps %%xmm0, %%xmm2") /* xmm2 = r0k i0k r1k i1k */
__ASM_EMIT("movaps %%xmm4, %%xmm6")
__ASM_EMIT("addps %%xmm1, %%xmm0") /* xmm0 = r0k+r2k i0k+i2k r1k+i3k i1k+r3k = d0 d4 d3 d5 */
__ASM_EMIT("addps %%xmm5, %%xmm4")
__ASM_EMIT("subps %%xmm1, %%xmm2") /* xmm2 = r0k-r2k i0k-i2k r1k-i3k i1k-r3k = d2 d6 d1 d7 */
__ASM_EMIT("subps %%xmm5, %%xmm6")
__ASM_EMIT("movaps %%xmm0, %%xmm1") /* xmm1 = d0 d4 d3 d5 */
__ASM_EMIT("movaps %%xmm4, %%xmm5")
__ASM_EMIT("shufps $0x88, %%xmm2, %%xmm0") /* xmm0 = d0 d3 d2 d1 */
__ASM_EMIT("shufps $0x88, %%xmm6, %%xmm4")
__ASM_EMIT("shufps $0xdd, %%xmm2, %%xmm1") /* xmm1 = d4 d5 d6 d7 */
__ASM_EMIT("shufps $0xdd, %%xmm6, %%xmm5")
__ASM_EMIT("shufps $0x6c, %%xmm0, %%xmm0") /* xmm0 = d0 d1 d2 d3 */
__ASM_EMIT("shufps $0x6c, %%xmm4, %%xmm4")
// Finally store result
__ASM_EMIT("movups %%xmm0, 0x00(%[tmp])")
__ASM_EMIT("movups %%xmm1, 0x10(%[tmp])")
__ASM_EMIT("movups %%xmm4, 0x20(%[tmp])")
__ASM_EMIT("movups %%xmm5, 0x30(%[tmp])")
__ASM_EMIT("add $0x40, %[tmp]")
__ASM_EMIT("add $0x40, %[c1]")
__ASM_EMIT("add $0x40, %[c2]")
__ASM_EMIT("sub $16, %[k]")
__ASM_EMIT("jnz 1b")
: [tmp] "+r" (tmp), [k] "+r" (items), [c1] "+r" (c1), [c2] "+r" (c2)
:
: "cc", "memory",
"%xmm0", "%xmm1", "%xmm2", "%xmm3",
"%xmm4", "%xmm5", "%xmm6", "%xmm7"
);
}
}
}
#endif /* PRIVATE_DSP_ARCH_X86_SSE_FASTCONV_APPLY_H_ */
|
