1 /*
2 * Copyright (c) 2003, 2007-14 Matteo Frigo
3 * Copyright (c) 2003, 2007-14 Massachusetts Institute of Technology
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 *
19 */
20
21 /* This file was automatically generated --- DO NOT EDIT */
22 /* Generated on Thu Dec 10 07:05:29 EST 2020 */
23
24 #include "dft/codelet-dft.h"
25
26 #if defined(ARCH_PREFERS_FMA) || defined(ISA_EXTENSION_PREFERS_FMA)
27
28 /* Generated by: ../../../genfft/gen_twiddle_c.native -fma -simd -compact -variables 4 -pipeline-latency 8 -n 15 -name t1bv_15 -include dft/simd/t1b.h -sign 1 */
29
30 /*
31 * This function contains 92 FP additions, 77 FP multiplications,
32 * (or, 50 additions, 35 multiplications, 42 fused multiply/add),
33 * 50 stack variables, 8 constants, and 30 memory accesses
34 */
35 #include "dft/simd/t1b.h"
36
t1bv_15(R * ri,R * ii,const R * W,stride rs,INT mb,INT me,INT ms)37 static void t1bv_15(R *ri, R *ii, const R *W, stride rs, INT mb, INT me, INT ms)
38 {
39 DVK(KP910592997, +0.910592997310029334643087372129977886038870291);
40 DVK(KP823639103, +0.823639103546331925877420039278190003029660514);
41 DVK(KP866025403, +0.866025403784438646763723170752936183471402627);
42 DVK(KP559016994, +0.559016994374947424102293417182819058860154590);
43 DVK(KP618033988, +0.618033988749894848204586834365638117720309180);
44 DVK(KP951056516, +0.951056516295153572116439333379382143405698634);
45 DVK(KP250000000, +0.250000000000000000000000000000000000000000000);
46 DVK(KP500000000, +0.500000000000000000000000000000000000000000000);
47 {
48 INT m;
49 R *x;
50 x = ii;
51 for (m = mb, W = W + (mb * ((TWVL / VL) * 28)); m < me; m = m + VL, x = x + (VL * ms), W = W + (TWVL * 28), MAKE_VOLATILE_STRIDE(15, rs)) {
52 V TV, T7, T1f, TM, TP, Tf, Tn, To, T1j, T1k, T1l, TW, TX, TY, Tw;
53 V TE, TF, T1g, T1h, T1i;
54 {
55 V T1, T5, T3, T4, T2, T6;
56 T1 = LD(&(x[0]), ms, &(x[0]));
57 T4 = LD(&(x[WS(rs, 10)]), ms, &(x[0]));
58 T5 = BYTW(&(W[TWVL * 18]), T4);
59 T2 = LD(&(x[WS(rs, 5)]), ms, &(x[WS(rs, 1)]));
60 T3 = BYTW(&(W[TWVL * 8]), T2);
61 TV = VSUB(T3, T5);
62 T6 = VADD(T3, T5);
63 T7 = VFNMS(LDK(KP500000000), T6, T1);
64 T1f = VADD(T1, T6);
65 }
66 {
67 V T9, Tq, Ty, Th, Te, TK, Tv, TN, TD, TO, Tm, TL;
68 {
69 V T8, Tp, Tx, Tg;
70 T8 = LD(&(x[WS(rs, 3)]), ms, &(x[WS(rs, 1)]));
71 T9 = BYTW(&(W[TWVL * 4]), T8);
72 Tp = LD(&(x[WS(rs, 6)]), ms, &(x[0]));
73 Tq = BYTW(&(W[TWVL * 10]), Tp);
74 Tx = LD(&(x[WS(rs, 9)]), ms, &(x[WS(rs, 1)]));
75 Ty = BYTW(&(W[TWVL * 16]), Tx);
76 Tg = LD(&(x[WS(rs, 12)]), ms, &(x[0]));
77 Th = BYTW(&(W[TWVL * 22]), Tg);
78 }
79 {
80 V Tb, Td, Ta, Tc;
81 Ta = LD(&(x[WS(rs, 8)]), ms, &(x[0]));
82 Tb = BYTW(&(W[TWVL * 14]), Ta);
83 Tc = LD(&(x[WS(rs, 13)]), ms, &(x[WS(rs, 1)]));
84 Td = BYTW(&(W[TWVL * 24]), Tc);
85 Te = VADD(Tb, Td);
86 TK = VSUB(Tb, Td);
87 }
88 {
89 V Ts, Tu, Tr, Tt;
90 Tr = LD(&(x[WS(rs, 11)]), ms, &(x[WS(rs, 1)]));
91 Ts = BYTW(&(W[TWVL * 20]), Tr);
92 Tt = LD(&(x[WS(rs, 1)]), ms, &(x[WS(rs, 1)]));
93 Tu = BYTW(&(W[0]), Tt);
94 Tv = VADD(Ts, Tu);
95 TN = VSUB(Ts, Tu);
96 }
97 {
98 V TA, TC, Tz, TB;
99 Tz = LD(&(x[WS(rs, 14)]), ms, &(x[0]));
100 TA = BYTW(&(W[TWVL * 26]), Tz);
101 TB = LD(&(x[WS(rs, 4)]), ms, &(x[0]));
102 TC = BYTW(&(W[TWVL * 6]), TB);
103 TD = VADD(TA, TC);
104 TO = VSUB(TA, TC);
105 }
106 {
107 V Tj, Tl, Ti, Tk;
108 Ti = LD(&(x[WS(rs, 2)]), ms, &(x[0]));
109 Tj = BYTW(&(W[TWVL * 2]), Ti);
110 Tk = LD(&(x[WS(rs, 7)]), ms, &(x[WS(rs, 1)]));
111 Tl = BYTW(&(W[TWVL * 12]), Tk);
112 Tm = VADD(Tj, Tl);
113 TL = VSUB(Tj, Tl);
114 }
115 TM = VSUB(TK, TL);
116 TP = VSUB(TN, TO);
117 Tf = VFNMS(LDK(KP500000000), Te, T9);
118 Tn = VFNMS(LDK(KP500000000), Tm, Th);
119 To = VADD(Tf, Tn);
120 T1j = VADD(Tq, Tv);
121 T1k = VADD(Ty, TD);
122 T1l = VADD(T1j, T1k);
123 TW = VADD(TK, TL);
124 TX = VADD(TN, TO);
125 TY = VADD(TW, TX);
126 Tw = VFNMS(LDK(KP500000000), Tv, Tq);
127 TE = VFNMS(LDK(KP500000000), TD, Ty);
128 TF = VADD(Tw, TE);
129 T1g = VADD(T9, Te);
130 T1h = VADD(Th, Tm);
131 T1i = VADD(T1g, T1h);
132 }
133 {
134 V T1o, T1m, T1n, T1s, T1u, T1q, T1r, T1t, T1p;
135 T1o = VSUB(T1i, T1l);
136 T1m = VADD(T1i, T1l);
137 T1n = VFNMS(LDK(KP250000000), T1m, T1f);
138 T1q = VSUB(T1j, T1k);
139 T1r = VSUB(T1g, T1h);
140 T1s = VMUL(LDK(KP951056516), VFNMS(LDK(KP618033988), T1r, T1q));
141 T1u = VMUL(LDK(KP951056516), VFMA(LDK(KP618033988), T1q, T1r));
142 ST(&(x[0]), VADD(T1f, T1m), ms, &(x[0]));
143 T1t = VFMA(LDK(KP559016994), T1o, T1n);
144 ST(&(x[WS(rs, 6)]), VFMAI(T1u, T1t), ms, &(x[0]));
145 ST(&(x[WS(rs, 9)]), VFNMSI(T1u, T1t), ms, &(x[WS(rs, 1)]));
146 T1p = VFNMS(LDK(KP559016994), T1o, T1n);
147 ST(&(x[WS(rs, 3)]), VFMAI(T1s, T1p), ms, &(x[WS(rs, 1)]));
148 ST(&(x[WS(rs, 12)]), VFNMSI(T1s, T1p), ms, &(x[0]));
149 }
150 {
151 V TQ, T16, T1e, T11, T19, TU, T18, TJ, T1d, T15, TZ, T10;
152 TQ = VFMA(LDK(KP618033988), TP, TM);
153 T16 = VFNMS(LDK(KP618033988), TM, TP);
154 T1e = VMUL(LDK(KP866025403), VADD(TV, TY));
155 TZ = VFNMS(LDK(KP250000000), TY, TV);
156 T10 = VSUB(TW, TX);
157 T11 = VFMA(LDK(KP559016994), T10, TZ);
158 T19 = VFNMS(LDK(KP559016994), T10, TZ);
159 {
160 V TS, TT, TI, TG, TH;
161 TS = VSUB(Tf, Tn);
162 TT = VSUB(Tw, TE);
163 TU = VFMA(LDK(KP618033988), TT, TS);
164 T18 = VFNMS(LDK(KP618033988), TS, TT);
165 TI = VSUB(To, TF);
166 TG = VADD(To, TF);
167 TH = VFNMS(LDK(KP250000000), TG, T7);
168 TJ = VFMA(LDK(KP559016994), TI, TH);
169 T1d = VADD(T7, TG);
170 T15 = VFNMS(LDK(KP559016994), TI, TH);
171 }
172 {
173 V TR, T12, T1b, T1c;
174 ST(&(x[WS(rs, 5)]), VFNMSI(T1e, T1d), ms, &(x[WS(rs, 1)]));
175 ST(&(x[WS(rs, 10)]), VFMAI(T1e, T1d), ms, &(x[0]));
176 TR = VFNMS(LDK(KP823639103), TQ, TJ);
177 T12 = VMUL(LDK(KP951056516), VFMA(LDK(KP910592997), T11, TU));
178 ST(&(x[WS(rs, 1)]), VFMAI(T12, TR), ms, &(x[WS(rs, 1)]));
179 ST(&(x[WS(rs, 14)]), VFNMSI(T12, TR), ms, &(x[0]));
180 T1b = VFMA(LDK(KP823639103), T16, T15);
181 T1c = VMUL(LDK(KP951056516), VFNMS(LDK(KP910592997), T19, T18));
182 ST(&(x[WS(rs, 7)]), VFNMSI(T1c, T1b), ms, &(x[WS(rs, 1)]));
183 ST(&(x[WS(rs, 8)]), VFMAI(T1c, T1b), ms, &(x[0]));
184 {
185 V T17, T1a, T13, T14;
186 T17 = VFNMS(LDK(KP823639103), T16, T15);
187 T1a = VMUL(LDK(KP951056516), VFMA(LDK(KP910592997), T19, T18));
188 ST(&(x[WS(rs, 2)]), VFNMSI(T1a, T17), ms, &(x[0]));
189 ST(&(x[WS(rs, 13)]), VFMAI(T1a, T17), ms, &(x[WS(rs, 1)]));
190 T13 = VFMA(LDK(KP823639103), TQ, TJ);
191 T14 = VMUL(LDK(KP951056516), VFNMS(LDK(KP910592997), T11, TU));
192 ST(&(x[WS(rs, 4)]), VFNMSI(T14, T13), ms, &(x[0]));
193 ST(&(x[WS(rs, 11)]), VFMAI(T14, T13), ms, &(x[WS(rs, 1)]));
194 }
195 }
196 }
197 }
198 }
199 VLEAVE();
200 }
201
202 static const tw_instr twinstr[] = {
203 VTW(0, 1),
204 VTW(0, 2),
205 VTW(0, 3),
206 VTW(0, 4),
207 VTW(0, 5),
208 VTW(0, 6),
209 VTW(0, 7),
210 VTW(0, 8),
211 VTW(0, 9),
212 VTW(0, 10),
213 VTW(0, 11),
214 VTW(0, 12),
215 VTW(0, 13),
216 VTW(0, 14),
217 { TW_NEXT, VL, 0 }
218 };
219
220 static const ct_desc desc = { 15, XSIMD_STRING("t1bv_15"), twinstr, &GENUS, { 50, 35, 42, 0 }, 0, 0, 0 };
221
XSIMD(codelet_t1bv_15)222 void XSIMD(codelet_t1bv_15) (planner *p) {
223 X(kdft_dit_register) (p, t1bv_15, &desc);
224 }
225 #else
226
227 /* Generated by: ../../../genfft/gen_twiddle_c.native -simd -compact -variables 4 -pipeline-latency 8 -n 15 -name t1bv_15 -include dft/simd/t1b.h -sign 1 */
228
229 /*
230 * This function contains 92 FP additions, 53 FP multiplications,
231 * (or, 78 additions, 39 multiplications, 14 fused multiply/add),
232 * 52 stack variables, 10 constants, and 30 memory accesses
233 */
234 #include "dft/simd/t1b.h"
235
t1bv_15(R * ri,R * ii,const R * W,stride rs,INT mb,INT me,INT ms)236 static void t1bv_15(R *ri, R *ii, const R *W, stride rs, INT mb, INT me, INT ms)
237 {
238 DVK(KP216506350, +0.216506350946109661690930792688234045867850657);
239 DVK(KP484122918, +0.484122918275927110647408174972799951354115213);
240 DVK(KP866025403, +0.866025403784438646763723170752936183471402627);
241 DVK(KP509036960, +0.509036960455127183450980863393907648510733164);
242 DVK(KP823639103, +0.823639103546331925877420039278190003029660514);
243 DVK(KP951056516, +0.951056516295153572116439333379382143405698634);
244 DVK(KP587785252, +0.587785252292473129168705954639072768597652438);
245 DVK(KP250000000, +0.250000000000000000000000000000000000000000000);
246 DVK(KP559016994, +0.559016994374947424102293417182819058860154590);
247 DVK(KP500000000, +0.500000000000000000000000000000000000000000000);
248 {
249 INT m;
250 R *x;
251 x = ii;
252 for (m = mb, W = W + (mb * ((TWVL / VL) * 28)); m < me; m = m + VL, x = x + (VL * ms), W = W + (TWVL * 28), MAKE_VOLATILE_STRIDE(15, rs)) {
253 V Ts, TV, T1f, TZ, T10, Tb, Tm, Tt, T1j, T1k, T1l, TI, TM, TR, Tz;
254 V TD, TQ, T1g, T1h, T1i;
255 {
256 V TT, Tr, Tp, Tq, To, TU;
257 TT = LD(&(x[0]), ms, &(x[0]));
258 Tq = LD(&(x[WS(rs, 10)]), ms, &(x[0]));
259 Tr = BYTW(&(W[TWVL * 18]), Tq);
260 To = LD(&(x[WS(rs, 5)]), ms, &(x[WS(rs, 1)]));
261 Tp = BYTW(&(W[TWVL * 8]), To);
262 Ts = VSUB(Tp, Tr);
263 TU = VADD(Tp, Tr);
264 TV = VFNMS(LDK(KP500000000), TU, TT);
265 T1f = VADD(TT, TU);
266 }
267 {
268 V Tx, TG, TK, TB, T5, Ty, Tg, TH, Tl, TL, Ta, TC;
269 {
270 V Tw, TF, TJ, TA;
271 Tw = LD(&(x[WS(rs, 3)]), ms, &(x[WS(rs, 1)]));
272 Tx = BYTW(&(W[TWVL * 4]), Tw);
273 TF = LD(&(x[WS(rs, 6)]), ms, &(x[0]));
274 TG = BYTW(&(W[TWVL * 10]), TF);
275 TJ = LD(&(x[WS(rs, 9)]), ms, &(x[WS(rs, 1)]));
276 TK = BYTW(&(W[TWVL * 16]), TJ);
277 TA = LD(&(x[WS(rs, 12)]), ms, &(x[0]));
278 TB = BYTW(&(W[TWVL * 22]), TA);
279 }
280 {
281 V T2, T4, T1, T3;
282 T1 = LD(&(x[WS(rs, 8)]), ms, &(x[0]));
283 T2 = BYTW(&(W[TWVL * 14]), T1);
284 T3 = LD(&(x[WS(rs, 13)]), ms, &(x[WS(rs, 1)]));
285 T4 = BYTW(&(W[TWVL * 24]), T3);
286 T5 = VSUB(T2, T4);
287 Ty = VADD(T2, T4);
288 }
289 {
290 V Td, Tf, Tc, Te;
291 Tc = LD(&(x[WS(rs, 11)]), ms, &(x[WS(rs, 1)]));
292 Td = BYTW(&(W[TWVL * 20]), Tc);
293 Te = LD(&(x[WS(rs, 1)]), ms, &(x[WS(rs, 1)]));
294 Tf = BYTW(&(W[0]), Te);
295 Tg = VSUB(Td, Tf);
296 TH = VADD(Td, Tf);
297 }
298 {
299 V Ti, Tk, Th, Tj;
300 Th = LD(&(x[WS(rs, 14)]), ms, &(x[0]));
301 Ti = BYTW(&(W[TWVL * 26]), Th);
302 Tj = LD(&(x[WS(rs, 4)]), ms, &(x[0]));
303 Tk = BYTW(&(W[TWVL * 6]), Tj);
304 Tl = VSUB(Ti, Tk);
305 TL = VADD(Ti, Tk);
306 }
307 {
308 V T7, T9, T6, T8;
309 T6 = LD(&(x[WS(rs, 2)]), ms, &(x[0]));
310 T7 = BYTW(&(W[TWVL * 2]), T6);
311 T8 = LD(&(x[WS(rs, 7)]), ms, &(x[WS(rs, 1)]));
312 T9 = BYTW(&(W[TWVL * 12]), T8);
313 Ta = VSUB(T7, T9);
314 TC = VADD(T7, T9);
315 }
316 TZ = VSUB(T5, Ta);
317 T10 = VSUB(Tg, Tl);
318 Tb = VADD(T5, Ta);
319 Tm = VADD(Tg, Tl);
320 Tt = VADD(Tb, Tm);
321 T1j = VADD(TG, TH);
322 T1k = VADD(TK, TL);
323 T1l = VADD(T1j, T1k);
324 TI = VFNMS(LDK(KP500000000), TH, TG);
325 TM = VFNMS(LDK(KP500000000), TL, TK);
326 TR = VADD(TI, TM);
327 Tz = VFNMS(LDK(KP500000000), Ty, Tx);
328 TD = VFNMS(LDK(KP500000000), TC, TB);
329 TQ = VADD(Tz, TD);
330 T1g = VADD(Tx, Ty);
331 T1h = VADD(TB, TC);
332 T1i = VADD(T1g, T1h);
333 }
334 {
335 V T1o, T1m, T1n, T1s, T1t, T1q, T1r, T1u, T1p;
336 T1o = VMUL(LDK(KP559016994), VSUB(T1i, T1l));
337 T1m = VADD(T1i, T1l);
338 T1n = VFNMS(LDK(KP250000000), T1m, T1f);
339 T1q = VSUB(T1g, T1h);
340 T1r = VSUB(T1j, T1k);
341 T1s = VBYI(VFNMS(LDK(KP951056516), T1r, VMUL(LDK(KP587785252), T1q)));
342 T1t = VBYI(VFMA(LDK(KP951056516), T1q, VMUL(LDK(KP587785252), T1r)));
343 ST(&(x[0]), VADD(T1f, T1m), ms, &(x[0]));
344 T1u = VADD(T1o, T1n);
345 ST(&(x[WS(rs, 6)]), VADD(T1t, T1u), ms, &(x[0]));
346 ST(&(x[WS(rs, 9)]), VSUB(T1u, T1t), ms, &(x[WS(rs, 1)]));
347 T1p = VSUB(T1n, T1o);
348 ST(&(x[WS(rs, 3)]), VSUB(T1p, T1s), ms, &(x[WS(rs, 1)]));
349 ST(&(x[WS(rs, 12)]), VADD(T1s, T1p), ms, &(x[0]));
350 }
351 {
352 V T11, T18, T1e, TO, T16, Tv, T15, TY, T1d, T19, TE, TN;
353 T11 = VFMA(LDK(KP823639103), TZ, VMUL(LDK(KP509036960), T10));
354 T18 = VFNMS(LDK(KP823639103), T10, VMUL(LDK(KP509036960), TZ));
355 T1e = VBYI(VMUL(LDK(KP866025403), VADD(Ts, Tt)));
356 TE = VSUB(Tz, TD);
357 TN = VSUB(TI, TM);
358 TO = VFMA(LDK(KP951056516), TE, VMUL(LDK(KP587785252), TN));
359 T16 = VFNMS(LDK(KP951056516), TN, VMUL(LDK(KP587785252), TE));
360 {
361 V Tn, Tu, TS, TW, TX;
362 Tn = VMUL(LDK(KP484122918), VSUB(Tb, Tm));
363 Tu = VFNMS(LDK(KP216506350), Tt, VMUL(LDK(KP866025403), Ts));
364 Tv = VADD(Tn, Tu);
365 T15 = VSUB(Tn, Tu);
366 TS = VMUL(LDK(KP559016994), VSUB(TQ, TR));
367 TW = VADD(TQ, TR);
368 TX = VFNMS(LDK(KP250000000), TW, TV);
369 TY = VADD(TS, TX);
370 T1d = VADD(TV, TW);
371 T19 = VSUB(TX, TS);
372 }
373 {
374 V TP, T12, T1b, T1c;
375 ST(&(x[WS(rs, 5)]), VSUB(T1d, T1e), ms, &(x[WS(rs, 1)]));
376 ST(&(x[WS(rs, 10)]), VADD(T1e, T1d), ms, &(x[0]));
377 TP = VBYI(VADD(Tv, TO));
378 T12 = VSUB(TY, T11);
379 ST(&(x[WS(rs, 1)]), VADD(TP, T12), ms, &(x[WS(rs, 1)]));
380 ST(&(x[WS(rs, 14)]), VSUB(T12, TP), ms, &(x[0]));
381 T1b = VBYI(VSUB(T16, T15));
382 T1c = VSUB(T19, T18);
383 ST(&(x[WS(rs, 7)]), VADD(T1b, T1c), ms, &(x[WS(rs, 1)]));
384 ST(&(x[WS(rs, 8)]), VSUB(T1c, T1b), ms, &(x[0]));
385 {
386 V T17, T1a, T13, T14;
387 T17 = VBYI(VADD(T15, T16));
388 T1a = VADD(T18, T19);
389 ST(&(x[WS(rs, 2)]), VADD(T17, T1a), ms, &(x[0]));
390 ST(&(x[WS(rs, 13)]), VSUB(T1a, T17), ms, &(x[WS(rs, 1)]));
391 T13 = VBYI(VSUB(Tv, TO));
392 T14 = VADD(T11, TY);
393 ST(&(x[WS(rs, 4)]), VADD(T13, T14), ms, &(x[0]));
394 ST(&(x[WS(rs, 11)]), VSUB(T14, T13), ms, &(x[WS(rs, 1)]));
395 }
396 }
397 }
398 }
399 }
400 VLEAVE();
401 }
402
403 static const tw_instr twinstr[] = {
404 VTW(0, 1),
405 VTW(0, 2),
406 VTW(0, 3),
407 VTW(0, 4),
408 VTW(0, 5),
409 VTW(0, 6),
410 VTW(0, 7),
411 VTW(0, 8),
412 VTW(0, 9),
413 VTW(0, 10),
414 VTW(0, 11),
415 VTW(0, 12),
416 VTW(0, 13),
417 VTW(0, 14),
418 { TW_NEXT, VL, 0 }
419 };
420
421 static const ct_desc desc = { 15, XSIMD_STRING("t1bv_15"), twinstr, &GENUS, { 78, 39, 14, 0 }, 0, 0, 0 };
422
XSIMD(codelet_t1bv_15)423 void XSIMD(codelet_t1bv_15) (planner *p) {
424 X(kdft_dit_register) (p, t1bv_15, &desc);
425 }
426 #endif
427