1 // Auto-generated file. Do not edit!
2 // Template: src/cs16-bfly4/scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2022 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include <xnnpack/math.h>
15 #include <xnnpack/fft.h>
16
17
xnn_cs16_bfly4_ukernel__scalar_x4(size_t samples,int16_t * data,const size_t stride,const int16_t * twiddle)18 void xnn_cs16_bfly4_ukernel__scalar_x4(
19 size_t samples,
20 int16_t* data,
21 const size_t stride,
22 const int16_t* twiddle)
23 {
24 const int16_t* tw1 = twiddle;
25 const int16_t* tw2 = twiddle;
26 const int16_t* tw3 = twiddle;
27 int16_t* data0 = data;
28 int16_t* data1 = data + samples * 2;
29 int16_t* data2 = data + samples * 4;
30 int16_t* data3 = data + samples * 6;
31
32 assert(samples != 0);
33 assert(data != NULL);
34 assert(stride != 0);
35 assert(twiddle != NULL);
36
37 for (; samples >= 4; samples -= 4) {
38 int32_t vout0r0 = (int32_t) data0[0];
39 int32_t vout0i0 = (int32_t) data0[1];
40 int32_t vout0r1 = (int32_t) data0[2];
41 int32_t vout0i1 = (int32_t) data0[3];
42 int32_t vout0r2 = (int32_t) data0[4];
43 int32_t vout0i2 = (int32_t) data0[5];
44 int32_t vout0r3 = (int32_t) data0[6];
45 int32_t vout0i3 = (int32_t) data0[7];
46 int32_t vout1r0 = (int32_t) data1[0];
47 int32_t vout1i0 = (int32_t) data1[1];
48 int32_t vout1r1 = (int32_t) data1[2];
49 int32_t vout1i1 = (int32_t) data1[3];
50 int32_t vout1r2 = (int32_t) data1[4];
51 int32_t vout1i2 = (int32_t) data1[5];
52 int32_t vout1r3 = (int32_t) data1[6];
53 int32_t vout1i3 = (int32_t) data1[7];
54 int32_t vout2r0 = (int32_t) data2[0];
55 int32_t vout2i0 = (int32_t) data2[1];
56 int32_t vout2r1 = (int32_t) data2[2];
57 int32_t vout2i1 = (int32_t) data2[3];
58 int32_t vout2r2 = (int32_t) data2[4];
59 int32_t vout2i2 = (int32_t) data2[5];
60 int32_t vout2r3 = (int32_t) data2[6];
61 int32_t vout2i3 = (int32_t) data2[7];
62 int32_t vout3r0 = (int32_t) data3[0];
63 int32_t vout3i0 = (int32_t) data3[1];
64 int32_t vout3r1 = (int32_t) data3[2];
65 int32_t vout3i1 = (int32_t) data3[3];
66 int32_t vout3r2 = (int32_t) data3[4];
67 int32_t vout3i2 = (int32_t) data3[5];
68 int32_t vout3r3 = (int32_t) data3[6];
69 int32_t vout3i3 = (int32_t) data3[7];
70
71 const int32_t vtw1r0 = (const int32_t) tw1[0];
72 const int32_t vtw1i0 = (const int32_t) tw1[1];
73 tw1 += stride * 2;
74 const int32_t vtw1r1 = (const int32_t) tw1[0];
75 const int32_t vtw1i1 = (const int32_t) tw1[1];
76 tw1 += stride * 2;
77 const int32_t vtw1r2 = (const int32_t) tw1[0];
78 const int32_t vtw1i2 = (const int32_t) tw1[1];
79 tw1 += stride * 2;
80 const int32_t vtw1r3 = (const int32_t) tw1[0];
81 const int32_t vtw1i3 = (const int32_t) tw1[1];
82 tw1 += stride * 2;
83 const int32_t vtw2r0 = (const int32_t) tw2[0];
84 const int32_t vtw2i0 = (const int32_t) tw2[1];
85 tw2 += stride * 4;
86 const int32_t vtw2r1 = (const int32_t) tw2[0];
87 const int32_t vtw2i1 = (const int32_t) tw2[1];
88 tw2 += stride * 4;
89 const int32_t vtw2r2 = (const int32_t) tw2[0];
90 const int32_t vtw2i2 = (const int32_t) tw2[1];
91 tw2 += stride * 4;
92 const int32_t vtw2r3 = (const int32_t) tw2[0];
93 const int32_t vtw2i3 = (const int32_t) tw2[1];
94 tw2 += stride * 4;
95 const int32_t vtw3r0 = (const int32_t) tw3[0];
96 const int32_t vtw3i0 = (const int32_t) tw3[1];
97 tw3 += stride * 6;
98 const int32_t vtw3r1 = (const int32_t) tw3[0];
99 const int32_t vtw3i1 = (const int32_t) tw3[1];
100 tw3 += stride * 6;
101 const int32_t vtw3r2 = (const int32_t) tw3[0];
102 const int32_t vtw3i2 = (const int32_t) tw3[1];
103 tw3 += stride * 6;
104 const int32_t vtw3r3 = (const int32_t) tw3[0];
105 const int32_t vtw3i3 = (const int32_t) tw3[1];
106 tw3 += stride * 6;
107
108 // Note 32767 / 4 = 8191. Should be 8192.
109 vout0r0 = math_asr_s32(vout0r0 * 8191 + 16384, 15);
110 vout0r1 = math_asr_s32(vout0r1 * 8191 + 16384, 15);
111 vout0r2 = math_asr_s32(vout0r2 * 8191 + 16384, 15);
112 vout0r3 = math_asr_s32(vout0r3 * 8191 + 16384, 15);
113 vout0i0 = math_asr_s32(vout0i0 * 8191 + 16384, 15);
114 vout0i1 = math_asr_s32(vout0i1 * 8191 + 16384, 15);
115 vout0i2 = math_asr_s32(vout0i2 * 8191 + 16384, 15);
116 vout0i3 = math_asr_s32(vout0i3 * 8191 + 16384, 15);
117 vout1r0 = math_asr_s32(vout1r0 * 8191 + 16384, 15);
118 vout1r1 = math_asr_s32(vout1r1 * 8191 + 16384, 15);
119 vout1r2 = math_asr_s32(vout1r2 * 8191 + 16384, 15);
120 vout1r3 = math_asr_s32(vout1r3 * 8191 + 16384, 15);
121 vout1i0 = math_asr_s32(vout1i0 * 8191 + 16384, 15);
122 vout1i1 = math_asr_s32(vout1i1 * 8191 + 16384, 15);
123 vout1i2 = math_asr_s32(vout1i2 * 8191 + 16384, 15);
124 vout1i3 = math_asr_s32(vout1i3 * 8191 + 16384, 15);
125 vout2r0 = math_asr_s32(vout2r0 * 8191 + 16384, 15);
126 vout2r1 = math_asr_s32(vout2r1 * 8191 + 16384, 15);
127 vout2r2 = math_asr_s32(vout2r2 * 8191 + 16384, 15);
128 vout2r3 = math_asr_s32(vout2r3 * 8191 + 16384, 15);
129 vout2i0 = math_asr_s32(vout2i0 * 8191 + 16384, 15);
130 vout2i1 = math_asr_s32(vout2i1 * 8191 + 16384, 15);
131 vout2i2 = math_asr_s32(vout2i2 * 8191 + 16384, 15);
132 vout2i3 = math_asr_s32(vout2i3 * 8191 + 16384, 15);
133 vout3r0 = math_asr_s32(vout3r0 * 8191 + 16384, 15);
134 vout3r1 = math_asr_s32(vout3r1 * 8191 + 16384, 15);
135 vout3r2 = math_asr_s32(vout3r2 * 8191 + 16384, 15);
136 vout3r3 = math_asr_s32(vout3r3 * 8191 + 16384, 15);
137 vout3i0 = math_asr_s32(vout3i0 * 8191 + 16384, 15);
138 vout3i1 = math_asr_s32(vout3i1 * 8191 + 16384, 15);
139 vout3i2 = math_asr_s32(vout3i2 * 8191 + 16384, 15);
140 vout3i3 = math_asr_s32(vout3i3 * 8191 + 16384, 15);
141
142 const int32_t vtmp0r0 = math_asr_s32(vout1r0 * vtw1r0 - vout1i0 * vtw1i0 + 16384, 15);
143 const int32_t vtmp0r1 = math_asr_s32(vout1r1 * vtw1r1 - vout1i1 * vtw1i1 + 16384, 15);
144 const int32_t vtmp0r2 = math_asr_s32(vout1r2 * vtw1r2 - vout1i2 * vtw1i2 + 16384, 15);
145 const int32_t vtmp0r3 = math_asr_s32(vout1r3 * vtw1r3 - vout1i3 * vtw1i3 + 16384, 15);
146 const int32_t vtmp0i0 = math_asr_s32(vout1r0 * vtw1i0 + vout1i0 * vtw1r0 + 16384, 15);
147 const int32_t vtmp0i1 = math_asr_s32(vout1r1 * vtw1i1 + vout1i1 * vtw1r1 + 16384, 15);
148 const int32_t vtmp0i2 = math_asr_s32(vout1r2 * vtw1i2 + vout1i2 * vtw1r2 + 16384, 15);
149 const int32_t vtmp0i3 = math_asr_s32(vout1r3 * vtw1i3 + vout1i3 * vtw1r3 + 16384, 15);
150 const int32_t vtmp1r0 = math_asr_s32(vout2r0 * vtw2r0 - vout2i0 * vtw2i0 + 16384, 15);
151 const int32_t vtmp1r1 = math_asr_s32(vout2r1 * vtw2r1 - vout2i1 * vtw2i1 + 16384, 15);
152 const int32_t vtmp1r2 = math_asr_s32(vout2r2 * vtw2r2 - vout2i2 * vtw2i2 + 16384, 15);
153 const int32_t vtmp1r3 = math_asr_s32(vout2r3 * vtw2r3 - vout2i3 * vtw2i3 + 16384, 15);
154 const int32_t vtmp1i0 = math_asr_s32(vout2r0 * vtw2i0 + vout2i0 * vtw2r0 + 16384, 15);
155 const int32_t vtmp1i1 = math_asr_s32(vout2r1 * vtw2i1 + vout2i1 * vtw2r1 + 16384, 15);
156 const int32_t vtmp1i2 = math_asr_s32(vout2r2 * vtw2i2 + vout2i2 * vtw2r2 + 16384, 15);
157 const int32_t vtmp1i3 = math_asr_s32(vout2r3 * vtw2i3 + vout2i3 * vtw2r3 + 16384, 15);
158 const int32_t vtmp2r0 = math_asr_s32(vout3r0 * vtw3r0 - vout3i0 * vtw3i0 + 16384, 15);
159 const int32_t vtmp2r1 = math_asr_s32(vout3r1 * vtw3r1 - vout3i1 * vtw3i1 + 16384, 15);
160 const int32_t vtmp2r2 = math_asr_s32(vout3r2 * vtw3r2 - vout3i2 * vtw3i2 + 16384, 15);
161 const int32_t vtmp2r3 = math_asr_s32(vout3r3 * vtw3r3 - vout3i3 * vtw3i3 + 16384, 15);
162 const int32_t vtmp2i0 = math_asr_s32(vout3r0 * vtw3i0 + vout3i0 * vtw3r0 + 16384, 15);
163 const int32_t vtmp2i1 = math_asr_s32(vout3r1 * vtw3i1 + vout3i1 * vtw3r1 + 16384, 15);
164 const int32_t vtmp2i2 = math_asr_s32(vout3r2 * vtw3i2 + vout3i2 * vtw3r2 + 16384, 15);
165 const int32_t vtmp2i3 = math_asr_s32(vout3r3 * vtw3i3 + vout3i3 * vtw3r3 + 16384, 15);
166
167 const int32_t vtmp5r0 = vout0r0 - vtmp1r0;
168 const int32_t vtmp5r1 = vout0r1 - vtmp1r1;
169 const int32_t vtmp5r2 = vout0r2 - vtmp1r2;
170 const int32_t vtmp5r3 = vout0r3 - vtmp1r3;
171 const int32_t vtmp5i0 = vout0i0 - vtmp1i0;
172 const int32_t vtmp5i1 = vout0i1 - vtmp1i1;
173 const int32_t vtmp5i2 = vout0i2 - vtmp1i2;
174 const int32_t vtmp5i3 = vout0i3 - vtmp1i3;
175 vout0r0 += vtmp1r0;
176 vout0r1 += vtmp1r1;
177 vout0r2 += vtmp1r2;
178 vout0r3 += vtmp1r3;
179 vout0i0 += vtmp1i0;
180 vout0i1 += vtmp1i1;
181 vout0i2 += vtmp1i2;
182 vout0i3 += vtmp1i3;
183 const int32_t vtmp3r0 = vtmp0r0 + vtmp2r0;
184 const int32_t vtmp3r1 = vtmp0r1 + vtmp2r1;
185 const int32_t vtmp3r2 = vtmp0r2 + vtmp2r2;
186 const int32_t vtmp3r3 = vtmp0r3 + vtmp2r3;
187 const int32_t vtmp3i0 = vtmp0i0 + vtmp2i0;
188 const int32_t vtmp3i1 = vtmp0i1 + vtmp2i1;
189 const int32_t vtmp3i2 = vtmp0i2 + vtmp2i2;
190 const int32_t vtmp3i3 = vtmp0i3 + vtmp2i3;
191 const int32_t vtmp4r0 = vtmp0r0 - vtmp2r0;
192 const int32_t vtmp4r1 = vtmp0r1 - vtmp2r1;
193 const int32_t vtmp4r2 = vtmp0r2 - vtmp2r2;
194 const int32_t vtmp4r3 = vtmp0r3 - vtmp2r3;
195 const int32_t vtmp4i0 = vtmp0i0 - vtmp2i0;
196 const int32_t vtmp4i1 = vtmp0i1 - vtmp2i1;
197 const int32_t vtmp4i2 = vtmp0i2 - vtmp2i2;
198 const int32_t vtmp4i3 = vtmp0i3 - vtmp2i3;
199 vout2r0 = vout0r0 - vtmp3r0;
200 vout2r1 = vout0r1 - vtmp3r1;
201 vout2r2 = vout0r2 - vtmp3r2;
202 vout2r3 = vout0r3 - vtmp3r3;
203 vout2i0 = vout0i0 - vtmp3i0;
204 vout2i1 = vout0i1 - vtmp3i1;
205 vout2i2 = vout0i2 - vtmp3i2;
206 vout2i3 = vout0i3 - vtmp3i3;
207 vout0r0 += vtmp3r0;
208 vout0r1 += vtmp3r1;
209 vout0r2 += vtmp3r2;
210 vout0r3 += vtmp3r3;
211 vout0i0 += vtmp3i0;
212 vout0i1 += vtmp3i1;
213 vout0i2 += vtmp3i2;
214 vout0i3 += vtmp3i3;
215 vout1r0 = vtmp5r0 + vtmp4i0;
216 vout1r1 = vtmp5r1 + vtmp4i1;
217 vout1r2 = vtmp5r2 + vtmp4i2;
218 vout1r3 = vtmp5r3 + vtmp4i3;
219 vout1i0 = vtmp5i0 - vtmp4r0;
220 vout1i1 = vtmp5i1 - vtmp4r1;
221 vout1i2 = vtmp5i2 - vtmp4r2;
222 vout1i3 = vtmp5i3 - vtmp4r3;
223 vout3r0 = vtmp5r0 - vtmp4i0;
224 vout3r1 = vtmp5r1 - vtmp4i1;
225 vout3r2 = vtmp5r2 - vtmp4i2;
226 vout3r3 = vtmp5r3 - vtmp4i3;
227 vout3i0 = vtmp5i0 + vtmp4r0;
228 vout3i1 = vtmp5i1 + vtmp4r1;
229 vout3i2 = vtmp5i2 + vtmp4r2;
230 vout3i3 = vtmp5i3 + vtmp4r3;
231
232 data0[0] = (int16_t) vout0r0;
233 data0[1] = (int16_t) vout0i0;
234 data0[2] = (int16_t) vout0r1;
235 data0[3] = (int16_t) vout0i1;
236 data0[4] = (int16_t) vout0r2;
237 data0[5] = (int16_t) vout0i2;
238 data0[6] = (int16_t) vout0r3;
239 data0[7] = (int16_t) vout0i3;
240 data0 += 4 * 2;
241 data1[0] = (int16_t) vout1r0;
242 data1[1] = (int16_t) vout1i0;
243 data1[2] = (int16_t) vout1r1;
244 data1[3] = (int16_t) vout1i1;
245 data1[4] = (int16_t) vout1r2;
246 data1[5] = (int16_t) vout1i2;
247 data1[6] = (int16_t) vout1r3;
248 data1[7] = (int16_t) vout1i3;
249 data1 += 4 * 2;
250 data2[0] = (int16_t) vout2r0;
251 data2[1] = (int16_t) vout2i0;
252 data2[2] = (int16_t) vout2r1;
253 data2[3] = (int16_t) vout2i1;
254 data2[4] = (int16_t) vout2r2;
255 data2[5] = (int16_t) vout2i2;
256 data2[6] = (int16_t) vout2r3;
257 data2[7] = (int16_t) vout2i3;
258 data2 += 4 * 2;
259 data3[0] = (int16_t) vout3r0;
260 data3[1] = (int16_t) vout3i0;
261 data3[2] = (int16_t) vout3r1;
262 data3[3] = (int16_t) vout3i1;
263 data3[4] = (int16_t) vout3r2;
264 data3[5] = (int16_t) vout3i2;
265 data3[6] = (int16_t) vout3r3;
266 data3[7] = (int16_t) vout3i3;
267 data3 += 4 * 2;
268 }
269
270 if XNN_UNLIKELY(samples != 0) {
271 do {
272 int32_t vout0r = (int32_t) data0[0];
273 int32_t vout0i = (int32_t) data0[1];
274 int32_t vout1r = (int32_t) data1[0];
275 int32_t vout1i = (int32_t) data1[1];
276 int32_t vout2r = (int32_t) data2[0];
277 int32_t vout2i = (int32_t) data2[1];
278 int32_t vout3r = (int32_t) data3[0];
279 int32_t vout3i = (int32_t) data3[1];
280
281 const int32_t vtw1r = (const int32_t) tw1[0];
282 const int32_t vtw1i = (const int32_t) tw1[1];
283 const int32_t vtw2r = (const int32_t) tw2[0];
284 const int32_t vtw2i = (const int32_t) tw2[1];
285 const int32_t vtw3r = (const int32_t) tw3[0];
286 const int32_t vtw3i = (const int32_t) tw3[1];
287 tw1 += stride * 2;
288 tw2 += stride * 4;
289 tw3 += stride * 6;
290
291 // Note 32767 / 4 = 8191. Should be 8192.
292 vout0r = math_asr_s32(vout0r * 8191 + 16384, 15);
293 vout0i = math_asr_s32(vout0i * 8191 + 16384, 15);
294 vout1r = math_asr_s32(vout1r * 8191 + 16384, 15);
295 vout1i = math_asr_s32(vout1i * 8191 + 16384, 15);
296 vout2r = math_asr_s32(vout2r * 8191 + 16384, 15);
297 vout2i = math_asr_s32(vout2i * 8191 + 16384, 15);
298 vout3r = math_asr_s32(vout3r * 8191 + 16384, 15);
299 vout3i = math_asr_s32(vout3i * 8191 + 16384, 15);
300
301 const int32_t vtmp0r = math_asr_s32(vout1r * vtw1r - vout1i * vtw1i + 16384, 15);
302 const int32_t vtmp0i = math_asr_s32(vout1r * vtw1i + vout1i * vtw1r + 16384, 15);
303 const int32_t vtmp1r = math_asr_s32(vout2r * vtw2r - vout2i * vtw2i + 16384, 15);
304 const int32_t vtmp1i = math_asr_s32(vout2r * vtw2i + vout2i * vtw2r + 16384, 15);
305 const int32_t vtmp2r = math_asr_s32(vout3r * vtw3r - vout3i * vtw3i + 16384, 15);
306 const int32_t vtmp2i = math_asr_s32(vout3r * vtw3i + vout3i * vtw3r + 16384, 15);
307
308 const int32_t vtmp5r = vout0r - vtmp1r;
309 const int32_t vtmp5i = vout0i - vtmp1i;
310 vout0r += vtmp1r;
311 vout0i += vtmp1i;
312 const int32_t vtmp3r = vtmp0r + vtmp2r;
313 const int32_t vtmp3i = vtmp0i + vtmp2i;
314 const int32_t vtmp4r = vtmp0r - vtmp2r;
315 const int32_t vtmp4i = vtmp0i - vtmp2i;
316 vout2r = vout0r - vtmp3r;
317 vout2i = vout0i - vtmp3i;
318
319 vout0r += vtmp3r;
320 vout0i += vtmp3i;
321
322 vout1r = vtmp5r + vtmp4i;
323 vout1i = vtmp5i - vtmp4r;
324 vout3r = vtmp5r - vtmp4i;
325 vout3i = vtmp5i + vtmp4r;
326
327 data0[0] = (int16_t) vout0r;
328 data0[1] = (int16_t) vout0i;
329 data1[0] = (int16_t) vout1r;
330 data1[1] = (int16_t) vout1i;
331 data2[0] = (int16_t) vout2r;
332 data2[1] = (int16_t) vout2i;
333 data3[0] = (int16_t) vout3r;
334 data3[1] = (int16_t) vout3i;
335 data0 += 2;
336 data1 += 2;
337 data2 += 2;
338 data3 += 2;
339 } while(--samples != 0);
340 }
341 }
342