1 /*
2 * Copyright © 2018-2019, VideoLAN and dav1d authors
3 * Copyright © 2018-2019, Two Orioles, LLC
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "config.h"
29
30 #include <stddef.h>
31 #include <stdint.h>
32
33 #include "common/intops.h"
34
35 #include "src/itx_1d.h"
36
37 #define CLIP(a) iclip(a, min, max)
38
39 /*
40 * In some places, we use the pattern like this:
41 * t2 = ((in1 * 1567 - in3 * (3784 - 4096) + 2048) >> 12) - in3;
42 * even though the reference code might use something like:
43 * t2 = (in1 * 1567 - in3 * 3784 + 2048) >> 12;
44 *
45 * The reason for this is that for 12 bits/component bitstreams (corrupt/
46 * invalid ones, but they are codable nonetheless), each coefficient or
47 * input can be 19(+sign) bits, and therefore if the combination of the
48 * two multipliers (each 12 bits) is >= 4096, the result of the add/sub
49 * after the pair of multiplies will exceed the 31+sign bit range. Signed
50 * integer overflows are UB in C, and we'd like to prevent that.
51 *
52 * To workaround this, we invert one of the two coefficients (or, if both are
53 * multiples of 2, we reduce their magnitude by one bit). It should be noted
54 * that SIMD implementations do not have to follow this exact behaviour. The
55 * AV1 spec clearly states that the result of the multiply/add pairs should
56 * fit in 31+sign bit intermediates, and that streams violating this convention
57 * are not AV1-compliant. So, as long as we don't trigger UB (which some people
58 * would consider a security vulnerability), we're fine. So, SIMD can simply
59 * use the faster implementation, even if that might in some cases result in
60 * integer overflows, since these are not considered valid AV1 anyway, and in
61 * e.g. x86 assembly, integer overflows are not considered UB, but they merely
62 * wrap around.
63 */
64
65 static NOINLINE void
inv_dct4_1d_internal_c(int32_t * const c,const ptrdiff_t stride,const int min,const int max,const int tx64)66 inv_dct4_1d_internal_c(int32_t *const c, const ptrdiff_t stride,
67 const int min, const int max, const int tx64)
68 {
69 assert(stride > 0);
70 const int in0 = c[0 * stride], in1 = c[1 * stride];
71
72 int t0, t1, t2, t3;
73 if (tx64) {
74 t0 = t1 = (in0 * 181 + 128) >> 8;
75 t2 = (in1 * 1567 + 2048) >> 12;
76 t3 = (in1 * 3784 + 2048) >> 12;
77 } else {
78 const int in2 = c[2 * stride], in3 = c[3 * stride];
79
80 t0 = ((in0 + in2) * 181 + 128) >> 8;
81 t1 = ((in0 - in2) * 181 + 128) >> 8;
82 t2 = ((in1 * 1567 - in3 * (3784 - 4096) + 2048) >> 12) - in3;
83 t3 = ((in1 * (3784 - 4096) + in3 * 1567 + 2048) >> 12) + in1;
84 }
85
86 c[0 * stride] = CLIP(t0 + t3);
87 c[1 * stride] = CLIP(t1 + t2);
88 c[2 * stride] = CLIP(t1 - t2);
89 c[3 * stride] = CLIP(t0 - t3);
90 }
91
inv_dct4_1d_c(int32_t * const c,const ptrdiff_t stride,const int min,const int max)92 static void inv_dct4_1d_c(int32_t *const c, const ptrdiff_t stride,
93 const int min, const int max)
94 {
95 inv_dct4_1d_internal_c(c, stride, min, max, 0);
96 }
97
98 static NOINLINE void
inv_dct8_1d_internal_c(int32_t * const c,const ptrdiff_t stride,const int min,const int max,const int tx64)99 inv_dct8_1d_internal_c(int32_t *const c, const ptrdiff_t stride,
100 const int min, const int max, const int tx64)
101 {
102 assert(stride > 0);
103 inv_dct4_1d_internal_c(c, stride << 1, min, max, tx64);
104
105 const int in1 = c[1 * stride], in3 = c[3 * stride];
106
107 int t4a, t5a, t6a, t7a;
108 if (tx64) {
109 t4a = (in1 * 799 + 2048) >> 12;
110 t5a = (in3 * -2276 + 2048) >> 12;
111 t6a = (in3 * 3406 + 2048) >> 12;
112 t7a = (in1 * 4017 + 2048) >> 12;
113 } else {
114 const int in5 = c[5 * stride], in7 = c[7 * stride];
115
116 t4a = ((in1 * 799 - in7 * (4017 - 4096) + 2048) >> 12) - in7;
117 t5a = (in5 * 1703 - in3 * 1138 + 1024) >> 11;
118 t6a = (in5 * 1138 + in3 * 1703 + 1024) >> 11;
119 t7a = ((in1 * (4017 - 4096) + in7 * 799 + 2048) >> 12) + in1;
120 }
121
122 const int t4 = CLIP(t4a + t5a);
123 t5a = CLIP(t4a - t5a);
124 const int t7 = CLIP(t7a + t6a);
125 t6a = CLIP(t7a - t6a);
126
127 const int t5 = ((t6a - t5a) * 181 + 128) >> 8;
128 const int t6 = ((t6a + t5a) * 181 + 128) >> 8;
129
130 const int t0 = c[0 * stride];
131 const int t1 = c[2 * stride];
132 const int t2 = c[4 * stride];
133 const int t3 = c[6 * stride];
134
135 c[0 * stride] = CLIP(t0 + t7);
136 c[1 * stride] = CLIP(t1 + t6);
137 c[2 * stride] = CLIP(t2 + t5);
138 c[3 * stride] = CLIP(t3 + t4);
139 c[4 * stride] = CLIP(t3 - t4);
140 c[5 * stride] = CLIP(t2 - t5);
141 c[6 * stride] = CLIP(t1 - t6);
142 c[7 * stride] = CLIP(t0 - t7);
143 }
144
inv_dct8_1d_c(int32_t * const c,const ptrdiff_t stride,const int min,const int max)145 static void inv_dct8_1d_c(int32_t *const c, const ptrdiff_t stride,
146 const int min, const int max)
147 {
148 inv_dct8_1d_internal_c(c, stride, min, max, 0);
149 }
150
151 static NOINLINE void
inv_dct16_1d_internal_c(int32_t * const c,const ptrdiff_t stride,const int min,const int max,int tx64)152 inv_dct16_1d_internal_c(int32_t *const c, const ptrdiff_t stride,
153 const int min, const int max, int tx64)
154 {
155 assert(stride > 0);
156 inv_dct8_1d_internal_c(c, stride << 1, min, max, tx64);
157
158 const int in1 = c[1 * stride], in3 = c[3 * stride];
159 const int in5 = c[5 * stride], in7 = c[7 * stride];
160
161 int t8a, t9a, t10a, t11a, t12a, t13a, t14a, t15a;
162 if (tx64) {
163 t8a = (in1 * 401 + 2048) >> 12;
164 t9a = (in7 * -2598 + 2048) >> 12;
165 t10a = (in5 * 1931 + 2048) >> 12;
166 t11a = (in3 * -1189 + 2048) >> 12;
167 t12a = (in3 * 3920 + 2048) >> 12;
168 t13a = (in5 * 3612 + 2048) >> 12;
169 t14a = (in7 * 3166 + 2048) >> 12;
170 t15a = (in1 * 4076 + 2048) >> 12;
171 } else {
172 const int in9 = c[ 9 * stride], in11 = c[11 * stride];
173 const int in13 = c[13 * stride], in15 = c[15 * stride];
174
175 t8a = ((in1 * 401 - in15 * (4076 - 4096) + 2048) >> 12) - in15;
176 t9a = (in9 * 1583 - in7 * 1299 + 1024) >> 11;
177 t10a = ((in5 * 1931 - in11 * (3612 - 4096) + 2048) >> 12) - in11;
178 t11a = ((in13 * (3920 - 4096) - in3 * 1189 + 2048) >> 12) + in13;
179 t12a = ((in13 * 1189 + in3 * (3920 - 4096) + 2048) >> 12) + in3;
180 t13a = ((in5 * (3612 - 4096) + in11 * 1931 + 2048) >> 12) + in5;
181 t14a = (in9 * 1299 + in7 * 1583 + 1024) >> 11;
182 t15a = ((in1 * (4076 - 4096) + in15 * 401 + 2048) >> 12) + in1;
183 }
184
185 int t8 = CLIP(t8a + t9a);
186 int t9 = CLIP(t8a - t9a);
187 int t10 = CLIP(t11a - t10a);
188 int t11 = CLIP(t11a + t10a);
189 int t12 = CLIP(t12a + t13a);
190 int t13 = CLIP(t12a - t13a);
191 int t14 = CLIP(t15a - t14a);
192 int t15 = CLIP(t15a + t14a);
193
194 t9a = (( t14 * 1567 - t9 * (3784 - 4096) + 2048) >> 12) - t9;
195 t14a = (( t14 * (3784 - 4096) + t9 * 1567 + 2048) >> 12) + t14;
196 t10a = ((-(t13 * (3784 - 4096) + t10 * 1567) + 2048) >> 12) - t13;
197 t13a = (( t13 * 1567 - t10 * (3784 - 4096) + 2048) >> 12) - t10;
198
199 t8a = CLIP(t8 + t11);
200 t9 = CLIP(t9a + t10a);
201 t10 = CLIP(t9a - t10a);
202 t11a = CLIP(t8 - t11);
203 t12a = CLIP(t15 - t12);
204 t13 = CLIP(t14a - t13a);
205 t14 = CLIP(t14a + t13a);
206 t15a = CLIP(t15 + t12);
207
208 t10a = ((t13 - t10) * 181 + 128) >> 8;
209 t13a = ((t13 + t10) * 181 + 128) >> 8;
210 t11 = ((t12a - t11a) * 181 + 128) >> 8;
211 t12 = ((t12a + t11a) * 181 + 128) >> 8;
212
213 const int t0 = c[ 0 * stride];
214 const int t1 = c[ 2 * stride];
215 const int t2 = c[ 4 * stride];
216 const int t3 = c[ 6 * stride];
217 const int t4 = c[ 8 * stride];
218 const int t5 = c[10 * stride];
219 const int t6 = c[12 * stride];
220 const int t7 = c[14 * stride];
221
222 c[ 0 * stride] = CLIP(t0 + t15a);
223 c[ 1 * stride] = CLIP(t1 + t14);
224 c[ 2 * stride] = CLIP(t2 + t13a);
225 c[ 3 * stride] = CLIP(t3 + t12);
226 c[ 4 * stride] = CLIP(t4 + t11);
227 c[ 5 * stride] = CLIP(t5 + t10a);
228 c[ 6 * stride] = CLIP(t6 + t9);
229 c[ 7 * stride] = CLIP(t7 + t8a);
230 c[ 8 * stride] = CLIP(t7 - t8a);
231 c[ 9 * stride] = CLIP(t6 - t9);
232 c[10 * stride] = CLIP(t5 - t10a);
233 c[11 * stride] = CLIP(t4 - t11);
234 c[12 * stride] = CLIP(t3 - t12);
235 c[13 * stride] = CLIP(t2 - t13a);
236 c[14 * stride] = CLIP(t1 - t14);
237 c[15 * stride] = CLIP(t0 - t15a);
238 }
239
inv_dct16_1d_c(int32_t * const c,const ptrdiff_t stride,const int min,const int max)240 static void inv_dct16_1d_c(int32_t *const c, const ptrdiff_t stride,
241 const int min, const int max)
242 {
243 inv_dct16_1d_internal_c(c, stride, min, max, 0);
244 }
245
246 static NOINLINE void
inv_dct32_1d_internal_c(int32_t * const c,const ptrdiff_t stride,const int min,const int max,const int tx64)247 inv_dct32_1d_internal_c(int32_t *const c, const ptrdiff_t stride,
248 const int min, const int max, const int tx64)
249 {
250 assert(stride > 0);
251 inv_dct16_1d_internal_c(c, stride << 1, min, max, tx64);
252
253 const int in1 = c[ 1 * stride], in3 = c[ 3 * stride];
254 const int in5 = c[ 5 * stride], in7 = c[ 7 * stride];
255 const int in9 = c[ 9 * stride], in11 = c[11 * stride];
256 const int in13 = c[13 * stride], in15 = c[15 * stride];
257
258 int t16a, t17a, t18a, t19a, t20a, t21a, t22a, t23a;
259 int t24a, t25a, t26a, t27a, t28a, t29a, t30a, t31a;
260 if (tx64) {
261 t16a = (in1 * 201 + 2048) >> 12;
262 t17a = (in15 * -2751 + 2048) >> 12;
263 t18a = (in9 * 1751 + 2048) >> 12;
264 t19a = (in7 * -1380 + 2048) >> 12;
265 t20a = (in5 * 995 + 2048) >> 12;
266 t21a = (in11 * -2106 + 2048) >> 12;
267 t22a = (in13 * 2440 + 2048) >> 12;
268 t23a = (in3 * -601 + 2048) >> 12;
269 t24a = (in3 * 4052 + 2048) >> 12;
270 t25a = (in13 * 3290 + 2048) >> 12;
271 t26a = (in11 * 3513 + 2048) >> 12;
272 t27a = (in5 * 3973 + 2048) >> 12;
273 t28a = (in7 * 3857 + 2048) >> 12;
274 t29a = (in9 * 3703 + 2048) >> 12;
275 t30a = (in15 * 3035 + 2048) >> 12;
276 t31a = (in1 * 4091 + 2048) >> 12;
277 } else {
278 const int in17 = c[17 * stride], in19 = c[19 * stride];
279 const int in21 = c[21 * stride], in23 = c[23 * stride];
280 const int in25 = c[25 * stride], in27 = c[27 * stride];
281 const int in29 = c[29 * stride], in31 = c[31 * stride];
282
283 t16a = ((in1 * 201 - in31 * (4091 - 4096) + 2048) >> 12) - in31;
284 t17a = ((in17 * (3035 - 4096) - in15 * 2751 + 2048) >> 12) + in17;
285 t18a = ((in9 * 1751 - in23 * (3703 - 4096) + 2048) >> 12) - in23;
286 t19a = ((in25 * (3857 - 4096) - in7 * 1380 + 2048) >> 12) + in25;
287 t20a = ((in5 * 995 - in27 * (3973 - 4096) + 2048) >> 12) - in27;
288 t21a = ((in21 * (3513 - 4096) - in11 * 2106 + 2048) >> 12) + in21;
289 t22a = (in13 * 1220 - in19 * 1645 + 1024) >> 11;
290 t23a = ((in29 * (4052 - 4096) - in3 * 601 + 2048) >> 12) + in29;
291 t24a = ((in29 * 601 + in3 * (4052 - 4096) + 2048) >> 12) + in3;
292 t25a = (in13 * 1645 + in19 * 1220 + 1024) >> 11;
293 t26a = ((in21 * 2106 + in11 * (3513 - 4096) + 2048) >> 12) + in11;
294 t27a = ((in5 * (3973 - 4096) + in27 * 995 + 2048) >> 12) + in5;
295 t28a = ((in25 * 1380 + in7 * (3857 - 4096) + 2048) >> 12) + in7;
296 t29a = ((in9 * (3703 - 4096) + in23 * 1751 + 2048) >> 12) + in9;
297 t30a = ((in17 * 2751 + in15 * (3035 - 4096) + 2048) >> 12) + in15;
298 t31a = ((in1 * (4091 - 4096) + in31 * 201 + 2048) >> 12) + in1;
299 }
300
301 int t16 = CLIP(t16a + t17a);
302 int t17 = CLIP(t16a - t17a);
303 int t18 = CLIP(t19a - t18a);
304 int t19 = CLIP(t19a + t18a);
305 int t20 = CLIP(t20a + t21a);
306 int t21 = CLIP(t20a - t21a);
307 int t22 = CLIP(t23a - t22a);
308 int t23 = CLIP(t23a + t22a);
309 int t24 = CLIP(t24a + t25a);
310 int t25 = CLIP(t24a - t25a);
311 int t26 = CLIP(t27a - t26a);
312 int t27 = CLIP(t27a + t26a);
313 int t28 = CLIP(t28a + t29a);
314 int t29 = CLIP(t28a - t29a);
315 int t30 = CLIP(t31a - t30a);
316 int t31 = CLIP(t31a + t30a);
317
318 t17a = (( t30 * 799 - t17 * (4017 - 4096) + 2048) >> 12) - t17;
319 t30a = (( t30 * (4017 - 4096) + t17 * 799 + 2048) >> 12) + t30;
320 t18a = ((-(t29 * (4017 - 4096) + t18 * 799) + 2048) >> 12) - t29;
321 t29a = (( t29 * 799 - t18 * (4017 - 4096) + 2048) >> 12) - t18;
322 t21a = ( t26 * 1703 - t21 * 1138 + 1024) >> 11;
323 t26a = ( t26 * 1138 + t21 * 1703 + 1024) >> 11;
324 t22a = (-(t25 * 1138 + t22 * 1703 ) + 1024) >> 11;
325 t25a = ( t25 * 1703 - t22 * 1138 + 1024) >> 11;
326
327 t16a = CLIP(t16 + t19);
328 t17 = CLIP(t17a + t18a);
329 t18 = CLIP(t17a - t18a);
330 t19a = CLIP(t16 - t19);
331 t20a = CLIP(t23 - t20);
332 t21 = CLIP(t22a - t21a);
333 t22 = CLIP(t22a + t21a);
334 t23a = CLIP(t23 + t20);
335 t24a = CLIP(t24 + t27);
336 t25 = CLIP(t25a + t26a);
337 t26 = CLIP(t25a - t26a);
338 t27a = CLIP(t24 - t27);
339 t28a = CLIP(t31 - t28);
340 t29 = CLIP(t30a - t29a);
341 t30 = CLIP(t30a + t29a);
342 t31a = CLIP(t31 + t28);
343
344 t18a = (( t29 * 1567 - t18 * (3784 - 4096) + 2048) >> 12) - t18;
345 t29a = (( t29 * (3784 - 4096) + t18 * 1567 + 2048) >> 12) + t29;
346 t19 = (( t28a * 1567 - t19a * (3784 - 4096) + 2048) >> 12) - t19a;
347 t28 = (( t28a * (3784 - 4096) + t19a * 1567 + 2048) >> 12) + t28a;
348 t20 = ((-(t27a * (3784 - 4096) + t20a * 1567) + 2048) >> 12) - t27a;
349 t27 = (( t27a * 1567 - t20a * (3784 - 4096) + 2048) >> 12) - t20a;
350 t21a = ((-(t26 * (3784 - 4096) + t21 * 1567) + 2048) >> 12) - t26;
351 t26a = (( t26 * 1567 - t21 * (3784 - 4096) + 2048) >> 12) - t21;
352
353 t16 = CLIP(t16a + t23a);
354 t17a = CLIP(t17 + t22);
355 t18 = CLIP(t18a + t21a);
356 t19a = CLIP(t19 + t20);
357 t20a = CLIP(t19 - t20);
358 t21 = CLIP(t18a - t21a);
359 t22a = CLIP(t17 - t22);
360 t23 = CLIP(t16a - t23a);
361 t24 = CLIP(t31a - t24a);
362 t25a = CLIP(t30 - t25);
363 t26 = CLIP(t29a - t26a);
364 t27a = CLIP(t28 - t27);
365 t28a = CLIP(t28 + t27);
366 t29 = CLIP(t29a + t26a);
367 t30a = CLIP(t30 + t25);
368 t31 = CLIP(t31a + t24a);
369
370 t20 = ((t27a - t20a) * 181 + 128) >> 8;
371 t27 = ((t27a + t20a) * 181 + 128) >> 8;
372 t21a = ((t26 - t21 ) * 181 + 128) >> 8;
373 t26a = ((t26 + t21 ) * 181 + 128) >> 8;
374 t22 = ((t25a - t22a) * 181 + 128) >> 8;
375 t25 = ((t25a + t22a) * 181 + 128) >> 8;
376 t23a = ((t24 - t23 ) * 181 + 128) >> 8;
377 t24a = ((t24 + t23 ) * 181 + 128) >> 8;
378
379 const int t0 = c[ 0 * stride];
380 const int t1 = c[ 2 * stride];
381 const int t2 = c[ 4 * stride];
382 const int t3 = c[ 6 * stride];
383 const int t4 = c[ 8 * stride];
384 const int t5 = c[10 * stride];
385 const int t6 = c[12 * stride];
386 const int t7 = c[14 * stride];
387 const int t8 = c[16 * stride];
388 const int t9 = c[18 * stride];
389 const int t10 = c[20 * stride];
390 const int t11 = c[22 * stride];
391 const int t12 = c[24 * stride];
392 const int t13 = c[26 * stride];
393 const int t14 = c[28 * stride];
394 const int t15 = c[30 * stride];
395
396 c[ 0 * stride] = CLIP(t0 + t31);
397 c[ 1 * stride] = CLIP(t1 + t30a);
398 c[ 2 * stride] = CLIP(t2 + t29);
399 c[ 3 * stride] = CLIP(t3 + t28a);
400 c[ 4 * stride] = CLIP(t4 + t27);
401 c[ 5 * stride] = CLIP(t5 + t26a);
402 c[ 6 * stride] = CLIP(t6 + t25);
403 c[ 7 * stride] = CLIP(t7 + t24a);
404 c[ 8 * stride] = CLIP(t8 + t23a);
405 c[ 9 * stride] = CLIP(t9 + t22);
406 c[10 * stride] = CLIP(t10 + t21a);
407 c[11 * stride] = CLIP(t11 + t20);
408 c[12 * stride] = CLIP(t12 + t19a);
409 c[13 * stride] = CLIP(t13 + t18);
410 c[14 * stride] = CLIP(t14 + t17a);
411 c[15 * stride] = CLIP(t15 + t16);
412 c[16 * stride] = CLIP(t15 - t16);
413 c[17 * stride] = CLIP(t14 - t17a);
414 c[18 * stride] = CLIP(t13 - t18);
415 c[19 * stride] = CLIP(t12 - t19a);
416 c[20 * stride] = CLIP(t11 - t20);
417 c[21 * stride] = CLIP(t10 - t21a);
418 c[22 * stride] = CLIP(t9 - t22);
419 c[23 * stride] = CLIP(t8 - t23a);
420 c[24 * stride] = CLIP(t7 - t24a);
421 c[25 * stride] = CLIP(t6 - t25);
422 c[26 * stride] = CLIP(t5 - t26a);
423 c[27 * stride] = CLIP(t4 - t27);
424 c[28 * stride] = CLIP(t3 - t28a);
425 c[29 * stride] = CLIP(t2 - t29);
426 c[30 * stride] = CLIP(t1 - t30a);
427 c[31 * stride] = CLIP(t0 - t31);
428 }
429
inv_dct32_1d_c(int32_t * const c,const ptrdiff_t stride,const int min,const int max)430 static void inv_dct32_1d_c(int32_t *const c, const ptrdiff_t stride,
431 const int min, const int max)
432 {
433 inv_dct32_1d_internal_c(c, stride, min, max, 0);
434 }
435
inv_dct64_1d_c(int32_t * const c,const ptrdiff_t stride,const int min,const int max)436 static void inv_dct64_1d_c(int32_t *const c, const ptrdiff_t stride,
437 const int min, const int max)
438 {
439 assert(stride > 0);
440 inv_dct32_1d_internal_c(c, stride << 1, min, max, 1);
441
442 const int in1 = c[ 1 * stride], in3 = c[ 3 * stride];
443 const int in5 = c[ 5 * stride], in7 = c[ 7 * stride];
444 const int in9 = c[ 9 * stride], in11 = c[11 * stride];
445 const int in13 = c[13 * stride], in15 = c[15 * stride];
446 const int in17 = c[17 * stride], in19 = c[19 * stride];
447 const int in21 = c[21 * stride], in23 = c[23 * stride];
448 const int in25 = c[25 * stride], in27 = c[27 * stride];
449 const int in29 = c[29 * stride], in31 = c[31 * stride];
450
451 int t32a = (in1 * 101 + 2048) >> 12;
452 int t33a = (in31 * -2824 + 2048) >> 12;
453 int t34a = (in17 * 1660 + 2048) >> 12;
454 int t35a = (in15 * -1474 + 2048) >> 12;
455 int t36a = (in9 * 897 + 2048) >> 12;
456 int t37a = (in23 * -2191 + 2048) >> 12;
457 int t38a = (in25 * 2359 + 2048) >> 12;
458 int t39a = (in7 * -700 + 2048) >> 12;
459 int t40a = (in5 * 501 + 2048) >> 12;
460 int t41a = (in27 * -2520 + 2048) >> 12;
461 int t42a = (in21 * 2019 + 2048) >> 12;
462 int t43a = (in11 * -1092 + 2048) >> 12;
463 int t44a = (in13 * 1285 + 2048) >> 12;
464 int t45a = (in19 * -1842 + 2048) >> 12;
465 int t46a = (in29 * 2675 + 2048) >> 12;
466 int t47a = (in3 * -301 + 2048) >> 12;
467 int t48a = (in3 * 4085 + 2048) >> 12;
468 int t49a = (in29 * 3102 + 2048) >> 12;
469 int t50a = (in19 * 3659 + 2048) >> 12;
470 int t51a = (in13 * 3889 + 2048) >> 12;
471 int t52a = (in11 * 3948 + 2048) >> 12;
472 int t53a = (in21 * 3564 + 2048) >> 12;
473 int t54a = (in27 * 3229 + 2048) >> 12;
474 int t55a = (in5 * 4065 + 2048) >> 12;
475 int t56a = (in7 * 4036 + 2048) >> 12;
476 int t57a = (in25 * 3349 + 2048) >> 12;
477 int t58a = (in23 * 3461 + 2048) >> 12;
478 int t59a = (in9 * 3996 + 2048) >> 12;
479 int t60a = (in15 * 3822 + 2048) >> 12;
480 int t61a = (in17 * 3745 + 2048) >> 12;
481 int t62a = (in31 * 2967 + 2048) >> 12;
482 int t63a = (in1 * 4095 + 2048) >> 12;
483
484 int t32 = CLIP(t32a + t33a);
485 int t33 = CLIP(t32a - t33a);
486 int t34 = CLIP(t35a - t34a);
487 int t35 = CLIP(t35a + t34a);
488 int t36 = CLIP(t36a + t37a);
489 int t37 = CLIP(t36a - t37a);
490 int t38 = CLIP(t39a - t38a);
491 int t39 = CLIP(t39a + t38a);
492 int t40 = CLIP(t40a + t41a);
493 int t41 = CLIP(t40a - t41a);
494 int t42 = CLIP(t43a - t42a);
495 int t43 = CLIP(t43a + t42a);
496 int t44 = CLIP(t44a + t45a);
497 int t45 = CLIP(t44a - t45a);
498 int t46 = CLIP(t47a - t46a);
499 int t47 = CLIP(t47a + t46a);
500 int t48 = CLIP(t48a + t49a);
501 int t49 = CLIP(t48a - t49a);
502 int t50 = CLIP(t51a - t50a);
503 int t51 = CLIP(t51a + t50a);
504 int t52 = CLIP(t52a + t53a);
505 int t53 = CLIP(t52a - t53a);
506 int t54 = CLIP(t55a - t54a);
507 int t55 = CLIP(t55a + t54a);
508 int t56 = CLIP(t56a + t57a);
509 int t57 = CLIP(t56a - t57a);
510 int t58 = CLIP(t59a - t58a);
511 int t59 = CLIP(t59a + t58a);
512 int t60 = CLIP(t60a + t61a);
513 int t61 = CLIP(t60a - t61a);
514 int t62 = CLIP(t63a - t62a);
515 int t63 = CLIP(t63a + t62a);
516
517 t33a = ((t33 * (4096 - 4076) + t62 * 401 + 2048) >> 12) - t33;
518 t34a = ((t34 * -401 + t61 * (4096 - 4076) + 2048) >> 12) - t61;
519 t37a = (t37 * -1299 + t58 * 1583 + 1024) >> 11;
520 t38a = (t38 * -1583 + t57 * -1299 + 1024) >> 11;
521 t41a = ((t41 * (4096 - 3612) + t54 * 1931 + 2048) >> 12) - t41;
522 t42a = ((t42 * -1931 + t53 * (4096 - 3612) + 2048) >> 12) - t53;
523 t45a = ((t45 * -1189 + t50 * (3920 - 4096) + 2048) >> 12) + t50;
524 t46a = ((t46 * (4096 - 3920) + t49 * -1189 + 2048) >> 12) - t46;
525 t49a = ((t46 * -1189 + t49 * (3920 - 4096) + 2048) >> 12) + t49;
526 t50a = ((t45 * (3920 - 4096) + t50 * 1189 + 2048) >> 12) + t45;
527 t53a = ((t42 * (4096 - 3612) + t53 * 1931 + 2048) >> 12) - t42;
528 t54a = ((t41 * 1931 + t54 * (3612 - 4096) + 2048) >> 12) + t54;
529 t57a = (t38 * -1299 + t57 * 1583 + 1024) >> 11;
530 t58a = (t37 * 1583 + t58 * 1299 + 1024) >> 11;
531 t61a = ((t34 * (4096 - 4076) + t61 * 401 + 2048) >> 12) - t34;
532 t62a = ((t33 * 401 + t62 * (4076 - 4096) + 2048) >> 12) + t62;
533
534 t32a = CLIP(t32 + t35);
535 t33 = CLIP(t33a + t34a);
536 t34 = CLIP(t33a - t34a);
537 t35a = CLIP(t32 - t35);
538 t36a = CLIP(t39 - t36);
539 t37 = CLIP(t38a - t37a);
540 t38 = CLIP(t38a + t37a);
541 t39a = CLIP(t39 + t36);
542 t40a = CLIP(t40 + t43);
543 t41 = CLIP(t41a + t42a);
544 t42 = CLIP(t41a - t42a);
545 t43a = CLIP(t40 - t43);
546 t44a = CLIP(t47 - t44);
547 t45 = CLIP(t46a - t45a);
548 t46 = CLIP(t46a + t45a);
549 t47a = CLIP(t47 + t44);
550 t48a = CLIP(t48 + t51);
551 t49 = CLIP(t49a + t50a);
552 t50 = CLIP(t49a - t50a);
553 t51a = CLIP(t48 - t51);
554 t52a = CLIP(t55 - t52);
555 t53 = CLIP(t54a - t53a);
556 t54 = CLIP(t54a + t53a);
557 t55a = CLIP(t55 + t52);
558 t56a = CLIP(t56 + t59);
559 t57 = CLIP(t57a + t58a);
560 t58 = CLIP(t57a - t58a);
561 t59a = CLIP(t56 - t59);
562 t60a = CLIP(t63 - t60);
563 t61 = CLIP(t62a - t61a);
564 t62 = CLIP(t62a + t61a);
565 t63a = CLIP(t63 + t60);
566
567 t34a = ((t34 * (4096 - 4017) + t61 * 799 + 2048) >> 12) - t34;
568 t35 = ((t35a * (4096 - 4017) + t60a * 799 + 2048) >> 12) - t35a;
569 t36 = ((t36a * -799 + t59a * (4096 - 4017) + 2048) >> 12) - t59a;
570 t37a = ((t37 * -799 + t58 * (4096 - 4017) + 2048) >> 12) - t58;
571 t42a = (t42 * -1138 + t53 * 1703 + 1024) >> 11;
572 t43 = (t43a * -1138 + t52a * 1703 + 1024) >> 11;
573 t44 = (t44a * -1703 + t51a * -1138 + 1024) >> 11;
574 t45a = (t45 * -1703 + t50 * -1138 + 1024) >> 11;
575 t50a = (t45 * -1138 + t50 * 1703 + 1024) >> 11;
576 t51 = (t44a * -1138 + t51a * 1703 + 1024) >> 11;
577 t52 = (t43a * 1703 + t52a * 1138 + 1024) >> 11;
578 t53a = (t42 * 1703 + t53 * 1138 + 1024) >> 11;
579 t58a = ((t37 * (4096 - 4017) + t58 * 799 + 2048) >> 12) - t37;
580 t59 = ((t36a * (4096 - 4017) + t59a * 799 + 2048) >> 12) - t36a;
581 t60 = ((t35a * 799 + t60a * (4017 - 4096) + 2048) >> 12) + t60a;
582 t61a = ((t34 * 799 + t61 * (4017 - 4096) + 2048) >> 12) + t61;
583
584 t32 = CLIP(t32a + t39a);
585 t33a = CLIP(t33 + t38);
586 t34 = CLIP(t34a + t37a);
587 t35a = CLIP(t35 + t36);
588 t36a = CLIP(t35 - t36);
589 t37 = CLIP(t34a - t37a);
590 t38a = CLIP(t33 - t38);
591 t39 = CLIP(t32a - t39a);
592 t40 = CLIP(t47a - t40a);
593 t41a = CLIP(t46 - t41);
594 t42 = CLIP(t45a - t42a);
595 t43a = CLIP(t44 - t43);
596 t44a = CLIP(t44 + t43);
597 t45 = CLIP(t45a + t42a);
598 t46a = CLIP(t46 + t41);
599 t47 = CLIP(t47a + t40a);
600 t48 = CLIP(t48a + t55a);
601 t49a = CLIP(t49 + t54);
602 t50 = CLIP(t50a + t53a);
603 t51a = CLIP(t51 + t52);
604 t52a = CLIP(t51 - t52);
605 t53 = CLIP(t50a - t53a);
606 t54a = CLIP(t49 - t54);
607 t55 = CLIP(t48a - t55a);
608 t56 = CLIP(t63a - t56a);
609 t57a = CLIP(t62 - t57);
610 t58 = CLIP(t61a - t58a);
611 t59a = CLIP(t60 - t59);
612 t60a = CLIP(t60 + t59);
613 t61 = CLIP(t61a + t58a);
614 t62a = CLIP(t62 + t57);
615 t63 = CLIP(t63a + t56a);
616
617 t36 = ((t36a * (4096 - 3784) + t59a * 1567 + 2048) >> 12) - t36a;
618 t37a = ((t37 * (4096 - 3784) + t58 * 1567 + 2048) >> 12) - t37;
619 t38 = ((t38a * (4096 - 3784) + t57a * 1567 + 2048) >> 12) - t38a;
620 t39a = ((t39 * (4096 - 3784) + t56 * 1567 + 2048) >> 12) - t39;
621 t40a = ((t40 * -1567 + t55 * (4096 - 3784) + 2048) >> 12) - t55;
622 t41 = ((t41a * -1567 + t54a * (4096 - 3784) + 2048) >> 12) - t54a;
623 t42a = ((t42 * -1567 + t53 * (4096 - 3784) + 2048) >> 12) - t53;
624 t43 = ((t43a * -1567 + t52a * (4096 - 3784) + 2048) >> 12) - t52a;
625 t52 = ((t43a * (4096 - 3784) + t52a * 1567 + 2048) >> 12) - t43a;
626 t53a = ((t42 * (4096 - 3784) + t53 * 1567 + 2048) >> 12) - t42;
627 t54 = ((t41a * (4096 - 3784) + t54a * 1567 + 2048) >> 12) - t41a;
628 t55a = ((t40 * (4096 - 3784) + t55 * 1567 + 2048) >> 12) - t40;
629 t56a = ((t39 * 1567 + t56 * (3784 - 4096) + 2048) >> 12) + t56;
630 t57 = ((t38a * 1567 + t57a * (3784 - 4096) + 2048) >> 12) + t57a;
631 t58a = ((t37 * 1567 + t58 * (3784 - 4096) + 2048) >> 12) + t58;
632 t59 = ((t36a * 1567 + t59a * (3784 - 4096) + 2048) >> 12) + t59a;
633
634 t32a = CLIP(t32 + t47);
635 t33 = CLIP(t33a + t46a);
636 t34a = CLIP(t34 + t45);
637 t35 = CLIP(t35a + t44a);
638 t36a = CLIP(t36 + t43);
639 t37 = CLIP(t37a + t42a);
640 t38a = CLIP(t38 + t41);
641 t39 = CLIP(t39a + t40a);
642 t40 = CLIP(t39a - t40a);
643 t41a = CLIP(t38 - t41);
644 t42 = CLIP(t37a - t42a);
645 t43a = CLIP(t36 - t43);
646 t44 = CLIP(t35a - t44a);
647 t45a = CLIP(t34 - t45);
648 t46 = CLIP(t33a - t46a);
649 t47a = CLIP(t32 - t47);
650 t48a = CLIP(t63 - t48);
651 t49 = CLIP(t62a - t49a);
652 t50a = CLIP(t61 - t50);
653 t51 = CLIP(t60a - t51a);
654 t52a = CLIP(t59 - t52);
655 t53 = CLIP(t58a - t53a);
656 t54a = CLIP(t57 - t54);
657 t55 = CLIP(t56a - t55a);
658 t56 = CLIP(t56a + t55a);
659 t57a = CLIP(t57 + t54);
660 t58 = CLIP(t58a + t53a);
661 t59a = CLIP(t59 + t52);
662 t60 = CLIP(t60a + t51a);
663 t61a = CLIP(t61 + t50);
664 t62 = CLIP(t62a + t49a);
665 t63a = CLIP(t63 + t48);
666
667 t40a = ((t55 - t40 ) * 181 + 128) >> 8;
668 t41 = ((t54a - t41a) * 181 + 128) >> 8;
669 t42a = ((t53 - t42 ) * 181 + 128) >> 8;
670 t43 = ((t52a - t43a) * 181 + 128) >> 8;
671 t44a = ((t51 - t44 ) * 181 + 128) >> 8;
672 t45 = ((t50a - t45a) * 181 + 128) >> 8;
673 t46a = ((t49 - t46 ) * 181 + 128) >> 8;
674 t47 = ((t48a - t47a) * 181 + 128) >> 8;
675 t48 = ((t47a + t48a) * 181 + 128) >> 8;
676 t49a = ((t46 + t49 ) * 181 + 128) >> 8;
677 t50 = ((t45a + t50a) * 181 + 128) >> 8;
678 t51a = ((t44 + t51 ) * 181 + 128) >> 8;
679 t52 = ((t43a + t52a) * 181 + 128) >> 8;
680 t53a = ((t42 + t53 ) * 181 + 128) >> 8;
681 t54 = ((t41a + t54a) * 181 + 128) >> 8;
682 t55a = ((t40 + t55 ) * 181 + 128) >> 8;
683
684 const int t0 = c[ 0 * stride];
685 const int t1 = c[ 2 * stride];
686 const int t2 = c[ 4 * stride];
687 const int t3 = c[ 6 * stride];
688 const int t4 = c[ 8 * stride];
689 const int t5 = c[10 * stride];
690 const int t6 = c[12 * stride];
691 const int t7 = c[14 * stride];
692 const int t8 = c[16 * stride];
693 const int t9 = c[18 * stride];
694 const int t10 = c[20 * stride];
695 const int t11 = c[22 * stride];
696 const int t12 = c[24 * stride];
697 const int t13 = c[26 * stride];
698 const int t14 = c[28 * stride];
699 const int t15 = c[30 * stride];
700 const int t16 = c[32 * stride];
701 const int t17 = c[34 * stride];
702 const int t18 = c[36 * stride];
703 const int t19 = c[38 * stride];
704 const int t20 = c[40 * stride];
705 const int t21 = c[42 * stride];
706 const int t22 = c[44 * stride];
707 const int t23 = c[46 * stride];
708 const int t24 = c[48 * stride];
709 const int t25 = c[50 * stride];
710 const int t26 = c[52 * stride];
711 const int t27 = c[54 * stride];
712 const int t28 = c[56 * stride];
713 const int t29 = c[58 * stride];
714 const int t30 = c[60 * stride];
715 const int t31 = c[62 * stride];
716
717 c[ 0 * stride] = CLIP(t0 + t63a);
718 c[ 1 * stride] = CLIP(t1 + t62);
719 c[ 2 * stride] = CLIP(t2 + t61a);
720 c[ 3 * stride] = CLIP(t3 + t60);
721 c[ 4 * stride] = CLIP(t4 + t59a);
722 c[ 5 * stride] = CLIP(t5 + t58);
723 c[ 6 * stride] = CLIP(t6 + t57a);
724 c[ 7 * stride] = CLIP(t7 + t56);
725 c[ 8 * stride] = CLIP(t8 + t55a);
726 c[ 9 * stride] = CLIP(t9 + t54);
727 c[10 * stride] = CLIP(t10 + t53a);
728 c[11 * stride] = CLIP(t11 + t52);
729 c[12 * stride] = CLIP(t12 + t51a);
730 c[13 * stride] = CLIP(t13 + t50);
731 c[14 * stride] = CLIP(t14 + t49a);
732 c[15 * stride] = CLIP(t15 + t48);
733 c[16 * stride] = CLIP(t16 + t47);
734 c[17 * stride] = CLIP(t17 + t46a);
735 c[18 * stride] = CLIP(t18 + t45);
736 c[19 * stride] = CLIP(t19 + t44a);
737 c[20 * stride] = CLIP(t20 + t43);
738 c[21 * stride] = CLIP(t21 + t42a);
739 c[22 * stride] = CLIP(t22 + t41);
740 c[23 * stride] = CLIP(t23 + t40a);
741 c[24 * stride] = CLIP(t24 + t39);
742 c[25 * stride] = CLIP(t25 + t38a);
743 c[26 * stride] = CLIP(t26 + t37);
744 c[27 * stride] = CLIP(t27 + t36a);
745 c[28 * stride] = CLIP(t28 + t35);
746 c[29 * stride] = CLIP(t29 + t34a);
747 c[30 * stride] = CLIP(t30 + t33);
748 c[31 * stride] = CLIP(t31 + t32a);
749 c[32 * stride] = CLIP(t31 - t32a);
750 c[33 * stride] = CLIP(t30 - t33);
751 c[34 * stride] = CLIP(t29 - t34a);
752 c[35 * stride] = CLIP(t28 - t35);
753 c[36 * stride] = CLIP(t27 - t36a);
754 c[37 * stride] = CLIP(t26 - t37);
755 c[38 * stride] = CLIP(t25 - t38a);
756 c[39 * stride] = CLIP(t24 - t39);
757 c[40 * stride] = CLIP(t23 - t40a);
758 c[41 * stride] = CLIP(t22 - t41);
759 c[42 * stride] = CLIP(t21 - t42a);
760 c[43 * stride] = CLIP(t20 - t43);
761 c[44 * stride] = CLIP(t19 - t44a);
762 c[45 * stride] = CLIP(t18 - t45);
763 c[46 * stride] = CLIP(t17 - t46a);
764 c[47 * stride] = CLIP(t16 - t47);
765 c[48 * stride] = CLIP(t15 - t48);
766 c[49 * stride] = CLIP(t14 - t49a);
767 c[50 * stride] = CLIP(t13 - t50);
768 c[51 * stride] = CLIP(t12 - t51a);
769 c[52 * stride] = CLIP(t11 - t52);
770 c[53 * stride] = CLIP(t10 - t53a);
771 c[54 * stride] = CLIP(t9 - t54);
772 c[55 * stride] = CLIP(t8 - t55a);
773 c[56 * stride] = CLIP(t7 - t56);
774 c[57 * stride] = CLIP(t6 - t57a);
775 c[58 * stride] = CLIP(t5 - t58);
776 c[59 * stride] = CLIP(t4 - t59a);
777 c[60 * stride] = CLIP(t3 - t60);
778 c[61 * stride] = CLIP(t2 - t61a);
779 c[62 * stride] = CLIP(t1 - t62);
780 c[63 * stride] = CLIP(t0 - t63a);
781 }
782
783 static NOINLINE void
inv_adst4_1d_internal_c(const int32_t * const in,const ptrdiff_t in_s,const int min,const int max,int32_t * const out,const ptrdiff_t out_s)784 inv_adst4_1d_internal_c(const int32_t *const in, const ptrdiff_t in_s,
785 const int min, const int max,
786 int32_t *const out, const ptrdiff_t out_s)
787 {
788 assert(in_s > 0 && out_s != 0);
789 const int in0 = in[0 * in_s], in1 = in[1 * in_s];
790 const int in2 = in[2 * in_s], in3 = in[3 * in_s];
791
792 out[0 * out_s] = (( 1321 * in0 + (3803 - 4096) * in2 +
793 (2482 - 4096) * in3 + (3344 - 4096) * in1 + 2048) >> 12) +
794 in2 + in3 + in1;
795 out[1 * out_s] = (((2482 - 4096) * in0 - 1321 * in2 -
796 (3803 - 4096) * in3 + (3344 - 4096) * in1 + 2048) >> 12) +
797 in0 - in3 + in1;
798 out[2 * out_s] = (209 * (in0 - in2 + in3) + 128) >> 8;
799 out[3 * out_s] = (((3803 - 4096) * in0 + (2482 - 4096) * in2 -
800 1321 * in3 - (3344 - 4096) * in1 + 2048) >> 12) +
801 in0 + in2 - in1;
802 }
803
804 static NOINLINE void
inv_adst8_1d_internal_c(const int32_t * const in,const ptrdiff_t in_s,const int min,const int max,int32_t * const out,const ptrdiff_t out_s)805 inv_adst8_1d_internal_c(const int32_t *const in, const ptrdiff_t in_s,
806 const int min, const int max,
807 int32_t *const out, const ptrdiff_t out_s)
808 {
809 assert(in_s > 0 && out_s != 0);
810 const int in0 = in[0 * in_s], in1 = in[1 * in_s];
811 const int in2 = in[2 * in_s], in3 = in[3 * in_s];
812 const int in4 = in[4 * in_s], in5 = in[5 * in_s];
813 const int in6 = in[6 * in_s], in7 = in[7 * in_s];
814
815 const int t0a = (((4076 - 4096) * in7 + 401 * in0 + 2048) >> 12) + in7;
816 const int t1a = (( 401 * in7 - (4076 - 4096) * in0 + 2048) >> 12) - in0;
817 const int t2a = (((3612 - 4096) * in5 + 1931 * in2 + 2048) >> 12) + in5;
818 const int t3a = (( 1931 * in5 - (3612 - 4096) * in2 + 2048) >> 12) - in2;
819 int t4a = ( 1299 * in3 + 1583 * in4 + 1024) >> 11;
820 int t5a = ( 1583 * in3 - 1299 * in4 + 1024) >> 11;
821 int t6a = (( 1189 * in1 + (3920 - 4096) * in6 + 2048) >> 12) + in6;
822 int t7a = (((3920 - 4096) * in1 - 1189 * in6 + 2048) >> 12) + in1;
823
824 const int t0 = CLIP(t0a + t4a);
825 const int t1 = CLIP(t1a + t5a);
826 int t2 = CLIP(t2a + t6a);
827 int t3 = CLIP(t3a + t7a);
828 const int t4 = CLIP(t0a - t4a);
829 const int t5 = CLIP(t1a - t5a);
830 int t6 = CLIP(t2a - t6a);
831 int t7 = CLIP(t3a - t7a);
832
833 t4a = (((3784 - 4096) * t4 + 1567 * t5 + 2048) >> 12) + t4;
834 t5a = (( 1567 * t4 - (3784 - 4096) * t5 + 2048) >> 12) - t5;
835 t6a = (((3784 - 4096) * t7 - 1567 * t6 + 2048) >> 12) + t7;
836 t7a = (( 1567 * t7 + (3784 - 4096) * t6 + 2048) >> 12) + t6;
837
838 out[0 * out_s] = CLIP(t0 + t2 );
839 out[7 * out_s] = -CLIP(t1 + t3 );
840 t2 = CLIP(t0 - t2 );
841 t3 = CLIP(t1 - t3 );
842 out[1 * out_s] = -CLIP(t4a + t6a);
843 out[6 * out_s] = CLIP(t5a + t7a);
844 t6 = CLIP(t4a - t6a);
845 t7 = CLIP(t5a - t7a);
846
847 out[3 * out_s] = -(((t2 + t3) * 181 + 128) >> 8);
848 out[4 * out_s] = ((t2 - t3) * 181 + 128) >> 8;
849 out[2 * out_s] = ((t6 + t7) * 181 + 128) >> 8;
850 out[5 * out_s] = -(((t6 - t7) * 181 + 128) >> 8);
851 }
852
853 static NOINLINE void
inv_adst16_1d_internal_c(const int32_t * const in,const ptrdiff_t in_s,const int min,const int max,int32_t * const out,const ptrdiff_t out_s)854 inv_adst16_1d_internal_c(const int32_t *const in, const ptrdiff_t in_s,
855 const int min, const int max,
856 int32_t *const out, const ptrdiff_t out_s)
857 {
858 assert(in_s > 0 && out_s != 0);
859 const int in0 = in[ 0 * in_s], in1 = in[ 1 * in_s];
860 const int in2 = in[ 2 * in_s], in3 = in[ 3 * in_s];
861 const int in4 = in[ 4 * in_s], in5 = in[ 5 * in_s];
862 const int in6 = in[ 6 * in_s], in7 = in[ 7 * in_s];
863 const int in8 = in[ 8 * in_s], in9 = in[ 9 * in_s];
864 const int in10 = in[10 * in_s], in11 = in[11 * in_s];
865 const int in12 = in[12 * in_s], in13 = in[13 * in_s];
866 const int in14 = in[14 * in_s], in15 = in[15 * in_s];
867
868 int t0 = ((in15 * (4091 - 4096) + in0 * 201 + 2048) >> 12) + in15;
869 int t1 = ((in15 * 201 - in0 * (4091 - 4096) + 2048) >> 12) - in0;
870 int t2 = ((in13 * (3973 - 4096) + in2 * 995 + 2048) >> 12) + in13;
871 int t3 = ((in13 * 995 - in2 * (3973 - 4096) + 2048) >> 12) - in2;
872 int t4 = ((in11 * (3703 - 4096) + in4 * 1751 + 2048) >> 12) + in11;
873 int t5 = ((in11 * 1751 - in4 * (3703 - 4096) + 2048) >> 12) - in4;
874 int t6 = (in9 * 1645 + in6 * 1220 + 1024) >> 11;
875 int t7 = (in9 * 1220 - in6 * 1645 + 1024) >> 11;
876 int t8 = ((in7 * 2751 + in8 * (3035 - 4096) + 2048) >> 12) + in8;
877 int t9 = ((in7 * (3035 - 4096) - in8 * 2751 + 2048) >> 12) + in7;
878 int t10 = ((in5 * 2106 + in10 * (3513 - 4096) + 2048) >> 12) + in10;
879 int t11 = ((in5 * (3513 - 4096) - in10 * 2106 + 2048) >> 12) + in5;
880 int t12 = ((in3 * 1380 + in12 * (3857 - 4096) + 2048) >> 12) + in12;
881 int t13 = ((in3 * (3857 - 4096) - in12 * 1380 + 2048) >> 12) + in3;
882 int t14 = ((in1 * 601 + in14 * (4052 - 4096) + 2048) >> 12) + in14;
883 int t15 = ((in1 * (4052 - 4096) - in14 * 601 + 2048) >> 12) + in1;
884
885 int t0a = CLIP(t0 + t8 );
886 int t1a = CLIP(t1 + t9 );
887 int t2a = CLIP(t2 + t10);
888 int t3a = CLIP(t3 + t11);
889 int t4a = CLIP(t4 + t12);
890 int t5a = CLIP(t5 + t13);
891 int t6a = CLIP(t6 + t14);
892 int t7a = CLIP(t7 + t15);
893 int t8a = CLIP(t0 - t8 );
894 int t9a = CLIP(t1 - t9 );
895 int t10a = CLIP(t2 - t10);
896 int t11a = CLIP(t3 - t11);
897 int t12a = CLIP(t4 - t12);
898 int t13a = CLIP(t5 - t13);
899 int t14a = CLIP(t6 - t14);
900 int t15a = CLIP(t7 - t15);
901
902 t8 = ((t8a * (4017 - 4096) + t9a * 799 + 2048) >> 12) + t8a;
903 t9 = ((t8a * 799 - t9a * (4017 - 4096) + 2048) >> 12) - t9a;
904 t10 = ((t10a * 2276 + t11a * (3406 - 4096) + 2048) >> 12) + t11a;
905 t11 = ((t10a * (3406 - 4096) - t11a * 2276 + 2048) >> 12) + t10a;
906 t12 = ((t13a * (4017 - 4096) - t12a * 799 + 2048) >> 12) + t13a;
907 t13 = ((t13a * 799 + t12a * (4017 - 4096) + 2048) >> 12) + t12a;
908 t14 = ((t15a * 2276 - t14a * (3406 - 4096) + 2048) >> 12) - t14a;
909 t15 = ((t15a * (3406 - 4096) + t14a * 2276 + 2048) >> 12) + t15a;
910
911 t0 = CLIP(t0a + t4a);
912 t1 = CLIP(t1a + t5a);
913 t2 = CLIP(t2a + t6a);
914 t3 = CLIP(t3a + t7a);
915 t4 = CLIP(t0a - t4a);
916 t5 = CLIP(t1a - t5a);
917 t6 = CLIP(t2a - t6a);
918 t7 = CLIP(t3a - t7a);
919 t8a = CLIP(t8 + t12);
920 t9a = CLIP(t9 + t13);
921 t10a = CLIP(t10 + t14);
922 t11a = CLIP(t11 + t15);
923 t12a = CLIP(t8 - t12);
924 t13a = CLIP(t9 - t13);
925 t14a = CLIP(t10 - t14);
926 t15a = CLIP(t11 - t15);
927
928 t4a = ((t4 * (3784 - 4096) + t5 * 1567 + 2048) >> 12) + t4;
929 t5a = ((t4 * 1567 - t5 * (3784 - 4096) + 2048) >> 12) - t5;
930 t6a = ((t7 * (3784 - 4096) - t6 * 1567 + 2048) >> 12) + t7;
931 t7a = ((t7 * 1567 + t6 * (3784 - 4096) + 2048) >> 12) + t6;
932 t12 = ((t12a * (3784 - 4096) + t13a * 1567 + 2048) >> 12) + t12a;
933 t13 = ((t12a * 1567 - t13a * (3784 - 4096) + 2048) >> 12) - t13a;
934 t14 = ((t15a * (3784 - 4096) - t14a * 1567 + 2048) >> 12) + t15a;
935 t15 = ((t15a * 1567 + t14a * (3784 - 4096) + 2048) >> 12) + t14a;
936
937 out[ 0 * out_s] = CLIP(t0 + t2 );
938 out[15 * out_s] = -CLIP(t1 + t3 );
939 t2a = CLIP(t0 - t2 );
940 t3a = CLIP(t1 - t3 );
941 out[ 3 * out_s] = -CLIP(t4a + t6a );
942 out[12 * out_s] = CLIP(t5a + t7a );
943 t6 = CLIP(t4a - t6a );
944 t7 = CLIP(t5a - t7a );
945 out[ 1 * out_s] = -CLIP(t8a + t10a);
946 out[14 * out_s] = CLIP(t9a + t11a);
947 t10 = CLIP(t8a - t10a);
948 t11 = CLIP(t9a - t11a);
949 out[ 2 * out_s] = CLIP(t12 + t14 );
950 out[13 * out_s] = -CLIP(t13 + t15 );
951 t14a = CLIP(t12 - t14 );
952 t15a = CLIP(t13 - t15 );
953
954 out[ 7 * out_s] = -(((t2a + t3a) * 181 + 128) >> 8);
955 out[ 8 * out_s] = ((t2a - t3a) * 181 + 128) >> 8;
956 out[ 4 * out_s] = ((t6 + t7) * 181 + 128) >> 8;
957 out[11 * out_s] = -(((t6 - t7) * 181 + 128) >> 8);
958 out[ 6 * out_s] = ((t10 + t11) * 181 + 128) >> 8;
959 out[ 9 * out_s] = -(((t10 - t11) * 181 + 128) >> 8);
960 out[ 5 * out_s] = -(((t14a + t15a) * 181 + 128) >> 8);
961 out[10 * out_s] = ((t14a - t15a) * 181 + 128) >> 8;
962 }
963
964 #define inv_adst_1d(sz) \
965 static void inv_adst##sz##_1d_c(int32_t *const c, const ptrdiff_t stride, \
966 const int min, const int max) \
967 { \
968 inv_adst##sz##_1d_internal_c(c, stride, min, max, c, stride); \
969 } \
970 static void inv_flipadst##sz##_1d_c(int32_t *const c, const ptrdiff_t stride, \
971 const int min, const int max) \
972 { \
973 inv_adst##sz##_1d_internal_c(c, stride, min, max, \
974 &c[(sz - 1) * stride], -stride); \
975 }
976
977 inv_adst_1d( 4)
978 inv_adst_1d( 8)
979 inv_adst_1d(16)
980
981 #undef inv_adst_1d
982
inv_identity4_1d_c(int32_t * const c,const ptrdiff_t stride,const int min,const int max)983 static void inv_identity4_1d_c(int32_t *const c, const ptrdiff_t stride,
984 const int min, const int max)
985 {
986 assert(stride > 0);
987 for (int i = 0; i < 4; i++) {
988 const int in = c[stride * i];
989 c[stride * i] = in + ((in * 1697 + 2048) >> 12);
990 }
991 }
992
inv_identity8_1d_c(int32_t * const c,const ptrdiff_t stride,const int min,const int max)993 static void inv_identity8_1d_c(int32_t *const c, const ptrdiff_t stride,
994 const int min, const int max)
995 {
996 assert(stride > 0);
997 for (int i = 0; i < 8; i++)
998 c[stride * i] *= 2;
999 }
1000
inv_identity16_1d_c(int32_t * const c,const ptrdiff_t stride,const int min,const int max)1001 static void inv_identity16_1d_c(int32_t *const c, const ptrdiff_t stride,
1002 const int min, const int max)
1003 {
1004 assert(stride > 0);
1005 for (int i = 0; i < 16; i++) {
1006 const int in = c[stride * i];
1007 c[stride * i] = 2 * in + ((in * 1697 + 1024) >> 11);
1008 }
1009 }
1010
inv_identity32_1d_c(int32_t * const c,const ptrdiff_t stride,const int min,const int max)1011 static void inv_identity32_1d_c(int32_t *const c, const ptrdiff_t stride,
1012 const int min, const int max)
1013 {
1014 assert(stride > 0);
1015 for (int i = 0; i < 32; i++)
1016 c[stride * i] *= 4;
1017 }
1018
1019 const itx_1d_fn dav1d_tx1d_fns[N_TX_SIZES][N_TX_1D_TYPES] = {
1020 [TX_4X4] = {
1021 [DCT] = inv_dct4_1d_c,
1022 [ADST] = inv_adst4_1d_c,
1023 [FLIPADST] = inv_flipadst4_1d_c,
1024 [IDENTITY] = inv_identity4_1d_c,
1025 }, [TX_8X8] = {
1026 [DCT] = inv_dct8_1d_c,
1027 [ADST] = inv_adst8_1d_c,
1028 [FLIPADST] = inv_flipadst8_1d_c,
1029 [IDENTITY] = inv_identity8_1d_c,
1030 }, [TX_16X16] = {
1031 [DCT] = inv_dct16_1d_c,
1032 [ADST] = inv_adst16_1d_c,
1033 [FLIPADST] = inv_flipadst16_1d_c,
1034 [IDENTITY] = inv_identity16_1d_c,
1035 }, [TX_32X32] = {
1036 [DCT] = inv_dct32_1d_c,
1037 [IDENTITY] = inv_identity32_1d_c,
1038 }, [TX_64X64] = {
1039 [DCT] = inv_dct64_1d_c,
1040 },
1041 };
1042
1043 const uint8_t /* enum Tx1dType */ dav1d_tx1d_types[N_TX_TYPES][2] = {
1044 [DCT_DCT] = { DCT, DCT },
1045 [ADST_DCT] = { ADST, DCT },
1046 [DCT_ADST] = { DCT, ADST },
1047 [ADST_ADST] = { ADST, ADST },
1048 [FLIPADST_DCT] = { FLIPADST, DCT },
1049 [DCT_FLIPADST] = { DCT, FLIPADST },
1050 [FLIPADST_FLIPADST] = { FLIPADST, FLIPADST },
1051 [ADST_FLIPADST] = { ADST, FLIPADST },
1052 [FLIPADST_ADST] = { FLIPADST, ADST },
1053 [IDTX] = { IDENTITY, IDENTITY },
1054 [V_DCT] = { DCT, IDENTITY },
1055 [H_DCT] = { IDENTITY, DCT },
1056 [V_ADST] = { ADST, IDENTITY },
1057 [H_ADST] = { IDENTITY, ADST },
1058 [V_FLIPADST] = { FLIPADST, IDENTITY },
1059 [H_FLIPADST] = { IDENTITY, FLIPADST },
1060 };
1061
1062 #if !(HAVE_ASM && TRIM_DSP_FUNCTIONS && ( \
1063 ARCH_AARCH64 || \
1064 (ARCH_ARM && (defined(__ARM_NEON) || defined(__APPLE__) || defined(_WIN32))) \
1065 ))
dav1d_inv_wht4_1d_c(int32_t * const c,const ptrdiff_t stride)1066 void dav1d_inv_wht4_1d_c(int32_t *const c, const ptrdiff_t stride) {
1067 assert(stride > 0);
1068 const int in0 = c[0 * stride], in1 = c[1 * stride];
1069 const int in2 = c[2 * stride], in3 = c[3 * stride];
1070
1071 const int t0 = in0 + in1;
1072 const int t2 = in2 - in3;
1073 const int t4 = (t0 - t2) >> 1;
1074 const int t3 = t4 - in3;
1075 const int t1 = t4 - in1;
1076
1077 c[0 * stride] = t0 - t3;
1078 c[1 * stride] = t3;
1079 c[2 * stride] = t1;
1080 c[3 * stride] = t2 + t1;
1081 }
1082 #endif
1083