1 /*
2 * Copyright (c) 2018, Alliance for Open Media. All rights reserved.
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #include <emmintrin.h>
13
14 #include "config/av1_rtcd.h"
15
16 #include "aom_dsp/aom_filter.h"
17 #include "aom_dsp/x86/convolve_sse2.h"
18 #include "aom_dsp/x86/synonyms.h"
19
av1_dist_wtd_convolve_x_sse2(const uint8_t * src,int src_stride,uint8_t * dst0,int dst_stride0,int w,int h,const InterpFilterParams * filter_params_x,const int subpel_x_qn,ConvolveParams * conv_params)20 void av1_dist_wtd_convolve_x_sse2(const uint8_t *src, int src_stride,
21 uint8_t *dst0, int dst_stride0, int w, int h,
22 const InterpFilterParams *filter_params_x,
23 const int subpel_x_qn,
24 ConvolveParams *conv_params) {
25 const int bd = 8;
26 CONV_BUF_TYPE *dst = conv_params->dst;
27 const int dst_stride = conv_params->dst_stride;
28 const int fo_horiz = filter_params_x->taps / 2 - 1;
29 const uint8_t *src_ptr = src - fo_horiz;
30 const int bits = FILTER_BITS - conv_params->round_1;
31 const __m128i left_shift = _mm_cvtsi32_si128(bits);
32 const __m128i round_const = _mm_set1_epi32((1 << conv_params->round_0) >> 1);
33 const __m128i round_shift = _mm_cvtsi32_si128(conv_params->round_0);
34 const int w0 = conv_params->fwd_offset;
35 const int w1 = conv_params->bck_offset;
36 const __m128i wt0 = _mm_set1_epi16(w0);
37 const __m128i wt1 = _mm_set1_epi16(w1);
38 const __m128i wt = _mm_unpacklo_epi16(wt0, wt1);
39 const int do_average = conv_params->do_average;
40 const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg;
41 const int offset_0 =
42 bd + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
43 const int offset = (1 << offset_0) + (1 << (offset_0 - 1));
44 const __m128i offset_const = _mm_set1_epi16(offset);
45 const int rounding_shift =
46 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
47 const __m128i rounding_const = _mm_set1_epi16((1 << rounding_shift) >> 1);
48 __m128i coeffs[4];
49
50 prepare_coeffs(filter_params_x, subpel_x_qn, coeffs);
51
52 if (w == 4) {
53 do {
54 const __m128i data = _mm_loadu_si128((__m128i *)src_ptr);
55 __m128i s[4];
56
57 s[0] = _mm_unpacklo_epi8(data, _mm_srli_si128(data, 1));
58 s[1] =
59 _mm_unpacklo_epi8(_mm_srli_si128(data, 2), _mm_srli_si128(data, 3));
60 s[2] =
61 _mm_unpacklo_epi8(_mm_srli_si128(data, 4), _mm_srli_si128(data, 5));
62 s[3] =
63 _mm_unpacklo_epi8(_mm_srli_si128(data, 6), _mm_srli_si128(data, 7));
64 const __m128i res_lo = convolve_lo_x(s, coeffs);
65 const __m128i res_lo_round =
66 _mm_sra_epi32(_mm_add_epi32(res_lo, round_const), round_shift);
67 const __m128i res_lo_shift = _mm_sll_epi32(res_lo_round, left_shift);
68
69 const __m128i res_16b = _mm_packs_epi32(res_lo_shift, res_lo_shift);
70 const __m128i res_unsigned = _mm_add_epi16(res_16b, offset_const);
71
72 // Accumulate values into the destination buffer
73 if (do_average) {
74 const __m128i data_ref_0 = _mm_loadu_si128((__m128i *)dst);
75
76 const __m128i comp_avg_res =
77 comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg);
78
79 const __m128i round_result = convolve_rounding(
80 &comp_avg_res, &offset_const, &rounding_const, rounding_shift);
81
82 const __m128i res_8 = _mm_packus_epi16(round_result, round_result);
83 *(int *)(&dst0[0]) = _mm_cvtsi128_si32(res_8);
84 } else {
85 _mm_store_si128((__m128i *)(&dst[0]), res_unsigned);
86 }
87 src_ptr += src_stride;
88 dst += dst_stride;
89 dst0 += dst_stride0;
90 } while (--h);
91 } else {
92 assert(!(w % 8));
93 int i = 0;
94 do {
95 int j = 0;
96 do {
97 const __m128i data =
98 _mm_loadu_si128((__m128i *)&src_ptr[i * src_stride + j]);
99 __m128i s[4];
100
101 // Filter even-index pixels
102 s[0] = data;
103 s[1] = _mm_srli_si128(data, 2);
104 s[2] = _mm_srli_si128(data, 4);
105 s[3] = _mm_srli_si128(data, 6);
106 const __m128i res_even = convolve_lo_x(s, coeffs);
107
108 // Filter odd-index pixels
109 s[0] = _mm_srli_si128(data, 1);
110 s[1] = _mm_srli_si128(data, 3);
111 s[2] = _mm_srli_si128(data, 5);
112 s[3] = _mm_srli_si128(data, 7);
113 const __m128i res_odd = convolve_lo_x(s, coeffs);
114
115 // Rearrange pixels back into the order 0 ... 7
116 const __m128i res_lo = _mm_unpacklo_epi32(res_even, res_odd);
117 const __m128i res_hi = _mm_unpackhi_epi32(res_even, res_odd);
118 const __m128i res_lo_round =
119 _mm_sra_epi32(_mm_add_epi32(res_lo, round_const), round_shift);
120 const __m128i res_hi_round =
121 _mm_sra_epi32(_mm_add_epi32(res_hi, round_const), round_shift);
122 const __m128i res_lo_shift = _mm_sll_epi32(res_lo_round, left_shift);
123 const __m128i res_hi_shift = _mm_sll_epi32(res_hi_round, left_shift);
124
125 const __m128i res_16b = _mm_packs_epi32(res_lo_shift, res_hi_shift);
126 const __m128i res_unsigned = _mm_add_epi16(res_16b, offset_const);
127
128 // Accumulate values into the destination buffer
129 if (do_average) {
130 const __m128i data_ref_0 =
131 _mm_loadu_si128((__m128i *)(&dst[i * dst_stride + j]));
132
133 const __m128i comp_avg_res =
134 comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg);
135
136 const __m128i round_result = convolve_rounding(
137 &comp_avg_res, &offset_const, &rounding_const, rounding_shift);
138
139 const __m128i res_8 = _mm_packus_epi16(round_result, round_result);
140 _mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_8);
141 } else {
142 _mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_unsigned);
143 }
144 j += 8;
145 } while (j < w);
146 } while (++i < h);
147 }
148 }
149
av1_dist_wtd_convolve_y_sse2(const uint8_t * src,int src_stride,uint8_t * dst0,int dst_stride0,int w,int h,const InterpFilterParams * filter_params_y,const int subpel_y_qn,ConvolveParams * conv_params)150 void av1_dist_wtd_convolve_y_sse2(const uint8_t *src, int src_stride,
151 uint8_t *dst0, int dst_stride0, int w, int h,
152 const InterpFilterParams *filter_params_y,
153 const int subpel_y_qn,
154 ConvolveParams *conv_params) {
155 const int bd = 8;
156 CONV_BUF_TYPE *dst = conv_params->dst;
157 const int dst_stride = conv_params->dst_stride;
158 const int fo_vert = filter_params_y->taps / 2 - 1;
159 const uint8_t *src_ptr = src - fo_vert * src_stride;
160 const int bits = FILTER_BITS - conv_params->round_0;
161 const __m128i left_shift = _mm_cvtsi32_si128(bits);
162 const __m128i wt0 = _mm_set1_epi16(conv_params->fwd_offset);
163 const __m128i wt1 = _mm_set1_epi16(conv_params->bck_offset);
164 const __m128i wt = _mm_unpacklo_epi16(wt0, wt1);
165 const int do_average = conv_params->do_average;
166 const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg;
167 const int offset_0 =
168 bd + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
169 const int offset = (1 << offset_0) + (1 << (offset_0 - 1));
170 const __m128i offset_const = _mm_set1_epi16(offset);
171 const int rounding_shift =
172 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
173 const __m128i rounding_const = _mm_set1_epi16((1 << rounding_shift) >> 1);
174 const __m128i round_const = _mm_set1_epi32((1 << conv_params->round_1) >> 1);
175 const __m128i round_shift = _mm_cvtsi32_si128(conv_params->round_1);
176 __m128i coeffs[4];
177
178 prepare_coeffs(filter_params_y, subpel_y_qn, coeffs);
179
180 if (w == 4) {
181 __m128i s[8], src6, res, res_shift;
182 s[0] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 0 * src_stride),
183 xx_loadl_32(src_ptr + 1 * src_stride));
184 s[1] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 1 * src_stride),
185 xx_loadl_32(src_ptr + 2 * src_stride));
186 s[2] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 2 * src_stride),
187 xx_loadl_32(src_ptr + 3 * src_stride));
188 s[3] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 3 * src_stride),
189 xx_loadl_32(src_ptr + 4 * src_stride));
190 s[4] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 4 * src_stride),
191 xx_loadl_32(src_ptr + 5 * src_stride));
192 src6 = xx_loadl_32(src_ptr + 6 * src_stride);
193 s[5] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 5 * src_stride), src6);
194
195 do {
196 s[6] = _mm_unpacklo_epi8(src6, xx_loadl_32(src_ptr + 7 * src_stride));
197 src6 = xx_loadl_32(src_ptr + 8 * src_stride);
198 s[7] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 7 * src_stride), src6);
199
200 res = convolve_lo_y(s + 0, coeffs);
201 res_shift = _mm_sll_epi32(res, left_shift);
202 res_shift =
203 _mm_sra_epi32(_mm_add_epi32(res_shift, round_const), round_shift);
204
205 __m128i res_16b = _mm_packs_epi32(res_shift, res_shift);
206 __m128i res_unsigned = _mm_add_epi16(res_16b, offset_const);
207
208 // Accumulate values into the destination buffer
209 if (do_average) {
210 const __m128i data_ref_0 = _mm_loadu_si128((__m128i *)dst);
211
212 const __m128i comp_avg_res =
213 comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg);
214
215 const __m128i round_result = convolve_rounding(
216 &comp_avg_res, &offset_const, &rounding_const, rounding_shift);
217
218 const __m128i res_8 = _mm_packus_epi16(round_result, round_result);
219 *(int *)(&dst0[0]) = _mm_cvtsi128_si32(res_8);
220
221 } else {
222 _mm_store_si128((__m128i *)dst, res_unsigned);
223 }
224
225 src_ptr += src_stride;
226 dst += dst_stride;
227 dst0 += dst_stride0;
228
229 res = convolve_lo_y(s + 1, coeffs);
230 res_shift = _mm_sll_epi32(res, left_shift);
231 res_shift =
232 _mm_sra_epi32(_mm_add_epi32(res_shift, round_const), round_shift);
233
234 res_16b = _mm_packs_epi32(res_shift, res_shift);
235 res_unsigned = _mm_add_epi16(res_16b, offset_const);
236
237 // Accumulate values into the destination buffer
238 if (do_average) {
239 const __m128i data_ref_0 = _mm_loadu_si128((__m128i *)dst);
240
241 const __m128i comp_avg_res =
242 comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg);
243
244 const __m128i round_result = convolve_rounding(
245 &comp_avg_res, &offset_const, &rounding_const, rounding_shift);
246
247 const __m128i res_8 = _mm_packus_epi16(round_result, round_result);
248 *(int *)(&dst0[0]) = _mm_cvtsi128_si32(res_8);
249
250 } else {
251 _mm_store_si128((__m128i *)dst, res_unsigned);
252 }
253
254 src_ptr += src_stride;
255 dst += dst_stride;
256 dst0 += dst_stride0;
257
258 s[0] = s[2];
259 s[1] = s[3];
260 s[2] = s[4];
261 s[3] = s[5];
262 s[4] = s[6];
263 s[5] = s[7];
264 h -= 2;
265 } while (h);
266 } else {
267 assert(!(w % 8));
268 int j = 0;
269 do {
270 __m128i s[8], src6, res_lo, res_hi, res_lo_shift, res_hi_shift;
271 const uint8_t *data = &src_ptr[j];
272
273 src6 = _mm_loadl_epi64((__m128i *)(data + 6 * src_stride));
274 s[0] = _mm_unpacklo_epi8(
275 _mm_loadl_epi64((__m128i *)(data + 0 * src_stride)),
276 _mm_loadl_epi64((__m128i *)(data + 1 * src_stride)));
277 s[1] = _mm_unpacklo_epi8(
278 _mm_loadl_epi64((__m128i *)(data + 1 * src_stride)),
279 _mm_loadl_epi64((__m128i *)(data + 2 * src_stride)));
280 s[2] = _mm_unpacklo_epi8(
281 _mm_loadl_epi64((__m128i *)(data + 2 * src_stride)),
282 _mm_loadl_epi64((__m128i *)(data + 3 * src_stride)));
283 s[3] = _mm_unpacklo_epi8(
284 _mm_loadl_epi64((__m128i *)(data + 3 * src_stride)),
285 _mm_loadl_epi64((__m128i *)(data + 4 * src_stride)));
286 s[4] = _mm_unpacklo_epi8(
287 _mm_loadl_epi64((__m128i *)(data + 4 * src_stride)),
288 _mm_loadl_epi64((__m128i *)(data + 5 * src_stride)));
289 s[5] = _mm_unpacklo_epi8(
290 _mm_loadl_epi64((__m128i *)(data + 5 * src_stride)), src6);
291
292 int i = 0;
293 do {
294 data = &src_ptr[i * src_stride + j];
295 s[6] = _mm_unpacklo_epi8(
296 src6, _mm_loadl_epi64((__m128i *)(data + 7 * src_stride)));
297 src6 = _mm_loadl_epi64((__m128i *)(data + 8 * src_stride));
298 s[7] = _mm_unpacklo_epi8(
299 _mm_loadl_epi64((__m128i *)(data + 7 * src_stride)), src6);
300
301 res_lo = convolve_lo_y(s, coeffs); // Filter low index pixels
302 res_hi = convolve_hi_y(s, coeffs); // Filter high index pixels
303 res_lo_shift = _mm_sll_epi32(res_lo, left_shift);
304 res_hi_shift = _mm_sll_epi32(res_hi, left_shift);
305 res_lo_shift = _mm_sra_epi32(_mm_add_epi32(res_lo_shift, round_const),
306 round_shift);
307 res_hi_shift = _mm_sra_epi32(_mm_add_epi32(res_hi_shift, round_const),
308 round_shift);
309
310 __m128i res_16b = _mm_packs_epi32(res_lo_shift, res_hi_shift);
311 __m128i res_unsigned = _mm_add_epi16(res_16b, offset_const);
312
313 // Accumulate values into the destination buffer
314 if (do_average) {
315 const __m128i data_ref_0 =
316 _mm_loadu_si128((__m128i *)(&dst[i * dst_stride + j]));
317
318 const __m128i comp_avg_res =
319 comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg);
320
321 const __m128i round_result = convolve_rounding(
322 &comp_avg_res, &offset_const, &rounding_const, rounding_shift);
323
324 const __m128i res_8 = _mm_packus_epi16(round_result, round_result);
325 _mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_8);
326 } else {
327 _mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_unsigned);
328 }
329 i++;
330
331 res_lo = convolve_lo_y(s + 1, coeffs); // Filter low index pixels
332 res_hi = convolve_hi_y(s + 1, coeffs); // Filter high index pixels
333 res_lo_shift = _mm_sll_epi32(res_lo, left_shift);
334 res_hi_shift = _mm_sll_epi32(res_hi, left_shift);
335 res_lo_shift = _mm_sra_epi32(_mm_add_epi32(res_lo_shift, round_const),
336 round_shift);
337 res_hi_shift = _mm_sra_epi32(_mm_add_epi32(res_hi_shift, round_const),
338 round_shift);
339 res_16b = _mm_packs_epi32(res_lo_shift, res_hi_shift);
340 res_unsigned = _mm_add_epi16(res_16b, offset_const);
341
342 // Accumulate values into the destination buffer
343 if (do_average) {
344 __m128i data_ref_0 =
345 _mm_loadu_si128((__m128i *)(&dst[i * dst_stride + j]));
346
347 const __m128i comp_avg_res =
348 comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg);
349
350 const __m128i round_result = convolve_rounding(
351 &comp_avg_res, &offset_const, &rounding_const, rounding_shift);
352
353 const __m128i res_8 = _mm_packus_epi16(round_result, round_result);
354 _mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_8);
355 } else {
356 _mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_unsigned);
357 }
358 i++;
359
360 s[0] = s[2];
361 s[1] = s[3];
362 s[2] = s[4];
363 s[3] = s[5];
364 s[4] = s[6];
365 s[5] = s[7];
366 } while (i < h);
367 j += 8;
368 } while (j < w);
369 }
370 }
371