1 /*
2 * Copyright (c) 2018, Alliance for Open Media. All rights reserved.
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #ifndef AOM_AOM_DSP_X86_CONVOLVE_COMMON_INTRIN_H_
13 #define AOM_AOM_DSP_X86_CONVOLVE_COMMON_INTRIN_H_
14
15 // Note:
16 // This header file should be put below any x86 intrinsics head file
17
add_store(CONV_BUF_TYPE * const dst,const __m128i * const res,const int do_average)18 static inline void add_store(CONV_BUF_TYPE *const dst, const __m128i *const res,
19 const int do_average) {
20 __m128i d;
21 if (do_average) {
22 d = _mm_load_si128((__m128i *)dst);
23 d = _mm_add_epi32(d, *res);
24 d = _mm_srai_epi32(d, 1);
25 } else {
26 d = *res;
27 }
28 _mm_store_si128((__m128i *)dst, d);
29 }
30
prepare_coeffs_12tap(const InterpFilterParams * filter_params,int subpel_q4,__m128i * coeffs)31 static inline void prepare_coeffs_12tap(const InterpFilterParams *filter_params,
32 int subpel_q4,
33 __m128i *coeffs /* [6] */) {
34 const int16_t *const y_filter = av1_get_interp_filter_subpel_kernel(
35 filter_params, subpel_q4 & SUBPEL_MASK);
36
37 __m128i coeffs_y = _mm_loadu_si128((__m128i *)y_filter);
38
39 coeffs[0] = _mm_shuffle_epi32(coeffs_y, 0); // coeffs 0 1 0 1 0 1 0 1
40 coeffs[1] = _mm_shuffle_epi32(coeffs_y, 85); // coeffs 2 3 2 3 2 3 2 3
41 coeffs[2] = _mm_shuffle_epi32(coeffs_y, 170); // coeffs 4 5 4 5 4 5 4 5
42 coeffs[3] = _mm_shuffle_epi32(coeffs_y, 255); // coeffs 6 7 6 7 6 7 6 7
43
44 coeffs_y = _mm_loadl_epi64((__m128i *)(y_filter + 8));
45
46 coeffs[4] = _mm_shuffle_epi32(coeffs_y, 0); // coeffs 8 9 8 9 8 9 8 9
47 coeffs[5] =
48 _mm_shuffle_epi32(coeffs_y, 85); // coeffs 10 11 10 11 10 11 10 11
49 }
50
convolve_12tap(const __m128i * s,const __m128i * coeffs)51 static inline __m128i convolve_12tap(const __m128i *s, const __m128i *coeffs) {
52 const __m128i d0 = _mm_madd_epi16(s[0], coeffs[0]);
53 const __m128i d1 = _mm_madd_epi16(s[1], coeffs[1]);
54 const __m128i d2 = _mm_madd_epi16(s[2], coeffs[2]);
55 const __m128i d3 = _mm_madd_epi16(s[3], coeffs[3]);
56 const __m128i d4 = _mm_madd_epi16(s[4], coeffs[4]);
57 const __m128i d5 = _mm_madd_epi16(s[5], coeffs[5]);
58 const __m128i d_0123 =
59 _mm_add_epi32(_mm_add_epi32(d0, d1), _mm_add_epi32(d2, d3));
60 const __m128i d = _mm_add_epi32(_mm_add_epi32(d4, d5), d_0123);
61 return d;
62 }
63
convolve_lo_x_12tap(const __m128i * s,const __m128i * coeffs,const __m128i zero)64 static inline __m128i convolve_lo_x_12tap(const __m128i *s,
65 const __m128i *coeffs,
66 const __m128i zero) {
67 __m128i ss[6];
68 ss[0] = _mm_unpacklo_epi8(s[0], zero); // 0 1 1 2 2 3 3 4
69 ss[1] = _mm_unpacklo_epi8(s[1], zero); // 2 3 3 4 4 5 5 6
70 ss[2] = _mm_unpacklo_epi8(s[2], zero); // 4 5 5 6 6 7 7 8
71 ss[3] = _mm_unpacklo_epi8(s[3], zero); // 6 7 7 8 8 9 9 10
72 ss[4] = _mm_unpackhi_epi8(s[2], zero); // 8 9 9 10 10 11 11 12
73 ss[5] = _mm_unpackhi_epi8(s[3], zero); // 10 11 11 12 12 13 13 14
74 return convolve_12tap(ss, coeffs);
75 }
76
convolve_lo_y_12tap(const __m128i * s,const __m128i * coeffs)77 static inline __m128i convolve_lo_y_12tap(const __m128i *s,
78 const __m128i *coeffs) {
79 __m128i ss[6];
80 const __m128i zero = _mm_setzero_si128();
81 ss[0] = _mm_unpacklo_epi8(s[0], zero);
82 ss[1] = _mm_unpacklo_epi8(s[2], zero);
83 ss[2] = _mm_unpacklo_epi8(s[4], zero);
84 ss[3] = _mm_unpacklo_epi8(s[6], zero);
85 ss[4] = _mm_unpacklo_epi8(s[8], zero);
86 ss[5] = _mm_unpacklo_epi8(s[10], zero);
87 return convolve_12tap(ss, coeffs);
88 }
89
convolve_hi_y_12tap(const __m128i * s,const __m128i * coeffs)90 static inline __m128i convolve_hi_y_12tap(const __m128i *s,
91 const __m128i *coeffs) {
92 __m128i ss[6];
93 const __m128i zero = _mm_setzero_si128();
94 ss[0] = _mm_unpackhi_epi8(s[0], zero);
95 ss[1] = _mm_unpackhi_epi8(s[2], zero);
96 ss[2] = _mm_unpackhi_epi8(s[4], zero);
97 ss[3] = _mm_unpackhi_epi8(s[6], zero);
98 ss[4] = _mm_unpackhi_epi8(s[8], zero);
99 ss[5] = _mm_unpackhi_epi8(s[10], zero);
100 return convolve_12tap(ss, coeffs);
101 }
102 #endif // AOM_AOM_DSP_X86_CONVOLVE_COMMON_INTRIN_H_
103