1 // Copyright 2016 Google Inc. All Rights Reserved.
2 //
3 // Use of this source code is governed by a BSD-style license
4 // that can be found in the COPYING file in the root of the source
5 // tree. An additional intellectual property rights grant can be found
6 // in the file PATENTS. All contributing project authors may
7 // be found in the AUTHORS file in the root of the source tree.
8 // -----------------------------------------------------------------------------
9 //
10 // MSA version of dsp functions
11 //
12 // Author(s): Prashant Patil ([email protected])
13
14
15 #include "src/dsp/dsp.h"
16
17 #if defined(WEBP_USE_MSA)
18
19 #include "src/dsp/msa_macro.h"
20
21 //------------------------------------------------------------------------------
22 // Transforms
23
24 #define IDCT_1D_W(in0, in1, in2, in3, out0, out1, out2, out3) { \
25 v4i32 a1_m, b1_m, c1_m, d1_m; \
26 v4i32 c_tmp1_m, c_tmp2_m, d_tmp1_m, d_tmp2_m; \
27 const v4i32 cospi8sqrt2minus1 = __msa_fill_w(20091); \
28 const v4i32 sinpi8sqrt2 = __msa_fill_w(35468); \
29 \
30 a1_m = in0 + in2; \
31 b1_m = in0 - in2; \
32 c_tmp1_m = (in1 * sinpi8sqrt2) >> 16; \
33 c_tmp2_m = in3 + ((in3 * cospi8sqrt2minus1) >> 16); \
34 c1_m = c_tmp1_m - c_tmp2_m; \
35 d_tmp1_m = in1 + ((in1 * cospi8sqrt2minus1) >> 16); \
36 d_tmp2_m = (in3 * sinpi8sqrt2) >> 16; \
37 d1_m = d_tmp1_m + d_tmp2_m; \
38 BUTTERFLY_4(a1_m, b1_m, c1_m, d1_m, out0, out1, out2, out3); \
39 }
40
TransformOne(const int16_t * in,uint8_t * dst)41 static void TransformOne(const int16_t* in, uint8_t* dst) {
42 v8i16 input0, input1;
43 v4i32 in0, in1, in2, in3, hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3;
44 v4i32 res0, res1, res2, res3;
45 const v16i8 zero = { 0 };
46 v16i8 dest0, dest1, dest2, dest3;
47
48 LD_SH2(in, 8, input0, input1);
49 UNPCK_SH_SW(input0, in0, in1);
50 UNPCK_SH_SW(input1, in2, in3);
51 IDCT_1D_W(in0, in1, in2, in3, hz0, hz1, hz2, hz3);
52 TRANSPOSE4x4_SW_SW(hz0, hz1, hz2, hz3, hz0, hz1, hz2, hz3);
53 IDCT_1D_W(hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3);
54 SRARI_W4_SW(vt0, vt1, vt2, vt3, 3);
55 TRANSPOSE4x4_SW_SW(vt0, vt1, vt2, vt3, vt0, vt1, vt2, vt3);
56 LD_SB4(dst, BPS, dest0, dest1, dest2, dest3);
57 ILVR_B4_SW(zero, dest0, zero, dest1, zero, dest2, zero, dest3,
58 res0, res1, res2, res3);
59 ILVR_H4_SW(zero, res0, zero, res1, zero, res2, zero, res3,
60 res0, res1, res2, res3);
61 ADD4(res0, vt0, res1, vt1, res2, vt2, res3, vt3, res0, res1, res2, res3);
62 CLIP_SW4_0_255(res0, res1, res2, res3);
63 PCKEV_B2_SW(res0, res1, res2, res3, vt0, vt1);
64 res0 = (v4i32)__msa_pckev_b((v16i8)vt0, (v16i8)vt1);
65 ST4x4_UB(res0, res0, 3, 2, 1, 0, dst, BPS);
66 }
67
TransformTwo(const int16_t * in,uint8_t * dst,int do_two)68 static void TransformTwo(const int16_t* in, uint8_t* dst, int do_two) {
69 TransformOne(in, dst);
70 if (do_two) {
71 TransformOne(in + 16, dst + 4);
72 }
73 }
74
TransformWHT(const int16_t * in,int16_t * out)75 static void TransformWHT(const int16_t* in, int16_t* out) {
76 v8i16 input0, input1;
77 const v8i16 mask0 = { 0, 1, 2, 3, 8, 9, 10, 11 };
78 const v8i16 mask1 = { 4, 5, 6, 7, 12, 13, 14, 15 };
79 const v8i16 mask2 = { 0, 4, 8, 12, 1, 5, 9, 13 };
80 const v8i16 mask3 = { 3, 7, 11, 15, 2, 6, 10, 14 };
81 v8i16 tmp0, tmp1, tmp2, tmp3;
82 v8i16 out0, out1;
83
84 LD_SH2(in, 8, input0, input1);
85 input1 = SLDI_SH(input1, input1, 8);
86 tmp0 = input0 + input1;
87 tmp1 = input0 - input1;
88 VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3);
89 out0 = tmp2 + tmp3;
90 out1 = tmp2 - tmp3;
91 VSHF_H2_SH(out0, out1, out0, out1, mask2, mask3, input0, input1);
92 tmp0 = input0 + input1;
93 tmp1 = input0 - input1;
94 VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3);
95 tmp0 = tmp2 + tmp3;
96 tmp1 = tmp2 - tmp3;
97 ADDVI_H2_SH(tmp0, 3, tmp1, 3, out0, out1);
98 SRAI_H2_SH(out0, out1, 3);
99 out[0] = __msa_copy_s_h(out0, 0);
100 out[16] = __msa_copy_s_h(out0, 4);
101 out[32] = __msa_copy_s_h(out1, 0);
102 out[48] = __msa_copy_s_h(out1, 4);
103 out[64] = __msa_copy_s_h(out0, 1);
104 out[80] = __msa_copy_s_h(out0, 5);
105 out[96] = __msa_copy_s_h(out1, 1);
106 out[112] = __msa_copy_s_h(out1, 5);
107 out[128] = __msa_copy_s_h(out0, 2);
108 out[144] = __msa_copy_s_h(out0, 6);
109 out[160] = __msa_copy_s_h(out1, 2);
110 out[176] = __msa_copy_s_h(out1, 6);
111 out[192] = __msa_copy_s_h(out0, 3);
112 out[208] = __msa_copy_s_h(out0, 7);
113 out[224] = __msa_copy_s_h(out1, 3);
114 out[240] = __msa_copy_s_h(out1, 7);
115 }
116
TransformDC(const int16_t * in,uint8_t * dst)117 static void TransformDC(const int16_t* in, uint8_t* dst) {
118 const int DC = (in[0] + 4) >> 3;
119 const v8i16 tmp0 = __msa_fill_h(DC);
120 ADDBLK_ST4x4_UB(tmp0, tmp0, tmp0, tmp0, dst, BPS);
121 }
122
TransformAC3(const int16_t * in,uint8_t * dst)123 static void TransformAC3(const int16_t* in, uint8_t* dst) {
124 const int a = in[0] + 4;
125 const int c4 = WEBP_TRANSFORM_AC3_MUL2(in[4]);
126 const int d4 = WEBP_TRANSFORM_AC3_MUL1(in[4]);
127 const int in2 = WEBP_TRANSFORM_AC3_MUL2(in[1]);
128 const int in3 = WEBP_TRANSFORM_AC3_MUL1(in[1]);
129 v4i32 tmp0 = { 0 };
130 v4i32 out0 = __msa_fill_w(a + d4);
131 v4i32 out1 = __msa_fill_w(a + c4);
132 v4i32 out2 = __msa_fill_w(a - c4);
133 v4i32 out3 = __msa_fill_w(a - d4);
134 v4i32 res0, res1, res2, res3;
135 const v4i32 zero = { 0 };
136 v16u8 dest0, dest1, dest2, dest3;
137
138 INSERT_W4_SW(in3, in2, -in2, -in3, tmp0);
139 ADD4(out0, tmp0, out1, tmp0, out2, tmp0, out3, tmp0,
140 out0, out1, out2, out3);
141 SRAI_W4_SW(out0, out1, out2, out3, 3);
142 LD_UB4(dst, BPS, dest0, dest1, dest2, dest3);
143 ILVR_B4_SW(zero, dest0, zero, dest1, zero, dest2, zero, dest3,
144 res0, res1, res2, res3);
145 ILVR_H4_SW(zero, res0, zero, res1, zero, res2, zero, res3,
146 res0, res1, res2, res3);
147 ADD4(res0, out0, res1, out1, res2, out2, res3, out3, res0, res1, res2, res3);
148 CLIP_SW4_0_255(res0, res1, res2, res3);
149 PCKEV_B2_SW(res0, res1, res2, res3, out0, out1);
150 res0 = (v4i32)__msa_pckev_b((v16i8)out0, (v16i8)out1);
151 ST4x4_UB(res0, res0, 3, 2, 1, 0, dst, BPS);
152 }
153
154 //------------------------------------------------------------------------------
155 // Edge filtering functions
156
157 #define FLIP_SIGN2(in0, in1, out0, out1) { \
158 out0 = (v16i8)__msa_xori_b(in0, 0x80); \
159 out1 = (v16i8)__msa_xori_b(in1, 0x80); \
160 }
161
162 #define FLIP_SIGN4(in0, in1, in2, in3, out0, out1, out2, out3) { \
163 FLIP_SIGN2(in0, in1, out0, out1); \
164 FLIP_SIGN2(in2, in3, out2, out3); \
165 }
166
167 #define FILT_VAL(q0_m, p0_m, mask, filt) do { \
168 v16i8 q0_sub_p0; \
169 q0_sub_p0 = __msa_subs_s_b(q0_m, p0_m); \
170 filt = __msa_adds_s_b(filt, q0_sub_p0); \
171 filt = __msa_adds_s_b(filt, q0_sub_p0); \
172 filt = __msa_adds_s_b(filt, q0_sub_p0); \
173 filt = filt & mask; \
174 } while (0)
175
176 #define FILT2(q_m, p_m, q, p) do { \
177 u_r = SRAI_H(temp1, 7); \
178 u_r = __msa_sat_s_h(u_r, 7); \
179 u_l = SRAI_H(temp3, 7); \
180 u_l = __msa_sat_s_h(u_l, 7); \
181 u = __msa_pckev_b((v16i8)u_l, (v16i8)u_r); \
182 q_m = __msa_subs_s_b(q_m, u); \
183 p_m = __msa_adds_s_b(p_m, u); \
184 q = __msa_xori_b((v16u8)q_m, 0x80); \
185 p = __msa_xori_b((v16u8)p_m, 0x80); \
186 } while (0)
187
188 #define LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev) do { \
189 v16i8 p1_m, p0_m, q0_m, q1_m; \
190 v16i8 filt, t1, t2; \
191 const v16i8 cnst4b = __msa_ldi_b(4); \
192 const v16i8 cnst3b = __msa_ldi_b(3); \
193 \
194 FLIP_SIGN4(p1, p0, q0, q1, p1_m, p0_m, q0_m, q1_m); \
195 filt = __msa_subs_s_b(p1_m, q1_m); \
196 filt = filt & hev; \
197 FILT_VAL(q0_m, p0_m, mask, filt); \
198 t1 = __msa_adds_s_b(filt, cnst4b); \
199 t1 = SRAI_B(t1, 3); \
200 t2 = __msa_adds_s_b(filt, cnst3b); \
201 t2 = SRAI_B(t2, 3); \
202 q0_m = __msa_subs_s_b(q0_m, t1); \
203 q0 = __msa_xori_b((v16u8)q0_m, 0x80); \
204 p0_m = __msa_adds_s_b(p0_m, t2); \
205 p0 = __msa_xori_b((v16u8)p0_m, 0x80); \
206 filt = __msa_srari_b(t1, 1); \
207 hev = __msa_xori_b(hev, 0xff); \
208 filt = filt & hev; \
209 q1_m = __msa_subs_s_b(q1_m, filt); \
210 q1 = __msa_xori_b((v16u8)q1_m, 0x80); \
211 p1_m = __msa_adds_s_b(p1_m, filt); \
212 p1 = __msa_xori_b((v16u8)p1_m, 0x80); \
213 } while (0)
214
215 #define LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev) do { \
216 v16i8 p2_m, p1_m, p0_m, q2_m, q1_m, q0_m; \
217 v16i8 u, filt, t1, t2, filt_sign; \
218 v8i16 filt_r, filt_l, u_r, u_l; \
219 v8i16 temp0, temp1, temp2, temp3; \
220 const v16i8 cnst4b = __msa_ldi_b(4); \
221 const v16i8 cnst3b = __msa_ldi_b(3); \
222 const v8i16 cnst9h = __msa_ldi_h(9); \
223 const v8i16 cnst63h = __msa_ldi_h(63); \
224 \
225 FLIP_SIGN4(p1, p0, q0, q1, p1_m, p0_m, q0_m, q1_m); \
226 filt = __msa_subs_s_b(p1_m, q1_m); \
227 FILT_VAL(q0_m, p0_m, mask, filt); \
228 FLIP_SIGN2(p2, q2, p2_m, q2_m); \
229 t2 = filt & hev; \
230 /* filt_val &= ~hev */ \
231 hev = __msa_xori_b(hev, 0xff); \
232 filt = filt & hev; \
233 t1 = __msa_adds_s_b(t2, cnst4b); \
234 t1 = SRAI_B(t1, 3); \
235 t2 = __msa_adds_s_b(t2, cnst3b); \
236 t2 = SRAI_B(t2, 3); \
237 q0_m = __msa_subs_s_b(q0_m, t1); \
238 p0_m = __msa_adds_s_b(p0_m, t2); \
239 filt_sign = __msa_clti_s_b(filt, 0); \
240 ILVRL_B2_SH(filt_sign, filt, filt_r, filt_l); \
241 /* update q2/p2 */ \
242 temp0 = filt_r * cnst9h; \
243 temp1 = temp0 + cnst63h; \
244 temp2 = filt_l * cnst9h; \
245 temp3 = temp2 + cnst63h; \
246 FILT2(q2_m, p2_m, q2, p2); \
247 /* update q1/p1 */ \
248 temp1 = temp1 + temp0; \
249 temp3 = temp3 + temp2; \
250 FILT2(q1_m, p1_m, q1, p1); \
251 /* update q0/p0 */ \
252 temp1 = temp1 + temp0; \
253 temp3 = temp3 + temp2; \
254 FILT2(q0_m, p0_m, q0, p0); \
255 } while (0)
256
257 #define LPF_MASK_HEV(p3_in, p2_in, p1_in, p0_in, \
258 q0_in, q1_in, q2_in, q3_in, \
259 limit_in, b_limit_in, thresh_in, \
260 hev_out, mask_out) do { \
261 v16u8 p3_asub_p2_m, p2_asub_p1_m, p1_asub_p0_m, q1_asub_q0_m; \
262 v16u8 p1_asub_q1_m, p0_asub_q0_m, q3_asub_q2_m, q2_asub_q1_m; \
263 v16u8 flat_out; \
264 \
265 /* absolute subtraction of pixel values */ \
266 p3_asub_p2_m = __msa_asub_u_b(p3_in, p2_in); \
267 p2_asub_p1_m = __msa_asub_u_b(p2_in, p1_in); \
268 p1_asub_p0_m = __msa_asub_u_b(p1_in, p0_in); \
269 q1_asub_q0_m = __msa_asub_u_b(q1_in, q0_in); \
270 q2_asub_q1_m = __msa_asub_u_b(q2_in, q1_in); \
271 q3_asub_q2_m = __msa_asub_u_b(q3_in, q2_in); \
272 p0_asub_q0_m = __msa_asub_u_b(p0_in, q0_in); \
273 p1_asub_q1_m = __msa_asub_u_b(p1_in, q1_in); \
274 /* calculation of hev */ \
275 flat_out = __msa_max_u_b(p1_asub_p0_m, q1_asub_q0_m); \
276 hev_out = (thresh_in < flat_out); \
277 /* calculation of mask */ \
278 p0_asub_q0_m = __msa_adds_u_b(p0_asub_q0_m, p0_asub_q0_m); \
279 p1_asub_q1_m = SRAI_B(p1_asub_q1_m, 1); \
280 p0_asub_q0_m = __msa_adds_u_b(p0_asub_q0_m, p1_asub_q1_m); \
281 mask_out = (b_limit_in < p0_asub_q0_m); \
282 mask_out = __msa_max_u_b(flat_out, mask_out); \
283 p3_asub_p2_m = __msa_max_u_b(p3_asub_p2_m, p2_asub_p1_m); \
284 mask_out = __msa_max_u_b(p3_asub_p2_m, mask_out); \
285 q2_asub_q1_m = __msa_max_u_b(q2_asub_q1_m, q3_asub_q2_m); \
286 mask_out = __msa_max_u_b(q2_asub_q1_m, mask_out); \
287 mask_out = (limit_in < mask_out); \
288 mask_out = __msa_xori_b(mask_out, 0xff); \
289 } while (0)
290
291 #define ST6x1_UB(in0, in0_idx, in1, in1_idx, pdst, stride) do { \
292 const uint16_t tmp0_h = __msa_copy_s_h((v8i16)in1, in1_idx); \
293 const uint32_t tmp0_w = __msa_copy_s_w((v4i32)in0, in0_idx); \
294 SW(tmp0_w, pdst); \
295 SH(tmp0_h, pdst + stride); \
296 } while (0)
297
298 #define ST6x4_UB(in0, start_in0_idx, in1, start_in1_idx, pdst, stride) do { \
299 uint8_t* ptmp1 = (uint8_t*)pdst; \
300 ST6x1_UB(in0, start_in0_idx, in1, start_in1_idx, ptmp1, 4); \
301 ptmp1 += stride; \
302 ST6x1_UB(in0, start_in0_idx + 1, in1, start_in1_idx + 1, ptmp1, 4); \
303 ptmp1 += stride; \
304 ST6x1_UB(in0, start_in0_idx + 2, in1, start_in1_idx + 2, ptmp1, 4); \
305 ptmp1 += stride; \
306 ST6x1_UB(in0, start_in0_idx + 3, in1, start_in1_idx + 3, ptmp1, 4); \
307 } while (0)
308
309 #define LPF_SIMPLE_FILT(p1_in, p0_in, q0_in, q1_in, mask) do { \
310 v16i8 p1_m, p0_m, q0_m, q1_m, filt, filt1, filt2; \
311 const v16i8 cnst4b = __msa_ldi_b(4); \
312 const v16i8 cnst3b = __msa_ldi_b(3); \
313 \
314 FLIP_SIGN4(p1_in, p0_in, q0_in, q1_in, p1_m, p0_m, q0_m, q1_m); \
315 filt = __msa_subs_s_b(p1_m, q1_m); \
316 FILT_VAL(q0_m, p0_m, mask, filt); \
317 filt1 = __msa_adds_s_b(filt, cnst4b); \
318 filt1 = SRAI_B(filt1, 3); \
319 filt2 = __msa_adds_s_b(filt, cnst3b); \
320 filt2 = SRAI_B(filt2, 3); \
321 q0_m = __msa_subs_s_b(q0_m, filt1); \
322 p0_m = __msa_adds_s_b(p0_m, filt2); \
323 q0_in = __msa_xori_b((v16u8)q0_m, 0x80); \
324 p0_in = __msa_xori_b((v16u8)p0_m, 0x80); \
325 } while (0)
326
327 #define LPF_SIMPLE_MASK(p1, p0, q0, q1, b_limit, mask) do { \
328 v16u8 p1_a_sub_q1, p0_a_sub_q0; \
329 \
330 p0_a_sub_q0 = __msa_asub_u_b(p0, q0); \
331 p1_a_sub_q1 = __msa_asub_u_b(p1, q1); \
332 p1_a_sub_q1 = (v16u8)__msa_srli_b((v16i8)p1_a_sub_q1, 1); \
333 p0_a_sub_q0 = __msa_adds_u_b(p0_a_sub_q0, p0_a_sub_q0); \
334 mask = __msa_adds_u_b(p0_a_sub_q0, p1_a_sub_q1); \
335 mask = (mask <= b_limit); \
336 } while (0)
337
VFilter16(uint8_t * src,int stride,int b_limit_in,int limit_in,int thresh_in)338 static void VFilter16(uint8_t* src, int stride,
339 int b_limit_in, int limit_in, int thresh_in) {
340 uint8_t* ptemp = src - 4 * stride;
341 v16u8 p3, p2, p1, p0, q3, q2, q1, q0;
342 v16u8 mask, hev;
343 const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in);
344 const v16u8 limit = (v16u8)__msa_fill_b(limit_in);
345 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
346
347 LD_UB8(ptemp, stride, p3, p2, p1, p0, q0, q1, q2, q3);
348 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh,
349 hev, mask);
350 LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev);
351 ptemp = src - 3 * stride;
352 ST_UB4(p2, p1, p0, q0, ptemp, stride);
353 ptemp += (4 * stride);
354 ST_UB2(q1, q2, ptemp, stride);
355 }
356
HFilter16(uint8_t * src,int stride,int b_limit_in,int limit_in,int thresh_in)357 static void HFilter16(uint8_t* src, int stride,
358 int b_limit_in, int limit_in, int thresh_in) {
359 uint8_t* ptmp = src - 4;
360 v16u8 p3, p2, p1, p0, q3, q2, q1, q0;
361 v16u8 mask, hev;
362 v16u8 row0, row1, row2, row3, row4, row5, row6, row7, row8;
363 v16u8 row9, row10, row11, row12, row13, row14, row15;
364 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
365 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
366 const v16u8 limit = (v16u8)__msa_fill_b(limit_in);
367 const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in);
368
369 LD_UB8(ptmp, stride, row0, row1, row2, row3, row4, row5, row6, row7);
370 ptmp += (8 * stride);
371 LD_UB8(ptmp, stride, row8, row9, row10, row11, row12, row13, row14, row15);
372 TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7,
373 row8, row9, row10, row11, row12, row13, row14, row15,
374 p3, p2, p1, p0, q0, q1, q2, q3);
375 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh,
376 hev, mask);
377 LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev);
378 ILVR_B2_SH(p1, p2, q0, p0, tmp0, tmp1);
379 ILVRL_H2_SH(tmp1, tmp0, tmp3, tmp4);
380 ILVL_B2_SH(p1, p2, q0, p0, tmp0, tmp1);
381 ILVRL_H2_SH(tmp1, tmp0, tmp6, tmp7);
382 ILVRL_B2_SH(q2, q1, tmp2, tmp5);
383 ptmp = src - 3;
384 ST6x1_UB(tmp3, 0, tmp2, 0, ptmp, 4);
385 ptmp += stride;
386 ST6x1_UB(tmp3, 1, tmp2, 1, ptmp, 4);
387 ptmp += stride;
388 ST6x1_UB(tmp3, 2, tmp2, 2, ptmp, 4);
389 ptmp += stride;
390 ST6x1_UB(tmp3, 3, tmp2, 3, ptmp, 4);
391 ptmp += stride;
392 ST6x1_UB(tmp4, 0, tmp2, 4, ptmp, 4);
393 ptmp += stride;
394 ST6x1_UB(tmp4, 1, tmp2, 5, ptmp, 4);
395 ptmp += stride;
396 ST6x1_UB(tmp4, 2, tmp2, 6, ptmp, 4);
397 ptmp += stride;
398 ST6x1_UB(tmp4, 3, tmp2, 7, ptmp, 4);
399 ptmp += stride;
400 ST6x1_UB(tmp6, 0, tmp5, 0, ptmp, 4);
401 ptmp += stride;
402 ST6x1_UB(tmp6, 1, tmp5, 1, ptmp, 4);
403 ptmp += stride;
404 ST6x1_UB(tmp6, 2, tmp5, 2, ptmp, 4);
405 ptmp += stride;
406 ST6x1_UB(tmp6, 3, tmp5, 3, ptmp, 4);
407 ptmp += stride;
408 ST6x1_UB(tmp7, 0, tmp5, 4, ptmp, 4);
409 ptmp += stride;
410 ST6x1_UB(tmp7, 1, tmp5, 5, ptmp, 4);
411 ptmp += stride;
412 ST6x1_UB(tmp7, 2, tmp5, 6, ptmp, 4);
413 ptmp += stride;
414 ST6x1_UB(tmp7, 3, tmp5, 7, ptmp, 4);
415 }
416
417 // on three inner edges
VFilterHorEdge16i(uint8_t * src,int stride,int b_limit,int limit,int thresh)418 static void VFilterHorEdge16i(uint8_t* src, int stride,
419 int b_limit, int limit, int thresh) {
420 v16u8 mask, hev;
421 v16u8 p3, p2, p1, p0, q3, q2, q1, q0;
422 const v16u8 thresh0 = (v16u8)__msa_fill_b(thresh);
423 const v16u8 b_limit0 = (v16u8)__msa_fill_b(b_limit);
424 const v16u8 limit0 = (v16u8)__msa_fill_b(limit);
425
426 LD_UB8((src - 4 * stride), stride, p3, p2, p1, p0, q0, q1, q2, q3);
427 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0,
428 hev, mask);
429 LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev);
430 ST_UB4(p1, p0, q0, q1, (src - 2 * stride), stride);
431 }
432
VFilter16i(uint8_t * src_y,int stride,int b_limit,int limit,int thresh)433 static void VFilter16i(uint8_t* src_y, int stride,
434 int b_limit, int limit, int thresh) {
435 VFilterHorEdge16i(src_y + 4 * stride, stride, b_limit, limit, thresh);
436 VFilterHorEdge16i(src_y + 8 * stride, stride, b_limit, limit, thresh);
437 VFilterHorEdge16i(src_y + 12 * stride, stride, b_limit, limit, thresh);
438 }
439
HFilterVertEdge16i(uint8_t * src,int stride,int b_limit,int limit,int thresh)440 static void HFilterVertEdge16i(uint8_t* src, int stride,
441 int b_limit, int limit, int thresh) {
442 v16u8 mask, hev;
443 v16u8 p3, p2, p1, p0, q3, q2, q1, q0;
444 v16u8 row0, row1, row2, row3, row4, row5, row6, row7;
445 v16u8 row8, row9, row10, row11, row12, row13, row14, row15;
446 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
447 const v16u8 thresh0 = (v16u8)__msa_fill_b(thresh);
448 const v16u8 b_limit0 = (v16u8)__msa_fill_b(b_limit);
449 const v16u8 limit0 = (v16u8)__msa_fill_b(limit);
450
451 LD_UB8(src - 4, stride, row0, row1, row2, row3, row4, row5, row6, row7);
452 LD_UB8(src - 4 + (8 * stride), stride,
453 row8, row9, row10, row11, row12, row13, row14, row15);
454 TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7,
455 row8, row9, row10, row11, row12, row13, row14, row15,
456 p3, p2, p1, p0, q0, q1, q2, q3);
457 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0,
458 hev, mask);
459 LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev);
460 ILVR_B2_SH(p0, p1, q1, q0, tmp0, tmp1);
461 ILVRL_H2_SH(tmp1, tmp0, tmp2, tmp3);
462 ILVL_B2_SH(p0, p1, q1, q0, tmp0, tmp1);
463 ILVRL_H2_SH(tmp1, tmp0, tmp4, tmp5);
464 src -= 2;
465 ST4x8_UB(tmp2, tmp3, src, stride);
466 src += (8 * stride);
467 ST4x8_UB(tmp4, tmp5, src, stride);
468 }
469
HFilter16i(uint8_t * src_y,int stride,int b_limit,int limit,int thresh)470 static void HFilter16i(uint8_t* src_y, int stride,
471 int b_limit, int limit, int thresh) {
472 HFilterVertEdge16i(src_y + 4, stride, b_limit, limit, thresh);
473 HFilterVertEdge16i(src_y + 8, stride, b_limit, limit, thresh);
474 HFilterVertEdge16i(src_y + 12, stride, b_limit, limit, thresh);
475 }
476
477 // 8-pixels wide variants, for chroma filtering
VFilter8(uint8_t * src_u,uint8_t * src_v,int stride,int b_limit_in,int limit_in,int thresh_in)478 static void VFilter8(uint8_t* src_u, uint8_t* src_v, int stride,
479 int b_limit_in, int limit_in, int thresh_in) {
480 uint8_t* ptmp_src_u = src_u - 4 * stride;
481 uint8_t* ptmp_src_v = src_v - 4 * stride;
482 uint64_t p2_d, p1_d, p0_d, q0_d, q1_d, q2_d;
483 v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev;
484 v16u8 p3_u, p2_u, p1_u, p0_u, q3_u, q2_u, q1_u, q0_u;
485 v16u8 p3_v, p2_v, p1_v, p0_v, q3_v, q2_v, q1_v, q0_v;
486 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
487 const v16u8 limit = (v16u8)__msa_fill_b(limit_in);
488 const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in);
489
490 LD_UB8(ptmp_src_u, stride, p3_u, p2_u, p1_u, p0_u, q0_u, q1_u, q2_u, q3_u);
491 LD_UB8(ptmp_src_v, stride, p3_v, p2_v, p1_v, p0_v, q0_v, q1_v, q2_v, q3_v);
492 ILVR_D4_UB(p3_v, p3_u, p2_v, p2_u, p1_v, p1_u, p0_v, p0_u, p3, p2, p1, p0);
493 ILVR_D4_UB(q0_v, q0_u, q1_v, q1_u, q2_v, q2_u, q3_v, q3_u, q0, q1, q2, q3);
494 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh,
495 hev, mask);
496 LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev);
497 p2_d = __msa_copy_s_d((v2i64)p2, 0);
498 p1_d = __msa_copy_s_d((v2i64)p1, 0);
499 p0_d = __msa_copy_s_d((v2i64)p0, 0);
500 q0_d = __msa_copy_s_d((v2i64)q0, 0);
501 q1_d = __msa_copy_s_d((v2i64)q1, 0);
502 q2_d = __msa_copy_s_d((v2i64)q2, 0);
503 ptmp_src_u += stride;
504 SD4(p2_d, p1_d, p0_d, q0_d, ptmp_src_u, stride);
505 ptmp_src_u += (4 * stride);
506 SD(q1_d, ptmp_src_u);
507 ptmp_src_u += stride;
508 SD(q2_d, ptmp_src_u);
509 p2_d = __msa_copy_s_d((v2i64)p2, 1);
510 p1_d = __msa_copy_s_d((v2i64)p1, 1);
511 p0_d = __msa_copy_s_d((v2i64)p0, 1);
512 q0_d = __msa_copy_s_d((v2i64)q0, 1);
513 q1_d = __msa_copy_s_d((v2i64)q1, 1);
514 q2_d = __msa_copy_s_d((v2i64)q2, 1);
515 ptmp_src_v += stride;
516 SD4(p2_d, p1_d, p0_d, q0_d, ptmp_src_v, stride);
517 ptmp_src_v += (4 * stride);
518 SD(q1_d, ptmp_src_v);
519 ptmp_src_v += stride;
520 SD(q2_d, ptmp_src_v);
521 }
522
HFilter8(uint8_t * src_u,uint8_t * src_v,int stride,int b_limit_in,int limit_in,int thresh_in)523 static void HFilter8(uint8_t* src_u, uint8_t* src_v, int stride,
524 int b_limit_in, int limit_in, int thresh_in) {
525 uint8_t* ptmp_src_u = src_u - 4;
526 uint8_t* ptmp_src_v = src_v - 4;
527 v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev;
528 v16u8 row0, row1, row2, row3, row4, row5, row6, row7, row8;
529 v16u8 row9, row10, row11, row12, row13, row14, row15;
530 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
531 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
532 const v16u8 limit = (v16u8)__msa_fill_b(limit_in);
533 const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in);
534
535 LD_UB8(ptmp_src_u, stride, row0, row1, row2, row3, row4, row5, row6, row7);
536 LD_UB8(ptmp_src_v, stride,
537 row8, row9, row10, row11, row12, row13, row14, row15);
538 TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7,
539 row8, row9, row10, row11, row12, row13, row14, row15,
540 p3, p2, p1, p0, q0, q1, q2, q3);
541 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh,
542 hev, mask);
543 LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev);
544 ILVR_B2_SH(p1, p2, q0, p0, tmp0, tmp1);
545 ILVRL_H2_SH(tmp1, tmp0, tmp3, tmp4);
546 ILVL_B2_SH(p1, p2, q0, p0, tmp0, tmp1);
547 ILVRL_H2_SH(tmp1, tmp0, tmp6, tmp7);
548 ILVRL_B2_SH(q2, q1, tmp2, tmp5);
549 ptmp_src_u += 1;
550 ST6x4_UB(tmp3, 0, tmp2, 0, ptmp_src_u, stride);
551 ptmp_src_u += 4 * stride;
552 ST6x4_UB(tmp4, 0, tmp2, 4, ptmp_src_u, stride);
553 ptmp_src_v += 1;
554 ST6x4_UB(tmp6, 0, tmp5, 0, ptmp_src_v, stride);
555 ptmp_src_v += 4 * stride;
556 ST6x4_UB(tmp7, 0, tmp5, 4, ptmp_src_v, stride);
557 }
558
VFilter8i(uint8_t * src_u,uint8_t * src_v,int stride,int b_limit_in,int limit_in,int thresh_in)559 static void VFilter8i(uint8_t* src_u, uint8_t* src_v, int stride,
560 int b_limit_in, int limit_in, int thresh_in) {
561 uint64_t p1_d, p0_d, q0_d, q1_d;
562 v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev;
563 v16u8 p3_u, p2_u, p1_u, p0_u, q3_u, q2_u, q1_u, q0_u;
564 v16u8 p3_v, p2_v, p1_v, p0_v, q3_v, q2_v, q1_v, q0_v;
565 const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in);
566 const v16u8 limit = (v16u8)__msa_fill_b(limit_in);
567 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
568
569 LD_UB8(src_u, stride, p3_u, p2_u, p1_u, p0_u, q0_u, q1_u, q2_u, q3_u);
570 src_u += (5 * stride);
571 LD_UB8(src_v, stride, p3_v, p2_v, p1_v, p0_v, q0_v, q1_v, q2_v, q3_v);
572 src_v += (5 * stride);
573 ILVR_D4_UB(p3_v, p3_u, p2_v, p2_u, p1_v, p1_u, p0_v, p0_u, p3, p2, p1, p0);
574 ILVR_D4_UB(q0_v, q0_u, q1_v, q1_u, q2_v, q2_u, q3_v, q3_u, q0, q1, q2, q3);
575 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh,
576 hev, mask);
577 LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev);
578 p1_d = __msa_copy_s_d((v2i64)p1, 0);
579 p0_d = __msa_copy_s_d((v2i64)p0, 0);
580 q0_d = __msa_copy_s_d((v2i64)q0, 0);
581 q1_d = __msa_copy_s_d((v2i64)q1, 0);
582 SD4(q1_d, q0_d, p0_d, p1_d, src_u, -stride);
583 p1_d = __msa_copy_s_d((v2i64)p1, 1);
584 p0_d = __msa_copy_s_d((v2i64)p0, 1);
585 q0_d = __msa_copy_s_d((v2i64)q0, 1);
586 q1_d = __msa_copy_s_d((v2i64)q1, 1);
587 SD4(q1_d, q0_d, p0_d, p1_d, src_v, -stride);
588 }
589
HFilter8i(uint8_t * src_u,uint8_t * src_v,int stride,int b_limit_in,int limit_in,int thresh_in)590 static void HFilter8i(uint8_t* src_u, uint8_t* src_v, int stride,
591 int b_limit_in, int limit_in, int thresh_in) {
592 v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev;
593 v16u8 row0, row1, row2, row3, row4, row5, row6, row7, row8;
594 v16u8 row9, row10, row11, row12, row13, row14, row15;
595 v4i32 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
596 const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in);
597 const v16u8 limit = (v16u8)__msa_fill_b(limit_in);
598 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
599
600 LD_UB8(src_u, stride, row0, row1, row2, row3, row4, row5, row6, row7);
601 LD_UB8(src_v, stride,
602 row8, row9, row10, row11, row12, row13, row14, row15);
603 TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7,
604 row8, row9, row10, row11, row12, row13, row14, row15,
605 p3, p2, p1, p0, q0, q1, q2, q3);
606 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh,
607 hev, mask);
608 LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev);
609 ILVR_B2_SW(p0, p1, q1, q0, tmp0, tmp1);
610 ILVRL_H2_SW(tmp1, tmp0, tmp2, tmp3);
611 ILVL_B2_SW(p0, p1, q1, q0, tmp0, tmp1);
612 ILVRL_H2_SW(tmp1, tmp0, tmp4, tmp5);
613 src_u += 2;
614 ST4x4_UB(tmp2, tmp2, 0, 1, 2, 3, src_u, stride);
615 src_u += 4 * stride;
616 ST4x4_UB(tmp3, tmp3, 0, 1, 2, 3, src_u, stride);
617 src_v += 2;
618 ST4x4_UB(tmp4, tmp4, 0, 1, 2, 3, src_v, stride);
619 src_v += 4 * stride;
620 ST4x4_UB(tmp5, tmp5, 0, 1, 2, 3, src_v, stride);
621 }
622
SimpleVFilter16(uint8_t * src,int stride,int b_limit_in)623 static void SimpleVFilter16(uint8_t* src, int stride, int b_limit_in) {
624 v16u8 p1, p0, q1, q0, mask;
625 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
626
627 LD_UB4(src - 2 * stride, stride, p1, p0, q0, q1);
628 LPF_SIMPLE_MASK(p1, p0, q0, q1, b_limit, mask);
629 LPF_SIMPLE_FILT(p1, p0, q0, q1, mask);
630 ST_UB2(p0, q0, src - stride, stride);
631 }
632
SimpleHFilter16(uint8_t * src,int stride,int b_limit_in)633 static void SimpleHFilter16(uint8_t* src, int stride, int b_limit_in) {
634 v16u8 p1, p0, q1, q0, mask, row0, row1, row2, row3, row4, row5, row6, row7;
635 v16u8 row8, row9, row10, row11, row12, row13, row14, row15;
636 v8i16 tmp0, tmp1;
637 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in);
638 uint8_t* ptemp_src = src - 2;
639
640 LD_UB8(ptemp_src, stride, row0, row1, row2, row3, row4, row5, row6, row7);
641 LD_UB8(ptemp_src + 8 * stride, stride,
642 row8, row9, row10, row11, row12, row13, row14, row15);
643 TRANSPOSE16x4_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7,
644 row8, row9, row10, row11, row12, row13, row14, row15,
645 p1, p0, q0, q1);
646 LPF_SIMPLE_MASK(p1, p0, q0, q1, b_limit, mask);
647 LPF_SIMPLE_FILT(p1, p0, q0, q1, mask);
648 ILVRL_B2_SH(q0, p0, tmp1, tmp0);
649 ptemp_src += 1;
650 ST2x4_UB(tmp1, 0, ptemp_src, stride);
651 ptemp_src += 4 * stride;
652 ST2x4_UB(tmp1, 4, ptemp_src, stride);
653 ptemp_src += 4 * stride;
654 ST2x4_UB(tmp0, 0, ptemp_src, stride);
655 ptemp_src += 4 * stride;
656 ST2x4_UB(tmp0, 4, ptemp_src, stride);
657 ptemp_src += 4 * stride;
658 }
659
SimpleVFilter16i(uint8_t * src_y,int stride,int b_limit_in)660 static void SimpleVFilter16i(uint8_t* src_y, int stride, int b_limit_in) {
661 SimpleVFilter16(src_y + 4 * stride, stride, b_limit_in);
662 SimpleVFilter16(src_y + 8 * stride, stride, b_limit_in);
663 SimpleVFilter16(src_y + 12 * stride, stride, b_limit_in);
664 }
665
SimpleHFilter16i(uint8_t * src_y,int stride,int b_limit_in)666 static void SimpleHFilter16i(uint8_t* src_y, int stride, int b_limit_in) {
667 SimpleHFilter16(src_y + 4, stride, b_limit_in);
668 SimpleHFilter16(src_y + 8, stride, b_limit_in);
669 SimpleHFilter16(src_y + 12, stride, b_limit_in);
670 }
671
672 //------------------------------------------------------------------------------
673 // Intra predictions
674 //------------------------------------------------------------------------------
675
676 // 4x4
677
DC4(uint8_t * dst)678 static void DC4(uint8_t* dst) { // DC
679 uint32_t dc = 4;
680 int i;
681 for (i = 0; i < 4; ++i) dc += dst[i - BPS] + dst[-1 + i * BPS];
682 dc >>= 3;
683 dc = dc | (dc << 8) | (dc << 16) | (dc << 24);
684 SW4(dc, dc, dc, dc, dst, BPS);
685 }
686
TM4(uint8_t * dst)687 static void TM4(uint8_t* dst) {
688 const uint8_t* const ptemp = dst - BPS - 1;
689 v8i16 T, d, r0, r1, r2, r3;
690 const v16i8 zero = { 0 };
691 const v8i16 TL = (v8i16)__msa_fill_h(ptemp[0 * BPS]);
692 const v8i16 L0 = (v8i16)__msa_fill_h(ptemp[1 * BPS]);
693 const v8i16 L1 = (v8i16)__msa_fill_h(ptemp[2 * BPS]);
694 const v8i16 L2 = (v8i16)__msa_fill_h(ptemp[3 * BPS]);
695 const v8i16 L3 = (v8i16)__msa_fill_h(ptemp[4 * BPS]);
696 const v16u8 T1 = LD_UB(ptemp + 1);
697
698 T = (v8i16)__msa_ilvr_b(zero, (v16i8)T1);
699 d = T - TL;
700 ADD4(d, L0, d, L1, d, L2, d, L3, r0, r1, r2, r3);
701 CLIP_SH4_0_255(r0, r1, r2, r3);
702 PCKEV_ST4x4_UB(r0, r1, r2, r3, dst, BPS);
703 }
704
VE4(uint8_t * dst)705 static void VE4(uint8_t* dst) { // vertical
706 const uint8_t* const ptop = dst - BPS - 1;
707 const uint32_t val0 = LW(ptop + 0);
708 const uint32_t val1 = LW(ptop + 4);
709 uint32_t out;
710 v16u8 A = { 0 }, B, C, AC, B2, R;
711
712 INSERT_W2_UB(val0, val1, A);
713 B = SLDI_UB(A, A, 1);
714 C = SLDI_UB(A, A, 2);
715 AC = __msa_ave_u_b(A, C);
716 B2 = __msa_ave_u_b(B, B);
717 R = __msa_aver_u_b(AC, B2);
718 out = __msa_copy_s_w((v4i32)R, 0);
719 SW4(out, out, out, out, dst, BPS);
720 }
721
RD4(uint8_t * dst)722 static void RD4(uint8_t* dst) { // Down-right
723 const uint8_t* const ptop = dst - 1 - BPS;
724 uint32_t val0 = LW(ptop + 0);
725 uint32_t val1 = LW(ptop + 4);
726 uint32_t val2, val3;
727 v16u8 A, B, C, AC, B2, R, A1 = { 0 };
728
729 INSERT_W2_UB(val0, val1, A1);
730 A = SLDI_UB(A1, A1, 12);
731 A = (v16u8)__msa_insert_b((v16i8)A, 3, ptop[1 * BPS]);
732 A = (v16u8)__msa_insert_b((v16i8)A, 2, ptop[2 * BPS]);
733 A = (v16u8)__msa_insert_b((v16i8)A, 1, ptop[3 * BPS]);
734 A = (v16u8)__msa_insert_b((v16i8)A, 0, ptop[4 * BPS]);
735 B = SLDI_UB(A, A, 1);
736 C = SLDI_UB(A, A, 2);
737 AC = __msa_ave_u_b(A, C);
738 B2 = __msa_ave_u_b(B, B);
739 R = __msa_aver_u_b(AC, B2);
740 val3 = __msa_copy_s_w((v4i32)R, 0);
741 R = SLDI_UB(R, R, 1);
742 val2 = __msa_copy_s_w((v4i32)R, 0);
743 R = SLDI_UB(R, R, 1);
744 val1 = __msa_copy_s_w((v4i32)R, 0);
745 R = SLDI_UB(R, R, 1);
746 val0 = __msa_copy_s_w((v4i32)R, 0);
747 SW4(val0, val1, val2, val3, dst, BPS);
748 }
749
LD4(uint8_t * dst)750 static void LD4(uint8_t* dst) { // Down-Left
751 const uint8_t* const ptop = dst - BPS;
752 uint32_t val0 = LW(ptop + 0);
753 uint32_t val1 = LW(ptop + 4);
754 uint32_t val2, val3;
755 v16u8 A = { 0 }, B, C, AC, B2, R;
756
757 INSERT_W2_UB(val0, val1, A);
758 B = SLDI_UB(A, A, 1);
759 C = SLDI_UB(A, A, 2);
760 C = (v16u8)__msa_insert_b((v16i8)C, 6, ptop[7]);
761 AC = __msa_ave_u_b(A, C);
762 B2 = __msa_ave_u_b(B, B);
763 R = __msa_aver_u_b(AC, B2);
764 val0 = __msa_copy_s_w((v4i32)R, 0);
765 R = SLDI_UB(R, R, 1);
766 val1 = __msa_copy_s_w((v4i32)R, 0);
767 R = SLDI_UB(R, R, 1);
768 val2 = __msa_copy_s_w((v4i32)R, 0);
769 R = SLDI_UB(R, R, 1);
770 val3 = __msa_copy_s_w((v4i32)R, 0);
771 SW4(val0, val1, val2, val3, dst, BPS);
772 }
773
774 // 16x16
775
DC16(uint8_t * dst)776 static void DC16(uint8_t* dst) { // DC
777 uint32_t dc = 16;
778 int i;
779 const v16u8 rtop = LD_UB(dst - BPS);
780 const v8u16 dctop = __msa_hadd_u_h(rtop, rtop);
781 v16u8 out;
782
783 for (i = 0; i < 16; ++i) {
784 dc += dst[-1 + i * BPS];
785 }
786 dc += HADD_UH_U32(dctop);
787 out = (v16u8)__msa_fill_b(dc >> 5);
788 ST_UB8(out, out, out, out, out, out, out, out, dst, BPS);
789 ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS);
790 }
791
TM16(uint8_t * dst)792 static void TM16(uint8_t* dst) {
793 int j;
794 v8i16 d1, d2;
795 const v16i8 zero = { 0 };
796 const v8i16 TL = (v8i16)__msa_fill_h(dst[-1 - BPS]);
797 const v16i8 T = LD_SB(dst - BPS);
798
799 ILVRL_B2_SH(zero, T, d1, d2);
800 SUB2(d1, TL, d2, TL, d1, d2);
801 for (j = 0; j < 16; j += 4) {
802 v16i8 t0, t1, t2, t3;
803 v8i16 r0, r1, r2, r3, r4, r5, r6, r7;
804 const v8i16 L0 = (v8i16)__msa_fill_h(dst[-1 + 0 * BPS]);
805 const v8i16 L1 = (v8i16)__msa_fill_h(dst[-1 + 1 * BPS]);
806 const v8i16 L2 = (v8i16)__msa_fill_h(dst[-1 + 2 * BPS]);
807 const v8i16 L3 = (v8i16)__msa_fill_h(dst[-1 + 3 * BPS]);
808 ADD4(d1, L0, d1, L1, d1, L2, d1, L3, r0, r1, r2, r3);
809 ADD4(d2, L0, d2, L1, d2, L2, d2, L3, r4, r5, r6, r7);
810 CLIP_SH4_0_255(r0, r1, r2, r3);
811 CLIP_SH4_0_255(r4, r5, r6, r7);
812 PCKEV_B4_SB(r4, r0, r5, r1, r6, r2, r7, r3, t0, t1, t2, t3);
813 ST_SB4(t0, t1, t2, t3, dst, BPS);
814 dst += 4 * BPS;
815 }
816 }
817
VE16(uint8_t * dst)818 static void VE16(uint8_t* dst) { // vertical
819 const v16u8 rtop = LD_UB(dst - BPS);
820 ST_UB8(rtop, rtop, rtop, rtop, rtop, rtop, rtop, rtop, dst, BPS);
821 ST_UB8(rtop, rtop, rtop, rtop, rtop, rtop, rtop, rtop, dst + 8 * BPS, BPS);
822 }
823
HE16(uint8_t * dst)824 static void HE16(uint8_t* dst) { // horizontal
825 int j;
826 for (j = 16; j > 0; j -= 4) {
827 const v16u8 L0 = (v16u8)__msa_fill_b(dst[-1 + 0 * BPS]);
828 const v16u8 L1 = (v16u8)__msa_fill_b(dst[-1 + 1 * BPS]);
829 const v16u8 L2 = (v16u8)__msa_fill_b(dst[-1 + 2 * BPS]);
830 const v16u8 L3 = (v16u8)__msa_fill_b(dst[-1 + 3 * BPS]);
831 ST_UB4(L0, L1, L2, L3, dst, BPS);
832 dst += 4 * BPS;
833 }
834 }
835
DC16NoTop(uint8_t * dst)836 static void DC16NoTop(uint8_t* dst) { // DC with top samples not available
837 int j;
838 uint32_t dc = 8;
839 v16u8 out;
840
841 for (j = 0; j < 16; ++j) {
842 dc += dst[-1 + j * BPS];
843 }
844 out = (v16u8)__msa_fill_b(dc >> 4);
845 ST_UB8(out, out, out, out, out, out, out, out, dst, BPS);
846 ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS);
847 }
848
DC16NoLeft(uint8_t * dst)849 static void DC16NoLeft(uint8_t* dst) { // DC with left samples not available
850 uint32_t dc = 8;
851 const v16u8 rtop = LD_UB(dst - BPS);
852 const v8u16 dctop = __msa_hadd_u_h(rtop, rtop);
853 v16u8 out;
854
855 dc += HADD_UH_U32(dctop);
856 out = (v16u8)__msa_fill_b(dc >> 4);
857 ST_UB8(out, out, out, out, out, out, out, out, dst, BPS);
858 ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS);
859 }
860
DC16NoTopLeft(uint8_t * dst)861 static void DC16NoTopLeft(uint8_t* dst) { // DC with nothing
862 const v16u8 out = (v16u8)__msa_fill_b(0x80);
863 ST_UB8(out, out, out, out, out, out, out, out, dst, BPS);
864 ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS);
865 }
866
867 // Chroma
868
869 #define STORE8x8(out, dst) do { \
870 SD4(out, out, out, out, dst + 0 * BPS, BPS); \
871 SD4(out, out, out, out, dst + 4 * BPS, BPS); \
872 } while (0)
873
DC8uv(uint8_t * dst)874 static void DC8uv(uint8_t* dst) { // DC
875 uint32_t dc = 8;
876 int i;
877 uint64_t out;
878 const v16u8 rtop = LD_UB(dst - BPS);
879 const v8u16 temp0 = __msa_hadd_u_h(rtop, rtop);
880 const v4u32 temp1 = __msa_hadd_u_w(temp0, temp0);
881 const v2u64 temp2 = __msa_hadd_u_d(temp1, temp1);
882 v16u8 dctemp;
883
884 for (i = 0; i < 8; ++i) {
885 dc += dst[-1 + i * BPS];
886 }
887 dc += __msa_copy_s_w((v4i32)temp2, 0);
888 dctemp = (v16u8)__msa_fill_b(dc >> 4);
889 out = __msa_copy_s_d((v2i64)dctemp, 0);
890 STORE8x8(out, dst);
891 }
892
TM8uv(uint8_t * dst)893 static void TM8uv(uint8_t* dst) {
894 int j;
895 const v16i8 T1 = LD_SB(dst - BPS);
896 const v16i8 zero = { 0 };
897 const v8i16 T = (v8i16)__msa_ilvr_b(zero, T1);
898 const v8i16 TL = (v8i16)__msa_fill_h(dst[-1 - BPS]);
899 const v8i16 d = T - TL;
900
901 for (j = 0; j < 8; j += 4) {
902 v16i8 t0, t1;
903 v8i16 r0 = (v8i16)__msa_fill_h(dst[-1 + 0 * BPS]);
904 v8i16 r1 = (v8i16)__msa_fill_h(dst[-1 + 1 * BPS]);
905 v8i16 r2 = (v8i16)__msa_fill_h(dst[-1 + 2 * BPS]);
906 v8i16 r3 = (v8i16)__msa_fill_h(dst[-1 + 3 * BPS]);
907 ADD4(d, r0, d, r1, d, r2, d, r3, r0, r1, r2, r3);
908 CLIP_SH4_0_255(r0, r1, r2, r3);
909 PCKEV_B2_SB(r1, r0, r3, r2, t0, t1);
910 ST4x4_UB(t0, t1, 0, 2, 0, 2, dst, BPS);
911 ST4x4_UB(t0, t1, 1, 3, 1, 3, dst + 4, BPS);
912 dst += 4 * BPS;
913 }
914 }
915
VE8uv(uint8_t * dst)916 static void VE8uv(uint8_t* dst) { // vertical
917 const v16u8 rtop = LD_UB(dst - BPS);
918 const uint64_t out = __msa_copy_s_d((v2i64)rtop, 0);
919 STORE8x8(out, dst);
920 }
921
HE8uv(uint8_t * dst)922 static void HE8uv(uint8_t* dst) { // horizontal
923 int j;
924 for (j = 0; j < 8; j += 4) {
925 const v16u8 L0 = (v16u8)__msa_fill_b(dst[-1 + 0 * BPS]);
926 const v16u8 L1 = (v16u8)__msa_fill_b(dst[-1 + 1 * BPS]);
927 const v16u8 L2 = (v16u8)__msa_fill_b(dst[-1 + 2 * BPS]);
928 const v16u8 L3 = (v16u8)__msa_fill_b(dst[-1 + 3 * BPS]);
929 const uint64_t out0 = __msa_copy_s_d((v2i64)L0, 0);
930 const uint64_t out1 = __msa_copy_s_d((v2i64)L1, 0);
931 const uint64_t out2 = __msa_copy_s_d((v2i64)L2, 0);
932 const uint64_t out3 = __msa_copy_s_d((v2i64)L3, 0);
933 SD4(out0, out1, out2, out3, dst, BPS);
934 dst += 4 * BPS;
935 }
936 }
937
DC8uvNoLeft(uint8_t * dst)938 static void DC8uvNoLeft(uint8_t* dst) { // DC with no left samples
939 const uint32_t dc = 4;
940 const v16u8 rtop = LD_UB(dst - BPS);
941 const v8u16 temp0 = __msa_hadd_u_h(rtop, rtop);
942 const v4u32 temp1 = __msa_hadd_u_w(temp0, temp0);
943 const v2u64 temp2 = __msa_hadd_u_d(temp1, temp1);
944 const uint32_t sum_m = __msa_copy_s_w((v4i32)temp2, 0);
945 const v16u8 dcval = (v16u8)__msa_fill_b((dc + sum_m) >> 3);
946 const uint64_t out = __msa_copy_s_d((v2i64)dcval, 0);
947 STORE8x8(out, dst);
948 }
949
DC8uvNoTop(uint8_t * dst)950 static void DC8uvNoTop(uint8_t* dst) { // DC with no top samples
951 uint32_t dc = 4;
952 int i;
953 uint64_t out;
954 v16u8 dctemp;
955
956 for (i = 0; i < 8; ++i) {
957 dc += dst[-1 + i * BPS];
958 }
959 dctemp = (v16u8)__msa_fill_b(dc >> 3);
960 out = __msa_copy_s_d((v2i64)dctemp, 0);
961 STORE8x8(out, dst);
962 }
963
DC8uvNoTopLeft(uint8_t * dst)964 static void DC8uvNoTopLeft(uint8_t* dst) { // DC with nothing
965 const uint64_t out = 0x8080808080808080ULL;
966 STORE8x8(out, dst);
967 }
968
969 //------------------------------------------------------------------------------
970 // Entry point
971
972 extern void VP8DspInitMSA(void);
973
VP8DspInitMSA(void)974 WEBP_TSAN_IGNORE_FUNCTION void VP8DspInitMSA(void) {
975 VP8TransformWHT = TransformWHT;
976 VP8Transform = TransformTwo;
977 VP8TransformDC = TransformDC;
978 VP8TransformAC3 = TransformAC3;
979
980 VP8VFilter16 = VFilter16;
981 VP8HFilter16 = HFilter16;
982 VP8VFilter16i = VFilter16i;
983 VP8HFilter16i = HFilter16i;
984 VP8VFilter8 = VFilter8;
985 VP8HFilter8 = HFilter8;
986 VP8VFilter8i = VFilter8i;
987 VP8HFilter8i = HFilter8i;
988 VP8SimpleVFilter16 = SimpleVFilter16;
989 VP8SimpleHFilter16 = SimpleHFilter16;
990 VP8SimpleVFilter16i = SimpleVFilter16i;
991 VP8SimpleHFilter16i = SimpleHFilter16i;
992
993 VP8PredLuma4[0] = DC4;
994 VP8PredLuma4[1] = TM4;
995 VP8PredLuma4[2] = VE4;
996 VP8PredLuma4[4] = RD4;
997 VP8PredLuma4[6] = LD4;
998 VP8PredLuma16[0] = DC16;
999 VP8PredLuma16[1] = TM16;
1000 VP8PredLuma16[2] = VE16;
1001 VP8PredLuma16[3] = HE16;
1002 VP8PredLuma16[4] = DC16NoTop;
1003 VP8PredLuma16[5] = DC16NoLeft;
1004 VP8PredLuma16[6] = DC16NoTopLeft;
1005 VP8PredChroma8[0] = DC8uv;
1006 VP8PredChroma8[1] = TM8uv;
1007 VP8PredChroma8[2] = VE8uv;
1008 VP8PredChroma8[3] = HE8uv;
1009 VP8PredChroma8[4] = DC8uvNoTop;
1010 VP8PredChroma8[5] = DC8uvNoLeft;
1011 VP8PredChroma8[6] = DC8uvNoTopLeft;
1012 }
1013
1014 #else // !WEBP_USE_MSA
1015
1016 WEBP_DSP_INIT_STUB(VP8DspInitMSA)
1017
1018 #endif // WEBP_USE_MSA
1019