1 // Copyright 2015 Google Inc. All Rights Reserved.
2 //
3 // Use of this source code is governed by a BSD-style license
4 // that can be found in the COPYING file in the root of the source
5 // tree. An additional intellectual property rights grant can be found
6 // in the file PATENTS. All contributing project authors may
7 // be found in the AUTHORS file in the root of the source tree.
8 // -----------------------------------------------------------------------------
9 //
10 // SSE4.1 variant of methods for lossless encoder
11 //
12 // Author: Skal ([email protected])
13
14 #include "src/dsp/dsp.h"
15
16 #if defined(WEBP_USE_SSE41)
17 #include <assert.h>
18 #include <smmintrin.h>
19 #include "src/dsp/lossless.h"
20
21 //------------------------------------------------------------------------------
22 // Cost operations.
23
HorizontalSum_SSE41(__m128i cost)24 static WEBP_INLINE uint32_t HorizontalSum_SSE41(__m128i cost) {
25 cost = _mm_add_epi32(cost, _mm_srli_si128(cost, 8));
26 cost = _mm_add_epi32(cost, _mm_srli_si128(cost, 4));
27 return _mm_cvtsi128_si32(cost);
28 }
29
ExtraCost_SSE41(const uint32_t * const a,int length)30 static uint32_t ExtraCost_SSE41(const uint32_t* const a, int length) {
31 int i;
32 __m128i cost = _mm_set_epi32(2 * a[7], 2 * a[6], a[5], a[4]);
33 assert(length % 8 == 0);
34
35 for (i = 8; i + 8 <= length; i += 8) {
36 const int j = (i - 2) >> 1;
37 const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i]);
38 const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]);
39 const __m128i w = _mm_set_epi32(j + 3, j + 2, j + 1, j);
40 const __m128i a2 = _mm_hadd_epi32(a0, a1);
41 const __m128i mul = _mm_mullo_epi32(a2, w);
42 cost = _mm_add_epi32(mul, cost);
43 }
44 return HorizontalSum_SSE41(cost);
45 }
46
ExtraCostCombined_SSE41(const uint32_t * const a,const uint32_t * const b,int length)47 static uint32_t ExtraCostCombined_SSE41(const uint32_t* const a,
48 const uint32_t* const b, int length) {
49 int i;
50 __m128i cost = _mm_add_epi32(_mm_set_epi32(2 * a[7], 2 * a[6], a[5], a[4]),
51 _mm_set_epi32(2 * b[7], 2 * b[6], b[5], b[4]));
52 assert(length % 8 == 0);
53
54 for (i = 8; i + 8 <= length; i += 8) {
55 const int j = (i - 2) >> 1;
56 const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i]);
57 const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]);
58 const __m128i b0 = _mm_loadu_si128((const __m128i*)&b[i]);
59 const __m128i b1 = _mm_loadu_si128((const __m128i*)&b[i + 4]);
60 const __m128i w = _mm_set_epi32(j + 3, j + 2, j + 1, j);
61 const __m128i a2 = _mm_hadd_epi32(a0, a1);
62 const __m128i b2 = _mm_hadd_epi32(b0, b1);
63 const __m128i mul = _mm_mullo_epi32(_mm_add_epi32(a2, b2), w);
64 cost = _mm_add_epi32(mul, cost);
65 }
66 return HorizontalSum_SSE41(cost);
67 }
68
69 //------------------------------------------------------------------------------
70 // Subtract-Green Transform
71
SubtractGreenFromBlueAndRed_SSE41(uint32_t * argb_data,int num_pixels)72 static void SubtractGreenFromBlueAndRed_SSE41(uint32_t* argb_data,
73 int num_pixels) {
74 int i;
75 const __m128i kCstShuffle = _mm_set_epi8(-1, 13, -1, 13, -1, 9, -1, 9,
76 -1, 5, -1, 5, -1, 1, -1, 1);
77 for (i = 0; i + 4 <= num_pixels; i += 4) {
78 const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]);
79 const __m128i in_0g0g = _mm_shuffle_epi8(in, kCstShuffle);
80 const __m128i out = _mm_sub_epi8(in, in_0g0g);
81 _mm_storeu_si128((__m128i*)&argb_data[i], out);
82 }
83 // fallthrough and finish off with plain-C
84 if (i != num_pixels) {
85 VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i);
86 }
87 }
88
89 //------------------------------------------------------------------------------
90 // Color Transform
91
92 // For sign-extended multiplying constants, pre-shifted by 5:
93 #define CST_5b(X) (((int16_t)((uint16_t)(X) << 8)) >> 5)
94
95 #define MK_CST_16(HI, LO) \
96 _mm_set1_epi32((int)(((uint32_t)(HI) << 16) | ((LO) & 0xffff)))
97
CollectColorBlueTransforms_SSE41(const uint32_t * argb,int stride,int tile_width,int tile_height,int green_to_blue,int red_to_blue,int histo[])98 static void CollectColorBlueTransforms_SSE41(const uint32_t* argb, int stride,
99 int tile_width, int tile_height,
100 int green_to_blue, int red_to_blue,
101 int histo[]) {
102 const __m128i mult =
103 MK_CST_16(CST_5b(red_to_blue) + 256,CST_5b(green_to_blue));
104 const __m128i perm =
105 _mm_setr_epi8(-1, 1, -1, 2, -1, 5, -1, 6, -1, 9, -1, 10, -1, 13, -1, 14);
106 if (tile_width >= 4) {
107 int y;
108 for (y = 0; y < tile_height; ++y) {
109 const uint32_t* const src = argb + y * stride;
110 const __m128i A1 = _mm_loadu_si128((const __m128i*)src);
111 const __m128i B1 = _mm_shuffle_epi8(A1, perm);
112 const __m128i C1 = _mm_mulhi_epi16(B1, mult);
113 const __m128i D1 = _mm_sub_epi16(A1, C1);
114 __m128i E = _mm_add_epi16(_mm_srli_epi32(D1, 16), D1);
115 int x;
116 for (x = 4; x + 4 <= tile_width; x += 4) {
117 const __m128i A2 = _mm_loadu_si128((const __m128i*)(src + x));
118 __m128i B2, C2, D2;
119 ++histo[_mm_extract_epi8(E, 0)];
120 B2 = _mm_shuffle_epi8(A2, perm);
121 ++histo[_mm_extract_epi8(E, 4)];
122 C2 = _mm_mulhi_epi16(B2, mult);
123 ++histo[_mm_extract_epi8(E, 8)];
124 D2 = _mm_sub_epi16(A2, C2);
125 ++histo[_mm_extract_epi8(E, 12)];
126 E = _mm_add_epi16(_mm_srli_epi32(D2, 16), D2);
127 }
128 ++histo[_mm_extract_epi8(E, 0)];
129 ++histo[_mm_extract_epi8(E, 4)];
130 ++histo[_mm_extract_epi8(E, 8)];
131 ++histo[_mm_extract_epi8(E, 12)];
132 }
133 }
134 {
135 const int left_over = tile_width & 3;
136 if (left_over > 0) {
137 VP8LCollectColorBlueTransforms_C(argb + tile_width - left_over, stride,
138 left_over, tile_height,
139 green_to_blue, red_to_blue, histo);
140 }
141 }
142 }
143
CollectColorRedTransforms_SSE41(const uint32_t * argb,int stride,int tile_width,int tile_height,int green_to_red,int histo[])144 static void CollectColorRedTransforms_SSE41(const uint32_t* argb, int stride,
145 int tile_width, int tile_height,
146 int green_to_red, int histo[]) {
147
148 const __m128i mult = MK_CST_16(0, CST_5b(green_to_red));
149 const __m128i mask_g = _mm_set1_epi32(0x0000ff00);
150 if (tile_width >= 4) {
151 int y;
152 for (y = 0; y < tile_height; ++y) {
153 const uint32_t* const src = argb + y * stride;
154 const __m128i A1 = _mm_loadu_si128((const __m128i*)src);
155 const __m128i B1 = _mm_and_si128(A1, mask_g);
156 const __m128i C1 = _mm_madd_epi16(B1, mult);
157 __m128i D = _mm_sub_epi16(A1, C1);
158 int x;
159 for (x = 4; x + 4 <= tile_width; x += 4) {
160 const __m128i A2 = _mm_loadu_si128((const __m128i*)(src + x));
161 __m128i B2, C2;
162 ++histo[_mm_extract_epi8(D, 2)];
163 B2 = _mm_and_si128(A2, mask_g);
164 ++histo[_mm_extract_epi8(D, 6)];
165 C2 = _mm_madd_epi16(B2, mult);
166 ++histo[_mm_extract_epi8(D, 10)];
167 ++histo[_mm_extract_epi8(D, 14)];
168 D = _mm_sub_epi16(A2, C2);
169 }
170 ++histo[_mm_extract_epi8(D, 2)];
171 ++histo[_mm_extract_epi8(D, 6)];
172 ++histo[_mm_extract_epi8(D, 10)];
173 ++histo[_mm_extract_epi8(D, 14)];
174 }
175 }
176 {
177 const int left_over = tile_width & 3;
178 if (left_over > 0) {
179 VP8LCollectColorRedTransforms_C(argb + tile_width - left_over, stride,
180 left_over, tile_height, green_to_red,
181 histo);
182 }
183 }
184 }
185
186 #undef MK_CST_16
187
188 //------------------------------------------------------------------------------
189 // Entry point
190
191 extern void VP8LEncDspInitSSE41(void);
192
VP8LEncDspInitSSE41(void)193 WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE41(void) {
194 VP8LExtraCost = ExtraCost_SSE41;
195 VP8LExtraCostCombined = ExtraCostCombined_SSE41;
196 VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed_SSE41;
197 VP8LCollectColorBlueTransforms = CollectColorBlueTransforms_SSE41;
198 VP8LCollectColorRedTransforms = CollectColorRedTransforms_SSE41;
199 }
200
201 #else // !WEBP_USE_SSE41
202
203 WEBP_DSP_INIT_STUB(VP8LEncDspInitSSE41)
204
205 #endif // WEBP_USE_SSE41
206