1 /*
2 * Copyright 2018 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8 #include "src/gpu/ganesh/effects/GrYUVtoRGBEffect.h"
9
10 #include "include/core/SkAlphaType.h"
11 #include "include/core/SkImageInfo.h"
12 #include "include/core/SkRect.h"
13 #include "include/core/SkSamplingOptions.h"
14 #include "include/core/SkYUVAInfo.h"
15 #include "include/private/SkSLSampleUsage.h"
16 #include "include/private/base/SkAssert.h"
17 #include "include/private/base/SkTo.h"
18 #include "include/private/gpu/ganesh/GrTypesPriv.h"
19 #include "src/core/SkSLTypeShared.h"
20 #include "src/core/SkYUVAInfoLocation.h"
21 #include "src/core/SkYUVMath.h"
22 #include "src/gpu/KeyBuilder.h"
23 #include "src/gpu/ganesh/GrSurfaceProxyView.h"
24 #include "src/gpu/ganesh/GrYUVATextureProxies.h"
25 #include "src/gpu/ganesh/effects/GrMatrixEffect.h"
26 #include "src/gpu/ganesh/effects/GrTextureEffect.h"
27 #include "src/gpu/ganesh/glsl/GrGLSLFragmentShaderBuilder.h"
28 #include "src/gpu/ganesh/glsl/GrGLSLProgramDataManager.h"
29 #include "src/gpu/ganesh/glsl/GrGLSLUniformHandler.h"
30
31 #include <algorithm>
32 #include <array>
33 #include <cmath>
34 #include <cstdint>
35 #include <string>
36 #include <utility>
37
38 struct GrShaderCaps;
39
border_colors(const GrYUVATextureProxies & yuvaProxies,float planeBorders[4][4])40 static void border_colors(const GrYUVATextureProxies& yuvaProxies, float planeBorders[4][4]) {
41 float m[20];
42 SkColorMatrix_RGB2YUV(yuvaProxies.yuvaInfo().yuvColorSpace(), m);
43 for (int i = 0; i < SkYUVAInfo::kYUVAChannelCount; ++i) {
44 auto [plane, channel] = yuvaProxies.yuvaLocations()[i];
45 if (plane == -1) {
46 return;
47 }
48 auto c = static_cast<int>(channel);
49 planeBorders[plane][c] = m[i*5 + 4];
50 }
51 }
52
Make(const GrYUVATextureProxies & yuvaProxies,GrSamplerState samplerState,const GrCaps & caps,const SkMatrix & localMatrix,const SkRect * subset,const SkRect * domain)53 std::unique_ptr<GrFragmentProcessor> GrYUVtoRGBEffect::Make(const GrYUVATextureProxies& yuvaProxies,
54 GrSamplerState samplerState,
55 const GrCaps& caps,
56 const SkMatrix& localMatrix,
57 const SkRect* subset,
58 const SkRect* domain) {
59 SkASSERT(!subset || SkRect::Make(yuvaProxies.yuvaInfo().dimensions()).contains(*subset));
60
61 int numPlanes = yuvaProxies.yuvaInfo().numPlanes();
62 if (!yuvaProxies.isValid()) {
63 return nullptr;
64 }
65
66 bool usesBorder = samplerState.wrapModeX() == GrSamplerState::WrapMode::kClampToBorder ||
67 samplerState.wrapModeY() == GrSamplerState::WrapMode::kClampToBorder;
68 float planeBorders[4][4] = {};
69 if (usesBorder) {
70 border_colors(yuvaProxies, planeBorders);
71 }
72
73 bool snap[2] = {false, false};
74 std::unique_ptr<GrFragmentProcessor> planeFPs[SkYUVAInfo::kMaxPlanes];
75 for (int i = 0; i < numPlanes; ++i) {
76 bool useSubset = SkToBool(subset);
77 GrSurfaceProxyView view = yuvaProxies.makeView(i);
78 SkMatrix planeMatrix = yuvaProxies.yuvaInfo().originMatrix();
79 // The returned matrix is a view matrix but we need a local matrix.
80 SkAssertResult(planeMatrix.invert(&planeMatrix));
81 SkRect planeSubset;
82 SkRect planeDomain;
83 bool makeLinearWithSnap = false;
84 auto [ssx, ssy] = yuvaProxies.yuvaInfo().planeSubsamplingFactors(i);
85 SkASSERT(ssx > 0 && ssx <= 4);
86 SkASSERT(ssy > 0 && ssy <= 2);
87 float scaleX = 1.f;
88 float scaleY = 1.f;
89 if (ssx > 1 || ssy > 1) {
90 scaleX = 1.f/ssx;
91 scaleY = 1.f/ssy;
92 // We would want to add a translation to this matrix to handle other sitings.
93 SkASSERT(yuvaProxies.yuvaInfo().sitingX() == SkYUVAInfo::Siting::kCentered);
94 SkASSERT(yuvaProxies.yuvaInfo().sitingY() == SkYUVAInfo::Siting::kCentered);
95 planeMatrix.postConcat(SkMatrix::Scale(scaleX, scaleY));
96 if (subset) {
97 planeSubset = {subset->fLeft *scaleX,
98 subset->fTop *scaleY,
99 subset->fRight *scaleX,
100 subset->fBottom*scaleY};
101 } else {
102 planeSubset = SkRect::Make(view.dimensions());
103 }
104 if (domain) {
105 planeDomain = {domain->fLeft *scaleX,
106 domain->fTop *scaleY,
107 domain->fRight *scaleX,
108 domain->fBottom*scaleY};
109 }
110 // If the image is not a multiple of the subsampling then the subsampled plane needs to
111 // be tiled at less than its full width/height. This only matters when the mode is not
112 // clamp.
113 if (samplerState.wrapModeX() != GrSamplerState::WrapMode::kClamp) {
114 int dx = (ssx*view.width() - yuvaProxies.yuvaInfo().width());
115 float maxRight = view.width() - dx*scaleX;
116 if (planeSubset.fRight > maxRight) {
117 planeSubset.fRight = maxRight;
118 useSubset = true;
119 }
120 }
121 if (samplerState.wrapModeY() != GrSamplerState::WrapMode::kClamp) {
122 int dy = (ssy*view.height() - yuvaProxies.yuvaInfo().height());
123 float maxBottom = view.height() - dy*scaleY;
124 if (planeSubset.fBottom > maxBottom) {
125 planeSubset.fBottom = maxBottom;
126 useSubset = true;
127 }
128 }
129 // This promotion of nearest to linear filtering for UV planes exists to mimic
130 // libjpeg[-turbo]'s do_fancy_upsampling option. We will filter the subsampled plane,
131 // however we want to filter at a fixed point for each logical image pixel to simulate
132 // nearest neighbor.
133 if (samplerState.filter() == GrSamplerState::Filter::kNearest) {
134 bool snapX = (ssx != 1),
135 snapY = (ssy != 1);
136 makeLinearWithSnap = snapX || snapY;
137 snap[0] |= snapX;
138 snap[1] |= snapY;
139 if (domain) {
140 // The outer YUVToRGB effect will ensure sampling happens at pixel centers
141 // within this plane.
142 planeDomain = {std::floor(planeDomain.fLeft) + 0.5f,
143 std::floor(planeDomain.fTop) + 0.5f,
144 std::floor(planeDomain.fRight) + 0.5f,
145 std::floor(planeDomain.fBottom) + 0.5f};
146 }
147 }
148 } else {
149 if (subset) {
150 planeSubset = *subset;
151 }
152 if (domain) {
153 planeDomain = *domain;
154 }
155 }
156 if (useSubset) {
157 if (makeLinearWithSnap) {
158 // The plane is subsampled and we have an overall subset on the image. We're
159 // emulating do_fancy_upsampling using linear filtering but snapping look ups to the
160 // y-plane pixel centers. Consider a logical image pixel at the edge of the subset.
161 // When computing the logical pixel color value we should use a 50/50 blend of two
162 // values from the subsampled plane. Depending on where the subset edge falls in
163 // actual subsampled plane, one of those values may come from outside the subset.
164 // Hence, we use this custom inset factory which applies the wrap mode to
165 // planeSubset but allows linear filtering to read pixels from the plane that are
166 // just outside planeSubset.
167 SkRect* domainRect = domain ? &planeDomain : nullptr;
168 planeFPs[i] = GrTextureEffect::MakeCustomLinearFilterInset(std::move(view),
169 kUnknown_SkAlphaType,
170 planeMatrix,
171 samplerState.wrapModeX(),
172 samplerState.wrapModeY(),
173 planeSubset,
174 domainRect,
175 {scaleX/2.f, scaleY/2.f},
176 caps,
177 planeBorders[i]);
178 } else if (domain) {
179 planeFPs[i] = GrTextureEffect::MakeSubset(std::move(view),
180 kUnknown_SkAlphaType,
181 planeMatrix,
182 samplerState,
183 planeSubset,
184 planeDomain,
185 caps,
186 planeBorders[i]);
187 } else {
188 planeFPs[i] = GrTextureEffect::MakeSubset(std::move(view),
189 kUnknown_SkAlphaType,
190 planeMatrix,
191 samplerState,
192 planeSubset,
193 caps,
194 planeBorders[i]);
195 }
196 } else {
197 GrSamplerState planeSampler = samplerState;
198 if (makeLinearWithSnap) {
199 planeSampler = GrSamplerState(samplerState.wrapModeX(),
200 samplerState.wrapModeY(),
201 GrSamplerState::Filter::kLinear,
202 samplerState.mipmapMode());
203 }
204 planeFPs[i] = GrTextureEffect::Make(std::move(view),
205 kUnknown_SkAlphaType,
206 planeMatrix,
207 planeSampler,
208 caps,
209 planeBorders[i]);
210 }
211 }
212 std::unique_ptr<GrFragmentProcessor> fp(
213 new GrYUVtoRGBEffect(planeFPs,
214 numPlanes,
215 yuvaProxies.yuvaLocations(),
216 snap,
217 yuvaProxies.yuvaInfo().yuvColorSpace()));
218 return GrMatrixEffect::Make(localMatrix, std::move(fp));
219 }
220
alpha_type(const SkYUVAInfo::YUVALocations locations)221 static SkAlphaType alpha_type(const SkYUVAInfo::YUVALocations locations) {
222 return locations[SkYUVAInfo::YUVAChannels::kA].fPlane >= 0 ? kPremul_SkAlphaType
223 : kOpaque_SkAlphaType;
224 }
225
GrYUVtoRGBEffect(std::unique_ptr<GrFragmentProcessor> planeFPs[4],int numPlanes,const SkYUVAInfo::YUVALocations & locations,const bool snap[2],SkYUVColorSpace yuvColorSpace)226 GrYUVtoRGBEffect::GrYUVtoRGBEffect(std::unique_ptr<GrFragmentProcessor> planeFPs[4],
227 int numPlanes,
228 const SkYUVAInfo::YUVALocations& locations,
229 const bool snap[2],
230 SkYUVColorSpace yuvColorSpace)
231 : GrFragmentProcessor(kGrYUVtoRGBEffect_ClassID,
232 ModulateForClampedSamplerOptFlags(alpha_type(locations)))
233 , fLocations(locations)
234 , fYUVColorSpace(yuvColorSpace) {
235 std::copy_n(snap, 2, fSnap);
236
237 if (fSnap[0] || fSnap[1]) {
238 // Need this so that we can access coords in SKSL to perform snapping.
239 this->setUsesSampleCoordsDirectly();
240 for (int i = 0; i < numPlanes; ++i) {
241 this->registerChild(std::move(planeFPs[i]), SkSL::SampleUsage::Explicit());
242 }
243 } else {
244 for (int i = 0; i < numPlanes; ++i) {
245 this->registerChild(std::move(planeFPs[i]));
246 }
247 }
248 }
249
250 #if defined(GPU_TEST_UTILS)
onDumpInfo() const251 SkString GrYUVtoRGBEffect::onDumpInfo() const {
252 SkString str("(");
253 for (int i = 0; i < SkYUVAInfo::kYUVAChannelCount; ++i) {
254 str.appendf("Locations[%d]=%d %d, ",
255 i, fLocations[i].fPlane, static_cast<int>(fLocations[i].fChannel));
256 }
257 str.appendf("YUVColorSpace=%d, snap=(%d, %d))",
258 static_cast<int>(fYUVColorSpace), fSnap[0], fSnap[1]);
259 return str;
260 }
261 #endif
262
onMakeProgramImpl() const263 std::unique_ptr<GrFragmentProcessor::ProgramImpl> GrYUVtoRGBEffect::onMakeProgramImpl() const {
264 class Impl : public ProgramImpl {
265 public:
266 void emitCode(EmitArgs& args) override {
267 GrGLSLFPFragmentBuilder* fragBuilder = args.fFragBuilder;
268 const GrYUVtoRGBEffect& yuvEffect = args.fFp.cast<GrYUVtoRGBEffect>();
269
270 int numPlanes = yuvEffect.numChildProcessors();
271
272 const char* sampleCoords = "";
273 if (yuvEffect.fSnap[0] || yuvEffect.fSnap[1]) {
274 fragBuilder->codeAppendf("float2 snappedCoords = %s;", args.fSampleCoord);
275 if (yuvEffect.fSnap[0]) {
276 fragBuilder->codeAppend("snappedCoords.x = floor(snappedCoords.x) + 0.5;");
277 }
278 if (yuvEffect.fSnap[1]) {
279 fragBuilder->codeAppend("snappedCoords.y = floor(snappedCoords.y) + 0.5;");
280 }
281 sampleCoords = "snappedCoords";
282 }
283
284 fragBuilder->codeAppendf("half4 color;");
285 const bool hasAlpha = yuvEffect.fLocations[SkYUVAInfo::YUVAChannels::kA].fPlane >= 0;
286
287 for (int planeIdx = 0; planeIdx < numPlanes; ++planeIdx) {
288 std::string colorChannel;
289 std::string planeChannel;
290 for (int locIdx = 0; locIdx < (hasAlpha ? 4 : 3); ++locIdx) {
291 auto [yuvPlane, yuvChannel] = yuvEffect.fLocations[locIdx];
292 if (yuvPlane == planeIdx) {
293 colorChannel.push_back("rgba"[locIdx]);
294 planeChannel.push_back("rgba"[static_cast<int>(yuvChannel)]);
295 }
296 }
297
298 SkASSERT(colorChannel.size() == planeChannel.size());
299 if (!colorChannel.empty()) {
300 fragBuilder->codeAppendf(
301 "color.%s = (%s).%s;",
302 colorChannel.c_str(),
303 this->invokeChild(planeIdx, args, sampleCoords).c_str(),
304 planeChannel.c_str());
305 }
306 }
307
308 if (!hasAlpha) {
309 fragBuilder->codeAppendf("color.a = 1;");
310 }
311
312 if (kIdentity_SkYUVColorSpace != yuvEffect.fYUVColorSpace) {
313 fColorSpaceMatrixVar = args.fUniformHandler->addUniform(&yuvEffect,
314 kFragment_GrShaderFlag, SkSLType::kHalf3x3, "colorSpaceMatrix");
315 fColorSpaceTranslateVar = args.fUniformHandler->addUniform(&yuvEffect,
316 kFragment_GrShaderFlag, SkSLType::kHalf3, "colorSpaceTranslate");
317 fragBuilder->codeAppendf(
318 "color.rgb = saturate(color.rgb * %s + %s);",
319 args.fUniformHandler->getUniformCStr(fColorSpaceMatrixVar),
320 args.fUniformHandler->getUniformCStr(fColorSpaceTranslateVar));
321 }
322 if (hasAlpha) {
323 // premultiply alpha
324 fragBuilder->codeAppendf("color.rgb *= color.a;");
325 }
326 fragBuilder->codeAppendf("return color;");
327 }
328
329 private:
330 void onSetData(const GrGLSLProgramDataManager& pdman,
331 const GrFragmentProcessor& proc) override {
332 const GrYUVtoRGBEffect& yuvEffect = proc.cast<GrYUVtoRGBEffect>();
333
334 if (yuvEffect.fYUVColorSpace != kIdentity_SkYUVColorSpace) {
335 SkASSERT(fColorSpaceMatrixVar.isValid());
336 float yuvM[20];
337 SkColorMatrix_YUV2RGB(yuvEffect.fYUVColorSpace, yuvM);
338 // We drop the fourth column entirely since the transformation
339 // should not depend on alpha. The fifth column is sent as a separate
340 // vector. The fourth row is also dropped entirely because alpha should
341 // never be modified.
342 SkASSERT(yuvM[3] == 0 && yuvM[8] == 0 && yuvM[13] == 0 && yuvM[18] == 1);
343 SkASSERT(yuvM[15] == 0 && yuvM[16] == 0 && yuvM[17] == 0 && yuvM[19] == 0);
344 float mtx[9] = {
345 yuvM[ 0], yuvM[ 1], yuvM[ 2],
346 yuvM[ 5], yuvM[ 6], yuvM[ 7],
347 yuvM[10], yuvM[11], yuvM[12],
348 };
349 float v[3] = {yuvM[4], yuvM[9], yuvM[14]};
350 pdman.setMatrix3f(fColorSpaceMatrixVar, mtx);
351 pdman.set3fv(fColorSpaceTranslateVar, 1, v);
352 }
353 }
354
355 UniformHandle fColorSpaceMatrixVar;
356 UniformHandle fColorSpaceTranslateVar;
357 };
358
359 return std::make_unique<Impl>();
360 }
361
onAddToKey(const GrShaderCaps & caps,skgpu::KeyBuilder * b) const362 void GrYUVtoRGBEffect::onAddToKey(const GrShaderCaps& caps, skgpu::KeyBuilder* b) const {
363 uint32_t packed = 0;
364 int i = 0;
365 for (auto [plane, channel] : fLocations) {
366 if (plane < 0) {
367 continue;
368 }
369
370 uint8_t chann = static_cast<int>(channel);
371
372 SkASSERT(plane < 4 && chann < 4);
373
374 packed |= (plane | (chann << 2)) << (i++ * 4);
375 }
376 if (fYUVColorSpace == kIdentity_SkYUVColorSpace) {
377 packed |= 1 << 16;
378 }
379 if (fSnap[0]) {
380 packed |= 1 << 17;
381 }
382 if (fSnap[1]) {
383 packed |= 1 << 18;
384 }
385 b->add32(packed);
386 }
387
onIsEqual(const GrFragmentProcessor & other) const388 bool GrYUVtoRGBEffect::onIsEqual(const GrFragmentProcessor& other) const {
389 const GrYUVtoRGBEffect& that = other.cast<GrYUVtoRGBEffect>();
390
391 return fLocations == that.fLocations &&
392 std::equal(fSnap, fSnap + 2, that.fSnap) &&
393 fYUVColorSpace == that.fYUVColorSpace;
394 }
395
GrYUVtoRGBEffect(const GrYUVtoRGBEffect & src)396 GrYUVtoRGBEffect::GrYUVtoRGBEffect(const GrYUVtoRGBEffect& src)
397 : GrFragmentProcessor(src)
398 , fLocations((src.fLocations))
399 , fYUVColorSpace(src.fYUVColorSpace) {
400 std::copy_n(src.fSnap, 2, fSnap);
401 }
402
clone() const403 std::unique_ptr<GrFragmentProcessor> GrYUVtoRGBEffect::clone() const {
404 return std::unique_ptr<GrFragmentProcessor>(new GrYUVtoRGBEffect(*this));
405 }
406