1 /*
2 * Copyright 2024 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "ultrahdr/ultrahdrcommon.h"
18 #include "ultrahdr/gainmapmath.h"
19 #include "ultrahdr/jpegr.h"
20
21 namespace ultrahdr {
22
23 extern const std::string vertex_shader = R"__SHADER__(#version 300 es
24 precision highp float;
25
26 layout(location = 0) in vec4 aPos;
27 layout(location = 1) in vec2 aTexCoord;
28
29 out vec2 TexCoord;
30
31 void main() {
32 gl_Position = aPos;
33 TexCoord = aTexCoord;
34 }
35 )__SHADER__";
36
37 static const std::string getYuv444PixelShader = R"__SHADER__(
38 uniform sampler2D yuvTexture;
39 uniform int pWidth, pHeight;
40
41 vec3 getYUVPixel() {
42 // Convert texCoord to pixel coordinates
43 ivec2 pixelCoord = ivec2(TexCoord * vec2(pWidth, pHeight));
44
45 float y = texelFetch(yuvTexture, ivec2(pixelCoord.r, pixelCoord.g), 0).r;
46 float u = texelFetch(yuvTexture, ivec2(pixelCoord.r, pixelCoord.g + pHeight), 0).r;
47 float v = texelFetch(yuvTexture, ivec2(pixelCoord.r, pixelCoord.g + 2 * pHeight), 0).r;
48
49 return vec3(y, u, v);
50 }
51 )__SHADER__";
52
53 static const std::string getYuv422PixelShader = R"__SHADER__(
54 uniform sampler2D yuvTexture;
55 uniform int pWidth, pHeight;
56
57 vec3 getYUVPixel() {
58 // Convert texCoord to pixel coordinates
59 ivec2 pixelCoord = ivec2(TexCoord * vec2(pWidth, pHeight));
60 ivec2 uvCoord = ivec2(pixelCoord.r / 2, pixelCoord.g);
61 int uvWidth = pWidth / 2;
62 int uvHeight = pHeight;
63 uint yPlaneSize = uint(pWidth) * uint(pHeight);
64 uint uPlaneSize = uint(uvWidth) * uint(uvHeight);
65 uint yIndex = uint(pixelCoord.g * pWidth + pixelCoord.r);
66 uint uIndex = yPlaneSize + uint(uvCoord.g * uvWidth + uvCoord.r);
67 uint vIndex = yPlaneSize + uPlaneSize + uint(uvCoord.g * uvWidth + uvCoord.r);
68
69 float y = texelFetch(yuvTexture, ivec2(yIndex % uint(pWidth), yIndex / uint(pWidth)), 0).r;
70 float u = texelFetch(yuvTexture, ivec2(uIndex % uint(pWidth), uIndex / uint(pWidth)), 0).r;
71 float v = texelFetch(yuvTexture, ivec2(vIndex % uint(pWidth), vIndex / uint(pWidth)), 0).r;
72
73 return vec3(y, u, v);
74 }
75 )__SHADER__";
76
77 static const std::string getYuv420PixelShader = R"__SHADER__(
78 uniform sampler2D yuvTexture;
79 uniform int pWidth, pHeight;
80
81 vec3 getYUVPixel() {
82 // Convert texCoord to pixel coordinates
83 ivec2 pixelCoord = ivec2(TexCoord * vec2(pWidth, pHeight));
84 ivec2 uvCoord = pixelCoord / 2;
85 int uvWidth = pWidth / 2;
86 int uvHeight = pHeight / 2;
87 uint yPlaneSize = uint(pWidth) * uint(pHeight);
88 uint uPlaneSize = uint(uvWidth) * uint(uvHeight);
89 uint yIndex = uint(pixelCoord.g * pWidth + pixelCoord.r);
90 uint uIndex = yPlaneSize + uint(uvCoord.g * uvWidth + uvCoord.r);
91 uint vIndex = yPlaneSize + uPlaneSize + uint(uvCoord.g * uvWidth + uvCoord.r);
92
93 float y = texelFetch(yuvTexture, ivec2(yIndex % uint(pWidth), yIndex / uint(pWidth)), 0).r;
94 float u = texelFetch(yuvTexture, ivec2(uIndex % uint(pWidth), uIndex / uint(pWidth)), 0).r;
95 float v = texelFetch(yuvTexture, ivec2(vIndex % uint(pWidth), vIndex / uint(pWidth)), 0).r;
96
97 return vec3(y, u, v);
98 }
99 )__SHADER__";
100
101 static const std::string p3YUVToRGBShader = R"__SHADER__(
102 vec3 p3YuvToRgb(const vec3 color) {
103 const vec3 offset = vec3(0.0, 128.0f / 255.0f, 128.0f / 255.0f);
104 const mat3 transform = mat3(
105 1.0, 1.0, 1.0,
106 0.0, -0.344136286, 1.772,
107 1.402, -0.714136286, 0.0);
108 return clamp(transform * (color - offset), 0.0, 1.0);
109 }
110 )__SHADER__";
111
112 static const std::string sRGBEOTFShader = R"__SHADER__(
113 float sRGBEOTF(float e_gamma) {
114 return e_gamma <= 0.04045 ? e_gamma / 12.92 : pow((e_gamma + 0.055) / 1.055, 2.4);
115 }
116
117 vec3 sRGBEOTF(const vec3 e_gamma) {
118 return vec3(sRGBEOTF(e_gamma.r), sRGBEOTF(e_gamma.g), sRGBEOTF(e_gamma.b));
119 }
120 )__SHADER__";
121
122 static const std::string getGainMapSampleSingleChannel = R"__SHADER__(
123 uniform sampler2D gainMapTexture;
124
125 vec3 sampleMap(sampler2D map) { return vec3(texture(map, TexCoord).r); }
126 )__SHADER__";
127
128 static const std::string getGainMapSampleMultiChannel = R"__SHADER__(
129 uniform sampler2D gainMapTexture;
130
131 vec3 sampleMap(sampler2D map) { return texture(map, TexCoord).rgb; }
132 )__SHADER__";
133
134 static const std::string applyGainMapShader = R"__SHADER__(
135 uniform float gamma;
136 uniform float logMinBoost;
137 uniform float logMaxBoost;
138 uniform float weight;
139 uniform float offsetSdr;
140 uniform float offsetHdr;
141 uniform float normalize;
142
143 float applyGainMapSample(const float channel, float gain) {
144 gain = pow(gain, 1.0f / gamma);
145 float logBoost = logMinBoost * (1.0f - gain) + logMaxBoost * gain;
146 logBoost = exp2(logBoost * weight);
147 return ((channel + offsetSdr) * logBoost - offsetHdr) / normalize;
148 }
149
150 vec3 applyGain(const vec3 color, const vec3 gain) {
151 return vec3(applyGainMapSample(color.r, gain.r),
152 applyGainMapSample(color.g, gain.g),
153 applyGainMapSample(color.b, gain.b));
154 }
155 )__SHADER__";
156
157 static const std::string linearOETFShader = R"__SHADER__(
158 vec3 OETF(const vec3 linear) { return linear; }
159 )__SHADER__";
160
161 static const std::string hlgOETFShader = R"__SHADER__(
162 float OETF(const float linear) {
163 const float kHlgA = 0.17883277;
164 const float kHlgB = 0.28466892;
165 const float kHlgC = 0.55991073;
166 return linear <= 1.0 / 12.0 ? sqrt(3.0 * linear) : kHlgA * log(12.0 * linear - kHlgB) + kHlgC;
167 }
168
169 vec3 OETF(const vec3 linear) {
170 return vec3(OETF(linear.r), OETF(linear.g), OETF(linear.b));
171 }
172 )__SHADER__";
173
174 static const std::string pqOETFShader = R"__SHADER__(
175 vec3 OETF(const vec3 linear) {
176 const float kPqM1 = (2610.0 / 4096.0) / 4.0;
177 const float kPqM2 = (2523.0 / 4096.0) * 128.0;
178 const float kPqC1 = (3424.0 / 4096.0);
179 const float kPqC2 = (2413.0 / 4096.0) * 32.0;
180 const float kPqC3 = (2392.0 / 4096.0) * 32.0;
181 vec3 tmp = pow(linear, vec3(kPqM1));
182 tmp = (kPqC1 + kPqC2 * tmp) / (1.0 + kPqC3 * tmp);
183 return pow(tmp, vec3(kPqM2));
184 }
185 )__SHADER__";
186
187 static const std::string hlgInverseOOTFShader = R"__SHADER__(
188 float InverseOOTF(const float linear) {
189 const float kOotfGamma = 1.2f;
190 return pow(linear, 1.0f / kOotfGamma);
191 }
192
193 vec3 InverseOOTF(const vec3 linear) {
194 return vec3(InverseOOTF(linear.r), InverseOOTF(linear.g), InverseOOTF(linear.b));
195 }
196 )__SHADER__";
197
198 static const std::string IdentityInverseOOTFShader = R"__SHADER__(
199 vec3 InverseOOTF(const vec3 linear) { return linear; }
200 )__SHADER__";
201
getApplyGainMapFragmentShader(uhdr_img_fmt sdr_fmt,uhdr_img_fmt gm_fmt,uhdr_color_transfer output_ct)202 std::string getApplyGainMapFragmentShader(uhdr_img_fmt sdr_fmt, uhdr_img_fmt gm_fmt,
203 uhdr_color_transfer output_ct) {
204 std::string shader_code = R"__SHADER__(#version 300 es
205 precision highp float;
206 precision highp int;
207
208 out vec4 FragColor;
209 in vec2 TexCoord;
210 )__SHADER__";
211
212 if (sdr_fmt == UHDR_IMG_FMT_24bppYCbCr444) {
213 shader_code.append(getYuv444PixelShader);
214 } else if (sdr_fmt == UHDR_IMG_FMT_16bppYCbCr422) {
215 shader_code.append(getYuv422PixelShader);
216 } else if (sdr_fmt == UHDR_IMG_FMT_12bppYCbCr420) {
217 shader_code.append(getYuv420PixelShader);
218 }
219 shader_code.append(p3YUVToRGBShader);
220 shader_code.append(sRGBEOTFShader);
221 shader_code.append(gm_fmt == UHDR_IMG_FMT_8bppYCbCr400 ? getGainMapSampleSingleChannel
222 : getGainMapSampleMultiChannel);
223 shader_code.append(applyGainMapShader);
224 if (output_ct == UHDR_CT_LINEAR) {
225 shader_code.append(IdentityInverseOOTFShader);
226 shader_code.append(linearOETFShader);
227 } else if (output_ct == UHDR_CT_HLG) {
228 shader_code.append(hlgInverseOOTFShader);
229 shader_code.append(hlgOETFShader);
230 } else if (output_ct == UHDR_CT_PQ) {
231 shader_code.append(IdentityInverseOOTFShader);
232 shader_code.append(pqOETFShader);
233 }
234
235 shader_code.append(R"__SHADER__(
236 void main() {
237 vec3 yuv_gamma_sdr = getYUVPixel();
238 vec3 rgb_gamma_sdr = p3YuvToRgb(yuv_gamma_sdr);
239 vec3 rgb_sdr = sRGBEOTF(rgb_gamma_sdr);
240 vec3 gain = sampleMap(gainMapTexture);
241 vec3 rgb_hdr = applyGain(rgb_sdr, gain);
242 rgb_hdr = InverseOOTF(rgb_hdr);
243 vec3 rgb_gamma_hdr = OETF(rgb_hdr);
244 FragColor = vec4(rgb_gamma_hdr, 1.0);
245 }
246 )__SHADER__");
247 return shader_code;
248 }
249
isBufferDataContiguous(uhdr_raw_image_t * img)250 bool isBufferDataContiguous(uhdr_raw_image_t* img) {
251 if (img->fmt == UHDR_IMG_FMT_32bppRGBA8888 || img->fmt == UHDR_IMG_FMT_24bppRGB888 ||
252 img->fmt == UHDR_IMG_FMT_8bppYCbCr400 || img->fmt == UHDR_IMG_FMT_32bppRGBA1010102 ||
253 img->fmt == UHDR_IMG_FMT_64bppRGBAHalfFloat) {
254 return img->stride[UHDR_PLANE_PACKED] == img->w;
255 } else if (img->fmt == UHDR_IMG_FMT_24bppYCbCrP010) {
256 uint16_t* y = static_cast<uint16_t*>(img->planes[UHDR_PLANE_Y]);
257 uint16_t* u = static_cast<uint16_t*>(img->planes[UHDR_PLANE_UV]);
258 std::ptrdiff_t sz = u - y;
259 long pixels = img->w * img->h;
260 return img->stride[UHDR_PLANE_Y] == img->w && img->stride[UHDR_PLANE_UV] == img->w &&
261 sz == pixels;
262 } else if (img->fmt == UHDR_IMG_FMT_12bppYCbCr420 || img->fmt == UHDR_IMG_FMT_24bppYCbCr444 ||
263 img->fmt == UHDR_IMG_FMT_16bppYCbCr422) {
264 int h_samp_factor = img->fmt == UHDR_IMG_FMT_24bppYCbCr444 ? 1 : 2;
265 int v_samp_factor = img->fmt == UHDR_IMG_FMT_12bppYCbCr420 ? 2 : 1;
266 uint8_t* y = static_cast<uint8_t*>(img->planes[UHDR_PLANE_Y]);
267 uint8_t* u = static_cast<uint8_t*>(img->planes[UHDR_PLANE_U]);
268 uint8_t* v = static_cast<uint8_t*>(img->planes[UHDR_PLANE_V]);
269 std::ptrdiff_t sz_a = u - y, sz_b = v - u;
270 long pixels = img->w * img->h;
271 return img->stride[UHDR_PLANE_Y] == img->w &&
272 img->stride[UHDR_PLANE_U] == img->w / h_samp_factor &&
273 img->stride[UHDR_PLANE_V] == img->w / h_samp_factor && sz_a == pixels &&
274 sz_b == pixels / (h_samp_factor * v_samp_factor);
275 }
276 return false;
277 }
278
applyGainMapGLES(uhdr_raw_image_t * sdr_intent,uhdr_raw_image_t * gainmap_img,uhdr_gainmap_metadata_ext_t * gainmap_metadata,uhdr_color_transfer_t output_ct,float display_boost,uhdr_raw_image_t * dest,uhdr_opengl_ctxt_t * opengl_ctxt)279 uhdr_error_info_t applyGainMapGLES(uhdr_raw_image_t* sdr_intent, uhdr_raw_image_t* gainmap_img,
280 uhdr_gainmap_metadata_ext_t* gainmap_metadata,
281 uhdr_color_transfer_t output_ct, float display_boost,
282 uhdr_raw_image_t* dest, uhdr_opengl_ctxt_t* opengl_ctxt) {
283 GLuint shaderProgram = 0; // shader program
284 GLuint yuvTexture = 0; // sdr intent texture
285 GLuint frameBuffer = 0;
286
287 #define RET_IF_ERR() \
288 if (opengl_ctxt->mErrorStatus.error_code != UHDR_CODEC_OK) { \
289 if (frameBuffer) glDeleteFramebuffers(1, &frameBuffer); \
290 if (yuvTexture) glDeleteTextures(1, &yuvTexture); \
291 if (shaderProgram) glDeleteProgram(shaderProgram); \
292 return opengl_ctxt->mErrorStatus; \
293 }
294
295 shaderProgram = opengl_ctxt->create_shader_program(
296 vertex_shader.c_str(),
297 getApplyGainMapFragmentShader(sdr_intent->fmt, gainmap_img->fmt, output_ct).c_str());
298 RET_IF_ERR()
299
300 yuvTexture = opengl_ctxt->create_texture(sdr_intent->fmt, sdr_intent->w, sdr_intent->h,
301 sdr_intent->planes[0]);
302 opengl_ctxt->mGainmapImgTexture = opengl_ctxt->create_texture(
303 gainmap_img->fmt, gainmap_img->w, gainmap_img->h, gainmap_img->planes[0]);
304 opengl_ctxt->mDecodedImgTexture = opengl_ctxt->create_texture(
305 output_ct == UHDR_CT_LINEAR ? UHDR_IMG_FMT_64bppRGBAHalfFloat : UHDR_IMG_FMT_32bppRGBA1010102,
306 sdr_intent->w, sdr_intent->h, nullptr);
307 RET_IF_ERR()
308
309 frameBuffer = opengl_ctxt->setup_framebuffer(opengl_ctxt->mDecodedImgTexture);
310 RET_IF_ERR()
311
312 glViewport(0, 0, sdr_intent->w, sdr_intent->h);
313 glUseProgram(shaderProgram);
314
315 // Get the location of the uniform variables
316 GLint pWidthLocation = glGetUniformLocation(shaderProgram, "pWidth");
317 GLint pHeightLocation = glGetUniformLocation(shaderProgram, "pHeight");
318 GLint gammaLocation = glGetUniformLocation(shaderProgram, "gamma");
319 GLint logMinBoostLocation = glGetUniformLocation(shaderProgram, "logMinBoost");
320 GLint logMaxBoostLocation = glGetUniformLocation(shaderProgram, "logMaxBoost");
321 GLint weightLocation = glGetUniformLocation(shaderProgram, "weight");
322 GLint offsetSdrLocation = glGetUniformLocation(shaderProgram, "offsetSdr");
323 GLint offsetHdrLocation = glGetUniformLocation(shaderProgram, "offsetHdr");
324 GLint normalizeLocation = glGetUniformLocation(shaderProgram, "normalize");
325
326 glUniform1i(pWidthLocation, sdr_intent->w);
327 glUniform1i(pHeightLocation, sdr_intent->h);
328 glUniform1f(gammaLocation, gainmap_metadata->gamma);
329 glUniform1f(logMinBoostLocation, log2(gainmap_metadata->min_content_boost));
330 glUniform1f(logMaxBoostLocation, log2(gainmap_metadata->max_content_boost));
331 glUniform1f(offsetSdrLocation, gainmap_metadata->offset_sdr);
332 glUniform1f(offsetHdrLocation, gainmap_metadata->offset_hdr);
333 float gainmap_weight;
334 if (display_boost != gainmap_metadata->hdr_capacity_max) {
335 gainmap_weight =
336 (log2(display_boost) - log2(gainmap_metadata->hdr_capacity_min)) /
337 (log2(gainmap_metadata->hdr_capacity_max) - log2(gainmap_metadata->hdr_capacity_min));
338 // avoid extrapolating the gain map to fill the displayable range
339 gainmap_weight = CLIP3(0.0f, gainmap_weight, 1.0f);
340 } else {
341 gainmap_weight = 1.0f;
342 }
343 glUniform1f(weightLocation, gainmap_weight);
344 float normalize = 1.0f;
345 if (output_ct == UHDR_CT_HLG) normalize = kHlgMaxNits / kSdrWhiteNits;
346 else if (output_ct == UHDR_CT_PQ) normalize = kPqMaxNits / kSdrWhiteNits;
347 glUniform1f(normalizeLocation, normalize);
348
349 glActiveTexture(GL_TEXTURE0);
350 glBindTexture(GL_TEXTURE_2D, yuvTexture);
351 glUniform1i(glGetUniformLocation(shaderProgram, "yuvTexture"), 0);
352
353 glActiveTexture(GL_TEXTURE1);
354 glBindTexture(GL_TEXTURE_2D, opengl_ctxt->mGainmapImgTexture);
355 glUniform1i(glGetUniformLocation(shaderProgram, "gainMapTexture"), 1);
356
357 opengl_ctxt->check_gl_errors("binding values to uniforms");
358 RET_IF_ERR()
359
360 glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
361
362 glBindFramebuffer(GL_FRAMEBUFFER, 0);
363
364 opengl_ctxt->check_gl_errors("reading gles output");
365 RET_IF_ERR()
366
367 dest->cg = sdr_intent->cg;
368
369 if (frameBuffer) glDeleteFramebuffers(1, &frameBuffer);
370 if (yuvTexture) glDeleteTextures(1, &yuvTexture);
371 if (shaderProgram) glDeleteProgram(shaderProgram);
372
373 return opengl_ctxt->mErrorStatus;
374 }
375
376 } // namespace ultrahdr
377