1 /*
2 * Copyright 2021 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8 #include "src/codec/SkJpegxlCodec.h"
9
10 #include "include/codec/SkCodec.h"
11 #include "include/codec/SkJpegxlDecoder.h"
12 #include "include/core/SkColorType.h"
13 #include "include/core/SkData.h"
14 #include "include/core/SkImageInfo.h"
15 #include "include/core/SkRefCnt.h"
16 #include "include/core/SkStream.h"
17 #include "include/core/SkTypes.h"
18 #include "include/private/SkEncodedInfo.h"
19 #include "include/private/base/SkTFitsIn.h"
20 #include "include/private/base/SkTemplates.h"
21 #include "include/private/base/SkTo.h"
22 #include "modules/skcms/skcms.h"
23 #include "src/codec/SkFrameHolder.h"
24 #include "src/core/SkStreamPriv.h"
25 #include "src/core/SkSwizzlePriv.h"
26
27 #include "jxl/codestream_header.h" // NO_G3_REWRITE
28 #include "jxl/decode.h" // NO_G3_REWRITE
29 #include "jxl/decode_cxx.h" // NO_G3_REWRITE
30 #include "jxl/types.h" // NO_G3_REWRITE
31
32 #include <cstdint>
33 #include <cstring>
34 #include <limits>
35 #include <utility>
36 #include <vector>
37
38 namespace {
39
40 class Frame : public SkFrame {
41 public:
Frame(int i,SkEncodedInfo::Alpha alpha)42 explicit Frame(int i, SkEncodedInfo::Alpha alpha) : INHERITED(i), fReportedAlpha(alpha) {}
onReportedAlpha() const43 SkEncodedInfo::Alpha onReportedAlpha() const override { return fReportedAlpha; }
44
45 private:
46 const SkEncodedInfo::Alpha fReportedAlpha;
47
48 using INHERITED = SkFrame;
49 };
50
51 } // namespace
52
IsJpegxl(const void * buffer,size_t bytesRead)53 bool SkJpegxlCodec::IsJpegxl(const void* buffer, size_t bytesRead) {
54 JxlSignature result = JxlSignatureCheck(reinterpret_cast<const uint8_t*>(buffer), bytesRead);
55 return (result == JXL_SIG_CODESTREAM) || (result == JXL_SIG_CONTAINER);
56 }
57
58 class SkJpegxlCodecPriv : public SkFrameHolder {
59 public:
SkJpegxlCodecPriv()60 SkJpegxlCodecPriv() : fDecoder(JxlDecoderMake(/* memory_manager= */ nullptr)) {}
61 JxlDecoderPtr fDecoder; // unique_ptr with custom destructor
62 JxlBasicInfo fInfo;
63 bool fSeenAllFrames = false;
64 std::vector<Frame> fFrames;
65 int fLastProcessedFrame = SkCodec::kNoFrame;
66 void* fDst;
67 size_t fPixelShift;
68 size_t fRowBytes;
69 SkColorType fDstColorType;
70
71 protected:
onGetFrame(int i) const72 const SkFrame* onGetFrame(int i) const override {
73 SkASSERT(i >= 0 && static_cast<size_t>(i) < fFrames.size());
74 return static_cast<const SkFrame*>(&fFrames[i]);
75 }
76 };
77
SkJpegxlCodec(std::unique_ptr<SkJpegxlCodecPriv> codec,SkEncodedInfo && info,std::unique_ptr<SkStream> stream,sk_sp<SkData> data)78 SkJpegxlCodec::SkJpegxlCodec(std::unique_ptr<SkJpegxlCodecPriv> codec,
79 SkEncodedInfo&& info,
80 std::unique_ptr<SkStream> stream,
81 sk_sp<SkData> data)
82 : INHERITED(std::move(info), skcms_PixelFormat_RGBA_16161616LE, std::move(stream))
83 , fCodec(std::move(codec))
84 , fData(std::move(data)) {}
85
MakeFromStream(std::unique_ptr<SkStream> stream,Result * result)86 std::unique_ptr<SkCodec> SkJpegxlCodec::MakeFromStream(std::unique_ptr<SkStream> stream,
87 Result* result) {
88 SkASSERT(result);
89 if (!stream) {
90 *result = SkCodec::kInvalidInput;
91 return nullptr;
92 }
93 *result = kInternalError;
94 // Either wrap or copy stream data.
95 sk_sp<SkData> data = nullptr;
96 if (stream->getMemoryBase()) {
97 data = SkData::MakeWithoutCopy(stream->getMemoryBase(), stream->getLength());
98 } else {
99 data = SkCopyStreamToData(stream.get());
100 // Data is copied; stream can be released now.
101 stream.reset(nullptr);
102 }
103
104 auto priv = std::make_unique<SkJpegxlCodecPriv>();
105 JxlDecoder* dec = priv->fDecoder.get();
106
107 // Only query metadata this time.
108 auto status = JxlDecoderSubscribeEvents(dec, JXL_DEC_BASIC_INFO | JXL_DEC_COLOR_ENCODING);
109 if (status != JXL_DEC_SUCCESS) {
110 // Fresh instance must accept request for subscription.
111 SkDEBUGFAIL("libjxl returned unexpected status");
112 return nullptr;
113 }
114
115 status = JxlDecoderSetInput(dec, data->bytes(), data->size());
116 if (status != JXL_DEC_SUCCESS) {
117 // Fresh instance must accept first chunk of input.
118 SkDEBUGFAIL("libjxl returned unexpected status");
119 return nullptr;
120 }
121
122 status = JxlDecoderProcessInput(dec);
123 if (status == JXL_DEC_NEED_MORE_INPUT) {
124 *result = kIncompleteInput;
125 return nullptr;
126 }
127 if (status != JXL_DEC_BASIC_INFO) {
128 *result = kInvalidInput;
129 return nullptr;
130 }
131 JxlBasicInfo& info = priv->fInfo;
132 status = JxlDecoderGetBasicInfo(dec, &info);
133 if (status != JXL_DEC_SUCCESS) {
134 // Current event is "JXL_DEC_BASIC_INFO" -> can't fail.
135 SkDEBUGFAIL("libjxl returned unexpected status");
136 return nullptr;
137 }
138
139 // Check that image dimensions are not too large.
140 if (!SkTFitsIn<int32_t>(info.xsize) || !SkTFitsIn<int32_t>(info.ysize)) {
141 *result = kInvalidInput;
142 return nullptr;
143 }
144 int32_t width = SkTo<int32_t>(info.xsize);
145 int32_t height = SkTo<int32_t>(info.ysize);
146
147 bool hasAlpha = (info.alpha_bits != 0);
148 bool isGray = (info.num_color_channels == 1);
149 SkEncodedInfo::Alpha alpha =
150 hasAlpha ? SkEncodedInfo::kUnpremul_Alpha : SkEncodedInfo::kOpaque_Alpha;
151 SkEncodedInfo::Color color;
152 if (hasAlpha) {
153 color = isGray ? SkEncodedInfo::kGrayAlpha_Color : SkEncodedInfo::kRGBA_Color;
154 } else {
155 color = isGray ? SkEncodedInfo::kGray_Color : SkEncodedInfo::kRGB_Color;
156 }
157
158 status = JxlDecoderProcessInput(dec);
159 if (status != JXL_DEC_COLOR_ENCODING) {
160 *result = kInvalidInput;
161 return nullptr;
162 }
163
164 size_t iccSize = 0;
165 // TODO(eustas): format field is currently ignored by decoder.
166 status = JxlDecoderGetICCProfileSize(
167 dec, /* format = */ nullptr, JXL_COLOR_PROFILE_TARGET_DATA, &iccSize);
168 if (status != JXL_DEC_SUCCESS) {
169 // Likely incompatible colorspace.
170 iccSize = 0;
171 }
172 std::unique_ptr<SkEncodedInfo::ICCProfile> profile = nullptr;
173 if (iccSize) {
174 auto icc = SkData::MakeUninitialized(iccSize);
175 // TODO(eustas): format field is currently ignored by decoder.
176 status = JxlDecoderGetColorAsICCProfile(dec,
177 /* format = */ nullptr,
178 JXL_COLOR_PROFILE_TARGET_DATA,
179 reinterpret_cast<uint8_t*>(icc->writable_data()),
180 iccSize);
181 if (status != JXL_DEC_SUCCESS) {
182 // Current event is JXL_DEC_COLOR_ENCODING -> can't fail.
183 SkDEBUGFAIL("libjxl returned unexpected status");
184 return nullptr;
185 }
186 profile = SkEncodedInfo::ICCProfile::Make(std::move(icc));
187 }
188
189 int bitsPerChannel = 16;
190
191 *result = kSuccess;
192 SkEncodedInfo encodedInfo =
193 SkEncodedInfo::Make(width, height, color, alpha, bitsPerChannel, std::move(profile));
194
195 return std::unique_ptr<SkCodec>(new SkJpegxlCodec(
196 std::move(priv), std::move(encodedInfo), std::move(stream), std::move(data)));
197 }
198
onGetPixels(const SkImageInfo & dstInfo,void * dst,size_t rowBytes,const Options & options,int * rowsDecodedPtr)199 SkCodec::Result SkJpegxlCodec::onGetPixels(const SkImageInfo& dstInfo, void* dst, size_t rowBytes,
200 const Options& options, int* rowsDecodedPtr) {
201 // TODO(eustas): implement
202 if (options.fSubset) {
203 return kUnimplemented;
204 }
205 auto& codec = *fCodec.get();
206 const int index = options.fFrameIndex;
207 SkASSERT(0 == index || static_cast<size_t>(index) < codec.fFrames.size());
208 auto* dec = codec.fDecoder.get();
209 JxlDecoderStatus status;
210
211 if ((codec.fLastProcessedFrame >= index) || (codec.fLastProcessedFrame = SkCodec::kNoFrame)) {
212 codec.fLastProcessedFrame = SkCodec::kNoFrame;
213 JxlDecoderRewind(dec);
214 status = JxlDecoderSubscribeEvents(dec, JXL_DEC_FRAME | JXL_DEC_FULL_IMAGE);
215 if (status != JXL_DEC_SUCCESS) {
216 // Fresh decoder instance (after rewind) must accept subscription request.
217 SkDEBUGFAIL("libjxl returned unexpected status");
218 return kInternalError;
219 }
220 status = JxlDecoderSetInput(dec, fData->bytes(), fData->size());
221 if (status != JXL_DEC_SUCCESS) {
222 // Fresh decoder instance (after rewind) must accept first data chunk.
223 SkDEBUGFAIL("libjxl returned unexpected status");
224 return kInternalError;
225 }
226 SkASSERT(codec.fLastProcessedFrame + 1 == 0);
227 }
228
229 int nextFrame = codec.fLastProcessedFrame + 1;
230 if (nextFrame < index) {
231 JxlDecoderSkipFrames(dec, index - nextFrame);
232 }
233
234 // Decode till the frame start.
235 status = JxlDecoderProcessInput(dec);
236 // TODO(eustas): actually, frame is not completely processed; for streaming / partial decoding
237 // we should also add a flag that "last processed frame" is still incomplete, and
238 // flip that flag when frame decoding is over.
239 codec.fLastProcessedFrame = index;
240 if (status != JXL_DEC_FRAME) {
241 // TODO(eustas): check status: it might be either corrupted or incomplete input.
242 return kInternalError;
243 }
244
245 codec.fDst = dst;
246 codec.fRowBytes = rowBytes;
247
248 // TODO(eustas): consider grayscale.
249 uint32_t numColorChannels = 3;
250 // TODO(eustas): consider no-alpha.
251 uint32_t numAlphaChannels = 1;
252 // NB: SKIA works with little-endian F16s.
253 auto endianness = JXL_LITTLE_ENDIAN;
254
255 // Internally JXL does most processing in floats. By "default" we request
256 // output data type to be U8; it takes less memory, but results in some precision loss.
257 // We request F16 in two cases:
258 // - destination type is F16
259 // - color transformation is required; in this case values are remapped,
260 // and with 8-bit precision it is likely that visual artefact will appear
261 // (like banding, etc.)
262 bool halfFloatOutput = false;
263 if (fCodec->fDstColorType == kRGBA_F16_SkColorType) halfFloatOutput = true;
264 if (colorXform()) halfFloatOutput = true;
265 auto dataType = halfFloatOutput ? JXL_TYPE_FLOAT16 : JXL_TYPE_UINT8;
266
267 JxlPixelFormat format =
268 {numColorChannels + numAlphaChannels, dataType, endianness, /* align = */ 0};
269 status = JxlDecoderSetImageOutCallback(dec, &format, SkJpegxlCodec::imageOutCallback, this);
270 if (status != JXL_DEC_SUCCESS) {
271 // Current event is JXL_DEC_FRAME -> decoder must accept callback.
272 SkDEBUGFAIL("libjxl returned unexpected status");
273 return kInternalError;
274 }
275
276 // Decode till the frame start.
277 status = JxlDecoderProcessInput(dec);
278 if (status != JXL_DEC_FULL_IMAGE) {
279 // TODO(eustas): check status: it might be either corrupted or incomplete input.
280 return kInternalError;
281 }
282 // TODO(eustas): currently it is supposed that complete input is accessible;
283 // when streaming support is added JXL_DEC_NEED_MORE_INPUT would also
284 // become a legal outcome; amount of decoded scanlines should be calculated
285 // based on callback invocations / render-pipeline API.
286 *rowsDecodedPtr = dstInfo.height();
287
288 return kSuccess;
289 }
290
onRewind()291 bool SkJpegxlCodec::onRewind() {
292 JxlDecoderRewind(fCodec->fDecoder.get());
293 return true;
294 }
295
conversionSupported(const SkImageInfo & dstInfo,bool srcIsOpaque,bool needsColorXform)296 bool SkJpegxlCodec::conversionSupported(const SkImageInfo& dstInfo, bool srcIsOpaque,
297 bool needsColorXform) {
298 fCodec->fDstColorType = dstInfo.colorType();
299 switch (dstInfo.colorType()) {
300 case kRGBA_8888_SkColorType:
301 return true; // memcpy
302 case kBGRA_8888_SkColorType:
303 return true; // rgba->bgra
304
305 case kRGBA_F16_SkColorType:
306 SkASSERT(needsColorXform); // TODO(eustas): not necessary for JXL.
307 return true; // memcpy
308
309 // TODO(eustas): implement
310 case kRGB_565_SkColorType:
311 return false;
312 case kGray_8_SkColorType:
313 return false;
314 case kAlpha_8_SkColorType:
315 return false;
316
317 default:
318 return false;
319 }
320 return true;
321 }
322
imageOutCallback(void * opaque,size_t x,size_t y,size_t num_pixels,const void * pixels)323 void SkJpegxlCodec::imageOutCallback(void* opaque, size_t x, size_t y,
324 size_t num_pixels, const void* pixels) {
325 SkJpegxlCodec* instance = reinterpret_cast<SkJpegxlCodec*>(opaque);
326 auto& codec = *instance->fCodec.get();
327 size_t offset = y * codec.fRowBytes + (x << codec.fPixelShift);
328 void* dst = SkTAddOffset<void>(codec.fDst, offset);
329 if (instance->colorXform()) {
330 instance->applyColorXform(dst, pixels, num_pixels);
331 return;
332 }
333 switch (codec.fDstColorType) {
334 case kRGBA_8888_SkColorType:
335 memcpy(dst, pixels, 4 * num_pixels);
336 return;
337 case kBGRA_8888_SkColorType:
338 SkOpts::RGBA_to_bgrA((uint32_t*) dst, (const uint32_t*)(pixels), num_pixels);
339 return;
340 case kRGBA_F16_SkColorType:
341 memcpy(dst, pixels, 8 * num_pixels);
342 return;
343 default:
344 SK_ABORT("Selected output format is not supported yet");
345 return;
346 }
347 }
348
scanFrames()349 bool SkJpegxlCodec::scanFrames() {
350 auto decoder = JxlDecoderMake(/* memory_manager = */ nullptr);
351 JxlDecoder* dec = decoder.get();
352 auto* frameHolder = fCodec.get();
353 auto& frames = frameHolder->fFrames;
354 const auto& info = fCodec->fInfo;
355 frames.clear();
356
357 auto alpha = (info.alpha_bits != 0) ? SkEncodedInfo::Alpha::kUnpremul_Alpha
358 : SkEncodedInfo::Alpha::kOpaque_Alpha;
359
360 auto status = JxlDecoderSubscribeEvents(dec, JXL_DEC_FRAME);
361 if (status != JXL_DEC_SUCCESS) {
362 // Fresh instance must accept request for subscription.
363 SkDEBUGFAIL("libjxl returned unexpected status");
364 return true;
365 }
366
367 status = JxlDecoderSetInput(dec, fData->bytes(), fData->size());
368 if (status != JXL_DEC_SUCCESS) {
369 // Fresh instance must accept first input chunk.
370 SkDEBUGFAIL("libjxl returned unexpected status");
371 return true;
372 }
373
374 while (true) {
375 status = JxlDecoderProcessInput(dec);
376 switch (status) {
377 case JXL_DEC_FRAME: {
378 size_t frameId = frames.size();
379 JxlFrameHeader frameHeader;
380 if (JxlDecoderGetFrameHeader(dec, &frameHeader) != JXL_DEC_SUCCESS) {
381 return true;
382 }
383 frames.emplace_back(static_cast<int>(frameId), alpha);
384 auto& frame = frames.back();
385 // TODO(eustas): for better consistency we need to track total duration and report
386 // frame duration as delta to previous frame.
387 int duration = (1000 * frameHeader.duration * info.animation.tps_denominator) /
388 info.animation.tps_numerator;
389 frame.setDuration(duration);
390 frameHolder->setAlphaAndRequiredFrame(&frame);
391 break;
392 }
393 case JXL_DEC_SUCCESS: {
394 return true;
395 }
396 default: {
397 return false;
398 }
399 }
400 }
401 }
402
onGetFrameCount()403 int SkJpegxlCodec::onGetFrameCount() {
404 if (!fCodec->fInfo.have_animation) {
405 return 1;
406 }
407
408 if (!fCodec->fSeenAllFrames) {
409 fCodec->fSeenAllFrames = scanFrames();
410 }
411
412 return fCodec->fFrames.size();
413 }
414
onGetFrameInfo(int index,FrameInfo * frameInfo) const415 bool SkJpegxlCodec::onGetFrameInfo(int index, FrameInfo* frameInfo) const {
416 if (index < 0) {
417 return false;
418 }
419 if (static_cast<size_t>(index) >= fCodec->fFrames.size()) {
420 return false;
421 }
422 fCodec->fFrames[index].fillIn(frameInfo, true);
423 return true;
424 }
425
onGetRepetitionCount()426 int SkJpegxlCodec::onGetRepetitionCount() {
427 JxlBasicInfo& info = fCodec->fInfo;
428 if (!info.have_animation) {
429 return 0;
430 }
431
432 if (info.animation.num_loops == 0) {
433 return kRepetitionCountInfinite;
434 }
435
436 if (SkTFitsIn<int>(info.animation.num_loops)) {
437 return info.animation.num_loops - 1;
438 }
439
440 // Largest "non-infinite" value.
441 return std::numeric_limits<int>::max();
442 }
443
getFrameHolder() const444 const SkFrameHolder* SkJpegxlCodec::getFrameHolder() const {
445 return fCodec.get();
446 }
447
448 // TODO(eustas): implement
449 // SkCodec::Result SkJpegxlCodec::onStartScanlineDecode(
450 // const SkImageInfo& /*dstInfo*/, const Options& /*options*/) { return kUnimplemented; }
451
452 // TODO(eustas): implement
453 // SkCodec::Result SkJpegxlCodec::onStartIncrementalDecode(
454 // const SkImageInfo& /*dstInfo*/, void*, size_t, const Options&) { return kUnimplemented; }
455
456 // TODO(eustas): implement
457 // SkCodec::Result SkJpegxlCodec::onIncrementalDecode(int*) { return kUnimplemented; }
458
459 // TODO(eustas): implement
460 // bool SkJpegxlCodec::onSkipScanlines(int /*countLines*/) { return false; }
461
462 // TODO(eustas): implement
463 // int SkJpegxlCodec::onGetScanlines(
464 // void* /*dst*/, int /*countLines*/, size_t /*rowBytes*/) { return 0; }
465
466 // TODO(eustas): implement
467 // SkSampler* SkJpegxlCodec::getSampler(bool /*createIfNecessary*/) { return nullptr; }
468
469 namespace SkJpegxlDecoder {
IsJpegxl(const void * data,size_t len)470 bool IsJpegxl(const void* data, size_t len) {
471 return SkJpegxlCodec::IsJpegxl(data, len);
472 }
473
Decode(std::unique_ptr<SkStream> stream,SkCodec::Result * outResult,SkCodecs::DecodeContext)474 std::unique_ptr<SkCodec> Decode(std::unique_ptr<SkStream> stream,
475 SkCodec::Result* outResult,
476 SkCodecs::DecodeContext) {
477 SkCodec::Result resultStorage;
478 if (!outResult) {
479 outResult = &resultStorage;
480 }
481 return SkJpegxlCodec::MakeFromStream(std::move(stream), outResult);
482 }
483
Decode(sk_sp<SkData> data,SkCodec::Result * outResult,SkCodecs::DecodeContext)484 std::unique_ptr<SkCodec> Decode(sk_sp<SkData> data,
485 SkCodec::Result* outResult,
486 SkCodecs::DecodeContext) {
487 if (!data) {
488 if (outResult) {
489 *outResult = SkCodec::kInvalidInput;
490 }
491 return nullptr;
492 }
493 return Decode(SkMemoryStream::Make(std::move(data)), outResult, nullptr);
494 }
495 } // namespace SkJpegDecoder
496