1 /*
2 * Copyright 2019 Google LLC
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8 #include "modules/skresources/include/SkResources.h"
9
10 #include "include/codec/SkCodec.h"
11 #include "include/core/SkBitmap.h"
12 #include "include/core/SkData.h"
13 #include "include/core/SkFontMgr.h"
14 #include "include/core/SkImage.h"
15 #include "include/private/base/SkTPin.h"
16 #include "modules/skresources/src/SkAnimCodecPlayer.h"
17 #include "src/base/SkBase64.h"
18 #include "src/core/SkOSFile.h"
19 #include "src/utils/SkOSPath.h"
20
21 #include <cmath>
22
23 #if defined(HAVE_VIDEO_DECODER)
24 #include "experimental/ffmpeg/SkVideoDecoder.h"
25 #include "include/core/SkStream.h"
26 #endif
27
28 namespace skresources {
29 namespace {
30
31 #if defined(HAVE_VIDEO_DECODER)
32
33 class VideoAsset final : public ImageAsset {
34 public:
Make(sk_sp<SkData> data)35 static sk_sp<VideoAsset> Make(sk_sp<SkData> data) {
36 auto decoder = std::make_unique<SkVideoDecoder>();
37
38 if (!decoder->loadStream(SkMemoryStream::Make(std::move(data))) ||
39 decoder->duration() <= 0) {
40 return nullptr;
41 }
42
43 return sk_sp<VideoAsset>(new VideoAsset(std::move(decoder)));
44 }
45
46 private:
VideoAsset(std::unique_ptr<SkVideoDecoder> decoder)47 explicit VideoAsset(std::unique_ptr<SkVideoDecoder> decoder)
48 : fDecoder(std::move(decoder)) {
49 }
50
isMultiFrame()51 bool isMultiFrame() override { return true; }
52
53 // Each frame has a presentation timestamp
54 // => the timespan for frame N is [stamp_N .. stamp_N+1)
55 // => we use a two-frame sliding window to track the current interval.
advance()56 void advance() {
57 fWindow[0] = std::move(fWindow[1]);
58 fWindow[1].frame = fDecoder->nextImage(&fWindow[1].stamp);
59 fEof = !fWindow[1].frame;
60 }
61
getFrame(float t_float)62 sk_sp<SkImage> getFrame(float t_float) override {
63 const auto t = SkTPin(static_cast<double>(t_float), 0.0, fDecoder->duration());
64
65 if (t < fWindow[0].stamp) {
66 // seeking back requires a full rewind
67 fDecoder->rewind();
68 fWindow[0].stamp = fWindow[1].stamp = 0;
69 fEof = 0;
70 }
71
72 while (!fEof && t >= fWindow[1].stamp) {
73 this->advance();
74 }
75
76 SkASSERT(fWindow[0].stamp <= t && (fEof || t < fWindow[1].stamp));
77
78 return fWindow[0].frame;
79 }
80
81 const std::unique_ptr<SkVideoDecoder> fDecoder;
82
83 struct FrameRec {
84 sk_sp<SkImage> frame;
85 double stamp = 0;
86 };
87
88 FrameRec fWindow[2];
89 bool fEof = false;
90 };
91
92 #endif // defined(HAVE_VIDEO_DECODER)
93
94 } // namespace
95
getFrame(float t)96 sk_sp<SkImage> ImageAsset::getFrame(float t) {
97 return nullptr;
98 }
99
getFrameData(float t)100 ImageAsset::FrameData ImageAsset::getFrameData(float t) {
101 // legacy behavior
102 return {
103 this->getFrame(t),
104 SkSamplingOptions(SkFilterMode::kLinear, SkMipmapMode::kNearest),
105 SkMatrix::I(),
106 SizeFit::kCenter,
107 };
108 }
109
Make(sk_sp<SkData> data,ImageDecodeStrategy strat)110 sk_sp<MultiFrameImageAsset> MultiFrameImageAsset::Make(sk_sp<SkData> data, ImageDecodeStrategy strat) {
111 if (auto codec = SkCodec::MakeFromData(std::move(data))) {
112 return sk_sp<MultiFrameImageAsset>(new MultiFrameImageAsset(
113 std::make_unique<SkAnimCodecPlayer>(std::move(codec)), strat));
114 }
115
116 return nullptr;
117 }
118
Make(std::unique_ptr<SkCodec> codec,ImageDecodeStrategy strat)119 sk_sp<MultiFrameImageAsset> MultiFrameImageAsset::Make(std::unique_ptr<SkCodec> codec, ImageDecodeStrategy strat) {
120 SkASSERT(codec);
121 return sk_sp<MultiFrameImageAsset>(new MultiFrameImageAsset(
122 std::make_unique<SkAnimCodecPlayer>(std::move(codec)), strat));
123 }
124
MultiFrameImageAsset(std::unique_ptr<SkAnimCodecPlayer> player,ImageDecodeStrategy strat)125 MultiFrameImageAsset::MultiFrameImageAsset(std::unique_ptr<SkAnimCodecPlayer> player,
126 ImageDecodeStrategy strat)
127 : fPlayer(std::move(player)), fStrategy(strat) {
128 SkASSERT(fPlayer);
129 }
130
isMultiFrame()131 bool MultiFrameImageAsset::isMultiFrame() {
132 return fPlayer->duration() > 0;
133 }
134
generateFrame(float t)135 sk_sp<SkImage> MultiFrameImageAsset::generateFrame(float t) {
136 auto decode = [](sk_sp<SkImage> image) {
137 SkASSERT(image->isLazyGenerated());
138
139 static constexpr size_t kMaxArea = 2048 * 2048;
140 const auto image_area = SkToSizeT(image->width() * image->height());
141
142 if (image_area > kMaxArea) {
143 // When the image is too large, decode and scale down to a reasonable size.
144 const auto scale = std::sqrt(static_cast<float>(kMaxArea) / image_area);
145 const auto info = SkImageInfo::MakeN32Premul(scale * image->width(),
146 scale * image->height());
147 SkBitmap bm;
148 if (bm.tryAllocPixels(info, info.minRowBytes()) &&
149 image->scalePixels(bm.pixmap(),
150 SkSamplingOptions(SkFilterMode::kLinear,
151 SkMipmapMode::kNearest),
152 SkImage::kDisallow_CachingHint)) {
153 image = bm.asImage();
154 }
155 } else {
156 // When the image size is OK, just force-decode.
157 image = image->makeRasterImage();
158 }
159
160 return image;
161 };
162
163 fPlayer->seek(static_cast<uint32_t>(t * 1000));
164 auto frame = fPlayer->getFrame();
165
166 if (fStrategy == ImageDecodeStrategy::kPreDecode && frame && frame->isLazyGenerated()) {
167 // The multi-frame decoder should never return lazy images.
168 SkASSERT(!this->isMultiFrame());
169 frame = decode(std::move(frame));
170 }
171
172 return frame;
173 }
174
getFrame(float t)175 sk_sp<SkImage> MultiFrameImageAsset::getFrame(float t) {
176 // For static images we can reuse the cached frame
177 // (which includes the optional pre-decode step).
178 if (!fCachedFrame || this->isMultiFrame()) {
179 fCachedFrame = this->generateFrame(t);
180 }
181
182 return fCachedFrame;
183 }
184
Make(SkString base_dir,ImageDecodeStrategy strat)185 sk_sp<FileResourceProvider> FileResourceProvider::Make(SkString base_dir, ImageDecodeStrategy strat) {
186 return sk_isdir(base_dir.c_str()) ? sk_sp<FileResourceProvider>(new FileResourceProvider(
187 std::move(base_dir), strat))
188 : nullptr;
189 }
190
FileResourceProvider(SkString base_dir,ImageDecodeStrategy strat)191 FileResourceProvider::FileResourceProvider(SkString base_dir, ImageDecodeStrategy strat)
192 : fDir(std::move(base_dir)), fStrategy(strat) {}
193
load(const char resource_path[],const char resource_name[]) const194 sk_sp<SkData> FileResourceProvider::load(const char resource_path[],
195 const char resource_name[]) const {
196 const auto full_dir = SkOSPath::Join(fDir.c_str() , resource_path),
197 full_path = SkOSPath::Join(full_dir.c_str(), resource_name);
198 return SkData::MakeFromFileName(full_path.c_str());
199 }
200
loadImageAsset(const char resource_path[],const char resource_name[],const char[]) const201 sk_sp<ImageAsset> FileResourceProvider::loadImageAsset(const char resource_path[],
202 const char resource_name[],
203 const char[]) const {
204 auto data = this->load(resource_path, resource_name);
205
206 if (auto image = MultiFrameImageAsset::Make(data, fStrategy)) {
207 return std::move(image);
208 }
209
210 #if defined(HAVE_VIDEO_DECODER)
211 if (auto video = VideoAsset::Make(data)) {
212 return std::move(video);
213 }
214 #endif
215
216 return nullptr;
217 }
218
ResourceProviderProxyBase(sk_sp<ResourceProvider> rp)219 ResourceProviderProxyBase::ResourceProviderProxyBase(sk_sp<ResourceProvider> rp)
220 : fProxy(std::move(rp)) {}
221
load(const char resource_path[],const char resource_name[]) const222 sk_sp<SkData> ResourceProviderProxyBase::load(const char resource_path[],
223 const char resource_name[]) const {
224 return fProxy ? fProxy->load(resource_path, resource_name)
225 : nullptr;
226 }
227
loadImageAsset(const char rpath[],const char rname[],const char rid[]) const228 sk_sp<ImageAsset> ResourceProviderProxyBase::loadImageAsset(const char rpath[],
229 const char rname[],
230 const char rid[]) const {
231 return fProxy ? fProxy->loadImageAsset(rpath, rname, rid)
232 : nullptr;
233 }
234
loadTypeface(const char name[],const char url[]) const235 sk_sp<SkTypeface> ResourceProviderProxyBase::loadTypeface(const char name[],
236 const char url[]) const {
237 return fProxy ? fProxy->loadTypeface(name, url)
238 : nullptr;
239 }
240
loadFont(const char name[],const char url[]) const241 sk_sp<SkData> ResourceProviderProxyBase::loadFont(const char name[], const char url[]) const {
242 return fProxy ? fProxy->loadFont(name, url)
243 : nullptr;
244 }
245
loadAudioAsset(const char path[],const char name[],const char id[])246 sk_sp<ExternalTrackAsset> ResourceProviderProxyBase::loadAudioAsset(const char path[],
247 const char name[],
248 const char id[]) {
249 return fProxy ? fProxy->loadAudioAsset(path, name, id)
250 : nullptr;
251 }
252
CachingResourceProvider(sk_sp<ResourceProvider> rp)253 CachingResourceProvider::CachingResourceProvider(sk_sp<ResourceProvider> rp)
254 : INHERITED(std::move(rp)) {}
255
loadImageAsset(const char resource_path[],const char resource_name[],const char resource_id[]) const256 sk_sp<ImageAsset> CachingResourceProvider::loadImageAsset(const char resource_path[],
257 const char resource_name[],
258 const char resource_id[]) const {
259 SkAutoMutexExclusive amx(fMutex);
260
261 const SkString key(resource_id);
262 if (const auto* asset = fImageCache.find(key)) {
263 return *asset;
264 }
265
266 auto asset = this->INHERITED::loadImageAsset(resource_path, resource_name, resource_id);
267 fImageCache.set(key, asset);
268
269 return asset;
270 }
271
Make(sk_sp<ResourceProvider> rp,ImageDecodeStrategy strat,sk_sp<const SkFontMgr> mgr)272 sk_sp<DataURIResourceProviderProxy> DataURIResourceProviderProxy::Make(sk_sp<ResourceProvider> rp,
273 ImageDecodeStrategy strat,
274 sk_sp<const SkFontMgr> mgr) {
275 return sk_sp<DataURIResourceProviderProxy>(
276 new DataURIResourceProviderProxy(std::move(rp), strat, std::move(mgr)));
277 }
278
DataURIResourceProviderProxy(sk_sp<ResourceProvider> rp,ImageDecodeStrategy strat,sk_sp<const SkFontMgr> mgr)279 DataURIResourceProviderProxy::DataURIResourceProviderProxy(sk_sp<ResourceProvider> rp,
280 ImageDecodeStrategy strat,
281 sk_sp<const SkFontMgr> mgr)
282 : INHERITED(std::move(rp)), fStrategy(strat), fFontMgr(std::move(mgr)) {}
283
decode_datauri(const char prefix[],const char uri[])284 static sk_sp<SkData> decode_datauri(const char prefix[], const char uri[]) {
285 // We only handle B64 encoded image dataURIs: data:image/<type>;base64,<data>
286 // (https://en.wikipedia.org/wiki/Data_URI_scheme)
287 static constexpr char kDataURIEncodingStr[] = ";base64,";
288
289 const size_t prefixLen = strlen(prefix);
290 if (strncmp(uri, prefix, prefixLen) != 0) {
291 return nullptr;
292 }
293
294 const char* encoding = strstr(uri + prefixLen, kDataURIEncodingStr);
295 if (!encoding) {
296 return nullptr;
297 }
298
299 const char* b64Data = encoding + std::size(kDataURIEncodingStr) - 1;
300 size_t b64DataLen = strlen(b64Data);
301 size_t dataLen;
302 if (SkBase64::Decode(b64Data, b64DataLen, nullptr, &dataLen) != SkBase64::kNoError) {
303 return nullptr;
304 }
305
306 sk_sp<SkData> data = SkData::MakeUninitialized(dataLen);
307 void* rawData = data->writable_data();
308 if (SkBase64::Decode(b64Data, b64DataLen, rawData, &dataLen) != SkBase64::kNoError) {
309 return nullptr;
310 }
311
312 return data;
313 }
314
loadImageAsset(const char rpath[],const char rname[],const char rid[]) const315 sk_sp<ImageAsset> DataURIResourceProviderProxy::loadImageAsset(const char rpath[],
316 const char rname[],
317 const char rid[]) const {
318 // First try to decode the data as base64 using codecs registered with SkCodecs::Register()
319 if (auto data = decode_datauri("data:image/", rname)) {
320 return MultiFrameImageAsset::Make(std::move(data), fStrategy);
321 }
322 // Fallback to the asking the ProviderProxy to load this image for us.
323 return this->INHERITED::loadImageAsset(rpath, rname, rid);
324 }
325
loadTypeface(const char name[],const char url[]) const326 sk_sp<SkTypeface> DataURIResourceProviderProxy::loadTypeface(const char name[],
327 const char url[]) const {
328 if (fFontMgr) {
329 if (auto data = decode_datauri("data:font/", url)) {
330 return fFontMgr->makeFromData(std::move(data));
331 }
332 }
333
334 return this->INHERITED::loadTypeface(name, url);
335 }
336
337 } // namespace skresources
338