1 /*
2 ** Copyright 2023, The Android Open Source Project
3 **
4 ** Licensed under the Apache License, Version 2.0 (the "License");
5 ** you may not use this file except in compliance with the License.
6 ** You may obtain a copy of the License at
7 **
8 ** http://www.apache.org/licenses/LICENSE-2.0
9 **
10 ** Unless required by applicable law or agreed to in writing, software
11 ** distributed under the License is distributed on an "AS IS" BASIS,
12 ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 ** See the License for the specific language governing permissions and
14 ** limitations under the License.
15 */
16
17 #include "MultifileBlobCache.h"
18
19 #include <android-base/properties.h>
20 #include <android-base/test_utils.h>
21 #include <fcntl.h>
22 #include <gtest/gtest.h>
23 #include <stdio.h>
24 #include <utils/JenkinsHash.h>
25
26 #include <fstream>
27 #include <memory>
28
29 #include <com_android_graphics_egl_flags.h>
30
31 using namespace com::android::graphics::egl;
32
33 using namespace std::literals;
34
35 namespace android {
36
37 template <typename T>
38 using sp = std::shared_ptr<T>;
39
40 constexpr size_t kMaxKeySize = 2 * 1024;
41 constexpr size_t kMaxValueSize = 6 * 1024;
42 constexpr size_t kMaxTotalSize = 32 * 1024;
43 constexpr size_t kMaxTotalEntries = 64;
44
45 class MultifileBlobCacheTest : public ::testing::Test {
46 protected:
SetUp()47 virtual void SetUp() {
48 clearProperties();
49 mTempFile.reset(new TemporaryFile());
50 mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize,
51 kMaxTotalEntries, &mTempFile->path[0]));
52 }
53
TearDown()54 virtual void TearDown() {
55 clearProperties();
56 mMBC.reset();
57 }
58
59 int getFileDescriptorCount();
60 std::vector<std::string> getCacheEntries();
61
62 void clearProperties();
63 bool clearCache();
64
65 std::unique_ptr<TemporaryFile> mTempFile;
66 std::unique_ptr<MultifileBlobCache> mMBC;
67 };
68
clearProperties()69 void MultifileBlobCacheTest::clearProperties() {
70 // Clear any debug properties used in the tests
71 base::SetProperty("debug.egl.blobcache.cache_version", "");
72 base::WaitForProperty("debug.egl.blobcache.cache_version", "");
73
74 base::SetProperty("debug.egl.blobcache.build_id", "");
75 base::WaitForProperty("debug.egl.blobcache.build_id", "");
76 }
77
TEST_F(MultifileBlobCacheTest,CacheSingleValueSucceeds)78 TEST_F(MultifileBlobCacheTest, CacheSingleValueSucceeds) {
79 unsigned char buf[4] = {0xee, 0xee, 0xee, 0xee};
80 mMBC->set("abcd", 4, "efgh", 4);
81 ASSERT_EQ(size_t(4), mMBC->get("abcd", 4, buf, 4));
82 ASSERT_EQ('e', buf[0]);
83 ASSERT_EQ('f', buf[1]);
84 ASSERT_EQ('g', buf[2]);
85 ASSERT_EQ('h', buf[3]);
86 }
87
TEST_F(MultifileBlobCacheTest,CacheTwoValuesSucceeds)88 TEST_F(MultifileBlobCacheTest, CacheTwoValuesSucceeds) {
89 unsigned char buf[2] = {0xee, 0xee};
90 mMBC->set("ab", 2, "cd", 2);
91 mMBC->set("ef", 2, "gh", 2);
92 ASSERT_EQ(size_t(2), mMBC->get("ab", 2, buf, 2));
93 ASSERT_EQ('c', buf[0]);
94 ASSERT_EQ('d', buf[1]);
95 ASSERT_EQ(size_t(2), mMBC->get("ef", 2, buf, 2));
96 ASSERT_EQ('g', buf[0]);
97 ASSERT_EQ('h', buf[1]);
98 }
99
TEST_F(MultifileBlobCacheTest,GetSetTwiceSucceeds)100 TEST_F(MultifileBlobCacheTest, GetSetTwiceSucceeds) {
101 unsigned char buf[2] = {0xee, 0xee};
102 mMBC->set("ab", 2, "cd", 2);
103 ASSERT_EQ(size_t(2), mMBC->get("ab", 2, buf, 2));
104 ASSERT_EQ('c', buf[0]);
105 ASSERT_EQ('d', buf[1]);
106 // Use the same key, but different value
107 mMBC->set("ab", 2, "ef", 2);
108 ASSERT_EQ(size_t(2), mMBC->get("ab", 2, buf, 2));
109 ASSERT_EQ('e', buf[0]);
110 ASSERT_EQ('f', buf[1]);
111 }
112
TEST_F(MultifileBlobCacheTest,GetOnlyWritesInsideBounds)113 TEST_F(MultifileBlobCacheTest, GetOnlyWritesInsideBounds) {
114 unsigned char buf[6] = {0xee, 0xee, 0xee, 0xee, 0xee, 0xee};
115 mMBC->set("abcd", 4, "efgh", 4);
116 ASSERT_EQ(size_t(4), mMBC->get("abcd", 4, buf + 1, 4));
117 ASSERT_EQ(0xee, buf[0]);
118 ASSERT_EQ('e', buf[1]);
119 ASSERT_EQ('f', buf[2]);
120 ASSERT_EQ('g', buf[3]);
121 ASSERT_EQ('h', buf[4]);
122 ASSERT_EQ(0xee, buf[5]);
123 }
124
TEST_F(MultifileBlobCacheTest,GetOnlyWritesIfBufferIsLargeEnough)125 TEST_F(MultifileBlobCacheTest, GetOnlyWritesIfBufferIsLargeEnough) {
126 unsigned char buf[3] = {0xee, 0xee, 0xee};
127 mMBC->set("abcd", 4, "efgh", 4);
128 ASSERT_EQ(size_t(4), mMBC->get("abcd", 4, buf, 3));
129 ASSERT_EQ(0xee, buf[0]);
130 ASSERT_EQ(0xee, buf[1]);
131 ASSERT_EQ(0xee, buf[2]);
132 }
133
TEST_F(MultifileBlobCacheTest,GetDoesntAccessNullBuffer)134 TEST_F(MultifileBlobCacheTest, GetDoesntAccessNullBuffer) {
135 mMBC->set("abcd", 4, "efgh", 4);
136 ASSERT_EQ(size_t(4), mMBC->get("abcd", 4, nullptr, 0));
137 }
138
TEST_F(MultifileBlobCacheTest,MultipleSetsCacheLatestValue)139 TEST_F(MultifileBlobCacheTest, MultipleSetsCacheLatestValue) {
140 unsigned char buf[4] = {0xee, 0xee, 0xee, 0xee};
141 mMBC->set("abcd", 4, "efgh", 4);
142 mMBC->set("abcd", 4, "ijkl", 4);
143 ASSERT_EQ(size_t(4), mMBC->get("abcd", 4, buf, 4));
144 ASSERT_EQ('i', buf[0]);
145 ASSERT_EQ('j', buf[1]);
146 ASSERT_EQ('k', buf[2]);
147 ASSERT_EQ('l', buf[3]);
148 }
149
TEST_F(MultifileBlobCacheTest,SecondSetKeepsFirstValueIfTooLarge)150 TEST_F(MultifileBlobCacheTest, SecondSetKeepsFirstValueIfTooLarge) {
151 unsigned char buf[kMaxValueSize + 1] = {0xee, 0xee, 0xee, 0xee};
152 mMBC->set("abcd", 4, "efgh", 4);
153 mMBC->set("abcd", 4, buf, kMaxValueSize + 1);
154 ASSERT_EQ(size_t(4), mMBC->get("abcd", 4, buf, 4));
155 ASSERT_EQ('e', buf[0]);
156 ASSERT_EQ('f', buf[1]);
157 ASSERT_EQ('g', buf[2]);
158 ASSERT_EQ('h', buf[3]);
159 }
160
TEST_F(MultifileBlobCacheTest,DoesntCacheIfKeyIsTooBig)161 TEST_F(MultifileBlobCacheTest, DoesntCacheIfKeyIsTooBig) {
162 char key[kMaxKeySize + 1];
163 unsigned char buf[4] = {0xee, 0xee, 0xee, 0xee};
164 for (int i = 0; i < kMaxKeySize + 1; i++) {
165 key[i] = 'a';
166 }
167 mMBC->set(key, kMaxKeySize + 1, "bbbb", 4);
168 ASSERT_EQ(size_t(0), mMBC->get(key, kMaxKeySize + 1, buf, 4));
169 ASSERT_EQ(0xee, buf[0]);
170 ASSERT_EQ(0xee, buf[1]);
171 ASSERT_EQ(0xee, buf[2]);
172 ASSERT_EQ(0xee, buf[3]);
173 }
174
TEST_F(MultifileBlobCacheTest,DoesntCacheIfValueIsTooBig)175 TEST_F(MultifileBlobCacheTest, DoesntCacheIfValueIsTooBig) {
176 char buf[kMaxValueSize + 1];
177 for (int i = 0; i < kMaxValueSize + 1; i++) {
178 buf[i] = 'b';
179 }
180 mMBC->set("abcd", 4, buf, kMaxValueSize + 1);
181 for (int i = 0; i < kMaxValueSize + 1; i++) {
182 buf[i] = 0xee;
183 }
184 ASSERT_EQ(size_t(0), mMBC->get("abcd", 4, buf, kMaxValueSize + 1));
185 for (int i = 0; i < kMaxValueSize + 1; i++) {
186 SCOPED_TRACE(i);
187 ASSERT_EQ(0xee, buf[i]);
188 }
189 }
190
TEST_F(MultifileBlobCacheTest,CacheMaxKeySizeSucceeds)191 TEST_F(MultifileBlobCacheTest, CacheMaxKeySizeSucceeds) {
192 char key[kMaxKeySize];
193 unsigned char buf[4] = {0xee, 0xee, 0xee, 0xee};
194 for (int i = 0; i < kMaxKeySize; i++) {
195 key[i] = 'a';
196 }
197 mMBC->set(key, kMaxKeySize, "wxyz", 4);
198 ASSERT_EQ(size_t(4), mMBC->get(key, kMaxKeySize, buf, 4));
199 ASSERT_EQ('w', buf[0]);
200 ASSERT_EQ('x', buf[1]);
201 ASSERT_EQ('y', buf[2]);
202 ASSERT_EQ('z', buf[3]);
203 }
204
TEST_F(MultifileBlobCacheTest,CacheMaxValueSizeSucceeds)205 TEST_F(MultifileBlobCacheTest, CacheMaxValueSizeSucceeds) {
206 char buf[kMaxValueSize];
207 for (int i = 0; i < kMaxValueSize; i++) {
208 buf[i] = 'b';
209 }
210 mMBC->set("abcd", 4, buf, kMaxValueSize);
211 for (int i = 0; i < kMaxValueSize; i++) {
212 buf[i] = 0xee;
213 }
214 mMBC->get("abcd", 4, buf, kMaxValueSize);
215 for (int i = 0; i < kMaxValueSize; i++) {
216 SCOPED_TRACE(i);
217 ASSERT_EQ('b', buf[i]);
218 }
219 }
220
TEST_F(MultifileBlobCacheTest,CacheMaxKeyAndValueSizeSucceeds)221 TEST_F(MultifileBlobCacheTest, CacheMaxKeyAndValueSizeSucceeds) {
222 char key[kMaxKeySize];
223 for (int i = 0; i < kMaxKeySize; i++) {
224 key[i] = 'a';
225 }
226 char buf[kMaxValueSize];
227 for (int i = 0; i < kMaxValueSize; i++) {
228 buf[i] = 'b';
229 }
230 mMBC->set(key, kMaxKeySize, buf, kMaxValueSize);
231 for (int i = 0; i < kMaxValueSize; i++) {
232 buf[i] = 0xee;
233 }
234 mMBC->get(key, kMaxKeySize, buf, kMaxValueSize);
235 for (int i = 0; i < kMaxValueSize; i++) {
236 SCOPED_TRACE(i);
237 ASSERT_EQ('b', buf[i]);
238 }
239 }
240
TEST_F(MultifileBlobCacheTest,CacheMaxEntrySucceeds)241 TEST_F(MultifileBlobCacheTest, CacheMaxEntrySucceeds) {
242 // Fill the cache with max entries
243 int i = 0;
244 for (i = 0; i < kMaxTotalEntries; i++) {
245 mMBC->set(std::to_string(i).c_str(), sizeof(i), std::to_string(i).c_str(), sizeof(i));
246 }
247
248 // Ensure it is full
249 ASSERT_EQ(mMBC->getTotalEntries(), kMaxTotalEntries);
250
251 // Add another entry
252 mMBC->set(std::to_string(i).c_str(), sizeof(i), std::to_string(i).c_str(), sizeof(i));
253
254 // Ensure total entries is cut in half + 1
255 ASSERT_EQ(mMBC->getTotalEntries(), kMaxTotalEntries / 2 + 1);
256 }
257
TEST_F(MultifileBlobCacheTest,CacheMinKeyAndValueSizeSucceeds)258 TEST_F(MultifileBlobCacheTest, CacheMinKeyAndValueSizeSucceeds) {
259 unsigned char buf[1] = {0xee};
260 mMBC->set("x", 1, "y", 1);
261 ASSERT_EQ(size_t(1), mMBC->get("x", 1, buf, 1));
262 ASSERT_EQ('y', buf[0]);
263 }
264
getFileDescriptorCount()265 int MultifileBlobCacheTest::getFileDescriptorCount() {
266 DIR* directory = opendir("/proc/self/fd");
267
268 int fileCount = 0;
269 struct dirent* entry;
270 while ((entry = readdir(directory)) != NULL) {
271 fileCount++;
272 // printf("File: %s\n", entry->d_name);
273 }
274
275 closedir(directory);
276 return fileCount;
277 }
278
TEST_F(MultifileBlobCacheTest,EnsureFileDescriptorsClosed)279 TEST_F(MultifileBlobCacheTest, EnsureFileDescriptorsClosed) {
280 // Populate the cache with a bunch of entries
281 for (int i = 0; i < kMaxTotalEntries; i++) {
282 // printf("Caching: %i", i);
283
284 // Use the index as the key and value
285 mMBC->set(&i, sizeof(i), &i, sizeof(i));
286
287 int result = 0;
288 ASSERT_EQ(sizeof(i), mMBC->get(&i, sizeof(i), &result, sizeof(result)));
289 ASSERT_EQ(i, result);
290 }
291
292 // Ensure we don't have a bunch of open fds
293 ASSERT_LT(getFileDescriptorCount(), kMaxTotalEntries / 2);
294
295 // Close the cache so everything writes out
296 mMBC->finish();
297 mMBC.reset();
298
299 // Now open it again and ensure we still don't have a bunch of open fds
300 mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries,
301 &mTempFile->path[0]));
302
303 // Check after initialization
304 ASSERT_LT(getFileDescriptorCount(), kMaxTotalEntries / 2);
305
306 for (int i = 0; i < kMaxTotalEntries; i++) {
307 int result = 0;
308 ASSERT_EQ(sizeof(i), mMBC->get(&i, sizeof(i), &result, sizeof(result)));
309 ASSERT_EQ(i, result);
310 }
311
312 // And again after we've actually used it
313 ASSERT_LT(getFileDescriptorCount(), kMaxTotalEntries / 2);
314 }
315
getCacheEntries()316 std::vector<std::string> MultifileBlobCacheTest::getCacheEntries() {
317 std::string cachePath = &mTempFile->path[0];
318 std::string multifileDirName = cachePath + ".multifile";
319 std::vector<std::string> cacheEntries;
320
321 struct stat info;
322 if (stat(multifileDirName.c_str(), &info) == 0) {
323 // We have a multifile dir. Skip the status file and return the entries.
324 DIR* dir;
325 struct dirent* entry;
326 if ((dir = opendir(multifileDirName.c_str())) != nullptr) {
327 while ((entry = readdir(dir)) != nullptr) {
328 if (entry->d_name == "."s || entry->d_name == ".."s) {
329 continue;
330 }
331 if (strcmp(entry->d_name, kMultifileBlobCacheStatusFile) == 0) {
332 continue;
333 }
334 // printf("Found entry: %s\n", entry->d_name);
335 cacheEntries.push_back(multifileDirName + "/" + entry->d_name);
336 }
337 } else {
338 printf("Unable to open %s, error: %s\n", multifileDirName.c_str(),
339 std::strerror(errno));
340 }
341 } else {
342 printf("Unable to stat %s, error: %s\n", multifileDirName.c_str(), std::strerror(errno));
343 }
344
345 return cacheEntries;
346 }
347
TEST_F(MultifileBlobCacheTest,CacheContainsStatus)348 TEST_F(MultifileBlobCacheTest, CacheContainsStatus) {
349 struct stat info;
350 std::stringstream statusFile;
351 statusFile << &mTempFile->path[0] << ".multifile/" << kMultifileBlobCacheStatusFile;
352
353 // After INIT, cache should have a status
354 ASSERT_TRUE(stat(statusFile.str().c_str(), &info) == 0);
355
356 // Set one entry
357 mMBC->set("abcd", 4, "efgh", 4);
358
359 // Close the cache so everything writes out
360 mMBC->finish();
361 mMBC.reset();
362
363 // Ensure status lives after closing the cache
364 ASSERT_TRUE(stat(statusFile.str().c_str(), &info) == 0);
365
366 // Open the cache again
367 mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries,
368 &mTempFile->path[0]));
369
370 // Ensure we still have a status
371 ASSERT_TRUE(stat(statusFile.str().c_str(), &info) == 0);
372 }
373
374 // Verify missing cache status file causes cache the be cleared
TEST_F(MultifileBlobCacheTest,MissingCacheStatusClears)375 TEST_F(MultifileBlobCacheTest, MissingCacheStatusClears) {
376 // Set one entry
377 mMBC->set("abcd", 4, "efgh", 4);
378
379 // Close the cache so everything writes out
380 mMBC->finish();
381 mMBC.reset();
382
383 // Ensure there is one cache entry
384 ASSERT_EQ(getCacheEntries().size(), 1);
385
386 // Delete the status file
387 std::stringstream statusFile;
388 statusFile << &mTempFile->path[0] << ".multifile/" << kMultifileBlobCacheStatusFile;
389 remove(statusFile.str().c_str());
390
391 // Open the cache again and ensure no cache hits
392 mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries,
393 &mTempFile->path[0]));
394
395 // Ensure we have no entries
396 ASSERT_EQ(getCacheEntries().size(), 0);
397 }
398
399 // Verify modified cache status file BEGIN causes cache to be cleared
TEST_F(MultifileBlobCacheTest,ModifiedCacheStatusBeginClears)400 TEST_F(MultifileBlobCacheTest, ModifiedCacheStatusBeginClears) {
401 // Set one entry
402 mMBC->set("abcd", 4, "efgh", 4);
403
404 // Close the cache so everything writes out
405 mMBC->finish();
406 mMBC.reset();
407
408 // Ensure there is one cache entry
409 ASSERT_EQ(getCacheEntries().size(), 1);
410
411 // Modify the status file
412 std::stringstream statusFile;
413 statusFile << &mTempFile->path[0] << ".multifile/" << kMultifileBlobCacheStatusFile;
414
415 // Stomp on the beginning of the cache file
416 const char* stomp = "BADF00D";
417 std::fstream fs(statusFile.str());
418 fs.seekp(0, std::ios_base::beg);
419 fs.write(stomp, strlen(stomp));
420 fs.flush();
421 fs.close();
422
423 // Open the cache again and ensure no cache hits
424 mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries,
425 &mTempFile->path[0]));
426
427 // Ensure we have no entries
428 ASSERT_EQ(getCacheEntries().size(), 0);
429 }
430
431 // Verify modified cache status file END causes cache to be cleared
TEST_F(MultifileBlobCacheTest,ModifiedCacheStatusEndClears)432 TEST_F(MultifileBlobCacheTest, ModifiedCacheStatusEndClears) {
433 // Set one entry
434 mMBC->set("abcd", 4, "efgh", 4);
435
436 // Close the cache so everything writes out
437 mMBC->finish();
438 mMBC.reset();
439
440 // Ensure there is one cache entry
441 ASSERT_EQ(getCacheEntries().size(), 1);
442
443 // Modify the status file
444 std::stringstream statusFile;
445 statusFile << &mTempFile->path[0] << ".multifile/" << kMultifileBlobCacheStatusFile;
446
447 // Stomp on the END of the cache status file, modifying its contents
448 const char* stomp = "BADF00D";
449 std::fstream fs(statusFile.str());
450 fs.seekp(-strlen(stomp), std::ios_base::end);
451 fs.write(stomp, strlen(stomp));
452 fs.flush();
453 fs.close();
454
455 // Open the cache again and ensure no cache hits
456 mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries,
457 &mTempFile->path[0]));
458
459 // Ensure we have no entries
460 ASSERT_EQ(getCacheEntries().size(), 0);
461 }
462
463 // Verify mismatched cacheVersion causes cache to be cleared
TEST_F(MultifileBlobCacheTest,MismatchedCacheVersionClears)464 TEST_F(MultifileBlobCacheTest, MismatchedCacheVersionClears) {
465 // Set one entry
466 mMBC->set("abcd", 4, "efgh", 4);
467
468 uint32_t initialCacheVersion = mMBC->getCurrentCacheVersion();
469
470 // Close the cache so everything writes out
471 mMBC->finish();
472 mMBC.reset();
473
474 // Ensure there is one cache entry
475 ASSERT_EQ(getCacheEntries().size(), 1);
476
477 // Set a debug cacheVersion
478 std::string newCacheVersion = std::to_string(initialCacheVersion + 1);
479 ASSERT_TRUE(base::SetProperty("debug.egl.blobcache.cache_version", newCacheVersion.c_str()));
480 ASSERT_TRUE(
481 base::WaitForProperty("debug.egl.blobcache.cache_version", newCacheVersion.c_str()));
482
483 // Open the cache again and ensure no cache hits
484 mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries,
485 &mTempFile->path[0]));
486
487 // Ensure we have no entries
488 ASSERT_EQ(getCacheEntries().size(), 0);
489 }
490
491 // Verify mismatched buildId causes cache to be cleared
TEST_F(MultifileBlobCacheTest,MismatchedBuildIdClears)492 TEST_F(MultifileBlobCacheTest, MismatchedBuildIdClears) {
493 // Set one entry
494 mMBC->set("abcd", 4, "efgh", 4);
495
496 // Close the cache so everything writes out
497 mMBC->finish();
498 mMBC.reset();
499
500 // Ensure there is one cache entry
501 ASSERT_EQ(getCacheEntries().size(), 1);
502
503 // Set a debug buildId
504 base::SetProperty("debug.egl.blobcache.build_id", "foo");
505 base::WaitForProperty("debug.egl.blobcache.build_id", "foo");
506
507 // Open the cache again and ensure no cache hits
508 mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries,
509 &mTempFile->path[0]));
510
511 // Ensure we have no entries
512 ASSERT_EQ(getCacheEntries().size(), 0);
513 }
514
515 // Ensure cache is correct when a key is reused
TEST_F(MultifileBlobCacheTest,SameKeyDifferentValues)516 TEST_F(MultifileBlobCacheTest, SameKeyDifferentValues) {
517 if (!flags::multifile_blobcache_advanced_usage()) {
518 GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag";
519 }
520
521 unsigned char buf[4] = {0xee, 0xee, 0xee, 0xee};
522
523 size_t startingSize = mMBC->getTotalSize();
524
525 // New cache should be empty
526 ASSERT_EQ(startingSize, 0);
527
528 // Set an initial value
529 mMBC->set("ab", 2, "cdef", 4);
530
531 // Grab the new size
532 size_t firstSize = mMBC->getTotalSize();
533
534 // Ensure the size went up
535 // Note: Checking for an exact size is challenging, as the
536 // file size can differ between platforms.
537 ASSERT_GT(firstSize, startingSize);
538
539 // Verify the cache is correct
540 ASSERT_EQ(size_t(4), mMBC->get("ab", 2, buf, 4));
541 ASSERT_EQ('c', buf[0]);
542 ASSERT_EQ('d', buf[1]);
543 ASSERT_EQ('e', buf[2]);
544 ASSERT_EQ('f', buf[3]);
545
546 // Now reuse the key with a smaller value
547 mMBC->set("ab", 2, "gh", 2);
548
549 // Grab the new size
550 size_t secondSize = mMBC->getTotalSize();
551
552 // Ensure it decreased in size
553 ASSERT_LT(secondSize, firstSize);
554
555 // Verify the cache is correct
556 ASSERT_EQ(size_t(2), mMBC->get("ab", 2, buf, 2));
557 ASSERT_EQ('g', buf[0]);
558 ASSERT_EQ('h', buf[1]);
559
560 // Now put back the original value
561 mMBC->set("ab", 2, "cdef", 4);
562
563 // And we should get back a stable size
564 size_t finalSize = mMBC->getTotalSize();
565 ASSERT_EQ(firstSize, finalSize);
566 }
567
568 // Ensure cache is correct when a key is reused with large value size
TEST_F(MultifileBlobCacheTest,SameKeyLargeValues)569 TEST_F(MultifileBlobCacheTest, SameKeyLargeValues) {
570 if (!flags::multifile_blobcache_advanced_usage()) {
571 GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag";
572 }
573
574 // Create the cache with larger limits to stress test reuse
575 constexpr uint32_t kLocalMaxKeySize = 1 * 1024 * 1024;
576 constexpr uint32_t kLocalMaxValueSize = 4 * 1024 * 1024;
577 constexpr uint32_t kLocalMaxTotalSize = 32 * 1024 * 1024;
578 mMBC.reset(new MultifileBlobCache(kLocalMaxKeySize, kLocalMaxValueSize, kLocalMaxTotalSize,
579 kMaxTotalEntries, &mTempFile->path[0]));
580
581 constexpr uint32_t kLargeValueCount = 8;
582 constexpr uint32_t kLargeValueSize = 64 * 1024;
583
584 // Create a several really large values
585 unsigned char largeValue[kLargeValueCount][kLargeValueSize];
586 for (int i = 0; i < kLargeValueCount; i++) {
587 for (int j = 0; j < kLargeValueSize; j++) {
588 // Fill the value with the index for uniqueness
589 largeValue[i][j] = i;
590 }
591 }
592
593 size_t startingSize = mMBC->getTotalSize();
594
595 // New cache should be empty
596 ASSERT_EQ(startingSize, 0);
597
598 // Cycle through the values and set them all in sequence
599 for (int i = 0; i < kLargeValueCount; i++) {
600 mMBC->set("abcd", 4, largeValue[i], kLargeValueSize);
601 }
602
603 // Ensure we get the last one back
604 unsigned char outBuf[kLargeValueSize];
605 mMBC->get("abcd", 4, outBuf, kLargeValueSize);
606
607 for (int i = 0; i < kLargeValueSize; i++) {
608 // Buffer should contain highest index value
609 ASSERT_EQ(kLargeValueCount - 1, outBuf[i]);
610 }
611 }
612
613 // Ensure cache eviction is LRU
TEST_F(MultifileBlobCacheTest,CacheEvictionIsLRU)614 TEST_F(MultifileBlobCacheTest, CacheEvictionIsLRU) {
615 if (!flags::multifile_blobcache_advanced_usage()) {
616 GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag";
617 }
618
619 // Fill the cache with exactly how much it can hold
620 int entry = 0;
621 for (entry = 0; entry < kMaxTotalEntries; entry++) {
622 // Use the index as the key and value
623 mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry));
624
625 int result = 0;
626 ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
627 ASSERT_EQ(entry, result);
628 }
629
630 // Ensure the cache is full
631 ASSERT_EQ(mMBC->getTotalEntries(), kMaxTotalEntries);
632
633 // Add one more entry to trigger eviction
634 size_t overflowEntry = kMaxTotalEntries;
635 mMBC->set(&overflowEntry, sizeof(overflowEntry), &overflowEntry, sizeof(overflowEntry));
636
637 // Verify it contains the right amount, which will be one more than reduced size
638 // because we evict the cache before adding a new entry
639 size_t evictionLimit = kMaxTotalEntries / mMBC->getTotalCacheSizeDivisor();
640 ASSERT_EQ(mMBC->getTotalEntries(), evictionLimit + 1);
641
642 // Ensure cache is as expected, with old entries removed, newer entries remaining
643 for (entry = 0; entry < kMaxTotalEntries; entry++) {
644 int result = 0;
645 mMBC->get(&entry, sizeof(entry), &result, sizeof(result));
646
647 if (entry < evictionLimit) {
648 // We should get no hits on evicted entries, i.e. the first added
649 ASSERT_EQ(result, 0);
650 } else {
651 // Above the limit should still be present
652 ASSERT_EQ(result, entry);
653 }
654 }
655 }
656
657 // Ensure calling GET on an entry updates its access time, even if already in hotcache
TEST_F(MultifileBlobCacheTest,GetUpdatesAccessTime)658 TEST_F(MultifileBlobCacheTest, GetUpdatesAccessTime) {
659 if (!flags::multifile_blobcache_advanced_usage()) {
660 GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag";
661 }
662
663 // Fill the cache with exactly how much it can hold
664 int entry = 0;
665 int result = 0;
666 for (entry = 0; entry < kMaxTotalEntries; entry++) {
667 // Use the index as the key and value
668 mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry));
669 ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
670 ASSERT_EQ(entry, result);
671 }
672
673 // Ensure the cache is full
674 ASSERT_EQ(mMBC->getTotalEntries(), kMaxTotalEntries);
675
676 // GET the first few entries to update their access time
677 std::vector<int> accessedEntries = {1, 2, 3};
678 for (int i = 0; i < accessedEntries.size(); i++) {
679 entry = accessedEntries[i];
680 ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
681 }
682
683 // Add one more entry to trigger eviction
684 size_t overflowEntry = kMaxTotalEntries;
685 mMBC->set(&overflowEntry, sizeof(overflowEntry), &overflowEntry, sizeof(overflowEntry));
686
687 size_t evictionLimit = kMaxTotalEntries / mMBC->getTotalCacheSizeDivisor();
688
689 // Ensure cache is as expected, with old entries removed, newer entries remaining
690 for (entry = 0; entry < kMaxTotalEntries; entry++) {
691 int result = 0;
692 mMBC->get(&entry, sizeof(entry), &result, sizeof(result));
693
694 if (std::find(accessedEntries.begin(), accessedEntries.end(), entry) !=
695 accessedEntries.end()) {
696 // If this is one of the handful we accessed after filling the cache,
697 // they should still be in the cache because LRU
698 ASSERT_EQ(result, entry);
699 } else if (entry >= (evictionLimit + accessedEntries.size())) {
700 // If they were above the eviction limit (plus three for our updated entries),
701 // they should still be present
702 ASSERT_EQ(result, entry);
703 } else {
704 // Otherwise, they shold be evicted and no longer present
705 ASSERT_EQ(result, 0);
706 }
707 }
708
709 // Close the cache so everything writes out
710 mMBC->finish();
711 mMBC.reset();
712
713 // Open the cache again
714 mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries,
715 &mTempFile->path[0]));
716
717 // Check the cache again, ensuring the updated access time made it to disk
718 for (entry = 0; entry < kMaxTotalEntries; entry++) {
719 int result = 0;
720 mMBC->get(&entry, sizeof(entry), &result, sizeof(result));
721 if (std::find(accessedEntries.begin(), accessedEntries.end(), entry) !=
722 accessedEntries.end()) {
723 ASSERT_EQ(result, entry);
724 } else if (entry >= (evictionLimit + accessedEntries.size())) {
725 ASSERT_EQ(result, entry);
726 } else {
727 ASSERT_EQ(result, 0);
728 }
729 }
730 }
731
clearCache()732 bool MultifileBlobCacheTest::clearCache() {
733 std::string cachePath = &mTempFile->path[0];
734 std::string multifileDirName = cachePath + ".multifile";
735
736 DIR* dir = opendir(multifileDirName.c_str());
737 if (dir == nullptr) {
738 printf("Error opening directory: %s\n", multifileDirName.c_str());
739 return false;
740 }
741
742 struct dirent* entry;
743 while ((entry = readdir(dir)) != nullptr) {
744 // Skip "." and ".." entries
745 if (std::string(entry->d_name) == "." || std::string(entry->d_name) == "..") {
746 continue;
747 }
748
749 std::string entryPath = multifileDirName + "/" + entry->d_name;
750
751 // Delete the entry (we assert it's a file, nothing nested here)
752 if (unlink(entryPath.c_str()) != 0) {
753 printf("Error deleting file: %s\n", entryPath.c_str());
754 closedir(dir);
755 return false;
756 }
757 }
758
759 closedir(dir);
760
761 // Delete the empty directory itself
762 if (rmdir(multifileDirName.c_str()) != 0) {
763 printf("Error deleting directory %s, error %s\n", multifileDirName.c_str(),
764 std::strerror(errno));
765 return false;
766 }
767
768 return true;
769 }
770
771 // Recover from lost cache in the case of app clearing it
TEST_F(MultifileBlobCacheTest,RecoverFromLostCache)772 TEST_F(MultifileBlobCacheTest, RecoverFromLostCache) {
773 if (!flags::multifile_blobcache_advanced_usage()) {
774 GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag";
775 }
776
777 int entry = 0;
778 int result = 0;
779
780 uint32_t kEntryCount = 10;
781
782 // Add some entries
783 for (entry = 0; entry < kEntryCount; entry++) {
784 mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry));
785 ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
786 ASSERT_EQ(entry, result);
787 }
788
789 // For testing, wait until the entries have completed writing
790 mMBC->finish();
791
792 // Manually delete the cache!
793 ASSERT_TRUE(clearCache());
794
795 // Cache should not contain any entries
796 for (entry = 0; entry < kEntryCount; entry++) {
797 ASSERT_EQ(size_t(0), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
798 }
799
800 // Ensure we can still add new ones
801 for (entry = kEntryCount; entry < kEntryCount * 2; entry++) {
802 mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry));
803 ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
804 ASSERT_EQ(entry, result);
805 }
806
807 // Close the cache so everything writes out
808 mMBC->finish();
809 mMBC.reset();
810
811 // Open the cache again
812 mMBC.reset(new MultifileBlobCache(kMaxKeySize, kMaxValueSize, kMaxTotalSize, kMaxTotalEntries,
813 &mTempFile->path[0]));
814
815 // Before fixes, writing the second entries to disk should have failed due to missing
816 // cache dir. But now they should have survived our shutdown above.
817 for (entry = kEntryCount; entry < kEntryCount * 2; entry++) {
818 ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
819 ASSERT_EQ(entry, result);
820 }
821 }
822
823 // Ensure cache eviction succeeds if the cache is deleted
TEST_F(MultifileBlobCacheTest,EvictAfterLostCache)824 TEST_F(MultifileBlobCacheTest, EvictAfterLostCache) {
825 if (!flags::multifile_blobcache_advanced_usage()) {
826 GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag";
827 }
828
829 int entry = 0;
830 int result = 0;
831
832 uint32_t kEntryCount = 10;
833
834 // Add some entries
835 for (entry = 0; entry < kEntryCount; entry++) {
836 mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry));
837 ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
838 ASSERT_EQ(entry, result);
839 }
840
841 // For testing, wait until the entries have completed writing
842 mMBC->finish();
843
844 // Manually delete the cache!
845 ASSERT_TRUE(clearCache());
846
847 // Now start adding entries to trigger eviction, cache should survive
848 for (entry = kEntryCount; entry < 2 * kMaxTotalEntries; entry++) {
849 mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry));
850 ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
851 ASSERT_EQ(entry, result);
852 }
853
854 // We should have triggered multiple evictions above and remain at or below the
855 // max amount of entries
856 ASSERT_LE(getCacheEntries().size(), kMaxTotalEntries);
857 }
858
859 // Remove from cache when size is zero
TEST_F(MultifileBlobCacheTest,ZeroSizeRemovesEntry)860 TEST_F(MultifileBlobCacheTest, ZeroSizeRemovesEntry) {
861 if (!flags::multifile_blobcache_advanced_usage()) {
862 GTEST_SKIP() << "Skipping test that requires multifile_blobcache_advanced_usage flag";
863 }
864
865 // Put some entries in
866 int entry = 0;
867 int result = 0;
868
869 uint32_t kEntryCount = 20;
870
871 // Add some entries
872 for (entry = 0; entry < kEntryCount; entry++) {
873 mMBC->set(&entry, sizeof(entry), &entry, sizeof(entry));
874 ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
875 ASSERT_EQ(entry, result);
876 }
877
878 // Send some of them again with size zero
879 std::vector<int> removedEntries = {5, 10, 18};
880 for (int i = 0; i < removedEntries.size(); i++) {
881 entry = removedEntries[i];
882 mMBC->set(&entry, sizeof(entry), nullptr, 0);
883 }
884
885 // Ensure they do not get a hit
886 for (int i = 0; i < removedEntries.size(); i++) {
887 entry = removedEntries[i];
888 ASSERT_EQ(size_t(0), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
889 }
890
891 // And have been removed from disk
892 std::vector<std::string> diskEntries = getCacheEntries();
893 ASSERT_EQ(diskEntries.size(), kEntryCount - removedEntries.size());
894 for (int i = 0; i < removedEntries.size(); i++) {
895 entry = removedEntries[i];
896 // Generate a hash for our removed entries and ensure they are not contained
897 // Note our entry and key and the same here, so we're hashing the key just like
898 // the multifile blobcache does.
899 uint32_t entryHash =
900 android::JenkinsHashMixBytes(0, reinterpret_cast<uint8_t*>(&entry), sizeof(entry));
901 ASSERT_EQ(std::find(diskEntries.begin(), diskEntries.end(), std::to_string(entryHash)),
902 diskEntries.end());
903 }
904
905 // Ensure the others are still present
906 for (entry = 0; entry < kEntryCount; entry++) {
907 if (std::find(removedEntries.begin(), removedEntries.end(), entry) ==
908 removedEntries.end()) {
909 ASSERT_EQ(sizeof(entry), mMBC->get(&entry, sizeof(entry), &result, sizeof(result)));
910 ASSERT_EQ(result, entry);
911 }
912 }
913 }
914
915 } // namespace android
916