1 // Copyright (C) 2019 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 // Test for Filesystem class and utils.
16
17 #include "icing/file/filesystem.h"
18
19 #include <algorithm>
20 #include <cstdint>
21 #include <string>
22 #include <unordered_set>
23 #include <vector>
24
25 #include "gmock/gmock.h"
26 #include "gtest/gtest.h"
27 #include "icing/testing/tmp-directory.h"
28
29 #if defined(__APPLE__)
30 #include <TargetConditionals.h>
31 #endif
32
33 using std::sort;
34 using std::vector;
35 using ::testing::Eq;
36 using ::testing::Ge;
37 using ::testing::Gt;
38 using ::testing::Le;
39 using ::testing::Ne;
40 using ::testing::UnorderedElementsAre;
41 using ::testing::UnorderedElementsAreArray;
42
43 namespace icing {
44 namespace lib {
45
46 namespace {
47 // Create some test files in the specified directory. "test data" plus the
48 // relative path name + "\n" is written to each.
CreateTestFiles(const vector<std::string> & file_names,const std::string & append_dir)49 void CreateTestFiles(const vector<std::string>& file_names,
50 const std::string& append_dir) {
51 Filesystem filesystem;
52 for (const std::string& one_file_name : file_names) {
53 // Write the filename to the file
54 std::string one_file_path = append_dir + "/" + one_file_name;
55 int fd = filesystem.OpenForWrite(one_file_path.c_str());
56 ASSERT_THAT(fd, Gt(0));
57 std::string test_data = "test data " + one_file_name + "\n";
58 EXPECT_TRUE(
59 filesystem.Write(fd, test_data.c_str(), strlen(test_data.c_str())));
60 EXPECT_THAT(close(fd), Eq(0));
61 }
62 }
63 } // namespace
64
65 // Indicates if the file system supports Sparse Files.
66 // 'Sparse files' are essentially pre-allocated files of big sizes which do not
67 // yet use any blocks. A few tests validate that disk-usage is accounted
68 // correctly in those cases as zero.
69 // However, on HFS+ file system sparse files are not supported.
70 // The new AFS supports sparse files, but as of 2017-09 all simulators in prod
71 // are running on MacOS using HFS+.
FileSystemSupportsSparseFiles()72 bool FileSystemSupportsSparseFiles() {
73 #ifdef TARGET_IPHONE_SIMULATOR
74 return false;
75 #else
76 return true;
77 #endif
78 }
79
80 class FilesystemTest : public testing::Test {
81 protected:
SetUp()82 void SetUp() override {
83 temp_dir_ = GetTestTempDir() + "/icing_filesystem";
84 Filesystem filesystem;
85 ASSERT_TRUE(filesystem.CreateDirectoryRecursively(temp_dir_.c_str()));
86 }
87
TearDown()88 void TearDown() override {
89 Filesystem filesystem;
90 EXPECT_TRUE(filesystem.DeleteDirectoryRecursively(temp_dir_.c_str()));
91 }
92
93 // Write junk data of given size to the given file descriptor
WriteJunk(int fd,size_t size)94 void WriteJunk(int fd, size_t size) {
95 const int kBufLen = 1024;
96 int buf[kBufLen];
97 for (int i = 0; i < kBufLen; ++i) {
98 buf[i] = i;
99 }
100 const int kBufSize = kBufLen * sizeof(int);
101
102 Filesystem filesystem;
103 for (size_t i = 0; i < size / kBufSize; ++i) {
104 EXPECT_TRUE(filesystem.Write(fd, buf, kBufSize));
105 }
106 if (size % kBufSize) {
107 EXPECT_TRUE(filesystem.Write(fd, buf, size % kBufSize));
108 }
109 }
110
111 std::string temp_dir_;
112 };
113
TEST_F(FilesystemTest,Names)114 TEST_F(FilesystemTest, Names) {
115 const std::string filename("/foo/bar/README.txt");
116 Filesystem filesystem;
117
118 std::string basename = filesystem.GetBasename(filename.c_str());
119 EXPECT_THAT(basename, Eq("README.txt"));
120
121 std::string dirname = filesystem.GetDirname(filename.c_str());
122 EXPECT_THAT(dirname, Eq("/foo/bar"));
123
124 basename = filesystem.GetBasename(dirname.c_str());
125 EXPECT_THAT(basename, Eq("bar"));
126
127 dirname = filesystem.GetDirname(dirname.c_str());
128 EXPECT_THAT(dirname, Eq("/foo"));
129
130 basename = filesystem.GetBasename(dirname.c_str());
131 EXPECT_THAT(basename, Eq("foo"));
132
133 dirname = filesystem.GetDirname(dirname.c_str());
134 EXPECT_THAT(dirname, Eq(""));
135 }
136
TEST_F(FilesystemTest,OneLetter)137 TEST_F(FilesystemTest, OneLetter) {
138 Filesystem filesystem;
139
140 const std::string basename = filesystem.GetDirname("a");
141 EXPECT_THAT(basename, Eq(""));
142
143 const std::string dirname = filesystem.GetDirname("a");
144 EXPECT_THAT(dirname, Eq(""));
145 }
146
TEST_F(FilesystemTest,Directory)147 TEST_F(FilesystemTest, Directory) {
148 Filesystem filesystem;
149
150 const std::string foo_str = temp_dir_ + "/foo";
151 const std::string bar_str = foo_str + "/bar";
152 const char* foo_dir = foo_str.c_str();
153 const char* bar_dir = bar_str.c_str();
154
155 EXPECT_TRUE(filesystem.CreateDirectory(foo_dir));
156 EXPECT_TRUE(filesystem.DirectoryExists(foo_dir));
157 EXPECT_TRUE(filesystem.DeleteDirectory(foo_dir));
158 EXPECT_FALSE(filesystem.DirectoryExists(foo_dir));
159
160 EXPECT_FALSE(filesystem.CreateDirectory(bar_dir));
161 EXPECT_FALSE(filesystem.DirectoryExists(foo_dir));
162 EXPECT_FALSE(filesystem.DirectoryExists(bar_dir));
163 EXPECT_TRUE(filesystem.CreateDirectoryRecursively(bar_dir));
164 EXPECT_TRUE(filesystem.DirectoryExists(foo_dir));
165 EXPECT_TRUE(filesystem.DirectoryExists(bar_dir));
166
167 EXPECT_FALSE(filesystem.DeleteDirectory(foo_dir));
168 EXPECT_TRUE(filesystem.DeleteDirectoryRecursively(foo_dir));
169 EXPECT_FALSE(filesystem.DirectoryExists(foo_dir));
170 EXPECT_FALSE(filesystem.DirectoryExists(bar_dir));
171
172 // Deleting a non-existing directory returns true.
173 EXPECT_TRUE(filesystem.DeleteDirectory(foo_dir));
174 EXPECT_TRUE(filesystem.DeleteDirectoryRecursively(foo_dir));
175 }
176
TEST_F(FilesystemTest,FSync)177 TEST_F(FilesystemTest, FSync) {
178 Filesystem filesystem;
179 const std::string foo_file = temp_dir_ + "/foo_file";
180 int fd = filesystem.OpenForWrite(foo_file.c_str());
181 ASSERT_THAT(fd, Ne(-1));
182 EXPECT_TRUE(filesystem.DataSync(fd));
183 close(fd);
184 }
185
TEST_F(FilesystemTest,Truncate)186 TEST_F(FilesystemTest, Truncate) {
187 Filesystem filesystem;
188 const std::string foo_file = temp_dir_ + "/foo_file";
189 const char* filename = foo_file.c_str();
190 int fd = filesystem.OpenForWrite(filename);
191 ASSERT_THAT(fd, Ne(-1));
192 char data[10000] = {0}; // Zero-init to satisfy msan.
193 EXPECT_TRUE(filesystem.Write(fd, data, sizeof(data)));
194 close(fd);
195 EXPECT_THAT(filesystem.GetFileSize(filename), Eq(sizeof(data)));
196 EXPECT_TRUE(filesystem.Truncate(filename, sizeof(data) / 2));
197 EXPECT_THAT(filesystem.GetFileSize(filename), Eq(sizeof(data) / 2));
198 EXPECT_TRUE(filesystem.Truncate(filename, 0));
199 EXPECT_THAT(filesystem.GetFileSize(filename), Eq(0u));
200 }
201
TEST_F(FilesystemTest,GetMatchingFiles)202 TEST_F(FilesystemTest, GetMatchingFiles) {
203 Filesystem filesystem;
204 const std::string foo_dir = temp_dir_ + "/foo";
205 const std::string glob = foo_dir + "/p_*_q";
206 vector<std::string> matches;
207
208 // Non existing directory
209 EXPECT_TRUE(filesystem.GetMatchingFiles(glob.c_str(), &matches));
210 EXPECT_THAT(matches.size(), Eq(0u));
211
212 // Existing directory
213 matches.clear();
214 ASSERT_TRUE(filesystem.CreateDirectoryRecursively(foo_dir.c_str()));
215 EXPECT_TRUE(filesystem.GetMatchingFiles(glob.c_str(), &matches));
216 EXPECT_THAT(matches.size(), Eq(0u));
217
218 // With some files
219 matches.clear();
220 const int files_size = 4;
221 const char* files[files_size] = {"p_1_q", "p_2_q", "p_3", "4_q"};
222 for (size_t i = 0; i < files_size; ++i) {
223 ScopedFd file(filesystem.OpenForWrite((foo_dir + "/" + files[i]).c_str()));
224 }
225 const int good_size = 2;
226 const std::string good[good_size] = {foo_dir + "/p_1_q", foo_dir + "/p_2_q"};
227 vector<std::string> expected(good, good + good_size);
228 EXPECT_TRUE(filesystem.GetMatchingFiles(glob.c_str(), &matches));
229 sort(matches.begin(), matches.end());
230 EXPECT_THAT(matches, Eq(expected));
231 }
232
TEST_F(FilesystemTest,IncrementByOrSetInvalid)233 TEST_F(FilesystemTest, IncrementByOrSetInvalid) {
234 int64_t to_increment = 1;
235 Filesystem::IncrementByOrSetInvalid(2, &to_increment);
236 EXPECT_THAT(to_increment, Eq(3));
237
238 Filesystem::IncrementByOrSetInvalid(Filesystem::kBadFileSize, &to_increment);
239 EXPECT_THAT(to_increment, Eq(Filesystem::kBadFileSize));
240
241 to_increment = Filesystem::kBadFileSize;
242 Filesystem::IncrementByOrSetInvalid(2, &to_increment);
243 EXPECT_THAT(to_increment, Eq(Filesystem::kBadFileSize));
244 }
245
TEST_F(FilesystemTest,GetDiskUsage)246 TEST_F(FilesystemTest, GetDiskUsage) {
247 Filesystem filesystem;
248 const std::string foo_dir = temp_dir_ + "/foo";
249
250 const int64_t kCluster = 4096; // at least the anticipated fs cluster
251
252 ASSERT_TRUE(filesystem.CreateDirectoryRecursively(foo_dir.c_str()));
253
254 // Grow a sparse file, and then append to it.
255 const std::string filename = foo_dir + "/myfile";
256 // Size to expand the sparse file to.
257 const int64_t kExpandedSize = 100 * kCluster - 5;
258 // Actual data to write to the file.
259 const int64_t kJunkSize = 5 * kCluster - 10;
260
261 EXPECT_TRUE(filesystem.Truncate(filename.c_str(), kExpandedSize));
262 ScopedFd fd(filesystem.OpenForWrite(filename.c_str()));
263 WriteJunk(*fd, kJunkSize);
264
265 int64_t size = filesystem.GetDiskUsage(*fd);
266 EXPECT_THAT(size, Ge(kJunkSize));
267 if (FileSystemSupportsSparseFiles()) {
268 EXPECT_THAT(size, Le(kExpandedSize));
269 }
270 }
271
TEST_F(FilesystemTest,GetDiskUsagePath)272 TEST_F(FilesystemTest, GetDiskUsagePath) {
273 Filesystem filesystem;
274 const std::string foo_dir = temp_dir_ + "/foo";
275
276 const int64_t kCluster = 4096; // at least the anticipated fs cluster
277
278 // Non-existing
279 {
280 EXPECT_THAT(filesystem.GetDiskUsage(foo_dir.c_str()),
281 Eq(Filesystem::kBadFileSize));
282 }
283
284 // A single directory
285 {
286 ASSERT_TRUE(filesystem.CreateDirectoryRecursively(foo_dir.c_str()));
287 int64_t size = filesystem.GetDiskUsage(foo_dir.c_str());
288 EXPECT_THAT(size, Ge(0 * kCluster));
289 EXPECT_THAT(size, Le(1 * kCluster));
290 }
291
292 // Nested directories
293 const std::string bar_dir = foo_dir + "/bar";
294 {
295 ASSERT_TRUE(filesystem.CreateDirectoryRecursively(bar_dir.c_str()));
296 int64_t size = filesystem.GetDiskUsage(bar_dir.c_str());
297 EXPECT_THAT(size, Ge(0 * kCluster));
298 EXPECT_THAT(size, Le(2 * kCluster));
299 }
300
301 // Two regular files
302 const std::string reg1 = bar_dir + "/f1";
303 const std::string reg2 = bar_dir + "/f2";
304 {
305 {
306 ScopedFd f1(filesystem.OpenForWrite(reg1.c_str()));
307 ScopedFd f2(filesystem.OpenForWrite(reg2.c_str()));
308 WriteJunk(*f1, 5 * kCluster - 10);
309 WriteJunk(*f2, 8 * kCluster - 10);
310 }
311 int64_t size = filesystem.GetDiskUsage(foo_dir.c_str());
312 EXPECT_THAT(size, Ge(13 * kCluster));
313 EXPECT_THAT(size, Le(15 * kCluster));
314 }
315
316 // Two sparse files
317 const std::string sparse1 = foo_dir + "/s1";
318 const std::string sparse2 = foo_dir + "/s2";
319 {
320 EXPECT_TRUE(filesystem.Truncate(sparse1.c_str(), 100 * kCluster - 5));
321 EXPECT_TRUE(filesystem.Truncate(sparse2.c_str(), 200 * kCluster - 123));
322 int64_t size = filesystem.GetDiskUsage(foo_dir.c_str());
323 EXPECT_THAT(size, Ge(13 * kCluster));
324 if (FileSystemSupportsSparseFiles()) {
325 EXPECT_THAT(size, Le(17 * kCluster));
326 } else {
327 EXPECT_THAT(size, Le(313 * kCluster));
328 }
329 }
330
331 // Some junk in the sparse files
332 {
333 {
334 ScopedFd f1(filesystem.OpenForWrite(sparse1.c_str()));
335 ScopedFd f2(filesystem.OpenForWrite(sparse2.c_str()));
336 WriteJunk(*f1, 5 * kCluster - 10);
337 WriteJunk(*f2, 8 * kCluster - 10);
338 }
339 int64_t size = filesystem.GetDiskUsage(foo_dir.c_str());
340 EXPECT_THAT(size, Ge(26 * kCluster));
341 if (FileSystemSupportsSparseFiles()) {
342 EXPECT_THAT(size, Le(30 * kCluster));
343 } else {
344 EXPECT_THAT(size, Le(313 * kCluster));
345 }
346 }
347 }
348
349 // TODO(b/112435354): Add test case for original (non-recursive) ListDirectory()
350
351 // Tests ListDirectory() with recursive dir search, with no exclusions
352 // (simple test).
TEST_F(FilesystemTest,ListDirectoryRecursiveSimple)353 TEST_F(FilesystemTest, ListDirectoryRecursiveSimple) {
354 Filesystem filesystem;
355 const std::string append_dir = temp_dir_ + "/append_test";
356 const std::string dir1_name = "dir1";
357 const std::string dir1_path = append_dir + "/" + dir1_name;
358 vector<std::string> some_files = {"file1", "file2", dir1_name + "/file3"};
359
360 // Make sure there is no pre-existing test-dir structure
361 ASSERT_TRUE(filesystem.DeleteDirectoryRecursively(append_dir.c_str()));
362
363 // Setup a test-dir structure
364 ASSERT_TRUE(filesystem.CreateDirectoryRecursively(
365 dir1_path.c_str())); // deepest path for test
366 CreateTestFiles(some_files, append_dir);
367
368 // Call the ListDirectory API with recursive dir-search, no exclusions.
369 vector<std::string> result;
370 EXPECT_TRUE(filesystem.ListDirectory(append_dir.c_str(), /*exclude=*/{},
371 /*recursive=*/true, &result));
372
373 // Verify that all files are returned, and no extras.
374 EXPECT_THAT(result, UnorderedElementsAre(some_files[0], some_files[1],
375 dir1_name, some_files[2]));
376
377 // Clean up
378 ASSERT_TRUE(filesystem.DeleteDirectoryRecursively(append_dir.c_str()));
379 }
380
381 // Tests ListDirectory() with recursive dir search, with exclusions.
382 // This test is similar in structure to ListDirectory_recursive_simple, but with
383 // exclusions.
TEST_F(FilesystemTest,ListDirectoryRecursiveExclude)384 TEST_F(FilesystemTest, ListDirectoryRecursiveExclude) {
385 Filesystem filesystem;
386 const std::string append_dir = temp_dir_ + "/append_test";
387 const std::string dir1_name = "dir1";
388 const std::string dir1_path = append_dir + "/" + dir1_name;
389 vector<std::string> some_files = {"file1", "file2", dir1_name + "/file3"};
390
391 // Make sure there is no pre-existing test-dir structure
392 ASSERT_TRUE(filesystem.DeleteDirectoryRecursively(append_dir.c_str()));
393
394 // Setup a test-dir structure
395 ASSERT_TRUE(filesystem.CreateDirectoryRecursively(
396 dir1_path.c_str())); // deepest path for test
397 CreateTestFiles(some_files, append_dir);
398
399 // Call the ListDirectory API with recursive dir-search, but exclude dir1.
400 vector<std::string> result;
401 std::unordered_set<std::string> exclude;
402 bool success = filesystem.ListDirectory(append_dir.c_str(),
403 /*exclude=*/{dir1_name.c_str()},
404 /*recursive=*/true, &result);
405
406 // Verify that all files are returned, and no extras.
407 EXPECT_TRUE(success);
408 EXPECT_THAT(result, UnorderedElementsAre(some_files[0], some_files[1]));
409
410 // Clean up
411 ASSERT_TRUE(filesystem.DeleteDirectoryRecursively(append_dir.c_str()));
412 }
413
TEST_F(FilesystemTest,ReadWrite)414 TEST_F(FilesystemTest, ReadWrite) {
415 Filesystem filesystem;
416 const std::string foo_file = temp_dir_ + "/foo_file";
417 int fd = filesystem.OpenForWrite(foo_file.c_str());
418 const std::string data = "hello world";
419 EXPECT_TRUE(filesystem.Write(fd, data.c_str(), strlen(data.c_str())));
420
421 std::string hello;
422 hello.resize(strlen("hello"));
423 EXPECT_TRUE(filesystem.Read(foo_file.c_str(), &hello[0], strlen("hello")));
424 EXPECT_THAT(hello, Eq("hello"));
425
426 // Read starts from wherever file offset is at the moment.
427 filesystem.SetPosition(fd, 0);
428 hello.clear();
429 hello.resize(strlen("hello"));
430 EXPECT_TRUE(filesystem.Read(fd, &hello[0], strlen("hello")));
431 EXPECT_THAT(hello, Eq("hello"));
432
433 // Shouldn't need to move file offset anymore since file offset gets updated
434 // after the read.
435 std::string world;
436 world.resize(strlen(" world"));
437 EXPECT_TRUE(filesystem.Read(fd, &world[0], strlen(" world")));
438 EXPECT_THAT(world, Eq(" world"));
439
440 // PRead should not be dependent on the file offset
441 world.clear();
442 world.resize(strlen(" world"));
443 EXPECT_TRUE(
444 filesystem.PRead(fd, &world[0], strlen(" world"), strlen("hello")));
445 EXPECT_THAT(world, Eq(" world"));
446
447 hello.clear();
448 hello.resize(strlen("hello"));
449 EXPECT_TRUE(
450 filesystem.PRead(foo_file.c_str(), &hello[0], strlen("hello"), 0));
451 EXPECT_THAT(hello, Eq("hello"));
452 }
453
TEST_F(FilesystemTest,CopyDirectory)454 TEST_F(FilesystemTest, CopyDirectory) {
455 Filesystem filesystem;
456
457 // File structure:
458 // <temp_dir>/
459 // src_dir/
460 // file1
461 // file2
462 // sub_dir/
463 // file3
464 const std::string src_dir = temp_dir_ + "/src_dir";
465 const std::string sub_dir = "sub_dir";
466 const std::string sub_dir_path = src_dir + "/" + sub_dir;
467 vector<std::string> some_files = {"file1", "file2", sub_dir + "/file3"};
468
469 // Make sure there is no pre-existing test-dir structure
470 ASSERT_TRUE(filesystem.DeleteDirectoryRecursively(src_dir.c_str()));
471
472 // Setup a test-dir structure
473 ASSERT_TRUE(filesystem.CreateDirectoryRecursively(
474 sub_dir_path.c_str())); // deepest path for test
475 CreateTestFiles(some_files, src_dir);
476
477 const std::string dst_dir = temp_dir_ + "/dst_dir";
478 EXPECT_TRUE(filesystem.CopyDirectory(src_dir.c_str(), dst_dir.c_str(),
479 /*recursive=*/true));
480
481 vector<std::string> src_dir_files;
482 EXPECT_TRUE(filesystem.ListDirectory(src_dir.c_str(), /*exclude=*/{},
483 /*recursive=*/true, &src_dir_files));
484
485 vector<std::string> dst_dir_files;
486 EXPECT_TRUE(filesystem.ListDirectory(dst_dir.c_str(), /*exclude=*/{},
487 /*recursive=*/true, &dst_dir_files));
488
489 EXPECT_THAT(dst_dir_files, UnorderedElementsAreArray(src_dir_files));
490
491 // Clean up
492 ASSERT_TRUE(filesystem.DeleteDirectoryRecursively(src_dir.c_str()));
493 ASSERT_TRUE(filesystem.DeleteDirectoryRecursively(dst_dir.c_str()));
494 }
495
496 } // namespace lib
497 } // namespace icing
498