1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/core/platform/cloud/gcs_file_system.h"
17
18 #include <fstream>
19
20 #include "tensorflow/core/lib/core/errors.h"
21 #include "tensorflow/core/lib/core/status_test_util.h"
22 #include "tensorflow/core/platform/cloud/http_request_fake.h"
23 #include "tensorflow/core/platform/errors.h"
24 #include "tensorflow/core/platform/str_util.h"
25 #include "tensorflow/core/platform/strcat.h"
26 #include "tensorflow/core/platform/test.h"
27
28 // Undef DeleteFile macro defined in wndows.h.
29 #ifdef PLATFORM_WINDOWS
30 #undef DeleteFile
31 #endif
32
33 namespace tensorflow {
34 namespace {
35
36 static GcsFileSystem::TimeoutConfig kTestTimeoutConfig(5, 1, 10, 20, 30);
37 static RetryConfig kTestRetryConfig(0 /* init_delay_time_us */);
38
39 // Default (empty) constraint config
40 static std::unordered_set<string>* kAllowedLocationsDefault =
41 new std::unordered_set<string>();
42 // Constraint config if bucket location constraint is turned on, with no
43 // custom list
44 static std::unordered_set<string>* kAllowedLocationsAuto =
45 new std::unordered_set<string>({"auto"});
46
47 class FakeAuthProvider : public AuthProvider {
48 public:
GetToken(string * token)49 Status GetToken(string* token) override {
50 *token = "fake_token";
51 return OkStatus();
52 }
53 };
54
55 class FakeZoneProvider : public ZoneProvider {
56 public:
GetZone(string * zone)57 Status GetZone(string* zone) override {
58 *zone = "us-east1-b";
59 return OkStatus();
60 }
61 };
62
TEST(GcsFileSystemTest,NewRandomAccessFile_NoBlockCache)63 TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) {
64 std::vector<HttpRequest*> requests(
65 {new FakeHttpRequest(
66 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
67 "Auth Token: fake_token\n"
68 "Range: 0-5\n"
69 "Timeouts: 5 1 20\n",
70 "012345"),
71 new FakeHttpRequest(
72 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
73 "Auth Token: fake_token\n"
74 "Range: 6-11\n"
75 "Timeouts: 5 1 20\n",
76 "6789")});
77 GcsFileSystem fs(
78 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
79 std::unique_ptr<HttpRequest::Factory>(
80 new FakeHttpRequestFactory(&requests)),
81 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
82 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
83 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
84 0 /* matching paths cache max entries */, kTestRetryConfig,
85 kTestTimeoutConfig, *kAllowedLocationsDefault,
86 nullptr /* gcs additional header */, false /* compose append */);
87
88 std::unique_ptr<RandomAccessFile> file;
89 TF_EXPECT_OK(
90 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
91
92 StringPiece filename;
93 TF_EXPECT_OK(file->Name(&filename));
94 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
95
96 char scratch[6];
97 StringPiece result;
98
99 // Read the first chunk.
100 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
101 EXPECT_EQ("012345", result);
102
103 // Read the second chunk.
104 EXPECT_TRUE(errors::IsOutOfRange(
105 file->Read(sizeof(scratch), sizeof(scratch), &result, scratch)));
106 EXPECT_EQ("6789", result);
107 }
108
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered)109 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered) {
110 std::vector<HttpRequest*> requests({
111 new FakeHttpRequest(
112 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
113 "Auth Token: fake_token\n"
114 "Range: 0-9\n"
115 "Timeouts: 5 1 20\n",
116 "0123456789"),
117 new FakeHttpRequest(
118 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
119 "Auth Token: fake_token\n"
120 "Range: 10-19\n"
121 "Timeouts: 5 1 20\n",
122 ""),
123 });
124 GcsFileSystem fs(
125 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
126 std::unique_ptr<HttpRequest::Factory>(
127 new FakeHttpRequestFactory(&requests)),
128 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
129 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
130 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
131 0 /* matching paths cache max entries */, kTestRetryConfig,
132 kTestTimeoutConfig, *kAllowedLocationsDefault,
133 nullptr /* gcs additional header */, false /* compose append */);
134
135 std::unique_ptr<RandomAccessFile> file;
136 TF_EXPECT_OK(
137 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
138
139 StringPiece filename;
140 TF_EXPECT_OK(file->Name(&filename));
141 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
142
143 char scratch[6];
144 StringPiece result;
145
146 // Read the first chunk.
147 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
148 EXPECT_EQ("012345", result);
149
150 // Read the second chunk.
151 EXPECT_TRUE(errors::IsOutOfRange(
152 file->Read(sizeof(scratch), sizeof(scratch), &result, scratch)));
153 EXPECT_EQ("6789", result);
154 }
155
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered_Errors)156 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_Errors) {
157 std::vector<HttpRequest*> requests({
158 new FakeHttpRequest(
159 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
160 "Auth Token: fake_token\n"
161 "Range: 0-9\n"
162 "Timeouts: 5 1 20\n",
163 "Server Not", errors::Unavailable("important HTTP error 308"),
164 nullptr, {}, 308),
165 new FakeHttpRequest(
166 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
167 "Auth Token: fake_token\n"
168 "Range: 6-15\n"
169 "Timeouts: 5 1 20\n",
170 "123"),
171 });
172 GcsFileSystem fs(
173 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
174 std::unique_ptr<HttpRequest::Factory>(
175 new FakeHttpRequestFactory(&requests)),
176 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
177 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
178 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
179 0 /* matching paths cache max entries */, kTestRetryConfig,
180 kTestTimeoutConfig, *kAllowedLocationsDefault,
181 nullptr /* gcs additional header */, false /* compose append */);
182
183 std::unique_ptr<RandomAccessFile> file;
184 TF_EXPECT_OK(
185 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
186
187 StringPiece filename;
188 TF_EXPECT_OK(file->Name(&filename));
189 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
190
191 char scratch[6];
192 StringPiece result;
193
194 // Read the first chunk.
195 EXPECT_TRUE(
196 errors::IsUnavailable(file->Read(0, sizeof(scratch), &result, scratch)));
197 EXPECT_EQ("", result);
198
199 // Read the second chunk.
200 EXPECT_TRUE(errors::IsOutOfRange(
201 file->Read(sizeof(scratch), sizeof(scratch), &result, scratch)));
202 EXPECT_EQ("123", result);
203 }
204
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered_ReadAtEOF)205 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_ReadAtEOF) {
206 std::vector<HttpRequest*> requests(
207 {new FakeHttpRequest(
208 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
209 "Auth Token: fake_token\n"
210 "Range: 0-9\n"
211 "Timeouts: 5 1 20\n",
212 "0123456789"),
213 new FakeHttpRequest(
214 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
215 "Auth Token: fake_token\n"
216 "Range: 10-19\n"
217 "Timeouts: 5 1 20\n",
218 "")});
219 GcsFileSystem fs(
220 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
221 std::unique_ptr<HttpRequest::Factory>(
222 new FakeHttpRequestFactory(&requests)),
223 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
224 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
225 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
226 0 /* matching paths cache max entries */, kTestRetryConfig,
227 kTestTimeoutConfig, *kAllowedLocationsDefault,
228 nullptr /* gcs additional header */, false /* compose append */);
229
230 std::unique_ptr<RandomAccessFile> file;
231 TF_EXPECT_OK(
232 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
233
234 StringPiece filename;
235 TF_EXPECT_OK(file->Name(&filename));
236 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
237
238 char scratch[10];
239 StringPiece result;
240
241 // Read the first chunk.
242 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
243 EXPECT_EQ("0123456789", result);
244
245 // Read the second chunk.
246 EXPECT_TRUE(errors::IsOutOfRange(
247 file->Read(sizeof(scratch), sizeof(scratch), &result, scratch)));
248 EXPECT_EQ("", result);
249 }
250
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered_CachedOutOfRange)251 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_CachedOutOfRange) {
252 // In this test, there is only one backend request since we cache the file
253 // size.
254 std::vector<HttpRequest*> requests({new FakeHttpRequest(
255 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
256 "Auth Token: fake_token\n"
257 "Range: 0-9\n"
258 "Timeouts: 5 1 20\n",
259 "012345678")});
260 GcsFileSystem fs(
261 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
262 std::unique_ptr<HttpRequest::Factory>(
263 new FakeHttpRequestFactory(&requests)),
264 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
265 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
266 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
267 0 /* matching paths cache max entries */, kTestRetryConfig,
268 kTestTimeoutConfig, *kAllowedLocationsDefault,
269 nullptr /* gcs additional header */, false /* compose append */);
270
271 std::unique_ptr<RandomAccessFile> file;
272 TF_EXPECT_OK(
273 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
274
275 StringPiece filename;
276 TF_EXPECT_OK(file->Name(&filename));
277 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
278
279 char scratch[5];
280 StringPiece result;
281
282 // Read the first chunk. Even though the backend response is out-of-range,
283 // we should get a OK status since we're just reading the first 5 bytes.
284 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
285 EXPECT_EQ("01234", result);
286
287 TF_EXPECT_OK(file->Read(4, sizeof(scratch), &result, scratch));
288 EXPECT_EQ("45678", result);
289
290 // Return the cached error once the user starts reading out of range.
291 EXPECT_TRUE(
292 errors::IsOutOfRange(file->Read(5, sizeof(scratch), &result, scratch)));
293 EXPECT_EQ("5678", result);
294 }
295
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered_CachedNotSequential)296 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_CachedNotSequential) {
297 // In this test, the second read is seeking backwards, so it should trigger
298 // a backend request.
299 std::vector<HttpRequest*> requests(
300 {new FakeHttpRequest(
301 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
302 "Auth Token: fake_token\n"
303 "Range: 1-10\n"
304 "Timeouts: 5 1 20\n",
305 "12345678"),
306 new FakeHttpRequest(
307 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
308 "Auth Token: fake_token\n"
309 "Range: 0-9\n"
310 "Timeouts: 5 1 20\n",
311 "012345678")});
312 GcsFileSystem fs(
313 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
314 std::unique_ptr<HttpRequest::Factory>(
315 new FakeHttpRequestFactory(&requests)),
316 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
317 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
318 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
319 0 /* matching paths cache max entries */, kTestRetryConfig,
320 kTestTimeoutConfig, *kAllowedLocationsDefault,
321 nullptr /* gcs additional header */, false /* compose append */);
322
323 std::unique_ptr<RandomAccessFile> file;
324 TF_EXPECT_OK(
325 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
326
327 StringPiece filename;
328 TF_EXPECT_OK(file->Name(&filename));
329 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
330
331 char scratch[5];
332 StringPiece result;
333
334 TF_EXPECT_OK(file->Read(1, sizeof(scratch), &result, scratch));
335 EXPECT_EQ("12345", result);
336 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
337 EXPECT_EQ("01234", result);
338 }
339
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered_Growing)340 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_Growing) {
341 std::vector<HttpRequest*> requests(
342 {new FakeHttpRequest(
343 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
344 "Auth Token: fake_token\n"
345 "Range: 0-9\n"
346 "Timeouts: 5 1 20\n",
347 "012345678"),
348 new FakeHttpRequest(
349 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
350 "Auth Token: fake_token\n"
351 "Range: 9-18\n"
352 "Timeouts: 5 1 20\n",
353 "9")});
354 GcsFileSystem fs(
355 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
356 std::unique_ptr<HttpRequest::Factory>(
357 new FakeHttpRequestFactory(&requests)),
358 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
359 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
360 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
361 0 /* matching paths cache max entries */, kTestRetryConfig,
362 kTestTimeoutConfig, *kAllowedLocationsDefault,
363 nullptr /* gcs additional header */, false /* compose append */);
364
365 std::unique_ptr<RandomAccessFile> file;
366 TF_EXPECT_OK(
367 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
368
369 StringPiece filename;
370 TF_EXPECT_OK(file->Name(&filename));
371 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
372
373 char scratch[10];
374 StringPiece result;
375
376 // Read the first chunk. Since the first read is out-of-range,
377 // we don't cache the out-of-range flag and each subsequent read triggers a
378 // backend call.
379 EXPECT_TRUE(
380 errors::IsOutOfRange(file->Read(0, sizeof(scratch), &result, scratch)));
381 EXPECT_EQ("012345678", result);
382
383 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
384 EXPECT_EQ("0123456789", result);
385 }
386
TEST(GcsFileSystemTest,NewRandomAccessFile_Buffered_ReadBackwards)387 TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_ReadBackwards) {
388 // Go backwards in the file. It should trigger a new read.
389 std::vector<HttpRequest*> requests(
390 {new FakeHttpRequest(
391 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
392 "Auth Token: fake_token\n"
393 "Range: 5-14\n"
394 "Timeouts: 5 1 20\n",
395 "56789"),
396 new FakeHttpRequest(
397 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
398 "Auth Token: fake_token\n"
399 "Range: 0-9\n"
400 "Timeouts: 5 1 20\n",
401 "0123456789")});
402 GcsFileSystem fs(
403 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
404 std::unique_ptr<HttpRequest::Factory>(
405 new FakeHttpRequestFactory(&requests)),
406 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 10 /* block size */,
407 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
408 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
409 0 /* matching paths cache max entries */, kTestRetryConfig,
410 kTestTimeoutConfig, *kAllowedLocationsDefault,
411 nullptr /* gcs additional header */, false /* compose append */);
412
413 std::unique_ptr<RandomAccessFile> file;
414 TF_EXPECT_OK(
415 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
416
417 StringPiece filename;
418 TF_EXPECT_OK(file->Name(&filename));
419 EXPECT_EQ(filename, "gs://bucket/random_access.txt");
420
421 char scratch[10];
422 StringPiece result;
423
424 // Read the first chunk.
425 EXPECT_TRUE(
426 errors::IsOutOfRange(file->Read(5, sizeof(scratch), &result, scratch)));
427 EXPECT_EQ("56789", result);
428
429 // Go back and read from the beginning of the file.
430 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
431 EXPECT_EQ("0123456789", result);
432 }
433
TEST(GcsFileSystemTest,NewRandomAccessFile_WithLocationConstraintInSameLocation)434 TEST(GcsFileSystemTest,
435 NewRandomAccessFile_WithLocationConstraintInSameLocation) {
436 std::vector<HttpRequest*> requests({new FakeHttpRequest(
437 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
438 "Auth Token: fake_token\n"
439 "Timeouts: 5 1 10\n",
440 R"(
441 {
442 "location":"US-EAST1"
443 })")});
444
445 GcsFileSystem fs(
446 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
447 std::unique_ptr<HttpRequest::Factory>(
448 new FakeHttpRequestFactory(&requests)),
449 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
450 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
451 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
452 0 /* matching paths cache max entries */, kTestRetryConfig,
453 kTestTimeoutConfig, *kAllowedLocationsAuto,
454 nullptr /* gcs additional header */, false /* compose append */);
455
456 std::unique_ptr<RandomAccessFile> file;
457 TF_EXPECT_OK(
458 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
459 }
460
TEST(GcsFileSystemTest,NewRandomAccessFile_WithLocationConstraintCaching)461 TEST(GcsFileSystemTest, NewRandomAccessFile_WithLocationConstraintCaching) {
462 std::vector<HttpRequest*> requests(
463 {new FakeHttpRequest(
464 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
465 "Auth Token: fake_token\n"
466 "Timeouts: 5 1 10\n",
467 R"(
468 {
469 "location":"US-EAST1"
470 })"),
471 new FakeHttpRequest(
472 "Uri: https://www.googleapis.com/storage/v1/b/anotherbucket\n"
473 "Auth Token: fake_token\n"
474 "Timeouts: 5 1 10\n",
475 R"(
476 {
477 "location":"US-EAST1"
478 })"),
479 new FakeHttpRequest(
480 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
481 "Auth Token: fake_token\n"
482 "Timeouts: 5 1 10\n",
483 R"(
484 {
485 "location":"US-EAST1"
486 })")});
487
488 GcsFileSystem fs(
489 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
490 std::unique_ptr<HttpRequest::Factory>(
491 new FakeHttpRequestFactory(&requests)),
492 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
493 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
494 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
495 0 /* matching paths cache max entries */, kTestRetryConfig,
496 kTestTimeoutConfig, *kAllowedLocationsAuto,
497 nullptr /* gcs additional header */, false /* compose append */);
498
499 std::unique_ptr<RandomAccessFile> file;
500
501 string bucket = "gs://bucket/random_access.txt";
502 string another_bucket = "gs://anotherbucket/random_access.txt";
503 // Multiple calls should only cause one request to the location API.
504 TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, nullptr, &file));
505 TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, nullptr, &file));
506
507 // A new bucket should have one cache miss
508 TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, nullptr, &file));
509 // And then future calls to both should be cached
510 TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, nullptr, &file));
511 TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, nullptr, &file));
512
513 // Trigger a flush, should then require one more call
514 fs.FlushCaches(nullptr);
515 TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, nullptr, &file));
516 }
517
TEST(GcsFileSystemTest,NewRandomAccessFile_WithLocationConstraintInDifferentLocation)518 TEST(GcsFileSystemTest,
519 NewRandomAccessFile_WithLocationConstraintInDifferentLocation) {
520 std::vector<HttpRequest*> requests({new FakeHttpRequest(
521 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
522 "Auth Token: fake_token\n"
523 "Timeouts: 5 1 10\n",
524 R"(
525 {
526 "location":"BARFOO"
527 })")});
528
529 GcsFileSystem fs(
530 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
531 std::unique_ptr<HttpRequest::Factory>(
532 new FakeHttpRequestFactory(&requests)),
533 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
534 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
535 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
536 0 /* matching paths cache max entries */, kTestRetryConfig,
537 kTestTimeoutConfig, *kAllowedLocationsAuto,
538 nullptr /* gcs additional header */, false /* compose append */);
539
540 std::unique_ptr<RandomAccessFile> file;
541 EXPECT_EQ(
542 tensorflow::errors::FailedPrecondition(
543 "Bucket 'bucket' is in 'barfoo' location, allowed locations "
544 "are: (us-east1)."),
545 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
546 }
547
TEST(GcsFileSystemTest,NewRandomAccessFile_NoBlockCache_DifferentN)548 TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_DifferentN) {
549 std::vector<HttpRequest*> requests(
550 {new FakeHttpRequest(
551 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
552 "Auth Token: fake_token\n"
553 "Range: 0-2\n"
554 "Timeouts: 5 1 20\n",
555 "012"),
556 new FakeHttpRequest(
557 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
558 "Auth Token: fake_token\n"
559 "Range: 3-12\n"
560 "Timeouts: 5 1 20\n",
561 "3456789")});
562 GcsFileSystem fs(
563 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
564 std::unique_ptr<HttpRequest::Factory>(
565 new FakeHttpRequestFactory(&requests)),
566 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
567 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
568 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
569 0 /* matching paths cache max entries */, kTestRetryConfig,
570 kTestTimeoutConfig, *kAllowedLocationsDefault,
571 nullptr /* gcs additional header */, false /* compose append */);
572
573 std::unique_ptr<RandomAccessFile> file;
574 TF_EXPECT_OK(
575 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
576
577 char small_scratch[3];
578 StringPiece result;
579
580 // Read the first chunk.
581 TF_EXPECT_OK(file->Read(0, sizeof(small_scratch), &result, small_scratch));
582 EXPECT_EQ("012", result);
583
584 // Read the second chunk that is larger. Requires allocation of new buffer.
585 char large_scratch[10];
586
587 EXPECT_TRUE(errors::IsOutOfRange(file->Read(
588 sizeof(small_scratch), sizeof(large_scratch), &result, large_scratch)));
589 EXPECT_EQ("3456789", result);
590 }
591
TEST(GcsFileSystemTest,NewRandomAccessFile_WithBlockCache)592 TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache) {
593 // Our underlying file in this test is a 15 byte file with contents
594 // "0123456789abcde".
595 std::vector<HttpRequest*> requests(
596 {new FakeHttpRequest(
597 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
598 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
599 "Auth Token: fake_token\n"
600 "Timeouts: 5 1 10\n",
601 strings::StrCat("{\"size\": \"15\",\"generation\": \"1\","
602 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
603 new FakeHttpRequest(
604 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
605 "Auth Token: fake_token\n"
606 "Range: 0-8\n"
607 "Timeouts: 5 1 20\n",
608 "012345678"),
609 new FakeHttpRequest(
610 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
611 "Auth Token: fake_token\n"
612 "Range: 9-17\n"
613 "Timeouts: 5 1 20\n",
614 "9abcde"),
615 new FakeHttpRequest(
616 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
617 "Auth Token: fake_token\n"
618 "Range: 18-26\n"
619 "Timeouts: 5 1 20\n",
620 "")});
621 GcsFileSystem fs(
622 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
623 std::unique_ptr<HttpRequest::Factory>(
624 new FakeHttpRequestFactory(&requests)),
625 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 9 /* block size */,
626 18 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
627 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
628 0 /* matching paths cache max entries */, kTestRetryConfig,
629 kTestTimeoutConfig, *kAllowedLocationsDefault,
630 nullptr /* gcs additional header */, false /* compose append */);
631
632 char scratch[100];
633 StringPiece result;
634 {
635 // We are instantiating this in an enclosed scope to make sure after the
636 // unique ptr goes out of scope, we can still access result.
637 std::unique_ptr<RandomAccessFile> file;
638 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt",
639 nullptr, &file));
640
641 // Read the first chunk. The cache will be populated with the first block of
642 // 9 bytes.
643 scratch[5] = 'x';
644 TF_EXPECT_OK(file->Read(0, 4, &result, scratch));
645 EXPECT_EQ("0123", result);
646 EXPECT_EQ(scratch[5], 'x'); // Make sure we only copied 4 bytes.
647
648 // The second chunk will be fully loaded from the cache, no requests are
649 // made.
650 TF_EXPECT_OK(file->Read(4, 4, &result, scratch));
651 EXPECT_EQ("4567", result);
652
653 // The chunk is only partially cached -- the request will be made to fetch
654 // the next block. 9 bytes will be requested, starting at offset 9.
655 TF_EXPECT_OK(file->Read(6, 5, &result, scratch));
656 EXPECT_EQ("6789a", result);
657
658 // The range can only be partially satisfied, as the second block contains
659 // only 6 bytes for a total of 9 + 6 = 15 bytes in the file.
660 EXPECT_TRUE(errors::IsOutOfRange(file->Read(6, 10, &result, scratch)));
661 EXPECT_EQ("6789abcde", result);
662
663 // The range cannot be satisfied, and the requested offset is past the end
664 // of the cache. A new request will be made to read 9 bytes starting at
665 // offset 18. This request will return an empty response, and there will not
666 // be another request.
667 EXPECT_TRUE(errors::IsOutOfRange(file->Read(20, 10, &result, scratch)));
668 EXPECT_TRUE(result.empty());
669
670 // The beginning of the file should still be in the LRU cache. There should
671 // not be another request. The buffer size is still 15.
672 TF_EXPECT_OK(file->Read(0, 4, &result, scratch));
673 }
674
675 EXPECT_EQ("0123", result);
676 }
677
TEST(GcsFileSystemTest,NewRandomAccessFile_WithBlockCache_Flush)678 TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_Flush) {
679 // Our underlying file in this test is a 15 byte file with contents
680 // "0123456789abcde".
681 std::vector<HttpRequest*> requests(
682 {new FakeHttpRequest(
683 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
684 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
685 "Auth Token: fake_token\n"
686 "Timeouts: 5 1 10\n",
687 strings::StrCat("{\"size\": \"15\",\"generation\": \"1\","
688 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
689 new FakeHttpRequest(
690 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
691 "Auth Token: fake_token\n"
692 "Range: 0-8\n"
693 "Timeouts: 5 1 20\n",
694 "012345678"),
695 new FakeHttpRequest(
696 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
697 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
698 "Auth Token: fake_token\n"
699 "Timeouts: 5 1 10\n",
700 strings::StrCat("{\"size\": \"15\",\"generation\": \"1\","
701 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
702 new FakeHttpRequest(
703 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
704 "Auth Token: fake_token\n"
705 "Range: 0-8\n"
706 "Timeouts: 5 1 20\n",
707 "012345678")});
708 GcsFileSystem fs(
709 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
710 std::unique_ptr<HttpRequest::Factory>(
711 new FakeHttpRequestFactory(&requests)),
712 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 9 /* block size */,
713 18 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
714 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
715 0 /* matching paths cache max entries */, kTestRetryConfig,
716 kTestTimeoutConfig, *kAllowedLocationsDefault,
717 nullptr /* gcs additional header */, false /* compose append */);
718
719 char scratch[100];
720 StringPiece result;
721 std::unique_ptr<RandomAccessFile> file;
722 TF_EXPECT_OK(
723 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
724 // Read the first chunk. The cache will be populated with the first block of
725 // 9 bytes.
726 scratch[5] = 'x';
727 TF_EXPECT_OK(file->Read(0, 4, &result, scratch));
728 EXPECT_EQ("0123", result);
729 EXPECT_EQ(scratch[5], 'x'); // Make sure we only copied 4 bytes.
730 // Flush caches and read the second chunk. This will be a cache miss, and
731 // the same block will be fetched again.
732 fs.FlushCaches(nullptr);
733 TF_EXPECT_OK(file->Read(4, 4, &result, scratch));
734 EXPECT_EQ("4567", result);
735 }
736
TEST(GcsFileSystemTest,NewRandomAccessFile_WithBlockCache_MaxStaleness)737 TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) {
738 // Our underlying file in this test is a 16 byte file with contents
739 // "0123456789abcdef".
740 std::vector<HttpRequest*> requests(
741 {new FakeHttpRequest(
742 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
743 "object?fields=size%2Cgeneration%2Cupdated\n"
744 "Auth Token: fake_token\n"
745 "Timeouts: 5 1 10\n",
746 strings::StrCat("{\"size\": \"16\",\"generation\": \"1\","
747 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
748 new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
749 "Auth Token: fake_token\n"
750 "Range: 0-7\n"
751 "Timeouts: 5 1 20\n",
752 "01234567"),
753 new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
754 "Auth Token: fake_token\n"
755 "Range: 8-15\n"
756 "Timeouts: 5 1 20\n",
757 "89abcdef")});
758 GcsFileSystem fs(
759 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
760 std::unique_ptr<HttpRequest::Factory>(
761 new FakeHttpRequestFactory(&requests)),
762 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 8 /* block size */,
763 16 /* max bytes */, 3600 /* max staleness */,
764 3600 /* stat cache max age */, 0 /* stat cache max entries */,
765 0 /* matching paths cache max age */,
766 0 /* matching paths cache max entries */, kTestRetryConfig,
767 kTestTimeoutConfig, *kAllowedLocationsDefault,
768 nullptr /* gcs additional header */, false /* compose append */);
769 char scratch[100];
770 StringPiece result;
771 // There should only be two HTTP requests issued to GCS even though we iterate
772 // this loop 10 times. This shows that the underlying FileBlockCache persists
773 // across file close/open boundaries.
774 for (int i = 0; i < 10; i++) {
775 // Create two files. Since these files have the same name and the max
776 // staleness of the filesystem is > 0, they will share the same blocks.
777 std::unique_ptr<RandomAccessFile> file1;
778 std::unique_ptr<RandomAccessFile> file2;
779 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", nullptr, &file1));
780 TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", nullptr, &file2));
781 // Reading the first block from file1 should load it once.
782 TF_EXPECT_OK(file1->Read(0, 8, &result, scratch));
783 EXPECT_EQ("01234567", result);
784 // Reading the first block from file2 should not trigger a request to load
785 // the first block again, because the FileBlockCache shared by file1 and
786 // file2 already has the first block.
787 TF_EXPECT_OK(file2->Read(0, 8, &result, scratch));
788 EXPECT_EQ("01234567", result);
789 // Reading the second block from file2 should load it once.
790 TF_EXPECT_OK(file2->Read(8, 8, &result, scratch));
791 EXPECT_EQ("89abcdef", result);
792 // Reading the second block from file1 should not trigger a request to load
793 // the second block again, because the FileBlockCache shared by file1 and
794 // file2 already has the second block.
795 TF_EXPECT_OK(file1->Read(8, 8, &result, scratch));
796 EXPECT_EQ("89abcdef", result);
797 }
798 }
799
TEST(GcsFileSystemTest,NewRandomAccessFile_WithBlockCache_FileSignatureChanges)800 TEST(GcsFileSystemTest,
801 NewRandomAccessFile_WithBlockCache_FileSignatureChanges) {
802 std::vector<HttpRequest*> requests(
803 {new FakeHttpRequest(
804 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
805 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
806 "Auth Token: fake_token\n"
807 "Timeouts: 5 1 10\n",
808 strings::StrCat("{\"size\": \"5\",\"generation\": \"1\","
809 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
810 new FakeHttpRequest(
811 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
812 "Auth Token: fake_token\n"
813 "Range: 0-8\n"
814 "Timeouts: 5 1 20\n",
815 "01234"),
816 new FakeHttpRequest(
817 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
818 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
819 "Auth Token: fake_token\n"
820 "Timeouts: 5 1 10\n",
821 strings::StrCat("{\"size\": \"5\",\"generation\": \"2\","
822 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
823 new FakeHttpRequest(
824 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
825 "Auth Token: fake_token\n"
826 "Range: 0-8\n"
827 "Timeouts: 5 1 20\n",
828 "43210")});
829 GcsFileSystem fs(
830 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
831 std::unique_ptr<HttpRequest::Factory>(
832 new FakeHttpRequestFactory(&requests)),
833 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 9 /* block size */,
834 18 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
835 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
836 0 /* matching paths cache max entries */, kTestRetryConfig,
837 kTestTimeoutConfig, *kAllowedLocationsDefault,
838 nullptr /* gcs additional header */, false /* compose append */);
839
840 std::unique_ptr<RandomAccessFile> file;
841 TF_EXPECT_OK(
842 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
843
844 char scratch[5];
845 StringPiece result;
846
847 // First read.
848 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
849 EXPECT_EQ("01234", result);
850
851 // Second read. File signatures are different.
852 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
853 EXPECT_EQ("43210", result);
854 }
855
TEST(GcsFileSystemTest,NewRandomAccessFile_NoObjectName)856 TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) {
857 std::vector<HttpRequest*> requests;
858 GcsFileSystem fs(
859 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
860 std::unique_ptr<HttpRequest::Factory>(
861 new FakeHttpRequestFactory(&requests)),
862 std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
863 0 /* read ahead bytes */, 0 /* max bytes */, 0 /* max staleness */,
864 0 /* stat cache max age */, 0 /* stat cache max entries */,
865 0 /* matching paths cache max age */,
866 0 /* matching paths cache max entries */, kTestRetryConfig,
867 kTestTimeoutConfig, *kAllowedLocationsDefault,
868 nullptr /* gcs additional header */, false /* compose append */);
869
870 std::unique_ptr<RandomAccessFile> file;
871 EXPECT_TRUE(errors::IsInvalidArgument(
872 fs.NewRandomAccessFile("gs://bucket/", nullptr, &file)));
873 }
874
TEST(GcsFileSystemTest,NewRandomAccessFile_InconsistentRead)875 TEST(GcsFileSystemTest, NewRandomAccessFile_InconsistentRead) {
876 std::vector<HttpRequest*> requests(
877 {new FakeHttpRequest(
878 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
879 "random_access.txt?fields=size%2Cgeneration%2Cupdated\n"
880 "Auth Token: fake_token\n"
881 "Timeouts: 5 1 10\n",
882 strings::StrCat("{\"size\": \"6\",\"generation\": \"1\","
883 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
884 new FakeHttpRequest(
885 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
886 "Auth Token: fake_token\n"
887 "Range: 0-5\n"
888 "Timeouts: 5 1 20\n",
889 "012")});
890
891 // Set stat_cache_max_age to 1000s so that StatCache could work.
892 GcsFileSystem fs(
893 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
894 std::unique_ptr<HttpRequest::Factory>(
895 new FakeHttpRequestFactory(&requests)),
896 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
897 0 /* max bytes */, 0 /* max staleness */, 1e3 /* stat cache max age */,
898 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
899 0 /* matching paths cache max entries */, kTestRetryConfig,
900 kTestTimeoutConfig, *kAllowedLocationsDefault,
901 nullptr /* gcs additional header */, false /* compose append */);
902
903 // Stat the file first so that the file stats are cached.
904 FileStatistics stat;
905 TF_ASSERT_OK(fs.Stat("gs://bucket/random_access.txt", nullptr, &stat));
906
907 std::unique_ptr<RandomAccessFile> file;
908 TF_ASSERT_OK(
909 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
910
911 char scratch[6];
912 StringPiece result;
913
914 EXPECT_TRUE(
915 errors::IsInternal(file->Read(0, sizeof(scratch), &result, scratch)));
916 }
917
TEST(GcsFileSystemTest,NewWritableFile)918 TEST(GcsFileSystemTest, NewWritableFile) {
919 std::vector<HttpRequest*> requests(
920 {new FakeHttpRequest(
921 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
922 "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
923 "Auth Token: fake_token\n"
924 "Timeouts: 5 1 10\n",
925 strings::StrCat("{\"size\": \"16\",\"generation\": \"1\","
926 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
927 new FakeHttpRequest(
928 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
929 "Auth Token: fake_token\n"
930 "Range: 0-7\n"
931 "Timeouts: 5 1 20\n",
932 "01234567"),
933 new FakeHttpRequest(
934 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
935 "uploadType=resumable&name=path%2Fwriteable\n"
936 "Auth Token: fake_token\n"
937 "Header X-Upload-Content-Length: 17\n"
938 "Post: yes\n"
939 "Timeouts: 5 1 10\n",
940 "", {{"Location", "https://custom/upload/location"}}),
941 new FakeHttpRequest("Uri: https://custom/upload/location\n"
942 "Auth Token: fake_token\n"
943 "Header Content-Range: bytes 0-16/17\n"
944 "Timeouts: 5 1 30\n"
945 "Put body: content1,content2\n",
946 ""),
947 new FakeHttpRequest(
948 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
949 "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
950 "Auth Token: fake_token\n"
951 "Timeouts: 5 1 10\n",
952 strings::StrCat("{\"size\": \"33\",\"generation\": \"2\","
953 "\"updated\": \"2016-04-29T23:15:34.896Z\"}")),
954 new FakeHttpRequest(
955 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
956 "Auth Token: fake_token\n"
957 "Range: 0-7\n"
958 "Timeouts: 5 1 20\n",
959 "01234567")});
960 GcsFileSystem fs(
961 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
962 std::unique_ptr<HttpRequest::Factory>(
963 new FakeHttpRequestFactory(&requests)),
964 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 8 /* block size */,
965 8 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
966 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
967 0 /* matching paths cache max entries */, kTestRetryConfig,
968 kTestTimeoutConfig, *kAllowedLocationsDefault,
969 nullptr /* gcs additional header */, false /* compose append */);
970
971 // Read from the file first, to fill the block cache.
972 std::unique_ptr<RandomAccessFile> rfile;
973 TF_EXPECT_OK(
974 fs.NewRandomAccessFile("gs://bucket/path/writeable", nullptr, &rfile));
975 char scratch[100];
976 StringPiece result;
977 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
978 EXPECT_EQ("0123", result);
979 // Open the writable file.
980 std::unique_ptr<WritableFile> wfile;
981 TF_EXPECT_OK(
982 fs.NewWritableFile("gs://bucket/path/writeable", nullptr, &wfile));
983 TF_EXPECT_OK(wfile->Append("content1,"));
984 int64_t pos;
985 TF_EXPECT_OK(wfile->Tell(&pos));
986 EXPECT_EQ(9, pos);
987 TF_EXPECT_OK(wfile->Append("content2"));
988 TF_EXPECT_OK(wfile->Flush());
989 // Re-reading the file should trigger another HTTP request to GCS.
990 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
991 EXPECT_EQ("0123", result);
992 // The calls to flush, sync, and close below should not cause uploads because
993 // the file is not dirty.
994 TF_EXPECT_OK(wfile->Flush());
995 TF_EXPECT_OK(wfile->Sync());
996 TF_EXPECT_OK(wfile->Close());
997 }
998
TEST(GcsFileSystemTest,NewWritableFile_ResumeUploadSucceeds)999 TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) {
1000 std::vector<HttpRequest*> requests(
1001 {new FakeHttpRequest(
1002 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1003 "uploadType=resumable&name=path%2Fwriteable.txt\n"
1004 "Auth Token: fake_token\n"
1005 "Header X-Upload-Content-Length: 17\n"
1006 "Post: yes\n"
1007 "Timeouts: 5 1 10\n",
1008 "", {{"Location", "https://custom/upload/location"}}),
1009 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1010 "Auth Token: fake_token\n"
1011 "Header Content-Range: bytes 0-16/17\n"
1012 "Timeouts: 5 1 30\n"
1013 "Put body: content1,content2\n",
1014 "", errors::Unavailable("503"), 503),
1015 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1016 "Auth Token: fake_token\n"
1017 "Timeouts: 5 1 10\n"
1018 "Header Content-Range: bytes */17\n"
1019 "Put: yes\n",
1020 "", errors::Unavailable("308"), nullptr,
1021 {{"Range", "0-10"}}, 308),
1022 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1023 "Auth Token: fake_token\n"
1024 "Header Content-Range: bytes 11-16/17\n"
1025 "Timeouts: 5 1 30\n"
1026 "Put body: ntent2\n",
1027 "", errors::Unavailable("503"), 503),
1028 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1029 "Auth Token: fake_token\n"
1030 "Timeouts: 5 1 10\n"
1031 "Header Content-Range: bytes */17\n"
1032 "Put: yes\n",
1033 "", errors::Unavailable("308"), nullptr,
1034 {{"Range", "bytes=0-12"}}, 308),
1035 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1036 "Auth Token: fake_token\n"
1037 "Header Content-Range: bytes 13-16/17\n"
1038 "Timeouts: 5 1 30\n"
1039 "Put body: ent2\n",
1040 "", errors::Unavailable("308"), 308),
1041 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1042 "Auth Token: fake_token\n"
1043 "Timeouts: 5 1 10\n"
1044 "Header Content-Range: bytes */17\n"
1045 "Put: yes\n",
1046 "", errors::Unavailable("308"), nullptr,
1047 {{"Range", "bytes=0-14"}}, 308),
1048 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1049 "Auth Token: fake_token\n"
1050 "Header Content-Range: bytes 15-16/17\n"
1051 "Timeouts: 5 1 30\n"
1052 "Put body: t2\n",
1053 "")});
1054 GcsFileSystem fs(
1055 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1056 std::unique_ptr<HttpRequest::Factory>(
1057 new FakeHttpRequestFactory(&requests)),
1058 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1059 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1060 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1061 0 /* matching paths cache max entries */, kTestRetryConfig,
1062 kTestTimeoutConfig, *kAllowedLocationsDefault,
1063 nullptr /* gcs additional header */, false /* compose append */);
1064
1065 std::unique_ptr<WritableFile> file;
1066 TF_EXPECT_OK(
1067 fs.NewWritableFile("gs://bucket/path/writeable.txt", nullptr, &file));
1068
1069 TF_EXPECT_OK(file->Append("content1,"));
1070 TF_EXPECT_OK(file->Append("content2"));
1071 TF_EXPECT_OK(file->Close());
1072 }
1073
TEST(GcsFileSystemTest,NewWritableFile_ResumeUploadSucceedsOnGetStatus)1074 TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) {
1075 // This test also verifies that a file's blocks are purged from the cache when
1076 // the file is written, even when the write takes the "succeeds on get status"
1077 // path.
1078 std::vector<HttpRequest*> requests(
1079 {new FakeHttpRequest(
1080 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1081 "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
1082 "Auth Token: fake_token\n"
1083 "Timeouts: 5 1 10\n",
1084 strings::StrCat("{\"size\": \"16\",\"generation\": \"1\","
1085 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
1086 new FakeHttpRequest(
1087 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
1088 "Auth Token: fake_token\n"
1089 "Range: 0-7\n"
1090 "Timeouts: 5 1 20\n",
1091 "01234567"),
1092 new FakeHttpRequest(
1093 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1094 "uploadType=resumable&name=path%2Fwriteable\n"
1095 "Auth Token: fake_token\n"
1096 "Header X-Upload-Content-Length: 17\n"
1097 "Post: yes\n"
1098 "Timeouts: 5 1 10\n",
1099 "", {{"Location", "https://custom/upload/location"}}),
1100 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1101 "Auth Token: fake_token\n"
1102 "Header Content-Range: bytes 0-16/17\n"
1103 "Timeouts: 5 1 30\n"
1104 "Put body: content1,content2\n",
1105 "", errors::Unavailable("503"), 503),
1106 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1107 "Auth Token: fake_token\n"
1108 "Timeouts: 5 1 10\n"
1109 "Header Content-Range: bytes */17\n"
1110 "Put: yes\n",
1111 "", OkStatus(), nullptr, {}, 201),
1112 new FakeHttpRequest(
1113 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1114 "path%2Fwriteable?fields=size%2Cgeneration%2Cupdated\n"
1115 "Auth Token: fake_token\n"
1116 "Timeouts: 5 1 10\n",
1117 strings::StrCat("{\"size\": \"33\",\"generation\": \"2\","
1118 "\"updated\": \"2016-04-29T23:19:24.896Z\"}")),
1119 new FakeHttpRequest(
1120 "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
1121 "Auth Token: fake_token\n"
1122 "Range: 0-7\n"
1123 "Timeouts: 5 1 20\n",
1124 "01234567")});
1125 GcsFileSystem fs(
1126 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1127 std::unique_ptr<HttpRequest::Factory>(
1128 new FakeHttpRequestFactory(&requests)),
1129 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 8 /* block size */,
1130 8 /* max bytes */, 3600 /* max staleness */,
1131 3600 /* stat cache max age */, 0 /* stat cache max entries */,
1132 0 /* matching paths cache max age */,
1133 0 /* matching paths cache max entries */, kTestRetryConfig,
1134 kTestTimeoutConfig, *kAllowedLocationsDefault,
1135 nullptr /* gcs additional header */, false /* compose append */);
1136 // Pull the file's first block into the cache. This will trigger the first
1137 // HTTP request to GCS.
1138 std::unique_ptr<RandomAccessFile> rfile;
1139 TF_EXPECT_OK(
1140 fs.NewRandomAccessFile("gs://bucket/path/writeable", nullptr, &rfile));
1141 char scratch[100];
1142 StringPiece result;
1143 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
1144 EXPECT_EQ("0123", result);
1145 // Now write to the same file. Once the write succeeds, the cached block will
1146 // be flushed.
1147 std::unique_ptr<WritableFile> wfile;
1148 TF_EXPECT_OK(
1149 fs.NewWritableFile("gs://bucket/path/writeable", nullptr, &wfile));
1150 TF_EXPECT_OK(wfile->Append("content1,"));
1151 TF_EXPECT_OK(wfile->Append("content2"));
1152 // Appending doesn't invalidate the read cache - only flushing does. This read
1153 // will not trigger an HTTP request to GCS.
1154 TF_EXPECT_OK(rfile->Read(4, 4, &result, scratch));
1155 EXPECT_EQ("4567", result);
1156 // Closing the file triggers HTTP requests to GCS and invalidates the read
1157 // cache for the file.
1158 TF_EXPECT_OK(wfile->Close());
1159 // Reading the first block of the file goes to GCS again.
1160 TF_EXPECT_OK(rfile->Read(0, 8, &result, scratch));
1161 EXPECT_EQ("01234567", result);
1162 }
1163
TEST(GcsFileSystemTest,NewWritableFile_ResumeUploadAllAttemptsFail)1164 TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) {
1165 std::vector<HttpRequest*> requests(
1166 {new FakeHttpRequest(
1167 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1168 "uploadType=resumable&name=path%2Fwriteable.txt\n"
1169 "Auth Token: fake_token\n"
1170 "Header X-Upload-Content-Length: 17\n"
1171 "Post: yes\n"
1172 "Timeouts: 5 1 10\n",
1173 "", {{"Location", "https://custom/upload/location"}}),
1174 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1175 "Auth Token: fake_token\n"
1176 "Header Content-Range: bytes 0-16/17\n"
1177 "Timeouts: 5 1 30\n"
1178 "Put body: content1,content2\n",
1179 "", errors::Unavailable("503"), 503)});
1180 for (int i = 0; i < 10; i++) {
1181 requests.emplace_back(
1182 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1183 "Auth Token: fake_token\n"
1184 "Timeouts: 5 1 10\n"
1185 "Header Content-Range: bytes */17\n"
1186 "Put: yes\n",
1187 "", errors::Unavailable("important HTTP error 308"),
1188 nullptr, {{"Range", "0-10"}}, 308));
1189 requests.emplace_back(new FakeHttpRequest(
1190 "Uri: https://custom/upload/location\n"
1191 "Auth Token: fake_token\n"
1192 "Header Content-Range: bytes 11-16/17\n"
1193 "Timeouts: 5 1 30\n"
1194 "Put body: ntent2\n",
1195 "", errors::Unavailable("important HTTP error 503"), 503));
1196 }
1197 // These calls will be made in the Close() attempt from the destructor.
1198 // Letting the destructor succeed.
1199 requests.emplace_back(new FakeHttpRequest(
1200 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1201 "uploadType=resumable&name=path%2Fwriteable.txt\n"
1202 "Auth Token: fake_token\n"
1203 "Header X-Upload-Content-Length: 17\n"
1204 "Post: yes\n"
1205 "Timeouts: 5 1 10\n",
1206 "", {{"Location", "https://custom/upload/location"}}));
1207 requests.emplace_back(
1208 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1209 "Auth Token: fake_token\n"
1210 "Header Content-Range: bytes 0-16/17\n"
1211 "Timeouts: 5 1 30\n"
1212 "Put body: content1,content2\n",
1213 ""));
1214 GcsFileSystem fs(
1215 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1216 std::unique_ptr<HttpRequest::Factory>(
1217 new FakeHttpRequestFactory(&requests)),
1218 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1219 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1220 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1221 0 /* matching paths cache max entries */,
1222 RetryConfig(2 /* .init_delay_time_us */), kTestTimeoutConfig,
1223 *kAllowedLocationsDefault, nullptr /* gcs additional header */,
1224 false /* compose append */);
1225
1226 std::unique_ptr<WritableFile> file;
1227 TF_EXPECT_OK(
1228 fs.NewWritableFile("gs://bucket/path/writeable.txt", nullptr, &file));
1229
1230 TF_EXPECT_OK(file->Append("content1,"));
1231 TF_EXPECT_OK(file->Append("content2"));
1232 const auto& status = file->Close();
1233 EXPECT_TRUE(errors::IsAborted(status));
1234 EXPECT_TRUE(
1235 absl::StrContains(status.error_message(),
1236 "All 10 retry attempts failed. The last failure: "
1237 "important HTTP error 503"))
1238 << status;
1239 }
1240
TEST(GcsFileSystemTest,NewWritableFile_UploadReturns410)1241 TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) {
1242 std::vector<string> results;
1243 TF_EXPECT_OK(
1244 Env::Default()->GetMatchingPaths("/tmp/tmp_file_tensorflow*", &results));
1245 const int64_t tmp_files_before = results.size();
1246
1247 std::vector<HttpRequest*> requests(
1248 {new FakeHttpRequest(
1249 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1250 "uploadType=resumable&name=path%2Fwriteable.txt\n"
1251 "Auth Token: fake_token\n"
1252 "Header X-Upload-Content-Length: 17\n"
1253 "Post: yes\n"
1254 "Timeouts: 5 1 10\n",
1255 "", {{"Location", "https://custom/upload/location"}}),
1256 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1257 "Auth Token: fake_token\n"
1258 "Header Content-Range: bytes 0-16/17\n"
1259 "Timeouts: 5 1 30\n"
1260 "Put body: content1,content2\n",
1261 "", errors::NotFound("important HTTP error 410"),
1262 410),
1263 // These calls will be made in the Close() attempt from the destructor.
1264 // Letting the destructor succeed.
1265 new FakeHttpRequest(
1266 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1267 "uploadType=resumable&name=path%2Fwriteable.txt\n"
1268 "Auth Token: fake_token\n"
1269 "Header X-Upload-Content-Length: 17\n"
1270 "Post: yes\n"
1271 "Timeouts: 5 1 10\n",
1272 "", {{"Location", "https://custom/upload/location"}}),
1273 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1274 "Auth Token: fake_token\n"
1275 "Header Content-Range: bytes 0-16/17\n"
1276 "Timeouts: 5 1 30\n"
1277 "Put body: content1,content2\n",
1278 "")});
1279 GcsFileSystem fs(
1280 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1281 std::unique_ptr<HttpRequest::Factory>(
1282 new FakeHttpRequestFactory(&requests)),
1283 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1284 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1285 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1286 0 /* matching paths cache max entries */, kTestRetryConfig,
1287 kTestTimeoutConfig, *kAllowedLocationsDefault,
1288 nullptr /* gcs additional header */, false /* compose append */);
1289
1290 {
1291 std::unique_ptr<WritableFile> file;
1292 TF_EXPECT_OK(
1293 fs.NewWritableFile("gs://bucket/path/writeable.txt", nullptr, &file));
1294
1295 TF_EXPECT_OK(file->Append("content1,"));
1296 TF_EXPECT_OK(file->Append("content2"));
1297 const auto& status = file->Close();
1298 EXPECT_TRUE(errors::IsUnavailable(status));
1299 EXPECT_TRUE(
1300 absl::StrContains(status.error_message(),
1301 "Upload to gs://bucket/path/writeable.txt failed, "
1302 "caused by: important HTTP error 410"))
1303 << status;
1304 EXPECT_TRUE(
1305 absl::StrContains(status.error_message(),
1306 "when uploading gs://bucket/path/writeable.txt"))
1307 << status;
1308 }
1309
1310 // Check that no new tempfiles were left over after failure and destruction
1311 // of the file.
1312 results.clear();
1313 TF_EXPECT_OK(
1314 Env::Default()->GetMatchingPaths("/tmp/tmp_file_tensorflow*", &results));
1315 EXPECT_EQ(tmp_files_before, results.size());
1316 }
1317
TEST(GcsFileSystemTest,NewWritableFile_NoObjectName)1318 TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) {
1319 std::vector<HttpRequest*> requests;
1320 GcsFileSystem fs(
1321 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1322 std::unique_ptr<HttpRequest::Factory>(
1323 new FakeHttpRequestFactory(&requests)),
1324 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1325 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1326 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1327 0 /* matching paths cache max entries */, kTestRetryConfig,
1328 kTestTimeoutConfig, *kAllowedLocationsDefault,
1329 nullptr /* gcs additional header */, false /* compose append */);
1330
1331 std::unique_ptr<WritableFile> file;
1332 EXPECT_TRUE(errors::IsInvalidArgument(
1333 fs.NewWritableFile("gs://bucket/", nullptr, &file)));
1334 }
1335
TEST(GcsFileSystemTest,NewAppendableFile)1336 TEST(GcsFileSystemTest, NewAppendableFile) {
1337 std::vector<HttpRequest*> requests(
1338 {new FakeHttpRequest(
1339 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1340 "path%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
1341 "Auth Token: fake_token\n"
1342 "Timeouts: 5 1 10\n",
1343 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
1344 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
1345 new FakeHttpRequest(
1346 "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
1347 "Auth Token: fake_token\n"
1348 "Range: 0-1048575\n"
1349 "Timeouts: 5 1 20\n",
1350 "content1,"),
1351 new FakeHttpRequest(
1352 "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
1353 "Auth Token: fake_token\n"
1354 "Range: 0-31\n"
1355 "Timeouts: 5 1 20\n",
1356 "content1,"),
1357 new FakeHttpRequest(
1358 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
1359 "uploadType=resumable&name=path%2Fappendable\n"
1360 "Auth Token: fake_token\n"
1361 "Header X-Upload-Content-Length: 17\n"
1362 "Post: yes\n"
1363 "Timeouts: 5 1 10\n",
1364 "", {{"Location", "https://custom/upload/location"}}),
1365 new FakeHttpRequest("Uri: https://custom/upload/location\n"
1366 "Auth Token: fake_token\n"
1367 "Header Content-Range: bytes 0-16/17\n"
1368 "Timeouts: 5 1 30\n"
1369 "Put body: content1,content2\n",
1370 ""),
1371 new FakeHttpRequest(
1372 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1373 "path%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
1374 "Auth Token: fake_token\n"
1375 "Timeouts: 5 1 10\n",
1376 strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
1377 "\"updated\": \"2016-04-29T23:25:24.896Z\"}")),
1378 new FakeHttpRequest(
1379 "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
1380 "Auth Token: fake_token\n"
1381 "Range: 0-31\n"
1382 "Timeouts: 5 1 20\n",
1383 "01234567")});
1384 GcsFileSystem fs(
1385 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1386 std::unique_ptr<HttpRequest::Factory>(
1387 new FakeHttpRequestFactory(&requests)),
1388 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 32 /* block size */,
1389 32 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
1390 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1391 0 /* matching paths cache max entries */, kTestRetryConfig,
1392 kTestTimeoutConfig, *kAllowedLocationsDefault,
1393 nullptr /* gcs additional header */, false /* compose append */);
1394
1395 // Create an appendable file. This should read the file from GCS, and pull its
1396 // contents into the block cache.
1397 std::unique_ptr<WritableFile> wfile;
1398 TF_EXPECT_OK(
1399 fs.NewAppendableFile("gs://bucket/path/appendable", nullptr, &wfile));
1400 TF_EXPECT_OK(wfile->Append("content2"));
1401 // Verify that the file contents are in the block cache. This read should not
1402 // trigger an HTTP request to GCS.
1403 std::unique_ptr<RandomAccessFile> rfile;
1404 TF_EXPECT_OK(
1405 fs.NewRandomAccessFile("gs://bucket/path/appendable", nullptr, &rfile));
1406 char scratch[100];
1407 StringPiece result;
1408 TF_EXPECT_OK(rfile->Read(0, 8, &result, scratch));
1409 EXPECT_EQ("content1", result);
1410 // Closing the appendable file will flush its contents to GCS, triggering HTTP
1411 // requests.
1412 TF_EXPECT_OK(wfile->Close());
1413 // Redo the read. The block should be reloaded from GCS, causing one more HTTP
1414 // request to load it.
1415 TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
1416 EXPECT_EQ("0123", result);
1417 }
1418
TEST(GcsFileSystemTest,NewAppendableFile_NoObjectName)1419 TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) {
1420 std::vector<HttpRequest*> requests;
1421 GcsFileSystem fs(
1422 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1423 std::unique_ptr<HttpRequest::Factory>(
1424 new FakeHttpRequestFactory(&requests)),
1425 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1426 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1427 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1428 0 /* matching paths cache max entries */, kTestRetryConfig,
1429 kTestTimeoutConfig, *kAllowedLocationsDefault,
1430 nullptr /* gcs additional header */, false /* compose append */);
1431
1432 std::unique_ptr<WritableFile> file;
1433 EXPECT_TRUE(errors::IsInvalidArgument(
1434 fs.NewAppendableFile("gs://bucket/", nullptr, &file)));
1435 }
1436
TEST(GcsFileSystemTest,NewAppendableFile_ObjectDoesNotExist)1437 TEST(GcsFileSystemTest, NewAppendableFile_ObjectDoesNotExist) {
1438 std::vector<HttpRequest*> requests(
1439 {new FakeHttpRequest(
1440 "Uri: https://storage.googleapis.com/bucket/filename\n"
1441 "Auth Token: fake_token\n"
1442 "Range: 0-1048575\n"
1443 "Timeouts: 5 1 20\n",
1444 "", errors::NotFound("404"), 404),
1445 new FakeHttpRequest(
1446 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o"
1447 "?uploadType=resumable&name=filename\n"
1448 "Auth Token: fake_token\n"
1449 "Header X-Upload-Content-Length: 0\n"
1450 "Post: yes\n"
1451 "Timeouts: 5 1 10\n",
1452 "")});
1453 GcsFileSystem fs(
1454 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1455 std::unique_ptr<HttpRequest::Factory>(
1456 new FakeHttpRequestFactory(&requests)),
1457 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1458 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1459 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1460 0 /* matching paths cache max entries */, kTestRetryConfig,
1461 kTestTimeoutConfig, *kAllowedLocationsDefault,
1462 nullptr /* gcs additional header */, false /* compose append */);
1463
1464 std::unique_ptr<WritableFile> file;
1465 TF_EXPECT_OK(fs.NewAppendableFile("gs://bucket/filename", nullptr, &file));
1466 }
1467
TEST(GcsFileSystemTest,NewReadOnlyMemoryRegionFromFile)1468 TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) {
1469 const string content = "file content";
1470 std::vector<HttpRequest*> requests(
1471 {new FakeHttpRequest(
1472 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1473 "path%2Frandom_access.txt?fields=size%2Cgeneration%2Cupdated\n"
1474 "Auth Token: fake_token\n"
1475 "Timeouts: 5 1 10\n",
1476 strings::StrCat("{\"size\": \"", content.size(), "\"",
1477 ", \"generation\": \"1\"",
1478 ", \"updated\": \"2016-04-29T23:15:24.896Z\"}")),
1479 new FakeHttpRequest(
1480 strings::StrCat("Uri: https://storage.googleapis.com/bucket/"
1481 "path%2Frandom_access.txt\n"
1482 "Auth Token: fake_token\n"
1483 "Range: 0-",
1484 content.size() - 1, "\n", "Timeouts: 5 1 20\n"),
1485 content)});
1486 GcsFileSystem fs(
1487 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1488 std::unique_ptr<HttpRequest::Factory>(
1489 new FakeHttpRequestFactory(&requests)),
1490 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1491 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1492 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1493 0 /* matching paths cache max entries */, kTestRetryConfig,
1494 kTestTimeoutConfig, *kAllowedLocationsDefault,
1495 nullptr /* gcs additional header */, false /* compose append */);
1496
1497 std::unique_ptr<ReadOnlyMemoryRegion> region;
1498 TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile(
1499 "gs://bucket/path/random_access.txt", nullptr, ®ion));
1500
1501 EXPECT_EQ(content, StringPiece(reinterpret_cast<const char*>(region->data()),
1502 region->length()));
1503 }
1504
TEST(GcsFileSystemTest,NewReadOnlyMemoryRegionFromFile_NoObjectName)1505 TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile_NoObjectName) {
1506 std::vector<HttpRequest*> requests;
1507 GcsFileSystem fs(
1508 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1509 std::unique_ptr<HttpRequest::Factory>(
1510 new FakeHttpRequestFactory(&requests)),
1511 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1512 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1513 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1514 0 /* matching paths cache max entries */, kTestRetryConfig,
1515 kTestTimeoutConfig, *kAllowedLocationsDefault,
1516 nullptr /* gcs additional header */, false /* compose append */);
1517
1518 std::unique_ptr<ReadOnlyMemoryRegion> region;
1519 EXPECT_TRUE(errors::IsInvalidArgument(
1520 fs.NewReadOnlyMemoryRegionFromFile("gs://bucket/", nullptr, ®ion)));
1521 }
1522
TEST(GcsFileSystemTest,FileExists_YesAsObject)1523 TEST(GcsFileSystemTest, FileExists_YesAsObject) {
1524 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1525 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1526 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
1527 "Auth Token: fake_token\n"
1528 "Timeouts: 5 1 10\n",
1529 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
1530 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
1531 GcsFileSystem fs(
1532 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1533 std::unique_ptr<HttpRequest::Factory>(
1534 new FakeHttpRequestFactory(&requests)),
1535 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1536 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1537 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1538 0 /* matching paths cache max entries */, kTestRetryConfig,
1539 kTestTimeoutConfig, *kAllowedLocationsDefault,
1540 nullptr /* gcs additional header */, false /* compose append */);
1541
1542 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt", nullptr));
1543 }
1544
TEST(GcsFileSystemTest,FileExists_YesAsFolder)1545 TEST(GcsFileSystemTest, FileExists_YesAsFolder) {
1546 std::vector<HttpRequest*> requests(
1547 {new FakeHttpRequest(
1548 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1549 "path%2Fsubfolder?fields=size%2Cgeneration%2Cupdated\n"
1550 "Auth Token: fake_token\n"
1551 "Timeouts: 5 1 10\n",
1552 "", errors::NotFound("404"), 404),
1553 new FakeHttpRequest(
1554 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1555 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubfolder%2F"
1556 "&maxResults=1\n"
1557 "Auth Token: fake_token\n"
1558 "Timeouts: 5 1 10\n",
1559 "{\"items\": [ "
1560 " { \"name\": \"path/subfolder/\" }]}")});
1561 GcsFileSystem fs(
1562 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1563 std::unique_ptr<HttpRequest::Factory>(
1564 new FakeHttpRequestFactory(&requests)),
1565 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1566 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1567 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1568 0 /* matching paths cache max entries */, kTestRetryConfig,
1569 kTestTimeoutConfig, *kAllowedLocationsDefault,
1570 nullptr /* gcs additional header */, false /* compose append */);
1571
1572 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder", nullptr));
1573 }
1574
TEST(GcsFileSystemTest,FileExists_YesAsBucket)1575 TEST(GcsFileSystemTest, FileExists_YesAsBucket) {
1576 std::vector<HttpRequest*> requests(
1577 {new FakeHttpRequest(
1578 "Uri: https://www.googleapis.com/storage/v1/b/bucket1\n"
1579 "Auth Token: fake_token\n"
1580 "Timeouts: 5 1 10\n",
1581 "{\"size\": \"100\"}"),
1582 new FakeHttpRequest(
1583 "Uri: https://www.googleapis.com/storage/v1/b/bucket1\n"
1584 "Auth Token: fake_token\n"
1585 "Timeouts: 5 1 10\n",
1586 "{\"size\": \"100\"}")});
1587 GcsFileSystem fs(
1588 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1589 std::unique_ptr<HttpRequest::Factory>(
1590 new FakeHttpRequestFactory(&requests)),
1591 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1592 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1593 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1594 0 /* matching paths cache max entries */, kTestRetryConfig,
1595 kTestTimeoutConfig, *kAllowedLocationsDefault,
1596 nullptr /* gcs additional header */, false /* compose append */);
1597
1598 TF_EXPECT_OK(fs.FileExists("gs://bucket1", nullptr));
1599 TF_EXPECT_OK(fs.FileExists("gs://bucket1/", nullptr));
1600 }
1601
TEST(GcsFileSystemTest,FileExists_NotAsObjectOrFolder)1602 TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) {
1603 std::vector<HttpRequest*> requests(
1604 {new FakeHttpRequest(
1605 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1606 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
1607 "Auth Token: fake_token\n"
1608 "Timeouts: 5 1 10\n",
1609 "", errors::NotFound("404"), 404),
1610 new FakeHttpRequest(
1611 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1612 "fields=items%2Fname%2CnextPageToken&prefix=path%2Ffile1.txt%2F"
1613 "&maxResults=1\n"
1614 "Auth Token: fake_token\n"
1615 "Timeouts: 5 1 10\n",
1616 "{\"items\": []}")});
1617 GcsFileSystem fs(
1618 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1619 std::unique_ptr<HttpRequest::Factory>(
1620 new FakeHttpRequestFactory(&requests)),
1621 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1622 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1623 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1624 0 /* matching paths cache max entries */, kTestRetryConfig,
1625 kTestTimeoutConfig, *kAllowedLocationsDefault,
1626 nullptr /* gcs additional header */, false /* compose append */);
1627
1628 EXPECT_TRUE(
1629 errors::IsNotFound(fs.FileExists("gs://bucket/path/file1.txt", nullptr)));
1630 }
1631
TEST(GcsFileSystemTest,FileExists_NotAsBucket)1632 TEST(GcsFileSystemTest, FileExists_NotAsBucket) {
1633 std::vector<HttpRequest*> requests(
1634 {new FakeHttpRequest(
1635 "Uri: https://www.googleapis.com/storage/v1/b/bucket2\n"
1636 "Auth Token: fake_token\n"
1637 "Timeouts: 5 1 10\n",
1638 "", errors::NotFound("404"), 404),
1639 new FakeHttpRequest(
1640 "Uri: https://www.googleapis.com/storage/v1/b/bucket2\n"
1641 "Auth Token: fake_token\n"
1642 "Timeouts: 5 1 10\n",
1643 "", errors::NotFound("404"), 404)});
1644 GcsFileSystem fs(
1645 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1646 std::unique_ptr<HttpRequest::Factory>(
1647 new FakeHttpRequestFactory(&requests)),
1648 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1649 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1650 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1651 0 /* matching paths cache max entries */, kTestRetryConfig,
1652 kTestTimeoutConfig, *kAllowedLocationsDefault,
1653 nullptr /* gcs additional header */, false /* compose append */);
1654 EXPECT_TRUE(
1655 errors::IsInvalidArgument(fs.FileExists("gs://bucket2/", nullptr)));
1656 EXPECT_TRUE(
1657 errors::IsInvalidArgument(fs.FileExists("gs://bucket2", nullptr)));
1658 }
1659
TEST(GcsFileSystemTest,FileExists_StatCache)1660 TEST(GcsFileSystemTest, FileExists_StatCache) {
1661 std::vector<HttpRequest*> requests(
1662 {new FakeHttpRequest(
1663 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1664 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
1665 "Auth Token: fake_token\n"
1666 "Timeouts: 5 1 10\n",
1667 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
1668 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
1669 new FakeHttpRequest(
1670 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1671 "path%2Fsubfolder%2F?fields=size%2Cgeneration%2Cupdated\n"
1672 "Auth Token: fake_token\n"
1673 "Timeouts: 5 1 10\n",
1674 "", errors::NotFound("404"), 404),
1675 new FakeHttpRequest(
1676 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1677 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubfolder%2F"
1678 "&maxResults=1\n"
1679 "Auth Token: fake_token\n"
1680 "Timeouts: 5 1 10\n",
1681 "{\"items\": [ "
1682 " { \"name\": \"path/subfolder/\" }]}")});
1683 GcsFileSystem fs(
1684 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1685 std::unique_ptr<HttpRequest::Factory>(
1686 new FakeHttpRequestFactory(&requests)),
1687 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1688 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
1689 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1690 0 /* matching paths cache max entries */, kTestRetryConfig,
1691 kTestTimeoutConfig, *kAllowedLocationsDefault,
1692 nullptr /* gcs additional header */, false /* compose append */);
1693
1694 // The stat cache will ensure that repeated lookups don't trigger additional
1695 // HTTP requests.
1696 for (int i = 0; i < 10; i++) {
1697 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt", nullptr));
1698 TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder/", nullptr));
1699 }
1700 }
1701
TEST(GcsFileSystemTest,FileExists_DirectoryMark)1702 TEST(GcsFileSystemTest, FileExists_DirectoryMark) {
1703 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1704 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
1705 "dir%2F?fields=size%2Cgeneration%2Cupdated\n"
1706 "Auth Token: fake_token\n"
1707 "Timeouts: 5 1 10\n",
1708 strings::StrCat("{\"size\": \"5\",\"generation\": \"1\","
1709 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
1710 GcsFileSystem fs(
1711 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1712 std::unique_ptr<HttpRequest::Factory>(
1713 new FakeHttpRequestFactory(&requests)),
1714 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1715 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
1716 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1717 0 /* matching paths cache max entries */, kTestRetryConfig,
1718 kTestTimeoutConfig, *kAllowedLocationsDefault,
1719 nullptr /* gcs additional header */, false /* compose append */);
1720
1721 TF_EXPECT_OK(fs.FileExists("gs://bucket/dir/", nullptr));
1722 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/dir/", nullptr));
1723 }
1724
TEST(GcsFileSystemTest,GetChildren_NoItems)1725 TEST(GcsFileSystemTest, GetChildren_NoItems) {
1726 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1727 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1728 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1729 "path%2F\n"
1730 "Auth Token: fake_token\n"
1731 "Timeouts: 5 1 10\n",
1732 "{\"prefixes\": [\"path/subpath/\"]}")});
1733 GcsFileSystem fs(
1734 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1735 std::unique_ptr<HttpRequest::Factory>(
1736 new FakeHttpRequestFactory(&requests)),
1737 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1738 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1739 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1740 0 /* matching paths cache max entries */, kTestRetryConfig,
1741 kTestTimeoutConfig, *kAllowedLocationsDefault,
1742 nullptr /* gcs additional header */, false /* compose append */);
1743
1744 std::vector<string> children;
1745 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", nullptr, &children));
1746
1747 EXPECT_EQ(std::vector<string>({"subpath/"}), children);
1748 }
1749
TEST(GcsFileSystemTest,GetChildren_ThreeFiles)1750 TEST(GcsFileSystemTest, GetChildren_ThreeFiles) {
1751 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1752 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1753 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1754 "path%2F\n"
1755 "Auth Token: fake_token\n"
1756 "Timeouts: 5 1 10\n",
1757 "{\"items\": [ "
1758 " { \"name\": \"path/file1.txt\" },"
1759 " { \"name\": \"path/file3.txt\" }],"
1760 "\"prefixes\": [\"path/subpath/\"]}")});
1761 GcsFileSystem fs(
1762 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1763 std::unique_ptr<HttpRequest::Factory>(
1764 new FakeHttpRequestFactory(&requests)),
1765 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1766 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1767 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1768 0 /* matching paths cache max entries */, kTestRetryConfig,
1769 kTestTimeoutConfig, *kAllowedLocationsDefault,
1770 nullptr /* gcs additional header */, false /* compose append */);
1771
1772 std::vector<string> children;
1773 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", nullptr, &children));
1774
1775 EXPECT_EQ(std::vector<string>({"file1.txt", "file3.txt", "subpath/"}),
1776 children);
1777 }
1778
TEST(GcsFileSystemTest,GetChildren_SelfDirectoryMarker)1779 TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) {
1780 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1781 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1782 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1783 "path%2F\n"
1784 "Auth Token: fake_token\n"
1785 "Timeouts: 5 1 10\n",
1786 "{\"items\": [ "
1787 " { \"name\": \"path/\" },"
1788 " { \"name\": \"path/file3.txt\" }],"
1789 "\"prefixes\": [\"path/subpath/\"]}")});
1790 GcsFileSystem fs(
1791 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1792 std::unique_ptr<HttpRequest::Factory>(
1793 new FakeHttpRequestFactory(&requests)),
1794 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1795 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1796 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1797 0 /* matching paths cache max entries */, kTestRetryConfig,
1798 kTestTimeoutConfig, *kAllowedLocationsDefault,
1799 nullptr /* gcs additional header */, false /* compose append */);
1800
1801 std::vector<string> children;
1802 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", nullptr, &children));
1803
1804 EXPECT_EQ(std::vector<string>({"file3.txt", "subpath/"}), children);
1805 }
1806
TEST(GcsFileSystemTest,GetChildren_ThreeFiles_NoSlash)1807 TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) {
1808 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1809 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1810 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1811 "path%2F\n"
1812 "Auth Token: fake_token\n"
1813 "Timeouts: 5 1 10\n",
1814 "{\"items\": [ "
1815 " { \"name\": \"path/file1.txt\" },"
1816 " { \"name\": \"path/file3.txt\" }],"
1817 "\"prefixes\": [\"path/subpath/\"]}")});
1818 GcsFileSystem fs(
1819 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1820 std::unique_ptr<HttpRequest::Factory>(
1821 new FakeHttpRequestFactory(&requests)),
1822 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1823 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1824 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1825 0 /* matching paths cache max entries */, kTestRetryConfig,
1826 kTestTimeoutConfig, *kAllowedLocationsDefault,
1827 nullptr /* gcs additional header */, false /* compose append */);
1828
1829 std::vector<string> children;
1830 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", nullptr, &children));
1831
1832 EXPECT_EQ(std::vector<string>({"file1.txt", "file3.txt", "subpath/"}),
1833 children);
1834 }
1835
TEST(GcsFileSystemTest,GetChildren_Root)1836 TEST(GcsFileSystemTest, GetChildren_Root) {
1837 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1838 "Uri: https://www.googleapis.com/storage/v1/b/bucket-a-b-c/o?"
1839 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F\n"
1840 "Auth Token: fake_token\n"
1841 "Timeouts: 5 1 10\n",
1842 "{}")});
1843 GcsFileSystem fs(
1844 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1845 std::unique_ptr<HttpRequest::Factory>(
1846 new FakeHttpRequestFactory(&requests)),
1847 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1848 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1849 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1850 0 /* matching paths cache max entries */, kTestRetryConfig,
1851 kTestTimeoutConfig, *kAllowedLocationsDefault,
1852 nullptr /* gcs additional header */, false /* compose append */);
1853
1854 std::vector<string> children;
1855 TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", nullptr, &children));
1856
1857 EXPECT_EQ(0, children.size());
1858 }
1859
TEST(GcsFileSystemTest,GetChildren_Empty)1860 TEST(GcsFileSystemTest, GetChildren_Empty) {
1861 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1862 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1863 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
1864 "path%2F\n"
1865 "Auth Token: fake_token\n"
1866 "Timeouts: 5 1 10\n",
1867 "{}")});
1868 GcsFileSystem fs(
1869 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1870 std::unique_ptr<HttpRequest::Factory>(
1871 new FakeHttpRequestFactory(&requests)),
1872 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1873 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1874 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1875 0 /* matching paths cache max entries */, kTestRetryConfig,
1876 kTestTimeoutConfig, *kAllowedLocationsDefault,
1877 nullptr /* gcs additional header */, false /* compose append */);
1878
1879 std::vector<string> children;
1880 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", nullptr, &children));
1881
1882 EXPECT_EQ(0, children.size());
1883 }
1884
TEST(GcsFileSystemTest,GetChildren_Pagination)1885 TEST(GcsFileSystemTest, GetChildren_Pagination) {
1886 std::vector<HttpRequest*> requests(
1887 {new FakeHttpRequest(
1888 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1889 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&"
1890 "prefix=path%2F\n"
1891 "Auth Token: fake_token\n"
1892 "Timeouts: 5 1 10\n",
1893 "{\"nextPageToken\": \"ABCD==\", "
1894 "\"items\": [ "
1895 " { \"name\": \"path/file1.txt\" },"
1896 " { \"name\": \"path/file3.txt\" }],"
1897 "\"prefixes\": [\"path/subpath/\"]}"),
1898 new FakeHttpRequest(
1899 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1900 "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&"
1901 "prefix=path%2F"
1902 "&pageToken=ABCD==\n"
1903 "Auth Token: fake_token\n"
1904 "Timeouts: 5 1 10\n",
1905 "{\"items\": [ "
1906 " { \"name\": \"path/file4.txt\" },"
1907 " { \"name\": \"path/file5.txt\" }]}")});
1908
1909 GcsFileSystem fs(
1910 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1911 std::unique_ptr<HttpRequest::Factory>(
1912 new FakeHttpRequestFactory(&requests)),
1913 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1914 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1915 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1916 0 /* matching paths cache max entries */, kTestRetryConfig,
1917 kTestTimeoutConfig, *kAllowedLocationsDefault,
1918 nullptr /* gcs additional header */, false /* compose append */);
1919
1920 std::vector<string> children;
1921 TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", nullptr, &children));
1922
1923 EXPECT_EQ(std::vector<string>({"file1.txt", "file3.txt", "subpath/",
1924 "file4.txt", "file5.txt"}),
1925 children);
1926 }
1927
TEST(GcsFileSystemTest,GetMatchingPaths_NoWildcard)1928 TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) {
1929 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1930 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1931 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
1932 "Auth Token: fake_token\n"
1933 "Timeouts: 5 1 10\n",
1934 "{\"items\": [ "
1935 " { \"name\": \"path/subpath/file2.txt\" }]}")});
1936 GcsFileSystem fs(
1937 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1938 std::unique_ptr<HttpRequest::Factory>(
1939 new FakeHttpRequestFactory(&requests)),
1940 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1941 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1942 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1943 0 /* matching paths cache max entries */, kTestRetryConfig,
1944 kTestTimeoutConfig, *kAllowedLocationsDefault,
1945 nullptr /* gcs additional header */, false /* compose append */);
1946
1947 std::vector<string> result;
1948 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt",
1949 nullptr, &result));
1950 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
1951 result);
1952 }
1953
TEST(GcsFileSystemTest,GetMatchingPaths_BucketAndWildcard)1954 TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) {
1955 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1956 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1957 "fields=items%2Fname%2CnextPageToken\n"
1958 "Auth Token: fake_token\n"
1959 "Timeouts: 5 1 10\n",
1960 "{\"items\": [ "
1961 " { \"name\": \"path/file1.txt\" },"
1962 " { \"name\": \"path/subpath/file2.txt\" },"
1963 " { \"name\": \"path/file3.txt\" }]}")});
1964 GcsFileSystem fs(
1965 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1966 std::unique_ptr<HttpRequest::Factory>(
1967 new FakeHttpRequestFactory(&requests)),
1968 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1969 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1970 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
1971 0 /* matching paths cache max entries */, kTestRetryConfig,
1972 kTestTimeoutConfig, *kAllowedLocationsDefault,
1973 nullptr /* gcs additional header */, false /* compose append */);
1974
1975 std::vector<string> result;
1976 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", nullptr, &result));
1977 EXPECT_EQ(std::vector<string>({"gs://bucket/path/file1.txt",
1978 "gs://bucket/path/file3.txt",
1979 "gs://bucket/path/subpath"}),
1980 result);
1981 }
1982
TEST(GcsFileSystemTest,GetMatchingPaths_FolderAndWildcard_Matches)1983 TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) {
1984 std::vector<HttpRequest*> requests({new FakeHttpRequest(
1985 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
1986 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
1987 "Auth Token: fake_token\n"
1988 "Timeouts: 5 1 10\n",
1989 "{\"items\": [ "
1990 " { \"name\": \"path/file1.txt\" },"
1991 " { \"name\": \"path/subpath/file2.txt\" },"
1992 " { \"name\": \"path/file3.txt\" }]}")});
1993 GcsFileSystem fs(
1994 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
1995 std::unique_ptr<HttpRequest::Factory>(
1996 new FakeHttpRequestFactory(&requests)),
1997 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
1998 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
1999 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2000 0 /* matching paths cache max entries */, kTestRetryConfig,
2001 kTestTimeoutConfig, *kAllowedLocationsDefault,
2002 nullptr /* gcs additional header */, false /* compose append */);
2003
2004 std::vector<string> result;
2005 TF_EXPECT_OK(
2006 fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", nullptr, &result));
2007 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
2008 result);
2009 }
2010
TEST(GcsFileSystemTest,GetMatchingPaths_SelfDirectoryMarker)2011 TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) {
2012 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2013 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2014 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
2015 "Auth Token: fake_token\n"
2016 "Timeouts: 5 1 10\n",
2017 "{\"items\": [ "
2018 " { \"name\": \"path/\" },"
2019 " { \"name\": \"path/file3.txt\" }]}")});
2020 GcsFileSystem fs(
2021 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2022 std::unique_ptr<HttpRequest::Factory>(
2023 new FakeHttpRequestFactory(&requests)),
2024 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2025 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2026 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2027 0 /* matching paths cache max entries */, kTestRetryConfig,
2028 kTestTimeoutConfig, *kAllowedLocationsDefault,
2029 nullptr /* gcs additional header */, false /* compose append */);
2030
2031 std::vector<string> result;
2032 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", nullptr, &result));
2033 EXPECT_EQ(std::vector<string>({"gs://bucket/path/file3.txt"}), result);
2034 }
2035
TEST(GcsFileSystemTest,GetMatchingPaths_SlashInObjectName)2036 TEST(GcsFileSystemTest, GetMatchingPaths_SlashInObjectName) {
2037 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2038 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2039 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
2040 "Auth Token: fake_token\n"
2041 "Timeouts: 5 1 10\n",
2042 "{\"items\": [ "
2043 " { \"name\": \"path/\" },"
2044 " { \"name\": \"path//foo.txt\" }]}")});
2045 GcsFileSystem fs(
2046 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2047 std::unique_ptr<HttpRequest::Factory>(
2048 new FakeHttpRequestFactory(&requests)),
2049 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2050 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2051 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2052 0 /* matching paths cache max entries */, kTestRetryConfig,
2053 kTestTimeoutConfig, *kAllowedLocationsDefault,
2054 nullptr /* gcs additional header */, false /* compose append */);
2055
2056 std::vector<string> result;
2057 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", nullptr, &result));
2058 EXPECT_EQ(std::vector<string>(), result);
2059 }
2060
TEST(GcsFileSystemTest,GetMatchingPaths_SlashInObjectNameEscaped)2061 TEST(GcsFileSystemTest, GetMatchingPaths_SlashInObjectNameEscaped) {
2062 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2063 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2064 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
2065 "Auth Token: fake_token\n"
2066 "Timeouts: 5 1 10\n",
2067 "{\"items\": [ "
2068 " { \"name\": \"path/\" },"
2069 " { \"name\": \"path//foo.txt\" }]}")});
2070 GcsFileSystem fs(
2071 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2072 std::unique_ptr<HttpRequest::Factory>(
2073 new FakeHttpRequestFactory(&requests)),
2074 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2075 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2076 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2077 0 /* matching paths cache max entries */, kTestRetryConfig,
2078 kTestTimeoutConfig, *kAllowedLocationsDefault,
2079 nullptr /* gcs additional header */, false /* compose append */);
2080
2081 std::vector<string> result;
2082 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/\\/*", nullptr, &result));
2083 EXPECT_EQ(std::vector<string>({"gs://bucket/path//foo.txt"}), result);
2084 }
2085
TEST(GcsFileSystemTest,GetMatchingPaths_FolderAndWildcard_NoMatches)2086 TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) {
2087 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2088 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2089 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
2090 "Auth Token: fake_token\n"
2091 "Timeouts: 5 1 10\n",
2092 "{\"items\": [ "
2093 " { \"name\": \"path/file1.txt\" },"
2094 " { \"name\": \"path/subpath/file2.txt\" },"
2095 " { \"name\": \"path/file3.txt\" }]}")});
2096 GcsFileSystem fs(
2097 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2098 std::unique_ptr<HttpRequest::Factory>(
2099 new FakeHttpRequestFactory(&requests)),
2100 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2101 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2102 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2103 0 /* matching paths cache max entries */, kTestRetryConfig,
2104 kTestTimeoutConfig, *kAllowedLocationsDefault,
2105 nullptr /* gcs additional header */, false /* compose append */);
2106
2107 std::vector<string> result;
2108 TF_EXPECT_OK(
2109 fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", nullptr, &result));
2110 EXPECT_EQ(std::vector<string>(), result);
2111 }
2112
TEST(GcsFileSystemTest,GetMatchingPaths_OnlyWildcard)2113 TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) {
2114 std::vector<HttpRequest*> requests;
2115 GcsFileSystem fs(
2116 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2117 std::unique_ptr<HttpRequest::Factory>(
2118 new FakeHttpRequestFactory(&requests)),
2119 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2120 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2121 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2122 0 /* matching paths cache max entries */, kTestRetryConfig,
2123 kTestTimeoutConfig, *kAllowedLocationsDefault,
2124 nullptr /* gcs additional header */, false /* compose append */);
2125
2126 std::vector<string> result;
2127 EXPECT_TRUE(errors::IsInvalidArgument(
2128 fs.GetMatchingPaths("gs://*", nullptr, &result)));
2129 }
2130
TEST(GcsFileSystemTest,GetMatchingPaths_Cache)2131 TEST(GcsFileSystemTest, GetMatchingPaths_Cache) {
2132 std::vector<HttpRequest*> requests(
2133 {new FakeHttpRequest(
2134 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2135 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
2136 "Auth Token: fake_token\n"
2137 "Timeouts: 5 1 10\n",
2138 "{\"items\": [ "
2139 " { \"name\": \"path/subpath/file2.txt\" }]}"),
2140 new FakeHttpRequest(
2141 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2142 "fields=items%2Fname%2CnextPageToken\n"
2143 "Auth Token: fake_token\n"
2144 "Timeouts: 5 1 10\n",
2145 "{\"items\": [ "
2146 " { \"name\": \"path/file1.txt\" },"
2147 " { \"name\": \"path/subpath/file2.txt\" },"
2148 " { \"name\": \"path/file3.txt\" }]}")});
2149 GcsFileSystem fs(
2150 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2151 std::unique_ptr<HttpRequest::Factory>(
2152 new FakeHttpRequestFactory(&requests)),
2153 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2154 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2155 0 /* stat cache max entries */, 3600 /* matching paths cache max age */,
2156 0 /* matching paths cache max entries */, kTestRetryConfig,
2157 kTestTimeoutConfig, *kAllowedLocationsDefault,
2158 nullptr /* gcs additional header */, false /* compose append */);
2159
2160 // Repeated calls to fs.GetMatchingPaths on these patterns should not lead to
2161 // any additional HTTP requests to GCS.
2162 for (int i = 0; i < 10; i++) {
2163 std::vector<string> result;
2164 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt",
2165 nullptr, &result));
2166 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
2167 result);
2168 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", nullptr, &result));
2169 EXPECT_EQ(std::vector<string>({"gs://bucket/path/file1.txt",
2170 "gs://bucket/path/file3.txt",
2171 "gs://bucket/path/subpath"}),
2172 result);
2173 }
2174 }
2175
TEST(GcsFileSystemTest,GetMatchingPaths_Cache_Flush)2176 TEST(GcsFileSystemTest, GetMatchingPaths_Cache_Flush) {
2177 std::vector<HttpRequest*> requests(
2178 {new FakeHttpRequest(
2179 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2180 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
2181 "Auth Token: fake_token\n"
2182 "Timeouts: 5 1 10\n",
2183 "{\"items\": [ "
2184 " { \"name\": \"path/subpath/file2.txt\" }]}"),
2185 new FakeHttpRequest(
2186 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2187 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
2188 "Auth Token: fake_token\n"
2189 "Timeouts: 5 1 10\n",
2190 "{\"items\": [ "
2191 " { \"name\": \"path/subpath/file2.txt\" }]}")});
2192 GcsFileSystem fs(
2193 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2194 std::unique_ptr<HttpRequest::Factory>(
2195 new FakeHttpRequestFactory(&requests)),
2196 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2197 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2198 0 /* stat cache max entries */, 3600 /* matching paths cache max age */,
2199 0 /* matching paths cache max entries */, kTestRetryConfig,
2200 kTestTimeoutConfig, *kAllowedLocationsDefault,
2201 nullptr /* gcs additional header */, false /* compose append */);
2202
2203 // This loop should trigger the first HTTP request to GCS.
2204 for (int i = 0; i < 10; i++) {
2205 std::vector<string> result;
2206 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt",
2207 nullptr, &result));
2208 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
2209 result);
2210 }
2211 // After flushing caches, there should be another (identical) request to GCS.
2212 fs.FlushCaches(nullptr);
2213 for (int i = 0; i < 10; i++) {
2214 std::vector<string> result;
2215 TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt",
2216 nullptr, &result));
2217 EXPECT_EQ(std::vector<string>({"gs://bucket/path/subpath/file2.txt"}),
2218 result);
2219 }
2220 }
2221
TEST(GcsFileSystemTest,DeleteFile)2222 TEST(GcsFileSystemTest, DeleteFile) {
2223 std::vector<HttpRequest*> requests(
2224 {new FakeHttpRequest(
2225 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2226 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
2227 "Auth Token: fake_token\n"
2228 "Timeouts: 5 1 10\n",
2229 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
2230 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2231 new FakeHttpRequest(
2232 "Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n"
2233 "Auth Token: fake_token\n"
2234 "Range: 0-15\n"
2235 "Timeouts: 5 1 20\n",
2236 "01234567"),
2237 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2238 "/bucket/o/path%2Ffile1.txt\n"
2239 "Auth Token: fake_token\n"
2240 "Timeouts: 5 1 10\n"
2241 "Delete: yes\n",
2242 ""),
2243 new FakeHttpRequest(
2244 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2245 "path%2Ffile1.txt?fields=size%2Cgeneration%2Cupdated\n"
2246 "Auth Token: fake_token\n"
2247 "Timeouts: 5 1 10\n",
2248 strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
2249 "\"updated\": \"2016-04-29T23:19:24.896Z\"}")),
2250 new FakeHttpRequest(
2251 "Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n"
2252 "Auth Token: fake_token\n"
2253 "Range: 0-15\n"
2254 "Timeouts: 5 1 20\n",
2255 "76543210")});
2256 GcsFileSystem fs(
2257 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2258 std::unique_ptr<HttpRequest::Factory>(
2259 new FakeHttpRequestFactory(&requests)),
2260 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 16 /* block size */,
2261 16 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
2262 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2263 0 /* matching paths cache max entries */, kTestRetryConfig,
2264 kTestTimeoutConfig, *kAllowedLocationsDefault,
2265 nullptr /* gcs additional header */, false /* compose append */);
2266
2267 // Do an initial read of the file to load its contents into the block cache.
2268 char scratch[100];
2269 StringPiece result;
2270 std::unique_ptr<RandomAccessFile> file;
2271 TF_EXPECT_OK(
2272 fs.NewRandomAccessFile("gs://bucket/path/file1.txt", nullptr, &file));
2273 TF_EXPECT_OK(file->Read(0, 8, &result, scratch));
2274 EXPECT_EQ("01234567", result);
2275 // Deleting the file triggers the next HTTP request to GCS.
2276 TF_EXPECT_OK(fs.DeleteFile("gs://bucket/path/file1.txt", nullptr));
2277 // Re-reading the file causes its contents to be reloaded from GCS and not
2278 // from the block cache.
2279 TF_EXPECT_OK(file->Read(0, 8, &result, scratch));
2280 EXPECT_EQ("76543210", result);
2281 }
2282
TEST(GcsFileSystemTest,DeleteFile_NoObjectName)2283 TEST(GcsFileSystemTest, DeleteFile_NoObjectName) {
2284 std::vector<HttpRequest*> requests;
2285 GcsFileSystem fs(
2286 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2287 std::unique_ptr<HttpRequest::Factory>(
2288 new FakeHttpRequestFactory(&requests)),
2289 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2290 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2291 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2292 0 /* matching paths cache max entries */, kTestRetryConfig,
2293 kTestTimeoutConfig, *kAllowedLocationsDefault,
2294 nullptr /* gcs additional header */, false /* compose append */);
2295
2296 EXPECT_TRUE(
2297 errors::IsInvalidArgument(fs.DeleteFile("gs://bucket/", nullptr)));
2298 }
2299
TEST(GcsFileSystemTest,DeleteFile_StatCacheRemoved)2300 TEST(GcsFileSystemTest, DeleteFile_StatCacheRemoved) {
2301 std::vector<HttpRequest*> requests(
2302 {new FakeHttpRequest(
2303 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2304 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2305 "Auth Token: fake_token\n"
2306 "Timeouts: 5 1 10\n",
2307 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2308 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2309 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2310 "/bucket/o/file.txt\n"
2311 "Auth Token: fake_token\n"
2312 "Timeouts: 5 1 10\n"
2313 "Delete: yes\n",
2314 ""),
2315 new FakeHttpRequest(
2316 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2317 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2318 "Auth Token: fake_token\n"
2319 "Timeouts: 5 1 10\n",
2320 "", errors::NotFound("404"), 404),
2321 new FakeHttpRequest(
2322 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2323 "fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F"
2324 "&maxResults=1\n"
2325 "Auth Token: fake_token\n"
2326 "Timeouts: 5 1 10\n",
2327 "{}")});
2328 GcsFileSystem fs(
2329 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2330 std::unique_ptr<HttpRequest::Factory>(
2331 new FakeHttpRequestFactory(&requests)),
2332 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 16 /* block size */,
2333 16 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
2334 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2335 0 /* matching paths cache max entries */, kTestRetryConfig,
2336 kTestTimeoutConfig, *kAllowedLocationsDefault,
2337 nullptr /* gcs additional header */, false /* compose append */);
2338
2339 // Stats the file first so the stat is cached.
2340 FileStatistics stat_before_deletion;
2341 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat_before_deletion));
2342 EXPECT_EQ(1010, stat_before_deletion.length);
2343
2344 TF_EXPECT_OK(fs.DeleteFile("gs://bucket/file.txt", nullptr));
2345
2346 FileStatistics stat_after_deletion;
2347 EXPECT_EQ(
2348 error::Code::NOT_FOUND,
2349 fs.Stat("gs://bucket/file.txt", nullptr, &stat_after_deletion).code());
2350 }
2351
TEST(GcsFileSystemTest,DeleteDir_Empty)2352 TEST(GcsFileSystemTest, DeleteDir_Empty) {
2353 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2354 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2355 "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n"
2356 "Auth Token: fake_token\n"
2357 "Timeouts: 5 1 10\n",
2358 "{}")});
2359 GcsFileSystem fs(
2360 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2361 std::unique_ptr<HttpRequest::Factory>(
2362 new FakeHttpRequestFactory(&requests)),
2363 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2364 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2365 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2366 0 /* matching paths cache max entries */, kTestRetryConfig,
2367 kTestTimeoutConfig, *kAllowedLocationsDefault,
2368 nullptr /* gcs additional header */, false /* compose append */);
2369
2370 TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/", nullptr));
2371 }
2372
TEST(GcsFileSystemTest,DeleteDir_OnlyDirMarkerLeft)2373 TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) {
2374 std::vector<HttpRequest*> requests(
2375 {new FakeHttpRequest(
2376 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2377 "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n"
2378 "Auth Token: fake_token\n"
2379 "Timeouts: 5 1 10\n",
2380 "{\"items\": [ "
2381 " { \"name\": \"path/\" }]}"),
2382 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
2383 "/bucket/o/path%2F\n"
2384 "Auth Token: fake_token\n"
2385 "Timeouts: 5 1 10\n"
2386 "Delete: yes\n",
2387 "")});
2388 GcsFileSystem fs(
2389 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2390 std::unique_ptr<HttpRequest::Factory>(
2391 new FakeHttpRequestFactory(&requests)),
2392 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2393 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2394 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2395 0 /* matching paths cache max entries */, kTestRetryConfig,
2396 kTestTimeoutConfig, *kAllowedLocationsDefault,
2397 nullptr /* gcs additional header */, false /* compose append */);
2398
2399 TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/", nullptr));
2400 }
2401
TEST(GcsFileSystemTest,DeleteDir_BucketOnly)2402 TEST(GcsFileSystemTest, DeleteDir_BucketOnly) {
2403 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2404 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?fields=items%2F"
2405 "name%2CnextPageToken&maxResults=2\nAuth Token: fake_token\n"
2406 "Timeouts: 5 1 10\n",
2407 "{}")});
2408 GcsFileSystem fs(
2409 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2410 std::unique_ptr<HttpRequest::Factory>(
2411 new FakeHttpRequestFactory(&requests)),
2412 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2413 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2414 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2415 0 /* matching paths cache max entries */, kTestRetryConfig,
2416 kTestTimeoutConfig, *kAllowedLocationsDefault,
2417 nullptr /* gcs additional header */, false /* compose append */);
2418
2419 TF_EXPECT_OK(fs.DeleteDir("gs://bucket", nullptr));
2420 }
2421
TEST(GcsFileSystemTest,DeleteDir_NonEmpty)2422 TEST(GcsFileSystemTest, DeleteDir_NonEmpty) {
2423 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2424 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2425 "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n"
2426 "Auth Token: fake_token\n"
2427 "Timeouts: 5 1 10\n",
2428 "{\"items\": [ "
2429 " { \"name\": \"path/file1.txt\" }]}")});
2430 GcsFileSystem fs(
2431 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2432 std::unique_ptr<HttpRequest::Factory>(
2433 new FakeHttpRequestFactory(&requests)),
2434 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2435 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2436 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2437 0 /* matching paths cache max entries */, kTestRetryConfig,
2438 kTestTimeoutConfig, *kAllowedLocationsDefault,
2439 nullptr /* gcs additional header */, false /* compose append */);
2440
2441 EXPECT_EQ(error::Code::FAILED_PRECONDITION,
2442 fs.DeleteDir("gs://bucket/path/", nullptr).code());
2443 }
2444
TEST(GcsFileSystemTest,GetFileSize)2445 TEST(GcsFileSystemTest, GetFileSize) {
2446 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2447 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2448 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2449 "Auth Token: fake_token\n"
2450 "Timeouts: 5 1 10\n",
2451 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2452 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
2453 GcsFileSystem fs(
2454 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2455 std::unique_ptr<HttpRequest::Factory>(
2456 new FakeHttpRequestFactory(&requests)),
2457 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2458 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2459 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2460 0 /* matching paths cache max entries */, kTestRetryConfig,
2461 kTestTimeoutConfig, *kAllowedLocationsDefault,
2462 nullptr /* gcs additional header */, false /* compose append */);
2463
2464 uint64 size;
2465 TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", nullptr, &size));
2466 EXPECT_EQ(1010, size);
2467 }
2468
TEST(GcsFileSystemTest,GetFileSize_NoObjectName)2469 TEST(GcsFileSystemTest, GetFileSize_NoObjectName) {
2470 std::vector<HttpRequest*> requests;
2471 GcsFileSystem fs(
2472 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2473 std::unique_ptr<HttpRequest::Factory>(
2474 new FakeHttpRequestFactory(&requests)),
2475 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2476 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2477 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2478 0 /* matching paths cache max entries */, kTestRetryConfig,
2479 kTestTimeoutConfig, *kAllowedLocationsDefault,
2480 nullptr /* gcs additional header */, false /* compose append */);
2481
2482 uint64 size;
2483 EXPECT_TRUE(errors::IsInvalidArgument(
2484 fs.GetFileSize("gs://bucket/", nullptr, &size)));
2485 }
2486
TEST(GcsFileSystemTest,RenameFile_Folder)2487 TEST(GcsFileSystemTest, RenameFile_Folder) {
2488 std::vector<HttpRequest*> requests(
2489 {// Check if this is a folder or an object.
2490 new FakeHttpRequest(
2491 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2492 "fields=items%2Fname%2CnextPageToken&prefix=path1%2F"
2493 "&maxResults=1\n"
2494 "Auth Token: fake_token\n"
2495 "Timeouts: 5 1 10\n",
2496 "{\"items\": [ "
2497 " { \"name\": \"path1/subfolder/file1.txt\" }]}"),
2498 // Requesting the full list of files in the folder.
2499 new FakeHttpRequest(
2500 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2501 "fields=items%2Fname%2CnextPageToken&prefix=path1%2F\n"
2502 "Auth Token: fake_token\n"
2503 "Timeouts: 5 1 10\n",
2504 "{\"items\": [ "
2505 " { \"name\": \"path1/\" }," // A directory marker.
2506 " { \"name\": \"path1/subfolder/file1.txt\" },"
2507 " { \"name\": \"path1/file2.txt\" }]}"),
2508 // Copying the directory marker.
2509 new FakeHttpRequest(
2510 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2511 "path1%2F/rewriteTo/b/bucket/o/path2%2F\n"
2512 "Auth Token: fake_token\n"
2513 "Post: yes\n"
2514 "Timeouts: 5 1 10\n",
2515 "{\"done\": true}"),
2516 // Deleting the original directory marker.
2517 new FakeHttpRequest(
2518 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2519 "path1%2F\n"
2520 "Auth Token: fake_token\n"
2521 "Timeouts: 5 1 10\n"
2522 "Delete: yes\n",
2523 ""),
2524 // Copying the first file.
2525 new FakeHttpRequest(
2526 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2527 "path1%2Fsubfolder%2Ffile1.txt/rewriteTo/b/bucket/o/"
2528 "path2%2Fsubfolder%2Ffile1.txt\n"
2529 "Auth Token: fake_token\n"
2530 "Post: yes\n"
2531 "Timeouts: 5 1 10\n",
2532 "{\"done\": true}"),
2533 // Deleting the first original file.
2534 new FakeHttpRequest(
2535 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2536 "path1%2Fsubfolder%2Ffile1.txt\n"
2537 "Auth Token: fake_token\n"
2538 "Timeouts: 5 1 10\n"
2539 "Delete: yes\n",
2540 ""),
2541 // Copying the second file.
2542 new FakeHttpRequest(
2543 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2544 "path1%2Ffile2.txt/rewriteTo/b/bucket/o/path2%2Ffile2.txt\n"
2545 "Auth Token: fake_token\n"
2546 "Post: yes\n"
2547 "Timeouts: 5 1 10\n",
2548 "{\"done\": true}"),
2549 // Deleting the second original file.
2550 new FakeHttpRequest(
2551 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2552 "path1%2Ffile2.txt\n"
2553 "Auth Token: fake_token\n"
2554 "Timeouts: 5 1 10\n"
2555 "Delete: yes\n",
2556 "")});
2557 GcsFileSystem fs(
2558 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2559 std::unique_ptr<HttpRequest::Factory>(
2560 new FakeHttpRequestFactory(&requests)),
2561 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2562 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2563 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2564 0 /* matching paths cache max entries */, kTestRetryConfig,
2565 kTestTimeoutConfig, *kAllowedLocationsDefault,
2566 nullptr /* gcs additional header */, false /* compose append */);
2567
2568 TF_EXPECT_OK(
2569 fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/", nullptr));
2570 }
2571
TEST(GcsFileSystemTest,RenameFile_Object)2572 TEST(GcsFileSystemTest, RenameFile_Object) {
2573 std::vector<HttpRequest*> requests(
2574 {new FakeHttpRequest(
2575 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2576 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2577 "Auth Token: fake_token\n"
2578 "Timeouts: 5 1 10\n",
2579 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
2580 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2581 new FakeHttpRequest(
2582 "Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n"
2583 "Auth Token: fake_token\n"
2584 "Range: 0-15\n"
2585 "Timeouts: 5 1 20\n",
2586 "01234567"),
2587 new FakeHttpRequest(
2588 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2589 "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
2590 "Auth Token: fake_token\n"
2591 "Timeouts: 5 1 10\n",
2592 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
2593 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2594 new FakeHttpRequest(
2595 "Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n"
2596 "Auth Token: fake_token\n"
2597 "Range: 0-15\n"
2598 "Timeouts: 5 1 20\n",
2599 "76543210"),
2600 // IsDirectory is checking whether there are children objects.
2601 new FakeHttpRequest(
2602 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2603 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
2604 "&maxResults=1\n"
2605 "Auth Token: fake_token\n"
2606 "Timeouts: 5 1 10\n",
2607 "{}"),
2608 // Copying to the new location.
2609 new FakeHttpRequest(
2610 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2611 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
2612 "Auth Token: fake_token\n"
2613 "Post: yes\n"
2614 "Timeouts: 5 1 10\n",
2615 "{\"done\": true}"),
2616 // Deleting the original file.
2617 new FakeHttpRequest(
2618 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2619 "path%2Fsrc.txt\n"
2620 "Auth Token: fake_token\n"
2621 "Timeouts: 5 1 10\n"
2622 "Delete: yes\n",
2623 ""),
2624 new FakeHttpRequest(
2625 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2626 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2627 "Auth Token: fake_token\n"
2628 "Timeouts: 5 1 10\n",
2629 strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
2630 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2631 new FakeHttpRequest(
2632 "Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n"
2633 "Auth Token: fake_token\n"
2634 "Range: 0-15\n"
2635 "Timeouts: 5 1 20\n",
2636 "89abcdef"),
2637 new FakeHttpRequest(
2638 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2639 "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
2640 "Auth Token: fake_token\n"
2641 "Timeouts: 5 1 10\n",
2642 strings::StrCat("{\"size\": \"8\",\"generation\": \"2\","
2643 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2644 new FakeHttpRequest(
2645 "Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n"
2646 "Auth Token: fake_token\n"
2647 "Range: 0-15\n"
2648 "Timeouts: 5 1 20\n",
2649 "fedcba98")});
2650 GcsFileSystem fs(
2651 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2652 std::unique_ptr<HttpRequest::Factory>(
2653 new FakeHttpRequestFactory(&requests)),
2654 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 16 /* block size */,
2655 64 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
2656 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2657 0 /* matching paths cache max entries */, kTestRetryConfig,
2658 kTestTimeoutConfig, *kAllowedLocationsDefault,
2659 nullptr /* gcs additional header */, false /* compose append */);
2660 // Do an initial read of the source and destination files to load their
2661 // contents into the block cache.
2662 char scratch[100];
2663 StringPiece result;
2664 std::unique_ptr<RandomAccessFile> src;
2665 std::unique_ptr<RandomAccessFile> dst;
2666 TF_EXPECT_OK(
2667 fs.NewRandomAccessFile("gs://bucket/path/src.txt", nullptr, &src));
2668 TF_EXPECT_OK(src->Read(0, 8, &result, scratch));
2669 EXPECT_EQ("01234567", result);
2670 TF_EXPECT_OK(
2671 fs.NewRandomAccessFile("gs://bucket/path/dst.txt", nullptr, &dst));
2672 TF_EXPECT_OK(dst->Read(0, 8, &result, scratch));
2673 EXPECT_EQ("76543210", result);
2674 // Now rename src to dst. This should flush the block cache for both files.
2675 TF_EXPECT_OK(fs.RenameFile("gs://bucket/path/src.txt",
2676 "gs://bucket/path/dst.txt", nullptr));
2677 // Re-read both files. This should reload their contents from GCS.
2678 TF_EXPECT_OK(src->Read(0, 8, &result, scratch));
2679 EXPECT_EQ("89abcdef", result);
2680 TF_EXPECT_OK(dst->Read(0, 8, &result, scratch));
2681 EXPECT_EQ("fedcba98", result);
2682 }
2683
TEST(GcsFileSystemTest,RenameFile_Object_FlushTargetStatCache)2684 TEST(GcsFileSystemTest, RenameFile_Object_FlushTargetStatCache) {
2685 std::vector<HttpRequest*> requests(
2686 {// Stat the target file.
2687 new FakeHttpRequest(
2688 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2689 "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
2690 "Auth Token: fake_token\n"
2691 "Timeouts: 5 1 10\n",
2692 strings::StrCat("{\"size\": \"1000\",\"generation\": \"1\","
2693 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2694 // IsDirectory is checking whether there are children objects.
2695 new FakeHttpRequest(
2696 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2697 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
2698 "&maxResults=1\n"
2699 "Auth Token: fake_token\n"
2700 "Timeouts: 5 1 10\n",
2701 "{}"),
2702 // IsDirectory is checking if the path exists as an object.
2703 new FakeHttpRequest(
2704 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2705 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2706 "Auth Token: fake_token\n"
2707 "Timeouts: 5 1 10\n",
2708 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2709 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2710 // Copying to the new location.
2711 new FakeHttpRequest(
2712 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2713 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
2714 "Auth Token: fake_token\n"
2715 "Post: yes\n"
2716 "Timeouts: 5 1 10\n",
2717 "{\"done\": true}"),
2718 // Deleting the original file.
2719 new FakeHttpRequest(
2720 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2721 "path%2Fsrc.txt\n"
2722 "Auth Token: fake_token\n"
2723 "Timeouts: 5 1 10\n"
2724 "Delete: yes\n",
2725 ""),
2726 new FakeHttpRequest(
2727 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2728 "path%2Fdst.txt?fields=size%2Cgeneration%2Cupdated\n"
2729 "Auth Token: fake_token\n"
2730 "Timeouts: 5 1 10\n",
2731 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2732 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
2733 GcsFileSystem fs(
2734 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2735 std::unique_ptr<HttpRequest::Factory>(
2736 new FakeHttpRequestFactory(&requests)),
2737 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2738 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
2739 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2740 0 /* matching paths cache max entries */, kTestRetryConfig,
2741 kTestTimeoutConfig, *kAllowedLocationsDefault,
2742 nullptr /* gcs additional header */, false /* compose append */);
2743 // Do an initial stat of the destination file to load their contents into the
2744 // stat cache.
2745 FileStatistics stat_before_renaming;
2746 TF_EXPECT_OK(
2747 fs.Stat("gs://bucket/path/dst.txt", nullptr, &stat_before_renaming));
2748 EXPECT_EQ(1000, stat_before_renaming.length);
2749
2750 TF_EXPECT_OK(fs.RenameFile("gs://bucket/path/src.txt",
2751 "gs://bucket/path/dst.txt", nullptr));
2752
2753 FileStatistics stat_after_renaming;
2754 TF_EXPECT_OK(
2755 fs.Stat("gs://bucket/path/dst.txt", nullptr, &stat_after_renaming));
2756 EXPECT_EQ(1010, stat_after_renaming.length);
2757 }
2758
2759 /// Tests the scenario when deletion returns a failure, but actually succeeds.
TEST(GcsFileSystemTest,RenameFile_Object_DeletionRetried)2760 TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) {
2761 std::vector<HttpRequest*> requests(
2762 {// IsDirectory is checking whether there are children objects.
2763 new FakeHttpRequest(
2764 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2765 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
2766 "&maxResults=1\n"
2767 "Auth Token: fake_token\n"
2768 "Timeouts: 5 1 10\n",
2769 "{}"),
2770 // IsDirectory is checking if the path exists as an object.
2771 new FakeHttpRequest(
2772 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2773 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2774 "Auth Token: fake_token\n"
2775 "Timeouts: 5 1 10\n",
2776 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2777 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2778 // Copying to the new location.
2779 new FakeHttpRequest(
2780 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2781 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
2782 "Auth Token: fake_token\n"
2783 "Post: yes\n"
2784 "Timeouts: 5 1 10\n",
2785 "{\"done\": true}"),
2786 // Deleting the original file - the deletion returns a failure.
2787 new FakeHttpRequest(
2788 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2789 "path%2Fsrc.txt\n"
2790 "Auth Token: fake_token\n"
2791 "Timeouts: 5 1 10\n"
2792 "Delete: yes\n",
2793 "", errors::Unavailable("503"), 503),
2794 // Deleting the original file again - the deletion returns NOT_FOUND.
2795 new FakeHttpRequest(
2796 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2797 "path%2Fsrc.txt\n"
2798 "Auth Token: fake_token\n"
2799 "Timeouts: 5 1 10\n"
2800 "Delete: yes\n",
2801 "", errors::NotFound("404"), 404)});
2802 GcsFileSystem fs(
2803 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2804 std::unique_ptr<HttpRequest::Factory>(
2805 new FakeHttpRequestFactory(&requests)),
2806 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2807 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2808 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2809 0 /* matching paths cache max entries */, kTestRetryConfig,
2810 kTestTimeoutConfig, *kAllowedLocationsDefault,
2811 nullptr /* gcs additional header */, false /* compose append */);
2812
2813 TF_EXPECT_OK(fs.RenameFile("gs://bucket/path/src.txt",
2814 "gs://bucket/path/dst.txt", nullptr));
2815 }
2816
2817 /// Tests the case when rewrite couldn't complete in one RPC.
TEST(GcsFileSystemTest,RenameFile_Object_Incomplete)2818 TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) {
2819 std::vector<HttpRequest*> requests(
2820 {// IsDirectory is checking whether there are children objects.
2821 new FakeHttpRequest(
2822 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2823 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
2824 "&maxResults=1\n"
2825 "Auth Token: fake_token\n"
2826 "Timeouts: 5 1 10\n",
2827 "{}"),
2828 // IsDirectory is checking if the path exists as an object.
2829 new FakeHttpRequest(
2830 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2831 "path%2Fsrc.txt?fields=size%2Cgeneration%2Cupdated\n"
2832 "Auth Token: fake_token\n"
2833 "Timeouts: 5 1 10\n",
2834 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2835 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
2836 // Copying to the new location.
2837 new FakeHttpRequest(
2838 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2839 "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
2840 "Auth Token: fake_token\n"
2841 "Post: yes\n"
2842 "Timeouts: 5 1 10\n",
2843 "{\"done\": false}")});
2844 GcsFileSystem fs(
2845 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2846 std::unique_ptr<HttpRequest::Factory>(
2847 new FakeHttpRequestFactory(&requests)),
2848 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2849 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2850 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2851 0 /* matching paths cache max entries */, kTestRetryConfig,
2852 kTestTimeoutConfig, *kAllowedLocationsDefault,
2853 nullptr /* gcs additional header */, false /* compose append */);
2854
2855 EXPECT_TRUE(errors::IsUnimplemented(fs.RenameFile(
2856 "gs://bucket/path/src.txt", "gs://bucket/path/dst.txt", nullptr)));
2857 }
2858
TEST(GcsFileSystemTest,Stat_Object)2859 TEST(GcsFileSystemTest, Stat_Object) {
2860 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2861 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2862 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
2863 "Auth Token: fake_token\n"
2864 "Timeouts: 5 1 10\n",
2865 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
2866 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
2867 GcsFileSystem fs(
2868 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2869 std::unique_ptr<HttpRequest::Factory>(
2870 new FakeHttpRequestFactory(&requests)),
2871 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2872 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2873 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2874 0 /* matching paths cache max entries */, kTestRetryConfig,
2875 kTestTimeoutConfig, *kAllowedLocationsDefault,
2876 nullptr /* gcs additional header */, false /* compose append */);
2877
2878 FileStatistics stat;
2879 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat));
2880 EXPECT_EQ(1010, stat.length);
2881 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
2882 EXPECT_FALSE(stat.is_directory);
2883 }
2884
TEST(GcsFileSystemTest,Stat_Folder)2885 TEST(GcsFileSystemTest, Stat_Folder) {
2886 std::vector<HttpRequest*> requests(
2887 {new FakeHttpRequest(
2888 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2889 "subfolder?fields=size%2Cgeneration%2Cupdated\n"
2890 "Auth Token: fake_token\n"
2891 "Timeouts: 5 1 10\n",
2892 "", errors::NotFound("404"), 404),
2893 new FakeHttpRequest(
2894 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2895 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
2896 "&maxResults=1\n"
2897 "Auth Token: fake_token\n"
2898 "Timeouts: 5 1 10\n",
2899 "{\"items\": [ "
2900 " { \"name\": \"subfolder/\" }]}")});
2901 GcsFileSystem fs(
2902 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2903 std::unique_ptr<HttpRequest::Factory>(
2904 new FakeHttpRequestFactory(&requests)),
2905 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2906 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2907 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2908 0 /* matching paths cache max entries */, kTestRetryConfig,
2909 kTestTimeoutConfig, *kAllowedLocationsDefault,
2910 nullptr /* gcs additional header */, false /* compose append */);
2911
2912 FileStatistics stat;
2913 TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", nullptr, &stat));
2914 EXPECT_EQ(0, stat.length);
2915 EXPECT_EQ(0, stat.mtime_nsec);
2916 EXPECT_TRUE(stat.is_directory);
2917 }
2918
TEST(GcsFileSystemTest,Stat_ObjectOrFolderNotFound)2919 TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) {
2920 std::vector<HttpRequest*> requests(
2921 {new FakeHttpRequest(
2922 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
2923 "path?fields=size%2Cgeneration%2Cupdated\n"
2924 "Auth Token: fake_token\n"
2925 "Timeouts: 5 1 10\n",
2926 "", errors::NotFound("404"), 404),
2927 new FakeHttpRequest(
2928 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
2929 "fields=items%2Fname%2CnextPageToken&prefix=path%2F"
2930 "&maxResults=1\n"
2931 "Auth Token: fake_token\n"
2932 "Timeouts: 5 1 10\n",
2933 "{}")});
2934 GcsFileSystem fs(
2935 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2936 std::unique_ptr<HttpRequest::Factory>(
2937 new FakeHttpRequestFactory(&requests)),
2938 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2939 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2940 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2941 0 /* matching paths cache max entries */, kTestRetryConfig,
2942 kTestTimeoutConfig, *kAllowedLocationsDefault,
2943 nullptr /* gcs additional header */, false /* compose append */);
2944
2945 FileStatistics stat;
2946 EXPECT_EQ(error::Code::NOT_FOUND,
2947 fs.Stat("gs://bucket/path", nullptr, &stat).code());
2948 }
2949
TEST(GcsFileSystemTest,Stat_Bucket)2950 TEST(GcsFileSystemTest, Stat_Bucket) {
2951 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2952 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
2953 "Auth Token: fake_token\n"
2954 "Timeouts: 5 1 10\n",
2955 "{}")});
2956 GcsFileSystem fs(
2957 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2958 std::unique_ptr<HttpRequest::Factory>(
2959 new FakeHttpRequestFactory(&requests)),
2960 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2961 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2962 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2963 0 /* matching paths cache max entries */, kTestRetryConfig,
2964 kTestTimeoutConfig, *kAllowedLocationsDefault,
2965 nullptr /* gcs additional header */, false /* compose append */);
2966
2967 FileStatistics stat;
2968 TF_EXPECT_OK(fs.Stat("gs://bucket/", nullptr, &stat));
2969 EXPECT_EQ(0, stat.length);
2970 EXPECT_EQ(0, stat.mtime_nsec);
2971 EXPECT_TRUE(stat.is_directory);
2972 }
2973
TEST(GcsFileSystemTest,Stat_BucketNotFound)2974 TEST(GcsFileSystemTest, Stat_BucketNotFound) {
2975 std::vector<HttpRequest*> requests({new FakeHttpRequest(
2976 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
2977 "Auth Token: fake_token\n"
2978 "Timeouts: 5 1 10\n",
2979 "", errors::NotFound("404"), 404)});
2980 GcsFileSystem fs(
2981 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
2982 std::unique_ptr<HttpRequest::Factory>(
2983 new FakeHttpRequestFactory(&requests)),
2984 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
2985 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
2986 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
2987 0 /* matching paths cache max entries */, kTestRetryConfig,
2988 kTestTimeoutConfig, *kAllowedLocationsDefault,
2989 nullptr /* gcs additional header */, false /* compose append */);
2990
2991 FileStatistics stat;
2992 EXPECT_EQ(error::Code::NOT_FOUND,
2993 fs.Stat("gs://bucket/", nullptr, &stat).code());
2994 }
2995
TEST(GcsFileSystemTest,Stat_Cache)2996 TEST(GcsFileSystemTest, Stat_Cache) {
2997 std::vector<HttpRequest*> requests(
2998 {new FakeHttpRequest(
2999 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3000 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
3001 "Auth Token: fake_token\n"
3002 "Timeouts: 5 1 10\n",
3003 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
3004 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3005 new FakeHttpRequest(
3006 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3007 "subfolder%2F?fields=size%2Cgeneration%2Cupdated\n"
3008 "Auth Token: fake_token\n"
3009 "Timeouts: 5 1 10\n",
3010 "", errors::NotFound("404"), 404),
3011 new FakeHttpRequest(
3012 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3013 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
3014 "&maxResults=1\n"
3015 "Auth Token: fake_token\n"
3016 "Timeouts: 5 1 10\n",
3017 "{\"items\": [ "
3018 " { \"name\": \"subfolder/\" }]}")});
3019 GcsFileSystem fs(
3020 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3021 std::unique_ptr<HttpRequest::Factory>(
3022 new FakeHttpRequestFactory(&requests)),
3023 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3024 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
3025 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3026 0 /* matching paths cache max entries */, kTestRetryConfig,
3027 kTestTimeoutConfig, *kAllowedLocationsDefault,
3028 nullptr /* gcs additional header */, false /* compose append */);
3029
3030 // Repeated calls to fs.Stat on these paths should not lead to any additional
3031 // HTTP requests to GCS.
3032 for (int i = 0; i < 10; i++) {
3033 FileStatistics stat;
3034 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat));
3035 EXPECT_EQ(1010, stat.length);
3036 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
3037 EXPECT_FALSE(stat.is_directory);
3038 TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder/", nullptr, &stat));
3039 EXPECT_EQ(0, stat.length);
3040 EXPECT_EQ(0, stat.mtime_nsec);
3041 EXPECT_TRUE(stat.is_directory);
3042 }
3043 }
3044
TEST(GcsFileSystemTest,Stat_Cache_Flush)3045 TEST(GcsFileSystemTest, Stat_Cache_Flush) {
3046 std::vector<HttpRequest*> requests(
3047 {new FakeHttpRequest(
3048 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3049 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
3050 "Auth Token: fake_token\n"
3051 "Timeouts: 5 1 10\n",
3052 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
3053 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3054 new FakeHttpRequest(
3055 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3056 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
3057 "Auth Token: fake_token\n"
3058 "Timeouts: 5 1 10\n",
3059 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
3060 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
3061 GcsFileSystem fs(
3062 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3063 std::unique_ptr<HttpRequest::Factory>(
3064 new FakeHttpRequestFactory(&requests)),
3065 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3066 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
3067 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3068 0 /* matching paths cache max entries */, kTestRetryConfig,
3069 kTestTimeoutConfig, *kAllowedLocationsDefault,
3070 nullptr /* gcs additional header */, false /* compose append */);
3071 // There should be a single HTTP request to GCS for fs.Stat in this loop.
3072 for (int i = 0; i < 10; i++) {
3073 FileStatistics stat;
3074 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat));
3075 EXPECT_EQ(1010, stat.length);
3076 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
3077 EXPECT_FALSE(stat.is_directory);
3078 }
3079 // After flushing caches, there should be a second request to GCS for fs.Stat.
3080 fs.FlushCaches(nullptr);
3081 for (int i = 0; i < 10; i++) {
3082 FileStatistics stat;
3083 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat));
3084 EXPECT_EQ(1010, stat.length);
3085 EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
3086 EXPECT_FALSE(stat.is_directory);
3087 }
3088 }
3089
TEST(GcsFileSystemTest,Stat_FilenameEndingWithSlash)3090 TEST(GcsFileSystemTest, Stat_FilenameEndingWithSlash) {
3091 std::vector<HttpRequest*> requests({new FakeHttpRequest(
3092 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3093 "dir%2F?fields=size%2Cgeneration%2Cupdated\n"
3094 "Auth Token: fake_token\n"
3095 "Timeouts: 5 1 10\n",
3096 strings::StrCat("{\"size\": \"5\",\"generation\": \"1\","
3097 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
3098 GcsFileSystem fs(
3099 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3100 std::unique_ptr<HttpRequest::Factory>(
3101 new FakeHttpRequestFactory(&requests)),
3102 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3103 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3104 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3105 0 /* matching paths cache max entries */, kTestRetryConfig,
3106 kTestTimeoutConfig, *kAllowedLocationsDefault,
3107 nullptr /* gcs additional header */, false /* compose append */);
3108
3109 FileStatistics stat;
3110 TF_EXPECT_OK(fs.Stat("gs://bucket/dir/", nullptr, &stat));
3111 EXPECT_EQ(5, stat.length);
3112 EXPECT_TRUE(stat.is_directory);
3113 }
3114
TEST(GcsFileSystemTest,IsDirectory_NotFound)3115 TEST(GcsFileSystemTest, IsDirectory_NotFound) {
3116 std::vector<HttpRequest*> requests(
3117 {new FakeHttpRequest(
3118 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3119 "fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F"
3120 "&maxResults=1\n"
3121 "Auth Token: fake_token\n"
3122 "Timeouts: 5 1 10\n",
3123 "{}"),
3124 new FakeHttpRequest(
3125 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3126 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
3127 "Auth Token: fake_token\n"
3128 "Timeouts: 5 1 10\n",
3129 "", errors::NotFound("404"), 404)});
3130 GcsFileSystem fs(
3131 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3132 std::unique_ptr<HttpRequest::Factory>(
3133 new FakeHttpRequestFactory(&requests)),
3134 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3135 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3136 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3137 0 /* matching paths cache max entries */, kTestRetryConfig,
3138 kTestTimeoutConfig, *kAllowedLocationsDefault,
3139 nullptr /* gcs additional header */, false /* compose append */);
3140
3141 EXPECT_EQ(error::Code::NOT_FOUND,
3142 fs.IsDirectory("gs://bucket/file.txt", nullptr).code());
3143 }
3144
TEST(GcsFileSystemTest,IsDirectory_NotDirectoryButObject)3145 TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) {
3146 std::vector<HttpRequest*> requests(
3147 {new FakeHttpRequest(
3148 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3149 "fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F"
3150 "&maxResults=1\n"
3151 "Auth Token: fake_token\n"
3152 "Timeouts: 5 1 10\n",
3153 "{}"),
3154 new FakeHttpRequest(
3155 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3156 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
3157 "Auth Token: fake_token\n"
3158 "Timeouts: 5 1 10\n",
3159 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
3160 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
3161 GcsFileSystem fs(
3162 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3163 std::unique_ptr<HttpRequest::Factory>(
3164 new FakeHttpRequestFactory(&requests)),
3165 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3166 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3167 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3168 0 /* matching paths cache max entries */, kTestRetryConfig,
3169 kTestTimeoutConfig, *kAllowedLocationsDefault,
3170 nullptr /* gcs additional header */, false /* compose append */);
3171
3172 EXPECT_EQ(error::Code::FAILED_PRECONDITION,
3173 fs.IsDirectory("gs://bucket/file.txt", nullptr).code());
3174 }
3175
TEST(GcsFileSystemTest,IsDirectory_Yes)3176 TEST(GcsFileSystemTest, IsDirectory_Yes) {
3177 std::vector<HttpRequest*> requests(
3178 {new FakeHttpRequest(
3179 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3180 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
3181 "&maxResults=1\n"
3182 "Auth Token: fake_token\n"
3183 "Timeouts: 5 1 10\n",
3184 "{\"items\": [{\"name\": \"subfolder/\"}]}"),
3185 new FakeHttpRequest(
3186 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3187 "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
3188 "&maxResults=1\n"
3189 "Auth Token: fake_token\n"
3190 "Timeouts: 5 1 10\n",
3191 "{\"items\": [{\"name\": \"subfolder/\"}]}")});
3192 GcsFileSystem fs(
3193 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3194 std::unique_ptr<HttpRequest::Factory>(
3195 new FakeHttpRequestFactory(&requests)),
3196 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3197 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3198 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3199 0 /* matching paths cache max entries */, kTestRetryConfig,
3200 kTestTimeoutConfig, *kAllowedLocationsDefault,
3201 nullptr /* gcs additional header */, false /* compose append */);
3202
3203 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder", nullptr));
3204 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/", nullptr));
3205 }
3206
TEST(GcsFileSystemTest,IsDirectory_Bucket)3207 TEST(GcsFileSystemTest, IsDirectory_Bucket) {
3208 std::vector<HttpRequest*> requests(
3209 {new FakeHttpRequest(
3210 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
3211 "Auth Token: fake_token\n"
3212 "Timeouts: 5 1 10\n",
3213 "{}"),
3214 new FakeHttpRequest(
3215 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
3216 "Auth Token: fake_token\n"
3217 "Timeouts: 5 1 10\n",
3218 "{}")});
3219 GcsFileSystem fs(
3220 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3221 std::unique_ptr<HttpRequest::Factory>(
3222 new FakeHttpRequestFactory(&requests)),
3223 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3224 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3225 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3226 0 /* matching paths cache max entries */, kTestRetryConfig,
3227 kTestTimeoutConfig, *kAllowedLocationsDefault,
3228 nullptr /* gcs additional header */, false /* compose append */);
3229
3230 TF_EXPECT_OK(fs.IsDirectory("gs://bucket", nullptr));
3231 TF_EXPECT_OK(fs.IsDirectory("gs://bucket/", nullptr));
3232 }
3233
TEST(GcsFileSystemTest,IsDirectory_BucketNotFound)3234 TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) {
3235 std::vector<HttpRequest*> requests({new FakeHttpRequest(
3236 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
3237 "Auth Token: fake_token\n"
3238 "Timeouts: 5 1 10\n",
3239 "", errors::NotFound("404"), 404)});
3240 GcsFileSystem fs(
3241 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3242 std::unique_ptr<HttpRequest::Factory>(
3243 new FakeHttpRequestFactory(&requests)),
3244 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3245 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3246 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3247 0 /* matching paths cache max entries */, kTestRetryConfig,
3248 kTestTimeoutConfig, *kAllowedLocationsDefault,
3249 nullptr /* gcs additional header */, false /* compose append */);
3250
3251 EXPECT_EQ(error::Code::NOT_FOUND,
3252 fs.IsDirectory("gs://bucket/", nullptr).code());
3253 }
3254
TEST(GcsFileSystemTest,CreateDir_Folder)3255 TEST(GcsFileSystemTest, CreateDir_Folder) {
3256 std::vector<HttpRequest*> requests(
3257
3258 {
3259 // File doesn't exist.
3260 new FakeHttpRequest(
3261 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3262 "subpath%2F?fields=size%2Cgeneration%2Cupdated\n"
3263 "Auth Token: fake_token\n"
3264 "Timeouts: 5 1 10\n",
3265 "{}"),
3266 // Simple upload.
3267 new FakeHttpRequest(
3268 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
3269 "uploadType=media&name=subpath%2F&ifGenerationMatch=0\n"
3270 "Auth Token: fake_token\n"
3271 "Post: yes\n"
3272 "Timeouts: 5 1 10\n",
3273 ""),
3274 // File exists.
3275 new FakeHttpRequest(
3276 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3277 "subpath%2F?fields=size%2Cgeneration%2Cupdated\n"
3278 "Auth Token: fake_token\n"
3279 "Timeouts: 5 1 10\n",
3280 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
3281 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3282 // File doesn't exist again.
3283 new FakeHttpRequest(
3284 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3285 "subpath%2F?fields=size%2Cgeneration%2Cupdated\n"
3286 "Auth Token: fake_token\n"
3287 "Timeouts: 5 1 10\n",
3288 "{}"),
3289 // Simulate object uploaded in between.
3290 new FakeHttpRequest(
3291 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
3292 "uploadType=media&name=subpath%2F&ifGenerationMatch=0\n"
3293 "Auth Token: fake_token\n"
3294 "Post: yes\n"
3295 "Timeouts: 5 1 10\n",
3296 "", errors::FailedPrecondition("412"), 412),
3297 });
3298 GcsFileSystem fs(
3299 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3300 std::unique_ptr<HttpRequest::Factory>(
3301 new FakeHttpRequestFactory(&requests)),
3302 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3303 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3304 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3305 0 /* matching paths cache max entries */, kTestRetryConfig,
3306 kTestTimeoutConfig, *kAllowedLocationsDefault,
3307 nullptr /* gcs additional header */, false /* compose append */);
3308
3309 TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath", nullptr));
3310 // Check that when GCS returns the object already exists return that the
3311 // directory already exists.
3312 EXPECT_EQ(errors::AlreadyExists("gs://bucket/subpath"),
3313 fs.CreateDir("gs://bucket/subpath", nullptr));
3314 // Check that when GCS returns the object already has a version (failed
3315 // precondition) return directory already exists.
3316 EXPECT_EQ(errors::AlreadyExists("gs://bucket/subpath"),
3317 fs.CreateDir("gs://bucket/subpath", nullptr));
3318 }
3319
TEST(GcsFileSystemTest,CreateDir_Bucket)3320 TEST(GcsFileSystemTest, CreateDir_Bucket) {
3321 std::vector<HttpRequest*> requests(
3322 {new FakeHttpRequest(
3323 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
3324 "Auth Token: fake_token\n"
3325 "Timeouts: 5 1 10\n",
3326 ""),
3327 new FakeHttpRequest(
3328 "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
3329 "Auth Token: fake_token\n"
3330 "Timeouts: 5 1 10\n",
3331 "")});
3332 GcsFileSystem fs(
3333 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3334 std::unique_ptr<HttpRequest::Factory>(
3335 new FakeHttpRequestFactory(&requests)),
3336 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3337 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3338 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3339 0 /* matching paths cache max entries */, kTestRetryConfig,
3340 kTestTimeoutConfig, *kAllowedLocationsDefault,
3341 nullptr /* gcs additional header */, false /* compose append */);
3342
3343 TF_EXPECT_OK(fs.CreateDir("gs://bucket/", nullptr));
3344 TF_EXPECT_OK(fs.CreateDir("gs://bucket", nullptr));
3345 }
3346
TEST(GcsFileSystemTest,DeleteRecursively_Ok)3347 TEST(GcsFileSystemTest, DeleteRecursively_Ok) {
3348 std::vector<HttpRequest*> requests(
3349 {// IsDirectory is checking whether there are children objects.
3350 new FakeHttpRequest(
3351 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3352 "fields=items%2Fname%2CnextPageToken&prefix=path%2F"
3353 "&maxResults=1\n"
3354 "Auth Token: fake_token\n"
3355 "Timeouts: 5 1 10\n",
3356 "{\"items\": [ "
3357 " { \"name\": \"path/file1.txt\" }]}"),
3358 // GetChildren recursively.
3359 new FakeHttpRequest(
3360 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3361 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
3362 "Auth Token: fake_token\n"
3363 "Timeouts: 5 1 10\n",
3364 "{\"items\": [ "
3365 " { \"name\": \"path/\" }," // The current directory's marker.
3366 " { \"name\": \"path/file1.txt\" },"
3367 " { \"name\": \"path/subpath/file2.txt\" },"
3368 " { \"name\": \"path/file3.txt\" }]}"),
3369 // Delete the current directory's marker.
3370 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3371 "/bucket/o/path%2F\n"
3372 "Auth Token: fake_token\n"
3373 "Timeouts: 5 1 10\n"
3374 "Delete: yes\n",
3375 ""),
3376 // Delete the object - fails and will be retried.
3377 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3378 "/bucket/o/path%2Ffile1.txt\n"
3379 "Auth Token: fake_token\n"
3380 "Timeouts: 5 1 10\n"
3381 "Delete: yes\n",
3382 "", errors::Unavailable("500"), 500),
3383 // Delete the object again.
3384 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3385 "/bucket/o/path%2Ffile1.txt\n"
3386 "Auth Token: fake_token\n"
3387 "Timeouts: 5 1 10\n"
3388 "Delete: yes\n",
3389 ""),
3390 // Delete the object.
3391 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3392 "/bucket/o/path%2Fsubpath%2Ffile2.txt\n"
3393 "Auth Token: fake_token\n"
3394 "Timeouts: 5 1 10\n"
3395 "Delete: yes\n",
3396 ""),
3397 // Delete the object.
3398 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3399 "/bucket/o/path%2Ffile3.txt\n"
3400 "Auth Token: fake_token\n"
3401 "Timeouts: 5 1 10\n"
3402 "Delete: yes\n",
3403 "")});
3404 GcsFileSystem fs(
3405 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3406 std::unique_ptr<HttpRequest::Factory>(
3407 new FakeHttpRequestFactory(&requests)),
3408 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3409 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3410 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3411 0 /* matching paths cache max entries */, kTestRetryConfig,
3412 kTestTimeoutConfig, *kAllowedLocationsDefault,
3413 nullptr /* gcs additional header */, false /* compose append */);
3414
3415 int64_t undeleted_files, undeleted_dirs;
3416 TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", nullptr,
3417 &undeleted_files, &undeleted_dirs));
3418 EXPECT_EQ(0, undeleted_files);
3419 EXPECT_EQ(0, undeleted_dirs);
3420 }
3421
TEST(GcsFileSystemTest,DeleteRecursively_DeletionErrors)3422 TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) {
3423 std::vector<HttpRequest*> requests(
3424 {// IsDirectory is checking whether there are children objects.
3425 new FakeHttpRequest(
3426 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3427 "fields=items%2Fname%2CnextPageToken&prefix=path%2F"
3428 "&maxResults=1\n"
3429 "Auth Token: fake_token\n"
3430 "Timeouts: 5 1 10\n",
3431 "{\"items\": [ "
3432 " { \"name\": \"path/file1.txt\" }]}"),
3433 // Calling GetChildren recursively.
3434 new FakeHttpRequest(
3435 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3436 "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
3437 "Auth Token: fake_token\n"
3438 "Timeouts: 5 1 10\n",
3439 "{\"items\": [ "
3440 " { \"name\": \"path/file1.txt\" },"
3441 " { \"name\": \"path/subpath/\" },"
3442 " { \"name\": \"path/subpath/file2.txt\" },"
3443 " { \"name\": \"path/file3.txt\" }]}"),
3444 // Deleting the object.
3445 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3446 "/bucket/o/path%2Ffile1.txt\n"
3447 "Auth Token: fake_token\n"
3448 "Timeouts: 5 1 10\n"
3449 "Delete: yes\n",
3450 ""),
3451 // Deleting the directory marker gs://bucket/path/ - fails with 404.
3452 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3453 "/bucket/o/path%2Fsubpath%2F\n"
3454 "Auth Token: fake_token\n"
3455 "Timeouts: 5 1 10\n"
3456 "Delete: yes\n",
3457 "", errors::NotFound("404"), 404),
3458 // Checking if gs://bucket/path/subpath/ is a folder - it is.
3459 new FakeHttpRequest(
3460 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3461 "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F"
3462 "&maxResults=1\n"
3463 "Auth Token: fake_token\n"
3464 "Timeouts: 5 1 10\n",
3465 strings::StrCat("{\"items\": [ "
3466 " { \"name\": \"path/subpath/\" }]}")),
3467 // Deleting the object gs://bucket/path/subpath/file2.txt
3468 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3469 "/bucket/o/path%2Fsubpath%2Ffile2.txt\n"
3470 "Auth Token: fake_token\n"
3471 "Timeouts: 5 1 10\n"
3472 "Delete: yes\n",
3473 ""),
3474 // Deleting the object s://bucket/path/file3.txt - fails with 404.
3475 new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
3476 "/bucket/o/path%2Ffile3.txt\n"
3477 "Auth Token: fake_token\n"
3478 "Timeouts: 5 1 10\n"
3479 "Delete: yes\n",
3480 "", errors::NotFound("404"), 404),
3481 // Checking if gs://bucket/path/file3.txt/ is a folder - it's not.
3482 new FakeHttpRequest(
3483 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3484 "fields=items%2Fname%2CnextPageToken&prefix=path%2Ffile3.txt%2F"
3485 "&maxResults=1\n"
3486 "Auth Token: fake_token\n"
3487 "Timeouts: 5 1 10\n",
3488 "{}"),
3489 // Checking if gs://bucket/path/file3.txt is an object - fails with 404.
3490 new FakeHttpRequest(
3491 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3492 "path%2Ffile3.txt?fields=size%2Cgeneration%2Cupdated\n"
3493 "Auth Token: fake_token\n"
3494 "Timeouts: 5 1 10\n",
3495 "", errors::NotFound("404"), 404)});
3496
3497 GcsFileSystem fs(
3498 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3499 std::unique_ptr<HttpRequest::Factory>(
3500 new FakeHttpRequestFactory(&requests)),
3501 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3502 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3503 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3504 0 /* matching paths cache max entries */, kTestRetryConfig,
3505 kTestTimeoutConfig, *kAllowedLocationsDefault,
3506 nullptr /* gcs additional header */, false /* compose append */);
3507
3508 int64_t undeleted_files, undeleted_dirs;
3509 TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", nullptr,
3510 &undeleted_files, &undeleted_dirs));
3511 EXPECT_EQ(1, undeleted_files);
3512 EXPECT_EQ(1, undeleted_dirs);
3513 }
3514
TEST(GcsFileSystemTest,DeleteRecursively_NotAFolder)3515 TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) {
3516 std::vector<HttpRequest*> requests(
3517 {// IsDirectory is checking whether there are children objects.
3518 new FakeHttpRequest(
3519 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
3520 "fields=items%2Fname%2CnextPageToken&prefix=path%2F"
3521 "&maxResults=1\n"
3522 "Auth Token: fake_token\n"
3523 "Timeouts: 5 1 10\n",
3524 "{}"),
3525 // IsDirectory is checking if the path exists as an object.
3526 new FakeHttpRequest(
3527 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3528 "path?fields=size%2Cgeneration%2Cupdated\n"
3529 "Auth Token: fake_token\n"
3530 "Timeouts: 5 1 10\n",
3531 "", errors::NotFound("404"), 404)});
3532 GcsFileSystem fs(
3533 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3534 std::unique_ptr<HttpRequest::Factory>(
3535 new FakeHttpRequestFactory(&requests)),
3536 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3537 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3538 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3539 0 /* matching paths cache max entries */, kTestRetryConfig,
3540 kTestTimeoutConfig, *kAllowedLocationsDefault,
3541 nullptr /* gcs additional header */, false /* compose append */);
3542
3543 int64_t undeleted_files, undeleted_dirs;
3544 EXPECT_EQ(error::Code::NOT_FOUND,
3545 fs.DeleteRecursively("gs://bucket/path", nullptr, &undeleted_files,
3546 &undeleted_dirs)
3547 .code());
3548 EXPECT_EQ(0, undeleted_files);
3549 EXPECT_EQ(1, undeleted_dirs);
3550 }
3551
TEST(GcsFileSystemTest,NoConstraintsEnvironmentVariableTest)3552 TEST(GcsFileSystemTest, NoConstraintsEnvironmentVariableTest) {
3553 unsetenv("GCS_ALLOWED_BUCKET_LOCATIONS");
3554 // No constraints
3555 GcsFileSystem fs1;
3556 EXPECT_EQ(*kAllowedLocationsDefault, fs1.allowed_locations());
3557
3558 // Cover cache initialization code, any uninitialized cache will cause this to
3559 // fail
3560 fs1.FlushCaches(nullptr);
3561 }
3562
TEST(GcsFileSystemTest,BucketLocationConstraintEnvironmentVariableTest)3563 TEST(GcsFileSystemTest, BucketLocationConstraintEnvironmentVariableTest) {
3564 unsetenv("GCS_ALLOWED_BUCKET_LOCATIONS");
3565 setenv("GCS_ALLOWED_BUCKET_LOCATIONS", "auto", 1);
3566 GcsFileSystem fs1;
3567 EXPECT_EQ(*kAllowedLocationsAuto, fs1.allowed_locations());
3568
3569 setenv("GCS_ALLOWED_BUCKET_LOCATIONS", "CUSTOM,list", 1);
3570 GcsFileSystem fs2;
3571 EXPECT_EQ(std::unordered_set<string>({"custom", "list"}),
3572 fs2.allowed_locations());
3573 }
3574
TEST(GcsFileSystemTest,AdditionalRequestHeaderTest)3575 TEST(GcsFileSystemTest, AdditionalRequestHeaderTest) {
3576 GcsFileSystem fs1;
3577 EXPECT_EQ("", fs1.additional_header_name());
3578 EXPECT_EQ("", fs1.additional_header_value());
3579
3580 setenv("GCS_ADDITIONAL_REQUEST_HEADER",
3581 "X-Add-Header:My Additional Header Value", 1);
3582 GcsFileSystem fs2;
3583 EXPECT_EQ("X-Add-Header", fs2.additional_header_name());
3584 EXPECT_EQ("My Additional Header Value", fs2.additional_header_value());
3585
3586 setenv("GCS_ADDITIONAL_REQUEST_HEADER", "Someinvalidheadervalue", 1);
3587 GcsFileSystem fs3;
3588 EXPECT_EQ("", fs3.additional_header_name());
3589 EXPECT_EQ("", fs3.additional_header_value());
3590
3591 setenv("GCS_ADDITIONAL_REQUEST_HEADER", ":thisisinvalid", 1);
3592 GcsFileSystem fs4;
3593 EXPECT_EQ("", fs4.additional_header_name());
3594 EXPECT_EQ("", fs4.additional_header_value());
3595
3596 setenv("GCS_ADDITIONAL_REQUEST_HEADER", "soisthis:", 1);
3597 GcsFileSystem fs5;
3598 EXPECT_EQ("", fs5.additional_header_name());
3599 EXPECT_EQ("", fs5.additional_header_value());
3600
3601 setenv("GCS_ADDITIONAL_REQUEST_HEADER", "a:b", 1);
3602 GcsFileSystem fs6;
3603 EXPECT_EQ("a", fs6.additional_header_name());
3604 EXPECT_EQ("b", fs6.additional_header_value());
3605
3606 auto* add_header = new std::pair<const string, const string>(
3607 "mynewheader", "newheadercontents");
3608
3609 std::vector<HttpRequest*> requests(
3610 {// IsDirectory is checking whether there are children objects.
3611 new FakeHttpRequest("Uri: https://www.googleapis.com/fake\n"
3612 "Auth Token: fake_token\n"
3613 "Header mynewheader: newheadercontents\n"
3614 "Header Hello: world\n",
3615 "{}")});
3616 GcsFileSystem fs7(
3617 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3618 std::unique_ptr<HttpRequest::Factory>(
3619 new FakeHttpRequestFactory(&requests)),
3620 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3621 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3622 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3623 0 /* matching paths cache max entries */, kTestRetryConfig,
3624 kTestTimeoutConfig, *kAllowedLocationsDefault,
3625 add_header /* gcs additional header */, false /* compose append */);
3626
3627 std::unique_ptr<HttpRequest> request;
3628 TF_EXPECT_OK(fs7.CreateHttpRequest(&request));
3629 request->SetUri("https://www.googleapis.com/fake");
3630 request->AddHeader("Hello", "world");
3631 TF_EXPECT_OK(request->Send());
3632 }
3633
TEST(GcsFileSystemTest,OverrideCacheParameters)3634 TEST(GcsFileSystemTest, OverrideCacheParameters) {
3635 // Verify defaults are propagated correctly.
3636 setenv("GCS_READ_CACHE_BLOCK_SIZE_MB", "16", 1);
3637 setenv("GCS_READ_CACHE_MAX_SIZE_MB", "128", 1);
3638 GcsFileSystem fs1;
3639 EXPECT_EQ(16 * 1024 * 1024, fs1.block_size());
3640 EXPECT_EQ(128 * 1024 * 1024, fs1.max_bytes());
3641 EXPECT_EQ(0, fs1.max_staleness());
3642 EXPECT_EQ(120, fs1.timeouts().connect);
3643 EXPECT_EQ(60, fs1.timeouts().idle);
3644 EXPECT_EQ(3600, fs1.timeouts().metadata);
3645 EXPECT_EQ(3600, fs1.timeouts().read);
3646 EXPECT_EQ(3600, fs1.timeouts().write);
3647
3648 // Verify legacy readahead buffer override sets block size.
3649 unsetenv("GCS_READ_CACHE_BLOCK_SIZE_MB");
3650 setenv("GCS_READAHEAD_BUFFER_SIZE_BYTES", "123456789", 1);
3651 GcsFileSystem fs2;
3652 EXPECT_EQ(123456789L, fs2.block_size());
3653
3654 // Verify block size, max size, and max staleness overrides.
3655 setenv("GCS_READ_CACHE_BLOCK_SIZE_MB", "1", 1);
3656 setenv("GCS_READ_CACHE_MAX_SIZE_MB", "16", 1);
3657 setenv("GCS_READ_CACHE_MAX_STALENESS", "60", 1);
3658 GcsFileSystem fs3;
3659 EXPECT_EQ(1048576L, fs3.block_size());
3660 EXPECT_EQ(16 * 1024 * 1024, fs3.max_bytes());
3661 EXPECT_EQ(60, fs3.max_staleness());
3662
3663 // Verify StatCache and MatchingPathsCache overrides.
3664 setenv("GCS_STAT_CACHE_MAX_AGE", "60", 1);
3665 setenv("GCS_STAT_CACHE_MAX_ENTRIES", "32", 1);
3666 setenv("GCS_MATCHING_PATHS_CACHE_MAX_AGE", "30", 1);
3667 setenv("GCS_MATCHING_PATHS_CACHE_MAX_ENTRIES", "64", 1);
3668 GcsFileSystem fs4;
3669 EXPECT_EQ(60, fs4.stat_cache_max_age());
3670 EXPECT_EQ(32, fs4.stat_cache_max_entries());
3671 EXPECT_EQ(30, fs4.matching_paths_cache_max_age());
3672 EXPECT_EQ(64, fs4.matching_paths_cache_max_entries());
3673
3674 // Verify timeout overrides.
3675 setenv("GCS_REQUEST_CONNECTION_TIMEOUT_SECS", "10", 1);
3676 setenv("GCS_REQUEST_IDLE_TIMEOUT_SECS", "5", 1);
3677 setenv("GCS_METADATA_REQUEST_TIMEOUT_SECS", "20", 1);
3678 setenv("GCS_READ_REQUEST_TIMEOUT_SECS", "30", 1);
3679 setenv("GCS_WRITE_REQUEST_TIMEOUT_SECS", "40", 1);
3680 GcsFileSystem fs5;
3681 EXPECT_EQ(10, fs5.timeouts().connect);
3682 EXPECT_EQ(5, fs5.timeouts().idle);
3683 EXPECT_EQ(20, fs5.timeouts().metadata);
3684 EXPECT_EQ(30, fs5.timeouts().read);
3685 EXPECT_EQ(40, fs5.timeouts().write);
3686 }
3687
TEST(GcsFileSystemTest,CreateHttpRequest)3688 TEST(GcsFileSystemTest, CreateHttpRequest) {
3689 std::vector<HttpRequest*> requests(
3690 {// IsDirectory is checking whether there are children objects.
3691 new FakeHttpRequest("Uri: https://www.googleapis.com/fake\n"
3692 "Auth Token: fake_token\n"
3693 "Header Hello: world\n",
3694 "{}")});
3695 GcsFileSystem fs(
3696 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3697 std::unique_ptr<HttpRequest::Factory>(
3698 new FakeHttpRequestFactory(&requests)),
3699 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3700 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3701 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3702 0 /* matching paths cache max entries */, kTestRetryConfig,
3703 kTestTimeoutConfig, *kAllowedLocationsDefault,
3704 nullptr /* gcs additional header */, false /* compose append */);
3705
3706 std::unique_ptr<HttpRequest> request;
3707 TF_EXPECT_OK(fs.CreateHttpRequest(&request));
3708 request->SetUri("https://www.googleapis.com/fake");
3709 request->AddHeader("Hello", "world");
3710 TF_EXPECT_OK(request->Send());
3711 }
3712
3713 class TestGcsStats : public GcsStatsInterface {
3714 public:
Configure(GcsFileSystem * fs,GcsThrottle * throttle,const FileBlockCache * block_cache)3715 void Configure(GcsFileSystem* fs, GcsThrottle* throttle,
3716 const FileBlockCache* block_cache) override {
3717 CHECK(fs_ == nullptr);
3718 CHECK(throttle_ == nullptr);
3719 CHECK(block_cache_ == nullptr);
3720
3721 fs_ = fs;
3722 throttle_ = throttle;
3723 block_cache_ = block_cache;
3724 }
3725
RecordBlockLoadRequest(const string & file,size_t offset)3726 void RecordBlockLoadRequest(const string& file, size_t offset) override {
3727 block_load_request_file_ = file;
3728 }
3729
RecordBlockRetrieved(const string & file,size_t offset,size_t bytes_transferred)3730 void RecordBlockRetrieved(const string& file, size_t offset,
3731 size_t bytes_transferred) override {
3732 block_retrieved_file_ = file;
3733 block_retrieved_bytes_transferred_ = bytes_transferred;
3734 }
3735
RecordStatObjectRequest()3736 void RecordStatObjectRequest() override { stat_object_request_count_++; }
3737
HttpStats()3738 HttpRequest::RequestStats* HttpStats() override { return nullptr; }
3739
3740 GcsFileSystem* fs_ = nullptr;
3741 GcsThrottle* throttle_ = nullptr;
3742 const FileBlockCache* block_cache_ = nullptr;
3743
3744 string block_load_request_file_;
3745 string block_retrieved_file_;
3746 size_t block_retrieved_bytes_transferred_ = 0;
3747 int stat_object_request_count_ = 0;
3748 };
3749
TEST(GcsFileSystemTest,Stat_StatsRecording)3750 TEST(GcsFileSystemTest, Stat_StatsRecording) {
3751 std::vector<HttpRequest*> requests({new FakeHttpRequest(
3752 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3753 "file.txt?fields=size%2Cgeneration%2Cupdated\n"
3754 "Auth Token: fake_token\n"
3755 "Timeouts: 5 1 10\n",
3756 strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
3757 "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
3758 GcsFileSystem fs(
3759 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3760 std::unique_ptr<HttpRequest::Factory>(
3761 new FakeHttpRequestFactory(&requests)),
3762 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3763 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3764 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3765 0 /* matching paths cache max entries */, kTestRetryConfig,
3766 kTestTimeoutConfig, *kAllowedLocationsDefault,
3767 nullptr /* gcs additional header */, false /* compose append */);
3768
3769 TestGcsStats stats;
3770 fs.SetStats(&stats);
3771 EXPECT_EQ(stats.fs_, &fs);
3772
3773 FileStatistics stat;
3774 TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat));
3775 EXPECT_EQ(1, stats.stat_object_request_count_);
3776 }
3777
TEST(GcsFileSystemTest,NewRandomAccessFile_StatsRecording)3778 TEST(GcsFileSystemTest, NewRandomAccessFile_StatsRecording) {
3779 std::vector<HttpRequest*> requests({new FakeHttpRequest(
3780 "Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
3781 "Auth Token: fake_token\n"
3782 "Range: 0-5\n"
3783 "Timeouts: 5 1 20\n",
3784 "012345")});
3785 GcsFileSystem fs(
3786 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3787 std::unique_ptr<HttpRequest::Factory>(
3788 new FakeHttpRequestFactory(&requests)),
3789 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
3790 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
3791 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3792 0 /* matching paths cache max entries */, kTestRetryConfig,
3793 kTestTimeoutConfig, *kAllowedLocationsDefault,
3794 nullptr /* gcs additional header */, false /* compose append */);
3795
3796 TestGcsStats stats;
3797 fs.SetStats(&stats);
3798 EXPECT_EQ(stats.fs_, &fs);
3799
3800 std::unique_ptr<RandomAccessFile> file;
3801 TF_EXPECT_OK(
3802 fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file));
3803
3804 char scratch[6];
3805 StringPiece result;
3806
3807 TF_EXPECT_OK(file->Read(0, sizeof(scratch), &result, scratch));
3808 EXPECT_EQ("012345", result);
3809
3810 EXPECT_EQ("gs://bucket/random_access.txt", stats.block_load_request_file_);
3811 EXPECT_EQ("gs://bucket/random_access.txt", stats.block_retrieved_file_);
3812 EXPECT_EQ(6, stats.block_retrieved_bytes_transferred_);
3813 }
3814
TEST(GcsFileSystemTest,NewAppendableFile_MultipleFlushesWithCompose)3815 TEST(GcsFileSystemTest, NewAppendableFile_MultipleFlushesWithCompose) {
3816 std::vector<string> contents(
3817 {"content0,", "content1,", "content2,", "content3,"});
3818 std::vector<HttpRequest*> requests({
3819 // Fetch the file (stats and then content)
3820 new FakeHttpRequest(
3821 "Uri: "
3822 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3823 "some%2Fpath%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
3824 "Auth Token: fake_token\n"
3825 "Timeouts: 5 1 10\n",
3826 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
3827 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3828 new FakeHttpRequest(
3829 "Uri: "
3830 "https://storage.googleapis.com/bucket/some%2Fpath%2Fappendable\n"
3831 "Auth Token: fake_token\n"
3832 "Range: 0-1048575\n"
3833 "Timeouts: 5 1 20\n",
3834 contents[0]),
3835 // Upload entire file
3836 new FakeHttpRequest(
3837 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
3838 "uploadType=resumable&name=some%2Fpath%2Fappendable\n"
3839 "Auth Token: fake_token\n"
3840 "Header X-Upload-Content-Length: 18\n"
3841 "Post: yes\n"
3842 "Timeouts: 5 1 10\n",
3843 "", {{"Location", "https://custom/upload/location"}}),
3844 new FakeHttpRequest(
3845 strings::StrCat("Uri: https://custom/upload/location\n"
3846 "Auth Token: fake_token\n"
3847 "Header Content-Range: bytes 0-17/18\n"
3848 "Timeouts: 5 1 30\n"
3849 "Put body: ",
3850 contents[0], contents[1], "\n"),
3851 ""),
3852 // Upload new part to a temporary object
3853 new FakeHttpRequest(
3854 "Uri: "
3855 "https://www.googleapis.com/upload/storage/v1/b/bucket/"
3856 "o?uploadType=resumable&name=some%2Fpath%2F.tmpcompose%2Fappendable."
3857 "18\n"
3858 "Auth Token: fake_token\n"
3859 "Header X-Upload-Content-Length: 9\n"
3860 "Post: yes\n"
3861 "Timeouts: 5 1 10\n",
3862 "",
3863 {{"Location",
3864 "https://custom/upload/"
3865 "location"}}),
3866 new FakeHttpRequest(
3867 strings::StrCat("Uri: https://custom/upload/location\n"
3868 "Auth Token: fake_token\n"
3869 "Header Content-Range: bytes 0-8/9\n"
3870 "Timeouts: 5 1 30\n"
3871 "Put body: ",
3872 contents[2], "\n"),
3873 ""),
3874 // Fetch generation
3875 new FakeHttpRequest(
3876 "Uri: "
3877 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3878 "some%2Fpath%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
3879 "Auth Token: fake_token\n"
3880 "Timeouts: 5 1 10\n",
3881 strings::StrCat("{\"size\": \"8\",\"generation\": \"1234\","
3882 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3883 // Compose the new part at the end of the original object.
3884 new FakeHttpRequest("Uri: "
3885 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3886 "some%2Fpath%2Fappendable/compose\n"
3887 "Auth Token: fake_token\n"
3888 "Timeouts: 5 1 10\n"
3889 "Header content-type: application/json\n"
3890 "Post body: {'sourceObjects': [{'name': "
3891 "'some/path/"
3892 "appendable','objectPrecondition':{'"
3893 "ifGenerationMatch':1234}},{'name': "
3894 "'some/path/.tmpcompose/appendable.18'}]}\n",
3895 ""),
3896 // Delete the temporary object.
3897 new FakeHttpRequest("Uri: "
3898 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3899 "some%2Fpath%2F.tmpcompose%2Fappendable.18\n"
3900 "Auth Token: fake_token\n"
3901 "Timeouts: 5 1 10\n"
3902 "Delete: yes\n",
3903 ""),
3904 new FakeHttpRequest(
3905 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
3906 "uploadType=resumable&name=some%2Fpath%2F.tmpcompose%2Fappendable."
3907 "27\n"
3908 "Auth Token: fake_token\n"
3909 "Header X-Upload-Content-Length: 9\n"
3910 "Post: yes\n"
3911 "Timeouts: 5 1 10\n",
3912 "", {{"Location", "https://custom/upload/location"}}),
3913 new FakeHttpRequest(
3914 strings::StrCat("Uri: https://custom/upload/location\n"
3915 "Auth Token: fake_token\n"
3916 "Header Content-Range: bytes 0-8/9\n"
3917 "Timeouts: 5 1 30\n"
3918 "Put body: ",
3919 contents[3], "\n"),
3920 ""),
3921 // Fetch generation
3922 new FakeHttpRequest(
3923 "Uri: "
3924 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3925 "some%2Fpath%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
3926 "Auth Token: fake_token\n"
3927 "Timeouts: 5 1 10\n",
3928 strings::StrCat("{\"size\": \"8\",\"generation\": \"4567\","
3929 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3930 new FakeHttpRequest("Uri: "
3931 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3932 "some%2Fpath%2Fappendable/compose\n"
3933 "Auth Token: fake_token\n"
3934 "Timeouts: 5 1 10\n"
3935 "Header content-type: application/json\n"
3936 "Post body: {'sourceObjects': [{'name': "
3937 "'some/path/"
3938 "appendable','objectPrecondition':{'"
3939 "ifGenerationMatch':4567}},{'name': "
3940 "'some/path/.tmpcompose/appendable.27'}]}\n",
3941 ""),
3942 new FakeHttpRequest("Uri: "
3943 "https://www.googleapis.com/storage/v1/b/bucket/o/"
3944 "some%2Fpath%2F.tmpcompose%2Fappendable."
3945 "27\n"
3946 "Auth Token: fake_token\n"
3947 "Timeouts: 5 1 10\n"
3948 "Delete: yes\n",
3949 ""),
3950 });
3951 GcsFileSystem fs(
3952 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
3953 std::unique_ptr<HttpRequest::Factory>(
3954 new FakeHttpRequestFactory(&requests)),
3955 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 32 /* block size */,
3956 32 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
3957 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
3958 0 /* matching paths cache max entries */, kTestRetryConfig,
3959 kTestTimeoutConfig, *kAllowedLocationsDefault,
3960 nullptr /* gcs additional header */, true /* compose append */);
3961
3962 // Create an appendable file. This should read the file from GCS, and pull its
3963 // contents into the block cache.
3964 std::unique_ptr<WritableFile> wfile;
3965 TF_EXPECT_OK(fs.NewAppendableFile("gs://bucket/some/path/appendable", nullptr,
3966 &wfile));
3967 TF_EXPECT_OK(wfile->Append(contents[1]));
3968 TF_EXPECT_OK(wfile->Flush());
3969 TF_EXPECT_OK(wfile->Append(contents[2]));
3970 TF_EXPECT_OK(wfile->Flush());
3971 TF_EXPECT_OK(wfile->Append(contents[3]));
3972 TF_EXPECT_OK(wfile->Close());
3973 }
3974
TEST(GcsFileSystemTest,NewAppendableFile_MultipleFlushesWithoutCompose)3975 TEST(GcsFileSystemTest, NewAppendableFile_MultipleFlushesWithoutCompose) {
3976 std::vector<string> contents(
3977 {"content0,", "content1,", "content2,", "content3,"});
3978 std::vector<HttpRequest*> requests({
3979 new FakeHttpRequest(
3980 "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
3981 "path%2Fappendable?fields=size%2Cgeneration%2Cupdated\n"
3982 "Auth Token: fake_token\n"
3983 "Timeouts: 5 1 10\n",
3984 strings::StrCat("{\"size\": \"8\",\"generation\": \"1\","
3985 "\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
3986 new FakeHttpRequest(
3987 "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
3988 "Auth Token: fake_token\n"
3989 "Range: 0-1048575\n"
3990 "Timeouts: 5 1 20\n",
3991 contents[0]),
3992 new FakeHttpRequest(
3993 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
3994 "uploadType=resumable&name=path%2Fappendable\n"
3995 "Auth Token: fake_token\n"
3996 "Header X-Upload-Content-Length: 18\n"
3997 "Post: yes\n"
3998 "Timeouts: 5 1 10\n",
3999 "", {{"Location", "https://custom/upload/location"}}),
4000 // Uploads entire file.
4001 new FakeHttpRequest(
4002 strings::StrCat("Uri: https://custom/upload/location\n"
4003 "Auth Token: fake_token\n"
4004 "Header Content-Range: bytes 0-17/18\n"
4005 "Timeouts: 5 1 30\n"
4006 "Put body: ",
4007 contents[0], contents[1], "\n"),
4008 ""),
4009 new FakeHttpRequest("Uri: "
4010 "https://www.googleapis.com/upload/storage/v1/b/"
4011 "bucket/o?"
4012 "uploadType=resumable&name=path%2Fappendable\n"
4013 "Auth Token: fake_token\n"
4014 "Header X-Upload-Content-Length: 27\n"
4015 "Post: yes\n"
4016 "Timeouts: 5 1 10\n",
4017 "",
4018 {{"Location",
4019 "https://custom/upload/"
4020 "location"}}),
4021 // Uploads entire file again.
4022 new FakeHttpRequest(
4023 strings::StrCat("Uri: https://custom/upload/location\n"
4024 "Auth Token: fake_token\n"
4025 "Header Content-Range: bytes 0-26/27\n"
4026 "Timeouts: 5 1 30\n"
4027 "Put body: ",
4028 contents[0], contents[1], contents[2], "\n"),
4029 ""),
4030 new FakeHttpRequest(
4031 "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
4032 "uploadType=resumable&name=path%2Fappendable\n"
4033 "Auth Token: fake_token\n"
4034 "Header X-Upload-Content-Length: 36\n"
4035 "Post: yes\n"
4036 "Timeouts: 5 1 10\n",
4037 "", {{"Location", "https://custom/upload/location"}}),
4038 // Uploads entire file again.
4039 new FakeHttpRequest(
4040 strings::StrCat("Uri: https://custom/upload/location\n"
4041 "Auth Token: fake_token\n"
4042 "Header Content-Range: bytes 0-35/36\n"
4043 "Timeouts: 5 1 30\n"
4044 "Put body: ",
4045 contents[0], contents[1], contents[2], contents[3],
4046 "\n"),
4047 ""),
4048 });
4049 GcsFileSystem fs(
4050 std::unique_ptr<AuthProvider>(new FakeAuthProvider),
4051 std::unique_ptr<HttpRequest::Factory>(
4052 new FakeHttpRequestFactory(&requests)),
4053 std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 32 /* block size */,
4054 32 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
4055 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
4056 0 /* matching paths cache max entries */, kTestRetryConfig,
4057 kTestTimeoutConfig, *kAllowedLocationsDefault,
4058 nullptr /* gcs additional header */, false /* compose append */);
4059
4060 // Create an appendable file. This should read the file from GCS, and pull its
4061 // contents into the block cache.
4062 std::unique_ptr<WritableFile> wfile;
4063 TF_EXPECT_OK(
4064 fs.NewAppendableFile("gs://bucket/path/appendable", nullptr, &wfile));
4065 TF_EXPECT_OK(wfile->Append(contents[1]));
4066 TF_EXPECT_OK(wfile->Flush());
4067 TF_EXPECT_OK(wfile->Append(contents[2]));
4068 TF_EXPECT_OK(wfile->Flush());
4069 TF_EXPECT_OK(wfile->Append(contents[3]));
4070 TF_EXPECT_OK(wfile->Close());
4071 }
4072
TEST(GcsFileSystemTest,AppendModeCompose)4073 TEST(GcsFileSystemTest, AppendModeCompose) {
4074 unsetenv("GCS_APPEND_MODE");
4075 setenv("GCS_APPEND_MODE", "compose", 1);
4076 GcsFileSystem fs1;
4077 EXPECT_EQ(true, fs1.compose_append());
4078 }
4079
TEST(GcsFileSystemTest,AppendModeDefault)4080 TEST(GcsFileSystemTest, AppendModeDefault) {
4081 unsetenv("GCS_APPEND_MODE");
4082 GcsFileSystem fs1;
4083 EXPECT_EQ(false, fs1.compose_append());
4084 }
4085
4086 } // namespace
4087 } // namespace tensorflow
4088