xref: /aosp_15_r20/external/federated-compute/fcp/client/http/in_memory_request_response_test.cc (revision 14675a029014e728ec732f129a32e299b2da0601) 
1 /*
2  * Copyright 2021 Google LLC
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #include "fcp/client/http/in_memory_request_response.h"
17 
18 #include <cstdint>
19 #include <memory>
20 #include <optional>
21 #include <string>
22 #include <utility>
23 #include <vector>
24 
25 #include "google/protobuf/any.pb.h"
26 #include "gmock/gmock.h"
27 #include "gtest/gtest.h"
28 #include "absl/status/status.h"
29 #include "absl/status/statusor.h"
30 #include "absl/strings/cord.h"
31 #include "absl/strings/escaping.h"
32 #include "absl/strings/string_view.h"
33 #include "absl/synchronization/blocking_counter.h"
34 #include "absl/synchronization/notification.h"
35 #include "absl/time/time.h"
36 #include "fcp/base/monitoring.h"
37 #include "fcp/base/simulated_clock.h"
38 #include "fcp/client/cache/file_backed_resource_cache.h"
39 #include "fcp/client/cache/test_helpers.h"
40 #include "fcp/client/diag_codes.pb.h"
41 #include "fcp/client/http/http_client.h"
42 #include "fcp/client/http/http_client_util.h"
43 #include "fcp/client/http/http_resource_metadata.pb.h"
44 #include "fcp/client/http/testing/test_helpers.h"
45 #include "fcp/client/interruptible_runner.h"
46 #include "fcp/client/test_helpers.h"
47 #include "fcp/testing/testing.h"
48 #include "google/protobuf/io/gzip_stream.h"
49 #include "google/protobuf/io/zero_copy_stream_impl_lite.h"
50 
51 namespace fcp::client::http {
52 namespace {
53 
54 using ::fcp::IsCode;
55 using ::fcp::client::http::FakeHttpResponse;
56 using ::fcp::client::http::MockableHttpClient;
57 using ::fcp::client::http::MockHttpClient;
58 using ::testing::_;
59 using ::testing::ContainerEq;
60 using ::testing::Contains;
61 using ::testing::ElementsAre;
62 using ::testing::Eq;
63 using ::testing::FieldsAre;
64 using ::testing::Ge;
65 using ::testing::HasSubstr;
66 using ::testing::IsEmpty;
67 using ::testing::MockFunction;
68 using ::testing::Ne;
69 using ::testing::NiceMock;
70 using ::testing::Pair;
71 using ::testing::Return;
72 using ::testing::StrEq;
73 using ::testing::StrictMock;
74 
75 using CompressionFormat =
76     ::fcp::client::http::UriOrInlineData::InlineData::CompressionFormat;
77 
78 constexpr absl::string_view kOctetStream = "application/octet-stream";
79 int64_t kMaxCacheSizeBytes = 10000000;
80 
MetadataForUncompressedResource()81 google::protobuf::Any MetadataForUncompressedResource() {
82   HttpResourceMetadata metadata;
83   google::protobuf::Any any;
84   any.PackFrom(metadata);
85   return any;
86 }
87 
MetadataForCompressedResource()88 google::protobuf::Any MetadataForCompressedResource() {
89   HttpResourceMetadata metadata;
90   metadata.set_compression_format(
91       ResourceCompressionFormat::RESOURCE_COMPRESSION_FORMAT_GZIP);
92   google::protobuf::Any any;
93   any.PackFrom(metadata);
94   return any;
95 }
96 
TEST(InMemoryHttpRequestTest,NonHttpsUriFails)97 TEST(InMemoryHttpRequestTest, NonHttpsUriFails) {
98   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
99       InMemoryHttpRequest::Create("http://invalid.com",
100                                   HttpRequest::Method::kGet, {}, "",
101                                   /*use_compression=*/false);
102   EXPECT_THAT(request.status(), IsCode(INVALID_ARGUMENT));
103 }
104 
TEST(InMemoryHttpRequestTest,GetWithRequestBodyFails)105 TEST(InMemoryHttpRequestTest, GetWithRequestBodyFails) {
106   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
107       InMemoryHttpRequest::Create(
108           "https://valid.com", HttpRequest::Method::kGet, {},
109           "non_empty_request_body", /*use_compression=*/false);
110   EXPECT_THAT(request.status(), IsCode(INVALID_ARGUMENT));
111 }
112 
113 // Ensures that providing a Content-Length header results in an error (since it
114 // is automatically generated instead).
TEST(InMemoryHttpRequestTest,ContentLengthHeaderFails)115 TEST(InMemoryHttpRequestTest, ContentLengthHeaderFails) {
116   // Note that we purposely use a mixed-case "Content-length" header name, to
117   // ensure that the check is correctly case-insensitive.
118   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
119       InMemoryHttpRequest::Create(
120           "https://valid.com", HttpRequest::Method::kPost,
121           {{"Content-length", "1234"}}, "non_empty_request_body",
122           /*use_compression=*/false);
123   EXPECT_THAT(request.status(), IsCode(INVALID_ARGUMENT));
124 }
125 
TEST(InMemoryHttpRequestTest,ValidGetRequest)126 TEST(InMemoryHttpRequestTest, ValidGetRequest) {
127   const std::string expected_uri = "https://valid.com";
128   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
129       InMemoryHttpRequest::Create(expected_uri, HttpRequest::Method::kGet, {},
130                                   "", /*use_compression=*/false);
131   ASSERT_OK(request);
132   EXPECT_THAT((*request)->uri(), StrEq(expected_uri));
133   EXPECT_EQ((*request)->method(), HttpRequest::Method::kGet);
134   // Because no request body is present, the Content-Length header shouldn't
135   // have been added.
136   EXPECT_THAT((*request)->extra_headers(), IsEmpty());
137   EXPECT_FALSE((*request)->HasBody());
138 }
139 
TEST(InMemoryHttpRequestTest,ValidGetRequestWithHeaders)140 TEST(InMemoryHttpRequestTest, ValidGetRequestWithHeaders) {
141   const HeaderList expected_headers{{"Foo", "Bar"}, {"Baz", "123"}};
142   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
143       InMemoryHttpRequest::Create("https://valid.com",
144                                   HttpRequest::Method::kGet, expected_headers,
145                                   "", /*use_compression=*/false);
146   ASSERT_OK(request);
147   EXPECT_THAT((*request)->extra_headers(), ContainerEq(expected_headers));
148 }
149 
TEST(InMemoryHttpRequestTest,ValidPostRequestWithoutBody)150 TEST(InMemoryHttpRequestTest, ValidPostRequestWithoutBody) {
151   const std::string expected_uri = "https://valid.com";
152   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
153       InMemoryHttpRequest::Create(expected_uri, HttpRequest::Method::kPost, {},
154                                   "", /*use_compression=*/false);
155   ASSERT_OK(request);
156   EXPECT_THAT((*request)->uri(), StrEq(expected_uri));
157   EXPECT_EQ((*request)->method(), HttpRequest::Method::kPost);
158   // Because no request body is present, the Content-Length header shouldn't
159   // have been added.
160   EXPECT_THAT((*request)->extra_headers(), IsEmpty());
161   EXPECT_FALSE((*request)->HasBody());
162 }
163 
TEST(InMemoryHttpRequestTest,ValidPostRequestWithBody)164 TEST(InMemoryHttpRequestTest, ValidPostRequestWithBody) {
165   const std::string expected_uri = "https://valid.com";
166   const std::string expected_body = "request_body";
167   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
168       InMemoryHttpRequest::Create(expected_uri, HttpRequest::Method::kPost, {},
169                                   expected_body,
170                                   /*use_compression=*/false);
171   ASSERT_OK(request);
172   EXPECT_THAT((*request)->uri(), StrEq(expected_uri));
173   EXPECT_EQ((*request)->method(), HttpRequest::Method::kPost);
174   EXPECT_THAT((*request)->extra_headers(),
175               ElementsAre(Pair("Content-Length",
176                                std::to_string(expected_body.size()))));
177   EXPECT_TRUE((*request)->HasBody());
178 }
179 
180 // Checks that the automatically generated Content-Length header is
181 // appropriately added to any specified extra headers (rather than replacing
182 // them completely).
TEST(InMemoryHttpRequestTest,ValidPostRequestWithBodyAndHeaders)183 TEST(InMemoryHttpRequestTest, ValidPostRequestWithBodyAndHeaders) {
184   const HeaderList original_headers{{"Foo", "Bar"}, {"Baz", "123"}};
185   const std::string expected_body = "request_body";
186   const HeaderList expected_headers{
187       {"Foo", "Bar"},
188       {"Baz", "123"},
189       {"Content-Length", std::to_string(expected_body.size())}};
190   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
191       InMemoryHttpRequest::Create("https://valid.com",
192                                   HttpRequest::Method::kPost, original_headers,
193                                   expected_body,
194                                   /*use_compression=*/false);
195   ASSERT_OK(request);
196 
197   EXPECT_THAT((*request)->extra_headers(), ContainerEq(expected_headers));
198 }
199 
TEST(InMemoryHttpRequestTest,ReadBodySimple)200 TEST(InMemoryHttpRequestTest, ReadBodySimple) {
201   const std::string expected_body = "request_body";
202   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
203       InMemoryHttpRequest::Create("https://valid.com",
204                                   HttpRequest::Method::kPost, {}, expected_body,
205                                   /*use_compression=*/false);
206   ASSERT_OK(request);
207 
208   std::string actual_body;
209   actual_body.resize(expected_body.size());
210   // Read the body in one go (the "simple" case).
211   auto read_result =
212       (*request)->ReadBody(actual_body.data(), actual_body.size());
213   ASSERT_OK(read_result);
214   EXPECT_THAT(*read_result, actual_body.size());
215   EXPECT_THAT(actual_body, StrEq(expected_body));
216   // Expect the second read to indicate the end of the stream.
217   EXPECT_THAT((*request)->ReadBody(nullptr, 1), IsCode(OUT_OF_RANGE));
218 }
219 
TEST(InMemoryHttpRequestTest,ReadBodyChunked)220 TEST(InMemoryHttpRequestTest, ReadBodyChunked) {
221   // This test reads the body in chunks of 3 bytes at a time (rather than all at
222   // once, like previous test). To test some edge cases, we ensure that the
223   // request body's length is not evenly dividable by 3.
224   const std::string expected_body = "12345678";
225   ASSERT_THAT(expected_body.size() % 3, Ne(0));
226   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
227       InMemoryHttpRequest::Create("https://valid.com",
228                                   HttpRequest::Method::kPost, {}, expected_body,
229                                   /*use_compression=*/false);
230   ASSERT_OK(request);
231 
232   std::string actual_body;
233   // Pre-size the buffer with 'X' characters. This will allow us to check
234   // whether each read modifies the expected range of the buffer.
235   actual_body.resize(expected_body.size(), 'X');
236 
237   // Read the body 3 bytes at a time.
238   // The first read tests the case where less data is requested than is
239   // available.
240   absl::StatusOr<int64_t> read_result =
241       (*request)->ReadBody(actual_body.data(), 3);
242   ASSERT_OK(read_result);
243   EXPECT_THAT(*read_result, 3);
244   EXPECT_THAT(actual_body, StrEq("123XXXXX"));
245 
246   read_result = (*request)->ReadBody(actual_body.data() + 3, 3);
247   ASSERT_OK(read_result);
248   EXPECT_THAT(*read_result, 3);
249   EXPECT_THAT(actual_body, StrEq("123456XX"));
250 
251   // The last read should only read 2 bytes. This tests the case where more data
252   // is requested than is available. A correct implementation should not write
253   // more bytes than it has available, which should ensure that no writes will
254   // occur beyond actual_body.data()'s buffer size (which is only 8 bytes long).
255   read_result = (*request)->ReadBody(actual_body.data() + 6, 3);
256   ASSERT_OK(read_result);
257   EXPECT_THAT(*read_result, 2);
258   EXPECT_THAT(actual_body, StrEq(expected_body));
259 
260   // Expect the last read to indicate the end of the stream.
261   EXPECT_THAT((*request)->ReadBody(nullptr, 1), IsCode(OUT_OF_RANGE));
262 }
263 
TEST(InMemoryHttpRequestTest,RequestWithCompressedBody)264 TEST(InMemoryHttpRequestTest, RequestWithCompressedBody) {
265   const std::string uncompressed_body =
266       "request_body_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
267   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
268       InMemoryHttpRequest::Create("https://valid.com",
269                                   HttpRequest::Method::kPost, {},
270                                   uncompressed_body,
271                                   /*use_compression=*/true);
272   ASSERT_OK(request);
273   auto content_encoding_header =
274       FindHeader((*request)->extra_headers(), kContentEncodingHdr);
275   ASSERT_TRUE(content_encoding_header.has_value());
276   ASSERT_EQ(content_encoding_header.value(), kGzipEncodingHdrValue);
277 
278   auto content_length_header =
279       FindHeader((*request)->extra_headers(), kContentLengthHdr);
280   ASSERT_TRUE(content_length_header.has_value());
281   int compressed_length = std::stoi(content_length_header.value());
282   ASSERT_GT(compressed_length, 0);
283   ASSERT_LT(compressed_length, uncompressed_body.size());
284 
285   std::string actual_body;
286   actual_body.resize(compressed_length);
287   // Read the body in one go (the "simple" case).
288   auto read_result =
289       (*request)->ReadBody(actual_body.data(), actual_body.size());
290   ASSERT_OK(read_result);
291   EXPECT_THAT(*read_result, actual_body.size());
292 
293   // Expect the second read to indicate the end of the stream.
294   EXPECT_THAT((*request)->ReadBody(nullptr, 1), IsCode(OUT_OF_RANGE));
295 
296   auto recovered_body = internal::UncompressWithGzip(actual_body);
297   ASSERT_OK(recovered_body);
298   EXPECT_EQ(*recovered_body, uncompressed_body);
299 }
300 
TEST(InMemoryHttpRequestCallbackTest,ResponseFailsBeforeHeaders)301 TEST(InMemoryHttpRequestCallbackTest, ResponseFailsBeforeHeaders) {
302   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
303       InMemoryHttpRequest::Create("https://valid.com",
304                                   HttpRequest::Method::kGet, {}, "",
305                                   /*use_compression=*/false);
306   ASSERT_OK(request);
307 
308   InMemoryHttpRequestCallback callback;
309   callback.OnResponseError(**request, absl::UnimplementedError("foobar"));
310 
311   absl::StatusOr<InMemoryHttpResponse> actual_response = callback.Response();
312   EXPECT_THAT(actual_response, IsCode(UNIMPLEMENTED));
313   EXPECT_THAT(actual_response.status().message(), HasSubstr("foobar"));
314 }
315 
TEST(InMemoryHttpRequestCallbackTest,ResponseFailsAfterHeaders)316 TEST(InMemoryHttpRequestCallbackTest, ResponseFailsAfterHeaders) {
317   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
318       InMemoryHttpRequest::Create("https://valid.com",
319                                   HttpRequest::Method::kGet, {}, "",
320                                   /*use_compression=*/false);
321   ASSERT_OK(request);
322 
323   auto fake_response = FakeHttpResponse(kHttpOk, {});
324 
325   InMemoryHttpRequestCallback callback;
326   ASSERT_OK(callback.OnResponseStarted(**request, fake_response));
327   callback.OnResponseBodyError(**request, fake_response,
328                                absl::UnimplementedError("foobar"));
329 
330   absl::StatusOr<InMemoryHttpResponse> actual_response = callback.Response();
331   EXPECT_THAT(actual_response, IsCode(UNIMPLEMENTED));
332   EXPECT_THAT(actual_response.status().message(), HasSubstr("foobar"));
333 }
334 
TEST(InMemoryHttpRequestCallbackTest,ResponseFailsAfterPartialBody)335 TEST(InMemoryHttpRequestCallbackTest, ResponseFailsAfterPartialBody) {
336   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
337       InMemoryHttpRequest::Create("https://valid.com",
338                                   HttpRequest::Method::kGet, {}, "",
339                                   /*use_compression=*/false);
340   ASSERT_OK(request);
341 
342   auto fake_response = FakeHttpResponse(kHttpOk, {});
343 
344   InMemoryHttpRequestCallback callback;
345   ASSERT_OK(callback.OnResponseStarted(**request, fake_response));
346   ASSERT_OK(callback.OnResponseBody(**request, fake_response, "response_"));
347   callback.OnResponseBodyError(**request, fake_response,
348                                absl::UnimplementedError("foobar"));
349 
350   absl::StatusOr<InMemoryHttpResponse> actual_response = callback.Response();
351   EXPECT_THAT(actual_response, IsCode(UNIMPLEMENTED));
352   EXPECT_THAT(actual_response.status().message(), HasSubstr("foobar"));
353 }
354 
TEST(InMemoryHttpRequestCallbackTest,TestResponseWithContentLengthAndBodyTooShort)355 TEST(InMemoryHttpRequestCallbackTest,
356      TestResponseWithContentLengthAndBodyTooShort) {
357   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
358       InMemoryHttpRequest::Create("https://valid.com",
359                                   HttpRequest::Method::kGet, {}, "",
360                                   /*use_compression=*/false);
361   ASSERT_OK(request);
362 
363   auto fake_response = FakeHttpResponse(kHttpOk, {{"Content-Length", "5"}});
364 
365   InMemoryHttpRequestCallback callback;
366   ASSERT_OK(callback.OnResponseStarted(**request, fake_response));
367   // Return a partial response (only 4 characters instead of the expected 5
368   // indicated by the Content-Length header).
369   ASSERT_OK(callback.OnResponseBody(**request, fake_response, "12"));
370   ASSERT_OK(callback.OnResponseBody(**request, fake_response, "34"));
371   callback.OnResponseCompleted(**request, fake_response);
372 
373   absl::StatusOr<InMemoryHttpResponse> actual_response = callback.Response();
374   EXPECT_THAT(actual_response, IsCode(INVALID_ARGUMENT));
375   EXPECT_THAT(actual_response.status().message(),
376               HasSubstr("Too little response body data received"));
377 }
378 
TEST(InMemoryHttpRequestCallbackTest,TestResponseWithContentLengthAndBodyTooLong)379 TEST(InMemoryHttpRequestCallbackTest,
380      TestResponseWithContentLengthAndBodyTooLong) {
381   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
382       InMemoryHttpRequest::Create("https://valid.com",
383                                   HttpRequest::Method::kGet, {}, "",
384                                   /*use_compression=*/false);
385   ASSERT_OK(request);
386 
387   auto fake_response = FakeHttpResponse(kHttpOk, {{"Content-Length", "5"}});
388 
389   InMemoryHttpRequestCallback callback;
390   ASSERT_OK(callback.OnResponseStarted(**request, fake_response));
391   // Return a more response data than expected (6 characters instead of the
392   // expected 5 indicated by the Content-Length header).
393   ASSERT_OK(callback.OnResponseBody(**request, fake_response, "12"));
394   ASSERT_OK(callback.OnResponseBody(**request, fake_response, "34"));
395   absl::Status response_body_result =
396       callback.OnResponseBody(**request, fake_response, "56");
397   EXPECT_THAT(response_body_result, IsCode(OUT_OF_RANGE));
398   EXPECT_THAT(response_body_result.message(),
399               HasSubstr("Too much response body data received"));
400 }
401 
TEST(InMemoryHttpRequestCallbackTest,ResponseWithContentLengthNegative)402 TEST(InMemoryHttpRequestCallbackTest, ResponseWithContentLengthNegative) {
403   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
404       InMemoryHttpRequest::Create("https://valid.com",
405                                   HttpRequest::Method::kGet, {}, "",
406                                   /*use_compression=*/false);
407   ASSERT_OK(request);
408 
409   // Set the Content-Length to an invalid negative value.
410   auto fake_response = FakeHttpResponse(kHttpOk, {{"Content-Length", "-1"}});
411 
412   InMemoryHttpRequestCallback callback;
413   absl::Status response_started_result =
414       callback.OnResponseStarted(**request, fake_response);
415   EXPECT_THAT(response_started_result, IsCode(OUT_OF_RANGE));
416   EXPECT_THAT(response_started_result.message(),
417               HasSubstr("Invalid Content-Length response header"));
418 }
419 
TEST(InMemoryHttpRequestCallbackTest,ResponseWithContentLengthNonInteger)420 TEST(InMemoryHttpRequestCallbackTest, ResponseWithContentLengthNonInteger) {
421   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
422       InMemoryHttpRequest::Create("https://valid.com",
423                                   HttpRequest::Method::kGet, {}, "",
424                                   /*use_compression=*/false);
425   ASSERT_OK(request);
426 
427   // Set the Content-Length to an invalid non-integer, non-ASCII value.
428   auto fake_response =
429       FakeHttpResponse(kHttpOk, {{"Content-Length", "\U0001F600"}});
430 
431   InMemoryHttpRequestCallback callback;
432   absl::Status response_started_result =
433       callback.OnResponseStarted(**request, fake_response);
434   EXPECT_THAT(response_started_result, IsCode(INVALID_ARGUMENT));
435   EXPECT_THAT(response_started_result.message(),
436               HasSubstr("Could not parse Content-Length response header"));
437 }
438 
TEST(InMemoryHttpRequestCallbackTest,ResponseWithContentEncodingHeader)439 TEST(InMemoryHttpRequestCallbackTest, ResponseWithContentEncodingHeader) {
440   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
441       InMemoryHttpRequest::Create("https://valid.com",
442                                   HttpRequest::Method::kGet, {}, "",
443                                   /*use_compression=*/false);
444   ASSERT_OK(request);
445 
446   // Add a Content-Encoding header, which implementations should never provide
447   // to us, unless we advertised an Accept-Encoding header in the request.
448   auto fake_response = FakeHttpResponse(kHttpOk, {{"Content-Encoding", "foo"}});
449 
450   InMemoryHttpRequestCallback callback;
451   absl::Status response_started_result =
452       callback.OnResponseStarted(**request, fake_response);
453   EXPECT_THAT(response_started_result, IsCode(INVALID_ARGUMENT));
454   EXPECT_THAT(response_started_result.message(),
455               HasSubstr("Unexpected header: Content-Encoding"));
456 }
457 
TEST(InMemoryHttpRequestCallbackTest,ResponseWithTransferEncodingHeader)458 TEST(InMemoryHttpRequestCallbackTest, ResponseWithTransferEncodingHeader) {
459   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
460       InMemoryHttpRequest::Create("https://valid.com",
461                                   HttpRequest::Method::kGet, {}, "",
462                                   /*use_compression=*/false);
463   ASSERT_OK(request);
464 
465   // Add a Transfer-Encoding header, which implementations should never provide
466   // to us.
467   auto fake_response =
468       FakeHttpResponse(kHttpOk, {{"Transfer-Encoding", "foo"}});
469 
470   InMemoryHttpRequestCallback callback;
471   absl::Status response_started_result =
472       callback.OnResponseStarted(**request, fake_response);
473   EXPECT_THAT(response_started_result, IsCode(INVALID_ARGUMENT));
474   EXPECT_THAT(response_started_result.message(),
475               HasSubstr("Unexpected header: Transfer-Encoding"));
476 }
477 
TEST(InMemoryHttpRequestCallbackTest,OkResponseWithoutContentLength)478 TEST(InMemoryHttpRequestCallbackTest, OkResponseWithoutContentLength) {
479   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
480       InMemoryHttpRequest::Create("https://valid.com",
481                                   HttpRequest::Method::kGet, {}, "",
482                                   /*use_compression=*/false);
483   ASSERT_OK(request);
484 
485   // Note that the fake response does not contain a Content-Length header. The
486   // InMemoryHttpRequestCallback should be able to handle this (accumulating
487   // response data until OnResponseCompleted is called).
488   int expected_code = 201;  // "201 Created"
489   auto fake_response = FakeHttpResponse(expected_code, {});
490   const std::string expected_body = "response_body";
491 
492   InMemoryHttpRequestCallback callback;
493   ASSERT_OK(callback.OnResponseStarted(**request, fake_response));
494   // We return the response body in one go (the "simple" case).
495   ASSERT_OK(callback.OnResponseBody(**request, fake_response, expected_body));
496   callback.OnResponseCompleted(**request, fake_response);
497 
498   absl::StatusOr<InMemoryHttpResponse> actual_response = callback.Response();
499   ASSERT_OK(actual_response);
500   EXPECT_THAT(*actual_response, FieldsAre(expected_code, IsEmpty(), IsEmpty(),
501                                           StrEq(expected_body)));
502 }
503 
TEST(InMemoryHttpRequestCallbackTest,OkResponseWithContentLength)504 TEST(InMemoryHttpRequestCallbackTest, OkResponseWithContentLength) {
505   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
506       InMemoryHttpRequest::Create("https://valid.com",
507                                   HttpRequest::Method::kGet, {}, "",
508                                   /*use_compression=*/false);
509   ASSERT_OK(request);
510 
511   int expected_code = kHttpOk;
512   const std::string expected_body = "response_body";
513   auto fake_response = FakeHttpResponse(
514       expected_code,
515       {{"Content-Length", std::to_string(expected_body.size())}});
516 
517   InMemoryHttpRequestCallback callback;
518   ASSERT_OK(callback.OnResponseStarted(**request, fake_response));
519   ASSERT_OK(callback.OnResponseBody(**request, fake_response, expected_body));
520   callback.OnResponseCompleted(**request, fake_response);
521 
522   absl::StatusOr<InMemoryHttpResponse> actual_response = callback.Response();
523   ASSERT_OK(actual_response);
524   EXPECT_THAT(*actual_response, FieldsAre(expected_code, IsEmpty(), IsEmpty(),
525                                           StrEq(expected_body)));
526 }
527 
TEST(InMemoryHttpRequestCallbackTest,OkResponseChunkedBody)528 TEST(InMemoryHttpRequestCallbackTest, OkResponseChunkedBody) {
529   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
530       InMemoryHttpRequest::Create("https://valid.com",
531                                   HttpRequest::Method::kGet, {}, "",
532                                   /*use_compression=*/false);
533   ASSERT_OK(request);
534 
535   // Note that the fake response does not contain a Content-Length header. The
536   // InMemoryHttpRequestCallback should be able to handle this (accumulating
537   // response data until OnResponseCompleted is called).
538   auto fake_response = FakeHttpResponse(kHttpOk, {});
539 
540   InMemoryHttpRequestCallback callback;
541   ASSERT_OK(callback.OnResponseStarted(**request, fake_response));
542   // This test returns the body in chunks of 3 bytes at a time (rather than
543   // all at once, like previous test). To test some edge cases, we ensure that
544   // the request body's length is not evenly dividable by 3.
545   ASSERT_OK(callback.OnResponseBody(**request, fake_response, "123"));
546   ASSERT_OK(callback.OnResponseBody(**request, fake_response, "456"));
547   ASSERT_OK(callback.OnResponseBody(**request, fake_response, "78"));
548   callback.OnResponseCompleted(**request, fake_response);
549 
550   absl::StatusOr<InMemoryHttpResponse> actual_response = callback.Response();
551   ASSERT_OK(actual_response);
552   EXPECT_THAT(actual_response->body, StrEq("12345678"));
553 }
554 
TEST(InMemoryHttpRequestCallbackTest,TestOkResponseWithEmptyBodyWithoutContentLength)555 TEST(InMemoryHttpRequestCallbackTest,
556      TestOkResponseWithEmptyBodyWithoutContentLength) {
557   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
558       InMemoryHttpRequest::Create("https://valid.com",
559                                   HttpRequest::Method::kGet, {}, "",
560                                   /*use_compression=*/false);
561   ASSERT_OK(request);
562 
563   int expected_code = kHttpOk;
564   auto fake_response = FakeHttpResponse(expected_code, {});
565 
566   InMemoryHttpRequestCallback callback;
567   ASSERT_OK(callback.OnResponseStarted(**request, fake_response));
568   ASSERT_OK(callback.OnResponseBody(**request, fake_response, ""));
569   callback.OnResponseCompleted(**request, fake_response);
570 
571   absl::StatusOr<InMemoryHttpResponse> actual_response = callback.Response();
572   ASSERT_OK(actual_response);
573   EXPECT_THAT(*actual_response,
574               FieldsAre(expected_code, IsEmpty(), IsEmpty(), IsEmpty()));
575 }
576 
TEST(InMemoryHttpRequestCallbackTest,TestOkResponseWithEmptyBodyWithContentLength)577 TEST(InMemoryHttpRequestCallbackTest,
578      TestOkResponseWithEmptyBodyWithContentLength) {
579   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
580       InMemoryHttpRequest::Create("https://valid.com",
581                                   HttpRequest::Method::kGet, {}, "",
582                                   /*use_compression=*/false);
583   ASSERT_OK(request);
584 
585   int expected_code = kHttpOk;
586   auto fake_response =
587       FakeHttpResponse(expected_code, {{"Content-Length", "0"}});
588 
589   InMemoryHttpRequestCallback callback;
590   ASSERT_OK(callback.OnResponseStarted(**request, fake_response));
591   ASSERT_OK(callback.OnResponseBody(**request, fake_response, ""));
592   callback.OnResponseCompleted(**request, fake_response);
593 
594   absl::StatusOr<InMemoryHttpResponse> actual_response = callback.Response();
595   ASSERT_OK(actual_response);
596   EXPECT_THAT(*actual_response,
597               FieldsAre(expected_code, IsEmpty(), IsEmpty(), IsEmpty()));
598 }
599 
TEST(InMemoryHttpRequestCallbackTest,TestOkResponsWithAcceptEncodingRequestHeader)600 TEST(InMemoryHttpRequestCallbackTest,
601      TestOkResponsWithAcceptEncodingRequestHeader) {
602   const std::string expected_content_encoding = "foo";
603   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
604       InMemoryHttpRequest::Create(
605           "https://valid.com", HttpRequest::Method::kGet,
606           {{"Accept-Encoding", expected_content_encoding}}, "",
607           /*use_compression=*/false);
608   ASSERT_OK(request);
609 
610   // Note that the fake response contains a Content-Encoding header, which
611   // should be allowed because the request contained an Accept-Encoding header.
612   int expected_code = kHttpOk;
613   auto fake_response = FakeHttpResponse(
614       expected_code, {{"Some-Response-Header", "foo"},
615                       {"Content-Encoding", expected_content_encoding}});
616   const std::string expected_body = "response_body";
617 
618   InMemoryHttpRequestCallback callback;
619   ASSERT_OK(callback.OnResponseStarted(**request, fake_response));
620   // We return the response body in one go (the "simple" case).
621   ASSERT_OK(callback.OnResponseBody(**request, fake_response, expected_body));
622   callback.OnResponseCompleted(**request, fake_response);
623 
624   absl::StatusOr<InMemoryHttpResponse> actual_response = callback.Response();
625   ASSERT_OK(actual_response);
626   EXPECT_THAT(*actual_response,
627               FieldsAre(expected_code, StrEq(expected_content_encoding),
628                         IsEmpty(), StrEq(expected_body)));
629 }
630 
TEST(InMemoryHttpRequestCallbackTest,NotFoundResponse)631 TEST(InMemoryHttpRequestCallbackTest, NotFoundResponse) {
632   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
633       InMemoryHttpRequest::Create("https://valid.com",
634                                   HttpRequest::Method::kGet, {}, "",
635                                   /*use_compression=*/false);
636   ASSERT_OK(request);
637 
638   // Return an HTTP error response.
639   auto fake_response = FakeHttpResponse(kHttpNotFound, {});
640   const std::string expected_body = "response_body";
641 
642   InMemoryHttpRequestCallback callback;
643   ASSERT_OK(callback.OnResponseStarted(**request, fake_response));
644   ASSERT_OK(callback.OnResponseBody(**request, fake_response, expected_body));
645   callback.OnResponseCompleted(**request, fake_response);
646 
647   absl::StatusOr<InMemoryHttpResponse> actual_response = callback.Response();
648   EXPECT_THAT(actual_response, IsCode(NOT_FOUND));
649   EXPECT_THAT(actual_response.status().message(), HasSubstr("404"));
650 }
651 
652 class PerformRequestsTest : public ::testing::Test {
653  protected:
PerformRequestsTest()654   PerformRequestsTest()
655       : interruptible_runner_(
656             &mock_log_manager_, mock_should_abort_.AsStdFunction(),
657             InterruptibleRunner::TimingConfig{
658                 .polling_period = absl::ZeroDuration(),
659                 .graceful_shutdown_period = absl::InfiniteDuration(),
660                 .extended_shutdown_period = absl::InfiniteDuration()},
661             InterruptibleRunner::DiagnosticsConfig{
662                 .interrupted = ProdDiagCode::BACKGROUND_TRAINING_INTERRUPT_HTTP,
663                 .interrupt_timeout =
664                     ProdDiagCode::BACKGROUND_TRAINING_INTERRUPT_HTTP_TIMED_OUT,
665                 .interrupted_extended = ProdDiagCode::
666                     BACKGROUND_TRAINING_INTERRUPT_HTTP_EXTENDED_COMPLETED,
667                 .interrupt_timeout_extended = ProdDiagCode::
668                     BACKGROUND_TRAINING_INTERRUPT_HTTP_EXTENDED_TIMED_OUT}) {}
SetUp()669   void SetUp() override {
670     root_cache_dir_ = testing::TempDir();
671     root_files_dir_ = testing::TempDir();
672   }
673 
TearDown()674   void TearDown() override {
675     std::filesystem::remove_all(root_cache_dir_);
676     std::filesystem::remove_all(root_files_dir_);
677   }
678 
679   NiceMock<MockLogManager> mock_log_manager_;
680   NiceMock<MockFunction<bool()>> mock_should_abort_;
681   InterruptibleRunner interruptible_runner_;
682   StrictMock<MockHttpClient> mock_http_client_;
683   testing::StrictMock<MockLogManager> log_manager_;
684   SimulatedClock clock_;
685   std::string root_cache_dir_;
686   std::string root_files_dir_;
687 };
688 
TEST_F(PerformRequestsTest,PerformRequestInMemoryOk)689 TEST_F(PerformRequestsTest, PerformRequestInMemoryOk) {
690   std::string expected_request_body = "request_body";
691   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
692       InMemoryHttpRequest::Create(
693           "https://valid.com", HttpRequest::Method::kPost,
694           {{"Some-Request-Header", "foo"}}, expected_request_body,
695           /*use_compression=*/false);
696   ASSERT_OK(request);
697 
698   int expected_response_code = kHttpOk;
699   std::string expected_response_body = "response_body";
700   EXPECT_CALL(mock_http_client_,
701               PerformSingleRequest(
702                   FieldsAre((*request)->uri(), (*request)->method(),
703                             Contains(Header{"Some-Request-Header", "foo"}),
704                             StrEq(expected_request_body))))
705       .WillOnce(Return(FakeHttpResponse(expected_response_code,
706                                         {{"Some-Response-Header", "bar"}},
707                                         expected_response_body)));
708 
709   int64_t bytes_received = 0;
710   int64_t bytes_sent = 0;
711   absl::StatusOr<InMemoryHttpResponse> result = PerformRequestInMemory(
712       mock_http_client_, interruptible_runner_, *std::move(request),
713       // We pass in non-null pointers for the network stats, to ensure they are
714       // correctly updated.
715       &bytes_received, &bytes_sent);
716   ASSERT_OK(result);
717   EXPECT_THAT(*result, FieldsAre(expected_response_code, IsEmpty(), IsEmpty(),
718                                  StrEq(expected_response_body)));
719 
720   EXPECT_THAT(bytes_sent, Ne(bytes_received));
721   EXPECT_THAT(bytes_sent, Ge(expected_request_body.size()));
722   EXPECT_THAT(bytes_received, Ge(expected_response_body.size()));
723 }
724 
TEST_F(PerformRequestsTest,PerformRequestInMemoryNotFound)725 TEST_F(PerformRequestsTest, PerformRequestInMemoryNotFound) {
726   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
727       InMemoryHttpRequest::Create("https://valid.com",
728                                   HttpRequest::Method::kGet, {}, "",
729                                   /*use_compression=*/false);
730   ASSERT_OK(request);
731 
732   int expected_response_code = kHttpNotFound;
733   std::string expected_response_body = "response_body";
734   EXPECT_CALL(mock_http_client_,
735               PerformSingleRequest(
736                   FieldsAre((*request)->uri(), (*request)->method(), _, _)))
737       .WillOnce(Return(FakeHttpResponse(expected_response_code, {},
738                                         expected_response_body)));
739 
740   absl::StatusOr<InMemoryHttpResponse> result =
741       PerformRequestInMemory(mock_http_client_, interruptible_runner_,
742                              *std::move(request), nullptr, nullptr);
743   EXPECT_THAT(result, IsCode(NOT_FOUND));
744   EXPECT_THAT(result.status().message(), HasSubstr("404"));
745 }
746 
TEST_F(PerformRequestsTest,PerformRequestInMemoryEarlyError)747 TEST_F(PerformRequestsTest, PerformRequestInMemoryEarlyError) {
748   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
749       InMemoryHttpRequest::Create("https://valid.com",
750                                   HttpRequest::Method::kPost, {}, "",
751                                   /*use_compression=*/false);
752   ASSERT_OK(request);
753 
754   EXPECT_CALL(mock_http_client_,
755               PerformSingleRequest(
756                   FieldsAre((*request)->uri(), (*request)->method(), _, _)))
757       // Make the call to PerformSingleRequest (and therefore the call to
758       // HttpClient::PerformRequests) fail as a whole (rather than having the
759       // individual request return a failure).
760       .WillOnce(Return(absl::InvalidArgumentError("PerformRequests failed")));
761 
762   int64_t bytes_received = 0;
763   int64_t bytes_sent = 0;
764   absl::StatusOr<InMemoryHttpResponse> result =
765       PerformRequestInMemory(mock_http_client_, interruptible_runner_,
766                              *std::move(request), &bytes_received, &bytes_sent);
767   EXPECT_THAT(result, IsCode(INVALID_ARGUMENT));
768   EXPECT_THAT(result.status().message(), HasSubstr("PerformRequests failed"));
769 
770   // We know that MockHttpClient will have updated the 'sent' network stat
771   // before having called into PerformSingleRequest, so that should be
772   // reflected.
773   EXPECT_THAT(bytes_sent, Ge(0));
774   // The 'received' network stat should be 0 OTOH, since issuing the request
775   // failed.
776   EXPECT_EQ(bytes_received, 0);
777 }
778 
779 // Tests the case where the request gets interrupted.
TEST_F(PerformRequestsTest,PerformRequestInMemoryCancellation)780 TEST_F(PerformRequestsTest, PerformRequestInMemoryCancellation) {
781   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
782       InMemoryHttpRequest::Create("https://valid.com",
783                                   HttpRequest::Method::kGet, {}, "",
784                                   /*use_compression=*/false);
785   ASSERT_OK(request);
786 
787   absl::Notification request_issued;
788   // We expect one calls to the cancellation listener.
789   absl::BlockingCounter counter_should_abort(1);
790   // When the HttpClient receives a HttpRequestHandle::Cancel call, we decrement
791   // the counter.
792   mock_http_client_.SetCancellationListener(
793       [&counter_should_abort]() { counter_should_abort.DecrementCount(); });
794 
795   // Make HttpClient::PerformRequests() block until the counter is decremented.
796   EXPECT_CALL(mock_http_client_,
797               PerformSingleRequest(
798                   FieldsAre((*request)->uri(), (*request)->method(), _, _)))
799       .WillOnce([&request_issued, &counter_should_abort](
800                     MockableHttpClient::SimpleHttpRequest ignored) {
801         request_issued.Notify();
802         counter_should_abort.Wait();
803         return FakeHttpResponse(503, {}, "");
804       });
805   // Make should_abort return false until we know that the request was issued
806   // (i.e. once InterruptibleRunner has actually started running the code it was
807   // given), and then make it return true, triggering an abort sequence and
808   // unblocking the PerformRequests() call we caused to block above.
809   EXPECT_CALL(mock_should_abort_, Call()).WillRepeatedly([&request_issued] {
810     return request_issued.HasBeenNotified();
811   });
812 
813   EXPECT_CALL(mock_log_manager_,
814               LogDiag(ProdDiagCode::BACKGROUND_TRAINING_INTERRUPT_HTTP));
815 
816   int64_t bytes_received = 0;
817   int64_t bytes_sent = 0;
818   // The request should result in a CANCELLED outcome.
819   absl::StatusOr<InMemoryHttpResponse> result =
820       PerformRequestInMemory(mock_http_client_, interruptible_runner_,
821                              *std::move(request), &bytes_received, &bytes_sent);
822 
823   EXPECT_THAT(result, IsCode(CANCELLED));
824   EXPECT_THAT(result.status().message(),
825               HasSubstr("cancelled after graceful wait"));
826 
827   // The network stats should still have been updated though (to reflect the
828   // data sent and received up until the point of interruption).
829   EXPECT_THAT(bytes_sent, Ge(0));
830   EXPECT_THAT(bytes_received, Ge(0));
831 }
832 
TEST_F(PerformRequestsTest,PerformTwoRequestsInMemoryOk)833 TEST_F(PerformRequestsTest, PerformTwoRequestsInMemoryOk) {
834   std::string expected_request_body = "request_body";
835   absl::StatusOr<std::unique_ptr<HttpRequest>> request =
836       InMemoryHttpRequest::Create(
837           "https://valid.com", HttpRequest::Method::kPost,
838           {{"Some-Request-Header", "foo"}}, expected_request_body,
839           /*use_compression=*/false);
840   ASSERT_OK(request);
841   std::string another_expected_request_body = "request_body_2";
842   absl::StatusOr<std::unique_ptr<HttpRequest>> another_request =
843       InMemoryHttpRequest::Create("https://valid.com",
844                                   HttpRequest::Method::kPost,
845                                   {{"Some-Other-Request-Header", "foo2"}},
846                                   another_expected_request_body,
847                                   /*use_compression=*/false);
848   ASSERT_OK(another_request);
849 
850   std::string expected_response_body = "response_body";
851   EXPECT_CALL(mock_http_client_,
852               PerformSingleRequest(
853                   FieldsAre((*request)->uri(), (*request)->method(),
854                             Contains(Header{"Some-Request-Header", "foo"}),
855                             StrEq(expected_request_body))))
856       .WillOnce(Return(FakeHttpResponse(
857           kHttpOk, {{"Some-Response-Header", "bar"}}, expected_response_body)));
858   std::string another_expected_response_body = "another_response_body";
859   EXPECT_CALL(mock_http_client_,
860               PerformSingleRequest(FieldsAre(
861                   (*request)->uri(), (*request)->method(),
862                   Contains(Header{"Some-Other-Request-Header", "foo2"}),
863                   StrEq(another_expected_request_body))))
864       .WillOnce(Return(
865           FakeHttpResponse(kHttpOk, {{"Some-Other-Response-Header", "bar2"}},
866                            another_expected_response_body)));
867 
868   int64_t bytes_received = 0;
869   int64_t bytes_sent = 0;
870   std::vector<std::unique_ptr<HttpRequest>> requests;
871   requests.push_back(*std::move(request));
872   requests.push_back(*std::move(another_request));
873   absl::StatusOr<std::vector<absl::StatusOr<InMemoryHttpResponse>>> results =
874       PerformMultipleRequestsInMemory(
875           mock_http_client_, interruptible_runner_, std::move(requests),
876           // We pass in non-null pointers for the network
877           // stats, to ensure they are correctly updated.
878           &bytes_received, &bytes_sent);
879   ASSERT_OK(results);
880   ASSERT_EQ(results->size(), 2);
881 
882   auto first_response = (*results)[0];
883   ASSERT_OK(first_response);
884   EXPECT_THAT(*first_response, FieldsAre(kHttpOk, IsEmpty(), IsEmpty(),
885                                          StrEq(expected_response_body)));
886   auto second_response = (*results)[1];
887   ASSERT_OK(second_response);
888   EXPECT_THAT(*second_response,
889               FieldsAre(kHttpOk, IsEmpty(), IsEmpty(),
890                         StrEq(another_expected_response_body)));
891 
892   EXPECT_THAT(bytes_sent, Ne(bytes_received));
893   EXPECT_THAT(bytes_sent, Ge(expected_request_body.size() +
894                              another_expected_request_body.size()));
895   EXPECT_THAT(bytes_received, Ge(expected_response_body.size() +
896                                  another_expected_response_body.size()));
897 }
898 
TEST_F(PerformRequestsTest,PerformTwoRequestsWithOneFailedOneSuccess)899 TEST_F(PerformRequestsTest, PerformTwoRequestsWithOneFailedOneSuccess) {
900   std::string success_request_body = "success_request_body";
901   absl::StatusOr<std::unique_ptr<HttpRequest>> success_request =
902       InMemoryHttpRequest::Create(
903           "https://valid.com", HttpRequest::Method::kPost,
904           {{"Some-Request-Header", "foo"}}, success_request_body,
905           /*use_compression=*/false);
906   ASSERT_OK(success_request);
907   std::string failure_request_body = "failure_request_body";
908   absl::StatusOr<std::unique_ptr<HttpRequest>> failure_request =
909       InMemoryHttpRequest::Create(
910           "https://valid.com", HttpRequest::Method::kPost,
911           {{"Some-Other-Request-Header", "foo2"}}, failure_request_body,
912           /*use_compression=*/false);
913   ASSERT_OK(failure_request);
914 
915   int ok_response_code = kHttpOk;
916   std::string success_response_body = "response_body";
917   EXPECT_CALL(mock_http_client_,
918               PerformSingleRequest(FieldsAre(
919                   (*success_request)->uri(), (*success_request)->method(),
920                   Contains(Header{"Some-Request-Header", "foo"}),
921                   StrEq(success_request_body))))
922       .WillOnce(Return(FakeHttpResponse(ok_response_code,
923                                         {{"Some-Response-Header", "bar"}},
924                                         success_response_body)));
925   std::string failure_response_body = "failure_response_body";
926   EXPECT_CALL(mock_http_client_,
927               PerformSingleRequest(FieldsAre(
928                   (*failure_request)->uri(), (*failure_request)->method(),
929                   Contains(Header{"Some-Other-Request-Header", "foo2"}),
930                   StrEq(failure_request_body))))
931       .WillOnce(
932           Return(FakeHttpResponse(kHttpNotFound, {}, failure_response_body)));
933 
934   int64_t bytes_received = 0;
935   int64_t bytes_sent = 0;
936   std::vector<std::unique_ptr<HttpRequest>> requests;
937   requests.push_back(*std::move(success_request));
938   requests.push_back(*std::move(failure_request));
939   absl::StatusOr<std::vector<absl::StatusOr<InMemoryHttpResponse>>> results =
940       PerformMultipleRequestsInMemory(
941           mock_http_client_, interruptible_runner_, std::move(requests),
942           // We pass in non-null pointers for the network
943           // stats, to ensure they are correctly updated.
944           &bytes_received, &bytes_sent);
945   ASSERT_OK(results);
946   ASSERT_EQ(results->size(), 2);
947   auto first_response = (*results)[0];
948   ASSERT_OK(first_response);
949   EXPECT_THAT(*first_response, FieldsAre(ok_response_code, IsEmpty(), IsEmpty(),
950                                          StrEq(success_response_body)));
951 
952   EXPECT_THAT(results->at(1), IsCode(NOT_FOUND));
953   EXPECT_THAT(results->at(1).status().message(), HasSubstr("404"));
954 
955   EXPECT_THAT(bytes_sent, Ne(bytes_received));
956   EXPECT_THAT(bytes_sent,
957               Ge(success_request_body.size() + failure_request_body.size()));
958   EXPECT_THAT(bytes_received,
959               Ge(success_response_body.size() + failure_response_body.size()));
960 }
961 
962 // Tests the case where a zero-length vector of UriOrInlineData is passed in. It
963 // should result in a zero-length result vector (as opposed to an error or a
964 // crash).
TEST_F(PerformRequestsTest,FetchResourcesInMemoryEmptyInputVector)965 TEST_F(PerformRequestsTest, FetchResourcesInMemoryEmptyInputVector) {
966   auto result = FetchResourcesInMemory(mock_http_client_, interruptible_runner_,
967                                        {}, nullptr, nullptr,
968                                        /*resource_cache=*/nullptr);
969   ASSERT_OK(result);
970   EXPECT_THAT(*result, IsEmpty());
971 }
972 
973 // Tests the case where both fields of UriOrInlineData are empty. The empty
974 // inline_data field should be returned.
TEST_F(PerformRequestsTest,FetchResourcesInMemoryEmptyUriAndInline)975 TEST_F(PerformRequestsTest, FetchResourcesInMemoryEmptyUriAndInline) {
976   auto result = FetchResourcesInMemory(
977       mock_http_client_, interruptible_runner_,
978       {UriOrInlineData::CreateInlineData(absl::Cord(),
979                                          CompressionFormat::kUncompressed)},
980       nullptr, nullptr,
981       /*resource_cache=*/nullptr);
982   ASSERT_OK(result);
983 
984   ASSERT_OK((*result)[0]);
985   EXPECT_THAT(*(*result)[0],
986               FieldsAre(kHttpOk, IsEmpty(), kOctetStream, IsEmpty()));
987 }
988 
989 // Tests the case where one of the URIs is invalid. The whole request should
990 // result in an error in that case.
TEST_F(PerformRequestsTest,FetchResourcesInMemoryInvalidUri)991 TEST_F(PerformRequestsTest, FetchResourcesInMemoryInvalidUri) {
992   auto result = FetchResourcesInMemory(
993       mock_http_client_, interruptible_runner_,
994       {UriOrInlineData::CreateUri("https://valid.com", "",
995                                   absl::ZeroDuration()),
996        UriOrInlineData::CreateUri("http://invalid.com", "",
997                                   absl::ZeroDuration())},
998       nullptr, nullptr,
999       /*resource_cache=*/nullptr);
1000   EXPECT_THAT(result, IsCode(INVALID_ARGUMENT));
1001   EXPECT_THAT(result.status().message(), HasSubstr("Non-HTTPS"));
1002 }
1003 
1004 // Tests the case where all of the requested resources must be fetched via URI.
TEST_F(PerformRequestsTest,FetchResourcesInMemoryAllUris)1005 TEST_F(PerformRequestsTest, FetchResourcesInMemoryAllUris) {
1006   const std::string uri1 = "https://valid.com/1";
1007   const std::string uri2 = "https://valid.com/2";
1008   const std::string uri3 = "https://valid.com/3";
1009   const std::string uri4 = "https://valid.com/4";
1010   auto resource1 = UriOrInlineData::CreateUri(uri1, "", absl::ZeroDuration());
1011   auto resource2 = UriOrInlineData::CreateUri(uri2, "", absl::ZeroDuration());
1012   auto resource3 = UriOrInlineData::CreateUri(uri3, "", absl::ZeroDuration());
1013   auto resource4 = UriOrInlineData::CreateUri(uri4, "", absl::ZeroDuration());
1014 
1015   int expected_response_code1 = kHttpOk;
1016   int expected_response_code2 = kHttpNotFound;
1017   int expected_response_code3 = kHttpServiceUnavailable;
1018   int expected_response_code4 = 204;  // "204 No Content"
1019   std::string expected_response_body1 = "response_body1";
1020   std::string expected_response_body4 = "response_body4";
1021   EXPECT_CALL(mock_http_client_,
1022               PerformSingleRequest(
1023                   FieldsAre(uri1, HttpRequest::Method::kGet, HeaderList{}, "")))
1024       .WillOnce(Return(FakeHttpResponse(expected_response_code1, {},
1025                                         expected_response_body1)));
1026   EXPECT_CALL(mock_http_client_,
1027               PerformSingleRequest(
1028                   FieldsAre(uri2, HttpRequest::Method::kGet, HeaderList{}, "")))
1029       .WillOnce(Return(FakeHttpResponse(expected_response_code2, {}, "")));
1030   EXPECT_CALL(mock_http_client_,
1031               PerformSingleRequest(
1032                   FieldsAre(uri3, HttpRequest::Method::kGet, HeaderList{}, "")))
1033       .WillOnce(Return(FakeHttpResponse(expected_response_code3, {}, "")));
1034   EXPECT_CALL(mock_http_client_,
1035               PerformSingleRequest(
1036                   FieldsAre(uri4, HttpRequest::Method::kGet, HeaderList{}, "")))
1037       .WillOnce(Return(FakeHttpResponse(expected_response_code4, {},
1038                                         expected_response_body4)));
1039 
1040   int64_t bytes_received = 0;
1041   int64_t bytes_sent = 0;
1042   auto result =
1043       FetchResourcesInMemory(mock_http_client_, interruptible_runner_,
1044                              {resource1, resource2, resource3, resource4},
1045                              // We pass in non-null pointers for the network
1046                              // stats, to ensure they are correctly updated.
1047                              &bytes_received, &bytes_sent,
1048                              /*resource_cache=*/nullptr);
1049   ASSERT_OK(result);
1050 
1051   ASSERT_OK((*result)[0]);
1052   EXPECT_THAT(*(*result)[0],
1053               FieldsAre(expected_response_code1, IsEmpty(), IsEmpty(),
1054                         StrEq(expected_response_body1)));
1055   EXPECT_THAT((*result)[1], IsCode(NOT_FOUND));
1056   EXPECT_THAT((*result)[1].status().message(), HasSubstr("404"));
1057   EXPECT_THAT((*result)[2], IsCode(UNAVAILABLE));
1058   EXPECT_THAT((*result)[2].status().message(), HasSubstr("503"));
1059   ASSERT_OK((*result)[3]);
1060   EXPECT_THAT(*(*result)[3],
1061               FieldsAre(expected_response_code4, IsEmpty(), IsEmpty(),
1062                         StrEq(expected_response_body4)));
1063 
1064   EXPECT_THAT(bytes_sent, Ne(bytes_received));
1065   EXPECT_THAT(bytes_sent, Ge(0));
1066   EXPECT_THAT(bytes_received, Ge(0));
1067 }
1068 
1069 // Tests the case where some of the requested resources have inline data
1070 // available.
TEST_F(PerformRequestsTest,FetchResourcesInMemorySomeInlineData)1071 TEST_F(PerformRequestsTest, FetchResourcesInMemorySomeInlineData) {
1072   const std::string uri1 = "https://valid.com/1";
1073   const std::string uri3 = "https://valid.com/3";
1074   std::string expected_response_body2 = "response_body2";
1075   std::string expected_response_body4 = "response_body4";
1076   auto resource1 = UriOrInlineData::CreateUri(uri1, "", absl::ZeroDuration());
1077   auto resource2 = UriOrInlineData::CreateInlineData(
1078       absl::Cord(expected_response_body2), CompressionFormat::kUncompressed);
1079   auto resource3 = UriOrInlineData::CreateUri(uri3, "", absl::ZeroDuration());
1080   auto resource4 = UriOrInlineData::CreateInlineData(
1081       absl::Cord(expected_response_body4), CompressionFormat::kUncompressed);
1082 
1083   int expected_response_code1 = kHttpServiceUnavailable;
1084   int expected_response_code3 = 204;  // "204 No Content"
1085   std::string expected_response_body3 = "response_body3";
1086   EXPECT_CALL(mock_http_client_,
1087               PerformSingleRequest(
1088                   FieldsAre(uri1, HttpRequest::Method::kGet, HeaderList{}, "")))
1089       .WillOnce(Return(FakeHttpResponse(expected_response_code1, {}, "")));
1090   EXPECT_CALL(mock_http_client_,
1091               PerformSingleRequest(
1092                   FieldsAre(uri3, HttpRequest::Method::kGet, HeaderList{}, "")))
1093       .WillOnce(Return(FakeHttpResponse(expected_response_code3, {},
1094                                         expected_response_body3)));
1095 
1096   int64_t bytes_received = 0;
1097   int64_t bytes_sent = 0;
1098   auto result =
1099       FetchResourcesInMemory(mock_http_client_, interruptible_runner_,
1100                              {resource1, resource2, resource3, resource4},
1101                              &bytes_received, &bytes_sent,
1102                              /*resource_cache=*/nullptr);
1103   ASSERT_OK(result);
1104 
1105   EXPECT_THAT((*result)[0], IsCode(UNAVAILABLE));
1106   EXPECT_THAT((*result)[0].status().message(), HasSubstr("503"));
1107   ASSERT_OK((*result)[1]);
1108   EXPECT_THAT(*(*result)[1], FieldsAre(kHttpOk, IsEmpty(), kOctetStream,
1109                                        StrEq(expected_response_body2)));
1110   ASSERT_OK((*result)[2]);
1111   EXPECT_THAT(*(*result)[2],
1112               FieldsAre(expected_response_code3, IsEmpty(), IsEmpty(),
1113                         StrEq(expected_response_body3)));
1114   ASSERT_OK((*result)[3]);
1115   EXPECT_THAT(*(*result)[3], FieldsAre(kHttpOk, IsEmpty(), kOctetStream,
1116                                        StrEq(expected_response_body4)));
1117 
1118   EXPECT_THAT(bytes_sent, Ne(bytes_received));
1119   EXPECT_THAT(bytes_sent, Ge(0));
1120   EXPECT_THAT(bytes_received, Ge(0));
1121 }
1122 
1123 // Tests the case where all of the requested resources have inline data
1124 // available (and hence no HTTP requests are expected to be issued).
TEST_F(PerformRequestsTest,FetchResourcesInMemoryOnlyInlineData)1125 TEST_F(PerformRequestsTest, FetchResourcesInMemoryOnlyInlineData) {
1126   std::string expected_response_body1 = "response_body1";
1127   std::string expected_response_body2 = "response_body2";
1128   auto resource1 = UriOrInlineData::CreateInlineData(
1129       absl::Cord(expected_response_body1), CompressionFormat::kUncompressed);
1130   auto resource2 = UriOrInlineData::CreateInlineData(
1131       absl::Cord(expected_response_body2), CompressionFormat::kUncompressed);
1132 
1133   int64_t bytes_received = 0;
1134   int64_t bytes_sent = 0;
1135   auto result = FetchResourcesInMemory(mock_http_client_, interruptible_runner_,
1136                                        {resource1, resource2}, &bytes_received,
1137                                        &bytes_sent,
1138                                        /*resource_cache=*/nullptr);
1139   ASSERT_OK(result);
1140 
1141   ASSERT_OK((*result)[0]);
1142   EXPECT_THAT(*(*result)[0], FieldsAre(kHttpOk, IsEmpty(), kOctetStream,
1143                                        StrEq(expected_response_body1)));
1144   ASSERT_OK((*result)[1]);
1145   EXPECT_THAT(*(*result)[1], FieldsAre(kHttpOk, IsEmpty(), kOctetStream,
1146                                        StrEq(expected_response_body2)));
1147 
1148   // The network stats should be untouched, since no network requests were
1149   // issued.
1150   EXPECT_EQ(bytes_sent, 0);
1151   EXPECT_EQ(bytes_received, 0);
1152 }
1153 
1154 // Tests the case where the fetches get interrupted.
TEST_F(PerformRequestsTest,FetchResourcesInMemoryCancellation)1155 TEST_F(PerformRequestsTest, FetchResourcesInMemoryCancellation) {
1156   const std::string uri1 = "https://valid.com/1";
1157   const std::string uri2 = "https://valid.com/2";
1158   auto resource1 = UriOrInlineData::CreateUri(uri1, "", absl::ZeroDuration());
1159   auto resource2 = UriOrInlineData::CreateUri(uri2, "", absl::ZeroDuration());
1160 
1161   EXPECT_CALL(mock_http_client_, PerformSingleRequest(FieldsAre(uri1, _, _, _)))
1162       .WillOnce(Return(FakeHttpResponse(kHttpOk, {}, "")));
1163 
1164   absl::Notification request_issued;
1165   // We expect two calls to the cancellation listener, one for each request.
1166   absl::BlockingCounter counter_should_abort(2);
1167   // When the HttpClient receives a HttpRequestHandle::Cancel call, we decrement
1168   // the counter.
1169   mock_http_client_.SetCancellationListener(
1170       [&counter_should_abort]() { counter_should_abort.DecrementCount(); });
1171 
1172   // Make HttpClient::PerformRequests() block until the counter is decremented.
1173   EXPECT_CALL(mock_http_client_, PerformSingleRequest(FieldsAre(uri2, _, _, _)))
1174       .WillOnce([&request_issued, &counter_should_abort](
1175                     MockableHttpClient::SimpleHttpRequest ignored) {
1176         request_issued.Notify();
1177         counter_should_abort.Wait();
1178         return FakeHttpResponse(503, {}, "");
1179       });
1180   // Make should_abort return false until we know that the 2nd request was
1181   // issued (i.e. once InterruptibleRunner has actually started running the code
1182   // it was given), and then make it return true, triggering an abort sequence
1183   // and unblocking the PerformRequests() call we caused to block above.
1184   EXPECT_CALL(mock_should_abort_, Call()).WillRepeatedly([&request_issued] {
1185     return request_issued.HasBeenNotified();
1186   });
1187 
1188   EXPECT_CALL(mock_log_manager_,
1189               LogDiag(ProdDiagCode::BACKGROUND_TRAINING_INTERRUPT_HTTP));
1190 
1191   int64_t bytes_received = 0;
1192   int64_t bytes_sent = 0;
1193   // The request should result in an overall CANCELLED outcome.
1194   auto result = FetchResourcesInMemory(mock_http_client_, interruptible_runner_,
1195                                        {resource1, resource2}, &bytes_received,
1196                                        &bytes_sent,
1197                                        /*resource_cache=*/nullptr);
1198   EXPECT_THAT(result, IsCode(CANCELLED));
1199   EXPECT_THAT(result.status().message(),
1200               HasSubstr("cancelled after graceful wait"));
1201 
1202   // The network stats should still have been updated though (to reflect the
1203   // data sent and received up until the point of interruption).
1204   EXPECT_THAT(bytes_sent, Ge(0));
1205   EXPECT_THAT(bytes_received, Ge(0));
1206 }
1207 
TEST_F(PerformRequestsTest,FetchResourcesInMemoryCompressedResources)1208 TEST_F(PerformRequestsTest, FetchResourcesInMemoryCompressedResources) {
1209   const std::string uri = "https://valid.com/";
1210   auto resource1 = UriOrInlineData::CreateUri(uri, "", absl::ZeroDuration());
1211 
1212   int expected_response_code = kHttpOk;
1213   std::string content_type = "bytes+gzip";
1214   std::string expected_response_body = "response_body: AAAAAAAAAAAAAAAAAAAAA";
1215   auto compressed_response_body =
1216       internal::CompressWithGzip(expected_response_body);
1217   ASSERT_OK(compressed_response_body);
1218   EXPECT_CALL(mock_http_client_,
1219               PerformSingleRequest(
1220                   FieldsAre(uri, HttpRequest::Method::kGet, HeaderList{}, "")))
1221       .WillOnce(Return(FakeHttpResponse(expected_response_code,
1222                                         {{kContentTypeHdr, content_type}},
1223                                         *compressed_response_body)));
1224 
1225   auto resource2 = UriOrInlineData::CreateInlineData(
1226       absl::Cord(*compressed_response_body), CompressionFormat::kGzip);
1227 
1228   int64_t bytes_received = 0;
1229   int64_t bytes_sent = 0;
1230   auto result = FetchResourcesInMemory(
1231       mock_http_client_, interruptible_runner_, {resource1, resource2},
1232       // We pass in non-null pointers for the network
1233       // stats, to ensure they are correctly updated.
1234       &bytes_received, &bytes_sent,
1235       /*resource_cache=*/nullptr);
1236   ASSERT_OK(result);
1237 
1238   ASSERT_OK((*result)[0]);
1239   EXPECT_THAT(*(*result)[0],
1240               FieldsAre(expected_response_code, IsEmpty(), StrEq(content_type),
1241                         StrEq(expected_response_body)));
1242   EXPECT_THAT(*(*result)[1],
1243               FieldsAre(kHttpOk, IsEmpty(), absl::StrCat(kOctetStream, "+gzip"),
1244                         StrEq(expected_response_body)));
1245 
1246   EXPECT_THAT(bytes_sent, Ne(bytes_received));
1247   EXPECT_THAT(bytes_sent, Ge(0));
1248   EXPECT_THAT(bytes_received, Ge(2 * compressed_response_body->size()));
1249 }
1250 
TEST_F(PerformRequestsTest,FetchResourcesInMemoryCompressedResourcesFailToDecode)1251 TEST_F(PerformRequestsTest,
1252        FetchResourcesInMemoryCompressedResourcesFailToDecode) {
1253   const std::string uri = "https://valid.com/";
1254   auto resource1 = UriOrInlineData::CreateUri(uri, "", absl::ZeroDuration());
1255 
1256   int expected_response_code = kHttpOk;
1257   std::string content_type = "not-actually-gzipped+gzip";
1258   std::string expected_response_body = "I am not a valid gzipped body ლ(ಠ益ಠლ)";
1259   EXPECT_CALL(mock_http_client_,
1260               PerformSingleRequest(
1261                   FieldsAre(uri, HttpRequest::Method::kGet, HeaderList{}, "")))
1262       .WillOnce(Return(FakeHttpResponse(expected_response_code,
1263                                         {{kContentTypeHdr, content_type}},
1264                                         expected_response_body)));
1265 
1266   auto resource2 = UriOrInlineData::CreateInlineData(
1267       absl::Cord(expected_response_body), CompressionFormat::kGzip);
1268 
1269   int64_t bytes_received = 0;
1270   int64_t bytes_sent = 0;
1271   auto result = FetchResourcesInMemory(
1272       mock_http_client_, interruptible_runner_, {resource1, resource2},
1273       // We pass in non-null pointers for the network
1274       // stats, to ensure they are correctly updated.
1275       &bytes_received, &bytes_sent,
1276       /*resource_cache=*/nullptr);
1277   // Fetching will succeed
1278   ASSERT_OK(result);
1279 
1280   // ...but our responses will have failed to decode.
1281   EXPECT_THAT((*result)[0], IsCode(INTERNAL));
1282   EXPECT_THAT((*result)[1], IsCode(INTERNAL));
1283 
1284   EXPECT_THAT(bytes_sent, Ne(bytes_received));
1285   EXPECT_THAT(bytes_sent, Ge(0));
1286   EXPECT_THAT(bytes_received, Ge(2 * expected_response_body.size()));
1287 }
1288 
TEST_F(PerformRequestsTest,FetchResourcesInMemoryCachedResourceOk)1289 TEST_F(PerformRequestsTest, FetchResourcesInMemoryCachedResourceOk) {
1290   const std::string uri = "https://valid.com/1";
1291   const std::string cache_id = "(^˵◕ω◕˵^)";
1292   absl::Cord cached_resource("(((*°▽°*)八(*°▽°*)))");
1293   absl::Duration max_age = absl::Hours(1);
1294   int expected_response_code = kHttpOk;
1295   auto resource = UriOrInlineData::CreateUri(uri, cache_id, max_age);
1296   auto resource_cache = cache::FileBackedResourceCache::Create(
1297       root_files_dir_, root_cache_dir_, &log_manager_, &clock_,
1298       kMaxCacheSizeBytes);
1299   ASSERT_OK(resource_cache);
1300   ASSERT_OK((*resource_cache)
1301                 ->Put(cache_id, cached_resource,
1302                       MetadataForUncompressedResource(), max_age));
1303 
1304   int64_t bytes_received = 0;
1305   int64_t bytes_sent = 0;
1306   EXPECT_CALL(log_manager_, LogDiag(DebugDiagCode::RESOURCE_CACHE_HIT));
1307   auto result = FetchResourcesInMemory(
1308       mock_http_client_, interruptible_runner_, {resource},
1309       // We pass in non-null pointers for the network
1310       // stats, to ensure they are correctly updated.
1311       &bytes_received, &bytes_sent, resource_cache->get());
1312   ASSERT_OK(result);
1313 
1314   ASSERT_OK((*result)[0]);
1315   EXPECT_THAT(*(*result)[0], FieldsAre(expected_response_code, IsEmpty(),
1316                                        kOctetStream, StrEq(cached_resource)));
1317 
1318   // Fully from the cache!
1319   EXPECT_THAT(bytes_sent, Eq(0));
1320   EXPECT_THAT(bytes_received, Eq(0));
1321 }
1322 
TEST_F(PerformRequestsTest,FetchResourcesInMemoryCachedResourceOkAndCompressed)1323 TEST_F(PerformRequestsTest,
1324        FetchResourcesInMemoryCachedResourceOkAndCompressed) {
1325   const std::string uri = "https://valid.com/1";
1326   const std::string cache_id = "(^˵◕ω◕˵^)";
1327   absl::Cord cached_resource("(((*°▽°*)八(*°▽°*)))");
1328   absl::Cord compressed_cached_resource(
1329       *internal::CompressWithGzip(std::string(cached_resource)));
1330   absl::Duration max_age = absl::Hours(1);
1331   int expected_response_code = kHttpOk;
1332   auto resource = UriOrInlineData::CreateUri(uri, cache_id, max_age);
1333   auto resource_cache = cache::FileBackedResourceCache::Create(
1334       root_files_dir_, root_cache_dir_, &log_manager_, &clock_,
1335       kMaxCacheSizeBytes);
1336   ASSERT_OK(resource_cache);
1337   ASSERT_OK((*resource_cache)
1338                 ->Put(cache_id, compressed_cached_resource,
1339                       MetadataForCompressedResource(), max_age));
1340 
1341   int64_t bytes_received = 0;
1342   int64_t bytes_sent = 0;
1343   EXPECT_CALL(log_manager_, LogDiag(DebugDiagCode::RESOURCE_CACHE_HIT));
1344   auto result = FetchResourcesInMemory(
1345       mock_http_client_, interruptible_runner_, {resource},
1346       // We pass in non-null pointers for the network
1347       // stats, to ensure they are correctly updated.
1348       &bytes_received, &bytes_sent, resource_cache->get());
1349   ASSERT_OK(result);
1350 
1351   ASSERT_OK((*result)[0]);
1352   EXPECT_THAT(*(*result)[0], FieldsAre(expected_response_code, IsEmpty(),
1353                                        kOctetStream, StrEq(cached_resource)));
1354 
1355   // Fully from the cache!
1356   EXPECT_THAT(bytes_sent, Eq(0));
1357   EXPECT_THAT(bytes_received, Eq(0));
1358 }
1359 
TEST_F(PerformRequestsTest,FetchResourcesInMemoryNotCachedButThenPutInCache)1360 TEST_F(PerformRequestsTest, FetchResourcesInMemoryNotCachedButThenPutInCache) {
1361   const std::string uri = "https://valid.com/1";
1362   const std::string cache_id = "(^˵◕ω◕˵^)";
1363   absl::Cord expected_response_body("(((*°▽°*)八(*°▽°*)))");
1364   absl::Duration max_age = absl::Hours(1);
1365   int expected_response_code = kHttpOk;
1366   std::string content_type = "bytes";
1367   auto resource = UriOrInlineData::CreateUri(uri, cache_id, max_age);
1368 
1369   EXPECT_CALL(mock_http_client_,
1370               PerformSingleRequest(
1371                   FieldsAre(uri, HttpRequest::Method::kGet, HeaderList{}, "")))
1372       .WillOnce(Return(FakeHttpResponse(expected_response_code,
1373                                         {{kContentTypeHdr, content_type}},
1374                                         std::string(expected_response_body))));
1375 
1376   auto resource_cache = cache::FileBackedResourceCache::Create(
1377       root_files_dir_, root_cache_dir_, &log_manager_, &clock_,
1378       kMaxCacheSizeBytes);
1379   ASSERT_OK(resource_cache);
1380 
1381   int64_t bytes_received = 0;
1382   int64_t bytes_sent = 0;
1383   EXPECT_CALL(log_manager_, LogDiag(DebugDiagCode::RESOURCE_CACHE_MISS));
1384   auto result = FetchResourcesInMemory(
1385       mock_http_client_, interruptible_runner_, {resource},
1386       // We pass in non-null pointers for the network
1387       // stats, to ensure they are correctly updated.
1388       &bytes_received, &bytes_sent, resource_cache->get());
1389   ASSERT_OK(result);
1390 
1391   ASSERT_OK((*result)[0]);
1392   EXPECT_THAT(*(*result)[0],
1393               FieldsAre(expected_response_code, IsEmpty(), content_type,
1394                         StrEq(expected_response_body)));
1395 
1396   EXPECT_CALL(log_manager_, LogDiag(DebugDiagCode::RESOURCE_CACHE_HIT));
1397   auto stored_resource = (*resource_cache)->Get(cache_id, std::nullopt);
1398   ASSERT_OK(stored_resource);
1399   EXPECT_THAT(*stored_resource,
1400               FieldsAre(StrEq(expected_response_body),
1401                         EqualsProto(MetadataForUncompressedResource())));
1402 }
1403 
TEST_F(PerformRequestsTest,FetchResourcesInMemoryNotCachedButThenPutInCacheCompressed)1404 TEST_F(PerformRequestsTest,
1405        FetchResourcesInMemoryNotCachedButThenPutInCacheCompressed) {
1406   const std::string uri = "https://valid.com/1";
1407   const std::string cache_id = "(^˵◕ω◕˵^)";
1408   absl::Cord expected_response_body("(((*°▽°*)八(*°▽°*)))");
1409   absl::Duration max_age = absl::Hours(1);
1410   int expected_response_code = kHttpOk;
1411   std::string content_type = "bytes+gzip";
1412   absl::Cord compressed_response_body(
1413       *internal::CompressWithGzip(std::string(expected_response_body)));
1414   auto resource = UriOrInlineData::CreateUri(uri, cache_id, max_age);
1415 
1416   EXPECT_CALL(mock_http_client_,
1417               PerformSingleRequest(
1418                   FieldsAre(uri, HttpRequest::Method::kGet, HeaderList{}, "")))
1419       .WillOnce(Return(FakeHttpResponse(
1420           expected_response_code, {{kContentTypeHdr, content_type}},
1421           std::string(compressed_response_body))));
1422 
1423   auto resource_cache = cache::FileBackedResourceCache::Create(
1424       root_files_dir_, root_cache_dir_, &log_manager_, &clock_,
1425       kMaxCacheSizeBytes);
1426   ASSERT_OK(resource_cache);
1427 
1428   int64_t bytes_received = 0;
1429   int64_t bytes_sent = 0;
1430   EXPECT_CALL(log_manager_, LogDiag(DebugDiagCode::RESOURCE_CACHE_MISS));
1431   auto result = FetchResourcesInMemory(
1432       mock_http_client_, interruptible_runner_, {resource},
1433       // We pass in non-null pointers for the network
1434       // stats, to ensure they are correctly updated.
1435       &bytes_received, &bytes_sent, resource_cache->get());
1436   ASSERT_OK(result);
1437 
1438   ASSERT_OK((*result)[0]);
1439   EXPECT_THAT(*(*result)[0],
1440               FieldsAre(expected_response_code, IsEmpty(), content_type,
1441                         StrEq(expected_response_body)));
1442 
1443   EXPECT_CALL(log_manager_, LogDiag(DebugDiagCode::RESOURCE_CACHE_HIT));
1444   auto stored_resource = (*resource_cache)->Get(cache_id, std::nullopt);
1445   ASSERT_OK(stored_resource);
1446   EXPECT_THAT(*stored_resource,
1447               FieldsAre(StrEq(compressed_response_body),
1448                         EqualsProto(MetadataForCompressedResource())));
1449 }
1450 
TEST_F(PerformRequestsTest,FetchResourcesInMemoryNotCachedPutInCacheWithZeroMaxAgeDoesntCrash)1451 TEST_F(PerformRequestsTest,
1452        FetchResourcesInMemoryNotCachedPutInCacheWithZeroMaxAgeDoesntCrash) {
1453   const std::string uri = "https://valid.com/1";
1454   const std::string cache_id = "(^˵◕ω◕˵^)";
1455   absl::Cord expected_response_body("(((*°▽°*)八(*°▽°*)))");
1456   absl::Duration max_age = absl::ZeroDuration();
1457   int expected_response_code = kHttpOk;
1458   std::string content_type = "bytes";
1459   auto resource = UriOrInlineData::CreateUri(uri, cache_id, max_age);
1460 
1461   EXPECT_CALL(mock_http_client_,
1462               PerformSingleRequest(
1463                   FieldsAre(uri, HttpRequest::Method::kGet, HeaderList{}, "")))
1464       .WillOnce(Return(FakeHttpResponse(expected_response_code,
1465                                         {{kContentTypeHdr, content_type}},
1466                                         std::string(expected_response_body))));
1467 
1468   auto resource_cache = cache::FileBackedResourceCache::Create(
1469       root_files_dir_, root_cache_dir_, &log_manager_, &clock_,
1470       kMaxCacheSizeBytes);
1471   ASSERT_OK(resource_cache);
1472 
1473   int64_t bytes_received = 0;
1474   int64_t bytes_sent = 0;
1475   EXPECT_CALL(log_manager_, LogDiag(DebugDiagCode::RESOURCE_CACHE_MISS));
1476   auto result = FetchResourcesInMemory(
1477       mock_http_client_, interruptible_runner_, {resource},
1478       // We pass in non-null pointers for the network
1479       // stats, to ensure they are correctly updated.
1480       &bytes_received, &bytes_sent, resource_cache->get());
1481   ASSERT_OK(result);
1482 
1483   ASSERT_OK((*result)[0]);
1484   EXPECT_THAT(*(*result)[0],
1485               FieldsAre(expected_response_code, IsEmpty(), content_type,
1486                         StrEq(expected_response_body)));
1487   EXPECT_CALL(log_manager_, LogDiag(DebugDiagCode::RESOURCE_CACHE_HIT));
1488   auto stored_resource = (*resource_cache)->Get(cache_id, std::nullopt);
1489   ASSERT_OK(stored_resource);
1490   EXPECT_THAT(*stored_resource,
1491               FieldsAre(StrEq(expected_response_body),
1492                         EqualsProto(MetadataForUncompressedResource())));
1493 }
1494 
TEST_F(PerformRequestsTest,FetchResourcesInMemoryCachedResourceGetReturnsInternal)1495 TEST_F(PerformRequestsTest,
1496        FetchResourcesInMemoryCachedResourceGetReturnsInternal) {
1497   const std::string uri = "https://valid.com/1";
1498   const std::string cache_id = "(^˵◕ω◕˵^)";
1499   absl::Cord expected_response_body("(((*°▽°*)八(*°▽°*)))");
1500   absl::Duration max_age = absl::Hours(1);
1501   int expected_response_code = kHttpOk;
1502   std::string content_type = "bytes";
1503   auto resource = UriOrInlineData::CreateUri(uri, cache_id, max_age);
1504   StrictMock<cache::MockResourceCache> resource_cache;
1505 
1506   EXPECT_CALL(resource_cache, Get(cache_id, std::make_optional(max_age)))
1507       .WillOnce(Return(absl::InternalError("the cache exploded -_-")));
1508 
1509   EXPECT_CALL(mock_http_client_,
1510               PerformSingleRequest(
1511                   FieldsAre(uri, HttpRequest::Method::kGet, HeaderList{}, "")))
1512       .WillOnce(Return(FakeHttpResponse(expected_response_code,
1513                                         {{kContentTypeHdr, content_type}},
1514                                         std::string(expected_response_body))));
1515 
1516   EXPECT_CALL(resource_cache, Put(cache_id, expected_response_body, _, max_age))
1517       .WillOnce(Return(absl::OkStatus()));
1518 
1519   int64_t bytes_received = 0;
1520   int64_t bytes_sent = 0;
1521   auto result = FetchResourcesInMemory(
1522       mock_http_client_, interruptible_runner_, {resource},
1523       // We pass in non-null pointers for the network
1524       // stats, to ensure they are correctly updated.
1525       &bytes_received, &bytes_sent, &resource_cache);
1526   ASSERT_OK(result);
1527 
1528   ASSERT_OK((*result)[0]);
1529   EXPECT_THAT(*(*result)[0],
1530               FieldsAre(expected_response_code, IsEmpty(), content_type,
1531                         StrEq(expected_response_body)));
1532 }
1533 
TEST_F(PerformRequestsTest,FetchResourcesInMemoryPutInCacheReturnsInternalDoesntCrash)1534 TEST_F(PerformRequestsTest,
1535        FetchResourcesInMemoryPutInCacheReturnsInternalDoesntCrash) {
1536   const std::string uri = "https://valid.com/1";
1537   const std::string cache_id = "(^˵◕ω◕˵^)";
1538   absl::Cord expected_response_body("(((*°▽°*)八(*°▽°*)))");
1539   absl::Duration max_age = absl::Hours(1);
1540   int expected_response_code = kHttpOk;
1541   std::string content_type = "bytes";
1542   auto resource = UriOrInlineData::CreateUri(uri, cache_id, max_age);
1543   StrictMock<cache::MockResourceCache> resource_cache;
1544 
1545   EXPECT_CALL(resource_cache, Get(cache_id, std::make_optional(max_age)))
1546       .WillOnce(Return(absl::NotFoundError("not in the cache sorry!")));
1547 
1548   EXPECT_CALL(mock_http_client_,
1549               PerformSingleRequest(
1550                   FieldsAre(uri, HttpRequest::Method::kGet, HeaderList{}, "")))
1551       .WillOnce(Return(FakeHttpResponse(expected_response_code,
1552                                         {{kContentTypeHdr, content_type}},
1553                                         std::string(expected_response_body))));
1554 
1555   EXPECT_CALL(resource_cache,
1556               Put(cache_id, expected_response_body,
1557                   EqualsProto(MetadataForUncompressedResource()), max_age))
1558       .WillOnce(Return(absl::InternalError("the cache exploded -_-")));
1559 
1560   int64_t bytes_received = 0;
1561   int64_t bytes_sent = 0;
1562   auto result = FetchResourcesInMemory(
1563       mock_http_client_, interruptible_runner_, {resource},
1564       // We pass in non-null pointers for the network
1565       // stats, to ensure they are correctly updated.
1566       &bytes_received, &bytes_sent, &resource_cache);
1567   ASSERT_OK(result);
1568 
1569   ASSERT_OK((*result)[0]);
1570   EXPECT_THAT(*(*result)[0],
1571               FieldsAre(expected_response_code, IsEmpty(), content_type,
1572                         StrEq(expected_response_body)));
1573 }
1574 
1575 }  // namespace
1576 }  // namespace fcp::client::http
1577