1 // Copyright 2012 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "net/url_request/url_request_http_job.h"
6
7 #include <algorithm>
8 #include <iterator>
9 #include <memory>
10 #include <optional>
11 #include <string_view>
12 #include <utility>
13 #include <vector>
14
15 #include "base/base_switches.h"
16 #include "base/check_op.h"
17 #include "base/command_line.h"
18 #include "base/compiler_specific.h"
19 #include "base/containers/adapters.h"
20 #include "base/feature_list.h"
21 #include "base/file_version_info.h"
22 #include "base/functional/bind.h"
23 #include "base/functional/callback_helpers.h"
24 #include "base/location.h"
25 #include "base/memory/ptr_util.h"
26 #include "base/metrics/field_trial.h"
27 #include "base/metrics/histogram_functions.h"
28 #include "base/metrics/histogram_macros.h"
29 #include "base/numerics/safe_conversions.h"
30 #include "base/rand_util.h"
31 #include "base/strings/string_number_conversions.h"
32 #include "base/strings/string_util.h"
33 #include "base/strings/stringprintf.h"
34 #include "base/task/single_thread_task_runner.h"
35 #include "base/time/time.h"
36 #include "base/types/optional_util.h"
37 #include "base/values.h"
38 #include "build/build_config.h"
39 #include "net/base/host_port_pair.h"
40 #include "net/base/http_user_agent_settings.h"
41 #include "net/base/load_flags.h"
42 #include "net/base/net_errors.h"
43 #include "net/base/network_anonymization_key.h"
44 #include "net/base/network_delegate.h"
45 #include "net/base/network_isolation_key.h"
46 #include "net/base/privacy_mode.h"
47 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
48 #include "net/base/schemeful_site.h"
49 #include "net/base/trace_constants.h"
50 #include "net/base/tracing.h"
51 #include "net/base/url_util.h"
52 #include "net/cert/cert_status_flags.h"
53 #include "net/cert/ct_policy_status.h"
54 #include "net/cert/known_roots.h"
55 #include "net/cookies/canonical_cookie.h"
56 #include "net/cookies/cookie_access_delegate.h"
57 #include "net/cookies/cookie_constants.h"
58 #include "net/cookies/cookie_store.h"
59 #include "net/cookies/cookie_util.h"
60 #include "net/cookies/parsed_cookie.h"
61 #include "net/filter/brotli_source_stream.h"
62 #include "net/filter/filter_source_stream.h"
63 #include "net/filter/gzip_source_stream.h"
64 #include "net/filter/source_stream.h"
65 #include "net/filter/zstd_source_stream.h"
66 #include "net/first_party_sets/first_party_set_entry.h"
67 #include "net/first_party_sets/first_party_set_metadata.h"
68 #include "net/first_party_sets/first_party_sets_cache_filter.h"
69 #include "net/http/http_content_disposition.h"
70 #include "net/http/http_log_util.h"
71 #include "net/http/http_network_session.h"
72 #include "net/http/http_request_headers.h"
73 #include "net/http/http_response_headers.h"
74 #include "net/http/http_response_info.h"
75 #include "net/http/http_status_code.h"
76 #include "net/http/http_transaction.h"
77 #include "net/http/http_transaction_factory.h"
78 #include "net/http/http_util.h"
79 #include "net/http/transport_security_state.h"
80 #include "net/log/net_log.h"
81 #include "net/log/net_log_event_type.h"
82 #include "net/log/net_log_values.h"
83 #include "net/log/net_log_with_source.h"
84 #include "net/nqe/network_quality_estimator.h"
85 #include "net/proxy_resolution/proxy_info.h"
86 #include "net/proxy_resolution/proxy_resolution_service.h"
87 #include "net/proxy_resolution/proxy_retry_info.h"
88 #include "net/ssl/ssl_cert_request_info.h"
89 #include "net/ssl/ssl_config_service.h"
90 #include "net/ssl/ssl_connection_status_flags.h"
91 #include "net/url_request/clear_site_data.h"
92 #include "net/url_request/redirect_util.h"
93 #include "net/url_request/url_request.h"
94 #include "net/url_request/url_request_context.h"
95 #include "net/url_request/url_request_error_job.h"
96 #include "net/url_request/url_request_job_factory.h"
97 #include "net/url_request/url_request_redirect_job.h"
98 #include "net/url_request/websocket_handshake_userdata_key.h"
99 #include "url/gurl.h"
100 #include "url/origin.h"
101 #include "url/url_constants.h"
102
103 #if BUILDFLAG(IS_ANDROID)
104 #include "net/android/network_library.h"
105 #endif
106
107 #if BUILDFLAG(ENABLE_DEVICE_BOUND_SESSIONS)
108 #include "net/device_bound_sessions/device_bound_session_registration_fetcher_param.h"
109 #include "net/device_bound_sessions/device_bound_session_service.h"
110 #endif // BUILDFLAG(ENABLE_DEVICE_BOUND_SESSIONS)
111
112 namespace {
113
114 // These values are persisted to logs. Entries should not be renumbered and
115 // numeric values should never be reused.
116 enum class TpcdHeaderStatus {
117 kSet = 0,
118 kNoLabel = 1,
119 kNoCookie = 2,
120 kMaxValue = kNoCookie,
121 };
122
RecordTpcdHeaderStatus(TpcdHeaderStatus status)123 void RecordTpcdHeaderStatus(TpcdHeaderStatus status) {
124 base::UmaHistogramEnumeration("Privacy.3PCD.SecCookieDeprecationHeaderStatus",
125 status);
126 }
127
FirstPartySetMetadataNetLogParams(const net::FirstPartySetMetadata & first_party_set_metadata,const int64_t * const fps_cache_filter)128 base::Value::Dict FirstPartySetMetadataNetLogParams(
129 const net::FirstPartySetMetadata& first_party_set_metadata,
130 const int64_t* const fps_cache_filter) {
131 base::Value::Dict dict;
132 auto entry_or_empty =
133 [](const std::optional<net::FirstPartySetEntry>& entry) -> std::string {
134 return entry.has_value() ? entry->GetDebugString() : "none";
135 };
136
137 dict.Set("cache_filter",
138 fps_cache_filter ? base::NumberToString(*fps_cache_filter) : "none");
139 dict.Set("frame_entry",
140 entry_or_empty(first_party_set_metadata.frame_entry()));
141 dict.Set("top_frame_primary",
142 entry_or_empty(first_party_set_metadata.top_frame_entry()));
143 return dict;
144 }
145
CookieInclusionStatusNetLogParams(const std::string & operation,const std::string & cookie_name,const std::string & cookie_domain,const std::string & cookie_path,const net::CookieInclusionStatus & status,net::NetLogCaptureMode capture_mode)146 base::Value::Dict CookieInclusionStatusNetLogParams(
147 const std::string& operation,
148 const std::string& cookie_name,
149 const std::string& cookie_domain,
150 const std::string& cookie_path,
151 const net::CookieInclusionStatus& status,
152 net::NetLogCaptureMode capture_mode) {
153 base::Value::Dict dict;
154 dict.Set("operation", operation);
155 dict.Set("status", status.GetDebugString());
156 if (net::NetLogCaptureIncludesSensitive(capture_mode)) {
157 if (!cookie_name.empty())
158 dict.Set("name", cookie_name);
159 if (!cookie_domain.empty())
160 dict.Set("domain", cookie_domain);
161 if (!cookie_path.empty())
162 dict.Set("path", cookie_path);
163 }
164 return dict;
165 }
166
167 // Records details about the most-specific trust anchor in |spki_hashes|,
168 // which is expected to be ordered with the leaf cert first and the root cert
169 // last. This complements the per-verification histogram
170 // Net.Certificate.TrustAnchor.Verify
LogTrustAnchor(const net::HashValueVector & spki_hashes)171 void LogTrustAnchor(const net::HashValueVector& spki_hashes) {
172 // Don't record metrics if there are no hashes; this is true if the HTTP
173 // load did not come from an active network connection, such as the disk
174 // cache or a synthesized response.
175 if (spki_hashes.empty())
176 return;
177
178 int32_t id = 0;
179 for (const auto& hash : spki_hashes) {
180 id = net::GetNetTrustAnchorHistogramIdForSPKI(hash);
181 if (id != 0)
182 break;
183 }
184 base::UmaHistogramSparse("Net.Certificate.TrustAnchor.Request", id);
185 }
186
CreateCookieOptions(net::CookieOptions::SameSiteCookieContext same_site_context)187 net::CookieOptions CreateCookieOptions(
188 net::CookieOptions::SameSiteCookieContext same_site_context) {
189 net::CookieOptions options;
190 options.set_return_excluded_cookies();
191 options.set_include_httponly();
192 options.set_same_site_cookie_context(same_site_context);
193 return options;
194 }
195
IsTLS13OverTCP(const net::HttpResponseInfo & response_info)196 bool IsTLS13OverTCP(const net::HttpResponseInfo& response_info) {
197 // Although IETF QUIC also uses TLS 1.3, our QUIC connections report
198 // SSL_CONNECTION_VERSION_QUIC.
199 return net::SSLConnectionStatusToVersion(
200 response_info.ssl_info.connection_status) ==
201 net::SSL_CONNECTION_VERSION_TLS1_3;
202 }
203
UpgradeSchemeToCryptographic(const GURL & insecure_url)204 GURL UpgradeSchemeToCryptographic(const GURL& insecure_url) {
205 DCHECK(!insecure_url.SchemeIsCryptographic());
206 DCHECK(insecure_url.SchemeIs(url::kHttpScheme) ||
207 insecure_url.SchemeIs(url::kWsScheme));
208
209 GURL::Replacements replacements;
210 replacements.SetSchemeStr(insecure_url.SchemeIs(url::kHttpScheme)
211 ? url::kHttpsScheme
212 : url::kWssScheme);
213
214 GURL secure_url = insecure_url.ReplaceComponents(replacements);
215 DCHECK(secure_url.SchemeIsCryptographic());
216
217 return secure_url;
218 }
219
220 // These values are persisted to logs. Entries should not be renumbered and
221 // numeric values should never be reused.
222 enum class ContentEncodingType {
223 kUnknown = 0,
224 kBrotli = 1,
225 kGZip = 2,
226 kDeflate = 3,
227 kZstd = 4,
228 kMaxValue = kZstd,
229 };
230
IsSameSiteIgnoringWebSocketProtocol(const net::SchemefulSite & initiator,const GURL & request_url)231 bool IsSameSiteIgnoringWebSocketProtocol(const net::SchemefulSite& initiator,
232 const GURL& request_url) {
233 net::SchemefulSite request_site = net::SchemefulSite(
234 request_url.SchemeIsHTTPOrHTTPS()
235 ? request_url
236 : net::ChangeWebSocketSchemeToHttpScheme(request_url));
237 return initiator == request_site;
238 }
239
240 // These values are persisted to logs. Entries should not be renumbered and
241 // numeric values should never be reused.
242 enum class HttpRequestStsState {
243 kUnknown = 0,
244 kUnprotectedHttps = 1,
245 kProtectedHttps = 2,
246 kUnprotectedHttp = 3,
247 kProtectedHttp = 4,
248 kMaxValue = kProtectedHttp,
249 };
250
RecordSTSHistogram(bool sts_enabled,bool is_secure,int load_flags)251 void RecordSTSHistogram(bool sts_enabled, bool is_secure, int load_flags) {
252 // Embrace the layering violation and only record the histogram for main frame
253 // navigations. It's possible to record this outside of net/, but the code is
254 // a lot more complicated, and while this flag is deprecated, there are no
255 // current plans to remove it. See crbug.com/516499 .
256 if (!(load_flags & net::LOAD_MAIN_FRAME_DEPRECATED)) {
257 return;
258 }
259 HttpRequestStsState sts_state = HttpRequestStsState::kUnknown;
260 if (is_secure) {
261 sts_state = (sts_enabled ? HttpRequestStsState::kProtectedHttps
262 : HttpRequestStsState::kUnprotectedHttps);
263 } else {
264 sts_state = (sts_enabled ? HttpRequestStsState::kProtectedHttp
265 : HttpRequestStsState::kUnprotectedHttp);
266 }
267 UMA_HISTOGRAM_ENUMERATION("Net.HttpRequestStsState", sts_state);
268 }
269
270 } // namespace
271
272 namespace net {
273
Create(URLRequest * request)274 std::unique_ptr<URLRequestJob> URLRequestHttpJob::Create(URLRequest* request) {
275 const GURL& url = request->url();
276
277 // URLRequestContext must have been initialized.
278 DCHECK(request->context()->http_transaction_factory());
279 DCHECK(url.SchemeIsHTTPOrHTTPS() || url.SchemeIsWSOrWSS());
280
281 TransportSecurityState* hsts = request->context()->transport_security_state();
282 bool should_upgrade_to_ssl =
283 hsts && hsts->ShouldUpgradeToSSL(url.host(), request->net_log());
284
285 // Check for reasons not to return a URLRequestHttpJob. These don't apply to
286 // https and wss requests.
287 if (!url.SchemeIsCryptographic()) {
288 // If the request explicitly has been marked to bypass HSTS, ensure that
289 // the request is in no-credential mode so that the http site can't read
290 // or set cookies which are shared across http/https, then skip the
291 // upgrade.
292 if (((request->load_flags() & net::LOAD_SHOULD_BYPASS_HSTS) ==
293 net::LOAD_SHOULD_BYPASS_HSTS)) {
294 CHECK(request->allow_credentials() == false);
295 } else {
296 // Check for HSTS upgrade.
297 if (should_upgrade_to_ssl) {
298 RecordSTSHistogram(/*sts_enabled=*/true, /*is_secure=*/false,
299 request->load_flags());
300 return std::make_unique<URLRequestRedirectJob>(
301 request, UpgradeSchemeToCryptographic(url),
302 // Use status code 307 to preserve the method, so POST requests
303 // work.
304 RedirectUtil::ResponseCode::REDIRECT_307_TEMPORARY_REDIRECT,
305 "HSTS");
306 }
307 }
308
309 #if BUILDFLAG(IS_ANDROID)
310 // Check whether the app allows cleartext traffic to this host, and return
311 // ERR_CLEARTEXT_NOT_PERMITTED if not.
312 if (request->context()->check_cleartext_permitted() &&
313 !android::IsCleartextPermitted(url.host_piece())) {
314 RecordSTSHistogram(/*sts_enabled=*/false, /*is_secure=*/false,
315 request->load_flags());
316 return std::make_unique<URLRequestErrorJob>(request,
317 ERR_CLEARTEXT_NOT_PERMITTED);
318 }
319 #endif
320 }
321
322 RecordSTSHistogram(should_upgrade_to_ssl, url.SchemeIsCryptographic(),
323 request->load_flags());
324 return base::WrapUnique<URLRequestJob>(new URLRequestHttpJob(
325 request, request->context()->http_user_agent_settings()));
326 }
327
URLRequestHttpJob(URLRequest * request,const HttpUserAgentSettings * http_user_agent_settings)328 URLRequestHttpJob::URLRequestHttpJob(
329 URLRequest* request,
330 const HttpUserAgentSettings* http_user_agent_settings)
331 : URLRequestJob(request),
332 http_user_agent_settings_(http_user_agent_settings) {
333 ResetTimer();
334 }
335
~URLRequestHttpJob()336 URLRequestHttpJob::~URLRequestHttpJob() {
337 CHECK(!awaiting_callback_);
338
339 DoneWithRequest(ABORTED);
340 }
341
SetPriority(RequestPriority priority)342 void URLRequestHttpJob::SetPriority(RequestPriority priority) {
343 priority_ = priority;
344 if (transaction_)
345 transaction_->SetPriority(priority_);
346 }
347
Start()348 void URLRequestHttpJob::Start() {
349 DCHECK(!transaction_.get());
350
351 request_info_.url = request_->url();
352 request_info_.method = request_->method();
353
354 request_info_.network_isolation_key =
355 request_->isolation_info().network_isolation_key();
356 request_info_.network_anonymization_key =
357 request_->isolation_info().network_anonymization_key();
358 request_info_.possibly_top_frame_origin =
359 request_->isolation_info().top_frame_origin();
360 request_info_.frame_origin = request_->isolation_info().frame_origin();
361 request_info_.is_subframe_document_resource =
362 request_->isolation_info().request_type() ==
363 net::IsolationInfo::RequestType::kSubFrame;
364 request_info_.load_flags = request_->load_flags();
365 request_info_.priority_incremental = request_->priority_incremental();
366 request_info_.secure_dns_policy = request_->secure_dns_policy();
367 request_info_.traffic_annotation =
368 net::MutableNetworkTrafficAnnotationTag(request_->traffic_annotation());
369 request_info_.socket_tag = request_->socket_tag();
370 request_info_.idempotency = request_->GetIdempotency();
371 #if BUILDFLAG(ENABLE_REPORTING)
372 request_info_.reporting_upload_depth = request_->reporting_upload_depth();
373 #endif
374
375 // Add/remove the Storage Access override enum based on whether the request's
376 // url and initiator are same-site, to prevent cross-site sibling iframes
377 // benefit from each other's storage access API grants.
378 request()->cookie_setting_overrides().PutOrRemove(
379 net::CookieSettingOverride::kStorageAccessGrantEligible,
380 request()->has_storage_access() && request_initiator_site().has_value() &&
381 IsSameSiteIgnoringWebSocketProtocol(request_initiator_site().value(),
382 request()->url()));
383
384 UMA_HISTOGRAM_BOOLEAN("Net.HttpJob.CanIncludeCookies",
385 ShouldAddCookieHeader());
386
387 CookieStore* cookie_store = request()->context()->cookie_store();
388 const CookieAccessDelegate* delegate =
389 cookie_store ? cookie_store->cookie_access_delegate() : nullptr;
390
391 request_->net_log().BeginEvent(NetLogEventType::FIRST_PARTY_SETS_METADATA);
392
393 std::optional<
394 std::pair<FirstPartySetMetadata, FirstPartySetsCacheFilter::MatchInfo>>
395 maybe_metadata = cookie_util::ComputeFirstPartySetMetadataMaybeAsync(
396 SchemefulSite(request()->url()), request()->isolation_info(),
397 delegate,
398 base::BindOnce(&URLRequestHttpJob::OnGotFirstPartySetMetadata,
399 weak_factory_.GetWeakPtr()));
400
401 if (maybe_metadata.has_value()) {
402 auto [metadata, match_info] = std::move(maybe_metadata).value();
403 OnGotFirstPartySetMetadata(std::move(metadata), std::move(match_info));
404 }
405 }
406
OnGotFirstPartySetMetadata(FirstPartySetMetadata first_party_set_metadata,FirstPartySetsCacheFilter::MatchInfo match_info)407 void URLRequestHttpJob::OnGotFirstPartySetMetadata(
408 FirstPartySetMetadata first_party_set_metadata,
409 FirstPartySetsCacheFilter::MatchInfo match_info) {
410 first_party_set_metadata_ = std::move(first_party_set_metadata);
411 request_info_.fps_cache_filter = match_info.clear_at_run_id;
412 request_info_.browser_run_id = match_info.browser_run_id;
413
414 request_->net_log().EndEvent(
415 NetLogEventType::FIRST_PARTY_SETS_METADATA, [&]() {
416 return FirstPartySetMetadataNetLogParams(
417 first_party_set_metadata_,
418 base::OptionalToPtr(request_info_.fps_cache_filter));
419 });
420
421 // Privacy mode could still be disabled in SetCookieHeaderAndStart if we are
422 // going to send previously saved cookies.
423 request_info_.privacy_mode = DeterminePrivacyMode();
424 request()->net_log().AddEventWithStringParams(
425 NetLogEventType::COMPUTED_PRIVACY_MODE, "privacy_mode",
426 PrivacyModeToDebugString(request_info_.privacy_mode));
427
428 // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
429 // from overriding headers that are controlled using other means. Otherwise a
430 // plugin could set a referrer although sending the referrer is inhibited.
431 request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
432
433 // URLRequest::SetReferrer ensures that we do not send username and password
434 // fields in the referrer.
435 GURL referrer(request_->referrer());
436
437 // Our consumer should have made sure that this is a safe referrer (e.g. via
438 // URLRequestJob::ComputeReferrerForPolicy).
439 if (referrer.is_valid()) {
440 std::string referer_value = referrer.spec();
441 request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
442 referer_value);
443 }
444
445 request_info_.extra_headers.SetHeaderIfMissing(
446 HttpRequestHeaders::kUserAgent,
447 http_user_agent_settings_ ?
448 http_user_agent_settings_->GetUserAgent() : std::string());
449
450 AddExtraHeaders();
451
452 if (ShouldAddCookieHeader()) {
453 AddCookieHeaderAndStart();
454 } else {
455 StartTransaction();
456 }
457 }
458
Kill()459 void URLRequestHttpJob::Kill() {
460 weak_factory_.InvalidateWeakPtrs();
461 if (transaction_)
462 DestroyTransaction();
463 URLRequestJob::Kill();
464 }
465
GetConnectionAttempts() const466 ConnectionAttempts URLRequestHttpJob::GetConnectionAttempts() const {
467 if (transaction_)
468 return transaction_->GetConnectionAttempts();
469 return {};
470 }
471
CloseConnectionOnDestruction()472 void URLRequestHttpJob::CloseConnectionOnDestruction() {
473 DCHECK(transaction_);
474 transaction_->CloseConnectionOnDestruction();
475 }
476
NotifyConnectedCallback(const TransportInfo & info,CompletionOnceCallback callback)477 int URLRequestHttpJob::NotifyConnectedCallback(
478 const TransportInfo& info,
479 CompletionOnceCallback callback) {
480 return URLRequestJob::NotifyConnected(info, std::move(callback));
481 }
482
DeterminePrivacyMode() const483 PrivacyMode URLRequestHttpJob::DeterminePrivacyMode() const {
484 if (!request()->allow_credentials()) {
485 // |allow_credentials_| implies LOAD_DO_NOT_SAVE_COOKIES.
486 DCHECK(request_->load_flags() & LOAD_DO_NOT_SAVE_COOKIES);
487
488 // TODO(https://crbug.com/775438): Client certs should always be
489 // affirmatively omitted for these requests.
490 return request()->send_client_certs()
491 ? PRIVACY_MODE_ENABLED
492 : PRIVACY_MODE_ENABLED_WITHOUT_CLIENT_CERTS;
493 }
494
495 // Otherwise, check with the delegate if present, or base it off of
496 // |URLRequest::DefaultCanUseCookies()| if not.
497 // TODO(mmenke): Looks like |URLRequest::DefaultCanUseCookies()| is not too
498 // useful, with the network service - remove it.
499 NetworkDelegate::PrivacySetting privacy_setting =
500 URLRequest::DefaultCanUseCookies()
501 ? NetworkDelegate::PrivacySetting::kStateAllowed
502 : NetworkDelegate::PrivacySetting::kStateDisallowed;
503 if (request_->network_delegate()) {
504 privacy_setting =
505 request()->network_delegate()->ForcePrivacyMode(*request());
506 }
507 switch (privacy_setting) {
508 case NetworkDelegate::PrivacySetting::kStateAllowed:
509 return PRIVACY_MODE_DISABLED;
510 case NetworkDelegate::PrivacySetting::kPartitionedStateAllowedOnly:
511 return PRIVACY_MODE_ENABLED_PARTITIONED_STATE_ALLOWED;
512 case NetworkDelegate::PrivacySetting::kStateDisallowed:
513 return PRIVACY_MODE_ENABLED;
514 }
515 NOTREACHED();
516 return PRIVACY_MODE_ENABLED;
517 }
518
NotifyHeadersComplete()519 void URLRequestHttpJob::NotifyHeadersComplete() {
520 DCHECK(!response_info_);
521 DCHECK_EQ(0, num_cookie_lines_left_);
522 DCHECK(request_->maybe_stored_cookies().empty());
523
524 if (override_response_info_) {
525 DCHECK(!transaction_);
526 response_info_ = override_response_info_.get();
527 } else {
528 response_info_ = transaction_->GetResponseInfo();
529 }
530
531 ProcessStrictTransportSecurityHeader();
532 #if BUILDFLAG(ENABLE_DEVICE_BOUND_SESSIONS)
533 ProcessDeviceBoundSessionsHeader();
534 #endif // BUILDFLAG(ENABLE_DEVICE_BOUND_SESSIONS)
535
536 // Clear |set_cookie_access_result_list_| after any processing in case
537 // SaveCookiesAndNotifyHeadersComplete is called again.
538 request_->set_maybe_stored_cookies(std::move(set_cookie_access_result_list_));
539
540 // The HTTP transaction may be restarted several times for the purposes
541 // of sending authorization information. Each time it restarts, we get
542 // notified of the headers completion so that we can update the cookie store.
543 if (transaction_ && transaction_->IsReadyToRestartForAuth()) {
544 // TODO(battre): This breaks the webrequest API for
545 // URLRequestTestHTTP.BasicAuthWithCookies
546 // where OnBeforeStartTransaction -> OnStartTransaction ->
547 // OnBeforeStartTransaction occurs.
548 RestartTransactionWithAuth(AuthCredentials());
549 return;
550 }
551
552 URLRequestJob::NotifyHeadersComplete();
553 }
554
DestroyTransaction()555 void URLRequestHttpJob::DestroyTransaction() {
556 DCHECK(transaction_.get());
557
558 DoneWithRequest(ABORTED);
559
560 total_received_bytes_from_previous_transactions_ +=
561 transaction_->GetTotalReceivedBytes();
562 total_sent_bytes_from_previous_transactions_ +=
563 transaction_->GetTotalSentBytes();
564 response_info_ = nullptr;
565 transaction_.reset();
566 override_response_headers_ = nullptr;
567 receive_headers_end_ = base::TimeTicks();
568 }
569
StartTransaction()570 void URLRequestHttpJob::StartTransaction() {
571 DCHECK(!override_response_info_);
572
573 NetworkDelegate* network_delegate = request()->network_delegate();
574 if (network_delegate) {
575 OnCallToDelegate(
576 NetLogEventType::NETWORK_DELEGATE_BEFORE_START_TRANSACTION);
577 int rv = network_delegate->NotifyBeforeStartTransaction(
578 request_, request_info_.extra_headers,
579 base::BindOnce(&URLRequestHttpJob::NotifyBeforeStartTransactionCallback,
580 weak_factory_.GetWeakPtr()));
581 // If an extension blocks the request, we rely on the callback to
582 // MaybeStartTransactionInternal().
583 if (rv == ERR_IO_PENDING)
584 return;
585 MaybeStartTransactionInternal(rv);
586 return;
587 }
588 StartTransactionInternal();
589 }
590
NotifyBeforeStartTransactionCallback(int result,const std::optional<HttpRequestHeaders> & headers)591 void URLRequestHttpJob::NotifyBeforeStartTransactionCallback(
592 int result,
593 const std::optional<HttpRequestHeaders>& headers) {
594 // The request should not have been cancelled or have already completed.
595 DCHECK(!is_done());
596
597 if (headers)
598 request_info_.extra_headers = headers.value();
599 MaybeStartTransactionInternal(result);
600 }
601
MaybeStartTransactionInternal(int result)602 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
603 OnCallToDelegateComplete();
604 if (result == OK) {
605 StartTransactionInternal();
606 } else {
607 request_->net_log().AddEventWithStringParams(NetLogEventType::CANCELLED,
608 "source", "delegate");
609 // Don't call back synchronously to the delegate.
610 base::SingleThreadTaskRunner::GetCurrentDefault()->PostTask(
611 FROM_HERE, base::BindOnce(&URLRequestHttpJob::NotifyStartError,
612 weak_factory_.GetWeakPtr(), result));
613 }
614 }
615
StartTransactionInternal()616 void URLRequestHttpJob::StartTransactionInternal() {
617 DCHECK(!override_response_headers_);
618
619 // NOTE: This method assumes that request_info_ is already setup properly.
620
621 // If we already have a transaction, then we should restart the transaction
622 // with auth provided by auth_credentials_.
623
624 int rv;
625
626 // Notify NetworkQualityEstimator.
627 NetworkQualityEstimator* network_quality_estimator =
628 request()->context()->network_quality_estimator();
629 if (network_quality_estimator)
630 network_quality_estimator->NotifyStartTransaction(*request_);
631
632 if (transaction_.get()) {
633 rv = transaction_->RestartWithAuth(
634 auth_credentials_, base::BindOnce(&URLRequestHttpJob::OnStartCompleted,
635 base::Unretained(this)));
636 auth_credentials_ = AuthCredentials();
637 } else {
638 DCHECK(request_->context()->http_transaction_factory());
639
640 rv = request_->context()->http_transaction_factory()->CreateTransaction(
641 priority_, &transaction_);
642
643 if (rv == OK && request_info_.url.SchemeIsWSOrWSS()) {
644 base::SupportsUserData::Data* data =
645 request_->GetUserData(kWebSocketHandshakeUserDataKey);
646 if (data) {
647 transaction_->SetWebSocketHandshakeStreamCreateHelper(
648 static_cast<WebSocketHandshakeStreamBase::CreateHelper*>(data));
649 } else {
650 rv = ERR_DISALLOWED_URL_SCHEME;
651 }
652 }
653
654 if (rv == OK && request_info_.method == "CONNECT") {
655 // CONNECT has different kinds of targets than other methods (RFC 9110,
656 // section 9.3.6), which are incompatible with URLRequest.
657 rv = ERR_METHOD_NOT_SUPPORTED;
658 }
659
660 if (rv == OK) {
661 transaction_->SetConnectedCallback(base::BindRepeating(
662 &URLRequestHttpJob::NotifyConnectedCallback, base::Unretained(this)));
663 transaction_->SetRequestHeadersCallback(request_headers_callback_);
664 transaction_->SetEarlyResponseHeadersCallback(
665 early_response_headers_callback_);
666 transaction_->SetResponseHeadersCallback(response_headers_callback_);
667 if (is_shared_dictionary_read_allowed_callback_) {
668 transaction_->SetIsSharedDictionaryReadAllowedCallback(
669 is_shared_dictionary_read_allowed_callback_);
670 }
671
672 rv = transaction_->Start(
673 &request_info_,
674 base::BindOnce(&URLRequestHttpJob::OnStartCompleted,
675 base::Unretained(this)),
676 request_->net_log());
677 start_time_ = base::TimeTicks::Now();
678 }
679 }
680
681 if (rv == ERR_IO_PENDING)
682 return;
683
684 // The transaction started synchronously, but we need to notify the
685 // URLRequest delegate via the message loop.
686 base::SingleThreadTaskRunner::GetCurrentDefault()->PostTask(
687 FROM_HERE, base::BindOnce(&URLRequestHttpJob::OnStartCompleted,
688 weak_factory_.GetWeakPtr(), rv));
689 }
690
AddExtraHeaders()691 void URLRequestHttpJob::AddExtraHeaders() {
692 request_info_.extra_headers.SetAcceptEncodingIfMissing(
693 request()->url(), request()->accepted_stream_types(),
694 request()->context()->enable_brotli(),
695 request()->context()->enable_zstd());
696
697 if (http_user_agent_settings_) {
698 // Only add default Accept-Language if the request didn't have it
699 // specified.
700 std::string accept_language =
701 http_user_agent_settings_->GetAcceptLanguage();
702 if (!accept_language.empty()) {
703 request_info_.extra_headers.SetHeaderIfMissing(
704 HttpRequestHeaders::kAcceptLanguage,
705 accept_language);
706 }
707 }
708 }
709
AddCookieHeaderAndStart()710 void URLRequestHttpJob::AddCookieHeaderAndStart() {
711 CookieStore* cookie_store = request_->context()->cookie_store();
712 DCHECK(cookie_store);
713 DCHECK(ShouldAddCookieHeader());
714 bool force_ignore_site_for_cookies =
715 request_->force_ignore_site_for_cookies();
716 if (cookie_store->cookie_access_delegate() &&
717 cookie_store->cookie_access_delegate()->ShouldIgnoreSameSiteRestrictions(
718 request_->url(), request_->site_for_cookies())) {
719 force_ignore_site_for_cookies = true;
720 }
721 bool is_main_frame_navigation =
722 IsolationInfo::RequestType::kMainFrame ==
723 request_->isolation_info().request_type() ||
724 request_->force_main_frame_for_same_site_cookies();
725 CookieOptions::SameSiteCookieContext same_site_context =
726 net::cookie_util::ComputeSameSiteContextForRequest(
727 request_->method(), request_->url_chain(),
728 request_->site_for_cookies(), request_->initiator(),
729 is_main_frame_navigation, force_ignore_site_for_cookies);
730
731 CookieOptions options = CreateCookieOptions(same_site_context);
732
733 cookie_store->GetCookieListWithOptionsAsync(
734 request_->url(), options,
735 CookiePartitionKeyCollection::FromOptional(
736 request_->cookie_partition_key()),
737 base::BindOnce(&URLRequestHttpJob::SetCookieHeaderAndStart,
738 weak_factory_.GetWeakPtr(), options));
739 }
740
741 namespace {
742
ShouldBlockAllCookies(const PrivacyMode & privacy_mode)743 bool ShouldBlockAllCookies(const PrivacyMode& privacy_mode) {
744 return privacy_mode == PRIVACY_MODE_ENABLED ||
745 privacy_mode == PRIVACY_MODE_ENABLED_WITHOUT_CLIENT_CERTS;
746 }
747
748 } // namespace
749
SetCookieHeaderAndStart(const CookieOptions & options,const CookieAccessResultList & cookies_with_access_result_list,const CookieAccessResultList & excluded_list)750 void URLRequestHttpJob::SetCookieHeaderAndStart(
751 const CookieOptions& options,
752 const CookieAccessResultList& cookies_with_access_result_list,
753 const CookieAccessResultList& excluded_list) {
754 DCHECK(request_->maybe_sent_cookies().empty());
755
756 CookieAccessResultList maybe_included_cookies =
757 cookies_with_access_result_list;
758 CookieAccessResultList excluded_cookies = excluded_list;
759
760 if (ShouldBlockAllCookies(request_info_.privacy_mode)) {
761 // If cookies are blocked (without our needing to consult the delegate),
762 // we move them to `excluded_cookies` and ensure that they have the
763 // correct exclusion reason.
764 excluded_cookies.insert(
765 excluded_cookies.end(),
766 std::make_move_iterator(maybe_included_cookies.begin()),
767 std::make_move_iterator(maybe_included_cookies.end()));
768 maybe_included_cookies.clear();
769 for (auto& cookie : excluded_cookies) {
770 cookie.access_result.status.AddExclusionReason(
771 CookieInclusionStatus::EXCLUDE_USER_PREFERENCES);
772 }
773 } else {
774 // Consult the delegate to ensure that they have the correct exclusion
775 // reason.
776 AnnotateAndMoveUserBlockedCookies(maybe_included_cookies, excluded_cookies);
777 }
778
779 const bool cookie_deprecation_testing_enabled =
780 request_->context()->cookie_deprecation_label().has_value();
781 const bool cookie_deprecation_testing_has_label =
782 cookie_deprecation_testing_enabled &&
783 !request_->context()->cookie_deprecation_label().value().empty();
784 bool may_set_sec_cookie_deprecation_header =
785 cookie_deprecation_testing_has_label;
786
787 if (!maybe_included_cookies.empty()) {
788 std::string cookie_line =
789 CanonicalCookie::BuildCookieLine(maybe_included_cookies);
790 request_info_.extra_headers.SetHeader(HttpRequestHeaders::kCookie,
791 cookie_line);
792
793 size_t n_partitioned_cookies = 0;
794
795 // TODO(crbug.com/1031664): Reduce the number of times the cookie list
796 // is iterated over. Get metrics for every cookie which is included.
797 for (const auto& c : maybe_included_cookies) {
798 bool request_is_secure = request_->url().SchemeIsCryptographic();
799 net::CookieSourceScheme cookie_scheme = c.cookie.SourceScheme();
800 CookieRequestScheme cookie_request_schemes;
801
802 switch (cookie_scheme) {
803 case net::CookieSourceScheme::kSecure:
804 cookie_request_schemes =
805 request_is_secure
806 ? CookieRequestScheme::kSecureSetSecureRequest
807 : CookieRequestScheme::kSecureSetNonsecureRequest;
808 break;
809
810 case net::CookieSourceScheme::kNonSecure:
811 cookie_request_schemes =
812 request_is_secure
813 ? CookieRequestScheme::kNonsecureSetSecureRequest
814 : CookieRequestScheme::kNonsecureSetNonsecureRequest;
815 break;
816
817 case net::CookieSourceScheme::kUnset:
818 cookie_request_schemes = CookieRequestScheme::kUnsetCookieScheme;
819 break;
820 }
821
822 UMA_HISTOGRAM_ENUMERATION("Cookie.CookieSchemeRequestScheme",
823 cookie_request_schemes);
824 if (c.cookie.IsPartitioned()) {
825 ++n_partitioned_cookies;
826
827 if (may_set_sec_cookie_deprecation_header &&
828 c.cookie.Name() == "receive-cookie-deprecation" &&
829 c.cookie.IsHttpOnly() && c.cookie.SecureAttribute()) {
830 request_info_.extra_headers.SetHeader(
831 "Sec-Cookie-Deprecation",
832 *request_->context()->cookie_deprecation_label());
833 may_set_sec_cookie_deprecation_header = false;
834 }
835 }
836 }
837
838 if (ShouldRecordPartitionedCookieUsage()) {
839 base::UmaHistogramCounts100("Cookie.PartitionedCookiesInRequest",
840 n_partitioned_cookies);
841 }
842 }
843 if (cookie_deprecation_testing_enabled) {
844 if (!cookie_deprecation_testing_has_label) {
845 RecordTpcdHeaderStatus(TpcdHeaderStatus::kNoLabel);
846 } else if (may_set_sec_cookie_deprecation_header) {
847 RecordTpcdHeaderStatus(TpcdHeaderStatus::kNoCookie);
848 } else {
849 RecordTpcdHeaderStatus(TpcdHeaderStatus::kSet);
850 }
851 }
852
853 CookieAccessResultList maybe_sent_cookies = std::move(excluded_cookies);
854 maybe_sent_cookies.insert(
855 maybe_sent_cookies.end(),
856 std::make_move_iterator(maybe_included_cookies.begin()),
857 std::make_move_iterator(maybe_included_cookies.end()));
858 maybe_included_cookies.clear();
859
860 if (request_->net_log().IsCapturing()) {
861 for (const auto& cookie_with_access_result : maybe_sent_cookies) {
862 request_->net_log().AddEvent(
863 NetLogEventType::COOKIE_INCLUSION_STATUS,
864 [&](NetLogCaptureMode capture_mode) {
865 return CookieInclusionStatusNetLogParams(
866 "send", cookie_with_access_result.cookie.Name(),
867 cookie_with_access_result.cookie.Domain(),
868 cookie_with_access_result.cookie.Path(),
869 cookie_with_access_result.access_result.status, capture_mode);
870 });
871 }
872 }
873
874 request_->set_maybe_sent_cookies(std::move(maybe_sent_cookies));
875
876 StartTransaction();
877 }
878
AnnotateAndMoveUserBlockedCookies(CookieAccessResultList & maybe_included_cookies,CookieAccessResultList & excluded_cookies) const879 void URLRequestHttpJob::AnnotateAndMoveUserBlockedCookies(
880 CookieAccessResultList& maybe_included_cookies,
881 CookieAccessResultList& excluded_cookies) const {
882 DCHECK(!ShouldBlockAllCookies(request_info_.privacy_mode))
883 << request_info_.privacy_mode;
884
885 bool can_get_cookies = URLRequest::DefaultCanUseCookies();
886 if (request()->network_delegate()) {
887 can_get_cookies =
888 request()->network_delegate()->AnnotateAndMoveUserBlockedCookies(
889 *request(), first_party_set_metadata_, maybe_included_cookies,
890 excluded_cookies);
891 }
892
893 if (!can_get_cookies) {
894 request()->net_log().AddEvent(
895 NetLogEventType::COOKIE_GET_BLOCKED_BY_NETWORK_DELEGATE);
896 }
897 }
898
SaveCookiesAndNotifyHeadersComplete(int result)899 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
900 DCHECK(set_cookie_access_result_list_.empty());
901 // TODO(crbug.com/1186863): Turn this CHECK into DCHECK once the investigation
902 // is done.
903 CHECK_EQ(0, num_cookie_lines_left_);
904
905 // End of the call started in OnStartCompleted.
906 OnCallToDelegateComplete();
907
908 if (result != OK) {
909 request_->net_log().AddEventWithStringParams(NetLogEventType::CANCELLED,
910 "source", "delegate");
911 NotifyStartError(result);
912 return;
913 }
914
915 CookieStore* cookie_store = request_->context()->cookie_store();
916
917 if ((request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) || !cookie_store) {
918 NotifyHeadersComplete();
919 return;
920 }
921
922 HttpResponseHeaders* headers = GetResponseHeaders();
923
924 // If we're clearing the cookies as part of a clear-site-data header we must
925 // not also write new ones in the same response.
926 bool clear_site_data_prevents_cookies_from_being_stored = false;
927 std::string clear_site_data_header;
928 headers->GetNormalizedHeader(kClearSiteDataHeader, &clear_site_data_header);
929 std::vector<std::string> clear_site_data_types =
930 ClearSiteDataHeaderContents(clear_site_data_header);
931 std::set<std::string> clear_site_data_set(clear_site_data_types.begin(),
932 clear_site_data_types.end());
933 if (clear_site_data_set.find(kDatatypeCookies) != clear_site_data_set.end() ||
934 clear_site_data_set.find(kDatatypeWildcard) !=
935 clear_site_data_set.end()) {
936 clear_site_data_prevents_cookies_from_being_stored = true;
937 }
938
939 base::Time response_date;
940 std::optional<base::Time> server_time = std::nullopt;
941 if (GetResponseHeaders()->GetDateValue(&response_date))
942 server_time = std::make_optional(response_date);
943
944 bool force_ignore_site_for_cookies =
945 request_->force_ignore_site_for_cookies();
946 if (cookie_store->cookie_access_delegate() &&
947 cookie_store->cookie_access_delegate()->ShouldIgnoreSameSiteRestrictions(
948 request_->url(), request_->site_for_cookies())) {
949 force_ignore_site_for_cookies = true;
950 }
951 bool is_main_frame_navigation =
952 IsolationInfo::RequestType::kMainFrame ==
953 request_->isolation_info().request_type() ||
954 request_->force_main_frame_for_same_site_cookies();
955 CookieOptions::SameSiteCookieContext same_site_context =
956 net::cookie_util::ComputeSameSiteContextForResponse(
957 request_->url_chain(), request_->site_for_cookies(),
958 request_->initiator(), is_main_frame_navigation,
959 force_ignore_site_for_cookies);
960
961 CookieOptions options = CreateCookieOptions(same_site_context);
962
963 // Set all cookies, without waiting for them to be set. Any subsequent
964 // read will see the combined result of all cookie operation.
965 const std::string_view name("Set-Cookie");
966 std::string cookie_string;
967 size_t iter = 0;
968
969 // NotifyHeadersComplete needs to be called once and only once after the
970 // list has been fully processed, and it can either be called in the
971 // callback or after the loop is called, depending on how the last element
972 // was handled. |num_cookie_lines_left_| keeps track of how many async
973 // callbacks are currently out (starting from 1 to make sure the loop runs
974 // all the way through before trying to exit). If there are any callbacks
975 // still waiting when the loop ends, then NotifyHeadersComplete will be
976 // called when it reaches 0 in the callback itself.
977 num_cookie_lines_left_ = 1;
978 while (headers->EnumerateHeader(&iter, name, &cookie_string)) {
979 CookieInclusionStatus returned_status;
980
981 num_cookie_lines_left_++;
982
983 // For the block_truncated parameter, the value shouldn't matter here
984 // because HTTP requests containing NULLs causes an error before this code
985 // can be reached and unpaired carriage returns and line feed characters
986 // cause truncation during HTTP header processing before reaching this
987 // point, so DCHECK this assumption and just pass true for this parameter.
988 DCHECK(cookie_string.find('\0') == std::string::npos);
989 DCHECK(cookie_string.find('\r') == std::string::npos);
990 DCHECK(cookie_string.find('\n') == std::string::npos);
991 std::unique_ptr<CanonicalCookie> cookie = net::CanonicalCookie::Create(
992 request_->url(), cookie_string, base::Time::Now(), server_time,
993 request_->cookie_partition_key(),
994 /*block_truncated=*/true, net::CookieSourceType::kHTTP,
995 &returned_status);
996
997 std::optional<CanonicalCookie> cookie_to_return = std::nullopt;
998 if (returned_status.IsInclude()) {
999 DCHECK(cookie);
1000 // Make a copy of the cookie if we successfully made one.
1001 cookie_to_return = *cookie;
1002 }
1003
1004 // Check cookie accessibility with cookie_settings.
1005 if (cookie && !CanSetCookie(*cookie, &options, first_party_set_metadata_,
1006 &returned_status)) {
1007 // Cookie allowed by cookie_settings checks could be blocked explicitly,
1008 // e.g. via Android Webview APIs, we need to manually add exclusion reason
1009 // in this case.
1010 if (returned_status.IsInclude()) {
1011 returned_status.AddExclusionReason(
1012 net::CookieInclusionStatus::EXCLUDE_USER_PREFERENCES);
1013 }
1014 }
1015 if (clear_site_data_prevents_cookies_from_being_stored) {
1016 returned_status.AddExclusionReason(
1017 CookieInclusionStatus::EXCLUDE_FAILURE_TO_STORE);
1018 }
1019 if (!returned_status.IsInclude()) {
1020 OnSetCookieResult(options, cookie_to_return, std::move(cookie_string),
1021 CookieAccessResult(returned_status));
1022 continue;
1023 }
1024 CookieAccessResult cookie_access_result(returned_status);
1025 cookie_store->SetCanonicalCookieAsync(
1026 std::move(cookie), request_->url(), options,
1027 base::BindOnce(&URLRequestHttpJob::OnSetCookieResult,
1028 weak_factory_.GetWeakPtr(), options, cookie_to_return,
1029 cookie_string),
1030 std::move(cookie_access_result));
1031 }
1032 // Removing the 1 that |num_cookie_lines_left| started with, signifing that
1033 // loop has been exited.
1034 num_cookie_lines_left_--;
1035
1036 if (num_cookie_lines_left_ == 0)
1037 NotifyHeadersComplete();
1038 }
1039
OnSetCookieResult(const CookieOptions & options,std::optional<CanonicalCookie> cookie,std::string cookie_string,CookieAccessResult access_result)1040 void URLRequestHttpJob::OnSetCookieResult(const CookieOptions& options,
1041 std::optional<CanonicalCookie> cookie,
1042 std::string cookie_string,
1043 CookieAccessResult access_result) {
1044 if (request_->net_log().IsCapturing()) {
1045 request_->net_log().AddEvent(NetLogEventType::COOKIE_INCLUSION_STATUS,
1046 [&](NetLogCaptureMode capture_mode) {
1047 return CookieInclusionStatusNetLogParams(
1048 "store",
1049 cookie ? cookie.value().Name() : "",
1050 cookie ? cookie.value().Domain() : "",
1051 cookie ? cookie.value().Path() : "",
1052 access_result.status, capture_mode);
1053 });
1054 }
1055
1056 set_cookie_access_result_list_.emplace_back(
1057 std::move(cookie), std::move(cookie_string), access_result);
1058
1059 num_cookie_lines_left_--;
1060
1061 // If all the cookie lines have been handled, |set_cookie_access_result_list_|
1062 // now reflects the result of all Set-Cookie lines, and the request can be
1063 // continued.
1064 if (num_cookie_lines_left_ == 0)
1065 NotifyHeadersComplete();
1066 }
1067
1068 #if BUILDFLAG(ENABLE_DEVICE_BOUND_SESSIONS)
ProcessDeviceBoundSessionsHeader()1069 void URLRequestHttpJob::ProcessDeviceBoundSessionsHeader() {
1070 std::vector<DeviceBoundSessionRegistrationFetcherParam> params =
1071 DeviceBoundSessionRegistrationFetcherParam::CreateIfValid(
1072 request_->url(), GetResponseHeaders());
1073 if (auto* service = request_->context()->device_bound_session_service()) {
1074 for (const auto& param : params) {
1075 service->RegisterBoundSession(param);
1076 }
1077 }
1078 }
1079 #endif // BUILDFLAG(ENABLE_DEVICE_BOUND_SESSIONS)
1080
ProcessStrictTransportSecurityHeader()1081 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
1082 DCHECK(response_info_);
1083 TransportSecurityState* security_state =
1084 request_->context()->transport_security_state();
1085 const SSLInfo& ssl_info = response_info_->ssl_info;
1086
1087 // Only accept HSTS headers on HTTPS connections that have no
1088 // certificate errors.
1089 if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
1090 !security_state) {
1091 return;
1092 }
1093
1094 // Don't accept HSTS headers when the hostname is an IP address.
1095 if (request_info_.url.HostIsIPAddress())
1096 return;
1097
1098 // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
1099 //
1100 // If a UA receives more than one STS header field in a HTTP response
1101 // message over secure transport, then the UA MUST process only the
1102 // first such header field.
1103 HttpResponseHeaders* headers = GetResponseHeaders();
1104 std::string value;
1105 if (headers->EnumerateHeader(nullptr, "Strict-Transport-Security", &value))
1106 security_state->AddHSTSHeader(request_info_.url.host(), value);
1107 }
1108
OnStartCompleted(int result)1109 void URLRequestHttpJob::OnStartCompleted(int result) {
1110 TRACE_EVENT0(NetTracingCategory(), "URLRequestHttpJob::OnStartCompleted");
1111 RecordTimer();
1112
1113 // If the job is done (due to cancellation), can just ignore this
1114 // notification.
1115 if (done_)
1116 return;
1117
1118 receive_headers_end_ = base::TimeTicks::Now();
1119
1120 const URLRequestContext* context = request_->context();
1121
1122 if (transaction_ && transaction_->GetResponseInfo()) {
1123 const SSLInfo& ssl_info = transaction_->GetResponseInfo()->ssl_info;
1124 if (!IsCertificateError(result)) {
1125 LogTrustAnchor(ssl_info.public_key_hashes);
1126 }
1127 }
1128
1129 if (transaction_ && transaction_->GetResponseInfo()) {
1130 SetProxyChain(transaction_->GetResponseInfo()->proxy_chain);
1131 }
1132
1133 if (result == OK) {
1134 scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
1135
1136 NetworkDelegate* network_delegate = request()->network_delegate();
1137 if (network_delegate) {
1138 // Note that |this| may not be deleted until
1139 // |URLRequestHttpJob::OnHeadersReceivedCallback()| or
1140 // |NetworkDelegate::URLRequestDestroyed()| has been called.
1141 OnCallToDelegate(NetLogEventType::NETWORK_DELEGATE_HEADERS_RECEIVED);
1142 preserve_fragment_on_redirect_url_ = std::nullopt;
1143 IPEndPoint endpoint;
1144 if (transaction_)
1145 transaction_->GetRemoteEndpoint(&endpoint);
1146 // The NetworkDelegate must watch for OnRequestDestroyed and not modify
1147 // any of the arguments after it's called.
1148 // TODO(mattm): change the API to remove the out-params and take the
1149 // results as params of the callback.
1150 int error = network_delegate->NotifyHeadersReceived(
1151 request_,
1152 base::BindOnce(&URLRequestHttpJob::OnHeadersReceivedCallback,
1153 weak_factory_.GetWeakPtr()),
1154 headers.get(), &override_response_headers_, endpoint,
1155 &preserve_fragment_on_redirect_url_);
1156 if (error != OK) {
1157 if (error == ERR_IO_PENDING) {
1158 awaiting_callback_ = true;
1159 } else {
1160 request_->net_log().AddEventWithStringParams(
1161 NetLogEventType::CANCELLED, "source", "delegate");
1162 OnCallToDelegateComplete();
1163 NotifyStartError(error);
1164 }
1165 return;
1166 }
1167 }
1168
1169 SaveCookiesAndNotifyHeadersComplete(OK);
1170 } else if (IsCertificateError(result)) {
1171 // We encountered an SSL certificate error.
1172 // Maybe overridable, maybe not. Ask the delegate to decide.
1173 TransportSecurityState* state = context->transport_security_state();
1174 NotifySSLCertificateError(
1175 result, transaction_->GetResponseInfo()->ssl_info,
1176 state->ShouldSSLErrorsBeFatal(request_info_.url.host()) &&
1177 result != ERR_CERT_KNOWN_INTERCEPTION_BLOCKED);
1178 } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
1179 NotifyCertificateRequested(
1180 transaction_->GetResponseInfo()->cert_request_info.get());
1181 } else if (result == ERR_DNS_NAME_HTTPS_ONLY) {
1182 // If DNS indicated the name is HTTPS-only, synthesize a redirect to either
1183 // HTTPS or WSS.
1184 DCHECK(!request_->url().SchemeIsCryptographic());
1185
1186 base::Time request_time =
1187 transaction_ && transaction_->GetResponseInfo()
1188 ? transaction_->GetResponseInfo()->request_time
1189 : base::Time::Now();
1190 DestroyTransaction();
1191 override_response_info_ = std::make_unique<HttpResponseInfo>();
1192 override_response_info_->request_time = request_time;
1193
1194 override_response_info_->headers = RedirectUtil::SynthesizeRedirectHeaders(
1195 UpgradeSchemeToCryptographic(request_->url()),
1196 RedirectUtil::ResponseCode::REDIRECT_307_TEMPORARY_REDIRECT, "DNS",
1197 request_->extra_request_headers());
1198 NetLogResponseHeaders(
1199 request_->net_log(),
1200 NetLogEventType::URL_REQUEST_FAKE_RESPONSE_HEADERS_CREATED,
1201 override_response_info_->headers.get());
1202
1203 NotifyHeadersComplete();
1204 } else {
1205 // Even on an error, there may be useful information in the response
1206 // info (e.g. whether there's a cached copy).
1207 if (transaction_.get())
1208 response_info_ = transaction_->GetResponseInfo();
1209 NotifyStartError(result);
1210 }
1211 }
1212
OnHeadersReceivedCallback(int result)1213 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
1214 // The request should not have been cancelled or have already completed.
1215 DCHECK(!is_done());
1216
1217 awaiting_callback_ = false;
1218
1219 SaveCookiesAndNotifyHeadersComplete(result);
1220 }
1221
OnReadCompleted(int result)1222 void URLRequestHttpJob::OnReadCompleted(int result) {
1223 TRACE_EVENT0(NetTracingCategory(), "URLRequestHttpJob::OnReadCompleted");
1224 read_in_progress_ = false;
1225
1226 DCHECK_NE(ERR_IO_PENDING, result);
1227
1228 if (ShouldFixMismatchedContentLength(result))
1229 result = OK;
1230
1231 // EOF or error, done with this job.
1232 if (result <= 0)
1233 DoneWithRequest(FINISHED);
1234
1235 ReadRawDataComplete(result);
1236 }
1237
RestartTransactionWithAuth(const AuthCredentials & credentials)1238 void URLRequestHttpJob::RestartTransactionWithAuth(
1239 const AuthCredentials& credentials) {
1240 DCHECK(!override_response_info_);
1241
1242 auth_credentials_ = credentials;
1243
1244 // These will be reset in OnStartCompleted.
1245 response_info_ = nullptr;
1246 override_response_headers_ = nullptr; // See https://crbug.com/801237.
1247 receive_headers_end_ = base::TimeTicks();
1248
1249 ResetTimer();
1250
1251 // Update the cookies, since the cookie store may have been updated from the
1252 // headers in the 401/407. Since cookies were already appended to
1253 // extra_headers, we need to strip them out before adding them again.
1254 request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
1255
1256 // TODO(https://crbug.com/968327/): This is weird, as all other clearing is at
1257 // the URLRequest layer. Should this call into URLRequest so it can share
1258 // logic at that layer with SetAuth()?
1259 request_->set_maybe_sent_cookies({});
1260 request_->set_maybe_stored_cookies({});
1261
1262 if (ShouldAddCookieHeader()) {
1263 // Since `request_->isolation_info()` hasn't changed, we don't need to
1264 // recompute the cookie partition key.
1265 AddCookieHeaderAndStart();
1266 } else {
1267 StartTransaction();
1268 }
1269 }
1270
SetUpload(UploadDataStream * upload)1271 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
1272 DCHECK(!transaction_.get() && !override_response_info_)
1273 << "cannot change once started";
1274 request_info_.upload_data_stream = upload;
1275 }
1276
SetExtraRequestHeaders(const HttpRequestHeaders & headers)1277 void URLRequestHttpJob::SetExtraRequestHeaders(
1278 const HttpRequestHeaders& headers) {
1279 DCHECK(!transaction_.get() && !override_response_info_)
1280 << "cannot change once started";
1281 request_info_.extra_headers = headers;
1282 }
1283
GetLoadState() const1284 LoadState URLRequestHttpJob::GetLoadState() const {
1285 return transaction_.get() ?
1286 transaction_->GetLoadState() : LOAD_STATE_IDLE;
1287 }
1288
GetMimeType(std::string * mime_type) const1289 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
1290 DCHECK(transaction_.get() || override_response_info_);
1291
1292 if (!response_info_)
1293 return false;
1294
1295 HttpResponseHeaders* headers = GetResponseHeaders();
1296 if (!headers)
1297 return false;
1298 return headers->GetMimeType(mime_type);
1299 }
1300
GetCharset(std::string * charset)1301 bool URLRequestHttpJob::GetCharset(std::string* charset) {
1302 DCHECK(transaction_.get() || override_response_info_);
1303
1304 if (!response_info_)
1305 return false;
1306
1307 return GetResponseHeaders()->GetCharset(charset);
1308 }
1309
GetResponseInfo(HttpResponseInfo * info)1310 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
1311 if (override_response_info_) {
1312 DCHECK(!transaction_.get());
1313 *info = *override_response_info_;
1314 return;
1315 }
1316
1317 if (response_info_) {
1318 DCHECK(transaction_.get());
1319
1320 *info = *response_info_;
1321 if (override_response_headers_.get())
1322 info->headers = override_response_headers_;
1323 }
1324 }
1325
GetLoadTimingInfo(LoadTimingInfo * load_timing_info) const1326 void URLRequestHttpJob::GetLoadTimingInfo(
1327 LoadTimingInfo* load_timing_info) const {
1328 // If haven't made it far enough to receive any headers, don't return
1329 // anything. This makes for more consistent behavior in the case of errors.
1330 if (!transaction_ || receive_headers_end_.is_null())
1331 return;
1332 if (transaction_->GetLoadTimingInfo(load_timing_info))
1333 load_timing_info->receive_headers_end = receive_headers_end_;
1334 }
1335
GetTransactionRemoteEndpoint(IPEndPoint * endpoint) const1336 bool URLRequestHttpJob::GetTransactionRemoteEndpoint(
1337 IPEndPoint* endpoint) const {
1338 if (!transaction_)
1339 return false;
1340
1341 return transaction_->GetRemoteEndpoint(endpoint);
1342 }
1343
GetResponseCode() const1344 int URLRequestHttpJob::GetResponseCode() const {
1345 DCHECK(transaction_.get());
1346
1347 if (!response_info_)
1348 return -1;
1349
1350 return GetResponseHeaders()->response_code();
1351 }
1352
PopulateNetErrorDetails(NetErrorDetails * details) const1353 void URLRequestHttpJob::PopulateNetErrorDetails(
1354 NetErrorDetails* details) const {
1355 if (!transaction_)
1356 return;
1357 return transaction_->PopulateNetErrorDetails(details);
1358 }
1359
SetUpSourceStream()1360 std::unique_ptr<SourceStream> URLRequestHttpJob::SetUpSourceStream() {
1361 DCHECK(transaction_.get());
1362 if (!response_info_)
1363 return nullptr;
1364
1365 std::unique_ptr<SourceStream> upstream = URLRequestJob::SetUpSourceStream();
1366 HttpResponseHeaders* headers = GetResponseHeaders();
1367 std::vector<SourceStream::SourceType> types;
1368 size_t iter = 0;
1369 for (std::string type;
1370 headers->EnumerateHeader(&iter, "Content-Encoding", &type);) {
1371 SourceStream::SourceType source_type =
1372 FilterSourceStream::ParseEncodingType(type);
1373 switch (source_type) {
1374 case SourceStream::TYPE_BROTLI:
1375 case SourceStream::TYPE_DEFLATE:
1376 case SourceStream::TYPE_GZIP:
1377 case SourceStream::TYPE_ZSTD:
1378 if (request_->accepted_stream_types() &&
1379 !request_->accepted_stream_types()->contains(source_type)) {
1380 // If the source type is disabled, we treat it
1381 // in the same way as SourceStream::TYPE_UNKNOWN.
1382 return upstream;
1383 }
1384 types.push_back(source_type);
1385 break;
1386 case SourceStream::TYPE_NONE:
1387 // Identity encoding type. Pass through raw response body.
1388 return upstream;
1389 case SourceStream::TYPE_UNKNOWN:
1390 // Unknown encoding type. Pass through raw response body.
1391 // Request will not be canceled; though
1392 // it is expected that user will see malformed / garbage response.
1393 return upstream;
1394 }
1395 }
1396
1397 ContentEncodingType content_encoding_type = ContentEncodingType::kUnknown;
1398
1399 for (const auto& type : base::Reversed(types)) {
1400 std::unique_ptr<FilterSourceStream> downstream;
1401 switch (type) {
1402 case SourceStream::TYPE_BROTLI:
1403 downstream = CreateBrotliSourceStream(std::move(upstream));
1404 content_encoding_type = ContentEncodingType::kBrotli;
1405 break;
1406 case SourceStream::TYPE_GZIP:
1407 case SourceStream::TYPE_DEFLATE:
1408 downstream = GzipSourceStream::Create(std::move(upstream), type);
1409 content_encoding_type = type == SourceStream::TYPE_GZIP
1410 ? ContentEncodingType::kGZip
1411 : ContentEncodingType::kDeflate;
1412 break;
1413 case SourceStream::TYPE_ZSTD:
1414 downstream = CreateZstdSourceStream(std::move(upstream));
1415 content_encoding_type = ContentEncodingType::kZstd;
1416 break;
1417 case SourceStream::TYPE_NONE:
1418 case SourceStream::TYPE_UNKNOWN:
1419 NOTREACHED();
1420 return nullptr;
1421 }
1422 if (downstream == nullptr)
1423 return nullptr;
1424 upstream = std::move(downstream);
1425 }
1426
1427 // Note: If multiple encoding types were specified, this only records the last
1428 // encoding type.
1429 UMA_HISTOGRAM_ENUMERATION("Net.ContentEncodingType", content_encoding_type);
1430
1431 return upstream;
1432 }
1433
CopyFragmentOnRedirect(const GURL & location) const1434 bool URLRequestHttpJob::CopyFragmentOnRedirect(const GURL& location) const {
1435 // Allow modification of reference fragments by default, unless
1436 // |preserve_fragment_on_redirect_url_| is set and equal to the redirect URL.
1437 return !preserve_fragment_on_redirect_url_.has_value() ||
1438 preserve_fragment_on_redirect_url_ != location;
1439 }
1440
IsSafeRedirect(const GURL & location)1441 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1442 // HTTP is always safe.
1443 // TODO(pauljensen): Remove once crbug.com/146591 is fixed.
1444 if (location.is_valid() &&
1445 (location.scheme() == "http" || location.scheme() == "https")) {
1446 return true;
1447 }
1448 // Query URLRequestJobFactory as to whether |location| would be safe to
1449 // redirect to.
1450 return request_->context()->job_factory() &&
1451 request_->context()->job_factory()->IsSafeRedirectTarget(location);
1452 }
1453
NeedsAuth()1454 bool URLRequestHttpJob::NeedsAuth() {
1455 int code = GetResponseCode();
1456 if (code == -1)
1457 return false;
1458
1459 // Check if we need either Proxy or WWW Authentication. This could happen
1460 // because we either provided no auth info, or provided incorrect info.
1461 switch (code) {
1462 case 407:
1463 if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1464 return false;
1465 proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1466 return true;
1467 case 401:
1468 if (server_auth_state_ == AUTH_STATE_CANCELED)
1469 return false;
1470 server_auth_state_ = AUTH_STATE_NEED_AUTH;
1471 return true;
1472 }
1473 return false;
1474 }
1475
GetAuthChallengeInfo()1476 std::unique_ptr<AuthChallengeInfo> URLRequestHttpJob::GetAuthChallengeInfo() {
1477 DCHECK(transaction_.get());
1478 DCHECK(response_info_);
1479
1480 // sanity checks:
1481 DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1482 server_auth_state_ == AUTH_STATE_NEED_AUTH);
1483 DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1484 (GetResponseHeaders()->response_code() ==
1485 HTTP_PROXY_AUTHENTICATION_REQUIRED));
1486
1487 if (!response_info_->auth_challenge.has_value())
1488 return nullptr;
1489 return std::make_unique<AuthChallengeInfo>(
1490 response_info_->auth_challenge.value());
1491 }
1492
SetAuth(const AuthCredentials & credentials)1493 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1494 DCHECK(transaction_.get());
1495
1496 // Proxy gets set first, then WWW.
1497 if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1498 proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1499 } else {
1500 DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1501 server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1502 }
1503
1504 RestartTransactionWithAuth(credentials);
1505 }
1506
CancelAuth()1507 void URLRequestHttpJob::CancelAuth() {
1508 if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1509 proxy_auth_state_ = AUTH_STATE_CANCELED;
1510 } else {
1511 DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1512 server_auth_state_ = AUTH_STATE_CANCELED;
1513 }
1514
1515 // The above lines should ensure this is the case.
1516 DCHECK(!NeedsAuth());
1517
1518 // Let the consumer read the HTTP error page. NeedsAuth() should now return
1519 // false, so NotifyHeadersComplete() should not request auth from the client
1520 // again.
1521 //
1522 // Have to do this via PostTask to avoid re-entrantly calling into the
1523 // consumer.
1524 base::SingleThreadTaskRunner::GetCurrentDefault()->PostTask(
1525 FROM_HERE, base::BindOnce(&URLRequestHttpJob::NotifyFinalHeadersReceived,
1526 weak_factory_.GetWeakPtr()));
1527 }
1528
ContinueWithCertificate(scoped_refptr<X509Certificate> client_cert,scoped_refptr<SSLPrivateKey> client_private_key)1529 void URLRequestHttpJob::ContinueWithCertificate(
1530 scoped_refptr<X509Certificate> client_cert,
1531 scoped_refptr<SSLPrivateKey> client_private_key) {
1532 DCHECK(transaction_);
1533
1534 DCHECK(!response_info_) << "should not have a response yet";
1535 DCHECK(!override_response_headers_);
1536 receive_headers_end_ = base::TimeTicks();
1537
1538 ResetTimer();
1539
1540 int rv = transaction_->RestartWithCertificate(
1541 std::move(client_cert), std::move(client_private_key),
1542 base::BindOnce(&URLRequestHttpJob::OnStartCompleted,
1543 base::Unretained(this)));
1544 if (rv == ERR_IO_PENDING)
1545 return;
1546
1547 // The transaction started synchronously, but we need to notify the
1548 // URLRequest delegate via the message loop.
1549 base::SingleThreadTaskRunner::GetCurrentDefault()->PostTask(
1550 FROM_HERE, base::BindOnce(&URLRequestHttpJob::OnStartCompleted,
1551 weak_factory_.GetWeakPtr(), rv));
1552 }
1553
ContinueDespiteLastError()1554 void URLRequestHttpJob::ContinueDespiteLastError() {
1555 // If the transaction was destroyed, then the job was cancelled.
1556 if (!transaction_.get())
1557 return;
1558
1559 DCHECK(!response_info_) << "should not have a response yet";
1560 DCHECK(!override_response_headers_);
1561 receive_headers_end_ = base::TimeTicks();
1562
1563 ResetTimer();
1564
1565 int rv = transaction_->RestartIgnoringLastError(base::BindOnce(
1566 &URLRequestHttpJob::OnStartCompleted, base::Unretained(this)));
1567 if (rv == ERR_IO_PENDING)
1568 return;
1569
1570 // The transaction started synchronously, but we need to notify the
1571 // URLRequest delegate via the message loop.
1572 base::SingleThreadTaskRunner::GetCurrentDefault()->PostTask(
1573 FROM_HERE, base::BindOnce(&URLRequestHttpJob::OnStartCompleted,
1574 weak_factory_.GetWeakPtr(), rv));
1575 }
1576
ShouldFixMismatchedContentLength(int rv) const1577 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1578 // Some servers send the body compressed, but specify the content length as
1579 // the uncompressed size. Although this violates the HTTP spec we want to
1580 // support it (as IE and FireFox do), but *only* for an exact match.
1581 // See http://crbug.com/79694.
1582 if (rv == ERR_CONTENT_LENGTH_MISMATCH ||
1583 rv == ERR_INCOMPLETE_CHUNKED_ENCODING) {
1584 if (request_->response_headers()) {
1585 int64_t expected_length =
1586 request_->response_headers()->GetContentLength();
1587 VLOG(1) << __func__ << "() \"" << request_->url().spec() << "\""
1588 << " content-length = " << expected_length
1589 << " pre total = " << prefilter_bytes_read()
1590 << " post total = " << postfilter_bytes_read();
1591 if (postfilter_bytes_read() == expected_length) {
1592 // Clear the error.
1593 return true;
1594 }
1595 }
1596 }
1597 return false;
1598 }
1599
ReadRawData(IOBuffer * buf,int buf_size)1600 int URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size) {
1601 DCHECK_NE(buf_size, 0);
1602 DCHECK(!read_in_progress_);
1603
1604 int rv =
1605 transaction_->Read(buf, buf_size,
1606 base::BindOnce(&URLRequestHttpJob::OnReadCompleted,
1607 base::Unretained(this)));
1608
1609 if (ShouldFixMismatchedContentLength(rv))
1610 rv = OK;
1611
1612 if (rv == 0 || (rv < 0 && rv != ERR_IO_PENDING))
1613 DoneWithRequest(FINISHED);
1614
1615 if (rv == ERR_IO_PENDING)
1616 read_in_progress_ = true;
1617
1618 return rv;
1619 }
1620
GetTotalReceivedBytes() const1621 int64_t URLRequestHttpJob::GetTotalReceivedBytes() const {
1622 int64_t total_received_bytes =
1623 total_received_bytes_from_previous_transactions_;
1624 if (transaction_)
1625 total_received_bytes += transaction_->GetTotalReceivedBytes();
1626 return total_received_bytes;
1627 }
1628
GetTotalSentBytes() const1629 int64_t URLRequestHttpJob::GetTotalSentBytes() const {
1630 int64_t total_sent_bytes = total_sent_bytes_from_previous_transactions_;
1631 if (transaction_)
1632 total_sent_bytes += transaction_->GetTotalSentBytes();
1633 return total_sent_bytes;
1634 }
1635
DoneReading()1636 void URLRequestHttpJob::DoneReading() {
1637 if (transaction_) {
1638 transaction_->DoneReading();
1639 }
1640 DoneWithRequest(FINISHED);
1641 }
1642
DoneReadingRedirectResponse()1643 void URLRequestHttpJob::DoneReadingRedirectResponse() {
1644 if (transaction_) {
1645 DCHECK(!override_response_info_);
1646 if (transaction_->GetResponseInfo()->headers->IsRedirect(nullptr)) {
1647 // If the original headers indicate a redirect, go ahead and cache the
1648 // response, even if the |override_response_headers_| are a redirect to
1649 // another location.
1650 transaction_->DoneReading();
1651 } else {
1652 // Otherwise, |override_response_headers_| must be non-NULL and contain
1653 // bogus headers indicating a redirect.
1654 DCHECK(override_response_headers_.get());
1655 DCHECK(override_response_headers_->IsRedirect(nullptr));
1656 transaction_->StopCaching();
1657 }
1658 }
1659 DoneWithRequest(FINISHED);
1660 }
1661
GetResponseRemoteEndpoint() const1662 IPEndPoint URLRequestHttpJob::GetResponseRemoteEndpoint() const {
1663 return response_info_ ? response_info_->remote_endpoint : IPEndPoint();
1664 }
1665
RecordTimer()1666 void URLRequestHttpJob::RecordTimer() {
1667 if (request_creation_time_.is_null()) {
1668 NOTREACHED()
1669 << "The same transaction shouldn't start twice without new timing.";
1670 return;
1671 }
1672
1673 base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1674 request_creation_time_ = base::Time();
1675
1676 UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1677
1678 // Record additional metrics for TLS 1.3 servers for Google hosts. Most
1679 // Google hosts are known to implement 0-RTT, so this gives more targeted
1680 // metrics as we initially roll out client support. This is to help measure
1681 // the impact of enabling 0-RTT. The effects of 0-RTT will be muted because
1682 // not all TLS 1.3 servers enable 0-RTT, and only the first round-trip on a
1683 // connection makes use of 0-RTT. However, 0-RTT can affect how requests are
1684 // bound to connections and which connections offer resumption. We look at
1685 // all TLS 1.3 responses for an apples-to-apples comparison.
1686 // TODO(crbug.com/641225): Remove these metrics after launching 0-RTT.
1687 if (transaction_ && transaction_->GetResponseInfo() &&
1688 IsTLS13OverTCP(*transaction_->GetResponseInfo()) &&
1689 HasGoogleHost(request()->url())) {
1690 base::UmaHistogramMediumTimes("Net.HttpTimeToFirstByte.TLS13.Google",
1691 to_start);
1692 }
1693 }
1694
ResetTimer()1695 void URLRequestHttpJob::ResetTimer() {
1696 if (!request_creation_time_.is_null()) {
1697 NOTREACHED()
1698 << "The timer was reset before it was recorded.";
1699 return;
1700 }
1701 request_creation_time_ = base::Time::Now();
1702 }
1703
SetRequestHeadersCallback(RequestHeadersCallback callback)1704 void URLRequestHttpJob::SetRequestHeadersCallback(
1705 RequestHeadersCallback callback) {
1706 DCHECK(!transaction_);
1707 DCHECK(!request_headers_callback_);
1708 request_headers_callback_ = std::move(callback);
1709 }
1710
SetEarlyResponseHeadersCallback(ResponseHeadersCallback callback)1711 void URLRequestHttpJob::SetEarlyResponseHeadersCallback(
1712 ResponseHeadersCallback callback) {
1713 DCHECK(!transaction_);
1714 DCHECK(!early_response_headers_callback_);
1715 early_response_headers_callback_ = std::move(callback);
1716 }
1717
SetIsSharedDictionaryReadAllowedCallback(base::RepeatingCallback<bool ()> callback)1718 void URLRequestHttpJob::SetIsSharedDictionaryReadAllowedCallback(
1719 base::RepeatingCallback<bool()> callback) {
1720 DCHECK(!transaction_);
1721 DCHECK(!is_shared_dictionary_read_allowed_callback_);
1722 is_shared_dictionary_read_allowed_callback_ = std::move(callback);
1723 }
1724
SetResponseHeadersCallback(ResponseHeadersCallback callback)1725 void URLRequestHttpJob::SetResponseHeadersCallback(
1726 ResponseHeadersCallback callback) {
1727 DCHECK(!transaction_);
1728 DCHECK(!response_headers_callback_);
1729 response_headers_callback_ = std::move(callback);
1730 }
1731
RecordCompletionHistograms(CompletionCause reason)1732 void URLRequestHttpJob::RecordCompletionHistograms(CompletionCause reason) {
1733 if (start_time_.is_null())
1734 return;
1735
1736 base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1737 base::UmaHistogramTimes("Net.HttpJob.TotalTime", total_time);
1738
1739 if (reason == FINISHED) {
1740 base::UmaHistogramTimes(
1741 base::StringPrintf("Net.HttpJob.TotalTimeSuccess.Priority%d",
1742 request()->priority()),
1743 total_time);
1744 base::UmaHistogramTimes("Net.HttpJob.TotalTimeSuccess", total_time);
1745 } else {
1746 base::UmaHistogramTimes("Net.HttpJob.TotalTimeCancel", total_time);
1747 }
1748
1749 // These metrics are intended to replace some of the later IP
1750 // Protection-focused metrics below which require response_info_. These
1751 // metrics are only concerned with data that actually hits or perhaps should
1752 // have hit the network.
1753 //
1754 // We count towards these metrics even if the job has been aborted. Jobs
1755 // aborted before an end-to-end connection is established will have both
1756 // sent and received equal to zero.
1757 //
1758 // In addition, we don't want to ignore jobs where response_info_->was_cached
1759 // is true but the network was used to trigger cache usage as part of a 304
1760 // Not Modified response. However, cache hits which bypass the network
1761 // entirely should not be counted.
1762 //
1763 // GetTotalReceivedBytes measures HTTP stream bytes, which is more
1764 // comprehensive than PrefilterBytesRead, which measures (possibly compressed)
1765 // content's length only.
1766 const bool bypassedNetwork = response_info_ && response_info_->was_cached &&
1767 !response_info_->network_accessed &&
1768 GetTotalSentBytes() == 0 &&
1769 GetTotalReceivedBytes() == 0;
1770 if (!bypassedNetwork) {
1771 base::UmaHistogramCustomCounts("Net.HttpJob.BytesSent2",
1772 GetTotalSentBytes(), 1, 50000000, 50);
1773 base::UmaHistogramCustomCounts("Net.HttpJob.BytesReceived2",
1774 GetTotalReceivedBytes(), 1, 50000000, 50);
1775 // Having a transaction_ does not imply having a response_info_. This is
1776 // particularly the case in some aborted/cancelled jobs. The transaction is
1777 // the primary source of MDL match information.
1778 if ((transaction_ && transaction_->IsMdlMatchForMetrics()) ||
1779 (response_info_ && response_info_->was_mdl_match)) {
1780 base::UmaHistogramCustomCounts(
1781 "Net.HttpJob.IpProtection.AllowListMatch.BytesSent2",
1782 GetTotalSentBytes(), 1, 50000000, 50);
1783 base::UmaHistogramCustomCounts(
1784 "Net.HttpJob.IpProtection.AllowListMatch.BytesReceived2",
1785 GetTotalReceivedBytes(), 1, 50000000, 50);
1786 }
1787 }
1788
1789 if (response_info_) {
1790 // QUIC (by default) supports https scheme only, thus track https URLs only
1791 // for QUIC.
1792 bool is_https_google = request() && request()->url().SchemeIs("https") &&
1793 HasGoogleHost(request()->url());
1794 bool used_quic = response_info_->DidUseQuic();
1795 if (is_https_google) {
1796 if (used_quic) {
1797 base::UmaHistogramMediumTimes("Net.HttpJob.TotalTime.Secure.Quic",
1798 total_time);
1799 }
1800 }
1801
1802 // Record metrics for TLS 1.3 to measure the impact of 0-RTT. See comment in
1803 // RecordTimer().
1804 //
1805 // TODO(https://crbug.com/641225): Remove these metrics after launching
1806 // 0-RTT.
1807 if (IsTLS13OverTCP(*response_info_) && is_https_google) {
1808 base::UmaHistogramTimes("Net.HttpJob.TotalTime.TLS13.Google", total_time);
1809 }
1810
1811 base::UmaHistogramCustomCounts("Net.HttpJob.PrefilterBytesRead",
1812 prefilter_bytes_read(), 1, 50000000, 50);
1813 if (response_info_->was_cached) {
1814 base::UmaHistogramTimes("Net.HttpJob.TotalTimeCached", total_time);
1815 base::UmaHistogramCustomCounts("Net.HttpJob.PrefilterBytesRead.Cache",
1816 prefilter_bytes_read(), 1, 50000000, 50);
1817 } else {
1818 base::UmaHistogramTimes("Net.HttpJob.TotalTimeNotCached", total_time);
1819 if (response_info_->was_mdl_match) {
1820 base::UmaHistogramCustomCounts(
1821 "Net.HttpJob.IpProtection.AllowListMatch.BytesSent",
1822 GetTotalSentBytes(), 1, 50000000, 50);
1823
1824 base::UmaHistogramCustomCounts(
1825 "Net.HttpJob.IpProtection.AllowListMatch.PrefilterBytesRead.Net",
1826 prefilter_bytes_read(), 1, 50000000, 50);
1827 }
1828 if (response_info_->proxy_chain.is_for_ip_protection()) {
1829 base::UmaHistogramTimes("Net.HttpJob.IpProtection.TotalTimeNotCached",
1830 total_time);
1831 // Log specific times for non-zero chains. The zero chain is the
1832 // default and is still counted in the base `TotalTimeNotCached`.
1833 int chain_id = response_info_->proxy_chain.ip_protection_chain_id();
1834 if (chain_id != ProxyChain::kNotIpProtectionChainId) {
1835 UmaHistogramTimes(
1836 base::StrCat({"Net.HttpJob.IpProtection.TotalTimeNotCached.Chain",
1837 base::NumberToString(chain_id)}),
1838 total_time);
1839 }
1840
1841 base::UmaHistogramCustomCounts("Net.HttpJob.IpProtection.BytesSent",
1842 GetTotalSentBytes(), 1, 50000000, 50);
1843
1844 base::UmaHistogramCustomCounts(
1845 "Net.HttpJob.IpProtection.PrefilterBytesRead.Net",
1846 prefilter_bytes_read(), 1, 50000000, 50);
1847 }
1848 base::UmaHistogramCustomCounts("Net.HttpJob.PrefilterBytesRead.Net",
1849 prefilter_bytes_read(), 1, 50000000, 50);
1850
1851 if (request_->ad_tagged()) {
1852 base::UmaHistogramCustomCounts("Net.HttpJob.PrefilterBytesRead.Ads.Net",
1853 prefilter_bytes_read(), 1, 50000000, 50);
1854 }
1855
1856 if (is_https_google && used_quic) {
1857 base::UmaHistogramMediumTimes(
1858 "Net.HttpJob.TotalTimeNotCached.Secure.Quic", total_time);
1859 }
1860 }
1861 }
1862
1863 start_time_ = base::TimeTicks();
1864 }
1865
DoneWithRequest(CompletionCause reason)1866 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1867 if (done_)
1868 return;
1869 done_ = true;
1870
1871 // Notify NetworkQualityEstimator.
1872 NetworkQualityEstimator* network_quality_estimator =
1873 request()->context()->network_quality_estimator();
1874 if (network_quality_estimator) {
1875 network_quality_estimator->NotifyRequestCompleted(*request());
1876 }
1877
1878 RecordCompletionHistograms(reason);
1879 request()->set_received_response_content_length(prefilter_bytes_read());
1880 }
1881
GetResponseHeaders() const1882 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1883 if (override_response_info_) {
1884 DCHECK(!transaction_.get());
1885 return override_response_info_->headers.get();
1886 }
1887
1888 DCHECK(transaction_.get());
1889 DCHECK(transaction_->GetResponseInfo());
1890
1891 return override_response_headers_.get() ?
1892 override_response_headers_.get() :
1893 transaction_->GetResponseInfo()->headers.get();
1894 }
1895
NotifyURLRequestDestroyed()1896 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1897 awaiting_callback_ = false;
1898
1899 // Notify NetworkQualityEstimator.
1900 NetworkQualityEstimator* network_quality_estimator =
1901 request()->context()->network_quality_estimator();
1902 if (network_quality_estimator)
1903 network_quality_estimator->NotifyURLRequestDestroyed(*request());
1904 }
1905
ShouldAddCookieHeader() const1906 bool URLRequestHttpJob::ShouldAddCookieHeader() const {
1907 // Read cookies whenever allow_credentials() is true, even if the PrivacyMode
1908 // is being overridden by NetworkDelegate and will eventually block them, as
1909 // blocked cookies still need to be logged in that case.
1910 return request_->context()->cookie_store() && request_->allow_credentials();
1911 }
1912
ShouldRecordPartitionedCookieUsage() const1913 bool URLRequestHttpJob::ShouldRecordPartitionedCookieUsage() const {
1914 return request_->cookie_partition_key().has_value();
1915 }
1916
1917 } // namespace net
1918