1// Copyright 2011 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// HTTP client implementation. See RFC 7230 through 7235. 6// 7// This is the low-level Transport implementation of RoundTripper. 8// The high-level interface is in client.go. 9 10package http 11 12import ( 13 "bufio" 14 "compress/gzip" 15 "container/list" 16 "context" 17 "crypto/tls" 18 "errors" 19 "fmt" 20 "internal/godebug" 21 "io" 22 "log" 23 "net" 24 "net/http/httptrace" 25 "net/http/internal/ascii" 26 "net/textproto" 27 "net/url" 28 "reflect" 29 "strings" 30 "sync" 31 "sync/atomic" 32 "time" 33 _ "unsafe" 34 35 "golang.org/x/net/http/httpguts" 36 "golang.org/x/net/http/httpproxy" 37) 38 39// DefaultTransport is the default implementation of [Transport] and is 40// used by [DefaultClient]. It establishes network connections as needed 41// and caches them for reuse by subsequent calls. It uses HTTP proxies 42// as directed by the environment variables HTTP_PROXY, HTTPS_PROXY 43// and NO_PROXY (or the lowercase versions thereof). 44var DefaultTransport RoundTripper = &Transport{ 45 Proxy: ProxyFromEnvironment, 46 DialContext: defaultTransportDialContext(&net.Dialer{ 47 Timeout: 30 * time.Second, 48 KeepAlive: 30 * time.Second, 49 }), 50 ForceAttemptHTTP2: true, 51 MaxIdleConns: 100, 52 IdleConnTimeout: 90 * time.Second, 53 TLSHandshakeTimeout: 10 * time.Second, 54 ExpectContinueTimeout: 1 * time.Second, 55} 56 57// DefaultMaxIdleConnsPerHost is the default value of [Transport]'s 58// MaxIdleConnsPerHost. 59const DefaultMaxIdleConnsPerHost = 2 60 61// Transport is an implementation of [RoundTripper] that supports HTTP, 62// HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT). 63// 64// By default, Transport caches connections for future re-use. 65// This may leave many open connections when accessing many hosts. 66// This behavior can be managed using [Transport.CloseIdleConnections] method 67// and the [Transport.MaxIdleConnsPerHost] and [Transport.DisableKeepAlives] fields. 68// 69// Transports should be reused instead of created as needed. 70// Transports are safe for concurrent use by multiple goroutines. 71// 72// A Transport is a low-level primitive for making HTTP and HTTPS requests. 73// For high-level functionality, such as cookies and redirects, see [Client]. 74// 75// Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2 76// for HTTPS URLs, depending on whether the server supports HTTP/2, 77// and how the Transport is configured. The [DefaultTransport] supports HTTP/2. 78// To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2 79// and call ConfigureTransport. See the package docs for more about HTTP/2. 80// 81// Responses with status codes in the 1xx range are either handled 82// automatically (100 expect-continue) or ignored. The one 83// exception is HTTP status code 101 (Switching Protocols), which is 84// considered a terminal status and returned by [Transport.RoundTrip]. To see the 85// ignored 1xx responses, use the httptrace trace package's 86// ClientTrace.Got1xxResponse. 87// 88// Transport only retries a request upon encountering a network error 89// if the connection has been already been used successfully and if the 90// request is idempotent and either has no body or has its [Request.GetBody] 91// defined. HTTP requests are considered idempotent if they have HTTP methods 92// GET, HEAD, OPTIONS, or TRACE; or if their [Header] map contains an 93// "Idempotency-Key" or "X-Idempotency-Key" entry. If the idempotency key 94// value is a zero-length slice, the request is treated as idempotent but the 95// header is not sent on the wire. 96type Transport struct { 97 idleMu sync.Mutex 98 closeIdle bool // user has requested to close all idle conns 99 idleConn map[connectMethodKey][]*persistConn // most recently used at end 100 idleConnWait map[connectMethodKey]wantConnQueue // waiting getConns 101 idleLRU connLRU 102 103 reqMu sync.Mutex 104 reqCanceler map[*Request]context.CancelCauseFunc 105 106 altMu sync.Mutex // guards changing altProto only 107 altProto atomic.Value // of nil or map[string]RoundTripper, key is URI scheme 108 109 connsPerHostMu sync.Mutex 110 connsPerHost map[connectMethodKey]int 111 connsPerHostWait map[connectMethodKey]wantConnQueue // waiting getConns 112 dialsInProgress wantConnQueue 113 114 // Proxy specifies a function to return a proxy for a given 115 // Request. If the function returns a non-nil error, the 116 // request is aborted with the provided error. 117 // 118 // The proxy type is determined by the URL scheme. "http", 119 // "https", "socks5", and "socks5h" are supported. If the scheme is empty, 120 // "http" is assumed. 121 // "socks5" is treated the same as "socks5h". 122 // 123 // If the proxy URL contains a userinfo subcomponent, 124 // the proxy request will pass the username and password 125 // in a Proxy-Authorization header. 126 // 127 // If Proxy is nil or returns a nil *URL, no proxy is used. 128 Proxy func(*Request) (*url.URL, error) 129 130 // OnProxyConnectResponse is called when the Transport gets an HTTP response from 131 // a proxy for a CONNECT request. It's called before the check for a 200 OK response. 132 // If it returns an error, the request fails with that error. 133 OnProxyConnectResponse func(ctx context.Context, proxyURL *url.URL, connectReq *Request, connectRes *Response) error 134 135 // DialContext specifies the dial function for creating unencrypted TCP connections. 136 // If DialContext is nil (and the deprecated Dial below is also nil), 137 // then the transport dials using package net. 138 // 139 // DialContext runs concurrently with calls to RoundTrip. 140 // A RoundTrip call that initiates a dial may end up using 141 // a connection dialed previously when the earlier connection 142 // becomes idle before the later DialContext completes. 143 DialContext func(ctx context.Context, network, addr string) (net.Conn, error) 144 145 // Dial specifies the dial function for creating unencrypted TCP connections. 146 // 147 // Dial runs concurrently with calls to RoundTrip. 148 // A RoundTrip call that initiates a dial may end up using 149 // a connection dialed previously when the earlier connection 150 // becomes idle before the later Dial completes. 151 // 152 // Deprecated: Use DialContext instead, which allows the transport 153 // to cancel dials as soon as they are no longer needed. 154 // If both are set, DialContext takes priority. 155 Dial func(network, addr string) (net.Conn, error) 156 157 // DialTLSContext specifies an optional dial function for creating 158 // TLS connections for non-proxied HTTPS requests. 159 // 160 // If DialTLSContext is nil (and the deprecated DialTLS below is also nil), 161 // DialContext and TLSClientConfig are used. 162 // 163 // If DialTLSContext is set, the Dial and DialContext hooks are not used for HTTPS 164 // requests and the TLSClientConfig and TLSHandshakeTimeout 165 // are ignored. The returned net.Conn is assumed to already be 166 // past the TLS handshake. 167 DialTLSContext func(ctx context.Context, network, addr string) (net.Conn, error) 168 169 // DialTLS specifies an optional dial function for creating 170 // TLS connections for non-proxied HTTPS requests. 171 // 172 // Deprecated: Use DialTLSContext instead, which allows the transport 173 // to cancel dials as soon as they are no longer needed. 174 // If both are set, DialTLSContext takes priority. 175 DialTLS func(network, addr string) (net.Conn, error) 176 177 // TLSClientConfig specifies the TLS configuration to use with 178 // tls.Client. 179 // If nil, the default configuration is used. 180 // If non-nil, HTTP/2 support may not be enabled by default. 181 TLSClientConfig *tls.Config 182 183 // TLSHandshakeTimeout specifies the maximum amount of time to 184 // wait for a TLS handshake. Zero means no timeout. 185 TLSHandshakeTimeout time.Duration 186 187 // DisableKeepAlives, if true, disables HTTP keep-alives and 188 // will only use the connection to the server for a single 189 // HTTP request. 190 // 191 // This is unrelated to the similarly named TCP keep-alives. 192 DisableKeepAlives bool 193 194 // DisableCompression, if true, prevents the Transport from 195 // requesting compression with an "Accept-Encoding: gzip" 196 // request header when the Request contains no existing 197 // Accept-Encoding value. If the Transport requests gzip on 198 // its own and gets a gzipped response, it's transparently 199 // decoded in the Response.Body. However, if the user 200 // explicitly requested gzip it is not automatically 201 // uncompressed. 202 DisableCompression bool 203 204 // MaxIdleConns controls the maximum number of idle (keep-alive) 205 // connections across all hosts. Zero means no limit. 206 MaxIdleConns int 207 208 // MaxIdleConnsPerHost, if non-zero, controls the maximum idle 209 // (keep-alive) connections to keep per-host. If zero, 210 // DefaultMaxIdleConnsPerHost is used. 211 MaxIdleConnsPerHost int 212 213 // MaxConnsPerHost optionally limits the total number of 214 // connections per host, including connections in the dialing, 215 // active, and idle states. On limit violation, dials will block. 216 // 217 // Zero means no limit. 218 MaxConnsPerHost int 219 220 // IdleConnTimeout is the maximum amount of time an idle 221 // (keep-alive) connection will remain idle before closing 222 // itself. 223 // Zero means no limit. 224 IdleConnTimeout time.Duration 225 226 // ResponseHeaderTimeout, if non-zero, specifies the amount of 227 // time to wait for a server's response headers after fully 228 // writing the request (including its body, if any). This 229 // time does not include the time to read the response body. 230 ResponseHeaderTimeout time.Duration 231 232 // ExpectContinueTimeout, if non-zero, specifies the amount of 233 // time to wait for a server's first response headers after fully 234 // writing the request headers if the request has an 235 // "Expect: 100-continue" header. Zero means no timeout and 236 // causes the body to be sent immediately, without 237 // waiting for the server to approve. 238 // This time does not include the time to send the request header. 239 ExpectContinueTimeout time.Duration 240 241 // TLSNextProto specifies how the Transport switches to an 242 // alternate protocol (such as HTTP/2) after a TLS ALPN 243 // protocol negotiation. If Transport dials a TLS connection 244 // with a non-empty protocol name and TLSNextProto contains a 245 // map entry for that key (such as "h2"), then the func is 246 // called with the request's authority (such as "example.com" 247 // or "example.com:1234") and the TLS connection. The function 248 // must return a RoundTripper that then handles the request. 249 // If TLSNextProto is not nil, HTTP/2 support is not enabled 250 // automatically. 251 TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper 252 253 // ProxyConnectHeader optionally specifies headers to send to 254 // proxies during CONNECT requests. 255 // To set the header dynamically, see GetProxyConnectHeader. 256 ProxyConnectHeader Header 257 258 // GetProxyConnectHeader optionally specifies a func to return 259 // headers to send to proxyURL during a CONNECT request to the 260 // ip:port target. 261 // If it returns an error, the Transport's RoundTrip fails with 262 // that error. It can return (nil, nil) to not add headers. 263 // If GetProxyConnectHeader is non-nil, ProxyConnectHeader is 264 // ignored. 265 GetProxyConnectHeader func(ctx context.Context, proxyURL *url.URL, target string) (Header, error) 266 267 // MaxResponseHeaderBytes specifies a limit on how many 268 // response bytes are allowed in the server's response 269 // header. 270 // 271 // Zero means to use a default limit. 272 MaxResponseHeaderBytes int64 273 274 // WriteBufferSize specifies the size of the write buffer used 275 // when writing to the transport. 276 // If zero, a default (currently 4KB) is used. 277 WriteBufferSize int 278 279 // ReadBufferSize specifies the size of the read buffer used 280 // when reading from the transport. 281 // If zero, a default (currently 4KB) is used. 282 ReadBufferSize int 283 284 // nextProtoOnce guards initialization of TLSNextProto and 285 // h2transport (via onceSetNextProtoDefaults) 286 nextProtoOnce sync.Once 287 h2transport h2Transport // non-nil if http2 wired up 288 tlsNextProtoWasNil bool // whether TLSNextProto was nil when the Once fired 289 290 // ForceAttemptHTTP2 controls whether HTTP/2 is enabled when a non-zero 291 // Dial, DialTLS, or DialContext func or TLSClientConfig is provided. 292 // By default, use of any those fields conservatively disables HTTP/2. 293 // To use a custom dialer or TLS config and still attempt HTTP/2 294 // upgrades, set this to true. 295 ForceAttemptHTTP2 bool 296} 297 298func (t *Transport) writeBufferSize() int { 299 if t.WriteBufferSize > 0 { 300 return t.WriteBufferSize 301 } 302 return 4 << 10 303} 304 305func (t *Transport) readBufferSize() int { 306 if t.ReadBufferSize > 0 { 307 return t.ReadBufferSize 308 } 309 return 4 << 10 310} 311 312// Clone returns a deep copy of t's exported fields. 313func (t *Transport) Clone() *Transport { 314 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 315 t2 := &Transport{ 316 Proxy: t.Proxy, 317 OnProxyConnectResponse: t.OnProxyConnectResponse, 318 DialContext: t.DialContext, 319 Dial: t.Dial, 320 DialTLS: t.DialTLS, 321 DialTLSContext: t.DialTLSContext, 322 TLSHandshakeTimeout: t.TLSHandshakeTimeout, 323 DisableKeepAlives: t.DisableKeepAlives, 324 DisableCompression: t.DisableCompression, 325 MaxIdleConns: t.MaxIdleConns, 326 MaxIdleConnsPerHost: t.MaxIdleConnsPerHost, 327 MaxConnsPerHost: t.MaxConnsPerHost, 328 IdleConnTimeout: t.IdleConnTimeout, 329 ResponseHeaderTimeout: t.ResponseHeaderTimeout, 330 ExpectContinueTimeout: t.ExpectContinueTimeout, 331 ProxyConnectHeader: t.ProxyConnectHeader.Clone(), 332 GetProxyConnectHeader: t.GetProxyConnectHeader, 333 MaxResponseHeaderBytes: t.MaxResponseHeaderBytes, 334 ForceAttemptHTTP2: t.ForceAttemptHTTP2, 335 WriteBufferSize: t.WriteBufferSize, 336 ReadBufferSize: t.ReadBufferSize, 337 } 338 if t.TLSClientConfig != nil { 339 t2.TLSClientConfig = t.TLSClientConfig.Clone() 340 } 341 if !t.tlsNextProtoWasNil { 342 npm := map[string]func(authority string, c *tls.Conn) RoundTripper{} 343 for k, v := range t.TLSNextProto { 344 npm[k] = v 345 } 346 t2.TLSNextProto = npm 347 } 348 return t2 349} 350 351// h2Transport is the interface we expect to be able to call from 352// net/http against an *http2.Transport that's either bundled into 353// h2_bundle.go or supplied by the user via x/net/http2. 354// 355// We name it with the "h2" prefix to stay out of the "http2" prefix 356// namespace used by x/tools/cmd/bundle for h2_bundle.go. 357type h2Transport interface { 358 CloseIdleConnections() 359} 360 361func (t *Transport) hasCustomTLSDialer() bool { 362 return t.DialTLS != nil || t.DialTLSContext != nil 363} 364 365var http2client = godebug.New("http2client") 366 367// onceSetNextProtoDefaults initializes TLSNextProto. 368// It must be called via t.nextProtoOnce.Do. 369func (t *Transport) onceSetNextProtoDefaults() { 370 t.tlsNextProtoWasNil = (t.TLSNextProto == nil) 371 if http2client.Value() == "0" { 372 http2client.IncNonDefault() 373 return 374 } 375 376 // If they've already configured http2 with 377 // golang.org/x/net/http2 instead of the bundled copy, try to 378 // get at its http2.Transport value (via the "https" 379 // altproto map) so we can call CloseIdleConnections on it if 380 // requested. (Issue 22891) 381 altProto, _ := t.altProto.Load().(map[string]RoundTripper) 382 if rv := reflect.ValueOf(altProto["https"]); rv.IsValid() && rv.Type().Kind() == reflect.Struct && rv.Type().NumField() == 1 { 383 if v := rv.Field(0); v.CanInterface() { 384 if h2i, ok := v.Interface().(h2Transport); ok { 385 t.h2transport = h2i 386 return 387 } 388 } 389 } 390 391 if t.TLSNextProto != nil { 392 // This is the documented way to disable http2 on a 393 // Transport. 394 return 395 } 396 if !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialContext != nil || t.hasCustomTLSDialer()) { 397 // Be conservative and don't automatically enable 398 // http2 if they've specified a custom TLS config or 399 // custom dialers. Let them opt-in themselves via 400 // http2.ConfigureTransport so we don't surprise them 401 // by modifying their tls.Config. Issue 14275. 402 // However, if ForceAttemptHTTP2 is true, it overrides the above checks. 403 return 404 } 405 if omitBundledHTTP2 { 406 return 407 } 408 t2, err := http2configureTransports(t) 409 if err != nil { 410 log.Printf("Error enabling Transport HTTP/2 support: %v", err) 411 return 412 } 413 t.h2transport = t2 414 415 // Auto-configure the http2.Transport's MaxHeaderListSize from 416 // the http.Transport's MaxResponseHeaderBytes. They don't 417 // exactly mean the same thing, but they're close. 418 // 419 // TODO: also add this to x/net/http2.Configure Transport, behind 420 // a +build go1.7 build tag: 421 if limit1 := t.MaxResponseHeaderBytes; limit1 != 0 && t2.MaxHeaderListSize == 0 { 422 const h2max = 1<<32 - 1 423 if limit1 >= h2max { 424 t2.MaxHeaderListSize = h2max 425 } else { 426 t2.MaxHeaderListSize = uint32(limit1) 427 } 428 } 429} 430 431// ProxyFromEnvironment returns the URL of the proxy to use for a 432// given request, as indicated by the environment variables 433// HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions 434// thereof). Requests use the proxy from the environment variable 435// matching their scheme, unless excluded by NO_PROXY. 436// 437// The environment values may be either a complete URL or a 438// "host[:port]", in which case the "http" scheme is assumed. 439// An error is returned if the value is a different form. 440// 441// A nil URL and nil error are returned if no proxy is defined in the 442// environment, or a proxy should not be used for the given request, 443// as defined by NO_PROXY. 444// 445// As a special case, if req.URL.Host is "localhost" (with or without 446// a port number), then a nil URL and nil error will be returned. 447func ProxyFromEnvironment(req *Request) (*url.URL, error) { 448 return envProxyFunc()(req.URL) 449} 450 451// ProxyURL returns a proxy function (for use in a [Transport]) 452// that always returns the same URL. 453func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { 454 return func(*Request) (*url.URL, error) { 455 return fixedURL, nil 456 } 457} 458 459// transportRequest is a wrapper around a *Request that adds 460// optional extra headers to write and stores any error to return 461// from roundTrip. 462type transportRequest struct { 463 *Request // original request, not to be mutated 464 extra Header // extra headers to write, or nil 465 trace *httptrace.ClientTrace // optional 466 467 ctx context.Context // canceled when we are done with the request 468 cancel context.CancelCauseFunc 469 470 mu sync.Mutex // guards err 471 err error // first setError value for mapRoundTripError to consider 472} 473 474func (tr *transportRequest) extraHeaders() Header { 475 if tr.extra == nil { 476 tr.extra = make(Header) 477 } 478 return tr.extra 479} 480 481func (tr *transportRequest) setError(err error) { 482 tr.mu.Lock() 483 if tr.err == nil { 484 tr.err = err 485 } 486 tr.mu.Unlock() 487} 488 489// useRegisteredProtocol reports whether an alternate protocol (as registered 490// with Transport.RegisterProtocol) should be respected for this request. 491func (t *Transport) useRegisteredProtocol(req *Request) bool { 492 if req.URL.Scheme == "https" && req.requiresHTTP1() { 493 // If this request requires HTTP/1, don't use the 494 // "https" alternate protocol, which is used by the 495 // HTTP/2 code to take over requests if there's an 496 // existing cached HTTP/2 connection. 497 return false 498 } 499 return true 500} 501 502// alternateRoundTripper returns the alternate RoundTripper to use 503// for this request if the Request's URL scheme requires one, 504// or nil for the normal case of using the Transport. 505func (t *Transport) alternateRoundTripper(req *Request) RoundTripper { 506 if !t.useRegisteredProtocol(req) { 507 return nil 508 } 509 altProto, _ := t.altProto.Load().(map[string]RoundTripper) 510 return altProto[req.URL.Scheme] 511} 512 513func validateHeaders(hdrs Header) string { 514 for k, vv := range hdrs { 515 if !httpguts.ValidHeaderFieldName(k) { 516 return fmt.Sprintf("field name %q", k) 517 } 518 for _, v := range vv { 519 if !httpguts.ValidHeaderFieldValue(v) { 520 // Don't include the value in the error, 521 // because it may be sensitive. 522 return fmt.Sprintf("field value for %q", k) 523 } 524 } 525 } 526 return "" 527} 528 529// roundTrip implements a RoundTripper over HTTP. 530func (t *Transport) roundTrip(req *Request) (_ *Response, err error) { 531 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 532 ctx := req.Context() 533 trace := httptrace.ContextClientTrace(ctx) 534 535 if req.URL == nil { 536 req.closeBody() 537 return nil, errors.New("http: nil Request.URL") 538 } 539 if req.Header == nil { 540 req.closeBody() 541 return nil, errors.New("http: nil Request.Header") 542 } 543 scheme := req.URL.Scheme 544 isHTTP := scheme == "http" || scheme == "https" 545 if isHTTP { 546 // Validate the outgoing headers. 547 if err := validateHeaders(req.Header); err != "" { 548 req.closeBody() 549 return nil, fmt.Errorf("net/http: invalid header %s", err) 550 } 551 552 // Validate the outgoing trailers too. 553 if err := validateHeaders(req.Trailer); err != "" { 554 req.closeBody() 555 return nil, fmt.Errorf("net/http: invalid trailer %s", err) 556 } 557 } 558 559 origReq := req 560 req = setupRewindBody(req) 561 562 if altRT := t.alternateRoundTripper(req); altRT != nil { 563 if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol { 564 return resp, err 565 } 566 var err error 567 req, err = rewindBody(req) 568 if err != nil { 569 return nil, err 570 } 571 } 572 if !isHTTP { 573 req.closeBody() 574 return nil, badStringError("unsupported protocol scheme", scheme) 575 } 576 if req.Method != "" && !validMethod(req.Method) { 577 req.closeBody() 578 return nil, fmt.Errorf("net/http: invalid method %q", req.Method) 579 } 580 if req.URL.Host == "" { 581 req.closeBody() 582 return nil, errors.New("http: no Host in request URL") 583 } 584 585 // Transport request context. 586 // 587 // If RoundTrip returns an error, it cancels this context before returning. 588 // 589 // If RoundTrip returns no error: 590 // - For an HTTP/1 request, persistConn.readLoop cancels this context 591 // after reading the request body. 592 // - For an HTTP/2 request, RoundTrip cancels this context after the HTTP/2 593 // RoundTripper returns. 594 ctx, cancel := context.WithCancelCause(req.Context()) 595 596 // Convert Request.Cancel into context cancelation. 597 if origReq.Cancel != nil { 598 go awaitLegacyCancel(ctx, cancel, origReq) 599 } 600 601 // Convert Transport.CancelRequest into context cancelation. 602 // 603 // This is lamentably expensive. CancelRequest has been deprecated for a long time 604 // and doesn't work on HTTP/2 requests. Perhaps we should drop support for it entirely. 605 cancel = t.prepareTransportCancel(origReq, cancel) 606 607 defer func() { 608 if err != nil { 609 cancel(err) 610 } 611 }() 612 613 for { 614 select { 615 case <-ctx.Done(): 616 req.closeBody() 617 return nil, context.Cause(ctx) 618 default: 619 } 620 621 // treq gets modified by roundTrip, so we need to recreate for each retry. 622 treq := &transportRequest{Request: req, trace: trace, ctx: ctx, cancel: cancel} 623 cm, err := t.connectMethodForRequest(treq) 624 if err != nil { 625 req.closeBody() 626 return nil, err 627 } 628 629 // Get the cached or newly-created connection to either the 630 // host (for http or https), the http proxy, or the http proxy 631 // pre-CONNECTed to https server. In any case, we'll be ready 632 // to send it requests. 633 pconn, err := t.getConn(treq, cm) 634 if err != nil { 635 req.closeBody() 636 return nil, err 637 } 638 639 var resp *Response 640 if pconn.alt != nil { 641 // HTTP/2 path. 642 resp, err = pconn.alt.RoundTrip(req) 643 } else { 644 resp, err = pconn.roundTrip(treq) 645 } 646 if err == nil { 647 if pconn.alt != nil { 648 // HTTP/2 requests are not cancelable with CancelRequest, 649 // so we have no further need for the request context. 650 // 651 // On the HTTP/1 path, roundTrip takes responsibility for 652 // canceling the context after the response body is read. 653 cancel(errRequestDone) 654 } 655 resp.Request = origReq 656 return resp, nil 657 } 658 659 // Failed. Clean up and determine whether to retry. 660 if http2isNoCachedConnError(err) { 661 if t.removeIdleConn(pconn) { 662 t.decConnsPerHost(pconn.cacheKey) 663 } 664 } else if !pconn.shouldRetryRequest(req, err) { 665 // Issue 16465: return underlying net.Conn.Read error from peek, 666 // as we've historically done. 667 if e, ok := err.(nothingWrittenError); ok { 668 err = e.error 669 } 670 if e, ok := err.(transportReadFromServerError); ok { 671 err = e.err 672 } 673 if b, ok := req.Body.(*readTrackingBody); ok && !b.didClose { 674 // Issue 49621: Close the request body if pconn.roundTrip 675 // didn't do so already. This can happen if the pconn 676 // write loop exits without reading the write request. 677 req.closeBody() 678 } 679 return nil, err 680 } 681 testHookRoundTripRetried() 682 683 // Rewind the body if we're able to. 684 req, err = rewindBody(req) 685 if err != nil { 686 return nil, err 687 } 688 } 689} 690 691func awaitLegacyCancel(ctx context.Context, cancel context.CancelCauseFunc, req *Request) { 692 select { 693 case <-req.Cancel: 694 cancel(errRequestCanceled) 695 case <-ctx.Done(): 696 } 697} 698 699var errCannotRewind = errors.New("net/http: cannot rewind body after connection loss") 700 701type readTrackingBody struct { 702 io.ReadCloser 703 didRead bool 704 didClose bool 705} 706 707func (r *readTrackingBody) Read(data []byte) (int, error) { 708 r.didRead = true 709 return r.ReadCloser.Read(data) 710} 711 712func (r *readTrackingBody) Close() error { 713 r.didClose = true 714 return r.ReadCloser.Close() 715} 716 717// setupRewindBody returns a new request with a custom body wrapper 718// that can report whether the body needs rewinding. 719// This lets rewindBody avoid an error result when the request 720// does not have GetBody but the body hasn't been read at all yet. 721func setupRewindBody(req *Request) *Request { 722 if req.Body == nil || req.Body == NoBody { 723 return req 724 } 725 newReq := *req 726 newReq.Body = &readTrackingBody{ReadCloser: req.Body} 727 return &newReq 728} 729 730// rewindBody returns a new request with the body rewound. 731// It returns req unmodified if the body does not need rewinding. 732// rewindBody takes care of closing req.Body when appropriate 733// (in all cases except when rewindBody returns req unmodified). 734func rewindBody(req *Request) (rewound *Request, err error) { 735 if req.Body == nil || req.Body == NoBody || (!req.Body.(*readTrackingBody).didRead && !req.Body.(*readTrackingBody).didClose) { 736 return req, nil // nothing to rewind 737 } 738 if !req.Body.(*readTrackingBody).didClose { 739 req.closeBody() 740 } 741 if req.GetBody == nil { 742 return nil, errCannotRewind 743 } 744 body, err := req.GetBody() 745 if err != nil { 746 return nil, err 747 } 748 newReq := *req 749 newReq.Body = &readTrackingBody{ReadCloser: body} 750 return &newReq, nil 751} 752 753// shouldRetryRequest reports whether we should retry sending a failed 754// HTTP request on a new connection. The non-nil input error is the 755// error from roundTrip. 756func (pc *persistConn) shouldRetryRequest(req *Request, err error) bool { 757 if http2isNoCachedConnError(err) { 758 // Issue 16582: if the user started a bunch of 759 // requests at once, they can all pick the same conn 760 // and violate the server's max concurrent streams. 761 // Instead, match the HTTP/1 behavior for now and dial 762 // again to get a new TCP connection, rather than failing 763 // this request. 764 return true 765 } 766 if err == errMissingHost { 767 // User error. 768 return false 769 } 770 if !pc.isReused() { 771 // This was a fresh connection. There's no reason the server 772 // should've hung up on us. 773 // 774 // Also, if we retried now, we could loop forever 775 // creating new connections and retrying if the server 776 // is just hanging up on us because it doesn't like 777 // our request (as opposed to sending an error). 778 return false 779 } 780 if _, ok := err.(nothingWrittenError); ok { 781 // We never wrote anything, so it's safe to retry, if there's no body or we 782 // can "rewind" the body with GetBody. 783 return req.outgoingLength() == 0 || req.GetBody != nil 784 } 785 if !req.isReplayable() { 786 // Don't retry non-idempotent requests. 787 return false 788 } 789 if _, ok := err.(transportReadFromServerError); ok { 790 // We got some non-EOF net.Conn.Read failure reading 791 // the 1st response byte from the server. 792 return true 793 } 794 if err == errServerClosedIdle { 795 // The server replied with io.EOF while we were trying to 796 // read the response. Probably an unfortunately keep-alive 797 // timeout, just as the client was writing a request. 798 return true 799 } 800 return false // conservatively 801} 802 803// ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol. 804var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol") 805 806// RegisterProtocol registers a new protocol with scheme. 807// The [Transport] will pass requests using the given scheme to rt. 808// It is rt's responsibility to simulate HTTP request semantics. 809// 810// RegisterProtocol can be used by other packages to provide 811// implementations of protocol schemes like "ftp" or "file". 812// 813// If rt.RoundTrip returns [ErrSkipAltProtocol], the Transport will 814// handle the [Transport.RoundTrip] itself for that one request, as if the 815// protocol were not registered. 816func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { 817 t.altMu.Lock() 818 defer t.altMu.Unlock() 819 oldMap, _ := t.altProto.Load().(map[string]RoundTripper) 820 if _, exists := oldMap[scheme]; exists { 821 panic("protocol " + scheme + " already registered") 822 } 823 newMap := make(map[string]RoundTripper) 824 for k, v := range oldMap { 825 newMap[k] = v 826 } 827 newMap[scheme] = rt 828 t.altProto.Store(newMap) 829} 830 831// CloseIdleConnections closes any connections which were previously 832// connected from previous requests but are now sitting idle in 833// a "keep-alive" state. It does not interrupt any connections currently 834// in use. 835func (t *Transport) CloseIdleConnections() { 836 t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) 837 t.idleMu.Lock() 838 m := t.idleConn 839 t.idleConn = nil 840 t.closeIdle = true // close newly idle connections 841 t.idleLRU = connLRU{} 842 t.idleMu.Unlock() 843 for _, conns := range m { 844 for _, pconn := range conns { 845 pconn.close(errCloseIdleConns) 846 } 847 } 848 t.connsPerHostMu.Lock() 849 t.dialsInProgress.all(func(w *wantConn) { 850 if w.cancelCtx != nil && !w.waiting() { 851 w.cancelCtx() 852 } 853 }) 854 t.connsPerHostMu.Unlock() 855 if t2 := t.h2transport; t2 != nil { 856 t2.CloseIdleConnections() 857 } 858} 859 860// prepareTransportCancel sets up state to convert Transport.CancelRequest into context cancelation. 861func (t *Transport) prepareTransportCancel(req *Request, origCancel context.CancelCauseFunc) context.CancelCauseFunc { 862 // Historically, RoundTrip has not modified the Request in any way. 863 // We could avoid the need to keep a map of all in-flight requests by adding 864 // a field to the Request containing its cancel func, and setting that field 865 // while the request is in-flight. Callers aren't supposed to reuse a Request 866 // until after the response body is closed, so this wouldn't violate any 867 // concurrency guarantees. 868 cancel := func(err error) { 869 origCancel(err) 870 t.reqMu.Lock() 871 delete(t.reqCanceler, req) 872 t.reqMu.Unlock() 873 } 874 t.reqMu.Lock() 875 if t.reqCanceler == nil { 876 t.reqCanceler = make(map[*Request]context.CancelCauseFunc) 877 } 878 t.reqCanceler[req] = cancel 879 t.reqMu.Unlock() 880 return cancel 881} 882 883// CancelRequest cancels an in-flight request by closing its connection. 884// CancelRequest should only be called after [Transport.RoundTrip] has returned. 885// 886// Deprecated: Use [Request.WithContext] to create a request with a 887// cancelable context instead. CancelRequest cannot cancel HTTP/2 888// requests. This may become a no-op in a future release of Go. 889func (t *Transport) CancelRequest(req *Request) { 890 t.reqMu.Lock() 891 cancel := t.reqCanceler[req] 892 t.reqMu.Unlock() 893 if cancel != nil { 894 cancel(errRequestCanceled) 895 } 896} 897 898// 899// Private implementation past this point. 900// 901 902var ( 903 envProxyOnce sync.Once 904 envProxyFuncValue func(*url.URL) (*url.URL, error) 905) 906 907// envProxyFunc returns a function that reads the 908// environment variable to determine the proxy address. 909func envProxyFunc() func(*url.URL) (*url.URL, error) { 910 envProxyOnce.Do(func() { 911 envProxyFuncValue = httpproxy.FromEnvironment().ProxyFunc() 912 }) 913 return envProxyFuncValue 914} 915 916// resetProxyConfig is used by tests. 917func resetProxyConfig() { 918 envProxyOnce = sync.Once{} 919 envProxyFuncValue = nil 920} 921 922func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { 923 cm.targetScheme = treq.URL.Scheme 924 cm.targetAddr = canonicalAddr(treq.URL) 925 if t.Proxy != nil { 926 cm.proxyURL, err = t.Proxy(treq.Request) 927 } 928 cm.onlyH1 = treq.requiresHTTP1() 929 return cm, err 930} 931 932// proxyAuth returns the Proxy-Authorization header to set 933// on requests, if applicable. 934func (cm *connectMethod) proxyAuth() string { 935 if cm.proxyURL == nil { 936 return "" 937 } 938 if u := cm.proxyURL.User; u != nil { 939 username := u.Username() 940 password, _ := u.Password() 941 return "Basic " + basicAuth(username, password) 942 } 943 return "" 944} 945 946// error values for debugging and testing, not seen by users. 947var ( 948 errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled") 949 errConnBroken = errors.New("http: putIdleConn: connection is in bad state") 950 errCloseIdle = errors.New("http: putIdleConn: CloseIdleConnections was called") 951 errTooManyIdle = errors.New("http: putIdleConn: too many idle connections") 952 errTooManyIdleHost = errors.New("http: putIdleConn: too many idle connections for host") 953 errCloseIdleConns = errors.New("http: CloseIdleConnections called") 954 errReadLoopExiting = errors.New("http: persistConn.readLoop exiting") 955 errIdleConnTimeout = errors.New("http: idle connection timeout") 956 957 // errServerClosedIdle is not seen by users for idempotent requests, but may be 958 // seen by a user if the server shuts down an idle connection and sends its FIN 959 // in flight with already-written POST body bytes from the client. 960 // See https://github.com/golang/go/issues/19943#issuecomment-355607646 961 errServerClosedIdle = errors.New("http: server closed idle connection") 962) 963 964// transportReadFromServerError is used by Transport.readLoop when the 965// 1 byte peek read fails and we're actually anticipating a response. 966// Usually this is just due to the inherent keep-alive shut down race, 967// where the server closed the connection at the same time the client 968// wrote. The underlying err field is usually io.EOF or some 969// ECONNRESET sort of thing which varies by platform. But it might be 970// the user's custom net.Conn.Read error too, so we carry it along for 971// them to return from Transport.RoundTrip. 972type transportReadFromServerError struct { 973 err error 974} 975 976func (e transportReadFromServerError) Unwrap() error { return e.err } 977 978func (e transportReadFromServerError) Error() string { 979 return fmt.Sprintf("net/http: Transport failed to read from server: %v", e.err) 980} 981 982func (t *Transport) putOrCloseIdleConn(pconn *persistConn) { 983 if err := t.tryPutIdleConn(pconn); err != nil { 984 pconn.close(err) 985 } 986} 987 988func (t *Transport) maxIdleConnsPerHost() int { 989 if v := t.MaxIdleConnsPerHost; v != 0 { 990 return v 991 } 992 return DefaultMaxIdleConnsPerHost 993} 994 995// tryPutIdleConn adds pconn to the list of idle persistent connections awaiting 996// a new request. 997// If pconn is no longer needed or not in a good state, tryPutIdleConn returns 998// an error explaining why it wasn't registered. 999// tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that. 1000func (t *Transport) tryPutIdleConn(pconn *persistConn) error { 1001 if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 { 1002 return errKeepAlivesDisabled 1003 } 1004 if pconn.isBroken() { 1005 return errConnBroken 1006 } 1007 pconn.markReused() 1008 1009 t.idleMu.Lock() 1010 defer t.idleMu.Unlock() 1011 1012 // HTTP/2 (pconn.alt != nil) connections do not come out of the idle list, 1013 // because multiple goroutines can use them simultaneously. 1014 // If this is an HTTP/2 connection being “returned,” we're done. 1015 if pconn.alt != nil && t.idleLRU.m[pconn] != nil { 1016 return nil 1017 } 1018 1019 // Deliver pconn to goroutine waiting for idle connection, if any. 1020 // (They may be actively dialing, but this conn is ready first. 1021 // Chrome calls this socket late binding. 1022 // See https://www.chromium.org/developers/design-documents/network-stack#TOC-Connection-Management.) 1023 key := pconn.cacheKey 1024 if q, ok := t.idleConnWait[key]; ok { 1025 done := false 1026 if pconn.alt == nil { 1027 // HTTP/1. 1028 // Loop over the waiting list until we find a w that isn't done already, and hand it pconn. 1029 for q.len() > 0 { 1030 w := q.popFront() 1031 if w.tryDeliver(pconn, nil, time.Time{}) { 1032 done = true 1033 break 1034 } 1035 } 1036 } else { 1037 // HTTP/2. 1038 // Can hand the same pconn to everyone in the waiting list, 1039 // and we still won't be done: we want to put it in the idle 1040 // list unconditionally, for any future clients too. 1041 for q.len() > 0 { 1042 w := q.popFront() 1043 w.tryDeliver(pconn, nil, time.Time{}) 1044 } 1045 } 1046 if q.len() == 0 { 1047 delete(t.idleConnWait, key) 1048 } else { 1049 t.idleConnWait[key] = q 1050 } 1051 if done { 1052 return nil 1053 } 1054 } 1055 1056 if t.closeIdle { 1057 return errCloseIdle 1058 } 1059 if t.idleConn == nil { 1060 t.idleConn = make(map[connectMethodKey][]*persistConn) 1061 } 1062 idles := t.idleConn[key] 1063 if len(idles) >= t.maxIdleConnsPerHost() { 1064 return errTooManyIdleHost 1065 } 1066 for _, exist := range idles { 1067 if exist == pconn { 1068 log.Fatalf("dup idle pconn %p in freelist", pconn) 1069 } 1070 } 1071 t.idleConn[key] = append(idles, pconn) 1072 t.idleLRU.add(pconn) 1073 if t.MaxIdleConns != 0 && t.idleLRU.len() > t.MaxIdleConns { 1074 oldest := t.idleLRU.removeOldest() 1075 oldest.close(errTooManyIdle) 1076 t.removeIdleConnLocked(oldest) 1077 } 1078 1079 // Set idle timer, but only for HTTP/1 (pconn.alt == nil). 1080 // The HTTP/2 implementation manages the idle timer itself 1081 // (see idleConnTimeout in h2_bundle.go). 1082 if t.IdleConnTimeout > 0 && pconn.alt == nil { 1083 if pconn.idleTimer != nil { 1084 pconn.idleTimer.Reset(t.IdleConnTimeout) 1085 } else { 1086 pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle) 1087 } 1088 } 1089 pconn.idleAt = time.Now() 1090 return nil 1091} 1092 1093// queueForIdleConn queues w to receive the next idle connection for w.cm. 1094// As an optimization hint to the caller, queueForIdleConn reports whether 1095// it successfully delivered an already-idle connection. 1096func (t *Transport) queueForIdleConn(w *wantConn) (delivered bool) { 1097 if t.DisableKeepAlives { 1098 return false 1099 } 1100 1101 t.idleMu.Lock() 1102 defer t.idleMu.Unlock() 1103 1104 // Stop closing connections that become idle - we might want one. 1105 // (That is, undo the effect of t.CloseIdleConnections.) 1106 t.closeIdle = false 1107 1108 if w == nil { 1109 // Happens in test hook. 1110 return false 1111 } 1112 1113 // If IdleConnTimeout is set, calculate the oldest 1114 // persistConn.idleAt time we're willing to use a cached idle 1115 // conn. 1116 var oldTime time.Time 1117 if t.IdleConnTimeout > 0 { 1118 oldTime = time.Now().Add(-t.IdleConnTimeout) 1119 } 1120 1121 // Look for most recently-used idle connection. 1122 if list, ok := t.idleConn[w.key]; ok { 1123 stop := false 1124 delivered := false 1125 for len(list) > 0 && !stop { 1126 pconn := list[len(list)-1] 1127 1128 // See whether this connection has been idle too long, considering 1129 // only the wall time (the Round(0)), in case this is a laptop or VM 1130 // coming out of suspend with previously cached idle connections. 1131 tooOld := !oldTime.IsZero() && pconn.idleAt.Round(0).Before(oldTime) 1132 if tooOld { 1133 // Async cleanup. Launch in its own goroutine (as if a 1134 // time.AfterFunc called it); it acquires idleMu, which we're 1135 // holding, and does a synchronous net.Conn.Close. 1136 go pconn.closeConnIfStillIdle() 1137 } 1138 if pconn.isBroken() || tooOld { 1139 // If either persistConn.readLoop has marked the connection 1140 // broken, but Transport.removeIdleConn has not yet removed it 1141 // from the idle list, or if this persistConn is too old (it was 1142 // idle too long), then ignore it and look for another. In both 1143 // cases it's already in the process of being closed. 1144 list = list[:len(list)-1] 1145 continue 1146 } 1147 delivered = w.tryDeliver(pconn, nil, pconn.idleAt) 1148 if delivered { 1149 if pconn.alt != nil { 1150 // HTTP/2: multiple clients can share pconn. 1151 // Leave it in the list. 1152 } else { 1153 // HTTP/1: only one client can use pconn. 1154 // Remove it from the list. 1155 t.idleLRU.remove(pconn) 1156 list = list[:len(list)-1] 1157 } 1158 } 1159 stop = true 1160 } 1161 if len(list) > 0 { 1162 t.idleConn[w.key] = list 1163 } else { 1164 delete(t.idleConn, w.key) 1165 } 1166 if stop { 1167 return delivered 1168 } 1169 } 1170 1171 // Register to receive next connection that becomes idle. 1172 if t.idleConnWait == nil { 1173 t.idleConnWait = make(map[connectMethodKey]wantConnQueue) 1174 } 1175 q := t.idleConnWait[w.key] 1176 q.cleanFrontNotWaiting() 1177 q.pushBack(w) 1178 t.idleConnWait[w.key] = q 1179 return false 1180} 1181 1182// removeIdleConn marks pconn as dead. 1183func (t *Transport) removeIdleConn(pconn *persistConn) bool { 1184 t.idleMu.Lock() 1185 defer t.idleMu.Unlock() 1186 return t.removeIdleConnLocked(pconn) 1187} 1188 1189// t.idleMu must be held. 1190func (t *Transport) removeIdleConnLocked(pconn *persistConn) bool { 1191 if pconn.idleTimer != nil { 1192 pconn.idleTimer.Stop() 1193 } 1194 t.idleLRU.remove(pconn) 1195 key := pconn.cacheKey 1196 pconns := t.idleConn[key] 1197 var removed bool 1198 switch len(pconns) { 1199 case 0: 1200 // Nothing 1201 case 1: 1202 if pconns[0] == pconn { 1203 delete(t.idleConn, key) 1204 removed = true 1205 } 1206 default: 1207 for i, v := range pconns { 1208 if v != pconn { 1209 continue 1210 } 1211 // Slide down, keeping most recently-used 1212 // conns at the end. 1213 copy(pconns[i:], pconns[i+1:]) 1214 t.idleConn[key] = pconns[:len(pconns)-1] 1215 removed = true 1216 break 1217 } 1218 } 1219 return removed 1220} 1221 1222var zeroDialer net.Dialer 1223 1224func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) { 1225 if t.DialContext != nil { 1226 c, err := t.DialContext(ctx, network, addr) 1227 if c == nil && err == nil { 1228 err = errors.New("net/http: Transport.DialContext hook returned (nil, nil)") 1229 } 1230 return c, err 1231 } 1232 if t.Dial != nil { 1233 c, err := t.Dial(network, addr) 1234 if c == nil && err == nil { 1235 err = errors.New("net/http: Transport.Dial hook returned (nil, nil)") 1236 } 1237 return c, err 1238 } 1239 return zeroDialer.DialContext(ctx, network, addr) 1240} 1241 1242// A wantConn records state about a wanted connection 1243// (that is, an active call to getConn). 1244// The conn may be gotten by dialing or by finding an idle connection, 1245// or a cancellation may make the conn no longer wanted. 1246// These three options are racing against each other and use 1247// wantConn to coordinate and agree about the winning outcome. 1248type wantConn struct { 1249 cm connectMethod 1250 key connectMethodKey // cm.key() 1251 1252 // hooks for testing to know when dials are done 1253 // beforeDial is called in the getConn goroutine when the dial is queued. 1254 // afterDial is called when the dial is completed or canceled. 1255 beforeDial func() 1256 afterDial func() 1257 1258 mu sync.Mutex // protects ctx, done and sending of the result 1259 ctx context.Context // context for dial, cleared after delivered or canceled 1260 cancelCtx context.CancelFunc 1261 done bool // true after delivered or canceled 1262 result chan connOrError // channel to deliver connection or error 1263} 1264 1265type connOrError struct { 1266 pc *persistConn 1267 err error 1268 idleAt time.Time 1269} 1270 1271// waiting reports whether w is still waiting for an answer (connection or error). 1272func (w *wantConn) waiting() bool { 1273 w.mu.Lock() 1274 defer w.mu.Unlock() 1275 1276 return !w.done 1277} 1278 1279// getCtxForDial returns context for dial or nil if connection was delivered or canceled. 1280func (w *wantConn) getCtxForDial() context.Context { 1281 w.mu.Lock() 1282 defer w.mu.Unlock() 1283 1284 return w.ctx 1285} 1286 1287// tryDeliver attempts to deliver pc, err to w and reports whether it succeeded. 1288func (w *wantConn) tryDeliver(pc *persistConn, err error, idleAt time.Time) bool { 1289 w.mu.Lock() 1290 defer w.mu.Unlock() 1291 1292 if w.done { 1293 return false 1294 } 1295 if (pc == nil) == (err == nil) { 1296 panic("net/http: internal error: misuse of tryDeliver") 1297 } 1298 w.ctx = nil 1299 w.done = true 1300 1301 w.result <- connOrError{pc: pc, err: err, idleAt: idleAt} 1302 close(w.result) 1303 1304 return true 1305} 1306 1307// cancel marks w as no longer wanting a result (for example, due to cancellation). 1308// If a connection has been delivered already, cancel returns it with t.putOrCloseIdleConn. 1309func (w *wantConn) cancel(t *Transport, err error) { 1310 w.mu.Lock() 1311 var pc *persistConn 1312 if w.done { 1313 if r, ok := <-w.result; ok { 1314 pc = r.pc 1315 } 1316 } else { 1317 close(w.result) 1318 } 1319 w.ctx = nil 1320 w.done = true 1321 w.mu.Unlock() 1322 1323 if pc != nil { 1324 t.putOrCloseIdleConn(pc) 1325 } 1326} 1327 1328// A wantConnQueue is a queue of wantConns. 1329type wantConnQueue struct { 1330 // This is a queue, not a deque. 1331 // It is split into two stages - head[headPos:] and tail. 1332 // popFront is trivial (headPos++) on the first stage, and 1333 // pushBack is trivial (append) on the second stage. 1334 // If the first stage is empty, popFront can swap the 1335 // first and second stages to remedy the situation. 1336 // 1337 // This two-stage split is analogous to the use of two lists 1338 // in Okasaki's purely functional queue but without the 1339 // overhead of reversing the list when swapping stages. 1340 head []*wantConn 1341 headPos int 1342 tail []*wantConn 1343} 1344 1345// len returns the number of items in the queue. 1346func (q *wantConnQueue) len() int { 1347 return len(q.head) - q.headPos + len(q.tail) 1348} 1349 1350// pushBack adds w to the back of the queue. 1351func (q *wantConnQueue) pushBack(w *wantConn) { 1352 q.tail = append(q.tail, w) 1353} 1354 1355// popFront removes and returns the wantConn at the front of the queue. 1356func (q *wantConnQueue) popFront() *wantConn { 1357 if q.headPos >= len(q.head) { 1358 if len(q.tail) == 0 { 1359 return nil 1360 } 1361 // Pick up tail as new head, clear tail. 1362 q.head, q.headPos, q.tail = q.tail, 0, q.head[:0] 1363 } 1364 w := q.head[q.headPos] 1365 q.head[q.headPos] = nil 1366 q.headPos++ 1367 return w 1368} 1369 1370// peekFront returns the wantConn at the front of the queue without removing it. 1371func (q *wantConnQueue) peekFront() *wantConn { 1372 if q.headPos < len(q.head) { 1373 return q.head[q.headPos] 1374 } 1375 if len(q.tail) > 0 { 1376 return q.tail[0] 1377 } 1378 return nil 1379} 1380 1381// cleanFrontNotWaiting pops any wantConns that are no longer waiting from the head of the 1382// queue, reporting whether any were popped. 1383func (q *wantConnQueue) cleanFrontNotWaiting() (cleaned bool) { 1384 for { 1385 w := q.peekFront() 1386 if w == nil || w.waiting() { 1387 return cleaned 1388 } 1389 q.popFront() 1390 cleaned = true 1391 } 1392} 1393 1394// cleanFrontCanceled pops any wantConns with canceled dials from the head of the queue. 1395func (q *wantConnQueue) cleanFrontCanceled() { 1396 for { 1397 w := q.peekFront() 1398 if w == nil || w.cancelCtx != nil { 1399 return 1400 } 1401 q.popFront() 1402 } 1403} 1404 1405// all iterates over all wantConns in the queue. 1406// The caller must not modify the queue while iterating. 1407func (q *wantConnQueue) all(f func(*wantConn)) { 1408 for _, w := range q.head[q.headPos:] { 1409 f(w) 1410 } 1411 for _, w := range q.tail { 1412 f(w) 1413 } 1414} 1415 1416func (t *Transport) customDialTLS(ctx context.Context, network, addr string) (conn net.Conn, err error) { 1417 if t.DialTLSContext != nil { 1418 conn, err = t.DialTLSContext(ctx, network, addr) 1419 } else { 1420 conn, err = t.DialTLS(network, addr) 1421 } 1422 if conn == nil && err == nil { 1423 err = errors.New("net/http: Transport.DialTLS or DialTLSContext returned (nil, nil)") 1424 } 1425 return 1426} 1427 1428// getConn dials and creates a new persistConn to the target as 1429// specified in the connectMethod. This includes doing a proxy CONNECT 1430// and/or setting up TLS. If this doesn't return an error, the persistConn 1431// is ready to write requests to. 1432func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (_ *persistConn, err error) { 1433 req := treq.Request 1434 trace := treq.trace 1435 ctx := req.Context() 1436 if trace != nil && trace.GetConn != nil { 1437 trace.GetConn(cm.addr()) 1438 } 1439 1440 // Detach from the request context's cancellation signal. 1441 // The dial should proceed even if the request is canceled, 1442 // because a future request may be able to make use of the connection. 1443 // 1444 // We retain the request context's values. 1445 dialCtx, dialCancel := context.WithCancel(context.WithoutCancel(ctx)) 1446 1447 w := &wantConn{ 1448 cm: cm, 1449 key: cm.key(), 1450 ctx: dialCtx, 1451 cancelCtx: dialCancel, 1452 result: make(chan connOrError, 1), 1453 beforeDial: testHookPrePendingDial, 1454 afterDial: testHookPostPendingDial, 1455 } 1456 defer func() { 1457 if err != nil { 1458 w.cancel(t, err) 1459 } 1460 }() 1461 1462 // Queue for idle connection. 1463 if delivered := t.queueForIdleConn(w); !delivered { 1464 t.queueForDial(w) 1465 } 1466 1467 // Wait for completion or cancellation. 1468 select { 1469 case r := <-w.result: 1470 // Trace success but only for HTTP/1. 1471 // HTTP/2 calls trace.GotConn itself. 1472 if r.pc != nil && r.pc.alt == nil && trace != nil && trace.GotConn != nil { 1473 info := httptrace.GotConnInfo{ 1474 Conn: r.pc.conn, 1475 Reused: r.pc.isReused(), 1476 } 1477 if !r.idleAt.IsZero() { 1478 info.WasIdle = true 1479 info.IdleTime = time.Since(r.idleAt) 1480 } 1481 trace.GotConn(info) 1482 } 1483 if r.err != nil { 1484 // If the request has been canceled, that's probably 1485 // what caused r.err; if so, prefer to return the 1486 // cancellation error (see golang.org/issue/16049). 1487 select { 1488 case <-treq.ctx.Done(): 1489 err := context.Cause(treq.ctx) 1490 if err == errRequestCanceled { 1491 err = errRequestCanceledConn 1492 } 1493 return nil, err 1494 default: 1495 // return below 1496 } 1497 } 1498 return r.pc, r.err 1499 case <-treq.ctx.Done(): 1500 err := context.Cause(treq.ctx) 1501 if err == errRequestCanceled { 1502 err = errRequestCanceledConn 1503 } 1504 return nil, err 1505 } 1506} 1507 1508// queueForDial queues w to wait for permission to begin dialing. 1509// Once w receives permission to dial, it will do so in a separate goroutine. 1510func (t *Transport) queueForDial(w *wantConn) { 1511 w.beforeDial() 1512 1513 t.connsPerHostMu.Lock() 1514 defer t.connsPerHostMu.Unlock() 1515 1516 if t.MaxConnsPerHost <= 0 { 1517 t.startDialConnForLocked(w) 1518 return 1519 } 1520 1521 if n := t.connsPerHost[w.key]; n < t.MaxConnsPerHost { 1522 if t.connsPerHost == nil { 1523 t.connsPerHost = make(map[connectMethodKey]int) 1524 } 1525 t.connsPerHost[w.key] = n + 1 1526 t.startDialConnForLocked(w) 1527 return 1528 } 1529 1530 if t.connsPerHostWait == nil { 1531 t.connsPerHostWait = make(map[connectMethodKey]wantConnQueue) 1532 } 1533 q := t.connsPerHostWait[w.key] 1534 q.cleanFrontNotWaiting() 1535 q.pushBack(w) 1536 t.connsPerHostWait[w.key] = q 1537} 1538 1539// startDialConnFor calls dialConn in a new goroutine. 1540// t.connsPerHostMu must be held. 1541func (t *Transport) startDialConnForLocked(w *wantConn) { 1542 t.dialsInProgress.cleanFrontCanceled() 1543 t.dialsInProgress.pushBack(w) 1544 go func() { 1545 t.dialConnFor(w) 1546 t.connsPerHostMu.Lock() 1547 defer t.connsPerHostMu.Unlock() 1548 w.cancelCtx = nil 1549 }() 1550} 1551 1552// dialConnFor dials on behalf of w and delivers the result to w. 1553// dialConnFor has received permission to dial w.cm and is counted in t.connCount[w.cm.key()]. 1554// If the dial is canceled or unsuccessful, dialConnFor decrements t.connCount[w.cm.key()]. 1555func (t *Transport) dialConnFor(w *wantConn) { 1556 defer w.afterDial() 1557 ctx := w.getCtxForDial() 1558 if ctx == nil { 1559 t.decConnsPerHost(w.key) 1560 return 1561 } 1562 1563 pc, err := t.dialConn(ctx, w.cm) 1564 delivered := w.tryDeliver(pc, err, time.Time{}) 1565 if err == nil && (!delivered || pc.alt != nil) { 1566 // pconn was not passed to w, 1567 // or it is HTTP/2 and can be shared. 1568 // Add to the idle connection pool. 1569 t.putOrCloseIdleConn(pc) 1570 } 1571 if err != nil { 1572 t.decConnsPerHost(w.key) 1573 } 1574} 1575 1576// decConnsPerHost decrements the per-host connection count for key, 1577// which may in turn give a different waiting goroutine permission to dial. 1578func (t *Transport) decConnsPerHost(key connectMethodKey) { 1579 if t.MaxConnsPerHost <= 0 { 1580 return 1581 } 1582 1583 t.connsPerHostMu.Lock() 1584 defer t.connsPerHostMu.Unlock() 1585 n := t.connsPerHost[key] 1586 if n == 0 { 1587 // Shouldn't happen, but if it does, the counting is buggy and could 1588 // easily lead to a silent deadlock, so report the problem loudly. 1589 panic("net/http: internal error: connCount underflow") 1590 } 1591 1592 // Can we hand this count to a goroutine still waiting to dial? 1593 // (Some goroutines on the wait list may have timed out or 1594 // gotten a connection another way. If they're all gone, 1595 // we don't want to kick off any spurious dial operations.) 1596 if q := t.connsPerHostWait[key]; q.len() > 0 { 1597 done := false 1598 for q.len() > 0 { 1599 w := q.popFront() 1600 if w.waiting() { 1601 t.startDialConnForLocked(w) 1602 done = true 1603 break 1604 } 1605 } 1606 if q.len() == 0 { 1607 delete(t.connsPerHostWait, key) 1608 } else { 1609 // q is a value (like a slice), so we have to store 1610 // the updated q back into the map. 1611 t.connsPerHostWait[key] = q 1612 } 1613 if done { 1614 return 1615 } 1616 } 1617 1618 // Otherwise, decrement the recorded count. 1619 if n--; n == 0 { 1620 delete(t.connsPerHost, key) 1621 } else { 1622 t.connsPerHost[key] = n 1623 } 1624} 1625 1626// Add TLS to a persistent connection, i.e. negotiate a TLS session. If pconn is already a TLS 1627// tunnel, this function establishes a nested TLS session inside the encrypted channel. 1628// The remote endpoint's name may be overridden by TLSClientConfig.ServerName. 1629func (pconn *persistConn) addTLS(ctx context.Context, name string, trace *httptrace.ClientTrace) error { 1630 // Initiate TLS and check remote host name against certificate. 1631 cfg := cloneTLSConfig(pconn.t.TLSClientConfig) 1632 if cfg.ServerName == "" { 1633 cfg.ServerName = name 1634 } 1635 if pconn.cacheKey.onlyH1 { 1636 cfg.NextProtos = nil 1637 } 1638 plainConn := pconn.conn 1639 tlsConn := tls.Client(plainConn, cfg) 1640 errc := make(chan error, 2) 1641 var timer *time.Timer // for canceling TLS handshake 1642 if d := pconn.t.TLSHandshakeTimeout; d != 0 { 1643 timer = time.AfterFunc(d, func() { 1644 errc <- tlsHandshakeTimeoutError{} 1645 }) 1646 } 1647 go func() { 1648 if trace != nil && trace.TLSHandshakeStart != nil { 1649 trace.TLSHandshakeStart() 1650 } 1651 err := tlsConn.HandshakeContext(ctx) 1652 if timer != nil { 1653 timer.Stop() 1654 } 1655 errc <- err 1656 }() 1657 if err := <-errc; err != nil { 1658 plainConn.Close() 1659 if err == (tlsHandshakeTimeoutError{}) { 1660 // Now that we have closed the connection, 1661 // wait for the call to HandshakeContext to return. 1662 <-errc 1663 } 1664 if trace != nil && trace.TLSHandshakeDone != nil { 1665 trace.TLSHandshakeDone(tls.ConnectionState{}, err) 1666 } 1667 return err 1668 } 1669 cs := tlsConn.ConnectionState() 1670 if trace != nil && trace.TLSHandshakeDone != nil { 1671 trace.TLSHandshakeDone(cs, nil) 1672 } 1673 pconn.tlsState = &cs 1674 pconn.conn = tlsConn 1675 return nil 1676} 1677 1678type erringRoundTripper interface { 1679 RoundTripErr() error 1680} 1681 1682var testHookProxyConnectTimeout = context.WithTimeout 1683 1684func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (pconn *persistConn, err error) { 1685 pconn = &persistConn{ 1686 t: t, 1687 cacheKey: cm.key(), 1688 reqch: make(chan requestAndChan, 1), 1689 writech: make(chan writeRequest, 1), 1690 closech: make(chan struct{}), 1691 writeErrCh: make(chan error, 1), 1692 writeLoopDone: make(chan struct{}), 1693 } 1694 trace := httptrace.ContextClientTrace(ctx) 1695 wrapErr := func(err error) error { 1696 if cm.proxyURL != nil { 1697 // Return a typed error, per Issue 16997 1698 return &net.OpError{Op: "proxyconnect", Net: "tcp", Err: err} 1699 } 1700 return err 1701 } 1702 if cm.scheme() == "https" && t.hasCustomTLSDialer() { 1703 var err error 1704 pconn.conn, err = t.customDialTLS(ctx, "tcp", cm.addr()) 1705 if err != nil { 1706 return nil, wrapErr(err) 1707 } 1708 if tc, ok := pconn.conn.(*tls.Conn); ok { 1709 // Handshake here, in case DialTLS didn't. TLSNextProto below 1710 // depends on it for knowing the connection state. 1711 if trace != nil && trace.TLSHandshakeStart != nil { 1712 trace.TLSHandshakeStart() 1713 } 1714 if err := tc.HandshakeContext(ctx); err != nil { 1715 go pconn.conn.Close() 1716 if trace != nil && trace.TLSHandshakeDone != nil { 1717 trace.TLSHandshakeDone(tls.ConnectionState{}, err) 1718 } 1719 return nil, err 1720 } 1721 cs := tc.ConnectionState() 1722 if trace != nil && trace.TLSHandshakeDone != nil { 1723 trace.TLSHandshakeDone(cs, nil) 1724 } 1725 pconn.tlsState = &cs 1726 } 1727 } else { 1728 conn, err := t.dial(ctx, "tcp", cm.addr()) 1729 if err != nil { 1730 return nil, wrapErr(err) 1731 } 1732 pconn.conn = conn 1733 if cm.scheme() == "https" { 1734 var firstTLSHost string 1735 if firstTLSHost, _, err = net.SplitHostPort(cm.addr()); err != nil { 1736 return nil, wrapErr(err) 1737 } 1738 if err = pconn.addTLS(ctx, firstTLSHost, trace); err != nil { 1739 return nil, wrapErr(err) 1740 } 1741 } 1742 } 1743 1744 // Proxy setup. 1745 switch { 1746 case cm.proxyURL == nil: 1747 // Do nothing. Not using a proxy. 1748 case cm.proxyURL.Scheme == "socks5" || cm.proxyURL.Scheme == "socks5h": 1749 conn := pconn.conn 1750 d := socksNewDialer("tcp", conn.RemoteAddr().String()) 1751 if u := cm.proxyURL.User; u != nil { 1752 auth := &socksUsernamePassword{ 1753 Username: u.Username(), 1754 } 1755 auth.Password, _ = u.Password() 1756 d.AuthMethods = []socksAuthMethod{ 1757 socksAuthMethodNotRequired, 1758 socksAuthMethodUsernamePassword, 1759 } 1760 d.Authenticate = auth.Authenticate 1761 } 1762 if _, err := d.DialWithConn(ctx, conn, "tcp", cm.targetAddr); err != nil { 1763 conn.Close() 1764 return nil, err 1765 } 1766 case cm.targetScheme == "http": 1767 pconn.isProxy = true 1768 if pa := cm.proxyAuth(); pa != "" { 1769 pconn.mutateHeaderFunc = func(h Header) { 1770 h.Set("Proxy-Authorization", pa) 1771 } 1772 } 1773 case cm.targetScheme == "https": 1774 conn := pconn.conn 1775 var hdr Header 1776 if t.GetProxyConnectHeader != nil { 1777 var err error 1778 hdr, err = t.GetProxyConnectHeader(ctx, cm.proxyURL, cm.targetAddr) 1779 if err != nil { 1780 conn.Close() 1781 return nil, err 1782 } 1783 } else { 1784 hdr = t.ProxyConnectHeader 1785 } 1786 if hdr == nil { 1787 hdr = make(Header) 1788 } 1789 if pa := cm.proxyAuth(); pa != "" { 1790 hdr = hdr.Clone() 1791 hdr.Set("Proxy-Authorization", pa) 1792 } 1793 connectReq := &Request{ 1794 Method: "CONNECT", 1795 URL: &url.URL{Opaque: cm.targetAddr}, 1796 Host: cm.targetAddr, 1797 Header: hdr, 1798 } 1799 1800 // Set a (long) timeout here to make sure we don't block forever 1801 // and leak a goroutine if the connection stops replying after 1802 // the TCP connect. 1803 connectCtx, cancel := testHookProxyConnectTimeout(ctx, 1*time.Minute) 1804 defer cancel() 1805 1806 didReadResponse := make(chan struct{}) // closed after CONNECT write+read is done or fails 1807 var ( 1808 resp *Response 1809 err error // write or read error 1810 ) 1811 // Write the CONNECT request & read the response. 1812 go func() { 1813 defer close(didReadResponse) 1814 err = connectReq.Write(conn) 1815 if err != nil { 1816 return 1817 } 1818 // Okay to use and discard buffered reader here, because 1819 // TLS server will not speak until spoken to. 1820 br := bufio.NewReader(conn) 1821 resp, err = ReadResponse(br, connectReq) 1822 }() 1823 select { 1824 case <-connectCtx.Done(): 1825 conn.Close() 1826 <-didReadResponse 1827 return nil, connectCtx.Err() 1828 case <-didReadResponse: 1829 // resp or err now set 1830 } 1831 if err != nil { 1832 conn.Close() 1833 return nil, err 1834 } 1835 1836 if t.OnProxyConnectResponse != nil { 1837 err = t.OnProxyConnectResponse(ctx, cm.proxyURL, connectReq, resp) 1838 if err != nil { 1839 conn.Close() 1840 return nil, err 1841 } 1842 } 1843 1844 if resp.StatusCode != 200 { 1845 _, text, ok := strings.Cut(resp.Status, " ") 1846 conn.Close() 1847 if !ok { 1848 return nil, errors.New("unknown status code") 1849 } 1850 return nil, errors.New(text) 1851 } 1852 } 1853 1854 if cm.proxyURL != nil && cm.targetScheme == "https" { 1855 if err := pconn.addTLS(ctx, cm.tlsHost(), trace); err != nil { 1856 return nil, err 1857 } 1858 } 1859 1860 if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" { 1861 if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok { 1862 alt := next(cm.targetAddr, pconn.conn.(*tls.Conn)) 1863 if e, ok := alt.(erringRoundTripper); ok { 1864 // pconn.conn was closed by next (http2configureTransports.upgradeFn). 1865 return nil, e.RoundTripErr() 1866 } 1867 return &persistConn{t: t, cacheKey: pconn.cacheKey, alt: alt}, nil 1868 } 1869 } 1870 1871 pconn.br = bufio.NewReaderSize(pconn, t.readBufferSize()) 1872 pconn.bw = bufio.NewWriterSize(persistConnWriter{pconn}, t.writeBufferSize()) 1873 1874 go pconn.readLoop() 1875 go pconn.writeLoop() 1876 return pconn, nil 1877} 1878 1879// persistConnWriter is the io.Writer written to by pc.bw. 1880// It accumulates the number of bytes written to the underlying conn, 1881// so the retry logic can determine whether any bytes made it across 1882// the wire. 1883// This is exactly 1 pointer field wide so it can go into an interface 1884// without allocation. 1885type persistConnWriter struct { 1886 pc *persistConn 1887} 1888 1889func (w persistConnWriter) Write(p []byte) (n int, err error) { 1890 n, err = w.pc.conn.Write(p) 1891 w.pc.nwrite += int64(n) 1892 return 1893} 1894 1895// ReadFrom exposes persistConnWriter's underlying Conn to io.Copy and if 1896// the Conn implements io.ReaderFrom, it can take advantage of optimizations 1897// such as sendfile. 1898func (w persistConnWriter) ReadFrom(r io.Reader) (n int64, err error) { 1899 n, err = io.Copy(w.pc.conn, r) 1900 w.pc.nwrite += n 1901 return 1902} 1903 1904var _ io.ReaderFrom = (*persistConnWriter)(nil) 1905 1906// connectMethod is the map key (in its String form) for keeping persistent 1907// TCP connections alive for subsequent HTTP requests. 1908// 1909// A connect method may be of the following types: 1910// 1911// connectMethod.key().String() Description 1912// ------------------------------ ------------------------- 1913// |http|foo.com http directly to server, no proxy 1914// |https|foo.com https directly to server, no proxy 1915// |https,h1|foo.com https directly to server w/o HTTP/2, no proxy 1916// http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com 1917// http://proxy.com|http http to proxy, http to anywhere after that 1918// socks5://proxy.com|http|foo.com socks5 to proxy, then http to foo.com 1919// socks5://proxy.com|https|foo.com socks5 to proxy, then https to foo.com 1920// https://proxy.com|https|foo.com https to proxy, then CONNECT to foo.com 1921// https://proxy.com|http https to proxy, http to anywhere after that 1922type connectMethod struct { 1923 _ incomparable 1924 proxyURL *url.URL // nil for no proxy, else full proxy URL 1925 targetScheme string // "http" or "https" 1926 // If proxyURL specifies an http or https proxy, and targetScheme is http (not https), 1927 // then targetAddr is not included in the connect method key, because the socket can 1928 // be reused for different targetAddr values. 1929 targetAddr string 1930 onlyH1 bool // whether to disable HTTP/2 and force HTTP/1 1931} 1932 1933func (cm *connectMethod) key() connectMethodKey { 1934 proxyStr := "" 1935 targetAddr := cm.targetAddr 1936 if cm.proxyURL != nil { 1937 proxyStr = cm.proxyURL.String() 1938 if (cm.proxyURL.Scheme == "http" || cm.proxyURL.Scheme == "https") && cm.targetScheme == "http" { 1939 targetAddr = "" 1940 } 1941 } 1942 return connectMethodKey{ 1943 proxy: proxyStr, 1944 scheme: cm.targetScheme, 1945 addr: targetAddr, 1946 onlyH1: cm.onlyH1, 1947 } 1948} 1949 1950// scheme returns the first hop scheme: http, https, or socks5 1951func (cm *connectMethod) scheme() string { 1952 if cm.proxyURL != nil { 1953 return cm.proxyURL.Scheme 1954 } 1955 return cm.targetScheme 1956} 1957 1958// addr returns the first hop "host:port" to which we need to TCP connect. 1959func (cm *connectMethod) addr() string { 1960 if cm.proxyURL != nil { 1961 return canonicalAddr(cm.proxyURL) 1962 } 1963 return cm.targetAddr 1964} 1965 1966// tlsHost returns the host name to match against the peer's 1967// TLS certificate. 1968func (cm *connectMethod) tlsHost() string { 1969 h := cm.targetAddr 1970 if hasPort(h) { 1971 h = h[:strings.LastIndex(h, ":")] 1972 } 1973 return h 1974} 1975 1976// connectMethodKey is the map key version of connectMethod, with a 1977// stringified proxy URL (or the empty string) instead of a pointer to 1978// a URL. 1979type connectMethodKey struct { 1980 proxy, scheme, addr string 1981 onlyH1 bool 1982} 1983 1984func (k connectMethodKey) String() string { 1985 // Only used by tests. 1986 var h1 string 1987 if k.onlyH1 { 1988 h1 = ",h1" 1989 } 1990 return fmt.Sprintf("%s|%s%s|%s", k.proxy, k.scheme, h1, k.addr) 1991} 1992 1993// persistConn wraps a connection, usually a persistent one 1994// (but may be used for non-keep-alive requests as well) 1995type persistConn struct { 1996 // alt optionally specifies the TLS NextProto RoundTripper. 1997 // This is used for HTTP/2 today and future protocols later. 1998 // If it's non-nil, the rest of the fields are unused. 1999 alt RoundTripper 2000 2001 t *Transport 2002 cacheKey connectMethodKey 2003 conn net.Conn 2004 tlsState *tls.ConnectionState 2005 br *bufio.Reader // from conn 2006 bw *bufio.Writer // to conn 2007 nwrite int64 // bytes written 2008 reqch chan requestAndChan // written by roundTrip; read by readLoop 2009 writech chan writeRequest // written by roundTrip; read by writeLoop 2010 closech chan struct{} // closed when conn closed 2011 isProxy bool 2012 sawEOF bool // whether we've seen EOF from conn; owned by readLoop 2013 readLimit int64 // bytes allowed to be read; owned by readLoop 2014 // writeErrCh passes the request write error (usually nil) 2015 // from the writeLoop goroutine to the readLoop which passes 2016 // it off to the res.Body reader, which then uses it to decide 2017 // whether or not a connection can be reused. Issue 7569. 2018 writeErrCh chan error 2019 2020 writeLoopDone chan struct{} // closed when write loop ends 2021 2022 // Both guarded by Transport.idleMu: 2023 idleAt time.Time // time it last become idle 2024 idleTimer *time.Timer // holding an AfterFunc to close it 2025 2026 mu sync.Mutex // guards following fields 2027 numExpectedResponses int 2028 closed error // set non-nil when conn is closed, before closech is closed 2029 canceledErr error // set non-nil if conn is canceled 2030 broken bool // an error has happened on this connection; marked broken so it's not reused. 2031 reused bool // whether conn has had successful request/response and is being reused. 2032 // mutateHeaderFunc is an optional func to modify extra 2033 // headers on each outbound request before it's written. (the 2034 // original Request given to RoundTrip is not modified) 2035 mutateHeaderFunc func(Header) 2036} 2037 2038func (pc *persistConn) maxHeaderResponseSize() int64 { 2039 if v := pc.t.MaxResponseHeaderBytes; v != 0 { 2040 return v 2041 } 2042 return 10 << 20 // conservative default; same as http2 2043} 2044 2045func (pc *persistConn) Read(p []byte) (n int, err error) { 2046 if pc.readLimit <= 0 { 2047 return 0, fmt.Errorf("read limit of %d bytes exhausted", pc.maxHeaderResponseSize()) 2048 } 2049 if int64(len(p)) > pc.readLimit { 2050 p = p[:pc.readLimit] 2051 } 2052 n, err = pc.conn.Read(p) 2053 if err == io.EOF { 2054 pc.sawEOF = true 2055 } 2056 pc.readLimit -= int64(n) 2057 return 2058} 2059 2060// isBroken reports whether this connection is in a known broken state. 2061func (pc *persistConn) isBroken() bool { 2062 pc.mu.Lock() 2063 b := pc.closed != nil 2064 pc.mu.Unlock() 2065 return b 2066} 2067 2068// canceled returns non-nil if the connection was closed due to 2069// CancelRequest or due to context cancellation. 2070func (pc *persistConn) canceled() error { 2071 pc.mu.Lock() 2072 defer pc.mu.Unlock() 2073 return pc.canceledErr 2074} 2075 2076// isReused reports whether this connection has been used before. 2077func (pc *persistConn) isReused() bool { 2078 pc.mu.Lock() 2079 r := pc.reused 2080 pc.mu.Unlock() 2081 return r 2082} 2083 2084func (pc *persistConn) cancelRequest(err error) { 2085 pc.mu.Lock() 2086 defer pc.mu.Unlock() 2087 pc.canceledErr = err 2088 pc.closeLocked(errRequestCanceled) 2089} 2090 2091// closeConnIfStillIdle closes the connection if it's still sitting idle. 2092// This is what's called by the persistConn's idleTimer, and is run in its 2093// own goroutine. 2094func (pc *persistConn) closeConnIfStillIdle() { 2095 t := pc.t 2096 t.idleMu.Lock() 2097 defer t.idleMu.Unlock() 2098 if _, ok := t.idleLRU.m[pc]; !ok { 2099 // Not idle. 2100 return 2101 } 2102 t.removeIdleConnLocked(pc) 2103 pc.close(errIdleConnTimeout) 2104} 2105 2106// mapRoundTripError returns the appropriate error value for 2107// persistConn.roundTrip. 2108// 2109// The provided err is the first error that (*persistConn).roundTrip 2110// happened to receive from its select statement. 2111// 2112// The startBytesWritten value should be the value of pc.nwrite before the roundTrip 2113// started writing the request. 2114func (pc *persistConn) mapRoundTripError(req *transportRequest, startBytesWritten int64, err error) error { 2115 if err == nil { 2116 return nil 2117 } 2118 2119 // Wait for the writeLoop goroutine to terminate to avoid data 2120 // races on callers who mutate the request on failure. 2121 // 2122 // When resc in pc.roundTrip and hence rc.ch receives a responseAndError 2123 // with a non-nil error it implies that the persistConn is either closed 2124 // or closing. Waiting on pc.writeLoopDone is hence safe as all callers 2125 // close closech which in turn ensures writeLoop returns. 2126 <-pc.writeLoopDone 2127 2128 // If the request was canceled, that's better than network 2129 // failures that were likely the result of tearing down the 2130 // connection. 2131 if cerr := pc.canceled(); cerr != nil { 2132 return cerr 2133 } 2134 2135 // See if an error was set explicitly. 2136 req.mu.Lock() 2137 reqErr := req.err 2138 req.mu.Unlock() 2139 if reqErr != nil { 2140 return reqErr 2141 } 2142 2143 if err == errServerClosedIdle { 2144 // Don't decorate 2145 return err 2146 } 2147 2148 if _, ok := err.(transportReadFromServerError); ok { 2149 if pc.nwrite == startBytesWritten { 2150 return nothingWrittenError{err} 2151 } 2152 // Don't decorate 2153 return err 2154 } 2155 if pc.isBroken() { 2156 if pc.nwrite == startBytesWritten { 2157 return nothingWrittenError{err} 2158 } 2159 return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %w", err) 2160 } 2161 return err 2162} 2163 2164// errCallerOwnsConn is an internal sentinel error used when we hand 2165// off a writable response.Body to the caller. We use this to prevent 2166// closing a net.Conn that is now owned by the caller. 2167var errCallerOwnsConn = errors.New("read loop ending; caller owns writable underlying conn") 2168 2169func (pc *persistConn) readLoop() { 2170 closeErr := errReadLoopExiting // default value, if not changed below 2171 defer func() { 2172 pc.close(closeErr) 2173 pc.t.removeIdleConn(pc) 2174 }() 2175 2176 tryPutIdleConn := func(treq *transportRequest) bool { 2177 trace := treq.trace 2178 if err := pc.t.tryPutIdleConn(pc); err != nil { 2179 closeErr = err 2180 if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled { 2181 trace.PutIdleConn(err) 2182 } 2183 return false 2184 } 2185 if trace != nil && trace.PutIdleConn != nil { 2186 trace.PutIdleConn(nil) 2187 } 2188 return true 2189 } 2190 2191 // eofc is used to block caller goroutines reading from Response.Body 2192 // at EOF until this goroutines has (potentially) added the connection 2193 // back to the idle pool. 2194 eofc := make(chan struct{}) 2195 defer close(eofc) // unblock reader on errors 2196 2197 // Read this once, before loop starts. (to avoid races in tests) 2198 testHookMu.Lock() 2199 testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead 2200 testHookMu.Unlock() 2201 2202 alive := true 2203 for alive { 2204 pc.readLimit = pc.maxHeaderResponseSize() 2205 _, err := pc.br.Peek(1) 2206 2207 pc.mu.Lock() 2208 if pc.numExpectedResponses == 0 { 2209 pc.readLoopPeekFailLocked(err) 2210 pc.mu.Unlock() 2211 return 2212 } 2213 pc.mu.Unlock() 2214 2215 rc := <-pc.reqch 2216 trace := rc.treq.trace 2217 2218 var resp *Response 2219 if err == nil { 2220 resp, err = pc.readResponse(rc, trace) 2221 } else { 2222 err = transportReadFromServerError{err} 2223 closeErr = err 2224 } 2225 2226 if err != nil { 2227 if pc.readLimit <= 0 { 2228 err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize()) 2229 } 2230 2231 select { 2232 case rc.ch <- responseAndError{err: err}: 2233 case <-rc.callerGone: 2234 return 2235 } 2236 return 2237 } 2238 pc.readLimit = maxInt64 // effectively no limit for response bodies 2239 2240 pc.mu.Lock() 2241 pc.numExpectedResponses-- 2242 pc.mu.Unlock() 2243 2244 bodyWritable := resp.bodyIsWritable() 2245 hasBody := rc.treq.Request.Method != "HEAD" && resp.ContentLength != 0 2246 2247 if resp.Close || rc.treq.Request.Close || resp.StatusCode <= 199 || bodyWritable { 2248 // Don't do keep-alive on error if either party requested a close 2249 // or we get an unexpected informational (1xx) response. 2250 // StatusCode 100 is already handled above. 2251 alive = false 2252 } 2253 2254 if !hasBody || bodyWritable { 2255 // Put the idle conn back into the pool before we send the response 2256 // so if they process it quickly and make another request, they'll 2257 // get this same conn. But we use the unbuffered channel 'rc' 2258 // to guarantee that persistConn.roundTrip got out of its select 2259 // potentially waiting for this persistConn to close. 2260 alive = alive && 2261 !pc.sawEOF && 2262 pc.wroteRequest() && 2263 tryPutIdleConn(rc.treq) 2264 2265 if bodyWritable { 2266 closeErr = errCallerOwnsConn 2267 } 2268 2269 select { 2270 case rc.ch <- responseAndError{res: resp}: 2271 case <-rc.callerGone: 2272 return 2273 } 2274 2275 rc.treq.cancel(errRequestDone) 2276 2277 // Now that they've read from the unbuffered channel, they're safely 2278 // out of the select that also waits on this goroutine to die, so 2279 // we're allowed to exit now if needed (if alive is false) 2280 testHookReadLoopBeforeNextRead() 2281 continue 2282 } 2283 2284 waitForBodyRead := make(chan bool, 2) 2285 body := &bodyEOFSignal{ 2286 body: resp.Body, 2287 earlyCloseFn: func() error { 2288 waitForBodyRead <- false 2289 <-eofc // will be closed by deferred call at the end of the function 2290 return nil 2291 2292 }, 2293 fn: func(err error) error { 2294 isEOF := err == io.EOF 2295 waitForBodyRead <- isEOF 2296 if isEOF { 2297 <-eofc // see comment above eofc declaration 2298 } else if err != nil { 2299 if cerr := pc.canceled(); cerr != nil { 2300 return cerr 2301 } 2302 } 2303 return err 2304 }, 2305 } 2306 2307 resp.Body = body 2308 if rc.addedGzip && ascii.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") { 2309 resp.Body = &gzipReader{body: body} 2310 resp.Header.Del("Content-Encoding") 2311 resp.Header.Del("Content-Length") 2312 resp.ContentLength = -1 2313 resp.Uncompressed = true 2314 } 2315 2316 select { 2317 case rc.ch <- responseAndError{res: resp}: 2318 case <-rc.callerGone: 2319 return 2320 } 2321 2322 // Before looping back to the top of this function and peeking on 2323 // the bufio.Reader, wait for the caller goroutine to finish 2324 // reading the response body. (or for cancellation or death) 2325 select { 2326 case bodyEOF := <-waitForBodyRead: 2327 alive = alive && 2328 bodyEOF && 2329 !pc.sawEOF && 2330 pc.wroteRequest() && 2331 tryPutIdleConn(rc.treq) 2332 if bodyEOF { 2333 eofc <- struct{}{} 2334 } 2335 case <-rc.treq.ctx.Done(): 2336 alive = false 2337 pc.cancelRequest(context.Cause(rc.treq.ctx)) 2338 case <-pc.closech: 2339 alive = false 2340 } 2341 2342 rc.treq.cancel(errRequestDone) 2343 testHookReadLoopBeforeNextRead() 2344 } 2345} 2346 2347func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { 2348 if pc.closed != nil { 2349 return 2350 } 2351 if n := pc.br.Buffered(); n > 0 { 2352 buf, _ := pc.br.Peek(n) 2353 if is408Message(buf) { 2354 pc.closeLocked(errServerClosedIdle) 2355 return 2356 } else { 2357 log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr) 2358 } 2359 } 2360 if peekErr == io.EOF { 2361 // common case. 2362 pc.closeLocked(errServerClosedIdle) 2363 } else { 2364 pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %w", peekErr)) 2365 } 2366} 2367 2368// is408Message reports whether buf has the prefix of an 2369// HTTP 408 Request Timeout response. 2370// See golang.org/issue/32310. 2371func is408Message(buf []byte) bool { 2372 if len(buf) < len("HTTP/1.x 408") { 2373 return false 2374 } 2375 if string(buf[:7]) != "HTTP/1." { 2376 return false 2377 } 2378 return string(buf[8:12]) == " 408" 2379} 2380 2381// readResponse reads an HTTP response (or two, in the case of "Expect: 2382// 100-continue") from the server. It returns the final non-100 one. 2383// trace is optional. 2384func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTrace) (resp *Response, err error) { 2385 if trace != nil && trace.GotFirstResponseByte != nil { 2386 if peek, err := pc.br.Peek(1); err == nil && len(peek) == 1 { 2387 trace.GotFirstResponseByte() 2388 } 2389 } 2390 num1xx := 0 // number of informational 1xx headers received 2391 const max1xxResponses = 5 // arbitrary bound on number of informational responses 2392 2393 continueCh := rc.continueCh 2394 for { 2395 resp, err = ReadResponse(pc.br, rc.treq.Request) 2396 if err != nil { 2397 return 2398 } 2399 resCode := resp.StatusCode 2400 if continueCh != nil && resCode == StatusContinue { 2401 if trace != nil && trace.Got100Continue != nil { 2402 trace.Got100Continue() 2403 } 2404 continueCh <- struct{}{} 2405 continueCh = nil 2406 } 2407 is1xx := 100 <= resCode && resCode <= 199 2408 // treat 101 as a terminal status, see issue 26161 2409 is1xxNonTerminal := is1xx && resCode != StatusSwitchingProtocols 2410 if is1xxNonTerminal { 2411 num1xx++ 2412 if num1xx > max1xxResponses { 2413 return nil, errors.New("net/http: too many 1xx informational responses") 2414 } 2415 pc.readLimit = pc.maxHeaderResponseSize() // reset the limit 2416 if trace != nil && trace.Got1xxResponse != nil { 2417 if err := trace.Got1xxResponse(resCode, textproto.MIMEHeader(resp.Header)); err != nil { 2418 return nil, err 2419 } 2420 } 2421 continue 2422 } 2423 break 2424 } 2425 if resp.isProtocolSwitch() { 2426 resp.Body = newReadWriteCloserBody(pc.br, pc.conn) 2427 } 2428 if continueCh != nil { 2429 // We send an "Expect: 100-continue" header, but the server 2430 // responded with a terminal status and no 100 Continue. 2431 // 2432 // If we're going to keep using the connection, we need to send the request body. 2433 // Tell writeLoop to skip sending the body if we're going to close the connection, 2434 // or to send it otherwise. 2435 // 2436 // The case where we receive a 101 Switching Protocols response is a bit 2437 // ambiguous, since we don't know what protocol we're switching to. 2438 // Conceivably, it's one that doesn't need us to send the body. 2439 // Given that we'll send the body if ExpectContinueTimeout expires, 2440 // be consistent and always send it if we aren't closing the connection. 2441 if resp.Close || rc.treq.Request.Close { 2442 close(continueCh) // don't send the body; the connection will close 2443 } else { 2444 continueCh <- struct{}{} // send the body 2445 } 2446 } 2447 2448 resp.TLS = pc.tlsState 2449 return 2450} 2451 2452// waitForContinue returns the function to block until 2453// any response, timeout or connection close. After any of them, 2454// the function returns a bool which indicates if the body should be sent. 2455func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool { 2456 if continueCh == nil { 2457 return nil 2458 } 2459 return func() bool { 2460 timer := time.NewTimer(pc.t.ExpectContinueTimeout) 2461 defer timer.Stop() 2462 2463 select { 2464 case _, ok := <-continueCh: 2465 return ok 2466 case <-timer.C: 2467 return true 2468 case <-pc.closech: 2469 return false 2470 } 2471 } 2472} 2473 2474func newReadWriteCloserBody(br *bufio.Reader, rwc io.ReadWriteCloser) io.ReadWriteCloser { 2475 body := &readWriteCloserBody{ReadWriteCloser: rwc} 2476 if br.Buffered() != 0 { 2477 body.br = br 2478 } 2479 return body 2480} 2481 2482// readWriteCloserBody is the Response.Body type used when we want to 2483// give users write access to the Body through the underlying 2484// connection (TCP, unless using custom dialers). This is then 2485// the concrete type for a Response.Body on the 101 Switching 2486// Protocols response, as used by WebSockets, h2c, etc. 2487type readWriteCloserBody struct { 2488 _ incomparable 2489 br *bufio.Reader // used until empty 2490 io.ReadWriteCloser 2491} 2492 2493func (b *readWriteCloserBody) Read(p []byte) (n int, err error) { 2494 if b.br != nil { 2495 if n := b.br.Buffered(); len(p) > n { 2496 p = p[:n] 2497 } 2498 n, err = b.br.Read(p) 2499 if b.br.Buffered() == 0 { 2500 b.br = nil 2501 } 2502 return n, err 2503 } 2504 return b.ReadWriteCloser.Read(p) 2505} 2506 2507// nothingWrittenError wraps a write errors which ended up writing zero bytes. 2508type nothingWrittenError struct { 2509 error 2510} 2511 2512func (nwe nothingWrittenError) Unwrap() error { 2513 return nwe.error 2514} 2515 2516func (pc *persistConn) writeLoop() { 2517 defer close(pc.writeLoopDone) 2518 for { 2519 select { 2520 case wr := <-pc.writech: 2521 startBytesWritten := pc.nwrite 2522 err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh)) 2523 if bre, ok := err.(requestBodyReadError); ok { 2524 err = bre.error 2525 // Errors reading from the user's 2526 // Request.Body are high priority. 2527 // Set it here before sending on the 2528 // channels below or calling 2529 // pc.close() which tears down 2530 // connections and causes other 2531 // errors. 2532 wr.req.setError(err) 2533 } 2534 if err == nil { 2535 err = pc.bw.Flush() 2536 } 2537 if err != nil { 2538 if pc.nwrite == startBytesWritten { 2539 err = nothingWrittenError{err} 2540 } 2541 } 2542 pc.writeErrCh <- err // to the body reader, which might recycle us 2543 wr.ch <- err // to the roundTrip function 2544 if err != nil { 2545 pc.close(err) 2546 return 2547 } 2548 case <-pc.closech: 2549 return 2550 } 2551 } 2552} 2553 2554// maxWriteWaitBeforeConnReuse is how long the a Transport RoundTrip 2555// will wait to see the Request's Body.Write result after getting a 2556// response from the server. See comments in (*persistConn).wroteRequest. 2557// 2558// In tests, we set this to a large value to avoid flakiness from inconsistent 2559// recycling of connections. 2560var maxWriteWaitBeforeConnReuse = 50 * time.Millisecond 2561 2562// wroteRequest is a check before recycling a connection that the previous write 2563// (from writeLoop above) happened and was successful. 2564func (pc *persistConn) wroteRequest() bool { 2565 select { 2566 case err := <-pc.writeErrCh: 2567 // Common case: the write happened well before the response, so 2568 // avoid creating a timer. 2569 return err == nil 2570 default: 2571 // Rare case: the request was written in writeLoop above but 2572 // before it could send to pc.writeErrCh, the reader read it 2573 // all, processed it, and called us here. In this case, give the 2574 // write goroutine a bit of time to finish its send. 2575 // 2576 // Less rare case: We also get here in the legitimate case of 2577 // Issue 7569, where the writer is still writing (or stalled), 2578 // but the server has already replied. In this case, we don't 2579 // want to wait too long, and we want to return false so this 2580 // connection isn't re-used. 2581 t := time.NewTimer(maxWriteWaitBeforeConnReuse) 2582 defer t.Stop() 2583 select { 2584 case err := <-pc.writeErrCh: 2585 return err == nil 2586 case <-t.C: 2587 return false 2588 } 2589 } 2590} 2591 2592// responseAndError is how the goroutine reading from an HTTP/1 server 2593// communicates with the goroutine doing the RoundTrip. 2594type responseAndError struct { 2595 _ incomparable 2596 res *Response // else use this response (see res method) 2597 err error 2598} 2599 2600type requestAndChan struct { 2601 _ incomparable 2602 treq *transportRequest 2603 ch chan responseAndError // unbuffered; always send in select on callerGone 2604 2605 // whether the Transport (as opposed to the user client code) 2606 // added the Accept-Encoding gzip header. If the Transport 2607 // set it, only then do we transparently decode the gzip. 2608 addedGzip bool 2609 2610 // Optional blocking chan for Expect: 100-continue (for send). 2611 // If the request has an "Expect: 100-continue" header and 2612 // the server responds 100 Continue, readLoop send a value 2613 // to writeLoop via this chan. 2614 continueCh chan<- struct{} 2615 2616 callerGone <-chan struct{} // closed when roundTrip caller has returned 2617} 2618 2619// A writeRequest is sent by the caller's goroutine to the 2620// writeLoop's goroutine to write a request while the read loop 2621// concurrently waits on both the write response and the server's 2622// reply. 2623type writeRequest struct { 2624 req *transportRequest 2625 ch chan<- error 2626 2627 // Optional blocking chan for Expect: 100-continue (for receive). 2628 // If not nil, writeLoop blocks sending request body until 2629 // it receives from this chan. 2630 continueCh <-chan struct{} 2631} 2632 2633// httpTimeoutError represents a timeout. 2634// It implements net.Error and wraps context.DeadlineExceeded. 2635type timeoutError struct { 2636 err string 2637} 2638 2639func (e *timeoutError) Error() string { return e.err } 2640func (e *timeoutError) Timeout() bool { return true } 2641func (e *timeoutError) Temporary() bool { return true } 2642func (e *timeoutError) Is(err error) bool { return err == context.DeadlineExceeded } 2643 2644var errTimeout error = &timeoutError{"net/http: timeout awaiting response headers"} 2645 2646// errRequestCanceled is set to be identical to the one from h2 to facilitate 2647// testing. 2648var errRequestCanceled = http2errRequestCanceled 2649var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify? 2650 2651// errRequestDone is used to cancel the round trip Context after a request is successfully done. 2652// It should not be seen by the user. 2653var errRequestDone = errors.New("net/http: request completed") 2654 2655func nop() {} 2656 2657// testHooks. Always non-nil. 2658var ( 2659 testHookEnterRoundTrip = nop 2660 testHookWaitResLoop = nop 2661 testHookRoundTripRetried = nop 2662 testHookPrePendingDial = nop 2663 testHookPostPendingDial = nop 2664 2665 testHookMu sync.Locker = fakeLocker{} // guards following 2666 testHookReadLoopBeforeNextRead = nop 2667) 2668 2669func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) { 2670 testHookEnterRoundTrip() 2671 pc.mu.Lock() 2672 pc.numExpectedResponses++ 2673 headerFn := pc.mutateHeaderFunc 2674 pc.mu.Unlock() 2675 2676 if headerFn != nil { 2677 headerFn(req.extraHeaders()) 2678 } 2679 2680 // Ask for a compressed version if the caller didn't set their 2681 // own value for Accept-Encoding. We only attempt to 2682 // uncompress the gzip stream if we were the layer that 2683 // requested it. 2684 requestedGzip := false 2685 if !pc.t.DisableCompression && 2686 req.Header.Get("Accept-Encoding") == "" && 2687 req.Header.Get("Range") == "" && 2688 req.Method != "HEAD" { 2689 // Request gzip only, not deflate. Deflate is ambiguous and 2690 // not as universally supported anyway. 2691 // See: https://zlib.net/zlib_faq.html#faq39 2692 // 2693 // Note that we don't request this for HEAD requests, 2694 // due to a bug in nginx: 2695 // https://trac.nginx.org/nginx/ticket/358 2696 // https://golang.org/issue/5522 2697 // 2698 // We don't request gzip if the request is for a range, since 2699 // auto-decoding a portion of a gzipped document will just fail 2700 // anyway. See https://golang.org/issue/8923 2701 requestedGzip = true 2702 req.extraHeaders().Set("Accept-Encoding", "gzip") 2703 } 2704 2705 var continueCh chan struct{} 2706 if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() { 2707 continueCh = make(chan struct{}, 1) 2708 } 2709 2710 if pc.t.DisableKeepAlives && 2711 !req.wantsClose() && 2712 !isProtocolSwitchHeader(req.Header) { 2713 req.extraHeaders().Set("Connection", "close") 2714 } 2715 2716 gone := make(chan struct{}) 2717 defer close(gone) 2718 2719 const debugRoundTrip = false 2720 2721 // Write the request concurrently with waiting for a response, 2722 // in case the server decides to reply before reading our full 2723 // request body. 2724 startBytesWritten := pc.nwrite 2725 writeErrCh := make(chan error, 1) 2726 pc.writech <- writeRequest{req, writeErrCh, continueCh} 2727 2728 resc := make(chan responseAndError) 2729 pc.reqch <- requestAndChan{ 2730 treq: req, 2731 ch: resc, 2732 addedGzip: requestedGzip, 2733 continueCh: continueCh, 2734 callerGone: gone, 2735 } 2736 2737 handleResponse := func(re responseAndError) (*Response, error) { 2738 if (re.res == nil) == (re.err == nil) { 2739 panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil)) 2740 } 2741 if debugRoundTrip { 2742 req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err) 2743 } 2744 if re.err != nil { 2745 return nil, pc.mapRoundTripError(req, startBytesWritten, re.err) 2746 } 2747 return re.res, nil 2748 } 2749 2750 var respHeaderTimer <-chan time.Time 2751 ctxDoneChan := req.ctx.Done() 2752 pcClosed := pc.closech 2753 for { 2754 testHookWaitResLoop() 2755 select { 2756 case err := <-writeErrCh: 2757 if debugRoundTrip { 2758 req.logf("writeErrCh recv: %T/%#v", err, err) 2759 } 2760 if err != nil { 2761 pc.close(fmt.Errorf("write error: %w", err)) 2762 return nil, pc.mapRoundTripError(req, startBytesWritten, err) 2763 } 2764 if d := pc.t.ResponseHeaderTimeout; d > 0 { 2765 if debugRoundTrip { 2766 req.logf("starting timer for %v", d) 2767 } 2768 timer := time.NewTimer(d) 2769 defer timer.Stop() // prevent leaks 2770 respHeaderTimer = timer.C 2771 } 2772 case <-pcClosed: 2773 select { 2774 case re := <-resc: 2775 // The pconn closing raced with the response to the request, 2776 // probably after the server wrote a response and immediately 2777 // closed the connection. Use the response. 2778 return handleResponse(re) 2779 default: 2780 } 2781 if debugRoundTrip { 2782 req.logf("closech recv: %T %#v", pc.closed, pc.closed) 2783 } 2784 return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed) 2785 case <-respHeaderTimer: 2786 if debugRoundTrip { 2787 req.logf("timeout waiting for response headers.") 2788 } 2789 pc.close(errTimeout) 2790 return nil, errTimeout 2791 case re := <-resc: 2792 return handleResponse(re) 2793 case <-ctxDoneChan: 2794 select { 2795 case re := <-resc: 2796 // readLoop is responsible for canceling req.ctx after 2797 // it reads the response body. Check for a response racing 2798 // the context close, and use the response if available. 2799 return handleResponse(re) 2800 default: 2801 } 2802 pc.cancelRequest(context.Cause(req.ctx)) 2803 } 2804 } 2805} 2806 2807// tLogKey is a context WithValue key for test debugging contexts containing 2808// a t.Logf func. See export_test.go's Request.WithT method. 2809type tLogKey struct{} 2810 2811func (tr *transportRequest) logf(format string, args ...any) { 2812 if logf, ok := tr.Request.Context().Value(tLogKey{}).(func(string, ...any)); ok { 2813 logf(time.Now().Format(time.RFC3339Nano)+": "+format, args...) 2814 } 2815} 2816 2817// markReused marks this connection as having been successfully used for a 2818// request and response. 2819func (pc *persistConn) markReused() { 2820 pc.mu.Lock() 2821 pc.reused = true 2822 pc.mu.Unlock() 2823} 2824 2825// close closes the underlying TCP connection and closes 2826// the pc.closech channel. 2827// 2828// The provided err is only for testing and debugging; in normal 2829// circumstances it should never be seen by users. 2830func (pc *persistConn) close(err error) { 2831 pc.mu.Lock() 2832 defer pc.mu.Unlock() 2833 pc.closeLocked(err) 2834} 2835 2836func (pc *persistConn) closeLocked(err error) { 2837 if err == nil { 2838 panic("nil error") 2839 } 2840 pc.broken = true 2841 if pc.closed == nil { 2842 pc.closed = err 2843 pc.t.decConnsPerHost(pc.cacheKey) 2844 // Close HTTP/1 (pc.alt == nil) connection. 2845 // HTTP/2 closes its connection itself. 2846 if pc.alt == nil { 2847 if err != errCallerOwnsConn { 2848 pc.conn.Close() 2849 } 2850 close(pc.closech) 2851 } 2852 } 2853 pc.mutateHeaderFunc = nil 2854} 2855 2856var portMap = map[string]string{ 2857 "http": "80", 2858 "https": "443", 2859 "socks5": "1080", 2860 "socks5h": "1080", 2861} 2862 2863func idnaASCIIFromURL(url *url.URL) string { 2864 addr := url.Hostname() 2865 if v, err := idnaASCII(addr); err == nil { 2866 addr = v 2867 } 2868 return addr 2869} 2870 2871// canonicalAddr returns url.Host but always with a ":port" suffix. 2872func canonicalAddr(url *url.URL) string { 2873 port := url.Port() 2874 if port == "" { 2875 port = portMap[url.Scheme] 2876 } 2877 return net.JoinHostPort(idnaASCIIFromURL(url), port) 2878} 2879 2880// bodyEOFSignal is used by the HTTP/1 transport when reading response 2881// bodies to make sure we see the end of a response body before 2882// proceeding and reading on the connection again. 2883// 2884// It wraps a ReadCloser but runs fn (if non-nil) at most 2885// once, right before its final (error-producing) Read or Close call 2886// returns. fn should return the new error to return from Read or Close. 2887// 2888// If earlyCloseFn is non-nil and Close is called before io.EOF is 2889// seen, earlyCloseFn is called instead of fn, and its return value is 2890// the return value from Close. 2891type bodyEOFSignal struct { 2892 body io.ReadCloser 2893 mu sync.Mutex // guards following 4 fields 2894 closed bool // whether Close has been called 2895 rerr error // sticky Read error 2896 fn func(error) error // err will be nil on Read io.EOF 2897 earlyCloseFn func() error // optional alt Close func used if io.EOF not seen 2898} 2899 2900var errReadOnClosedResBody = errors.New("http: read on closed response body") 2901 2902func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { 2903 es.mu.Lock() 2904 closed, rerr := es.closed, es.rerr 2905 es.mu.Unlock() 2906 if closed { 2907 return 0, errReadOnClosedResBody 2908 } 2909 if rerr != nil { 2910 return 0, rerr 2911 } 2912 2913 n, err = es.body.Read(p) 2914 if err != nil { 2915 es.mu.Lock() 2916 defer es.mu.Unlock() 2917 if es.rerr == nil { 2918 es.rerr = err 2919 } 2920 err = es.condfn(err) 2921 } 2922 return 2923} 2924 2925func (es *bodyEOFSignal) Close() error { 2926 es.mu.Lock() 2927 defer es.mu.Unlock() 2928 if es.closed { 2929 return nil 2930 } 2931 es.closed = true 2932 if es.earlyCloseFn != nil && es.rerr != io.EOF { 2933 return es.earlyCloseFn() 2934 } 2935 err := es.body.Close() 2936 return es.condfn(err) 2937} 2938 2939// caller must hold es.mu. 2940func (es *bodyEOFSignal) condfn(err error) error { 2941 if es.fn == nil { 2942 return err 2943 } 2944 err = es.fn(err) 2945 es.fn = nil 2946 return err 2947} 2948 2949// gzipReader wraps a response body so it can lazily 2950// call gzip.NewReader on the first call to Read 2951type gzipReader struct { 2952 _ incomparable 2953 body *bodyEOFSignal // underlying HTTP/1 response body framing 2954 zr *gzip.Reader // lazily-initialized gzip reader 2955 zerr error // any error from gzip.NewReader; sticky 2956} 2957 2958func (gz *gzipReader) Read(p []byte) (n int, err error) { 2959 if gz.zr == nil { 2960 if gz.zerr == nil { 2961 gz.zr, gz.zerr = gzip.NewReader(gz.body) 2962 } 2963 if gz.zerr != nil { 2964 return 0, gz.zerr 2965 } 2966 } 2967 2968 gz.body.mu.Lock() 2969 if gz.body.closed { 2970 err = errReadOnClosedResBody 2971 } 2972 gz.body.mu.Unlock() 2973 2974 if err != nil { 2975 return 0, err 2976 } 2977 return gz.zr.Read(p) 2978} 2979 2980func (gz *gzipReader) Close() error { 2981 return gz.body.Close() 2982} 2983 2984type tlsHandshakeTimeoutError struct{} 2985 2986func (tlsHandshakeTimeoutError) Timeout() bool { return true } 2987func (tlsHandshakeTimeoutError) Temporary() bool { return true } 2988func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } 2989 2990// fakeLocker is a sync.Locker which does nothing. It's used to guard 2991// test-only fields when not under test, to avoid runtime atomic 2992// overhead. 2993type fakeLocker struct{} 2994 2995func (fakeLocker) Lock() {} 2996func (fakeLocker) Unlock() {} 2997 2998// cloneTLSConfig returns a shallow clone of cfg, or a new zero tls.Config if 2999// cfg is nil. This is safe to call even if cfg is in active use by a TLS 3000// client or server. 3001// 3002// cloneTLSConfig should be an internal detail, 3003// but widely used packages access it using linkname. 3004// Notable members of the hall of shame include: 3005// - github.com/searKing/golang 3006// 3007// Do not remove or change the type signature. 3008// See go.dev/issue/67401. 3009// 3010//go:linkname cloneTLSConfig 3011func cloneTLSConfig(cfg *tls.Config) *tls.Config { 3012 if cfg == nil { 3013 return &tls.Config{} 3014 } 3015 return cfg.Clone() 3016} 3017 3018type connLRU struct { 3019 ll *list.List // list.Element.Value type is of *persistConn 3020 m map[*persistConn]*list.Element 3021} 3022 3023// add adds pc to the head of the linked list. 3024func (cl *connLRU) add(pc *persistConn) { 3025 if cl.ll == nil { 3026 cl.ll = list.New() 3027 cl.m = make(map[*persistConn]*list.Element) 3028 } 3029 ele := cl.ll.PushFront(pc) 3030 if _, ok := cl.m[pc]; ok { 3031 panic("persistConn was already in LRU") 3032 } 3033 cl.m[pc] = ele 3034} 3035 3036func (cl *connLRU) removeOldest() *persistConn { 3037 ele := cl.ll.Back() 3038 pc := ele.Value.(*persistConn) 3039 cl.ll.Remove(ele) 3040 delete(cl.m, pc) 3041 return pc 3042} 3043 3044// remove removes pc from cl. 3045func (cl *connLRU) remove(pc *persistConn) { 3046 if ele, ok := cl.m[pc]; ok { 3047 cl.ll.Remove(ele) 3048 delete(cl.m, pc) 3049 } 3050} 3051 3052// len returns the number of items in the cache. 3053func (cl *connLRU) len() int { 3054 return len(cl.m) 3055} 3056