1
// Copyright 2011 The Go Authors. All rights reserved.
2
// Use of this source code is governed by a BSD-style
3
// license that can be found in the LICENSE file.
5
// HTTP client implementation. See RFC 2616.
7
// This is the low-level Transport implementation of RoundTripper.
8
// The high-level interface is in client.go.
29
// DefaultTransport is the default implementation of Transport and is
30
// used by DefaultClient. It establishes a new network connection for
31
// each call to Do and uses HTTP proxies as directed by the
32
// $HTTP_PROXY and $NO_PROXY (or $http_proxy and $no_proxy)
33
// environment variables.
34
var DefaultTransport RoundTripper = &Transport{Proxy: ProxyFromEnvironment}
36
// DefaultMaxIdleConnsPerHost is the default value of Transport's
37
// MaxIdleConnsPerHost.
38
const DefaultMaxIdleConnsPerHost = 2
40
// Transport is an implementation of RoundTripper that supports http,
41
// https, and http proxies (for either http or https with CONNECT).
42
// Transport can also cache connections for future re-use.
43
type Transport struct {
45
idleConn map[string][]*persistConn
46
altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper
48
// TODO: tunable on global max cached connections
49
// TODO: tunable on timeout on cached connections
50
// TODO: optional pipelining
52
// Proxy specifies a function to return a proxy for a given
53
// Request. If the function returns a non-nil error, the
54
// request is aborted with the provided error.
55
// If Proxy is nil or returns a nil *URL, no proxy is used.
56
Proxy func(*Request) (*url.URL, error)
58
// Dial specifies the dial function for creating TCP
60
// If Dial is nil, net.Dial is used.
61
Dial func(net, addr string) (c net.Conn, err error)
63
// TLSClientConfig specifies the TLS configuration to use with
64
// tls.Client. If nil, the default configuration is used.
65
TLSClientConfig *tls.Config
67
DisableKeepAlives bool
68
DisableCompression bool
70
// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
71
// (keep-alive) to keep to keep per-host. If zero,
72
// DefaultMaxIdleConnsPerHost is used.
73
MaxIdleConnsPerHost int
76
// ProxyFromEnvironment returns the URL of the proxy to use for a
77
// given request, as indicated by the environment variables
78
// $HTTP_PROXY and $NO_PROXY (or $http_proxy and $no_proxy).
79
// An error is returned if the proxy environment is invalid.
80
// A nil URL and nil error are returned if no proxy is defined in the
81
// environment, or a proxy should not be used for the given request.
82
func ProxyFromEnvironment(req *Request) (*url.URL, error) {
83
proxy := getenvEitherCase("HTTP_PROXY")
87
if !useProxy(canonicalAddr(req.URL)) {
90
proxyURL, err := url.Parse(proxy)
91
if err != nil || proxyURL.Scheme == "" {
92
if u, err := url.Parse("http://" + proxy); err == nil {
98
return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
103
// ProxyURL returns a proxy function (for use in a Transport)
104
// that always returns the same URL.
105
func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) {
106
return func(*Request) (*url.URL, error) {
111
// transportRequest is a wrapper around a *Request that adds
112
// optional extra headers to write.
113
type transportRequest struct {
114
*Request // original request, not to be mutated
115
extra Header // extra headers to write, or nil
118
func (tr *transportRequest) extraHeaders() Header {
120
tr.extra = make(Header)
125
// RoundTrip implements the RoundTripper interface.
126
func (t *Transport) RoundTrip(req *Request) (resp *Response, err error) {
128
return nil, errors.New("http: nil Request.URL")
130
if req.Header == nil {
131
return nil, errors.New("http: nil Request.Header")
133
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
136
if t.altProto != nil {
137
rt = t.altProto[req.URL.Scheme]
141
return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme}
143
return rt.RoundTrip(req)
145
treq := &transportRequest{Request: req}
146
cm, err := t.connectMethodForRequest(treq)
151
// Get the cached or newly-created connection to either the
152
// host (for http or https), the http proxy, or the http proxy
153
// pre-CONNECTed to https server. In any case, we'll be ready
154
// to send it requests.
155
pconn, err := t.getConn(cm)
160
return pconn.roundTrip(treq)
163
// RegisterProtocol registers a new protocol with scheme.
164
// The Transport will pass requests using the given scheme to rt.
165
// It is rt's responsibility to simulate HTTP request semantics.
167
// RegisterProtocol can be used by other packages to provide
168
// implementations of protocol schemes like "ftp" or "file".
169
func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) {
170
if scheme == "http" || scheme == "https" {
171
panic("protocol " + scheme + " already registered")
175
if t.altProto == nil {
176
t.altProto = make(map[string]RoundTripper)
178
if _, exists := t.altProto[scheme]; exists {
179
panic("protocol " + scheme + " already registered")
181
t.altProto[scheme] = rt
184
// CloseIdleConnections closes any connections which were previously
185
// connected from previous requests but are now sitting idle in
186
// a "keep-alive" state. It does not interrupt any connections currently
188
func (t *Transport) CloseIdleConnections() {
191
if t.idleConn == nil {
194
for _, conns := range t.idleConn {
195
for _, pconn := range conns {
199
t.idleConn = make(map[string][]*persistConn)
203
// Private implementation past this point.
206
func getenvEitherCase(k string) string {
207
if v := os.Getenv(strings.ToUpper(k)); v != "" {
210
return os.Getenv(strings.ToLower(k))
213
func (t *Transport) connectMethodForRequest(treq *transportRequest) (*connectMethod, error) {
214
cm := &connectMethod{
215
targetScheme: treq.URL.Scheme,
216
targetAddr: canonicalAddr(treq.URL),
220
cm.proxyURL, err = t.Proxy(treq.Request)
228
// proxyAuth returns the Proxy-Authorization header to set
229
// on requests, if applicable.
230
func (cm *connectMethod) proxyAuth() string {
231
if cm.proxyURL == nil {
234
if u := cm.proxyURL.User; u != nil {
235
return "Basic " + base64.URLEncoding.EncodeToString([]byte(u.String()))
240
// putIdleConn adds pconn to the list of idle persistent connections awaiting
242
// If pconn is no longer needed or not in a good state, putIdleConn
244
func (t *Transport) putIdleConn(pconn *persistConn) bool {
247
if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 {
251
if pconn.isBroken() {
254
key := pconn.cacheKey
255
max := t.MaxIdleConnsPerHost
257
max = DefaultMaxIdleConnsPerHost
259
if len(t.idleConn[key]) >= max {
263
t.idleConn[key] = append(t.idleConn[key], pconn)
267
func (t *Transport) getIdleConn(cm *connectMethod) (pconn *persistConn) {
270
if t.idleConn == nil {
271
t.idleConn = make(map[string][]*persistConn)
275
pconns, ok := t.idleConn[key]
279
if len(pconns) == 1 {
281
delete(t.idleConn, key)
283
// 2 or more cached connections; pop last
285
pconn = pconns[len(pconns)-1]
286
t.idleConn[key] = pconns[0 : len(pconns)-1]
288
if !pconn.isBroken() {
295
func (t *Transport) dial(network, addr string) (c net.Conn, err error) {
297
return t.Dial(network, addr)
299
return net.Dial(network, addr)
302
// getConn dials and creates a new persistConn to the target as
303
// specified in the connectMethod. This includes doing a proxy CONNECT
304
// and/or setting up TLS. If this doesn't return an error, the persistConn
305
// is ready to write requests to.
306
func (t *Transport) getConn(cm *connectMethod) (*persistConn, error) {
307
if pc := t.getIdleConn(cm); pc != nil {
311
conn, err := t.dial("tcp", cm.addr())
313
if cm.proxyURL != nil {
314
err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err)
321
pconn := &persistConn{
323
cacheKey: cm.String(),
325
reqch: make(chan requestAndChan, 50),
329
case cm.proxyURL == nil:
331
case cm.targetScheme == "http":
334
pconn.mutateHeaderFunc = func(h Header) {
335
h.Set("Proxy-Authorization", pa)
338
case cm.targetScheme == "https":
339
connectReq := &Request{
341
URL: &url.URL{Opaque: cm.targetAddr},
343
Header: make(Header),
346
connectReq.Header.Set("Proxy-Authorization", pa)
348
connectReq.Write(conn)
351
// Okay to use and discard buffered reader here, because
352
// TLS server will not speak until spoken to.
353
br := bufio.NewReader(conn)
354
resp, err := ReadResponse(br, connectReq)
359
if resp.StatusCode != 200 {
360
f := strings.SplitN(resp.Status, " ", 2)
362
return nil, errors.New(f[1])
366
if cm.targetScheme == "https" {
367
// Initiate TLS and check remote host name against certificate.
368
conn = tls.Client(conn, t.TLSClientConfig)
369
if err = conn.(*tls.Conn).Handshake(); err != nil {
372
if t.TLSClientConfig == nil || !t.TLSClientConfig.InsecureSkipVerify {
373
if err = conn.(*tls.Conn).VerifyHostname(cm.tlsHost()); err != nil {
380
pconn.br = bufio.NewReader(pconn.conn)
381
pconn.bw = bufio.NewWriter(pconn.conn)
386
// useProxy returns true if requests to addr should use a proxy,
387
// according to the NO_PROXY or no_proxy environment variable.
388
// addr is always a canonicalAddr with a host and port.
389
func useProxy(addr string) bool {
393
host, _, err := net.SplitHostPort(addr)
397
if host == "localhost" {
400
if ip := net.ParseIP(host); ip != nil {
406
no_proxy := getenvEitherCase("NO_PROXY")
411
addr = strings.ToLower(strings.TrimSpace(addr))
413
addr = addr[:strings.LastIndex(addr, ":")]
416
for _, p := range strings.Split(no_proxy, ",") {
417
p = strings.ToLower(strings.TrimSpace(p))
422
p = p[:strings.LastIndex(p, ":")]
424
if addr == p || (p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:])) {
431
// connectMethod is the map key (in its String form) for keeping persistent
432
// TCP connections alive for subsequent HTTP requests.
434
// A connect method may be of the following types:
436
// Cache key form Description
437
// ----------------- -------------------------
438
// ||http|foo.com http directly to server, no proxy
439
// ||https|foo.com https directly to server, no proxy
440
// http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com
441
// http://proxy.com|http http to proxy, http to anywhere after that
443
// Note: no support to https to the proxy yet.
445
type connectMethod struct {
446
proxyURL *url.URL // nil for no proxy, else full proxy URL
447
targetScheme string // "http" or "https"
448
targetAddr string // Not used if proxy + http targetScheme (4th example in table)
451
func (ck *connectMethod) String() string {
453
targetAddr := ck.targetAddr
454
if ck.proxyURL != nil {
455
proxyStr = ck.proxyURL.String()
456
if ck.targetScheme == "http" {
460
return strings.Join([]string{proxyStr, ck.targetScheme, targetAddr}, "|")
463
// addr returns the first hop "host:port" to which we need to TCP connect.
464
func (cm *connectMethod) addr() string {
465
if cm.proxyURL != nil {
466
return canonicalAddr(cm.proxyURL)
471
// tlsHost returns the host name to match against the peer's
473
func (cm *connectMethod) tlsHost() string {
476
h = h[:strings.LastIndex(h, ":")]
481
// persistConn wraps a connection, usually a persistent one
482
// (but may be used for non-keep-alive requests as well)
483
type persistConn struct {
485
cacheKey string // its connectMethod.String()
487
br *bufio.Reader // from conn
488
bw *bufio.Writer // to conn
489
reqch chan requestAndChan // written by roundTrip(); read by readLoop()
492
// mutateHeaderFunc is an optional func to modify extra
493
// headers on each outbound request before it's written. (the
494
// original Request given to RoundTrip is not modified)
495
mutateHeaderFunc func(Header)
497
lk sync.Mutex // guards numExpectedResponses and broken
498
numExpectedResponses int
499
broken bool // an error has happened on this connection; marked broken so it's not reused.
502
func (pc *persistConn) isBroken() bool {
508
var remoteSideClosedFunc func(error) bool // or nil to use default
510
func remoteSideClosed(err error) bool {
514
if remoteSideClosedFunc != nil {
515
return remoteSideClosedFunc(err)
520
func (pc *persistConn) readLoop() {
522
var lastbody io.ReadCloser // last response body, if any, read on this connection
525
pb, err := pc.br.Peek(1)
528
if pc.numExpectedResponses == 0 {
532
log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v",
541
// Advance past the previous response's body, if the
542
// caller hasn't done so.
544
lastbody.Close() // assumed idempotent
547
resp, err := ReadResponse(pc.br, rc.req)
552
hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0
553
if rc.addedGzip && hasBody && resp.Header.Get("Content-Encoding") == "gzip" {
554
resp.Header.Del("Content-Encoding")
555
resp.Header.Del("Content-Length")
556
resp.ContentLength = -1
557
gzReader, zerr := gzip.NewReader(resp.Body)
562
resp.Body = &readFirstCloseBoth{&discardOnCloseReadCloser{gzReader}, resp.Body}
565
resp.Body = &bodyEOFSignal{body: resp.Body}
568
if err != nil || resp.Close || rc.req.Close {
572
hasBody := resp != nil && resp.ContentLength != 0
573
var waitForBodyRead chan bool
577
waitForBodyRead = make(chan bool)
578
resp.Body.(*bodyEOFSignal).fn = func() {
579
if !pc.t.putIdleConn(pc) {
582
waitForBodyRead <- true
585
// When there's no response body, we immediately
586
// reuse the TCP connection (putIdleConn), but
587
// we need to prevent ClientConn.Read from
588
// closing the Response.Body on the next
589
// loop, otherwise it might close the body
590
// before the client code has had a chance to
591
// read it (even though it'll just be 0, EOF).
594
if !pc.t.putIdleConn(pc) {
600
rc.ch <- responseAndError{resp, err}
602
// Wait for the just-returned response body to be fully consumed
603
// before we race and peek on the underlying bufio reader.
604
if waitForBodyRead != nil {
610
type responseAndError struct {
615
type requestAndChan struct {
617
ch chan responseAndError
619
// did the Transport (as opposed to the client code) add an
620
// Accept-Encoding gzip header? only if it we set it do
621
// we transparently decode the gzip.
625
func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) {
626
if pc.mutateHeaderFunc != nil {
627
pc.mutateHeaderFunc(req.extraHeaders())
630
// Ask for a compressed version if the caller didn't set their
631
// own value for Accept-Encoding. We only attempted to
632
// uncompress the gzip stream if we were the layer that
634
requestedGzip := false
635
if !pc.t.DisableCompression && req.Header.Get("Accept-Encoding") == "" {
636
// Request gzip only, not deflate. Deflate is ambiguous and
637
// not as universally supported anyway.
638
// See: http://www.gzip.org/zlib/zlib_faq.html#faq38
640
req.extraHeaders().Set("Accept-Encoding", "gzip")
644
pc.numExpectedResponses++
647
err = req.Request.write(pc.bw, pc.isProxy, req.extra)
654
ch := make(chan responseAndError, 1)
655
pc.reqch <- requestAndChan{req.Request, ch, requestedGzip}
658
pc.numExpectedResponses--
661
return re.res, re.err
664
func (pc *persistConn) close() {
670
func (pc *persistConn) closeLocked() {
673
pc.mutateHeaderFunc = nil
676
var portMap = map[string]string{
681
// canonicalAddr returns url.Host but always with a ":port" suffix
682
func canonicalAddr(url *url.URL) string {
685
return addr + ":" + portMap[url.Scheme]
690
func responseIsKeepAlive(res *Response) bool {
691
// TODO: implement. for now just always shutting down the connection.
695
// bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most
696
// once, right before the final Read() or Close() call returns, but after
697
// EOF has been seen.
698
type bodyEOFSignal struct {
704
func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {
705
n, err = es.body.Read(p)
706
if es.isClosed && n > 0 {
707
panic("http: unexpected bodyEOFSignal Read after Close; see issue 1725")
709
if err == io.EOF && es.fn != nil {
716
func (es *bodyEOFSignal) Close() (err error) {
721
err = es.body.Close()
722
if err == nil && es.fn != nil {
729
type readFirstCloseBoth struct {
734
func (r *readFirstCloseBoth) Close() error {
735
if err := r.ReadCloser.Close(); err != nil {
739
if err := r.Closer.Close(); err != nil {
745
// discardOnCloseReadCloser consumes all its input on Close.
746
type discardOnCloseReadCloser struct {
750
func (d *discardOnCloseReadCloser) Close() error {
751
io.Copy(ioutil.Discard, d.ReadCloser) // ignore errors; likely invalid or already closed
752
return d.ReadCloser.Close()
added
removed
.pc/15-net-http-connection-close.patch/src/pkg/net/http/transport.go
