1
// Copyright 2009 The Go Authors. All rights reserved.
2
// Use of this source code is governed by a BSD-style
3
// license that can be found in the LICENSE file.
5
// HTTP Request reading and parsing.
28
chunkSize = 4 << 10 // 4 KB chunks
29
defaultMaxMemory = 32 << 20 // 32 MB
32
// ErrMissingFile is returned by FormFile when the provided file field name
33
// is either not present in the request or not a file field.
34
var ErrMissingFile = errors.New("http: no such file")
36
// HTTP request parsing errors.
37
type ProtocolError struct {
41
func (err *ProtocolError) Error() string { return err.ErrorString }
44
ErrHeaderTooLong = &ProtocolError{"header too long"}
45
ErrShortBody = &ProtocolError{"entity body too short"}
46
ErrNotSupported = &ProtocolError{"feature not supported"}
47
ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"}
48
ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"}
49
ErrNotMultipart = &ProtocolError{"request Content-Type isn't multipart/form-data"}
50
ErrMissingBoundary = &ProtocolError{"no multipart boundary param in Content-Type"}
53
type badStringError struct {
58
func (e *badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) }
60
// Headers that Request.Write handles itself and should be skipped.
61
var reqWriteExcludeHeader = map[string]bool{
62
"Host": true, // not in Header map anyway
64
"Content-Length": true,
65
"Transfer-Encoding": true,
69
// A Request represents an HTTP request received by a server
70
// or to be sent by a client.
72
Method string // GET, POST, PUT, etc.
74
// URL is created from the URI supplied on the Request-Line
75
// as stored in RequestURI.
77
// For most requests, fields other than Path and RawQuery
78
// will be empty. (See RFC 2616, Section 5.1.2)
81
// The protocol version for incoming requests.
82
// Outgoing requests always use HTTP/1.1.
83
Proto string // "HTTP/1.0"
87
// A header maps request lines to their values.
90
// accept-encoding: gzip, deflate
91
// Accept-Language: en-us
92
// Connection: keep-alive
96
// Header = map[string][]string{
97
// "Accept-Encoding": {"gzip, deflate"},
98
// "Accept-Language": {"en-us"},
99
// "Connection": {"keep-alive"},
102
// HTTP defines that header names are case-insensitive.
103
// The request parser implements this by canonicalizing the
104
// name, making the first character and any characters
105
// following a hyphen uppercase and the rest lowercase.
108
// Body is the request's body.
110
// For client requests, a nil body means the request has no
111
// body, such as a GET request. The HTTP Client's Transport
112
// is responsible for calling the Close method.
114
// For server requests, the Request Body is always non-nil
115
// but will return EOF immediately when no body is present.
116
// The Server will close the request body. The ServeHTTP
117
// Handler does not need to.
120
// ContentLength records the length of the associated content.
121
// The value -1 indicates that the length is unknown.
122
// Values >= 0 indicate that the given number of bytes may
123
// be read from Body.
124
// For outgoing requests, a value of 0 means unknown if Body is not nil.
127
// TransferEncoding lists the transfer encodings from outermost to
128
// innermost. An empty list denotes the "identity" encoding.
129
// TransferEncoding can usually be ignored; chunked encoding is
130
// automatically added and removed as necessary when sending and
131
// receiving requests.
132
TransferEncoding []string
134
// Close indicates whether to close the connection after
135
// replying to this request.
138
// The host on which the URL is sought.
139
// Per RFC 2616, this is either the value of the Host: header
140
// or the host name given in the URL itself.
141
// It may be of the form "host:port".
144
// Form contains the parsed form data, including both the URL
145
// field's query parameters and the POST or PUT form data.
146
// This field is only available after ParseForm is called.
147
// The HTTP client ignores Form and uses Body instead.
150
// PostForm contains the parsed form data from POST or PUT
152
// This field is only available after ParseForm is called.
153
// The HTTP client ignores PostForm and uses Body instead.
156
// MultipartForm is the parsed multipart form, including file uploads.
157
// This field is only available after ParseMultipartForm is called.
158
// The HTTP client ignores MultipartForm and uses Body instead.
159
MultipartForm *multipart.Form
161
// Trailer maps trailer keys to values. Like for Header, if the
162
// response has multiple trailer lines with the same key, they will be
163
// concatenated, delimited by commas.
164
// For server requests, Trailer is only populated after Body has been
165
// closed or fully consumed.
166
// Trailer support is only partially complete.
169
// RemoteAddr allows HTTP servers and other software to record
170
// the network address that sent the request, usually for
171
// logging. This field is not filled in by ReadRequest and
172
// has no defined format. The HTTP server in this package
173
// sets RemoteAddr to an "IP:port" address before invoking a
175
// This field is ignored by the HTTP client.
178
// RequestURI is the unmodified Request-URI of the
179
// Request-Line (RFC 2616, Section 5.1) as sent by the client
180
// to a server. Usually the URL field should be used instead.
181
// It is an error to set this field in an HTTP client request.
184
// TLS allows HTTP servers and other software to record
185
// information about the TLS connection on which the request
186
// was received. This field is not filled in by ReadRequest.
187
// The HTTP server in this package sets the field for
188
// TLS-enabled connections before invoking a handler;
189
// otherwise it leaves the field nil.
190
// This field is ignored by the HTTP client.
191
TLS *tls.ConnectionState
194
// ProtoAtLeast reports whether the HTTP protocol used
195
// in the request is at least major.minor.
196
func (r *Request) ProtoAtLeast(major, minor int) bool {
197
return r.ProtoMajor > major ||
198
r.ProtoMajor == major && r.ProtoMinor >= minor
201
// UserAgent returns the client's User-Agent, if sent in the request.
202
func (r *Request) UserAgent() string {
203
return r.Header.Get("User-Agent")
206
// Cookies parses and returns the HTTP cookies sent with the request.
207
func (r *Request) Cookies() []*Cookie {
208
return readCookies(r.Header, "")
211
var ErrNoCookie = errors.New("http: named cookie not present")
213
// Cookie returns the named cookie provided in the request or
214
// ErrNoCookie if not found.
215
func (r *Request) Cookie(name string) (*Cookie, error) {
216
for _, c := range readCookies(r.Header, name) {
219
return nil, ErrNoCookie
222
// AddCookie adds a cookie to the request. Per RFC 6265 section 5.4,
223
// AddCookie does not attach more than one Cookie header field. That
224
// means all cookies, if any, are written into the same line,
225
// separated by semicolon.
226
func (r *Request) AddCookie(c *Cookie) {
227
s := fmt.Sprintf("%s=%s", sanitizeCookieName(c.Name), sanitizeCookieValue(c.Value))
228
if c := r.Header.Get("Cookie"); c != "" {
229
r.Header.Set("Cookie", c+"; "+s)
231
r.Header.Set("Cookie", s)
235
// Referer returns the referring URL, if sent in the request.
237
// Referer is misspelled as in the request itself, a mistake from the
238
// earliest days of HTTP. This value can also be fetched from the
239
// Header map as Header["Referer"]; the benefit of making it available
240
// as a method is that the compiler can diagnose programs that use the
241
// alternate (correct English) spelling req.Referrer() but cannot
242
// diagnose programs that use Header["Referrer"].
243
func (r *Request) Referer() string {
244
return r.Header.Get("Referer")
247
// multipartByReader is a sentinel value.
248
// Its presence in Request.MultipartForm indicates that parsing of the request
249
// body has been handed off to a MultipartReader instead of ParseMultipartFrom.
250
var multipartByReader = &multipart.Form{
251
Value: make(map[string][]string),
252
File: make(map[string][]*multipart.FileHeader),
255
// MultipartReader returns a MIME multipart reader if this is a
256
// multipart/form-data POST request, else returns nil and an error.
257
// Use this function instead of ParseMultipartForm to
258
// process the request body as a stream.
259
func (r *Request) MultipartReader() (*multipart.Reader, error) {
260
if r.MultipartForm == multipartByReader {
261
return nil, errors.New("http: MultipartReader called twice")
263
if r.MultipartForm != nil {
264
return nil, errors.New("http: multipart handled by ParseMultipartForm")
266
r.MultipartForm = multipartByReader
267
return r.multipartReader()
270
func (r *Request) multipartReader() (*multipart.Reader, error) {
271
v := r.Header.Get("Content-Type")
273
return nil, ErrNotMultipart
275
d, params, err := mime.ParseMediaType(v)
276
if err != nil || d != "multipart/form-data" {
277
return nil, ErrNotMultipart
279
boundary, ok := params["boundary"]
281
return nil, ErrMissingBoundary
283
return multipart.NewReader(r.Body, boundary), nil
286
// Return value if nonempty, def otherwise.
287
func valueOrDefault(value, def string) string {
294
// NOTE: This is not intended to reflect the actual Go version being used.
295
// It was changed from "Go http package" to "Go 1.1 package http" at the
296
// time of the Go 1.1 release because the former User-Agent had ended up
297
// on a blacklist for some intrusion detection systems.
298
// See https://codereview.appspot.com/7532043.
299
const defaultUserAgent = "Go 1.1 package http"
301
// Write writes an HTTP/1.1 request -- header and body -- in wire format.
302
// This method consults the following fields of the request:
305
// Method (defaults to "GET")
311
// If Body is present, Content-Length is <= 0 and TransferEncoding
312
// hasn't been set to "identity", Write adds "Transfer-Encoding:
313
// chunked" to the header. Body is closed after it is sent.
314
func (r *Request) Write(w io.Writer) error {
315
return r.write(w, false, nil)
318
// WriteProxy is like Write but writes the request in the form
319
// expected by an HTTP proxy. In particular, WriteProxy writes the
320
// initial Request-URI line of the request with an absolute URI, per
321
// section 5.1.2 of RFC 2616, including the scheme and host.
322
// In either case, WriteProxy also writes a Host header, using
323
// either r.Host or r.URL.Host.
324
func (r *Request) WriteProxy(w io.Writer) error {
325
return r.write(w, true, nil)
328
// extraHeaders may be nil
329
func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header) error {
333
return errors.New("http: Request.Write on Request with no Host or URL set")
338
ruri := req.URL.RequestURI()
339
if usingProxy && req.URL.Scheme != "" && req.URL.Opaque == "" {
340
ruri = req.URL.Scheme + "://" + host + ruri
341
} else if req.Method == "CONNECT" && req.URL.Path == "" {
342
// CONNECT requests normally give just the host and port, not a full URL.
345
// TODO(bradfitz): escape at least newlines in ruri?
347
// Wrap the writer in a bufio Writer if it's not already buffered.
348
// Don't always call NewWriter, as that forces a bytes.Buffer
349
// and other small bufio Writers to have a minimum 4k buffer
352
if _, ok := w.(io.ByteWriter); !ok {
353
bw = bufio.NewWriter(w)
357
fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), ruri)
360
fmt.Fprintf(w, "Host: %s\r\n", host)
362
// Use the defaultUserAgent unless the Header contains one, which
363
// may be blank to not send the header.
364
userAgent := defaultUserAgent
365
if req.Header != nil {
366
if ua := req.Header["User-Agent"]; len(ua) > 0 {
371
fmt.Fprintf(w, "User-Agent: %s\r\n", userAgent)
374
// Process Body,ContentLength,Close,Trailer
375
tw, err := newTransferWriter(req)
379
err = tw.WriteHeader(w)
384
// TODO: split long values? (If so, should share code with Conn.Write)
385
err = req.Header.WriteSubset(w, reqWriteExcludeHeader)
390
if extraHeaders != nil {
391
err = extraHeaders.Write(w)
397
io.WriteString(w, "\r\n")
399
// Write body and trailer
400
err = tw.WriteBody(w)
411
// ParseHTTPVersion parses a HTTP version string.
412
// "HTTP/1.0" returns (1, 0, true).
413
func ParseHTTPVersion(vers string) (major, minor int, ok bool) {
414
const Big = 1000000 // arbitrary upper bound
421
if !strings.HasPrefix(vers, "HTTP/") {
424
dot := strings.Index(vers, ".")
428
major, err := strconv.Atoi(vers[5:dot])
429
if err != nil || major < 0 || major > Big {
432
minor, err = strconv.Atoi(vers[dot+1:])
433
if err != nil || minor < 0 || minor > Big {
436
return major, minor, true
439
// NewRequest returns a new Request given a method, URL, and optional body.
441
// If the provided body is also an io.Closer, the returned
442
// Request.Body is set to body and will be closed by the Client
443
// methods Do, Post, and PostForm, and Transport.RoundTrip.
444
func NewRequest(method, urlStr string, body io.Reader) (*Request, error) {
445
u, err := url.Parse(urlStr)
449
rc, ok := body.(io.ReadCloser)
450
if !ok && body != nil {
451
rc = ioutil.NopCloser(body)
459
Header: make(Header),
464
switch v := body.(type) {
466
req.ContentLength = int64(v.Len())
468
req.ContentLength = int64(v.Len())
469
case *strings.Reader:
470
req.ContentLength = int64(v.Len())
477
// SetBasicAuth sets the request's Authorization header to use HTTP
478
// Basic Authentication with the provided username and password.
480
// With HTTP Basic Authentication the provided username and password
481
// are not encrypted.
482
func (r *Request) SetBasicAuth(username, password string) {
483
r.Header.Set("Authorization", "Basic "+basicAuth(username, password))
486
// parseRequestLine parses "GET /foo HTTP/1.1" into its three parts.
487
func parseRequestLine(line string) (method, requestURI, proto string, ok bool) {
488
s1 := strings.Index(line, " ")
489
s2 := strings.Index(line[s1+1:], " ")
490
if s1 < 0 || s2 < 0 {
494
return line[:s1], line[s1+1 : s2], line[s2+1:], true
497
// TODO(bradfitz): use a sync.Cache when available
498
var textprotoReaderCache = make(chan *textproto.Reader, 4)
500
func newTextprotoReader(br *bufio.Reader) *textproto.Reader {
502
case r := <-textprotoReaderCache:
506
return textproto.NewReader(br)
510
func putTextprotoReader(r *textproto.Reader) {
513
case textprotoReaderCache <- r:
518
// ReadRequest reads and parses a request from b.
519
func ReadRequest(b *bufio.Reader) (req *Request, err error) {
521
tp := newTextprotoReader(b)
524
// First line: GET /index.html HTTP/1.0
526
if s, err = tp.ReadLine(); err != nil {
530
putTextprotoReader(tp)
532
err = io.ErrUnexpectedEOF
537
req.Method, req.RequestURI, req.Proto, ok = parseRequestLine(s)
539
return nil, &badStringError{"malformed HTTP request", s}
541
rawurl := req.RequestURI
542
if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok {
543
return nil, &badStringError{"malformed HTTP version", req.Proto}
546
// CONNECT requests are used two different ways, and neither uses a full URL:
547
// The standard use is to tunnel HTTPS through an HTTP proxy.
548
// It looks like "CONNECT www.google.com:443 HTTP/1.1", and the parameter is
549
// just the authority section of a URL. This information should go in req.URL.Host.
551
// The net/rpc package also uses CONNECT, but there the parameter is a path
552
// that starts with a slash. It can be parsed with the regular URL parser,
553
// and the path will end up in req.URL.Path, where it needs to be in order for
555
justAuthority := req.Method == "CONNECT" && !strings.HasPrefix(rawurl, "/")
557
rawurl = "http://" + rawurl
560
if req.URL, err = url.ParseRequestURI(rawurl); err != nil {
565
// Strip the bogus "http://" back off.
569
// Subsequent lines: Key: value.
570
mimeHeader, err := tp.ReadMIMEHeader()
574
req.Header = Header(mimeHeader)
576
// RFC2616: Must treat
577
// GET /index.html HTTP/1.1
578
// Host: www.google.com
580
// GET http://www.google.com/index.html HTTP/1.1
581
// Host: doesntmatter
582
// the same. In the second case, any Host line is ignored.
583
req.Host = req.URL.Host
585
req.Host = req.Header.get("Host")
587
delete(req.Header, "Host")
589
fixPragmaCacheControl(req.Header)
591
// TODO: Parse specific header values:
605
// If-Unmodified-Since
607
// Proxy-Authorization
609
// TE (transfer-codings)
617
err = readTransfer(req, b)
625
// MaxBytesReader is similar to io.LimitReader but is intended for
626
// limiting the size of incoming request bodies. In contrast to
627
// io.LimitReader, MaxBytesReader's result is a ReadCloser, returns a
628
// non-EOF error for a Read beyond the limit, and Closes the
629
// underlying reader when its Close method is called.
631
// MaxBytesReader prevents clients from accidentally or maliciously
632
// sending a large request and wasting server resources.
633
func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser {
634
return &maxBytesReader{w: w, r: r, n: n}
637
type maxBytesReader struct {
639
r io.ReadCloser // underlying reader
640
n int64 // max bytes remaining
644
func (l *maxBytesReader) Read(p []byte) (n int, err error) {
648
if res, ok := l.w.(*response); ok {
649
res.requestTooLarge()
652
return 0, errors.New("http: request body too large")
654
if int64(len(p)) > l.n {
662
func (l *maxBytesReader) Close() error {
666
func copyValues(dst, src url.Values) {
667
for k, vs := range src {
668
for _, value := range vs {
674
func parsePostForm(r *Request) (vs url.Values, err error) {
676
err = errors.New("missing form body")
679
ct := r.Header.Get("Content-Type")
680
// RFC 2616, section 7.2.1 - empty type
681
// SHOULD be treated as application/octet-stream
683
ct = "application/octet-stream"
685
ct, _, err = mime.ParseMediaType(ct)
687
case ct == "application/x-www-form-urlencoded":
688
var reader io.Reader = r.Body
689
maxFormSize := int64(1<<63 - 1)
690
if _, ok := r.Body.(*maxBytesReader); !ok {
691
maxFormSize = int64(10 << 20) // 10 MB is a lot of text.
692
reader = io.LimitReader(r.Body, maxFormSize+1)
694
b, e := ioutil.ReadAll(reader)
701
if int64(len(b)) > maxFormSize {
702
err = errors.New("http: POST too large")
705
vs, e = url.ParseQuery(string(b))
709
case ct == "multipart/form-data":
710
// handled by ParseMultipartForm (which is calling us, or should be)
711
// TODO(bradfitz): there are too many possible
712
// orders to call too many functions here.
713
// Clean this up and write more tests.
714
// request_test.go contains the start of this,
715
// in TestParseMultipartFormOrder and others.
720
// ParseForm parses the raw query from the URL and updates r.Form.
722
// For POST or PUT requests, it also parses the request body as a form and
723
// put the results into both r.PostForm and r.Form.
724
// POST and PUT body parameters take precedence over URL query string values
727
// If the request Body's size has not already been limited by MaxBytesReader,
728
// the size is capped at 10MB.
730
// ParseMultipartForm calls ParseForm automatically.
732
func (r *Request) ParseForm() error {
734
if r.PostForm == nil {
735
if r.Method == "POST" || r.Method == "PUT" || r.Method == "PATCH" {
736
r.PostForm, err = parsePostForm(r)
738
if r.PostForm == nil {
739
r.PostForm = make(url.Values)
743
if len(r.PostForm) > 0 {
744
r.Form = make(url.Values)
745
copyValues(r.Form, r.PostForm)
747
var newValues url.Values
750
newValues, e = url.ParseQuery(r.URL.RawQuery)
755
if newValues == nil {
756
newValues = make(url.Values)
761
copyValues(r.Form, newValues)
767
// ParseMultipartForm parses a request body as multipart/form-data.
768
// The whole request body is parsed and up to a total of maxMemory bytes of
769
// its file parts are stored in memory, with the remainder stored on
770
// disk in temporary files.
771
// ParseMultipartForm calls ParseForm if necessary.
772
// After one call to ParseMultipartForm, subsequent calls have no effect.
773
func (r *Request) ParseMultipartForm(maxMemory int64) error {
774
if r.MultipartForm == multipartByReader {
775
return errors.New("http: multipart handled by MultipartReader")
783
if r.MultipartForm != nil {
787
mr, err := r.multipartReader()
788
if err == ErrNotMultipart {
790
} else if err != nil {
794
f, err := mr.ReadForm(maxMemory)
798
for k, v := range f.Value {
799
r.Form[k] = append(r.Form[k], v...)
806
// FormValue returns the first value for the named component of the query.
807
// POST and PUT body parameters take precedence over URL query string values.
808
// FormValue calls ParseMultipartForm and ParseForm if necessary.
809
// To access multiple values of the same key use ParseForm.
810
func (r *Request) FormValue(key string) string {
812
r.ParseMultipartForm(defaultMaxMemory)
814
if vs := r.Form[key]; len(vs) > 0 {
820
// PostFormValue returns the first value for the named component of the POST
821
// or PUT request body. URL query parameters are ignored.
822
// PostFormValue calls ParseMultipartForm and ParseForm if necessary.
823
func (r *Request) PostFormValue(key string) string {
824
if r.PostForm == nil {
825
r.ParseMultipartForm(defaultMaxMemory)
827
if vs := r.PostForm[key]; len(vs) > 0 {
833
// FormFile returns the first file for the provided form key.
834
// FormFile calls ParseMultipartForm and ParseForm if necessary.
835
func (r *Request) FormFile(key string) (multipart.File, *multipart.FileHeader, error) {
836
if r.MultipartForm == multipartByReader {
837
return nil, nil, errors.New("http: multipart handled by MultipartReader")
839
if r.MultipartForm == nil {
840
err := r.ParseMultipartForm(defaultMaxMemory)
845
if r.MultipartForm != nil && r.MultipartForm.File != nil {
846
if fhs := r.MultipartForm.File[key]; len(fhs) > 0 {
847
f, err := fhs[0].Open()
848
return f, fhs[0], err
851
return nil, nil, ErrMissingFile
854
func (r *Request) expectsContinue() bool {
855
return hasToken(r.Header.get("Expect"), "100-continue")
858
func (r *Request) wantsHttp10KeepAlive() bool {
859
if r.ProtoMajor != 1 || r.ProtoMinor != 0 {
862
return hasToken(r.Header.get("Connection"), "keep-alive")
865
func (r *Request) wantsClose() bool {
866
return hasToken(r.Header.get("Connection"), "close")