1
// Copyright 2013 The Go Authors. All rights reserved.
2
// Use of this source code is governed by a BSD-style
3
// license that can be found in the LICENSE file.
14
// Scanner provides a convenient interface for reading data such as
15
// a file of newline-delimited lines of text. Successive calls to
16
// the Scan method will step through the 'tokens' of a file, skipping
17
// the bytes between the tokens. The specification of a token is
18
// defined by a split function of type SplitFunc; the default split
19
// function breaks the input into lines with line termination stripped. Split
20
// functions are defined in this package for scanning a file into
21
// lines, bytes, UTF-8-encoded runes, and space-delimited words. The
22
// client may instead provide a custom split function.
24
// Scanning stops unrecoverably at EOF, the first I/O error, or a token too
25
// large to fit in the buffer. When a scan stops, the reader may have
26
// advanced arbitrarily far past the last token. Programs that need more
27
// control over error handling or large tokens, or must run sequential scans
28
// on a reader, should use bufio.Reader instead.
31
r io.Reader // The reader provided by the client.
32
split SplitFunc // The function to split the tokens.
33
maxTokenSize int // Maximum size of a token; modified by tests.
34
token []byte // Last token returned by split.
35
buf []byte // Buffer used as argument to split.
36
start int // First non-processed byte in buf.
37
end int // End of data in buf.
38
err error // Sticky error.
41
// SplitFunc is the signature of the split function used to tokenize the
42
// input. The arguments are an initial substring of the remaining unprocessed
43
// data and a flag, atEOF, that reports whether the Reader has no more data
44
// to give. The return values are the number of bytes to advance the input
45
// and the next token to return to the user, plus an error, if any. If the
46
// data does not yet hold a complete token, for instance if it has no newline
47
// while scanning lines, SplitFunc can return (0, nil) to signal the Scanner
48
// to read more data into the slice and try again with a longer slice
49
// starting at the same point in the input.
51
// If the returned error is non-nil, scanning stops and the error
52
// is returned to the client.
54
// The function is never called with an empty data slice unless atEOF
55
// is true. If atEOF is true, however, data may be non-empty and,
56
// as always, holds unprocessed text.
57
type SplitFunc func(data []byte, atEOF bool) (advance int, token []byte, err error)
59
// Errors returned by Scanner.
61
ErrTooLong = errors.New("bufio.Scanner: token too long")
62
ErrNegativeAdvance = errors.New("bufio.Scanner: SplitFunc returns negative advance count")
63
ErrAdvanceTooFar = errors.New("bufio.Scanner: SplitFunc returns advance count beyond input")
67
// Maximum size used to buffer a token. The actual maximum token size
68
// may be smaller as the buffer may need to include, for instance, a newline.
69
MaxScanTokenSize = 64 * 1024
72
// NewScanner returns a new Scanner to read from r.
73
// The split function defaults to ScanLines.
74
func NewScanner(r io.Reader) *Scanner {
78
maxTokenSize: MaxScanTokenSize,
79
buf: make([]byte, 4096), // Plausible starting size; needn't be large.
83
// Err returns the first non-EOF error that was encountered by the Scanner.
84
func (s *Scanner) Err() error {
91
// Bytes returns the most recent token generated by a call to Scan.
92
// The underlying array may point to data that will be overwritten
93
// by a subsequent call to Scan. It does no allocation.
94
func (s *Scanner) Bytes() []byte {
98
// Text returns the most recent token generated by a call to Scan
99
// as a newly allocated string holding its bytes.
100
func (s *Scanner) Text() string {
101
return string(s.token)
104
// Scan advances the Scanner to the next token, which will then be
105
// available through the Bytes or Text method. It returns false when the
106
// scan stops, either by reaching the end of the input or an error.
107
// After Scan returns false, the Err method will return any error that
108
// occurred during scanning, except that if it was io.EOF, Err
110
func (s *Scanner) Scan() bool {
111
// Loop until we have a token.
113
// See if we can get a token with what we already have.
115
advance, token, err := s.split(s.buf[s.start:s.end], s.err != nil)
120
if !s.advance(advance) {
128
// We cannot generate a token with what we are holding.
129
// If we've already hit EOF or an I/O error, we are done.
136
// Must read more data.
137
// First, shift data to beginning of buffer if there's lots of empty space
138
// or space is neded.
139
if s.start > 0 && (s.end == len(s.buf) || s.start > len(s.buf)/2) {
140
copy(s.buf, s.buf[s.start:s.end])
144
// Is the buffer full? If so, resize.
145
if s.end == len(s.buf) {
146
if len(s.buf) >= s.maxTokenSize {
150
newSize := len(s.buf) * 2
151
if newSize > s.maxTokenSize {
152
newSize = s.maxTokenSize
154
newBuf := make([]byte, newSize)
155
copy(newBuf, s.buf[s.start:s.end])
161
// Finally we can read some input. Make sure we don't get stuck with
162
// a misbehaving Reader. Officially we don't need to do this, but let's
163
// be extra careful: Scanner is for safe, simple jobs.
165
n, err := s.r.Read(s.buf[s.end:len(s.buf)])
176
s.setErr(io.ErrNoProgress)
183
// advance consumes n bytes of the buffer. It reports whether the advance was legal.
184
func (s *Scanner) advance(n int) bool {
186
s.setErr(ErrNegativeAdvance)
189
if n > s.end-s.start {
190
s.setErr(ErrAdvanceTooFar)
197
// setErr records the first error encountered.
198
func (s *Scanner) setErr(err error) {
199
if s.err == nil || s.err == io.EOF {
204
// Split sets the split function for the Scanner. If called, it must be
205
// called before Scan. The default split function is ScanLines.
206
func (s *Scanner) Split(split SplitFunc) {
212
// ScanBytes is a split function for a Scanner that returns each byte as a token.
213
func ScanBytes(data []byte, atEOF bool) (advance int, token []byte, err error) {
214
if atEOF && len(data) == 0 {
217
return 1, data[0:1], nil
220
var errorRune = []byte(string(utf8.RuneError))
222
// ScanRunes is a split function for a Scanner that returns each
223
// UTF-8-encoded rune as a token. The sequence of runes returned is
224
// equivalent to that from a range loop over the input as a string, which
225
// means that erroneous UTF-8 encodings translate to U+FFFD = "\xef\xbf\xbd".
226
// Because of the Scan interface, this makes it impossible for the client to
227
// distinguish correctly encoded replacement runes from encoding errors.
228
func ScanRunes(data []byte, atEOF bool) (advance int, token []byte, err error) {
229
if atEOF && len(data) == 0 {
233
// Fast path 1: ASCII.
234
if data[0] < utf8.RuneSelf {
235
return 1, data[0:1], nil
238
// Fast path 2: Correct UTF-8 decode without error.
239
_, width := utf8.DecodeRune(data)
241
// It's a valid encoding. Width cannot be one for a correctly encoded
243
return width, data[0:width], nil
246
// We know it's an error: we have width==1 and implicitly r==utf8.RuneError.
247
// Is the error because there wasn't a full rune to be decoded?
248
// FullRune distinguishes correctly between erroneous and incomplete encodings.
249
if !atEOF && !utf8.FullRune(data) {
250
// Incomplete; get more bytes.
254
// We have a real UTF-8 encoding error. Return a properly encoded error rune
255
// but advance only one byte. This matches the behavior of a range loop over
256
// an incorrectly encoded string.
257
return 1, errorRune, nil
260
// dropCR drops a terminal \r from the data.
261
func dropCR(data []byte) []byte {
262
if len(data) > 0 && data[len(data)-1] == '\r' {
263
return data[0 : len(data)-1]
268
// ScanLines is a split function for a Scanner that returns each line of
269
// text, stripped of any trailing end-of-line marker. The returned line may
270
// be empty. The end-of-line marker is one optional carriage return followed
271
// by one mandatory newline. In regular expression notation, it is `\r?\n`.
272
// The last non-empty line of input will be returned even if it has no
274
func ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
275
if atEOF && len(data) == 0 {
278
if i := bytes.IndexByte(data, '\n'); i >= 0 {
279
// We have a full newline-terminated line.
280
return i + 1, dropCR(data[0:i]), nil
282
// If we're at EOF, we have a final, non-terminated line. Return it.
284
return len(data), dropCR(data), nil
286
// Request more data.
290
// isSpace returns whether the character is a Unicode white space character.
291
// We avoid dependency on the unicode package, but check validity of the implementation
293
func isSpace(r rune) bool {
295
// Obvious ASCII ones: \t through \r plus space. Plus two Latin-1 oddballs.
297
case ' ', '\t', '\n', '\v', '\f', '\r':
299
case '\u0085', '\u00A0':
305
if '\u2000' <= r && r <= '\u200a' {
309
case '\u1680', '\u180e', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000':
315
// ScanWords is a split function for a Scanner that returns each
316
// space-separated word of text, with surrounding spaces deleted. It will
317
// never return an empty string. The definition of space is set by
319
func ScanWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
320
// Skip leading spaces.
322
for width := 0; start < len(data); start += width {
324
r, width = utf8.DecodeRune(data[start:])
329
if atEOF && len(data) == 0 {
332
// Scan until space, marking end of word.
333
for width, i := 0, start; i < len(data); i += width {
335
r, width = utf8.DecodeRune(data[i:])
337
return i + width, data[start:i], nil
340
// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
341
if atEOF && len(data) > start {
342
return len(data), data[start:], nil
344
// Request more data.
added
removed
src/pkg/bufio/scan.go
