1
// Copyright 2011 The Go Authors. All rights reserved.
2
// Use of this source code is governed by a BSD-style
3
// license that can be found in the LICENSE file.
14
// item represents a token or text string returned from the scanner.
20
func (i item) String() string {
22
case i.typ == itemEOF:
24
case i.typ == itemError:
26
case i.typ > itemKeyword:
27
return fmt.Sprintf("<%s>", i.val)
29
return fmt.Sprintf("%.10q...", i.val)
31
return fmt.Sprintf("%q", i.val)
34
// itemType identifies the type of lex items.
38
itemError itemType = iota // error occurred; value is text of error
39
itemBool // boolean constant
40
itemComplex // complex constant (1+2i); imaginary is just a number
42
itemField // alphanumeric identifier, starting with '.', possibly chained ('.x.y')
43
itemIdentifier // alphanumeric identifier
44
itemLeftDelim // left action delimiter
45
itemNumber // simple number, including imaginary
46
itemPipe // pipe symbol
47
itemRawString // raw quoted string (includes quotes)
48
itemRightDelim // right action delimiter
49
itemString // quoted string (includes quotes)
50
itemText // plain text
51
// Keywords appear after all the rest.
52
itemKeyword // used only to delimit the keywords
53
itemDot // the cursor, spelled '.'.
54
itemDefine // define keyword
55
itemElse // else keyword
56
itemEnd // end keyword
58
itemRange // range keyword
59
itemTemplate // template keyword
60
itemWith // with keyword
63
// Make the types prettyprint.
64
var itemName = map[itemType]string{
67
itemComplex: "complex",
70
itemIdentifier: "identifier",
71
itemLeftDelim: "left delim",
74
itemRawString: "raw string",
75
itemRightDelim: "right delim",
84
itemTemplate: "template",
88
func (i itemType) String() string {
91
return fmt.Sprintf("item%d", int(i))
96
var key = map[string]itemType{
103
"template": itemTemplate,
109
// stateFn represents the state of the scanner as a function that returns the next state.
110
type stateFn func(*lexer) stateFn
112
// lexer holds the state of the scanner.
114
name string // the name of the input; used only for error reports.
115
input string // the string being scanned.
116
state stateFn // the next lexing function to enter
117
pos int // current position in the input.
118
start int // start position of this item.
119
width int // width of last rune read from input.
120
items chan item // channel of scanned items.
123
// next returns the next rune in the input.
124
func (l *lexer) next() (rune int) {
125
if l.pos >= len(l.input) {
129
rune, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
134
// peek returns but does not consume the next rune in the input.
135
func (l *lexer) peek() int {
141
// backup steps back one rune. Can only be called once per call of next.
142
func (l *lexer) backup() {
146
// emit passes an item back to the client.
147
func (l *lexer) emit(t itemType) {
148
l.items <- item{t, l.input[l.start:l.pos]}
152
// ignore skips over the pending input before this point.
153
func (l *lexer) ignore() {
157
// accept consumes the next rune if it's from the valid set.
158
func (l *lexer) accept(valid string) bool {
159
if strings.IndexRune(valid, l.next()) >= 0 {
166
// acceptRun consumes a run of runes from the valid set.
167
func (l *lexer) acceptRun(valid string) {
168
for strings.IndexRune(valid, l.next()) >= 0 {
173
// lineNumber reports which line we're on. Doing it this way
174
// means we don't have to worry about peek double counting.
175
func (l *lexer) lineNumber() int {
176
return 1 + strings.Count(l.input[:l.pos], "\n")
179
// error returns an error token and terminates the scan by passing
180
// back a nil pointer that will be the next state, terminating l.run.
181
func (l *lexer) errorf(format string, args ...interface{}) stateFn {
182
l.items <- item{itemError, fmt.Sprintf(format, args...)}
186
// nextItem returns the next item from the input.
187
func (l *lexer) nextItem() item {
190
case item := <-l.items:
199
// lex creates a new scanner for the input string.
200
func lex(name, input string) *lexer {
205
items: make(chan item, 2), // Two items of buffering is sufficient for all state functions
216
rightComment = "*/}}"
219
// lexText scans until an opening action delimiter, "{{".
220
func lexText(l *lexer) stateFn {
222
if strings.HasPrefix(l.input[l.pos:], leftDelim) {
232
// Correctly reached EOF.
240
// lexLeftDelim scans the left delimiter, which is known to be present.
241
func lexLeftDelim(l *lexer) stateFn {
242
if strings.HasPrefix(l.input[l.pos:], leftComment) {
245
l.pos += len(leftDelim)
246
l.emit(itemLeftDelim)
247
return lexInsideAction
250
// lexComment scans a comment. The left comment marker is known to be present.
251
func lexComment(l *lexer) stateFn {
252
i := strings.Index(l.input[l.pos:], rightComment)
254
return l.errorf("unclosed comment")
256
l.pos += i + len(rightComment)
261
// lexRightDelim scans the right delimiter, which is known to be present.
262
func lexRightDelim(l *lexer) stateFn {
263
l.pos += len(rightDelim)
264
l.emit(itemRightDelim)
268
// lexInsideAction scans the elements inside action delimiters.
269
func lexInsideAction(l *lexer) stateFn {
270
// Either number, quoted string, or identifier.
271
// Spaces separate and are ignored.
272
// Pipe symbols separate and are emitted.
274
if strings.HasPrefix(l.input[l.pos:], rightDelim) {
277
switch r := l.next(); {
278
case r == eof || r == '\n':
279
return l.errorf("unclosed action")
289
// special look-ahead for ".field" so we don't break l.backup().
290
if l.pos < len(l.input) {
292
if r < '0' || '9' < r {
293
return lexIdentifier // itemDot comes from the keyword table.
296
fallthrough // '.' can start a number.
297
case r == '+' || r == '-' || ('0' <= r && r <= '9'):
300
case isAlphaNumeric(r):
304
return l.errorf("unrecognized character in action: %#U", r)
310
// lexIdentifier scans an alphanumeric or field.
311
func lexIdentifier(l *lexer) stateFn {
314
switch r := l.next(); {
315
case isAlphaNumeric(r):
317
case r == '.' && l.input[l.start] == '.':
318
// field chaining; absorb into one token.
321
word := l.input[l.start:l.pos]
323
case key[word] > itemKeyword:
327
case word == "true", word == "false":
330
l.emit(itemIdentifier)
335
return lexInsideAction
338
// lexNumber scans a number: decimal, octal, hex, float, or imaginary. This
339
// isn't a perfect number scanner - for instance it accepts "." and "0x0.2"
340
// and "089" - but when it's wrong the input is invalid and the parser (via
341
// strconv) will notice.
342
func lexNumber(l *lexer) stateFn {
344
return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
346
if sign := l.peek(); sign == '+' || sign == '-' {
347
// Complex: 1+2i. No spaces, must end in 'i'.
348
if !l.scanNumber() || l.input[l.pos-1] != 'i' {
349
return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
355
return lexInsideAction
358
func (l *lexer) scanNumber() bool {
359
// Optional leading sign.
362
digits := "0123456789"
363
if l.accept("0") && l.accept("xX") {
364
digits = "0123456789abcdefABCDEF"
372
l.acceptRun("0123456789")
376
// Next thing mustn't be alphanumeric.
377
if isAlphaNumeric(l.peek()) {
384
// lexQuote scans a quoted string.
385
func lexQuote(l *lexer) stateFn {
390
if r := l.next(); r != eof && r != '\n' {
395
return l.errorf("unterminated quoted string")
401
return lexInsideAction
404
// lexRawQuote scans a raw quoted string.
405
func lexRawQuote(l *lexer) stateFn {
410
return l.errorf("unterminated raw quoted string")
415
l.emit(itemRawString)
416
return lexInsideAction
419
// isSpace reports whether r is a space character.
420
func isSpace(r int) bool {
422
case ' ', '\t', '\n', '\r':
428
// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.
429
func isAlphaNumeric(r int) bool {
430
return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
added
removed
src/pkg/exp/template/lex.go
