1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
package lexer
import (
"bytes"
"fmt"
"io"
"strconv"
"strings"
"text/scanner"
"unicode/utf8"
)
// TextScannerLexer is a lexer that uses the text/scanner module.
var (
TextScannerLexer Definition = &defaultDefinition{}
// DefaultDefinition defines properties for the default lexer.
DefaultDefinition = TextScannerLexer
)
type defaultDefinition struct{}
func (d *defaultDefinition) Lex(r io.Reader) (Lexer, error) {
return Lex(r), nil
}
func (d *defaultDefinition) Symbols() map[string]rune {
return map[string]rune{
"EOF": scanner.EOF,
"Char": scanner.Char,
"Ident": scanner.Ident,
"Int": scanner.Int,
"Float": scanner.Float,
"String": scanner.String,
"RawString": scanner.RawString,
"Comment": scanner.Comment,
}
}
// textScannerLexer is a Lexer based on text/scanner.Scanner
type textScannerLexer struct {
scanner *scanner.Scanner
filename string
err error
}
// Lex an io.Reader with text/scanner.Scanner.
//
// This provides very fast lexing of source code compatible with Go tokens.
//
// Note that this differs from text/scanner.Scanner in that string tokens will be unquoted.
func Lex(r io.Reader) Lexer {
s := &scanner.Scanner{}
s.Init(r)
lexer := lexWithScanner(r, s)
lexer.scanner.Error = func(s *scanner.Scanner, msg string) {
// This is to support single quoted strings. Hacky.
if !strings.HasSuffix(msg, "char literal") {
lexer.err = Errorf(Position(lexer.scanner.Pos()), msg)
}
}
return lexer
}
// LexWithScanner creates a Lexer from a user-provided scanner.Scanner.
//
// Useful if you need to customise the Scanner.
func LexWithScanner(r io.Reader, scan *scanner.Scanner) Lexer {
return lexWithScanner(r, scan)
}
func lexWithScanner(r io.Reader, scan *scanner.Scanner) *textScannerLexer {
lexer := &textScannerLexer{
filename: NameOfReader(r),
scanner: scan,
}
return lexer
}
// LexBytes returns a new default lexer over bytes.
func LexBytes(b []byte) Lexer {
return Lex(bytes.NewReader(b))
}
// LexString returns a new default lexer over a string.
func LexString(s string) Lexer {
return Lex(strings.NewReader(s))
}
func (t *textScannerLexer) Next() (Token, error) {
typ := t.scanner.Scan()
text := t.scanner.TokenText()
pos := Position(t.scanner.Position)
pos.Filename = t.filename
if t.err != nil {
return Token{}, t.err
}
return textScannerTransform(Token{
Type: typ,
Value: text,
Pos: pos,
})
}
func textScannerTransform(token Token) (Token, error) {
// Unquote strings.
switch token.Type {
case scanner.Char:
// FIXME(alec): This is pretty hacky...we convert a single quoted char into a double
// quoted string in order to support single quoted strings.
token.Value = fmt.Sprintf("\"%s\"", token.Value[1:len(token.Value)-1])
fallthrough
case scanner.String:
s, err := strconv.Unquote(token.Value)
if err != nil {
return Token{}, Errorf(token.Pos, "%s: %q", err.Error(), token.Value)
}
token.Value = s
if token.Type == scanner.Char && utf8.RuneCountInString(s) > 1 {
token.Type = scanner.String
}
case scanner.RawString:
token.Value = token.Value[1 : len(token.Value)-1]
}
return token, nil
}
|
