1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
|
package lexer
import (
"bytes"
"io"
"strings"
"text/scanner"
)
// TextScannerLexer is a lexer that uses the text/scanner module.
var (
TextScannerLexer Definition = &textScannerLexerDefinition{}
// DefaultDefinition defines properties for the default lexer.
DefaultDefinition = TextScannerLexer
)
// NewTextScannerLexer constructs a Definition that uses an underlying scanner.Scanner
//
// "configure" will be called after the scanner.Scanner.Init(r) is called. If "configure"
// is nil a default scanner.Scanner will be used.
func NewTextScannerLexer(configure func(*scanner.Scanner)) Definition {
return &textScannerLexerDefinition{configure: configure}
}
type textScannerLexerDefinition struct {
configure func(*scanner.Scanner)
}
func (d *textScannerLexerDefinition) Lex(filename string, r io.Reader) (Lexer, error) {
l := Lex(filename, r)
if d.configure != nil {
d.configure(l.(*textScannerLexer).scanner)
}
return l, nil
}
func (d *textScannerLexerDefinition) Symbols() map[string]TokenType {
return map[string]TokenType{
"EOF": EOF,
"Char": scanner.Char,
"Ident": scanner.Ident,
"Int": scanner.Int,
"Float": scanner.Float,
"String": scanner.String,
"RawString": scanner.RawString,
"Comment": scanner.Comment,
}
}
// textScannerLexer is a Lexer based on text/scanner.Scanner
type textScannerLexer struct {
scanner *scanner.Scanner
filename string
err error
}
// Lex an io.Reader with text/scanner.Scanner.
//
// This provides very fast lexing of source code compatible with Go tokens.
//
// Note that this differs from text/scanner.Scanner in that string tokens will be unquoted.
func Lex(filename string, r io.Reader) Lexer {
s := &scanner.Scanner{}
s.Init(r)
lexer := lexWithScanner(filename, s)
lexer.scanner.Error = func(s *scanner.Scanner, msg string) {
lexer.err = errorf(Position(lexer.scanner.Pos()), "%s", msg)
}
return lexer
}
// LexWithScanner creates a Lexer from a user-provided scanner.Scanner.
//
// Useful if you need to customise the Scanner.
func LexWithScanner(filename string, scan *scanner.Scanner) Lexer {
return lexWithScanner(filename, scan)
}
func lexWithScanner(filename string, scan *scanner.Scanner) *textScannerLexer {
scan.Filename = filename
lexer := &textScannerLexer{
filename: filename,
scanner: scan,
}
return lexer
}
// LexBytes returns a new default lexer over bytes.
func LexBytes(filename string, b []byte) Lexer {
return Lex(filename, bytes.NewReader(b))
}
// LexString returns a new default lexer over a string.
func LexString(filename, s string) Lexer {
return Lex(filename, strings.NewReader(s))
}
func (t *textScannerLexer) Next() (Token, error) {
typ := t.scanner.Scan()
text := t.scanner.TokenText()
pos := Position(t.scanner.Position)
pos.Filename = t.filename
if t.err != nil {
return Token{}, t.err
}
return Token{
Type: TokenType(typ),
Value: text,
Pos: pos,
}, nil
}
|
