File: scanner.go

package info (click to toggle)
golang-github-ganigeorgiev-fexpr 0.5.0-2
  • links: PTS, VCS
  • area: main
  • in suites:
  • size: 104 kB
  • sloc: makefile: 2
file content (679 lines) | stat: -rw-r--r-- 16,055 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
package fexpr

import (
	"bytes"
	"fmt"
	"strings"
	"unicode/utf8"
)

// eof represents a marker rune for the end of the reader.
const eof = rune(0)

// JoinOp represents a join type operator.
type JoinOp string

// supported join type operators
const (
	JoinAnd JoinOp = "&&"
	JoinOr  JoinOp = "||"
)

// SignOp represents an expression sign operator.
type SignOp string

// supported expression sign operators
const (
	SignEq    SignOp = "="
	SignNeq   SignOp = "!="
	SignLike  SignOp = "~"
	SignNlike SignOp = "!~"
	SignLt    SignOp = "<"
	SignLte   SignOp = "<="
	SignGt    SignOp = ">"
	SignGte   SignOp = ">="

	// array/any operators
	SignAnyEq    SignOp = "?="
	SignAnyNeq   SignOp = "?!="
	SignAnyLike  SignOp = "?~"
	SignAnyNlike SignOp = "?!~"
	SignAnyLt    SignOp = "?<"
	SignAnyLte   SignOp = "?<="
	SignAnyGt    SignOp = "?>"
	SignAnyGte   SignOp = "?>="
)

// TokenType represents a Token type.
type TokenType string

// token type constants
const (
	TokenUnexpected TokenType = "unexpected"
	TokenEOF        TokenType = "eof"
	TokenWS         TokenType = "whitespace"
	TokenJoin       TokenType = "join"
	TokenSign       TokenType = "sign"
	TokenIdentifier TokenType = "identifier" // variable, column name, placeholder, etc.
	TokenFunction   TokenType = "function"   // function
	TokenNumber     TokenType = "number"
	TokenText       TokenType = "text"  // ' or " quoted string
	TokenGroup      TokenType = "group" // groupped/nested tokens
	TokenComment    TokenType = "comment"
)

// Token represents a single scanned literal (one or more combined runes).
type Token struct {
	Meta    interface{}
	Type    TokenType
	Literal string
}

// NewScanner creates and returns a new scanner instance loaded with the specified data.
func NewScanner(data []byte) *Scanner {
	return &Scanner{
		data:         data,
		maxFuncDepth: 3,
	}
}

// Scanner represents a filter and lexical scanner.
type Scanner struct {
	data         []byte
	pos          int
	maxFuncDepth int
}

// Scan reads and returns the next available token value from the scanner's buffer.
func (s *Scanner) Scan() (Token, error) {
	ch := s.read()

	if ch == eof {
		return Token{Type: TokenEOF, Literal: ""}, nil
	}

	if isWhitespaceRune(ch) {
		s.unread()
		return s.scanWhitespace()
	}

	if isGroupStartRune(ch) {
		s.unread()
		return s.scanGroup()
	}

	if isIdentifierStartRune(ch) {
		s.unread()
		return s.scanIdentifier(s.maxFuncDepth)
	}

	if isNumberStartRune(ch) {
		s.unread()
		return s.scanNumber()
	}

	if isTextStartRune(ch) {
		s.unread()
		return s.scanText(false)
	}

	if isSignStartRune(ch) {
		s.unread()
		return s.scanSign()
	}

	if isJoinStartRune(ch) {
		s.unread()
		return s.scanJoin()
	}

	if isCommentStartRune(ch) {
		s.unread()
		return s.scanComment()
	}

	return Token{Type: TokenUnexpected, Literal: string(ch)}, fmt.Errorf("unexpected character %q", ch)
}

// scanWhitespace consumes all contiguous whitespace runes.
func (s *Scanner) scanWhitespace() (Token, error) {
	var buf bytes.Buffer

	// Reads every subsequent whitespace character into the buffer.
	// Non-whitespace runes and EOF will cause the loop to exit.
	for {
		ch := s.read()

		if ch == eof {
			break
		}

		if !isWhitespaceRune(ch) {
			s.unread()
			break
		}

		// write the whitespace rune
		buf.WriteRune(ch)
	}

	return Token{Type: TokenWS, Literal: buf.String()}, nil
}

// scanNumber consumes all contiguous digit runes
// (complex numbers and scientific notations are not supported).
func (s *Scanner) scanNumber() (Token, error) {
	var buf bytes.Buffer

	var hadDot bool

	// Read every subsequent digit rune into the buffer.
	// Non-digit runes and EOF will cause the loop to exit.
	for {
		ch := s.read()

		if ch == eof {
			break
		}

		// not a digit rune
		if !isDigitRune(ch) &&
			// minus sign but not at the beginning
			(ch != '-' || buf.Len() != 0) &&
			// dot but there was already another dot
			(ch != '.' || hadDot) {
			s.unread()
			break
		}

		// write the rune
		buf.WriteRune(ch)

		if ch == '.' {
			hadDot = true
		}
	}

	total := buf.Len()
	literal := buf.String()

	var err error
	// only "-" or starts with "." or ends with "."
	if (total == 1 && literal[0] == '-') || literal[0] == '.' || literal[total-1] == '.' {
		err = fmt.Errorf("invalid number %q", literal)
	}

	return Token{Type: TokenNumber, Literal: buf.String()}, err
}

// scanText consumes all contiguous quoted text runes.
func (s *Scanner) scanText(preserveQuotes bool) (Token, error) {
	var buf bytes.Buffer

	// read the first rune to determine the quotes type
	firstCh := s.read()
	buf.WriteRune(firstCh)
	var prevCh rune
	var hasMatchingQuotes bool

	// Read every subsequent text rune into the buffer.
	// EOF and matching unescaped ending quote will cause the loop to exit.
	for {
		ch := s.read()

		if ch == eof {
			break
		}

		// write the text rune
		buf.WriteRune(ch)

		// unescaped matching quote, aka. the end
		if ch == firstCh && prevCh != '\\' {
			hasMatchingQuotes = true
			break
		}

		prevCh = ch
	}

	literal := buf.String()

	var err error
	if !hasMatchingQuotes {
		err = fmt.Errorf("invalid quoted text %q", literal)
	} else if !preserveQuotes {
		// unquote
		literal = literal[1 : len(literal)-1]
		// remove escaped quotes prefix (aka. \)
		firstChStr := string(firstCh)
		literal = strings.ReplaceAll(literal, `\`+firstChStr, firstChStr)
	}

	return Token{Type: TokenText, Literal: literal}, err
}

// scanComment consumes all contiguous single line comment runes until
// a new character (\n) or EOF is reached.
func (s *Scanner) scanComment() (Token, error) {
	var buf bytes.Buffer

	// Read the first 2 characters without writting them to the buffer.
	if !isCommentStartRune(s.read()) || !isCommentStartRune(s.read()) {
		return Token{Type: TokenComment}, ErrInvalidComment
	}

	// Read every subsequent comment text rune into the buffer.
	// \n and EOF will cause the loop to exit.
	for i := 0; ; i++ {
		ch := s.read()

		if ch == eof || ch == '\n' {
			break
		}

		buf.WriteRune(ch)
	}

	return Token{Type: TokenComment, Literal: strings.TrimSpace(buf.String())}, nil
}

// scanIdentifier consumes all contiguous ident runes.
func (s *Scanner) scanIdentifier(funcDepth int) (Token, error) {
	var buf bytes.Buffer

	// read the first rune in case it is a special start identifier character
	buf.WriteRune(s.read())

	// Read every subsequent identifier rune into the buffer.
	// Non-ident runes and EOF will cause the loop to exit.
	for {
		ch := s.read()

		if ch == eof {
			break
		}

		// func
		if ch == '(' {
			funcName := buf.String()
			if funcDepth <= 0 {
				return Token{Type: TokenFunction, Literal: funcName}, fmt.Errorf("max nested function arguments reached (max: %d)", s.maxFuncDepth)
			}
			if !isValidIdentifier(funcName) {
				return Token{Type: TokenFunction, Literal: funcName}, fmt.Errorf("invalid function name %q", funcName)
			}
			s.unread()
			return s.scanFunctionArgs(funcName, funcDepth)
		}

		// not an identifier character
		if !isLetterRune(ch) && !isDigitRune(ch) && !isIdentifierCombineRune(ch) && ch != '_' {
			s.unread()
			break
		}

		// write the identifier rune
		buf.WriteRune(ch)
	}

	literal := buf.String()

	var err error
	if !isValidIdentifier(literal) {
		err = fmt.Errorf("invalid identifier %q", literal)
	}

	return Token{Type: TokenIdentifier, Literal: literal}, err
}

// scanSign consumes all contiguous sign operator runes.
func (s *Scanner) scanSign() (Token, error) {
	var buf bytes.Buffer

	// Read every subsequent sign rune into the buffer.
	// Non-sign runes and EOF will cause the loop to exit.
	for {
		ch := s.read()

		if ch == eof {
			break
		}

		if !isSignStartRune(ch) {
			s.unread()
			break
		}

		// write the sign rune
		buf.WriteRune(ch)
	}

	literal := buf.String()

	var err error
	if !isSignOperator(literal) {
		err = fmt.Errorf("invalid sign operator %q", literal)
	}

	return Token{Type: TokenSign, Literal: literal}, err
}

// scanJoin consumes all contiguous join operator runes.
func (s *Scanner) scanJoin() (Token, error) {
	var buf bytes.Buffer

	// Read every subsequent join operator rune into the buffer.
	// Non-join runes and EOF will cause the loop to exit.
	for {
		ch := s.read()

		if ch == eof {
			break
		}

		if !isJoinStartRune(ch) {
			s.unread()
			break
		}

		// write the join operator rune
		buf.WriteRune(ch)
	}

	literal := buf.String()

	var err error
	if !isJoinOperator(literal) {
		err = fmt.Errorf("invalid join operator %q", literal)
	}

	return Token{Type: TokenJoin, Literal: literal}, err
}

// scanGroup consumes all runes within a group/parenthesis.
func (s *Scanner) scanGroup() (Token, error) {
	var buf bytes.Buffer

	// read the first group bracket without writing it to the buffer
	firstChar := s.read()
	openGroups := 1

	// Read every subsequent text rune into the buffer.
	// EOF and matching unescaped ending quote will cause the loop to exit.
	for {
		ch := s.read()

		if ch == eof {
			break
		}

		if isGroupStartRune(ch) {
			// nested group
			openGroups++
			buf.WriteRune(ch)
		} else if isTextStartRune(ch) {
			s.unread()
			t, err := s.scanText(true) // with quotes to preserve the exact text start/end runes
			if err != nil {
				// write the errored literal as it is
				buf.WriteString(t.Literal)
				return Token{Type: TokenGroup, Literal: buf.String()}, err
			}

			buf.WriteString(t.Literal)
		} else if ch == ')' {
			openGroups--

			if openGroups <= 0 {
				// main group end
				break
			} else {
				buf.WriteRune(ch)
			}
		} else {
			buf.WriteRune(ch)
		}
	}

	literal := buf.String()

	var err error
	if !isGroupStartRune(firstChar) || openGroups > 0 {
		err = fmt.Errorf("invalid formatted group - missing %d closing bracket(s)", openGroups)
	}

	return Token{Type: TokenGroup, Literal: literal}, err
}

// scanFunctionArgs consumes all contiguous function call runes to
// extract its arguments and returns a function token with the found
// Token arguments loaded in Token.Meta.
func (s *Scanner) scanFunctionArgs(funcName string, funcDepth int) (Token, error) {
	var args []Token

	var expectComma, isComma, isClosed bool

	ch := s.read()
	if ch != '(' {
		return Token{Type: TokenFunction, Literal: funcName}, fmt.Errorf("invalid or incomplete function call %q", funcName)
	}

	// Read every subsequent rune until ')' or EOF has been reached.
	for {
		ch := s.read()

		if ch == eof {
			break
		}

		if ch == ')' {
			isClosed = true
			break
		}

		// skip whitespaces
		if isWhitespaceRune(ch) {
			_, err := s.scanWhitespace()
			if err != nil {
				return Token{Type: TokenFunction, Literal: funcName, Meta: args}, fmt.Errorf("failed to scan whitespaces in function %q: %w", funcName, err)
			}
			continue
		}

		// skip comments
		if isCommentStartRune(ch) {
			s.unread()
			_, err := s.scanComment()
			if err != nil {
				return Token{Type: TokenFunction, Literal: funcName, Meta: args}, fmt.Errorf("failed to scan comment in function %q: %w", funcName, err)
			}
			continue
		}

		isComma = ch == ','

		if expectComma && !isComma {
			return Token{Type: TokenFunction, Literal: funcName, Meta: args}, fmt.Errorf("expected comma after the last argument in function %q", funcName)
		}

		if !expectComma && isComma {
			return Token{Type: TokenFunction, Literal: funcName, Meta: args}, fmt.Errorf("unexpected comma in function %q", funcName)
		}

		expectComma = false // reset

		if isComma {
			continue
		}

		if isIdentifierStartRune(ch) {
			s.unread()
			t, err := s.scanIdentifier(funcDepth - 1)
			if err != nil {
				return Token{Type: TokenFunction, Literal: funcName, Meta: args}, fmt.Errorf("invalid identifier argument %q in function %q: %w", t.Literal, funcName, err)
			}
			args = append(args, t)
			expectComma = true
		} else if isNumberStartRune(ch) {
			s.unread()
			t, err := s.scanNumber()
			if err != nil {
				return Token{Type: TokenFunction, Literal: funcName, Meta: args}, fmt.Errorf("invalid number argument %q in function %q: %w", t.Literal, funcName, err)
			}
			args = append(args, t)
			expectComma = true
		} else if isTextStartRune(ch) {
			s.unread()
			t, err := s.scanText(false)
			if err != nil {
				return Token{Type: TokenFunction, Literal: funcName, Meta: args}, fmt.Errorf("invalid text argument %q in function %q: %w", t.Literal, funcName, err)
			}
			args = append(args, t)
			expectComma = true
		} else {
			return Token{Type: TokenFunction, Literal: funcName, Meta: args}, fmt.Errorf("unsupported argument character %q in function %q", ch, funcName)
		}
	}

	if !isClosed {
		return Token{Type: TokenFunction, Literal: funcName, Meta: args}, fmt.Errorf("invalid or incomplete function %q (expected ')')", funcName)
	}

	return Token{Type: TokenFunction, Literal: funcName, Meta: args}, nil
}

// unread unreads the last character and revert the position 1 step back.
func (s *Scanner) unread() {
	if s.pos > 0 {
		s.pos = s.pos - 1
	}
}

// read reads the next rune and moves the position forward.
func (s *Scanner) read() rune {
	if s.pos >= len(s.data) {
		return eof
	}

	ch, n := utf8.DecodeRune(s.data[s.pos:])
	s.pos += n

	return ch
}

// Lexical helpers:
// -------------------------------------------------------------------

// isWhitespaceRune checks if a rune is a space, tab, or newline.
func isWhitespaceRune(ch rune) bool { return ch == ' ' || ch == '\t' || ch == '\n' }

// isLetterRune checks if a rune is a letter.
func isLetterRune(ch rune) bool {
	return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')
}

// isDigitRune checks if a rune is a digit.
func isDigitRune(ch rune) bool {
	return (ch >= '0' && ch <= '9')
}

// isTextStartRune checks if a rune is a valid quoted text first character
// (aka. single or double quote).
func isTextStartRune(ch rune) bool {
	return ch == '\'' || ch == '"'
}

// isNumberStartRune checks if a rune is a valid number start character (aka. digit).
func isNumberStartRune(ch rune) bool {
	return ch == '-' || isDigitRune(ch)
}

// isSignStartRune checks if a rune is a valid sign operator start character.
func isSignStartRune(ch rune) bool {
	return ch == '=' ||
		ch == '?' ||
		ch == '!' ||
		ch == '>' ||
		ch == '<' ||
		ch == '~'
}

// isJoinStartRune checks if a rune is a valid join type start character.
func isJoinStartRune(ch rune) bool {
	return ch == '&' || ch == '|'
}

// isGroupStartRune checks if a rune is a valid group/parenthesis start character.
func isGroupStartRune(ch rune) bool {
	return ch == '('
}

// isCommentStartRune checks if a rune is a valid comment start character.
func isCommentStartRune(ch rune) bool {
	return ch == '/'
}

// isIdentifierStartRune checks if a rune is valid identifier's first character.
func isIdentifierStartRune(ch rune) bool {
	return isLetterRune(ch) || isIdentifierSpecialStartRune(ch)
}

// isIdentifierSpecialStartRune checks if a rune is valid identifier's first special character.
func isIdentifierSpecialStartRune(ch rune) bool {
	return ch == '@' || ch == '_' || ch == '#'
}

// isIdentifierCombineRune checks if a rune is valid identifier's combine character.
func isIdentifierCombineRune(ch rune) bool {
	return ch == '.' || ch == ':'
}

// isSignOperator checks if a literal is a valid sign operator.
func isSignOperator(literal string) bool {
	switch SignOp(literal) {
	case
		SignEq,
		SignNeq,
		SignLt,
		SignLte,
		SignGt,
		SignGte,
		SignLike,
		SignNlike,
		SignAnyEq,
		SignAnyNeq,
		SignAnyLike,
		SignAnyNlike,
		SignAnyLt,
		SignAnyLte,
		SignAnyGt,
		SignAnyGte:
		return true
	}

	return false
}

// isJoinOperator checks if a literal is a valid join type operator.
func isJoinOperator(literal string) bool {
	switch JoinOp(literal) {
	case
		JoinAnd,
		JoinOr:
		return true
	}

	return false
}

// isValidIdentifier validates the literal against common identifier requirements.
func isValidIdentifier(literal string) bool {
	length := len(literal)

	return (
	// doesn't end with combine rune
	!isIdentifierCombineRune(rune(literal[length-1])) &&
		// is not just a special start rune
		(length != 1 || !isIdentifierSpecialStartRune(rune(literal[0]))))
}