File: tokenize.go

package info (click to toggle)
golang-github-aws-aws-sdk-go-v2 1.24.1-2~bpo12%2B1
  • links: PTS, VCS
  • area: main
  • in suites: bookworm-backports
  • size: 554,032 kB
  • sloc: java: 15,941; makefile: 419; sh: 175
file content (92 lines) | stat: -rw-r--r-- 2,290 bytes parent folder | download | duplicates (7)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
package ini

import (
	"strings"
)

func tokenize(lines []string) ([]lineToken, error) {
	tokens := make([]lineToken, 0, len(lines))
	for _, line := range lines {
		if len(strings.TrimSpace(line)) == 0 || isLineComment(line) {
			continue
		}

		if tok := asProfile(line); tok != nil {
			tokens = append(tokens, tok)
		} else if tok := asProperty(line); tok != nil {
			tokens = append(tokens, tok)
		} else if tok := asSubProperty(line); tok != nil {
			tokens = append(tokens, tok)
		} else if tok := asContinuation(line); tok != nil {
			tokens = append(tokens, tok)
		} // unrecognized tokens are effectively ignored
	}
	return tokens, nil
}

func isLineComment(line string) bool {
	trimmed := strings.TrimLeft(line, " \t")
	return strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, ";")
}

func asProfile(line string) *lineTokenProfile { // " [ type name ] ; comment"
	trimmed := strings.TrimSpace(trimProfileComment(line)) // "[ type name ]"
	if !isBracketed(trimmed) {
		return nil
	}
	trimmed = trimmed[1 : len(trimmed)-1] // " type name " (or just " name ")
	trimmed = strings.TrimSpace(trimmed)  // "type name" / "name"
	typ, name := splitProfile(trimmed)
	return &lineTokenProfile{
		Type: typ,
		Name: name,
	}
}

func asProperty(line string) *lineTokenProperty {
	if isLineSpace(rune(line[0])) {
		return nil
	}

	trimmed := trimPropertyComment(line)
	trimmed = strings.TrimRight(trimmed, " \t")
	k, v, ok := splitProperty(trimmed)
	if !ok {
		return nil
	}

	return &lineTokenProperty{
		Key:   strings.ToLower(k), // LEGACY: normalize key case
		Value: legacyStrconv(v),   // LEGACY: see func docs
	}
}

func asSubProperty(line string) *lineTokenSubProperty {
	if !isLineSpace(rune(line[0])) {
		return nil
	}

	// comments on sub-properties are included in the value
	trimmed := strings.TrimLeft(line, " \t")
	k, v, ok := splitProperty(trimmed)
	if !ok {
		return nil
	}

	return &lineTokenSubProperty{ // same LEGACY constraints as in normal property
		Key:   strings.ToLower(k),
		Value: legacyStrconv(v),
	}
}

func asContinuation(line string) *lineTokenContinuation {
	if !isLineSpace(rune(line[0])) {
		return nil
	}

	// includes comments like sub-properties
	trimmed := strings.TrimLeft(line, " \t")
	return &lineTokenContinuation{
		Value: trimmed,
	}
}