1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
---input---
an_int = 42
a_hex_int = 0xC001
a_string = "cool"
a_list = ["put", "stuff", "here"]
a_map = {key1: "value1", key2: ["value2"]}
test_append = ["a", "b"]
test_append += ["c", "d"]
multiline_list = [
"snap",
// With comments
"crackle",
/* more comments */ "pop",
]
---tokens---
'an_int' Name.Variable
' ' Text.Whitespace
'=' Operator
' ' Text.Whitespace
'42' Literal.Number.Integer
'\n' Text.Whitespace
'\n' Text.Whitespace
'a_hex_int' Name.Variable
' ' Text.Whitespace
'=' Operator
' ' Text.Whitespace
'0xC001' Literal.Number.Hex
'\n' Text.Whitespace
'\n' Text.Whitespace
'a_string' Name.Variable
' ' Text.Whitespace
'=' Operator
' ' Text.Whitespace
'"cool"' Literal.String
'\n' Text.Whitespace
'\n' Text.Whitespace
'a_list' Name.Variable
' ' Text.Whitespace
'=' Operator
' ' Text.Whitespace
'[' Punctuation
'"put"' Literal.String
',' Punctuation
' ' Text.Whitespace
'"stuff"' Literal.String
',' Punctuation
' ' Text.Whitespace
'"here"' Literal.String
']' Punctuation
'\n' Text.Whitespace
'\n' Text.Whitespace
'a_map' Name.Variable
' ' Text.Whitespace
'=' Operator
' ' Text.Whitespace
'{' Punctuation
'key1' Name
':' Punctuation
' ' Text.Whitespace
'"value1"' Literal.String
',' Punctuation
' ' Text.Whitespace
'key2' Name
':' Punctuation
' ' Text.Whitespace
'[' Punctuation
'"value2"' Literal.String
']' Punctuation
'}' Punctuation
'\n' Text.Whitespace
'\n' Text.Whitespace
'test_append' Name.Variable
' ' Text.Whitespace
'=' Operator
' ' Text.Whitespace
'[' Punctuation
'"a"' Literal.String
',' Punctuation
' ' Text.Whitespace
'"b"' Literal.String
']' Punctuation
'\n' Text.Whitespace
'test_append' Name.Variable
' ' Text.Whitespace
'+=' Operator
' ' Text.Whitespace
'[' Punctuation
'"c"' Literal.String
',' Punctuation
' ' Text.Whitespace
'"d"' Literal.String
']' Punctuation
'\n' Text.Whitespace
'\n' Text.Whitespace
'multiline_list' Name.Variable
' ' Text.Whitespace
'=' Operator
' ' Text.Whitespace
'[' Punctuation
'\n ' Text.Whitespace
'"snap"' Literal.String
',' Punctuation
'\n ' Text.Whitespace
'// With comments' Comment.Single
'\n ' Text.Whitespace
'"crackle"' Literal.String
',' Punctuation
'\n ' Text.Whitespace
'/* more comments */' Comment.Multiline
' ' Text.Whitespace
'"pop"' Literal.String
',' Punctuation
'\n' Text.Whitespace
']' Punctuation
'\n' Text.Whitespace
|
