1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
|
---input---
foo = "this string should" # not extend into the comment that contains "quotes"
bar = 'same with a' # basic 'string'
baz = """same with a""" # multiline """basic string"""
spam = '''same with a''' # multiline '''literal string'''
---tokens---
'foo' Name
' ' Text.Whitespace
'=' Operator
' ' Text.Whitespace
'"' Literal.String.Double
'this string should' Literal.String.Double
'"' Literal.String.Double
' ' Text.Whitespace
'# not extend into the comment that contains "quotes"' Comment.Single
'\n' Text.Whitespace
'bar' Name
' ' Text.Whitespace
'=' Operator
' ' Text.Whitespace
"'" Literal.String.Single
"same with a'" Literal.String.Single
' ' Text.Whitespace
"# basic 'string'" Comment.Single
'\n' Text.Whitespace
'baz' Name
' ' Text.Whitespace
'=' Operator
' ' Text.Whitespace
'"""' Literal.String.Double
'same with a' Literal.String.Double
'"""' Literal.String.Double
' ' Text.Whitespace
'# multiline """basic string"""' Comment.Single
'\n' Text.Whitespace
'spam' Name
' ' Text.Whitespace
'=' Operator
' ' Text.Whitespace
"'''" Literal.String.Single
'same with a' Literal.String.Single
"'''" Literal.String.Single
' ' Text.Whitespace
"# multiline '''literal string'''" Comment.Single
'\n' Text.Whitespace
|
