1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
---input---
\documentclass{article}
% Comment
\ExplSyntaxOn
\@@_command:nTF { a } { b } { c }
\ExplSyntaxOff
\makeatletter
\def\example@command{example}
\makeatother
\begin{document}
text \LaTeX: text \
text \example{}[] text \\
text \example text. ``quote''; \123\% and: text.
$\alpha_i\in\{\alpha:\alpha\leq3\}\ \text{something}$
\end{document}
---tokens---
'\\documentclass' Keyword
'{' Name.Builtin
'article' Text
'}' Name.Builtin
'\n\n' Text
'% Comment\n' Comment
'\n' Text
'\\ExplSyntaxOn' Keyword
'\n ' Text
'\\@@_command:nTF' Keyword
' ' Text
'{' Name.Builtin
' a ' Text
'}' Name.Builtin
' ' Text
'{' Name.Builtin
' b ' Text
'}' Name.Builtin
' ' Text
'{' Name.Builtin
' c ' Text
'}' Name.Builtin
'\n' Text
'\\ExplSyntaxOff' Keyword
'\n\n' Text
'\\makeatletter' Keyword
'\n' Text
'\\def' Keyword
'\\example@command' Keyword
'{' Name.Builtin
'example' Text
'}' Name.Builtin
'\n' Text
'\\makeatother' Keyword
'\n\n' Text
'\\begin' Keyword
'{' Name.Builtin
'document' Text
'}' Name.Builtin
'\n\ntext ' Text
'\\LaTeX:' Keyword
' text ' Text
'\\' Keyword
'\n\ntext ' Text
'\\example' Keyword
'{' Name.Builtin
'}' Name.Builtin
'[] text ' Text
'\\\\' Keyword
'\n\ntext ' Text
'\\example' Keyword
" text. ``quote''; " Text
'\\1' Keyword
'23' Text
'\\%' Keyword
' and: text.\n\n' Text
'$' Literal.String
'\\alpha' Name.Variable
'_' Name.Builtin
'i' Name.Builtin
'\\in' Name.Variable
'\\{' Name.Variable
'\\alpha' Name.Variable
':' Name.Builtin
'\\alpha' Name.Variable
'\\leq' Name.Variable
'3' Literal.Number
'\\}' Name.Variable
'\\' Name.Variable
' ' Name.Builtin
'\\text' Name.Variable
'{' Name.Builtin
'something' Name.Builtin
'}' Name.Builtin
'$' Literal.String
'\n\n' Text
'\\end' Keyword
'{' Name.Builtin
'document' Text
'}' Name.Builtin
'\n' Text
|
