1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
|
syntax robotstxt
list -i field user-agent: disallow: allow: crawl-delay: sitemap: host:
state main code
char -b A-Za-z firstword
char # comment
char " \t" this
noeat remainder
state firstword
char -b A-Za-z- this
char -b : field
noeat -b remainder
state field
inlist field value keyword
recolor error
noeat remainder
state value
char # comment
char "\n" main
eat this
state remainder error
recolor error
char # comment
char "\n" main
eat this
state comment
char "\n" main
eat this
|
