1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
|
# 80legs
User-agent: 008
Disallow: /
User-Agent: *
Disallow: /*.json
Disallow: /*.json-compact
Disallow: /*.json-html
Disallow: /*.mobile
Disallow: /*.compact
Disallow: /*.rss
Disallow: /*.i
Disallow: /*.embed
Disallow: /v/*/submit
Disallow: /messages/compose*
Disallow: /api
Disallow: /post
Disallow: /submit
Disallow: /goto
Disallow: /*page=
Disallow: /domains/*
Disallow: /login
Disallow: /subverses/search
Disallow: /search
Disallow: /v/*/search
Allow: /
|
