1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
User-agent: *
#favorite routes
Disallow: /block-follow-user
#Disallow: /follow-user
Disallow: /unfollow-user
#forum routes
Disallow: /forum-check-title
Disallow: /forum-post-vote
Disallow: /forum-subscribe
Disallow: /forum-thread-check-title
Disallow: /forum-unsubscribe
Disallow: /forum-upload-post-photo
#page routes
Disallow: /full-website
Disallow: /mailing-list-subscribe
#partner routes
Disallow: /legitscript-approved-urls
Disallow: /rss
Disallow: /search-widget
#plugin routes
Disallow: /hide-all-extension-prompt
Disallow: /hide-footer-extension-prompt
Disallow: /hide-header-extension-prompt
Disallow: /hide-social-extension-prompt
#review routes
Disallow: /check-existing-review
Disallow: /check-order-id
#Disallow: /review-toggle-vote
Disallow: /set-fingerprint
Disallow: /share-fb
Disallow: /upload-review-photo
#url routes (non-pages)
#Disallow: /answer-toggle-vote
Disallow: /check-phone
Disallow: /check-url
#Disallow: /register-pageview
Disallow: /share-url
Disallow: /track-lead
Disallow: /track-search
Disallow: /track-url-suggestion-click
Disallow: /track-url-suggestion-view
#Disallow: /upload-url-answer-photo
Disallow: /upload-url-photo
Disallow: /test
#user routes
#Disallow: /block-user
Disallow: /check-email
Disallow: /check-email-preconditions
Disallow: /check-password
Disallow: /dismiss-url
Disallow: /keep-alive
Disallow: /track-click
#Disallow: /unblock-user
Disallow: /upload-photo
Disallow: /visit-thank-yous-tab
Disallow: /visit-votes-tab
# misc
Disallow: /post-purchase-demo.html
#adult content
Disallow: /search?q=adult
Crawl-delay: 1
User-agent: Fasterfox
Disallow: /
User-agent: Nutch
Disallow: /
User-agent: spock
Disallow: /
User-agent: OmniExplorer_Bot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: BecomeBot
Disallow: /
User-agent: genieBot
Disallow: /
User-agent: MLBot
Disallow: /
User-agent: 80bot
Disallow: /
|
