1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
User-agent: *
Disallow: /accounts
Disallow: /admin
Disallow: /campaign
Disallow: /captcha
Disallow: /common
Disallow: /download
Disallow: /editor
Disallow: /images/external
Disallow: /favorite
Disallow: /get
Disallow: /mailing
Disallow: /more
Disallow: /my
Disallow: /relaunch
Disallow: /istockphoto
Disallow: /translate
Disallow: /images/screenshots
Allow: /
User-agent: AhrefsBot
User-agent: dotbot
User-agent: exabot
User-agent: gigabot
User-agent: ia_archiver
User-agent: MJ12bot
User-agent: 007ac9
disallow: /
Allow: /
|
