1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
|
User-agent: *
Disallow: /showprofile.php?action=get_history&id=*
Disallow: /gallery.php*
Disallow: /results.php
Disallow: /*layout=grid*
Disallow: /ext/
Disallow: /track_spokeo.php
Disallow: /outgoing/*
Disallow: /addineyeV2.html
Disallow: /DARTIframe.html
Disallow: /provider/*
Disallow: /blog/cgi-bin/
Disallow: /blog/wp-admin/
Disallow: /blog/wp-content/
Disallow: /blog/wp-includes/
Disallow: /blog/trackback/
Disallow: /blog/xmlrpc.php
Disallow: /blog?wptheme=
Disallow: /blog/transfer/
Disallow: /blog/tweets/
Disallow: /blog/mint/
Disallow: /web_results/*
Disallow: /tools/
Allow: /blog/tag/mint/
Allow: /blog/tag/feed/
Allow: /blog/wp-content/online/
User-agent: Exabot
Crawl-delay: 5
User-agent: 008
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: Yandex
Disallow: /
User-agent: Slurp
Crawl-delay: 30
|
