1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
|
User-agent: *
Disallow: /apps
Disallow: /discussions
Disallow: /cgi-bin
Disallow: /search
Disallow: /sendit
Disallow: /soundoff
Disallow: /resources/inc500.5172005
Disallow: /resources/inc500.20050819
Disallow: /resources/inc500.postDART
Disallow: /resources/inc500.preDART
Disallow: /resources/inc500.tmp
Disallow: /inctv/video/featured-video
Disallow: /inctv/video/featured-audio
Disallow: /inc5000dev
Disallow: /adops
Disallow: /adsales
Disallow: /atlas
Disallow: /brandnet
Disallow: /cache
Disallow: /clients
Disallow: /conf
Disallow: /doubleclick
Disallow: /lib
Disallow: /models
Disallow: /modules
Disallow: /users
Disallow: /lib/twitter/start.php
#Disallow robots
User-Agent: Googlebot
Allow: /
User-agent: Twitterbot
Disallow:
User-Agent: Googlebot-Mobile
Allow: /
User-Agent: msnbot
Allow: /
User-Agent: bingbot
Allow: /
# Adsense
User-Agent: Mediapartners-Google
Allow: /
# Blekko
User-Agent: ScoutJet
Allow: /
User-Agent: Yandex
Allow: /
User-agent: baiduspider
Allow: /
User-agent: DuckDuckBot
Allow: /
User-agent: BomboraBot
Allow: /
User-agent: grapeshot
Allow: /
# CommonCrawl
User-agent: ccbot
Allow: /
User-Agent: *
Disallow: /
Sitemap: http://www.inc.com/sitemap/sm_1.xml
Sitemap: http://www.inc.com/sitemap/sm_2.xml
Sitemap: http://www.inc.com/sitemap/sm_5.xml
Sitemap: http://www.inc.com/sitemap/sm_6.xml
Sitemap: http://www.inc.com/sitemap/sm_7.xml
Sitemap: http://www.inc.com/sitemap/sm_415.xml
Sitemap: http://www.inc.com/sitemap/sm_468.xml
Sitemap: http://www.inc.com/sitemap/sm_n.xml
|
