1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
|
# BEGIN FPINFOMART
User-agent: CEDROM
Disallow: /
User-Agent: CyberAlert
Disallow: /
User-agent: Eureka
Disallow: /
User-agent: Europresse
Disallow: /
User-agent: libwww
Disallow: /
User-agent: libwww-perl
Disallow: /
User-agent: mag-crawl
Disallow: /
User-agent: Meltwater
Disallow: /
User-agent: Moreover
Disallow: /
User-agent: Moreoverbot
Disallow: /
User-agent: Newscan
Disallow: /
User-agent: R6_CommentReader
Disallow: /
User-agent: R6_FeedFetcher
Disallow: /
User-agent: Radian6
Disallow: /
User-agent: ScooperBot
Disallow: /
User-agent: Spinn3r
Disallow: /
User-agent: Vocus
Disallow: /
# END FPINFOMART
User-agent: *
Disallow: /search/
Disallow: /2010/07/17/sell-your-air-canada-stock
Disallow: /todays-paper/
Disallow: /news/story.html?id=9e919563-e44b-4ca2-9706-8af9cf743c95
Disallow: /scripts/story.html?id=2c07121b-85c2-4799-9aaf-0c2688bf5ca1&k=64805
Allow: /
|
