1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
|
User-agent: Googlebot-Mobile
Allow: /mobile/
Allow: /touch/
User-agent: YahooSeeker/M1A1-R2D2
Allow: /mobile/
Allow: /touch/
Disallow: /_doc/*
Disallow: /_staging/*
Disallow: /_tech/*
Disallow: /_data/*
Disallow: /search/search.html
Disallow: /travel/search/search.html
#
# BEGIN FPINFOMART
# 2010 Feb 25
#
# Meltwater
User-agent: Meltwater
Disallow: /
User-agent: libwww
Disallow: /
User-agent: libwww-perl
Disallow: /
User-agent: mag-crawl
Disallow: /
# Moreover
User-agent: Moreoverbot
Disallow: /
User-agent: Moreover
Disallow: /
# Vocus
User-agent: Vocus
Disallow: /
# CustomScoop
User-agent: ScooperBot
Disallow: /
# CyberAlert
User-Agent: CyberAlert
Disallow: /
# CEDROM
User-agent: CEDROM
Disallow: /
User-agent: Newscan
Disallow: /
User-agent: Eureka
Disallow: /
User-agent: Europresse
Disallow: /
# Radian6
User-agent: Radian6
Disallow: /
User-agent: R6_FeedFetcher
Disallow: /
User-agent: R6_CommentReader
Disallow: /
# Spinn3r
User-agent: Spinn3r
Disallow: /
#
# END FPINFOMART
#
User-agent: *
Disallow: /_doc/*
Disallow: /_staging/*
Disallow: /_tech/*
Disallow: /_data/*
Disallow: /search/
Disallow: /{videoId}/
Disallow: /*/post.html
Disallow: /*/view.html
Disallow: /*/mailto.html
Disallow: /*/story_print.html
Disallow: /pdf/1-windsorbanks.pdf
Sitemap: http://www.vancouversun.com/SP6SitemapIndex.xml?CatalogIDs=VASN
Sitemap: http://www.vancouversun.com/scripts/NewsSiteMap.ashx?CatalogIDs=VASN
Sitemap: http://www.vancouversun.com/videos/oo-sitemapIndex.xml
Sitemap: http://www.vancouversun.com/touch/xml/sitemap.xml
|
