1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
|
##ACAP version=1.0
User-agent: Mediapartners-Google
Disallow:
User-agent: Googlebot
Disallow: /search/
Disallow: /*.ece$
Disallow: /*startindex=
Disallow: /*from=*
Disallow: /*service=Print
Disallow: /*action=Email
Disallow: /*comment_form
Disallow: /*r=RSS
Disallow: /*FORM=ZZNR3
Disallow: /*WT.mc_id=*
Disallow: /*ibPage=*
Disallow: /*start=*
Disallow: /sponsored-features/
Disallow: /ThreeTrending/
Disallow: /tablet/morning-edition/
Disallow: /tablet/
Disallow: /sponsored/
Disallow: /qwerty/
Disallow: /unsorted/
Disallow: /7380
User-agent: Bingbot
Disallow: /search/
Disallow: /*.ece$
Disallow: /*startindex=
Disallow: /*from=*
Disallow: /*service=Print
Disallow: /*action=Email
Disallow: /*comment_form
Disallow: /*r=RSS
Disallow: /*FORM=ZZNR3
Disallow: /*WT.mc_id=*
Disallow: /*ibPage=*
Disallow: /*start=*
Disallow: /sponsored-features/
Disallow: /ThreeTrending/
Disallow: /tablet/morning-edition/
Disallow: /tablet/
Disallow: /sponsored/
Disallow: /qwerty/
Disallow: /unsorted/
Disallow: /7380
#Disallow Specific Robots
User-agent: omgilibot/0.3
Disallow: /
User-agent: WebVac
Disallow: /
User-agent: WebZip
Disallow: /
User-agent: psbot
Disallow: /
User-agent: daumoa
Disallow: /
User-agent: Isposure Agent
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: *
Disallow:
Sitemap: https://www.independent.ie/sitemap.xml.gz
Sitemap: https://www.independent.ie/sitemap_googlenews.xml.gz
Sitemap: https://www.independent.ie/sitemap_channels.xml.gz
ACAP-crawler: *
ACAP-disallow-crawl: /search/
ACAP-disallow-crawl: /*.ece$
ACAP-disallow-crawl: /*startindex=
ACAP-disallow-crawl: /*from=*
ACAP-disallow-crawl: /*service=Print
ACAP-disallow-crawl: /*action=Email
ACAP-disallow-crawl: /*comment_form
ACAP-disallow-crawl: /*r=RSS
ACAP-disallow-crawl: /*FORM=ZZNR3
ACAP-disallow-crawl: /*WT.mc_id=*
ACAP-disallow-crawl: /*ibPage=*
ACAP-disallow-crawl: /*start=*
ACAP-disallow-crawl: /sponsored-features/
ACAP-disallow-crawl: /ThreeTrending/
ACAP-disallow-crawl: /tablet/
ACAP-disallow-crawl: /sponsored/
ACAP-disallow-crawl: /qwerty/
ACAP-disallow-crawl: /unsorted/
ACAP-disallow-crawl: /7380
|
