1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
|
User-agent: *
Disallow: /gbuk/*send-a-friend
Disallow: /gbuk/*?intcmp
Disallow: /gbuk/*price-asc/
Disallow: /gbuk/*price-desc/
Disallow: /gbuk/*brand-asc/
Disallow: /gbuk/*brand-desc/
Disallow: /gbuk/*rating-desc/
Disallow: /gbuk/*relevance-desc/
Disallow: /gbuk/*discount-desc/
Disallow: /gbuk/*search-keywords/
Disallow: /gbuk/*compare/
Disallow: /gbuk/*?sp=fr
Disallow: /*?do=json
Disallow: /*xx_5-6
Disallow: /*xx_6-7
Disallow: /*xx_4-5
Disallow: /*xx_5-7
Disallow: /*xx_7-8
Disallow: /*xx_20-21
Disallow: /*xx_0-4
Disallow: /*xx_xx/xx_xx
Disallow: /*xx_xx_xx_xx_
Disallow: *-/xx_xx_xx
User-agent: TurnitinBot
User-agent: ConveraCrawler
User-agent: QuepasaCreep
User-agent: Jetbot
Disallow: /
Sitemap: https://www.pcworld.co.uk/sitemap.xml
|
