1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
|
User-agent: dotbot
Disallow: /
User-agent: Slurp
Crawl-delay: 5
User-agent: twiceler
Crawl-delay: 5
User-agent: ScoutJet
Disallow: /
User-agent: Fasterfox
Disallow: /
User-agent: *
# Disallow all unnecessary content from the search
Disallow: /iis/*
Disallow: /login/*
Disallow: /admin/*
Disallow: /map/*
Disallow: /email/*
Disallow: /my/*
Disallow: /gps/*
Disallow: /datastore/*
Disallow: /api/*
Disallow: /bait.asp
Disallow: /hotels/*
Sitemap: http://www.geocaching.com/sitemaps/sitemap-index.xml
|
