1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
# allow the google advertising bot full access
User-agent: bingbot
Crawl-delay: 1
User-agent: Mediapartners-Google
Allow: /
User-agent: TalkTalk Virus Alerts Scanning Engine
Disallow: /jobs/
User-agent: *
Disallow: /LearningCentre/Course/RequestDetails
Disallow: /linkclicked.html
Disallow: /api/
Disallow: /handlers/
# Courses
Disallow: /courses/api/
Disallow: /courses/errorlogging
Disallow: /courses/logging/logcourseview
# Salary Checker beta
Disallow: /salary-checker
# Sitemap URL:
Sitemap: https://www.reed.co.uk/sitemap
Sitemap: https://www.reed.co.uk/sitemaps/livejobs/sitemap_livejobs_index.xml
#Allow: WebSurge
|
