1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
|
# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /
User-Agent: *
Disallow: /collection/browse_results.php
Disallow: /collection/object.php
Disallow: /collection/artist.php
Disallow: /collection/theme.php
Disallow: /collection_lb/browse_results.php
Disallow: /collection_ge/browse_results.php
Disallow: /collection/search.php
Disallow: /search
Disallow: /visit/calendar/search
Disallow: /visit/calendar/filters
Disallow: /visit/calendar/films?day*
Disallow: /media/*
User-Agent: gsa-crawler
Allow: /
User-Agent: Googlebot
Disallow: /dist/robots.*.js
Disallow: /assets/robots-*.js
|
