1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
|
# This file is managed through WordPress - master site - robots_txt page
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /global-en/dotcom/
Disallow: /*/quickstart/$
Disallow: /*/search/
Disallow: /mobile-z5/
Disallow: /*/mobile-z5/
Disallow: /services/footer
Disallow: /services/example
Disallow: /spg.jsp
Disallow: /global-en/spg.jsp
Disallow: /se/spg.jsp
Disallow: /ch-fr/spg.jsp
Disallow: /au/spg.jsp
Disallow: /it/spg.jsp
User-agent:
User-agent: Nutch
Disallow: /
|
