1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
|
# Allow Mediapartners-Google
User-agent: Mediapartners-Google
Allow: /
User-agent: *
Disallow: /training
Disallow: /discard
Disallow: /qa
Disallow: /contests/
Disallow: /startouch.html
Disallow: /startouch.jsont.html
Disallow: /startouch/
Disallow: /life/insidersguide/sponsored_sections/
# Disallow Specific Robots
User-agent: Meltwater
Disallow: /
User-agent: brokencontrollers.com
Disallow: /
Sitemap: https://www.thestar.com/web-sitemap/sitemap-web-index.xml
Sitemap: https://www.thestar.com/news-sitemap.xml
|
