1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
|
# For all robots
User-agent: *
Allow: /
# Block access to specific groups of pages
# Folders
Disallow: /en/cart
Disallow: /en/checkout
Disallow: /en/my-account
Disallow: /en/search
Disallow: /en/compare
Disallow: /fr/cart
Disallow: /fr/checkout
Disallow: /fr/my-account
Disallow: /fr/search
Disallow: /fr/compare
# Parameters
Disallow: /*?b=*
Disallow: /*?org_text*
Disallow: /*?sort*
Disallow: /*?Ns*
Disallow: /*?h*
Disallow: /*?q*
Disallow: /*?No*
Crawl-delay: 5
# 5 seconds between page requests => Ignored by Googlebot => include this line only if you want to set up a crawl delay for other crawlers
# Allow search crawlers to discover the sitemap
Sitemap: https://www.mec.ca/sitemap.xml
# Block CazoodleBot as it does not present correct accept content headers
User-agent: CazoodleBot
Disallow: /
# Block MJ12bot as it is just noise
User-agent: MJ12bot
Disallow: /
# Block dotbot as it cannot parse base urls properly
User-agent: dotbot/1.0
Disallow: /
# Block Gigabot
User-agent: Gigabot
Disallow: /
|