1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
|
# tells Google not to include these URLs in the index even if they have already been crawled
User-Agent: Googlebot
Noindex: *d=pro*
Noindex: /dashboard*
Noindex: /signup*
Noindex: /feed*
Noindex: /login*
Noindex: /hootlet/social-share*
Noindex: /hootlet/load*
Noindex: /social-connect/send*
Noindex: /share-button*
Noindex: /app/social-network/*
Noindex: /ajax*
Noindex: /sso-redirect*
Noindex: /js*
# tells all engines not to crawl these URLs
User-Agent: *
Disallow: *d=pro*
Disallow: /dashboard*
Disallow: /signup*
Disallow: /feed*
Disallow: /login*
Disallow: /hootlet/social-share*
Disallow: /hootlet/load*
Disallow: /social-connect/send*
Disallow: /share-button*
Disallow: /app/social-network/*
Disallow: /ajax*
Disallow: /sso-redirect*
Disallow: /js*
# Don't allow web crawlers to index Craft CMS
User-agent: *
Noindex: /craft/
Disallow: /craft/
Noindex: /*.pdf$
Disallow: /*.pdf$
Sitemap: https://hootsuite.com/sitemap.xml
|
