1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
|
User-agent: Twitterbot
Disallow: /
User-agent: *
Disallow: /core/
Disallow: /cgi-bin/
Disallow: /includes/
Disallow: /flash/
Disallow: /plugins/
Disallow: /system/
Disallow: /widgets/
Disallow: /assets/
Disallow: /blogs/wp-content/
Disallow: /blog-galleries/
Disallow: /blogs/
Allow: /sitemap/
Allow: /api/
Allow: /static/js/
Allow: /static/css/
Allow: /js/
Allow: /css/
Allow: /tv/
Sitemap: https://www.complex.com/sitemap/news.xml
Sitemap: https://www.complex.com/sitemap/index.xml
|
