1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
Sitemap: https://arstechnica.com/sitemap.xml
# Google Image
User-agent: Googlebot-Image
Disallow:
Allow: /*
# Google AdSense
User-agent: Mediapartners-Google*
Disallow:
# Global
User-agent: *
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/
Disallow: /trackback/
Disallow: /comments/
Disallow: /category/*/*
Disallow: */trackback/
Disallow: */comments/
Allow: /wp-content/uploads/
# phpBB
Disallow: /civis/adm/
Disallow: /civis/mcp.php
Disallow: /civis/mcp.php*
Disallow: /civis/ucp.php
Disallow: /civis/ucp.php*
Disallow: /civis/download.php
Disallow: /civis/file.php
Disallow: /civis/images/
Disallow: /civis/includes/
Disallow: /civis/language/
Disallow: /civis/memberlist.php
Disallow: /civis/posting.php
Disallow: /civis/search.php
Disallow: /civis/styles/
Disallow: /civis/viewonline.php
Disallow: /civis/faq.php
Disallow: /civis/style.php
Disallow: /civis/style.php*
Disallow: /civis/memberlist.php*
Disallow: /civis/search.php*
Disallow: /civis/posting.php*
Disallow: /civis/report.php*
Disallow: /civis/download.php*
Disallow: /civis/file.php*
Disallow: /civis/*?sid=*
Disallow: /sponsored/
Disallow: /sponsors/
|
