1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
|
User-agent: *
Disallow: /em/
Disallow: /latest/
Disallow: /stats/
Disallow: /go/
Disallow: /wl/
Disallow: /g/
Disallow: /commonphp/
Disallow: /wp-admin/
Disallow: /thanks-for-subscribing/
Disallow: /131490/
Disallow: /167877/
Disallow: /178022/
Disallow: /178645/
Disallow: /wp-content/uploads/contentframe/
Allow: /
#
# Yahoo bot is evil.
#
User-agent: Slurp
Disallow: /
#
# Wut? 80 legs? Where do I get traffic from this?
#
User-agent: 008
Disallow: /
#
# Yahoo Pipes is for feeds not web pages.
#
User-agent: Yahoo Pipes 1.0
Disallow: /
#
# There's no need to scan the forums for images
#
User-agent: Googlebot-Image
Disallow: /forum/
Sitemap: https://www.howtogeek.com/sitemap.xml
|
