File: www.sitepoint.com

package info (click to toggle)
python-protego 0.5.0%2Bdfsg-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 30,052 kB
  • sloc: python: 1,579; perl: 190; cpp: 33; sh: 4; makefile: 3
file content (106 lines) | stat: -rw-r--r-- 3,123 bytes parent folder | download | duplicates (4)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
# References:
# https://en.wikipedia.org/wiki/Robots.txt
# https://support.google.com/webmasters/answer/1061943?hl=en
# https://blogs.bing.com/webmaster/2012/05/03/to-crawl-or-not-to-crawl-that-is-bingbots-question/

Sitemap: https://www.sitepoint.com/sitemap_index.xml

User-agent: *
Disallow: /?s=*
Disallow: /cgi-bin/
Disallow: /login/
Disallow: /trackback/
Disallow: /wp-admin/
Disallow: /wp-content/
Allow: /wp-content/themes/sitepoint/assets/
Disallow: /wp-includes/
Allow: /wp-includes/js/wp-embed.min.js
Allow: /wp-includes/js/wp-emoji-release.min.js
Disallow: /xmlrpc.php
Allow: /ads.txt
# SitePoint Premium-related
Disallow: /premium/admin/
Disallow: /premium/cert/
# Discourse-related
Disallow: /community/auth/cas
Disallow: /community/auth/facebook/callback
Disallow: /community/auth/twitter/callback
Disallow: /community/auth/google/callback
Disallow: /community/auth/yahoo/callback
Disallow: /community/auth/github/callback
Disallow: /community/auth/cas/callback
Disallow: /community/assets/browser-update*.js
Disallow: /community/users/
Disallow: /community/u/
Disallow: /community/badges/
Disallow: /community/search
Disallow: /community/search/
Disallow: /community/tags
Disallow: /community/tags/
Disallow: /community/email/
Disallow: /community/session
Disallow: /community/session/
Disallow: /community/admin
Disallow: /community/admin/
Disallow: /community/user-api-key
Disallow: /community/user-api-key/
Disallow: /community/*?api_key*
Disallow: /community/*?*api_key*
Disallow: /community/groups
Disallow: /community/groups/
Disallow: /community/t/*/*.rss
Disallow: /community/tags/*.rss
Disallow: /community/c/*.rss
# Store
Disallow: /store/

# Adds a Crawl-delay of 30 to bingbot. Otherwise, identical to the
# above. Requested by Discourse.
User-agent: bingbot
Crawl-delay: 30
Disallow: /?s=*
Disallow: /cgi-bin/
Disallow: /login/
Disallow: /trackback/
Disallow: /wp-admin/
Disallow: /wp-content/
Allow: /wp-content/themes/sitepoint/assets/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /ads.txt
# SitePoint Premium-related
Disallow: /premium/admin/
Disallow: /premium/l/
Disallow: /premium/cert/
# Discourse-related
Disallow: /community/auth/cas
Disallow: /community/auth/facebook/callback
Disallow: /community/auth/twitter/callback
Disallow: /community/auth/google/callback
Disallow: /community/auth/yahoo/callback
Disallow: /community/auth/github/callback
Disallow: /community/auth/cas/callback
Disallow: /community/assets/browser-update*.js
Disallow: /community/users/
Disallow: /community/u/
Disallow: /community/badges/
Disallow: /community/search
Disallow: /community/search/
Disallow: /community/tags
Disallow: /community/tags/
Disallow: /community/email/
Disallow: /community/session
Disallow: /community/session/
Disallow: /community/admin
Disallow: /community/admin/
Disallow: /community/user-api-key
Disallow: /community/user-api-key/
Disallow: /community/*?api_key*
Disallow: /community/*?*api_key*
Disallow: /community/groups
Disallow: /community/groups/
Disallow: /community/t/*/*.rss
Disallow: /community/tags/*.rss
Disallow: /community/c/*.rss
# Store
Disallow: /store/