1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
|
# robots.txt
User-agent: *
Disallow: /AA/
Disallow: /Admin/
Disallow: /App_Data/
Disallow: /bin/
Disallow: /casgi/
Disallow: /Checkout/
Disallow: /Config/
Disallow: /Controls/
Disallow: /MySite/
Disallow: /a/my-gunbroker
Disallow: /Registration/
Disallow: /Test/
Disallow: /Tools/
Disallow: /User/images/
Disallow: /webhandler/
Disallow: /Auction/BrowseItems.aspx
Disallow: /Auction/SearchResults.aspx
Disallow: /BMP/
Disallow: /BannerAdTestDefault.aspx
Disallow: /Auction/BannerAdTestBrowseItems.aspx
Disallow: /Auction/Browse.aspx
Disallow: /Auction/AddToWatchList.aspx
Disallow: /Auction/SendMailToUser.aspx
Disallow: /Auction/SendMailToFriendForm.aspx
Disallow: /Auction/EditFeedbackForm.aspx
Disallow: /Auction/DeleteFeedbackForm.aspx
Disallow: /Auction/ReplyToFeedbackForm.aspx
Disallow: /Auction/searchform.aspx
Disallow: /Auction/ViewUserContactInfo.aspx
Disallow: /ConfirmBuyNow.aspx
Disallow: /LogoutClearCookies.aspx
Disallow: /FFL/ContactDealer.aspx
Disallow: /ShowLogin.aspx
Disallow: /Auction/ViewBidHistory.aspx
Disallow: /Auction/SendMailToUser.aspx
Sitemap: http://www.gunbroker.com/gsmi.xml
# MSN/BING sometimes hammers us.
# Crawl-delay setting 1=Slow, 5=Very slow, 10=Extremely slow
User-agent: msnbot
Crawl-delay: 1
# TurninBot is some crawler that searches the web for 'plagarism'
# This does not seem like a legitimate reason to waste our bandwidth and CPU time
User-agent: TurnitinBot
Disallow: /
|
