1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
|
# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism
User-agent: *
Disallow: /anime
Disallow: /bin
Disallow: /covers
Disallow: /design
Disallow: /freedvd
Disallow: /graphics
Disallow: /ie6update
Disallow: /images
Disallow: /img
Disallow: /lightboximage
Disallow: /oxman
Disallow: /sample
Disallow: /size-chart
Disallow: /uploadimage
Disallow: /western
Disallow: /app_data
Disallow: /css
Disallow: /faq
Disallow: /headerfolder
Disallow: /js
Disallow: /Xmlmailcontent
Disallow: /aspnet_client
Disallow: /component
Disallow: /xmlpagecontext
Disallow: /addlist.aspx
Disallow: /addprod.aspx
Disallow: /cartsignin.aspx
Disallow: /checkoutnow.aspx
Disallow: /customer_info.aspx
Disallow: /error_found.aspx
Disallow: /getpassword.aspx
Disallow: /login.aspx
Disallow: /logout.aspx
Disallow: /m_billing.aspx
Disallow: /m_password.aspx
Disallow: /m_payment.aspx
Disallow: /managersubscription.aspx
Disallow: /multiss.aspx
Disallow: /myaccount.aspx
Disallow: /n_billing.aspx
Disallow: /n_payment.aspx
Disallow: /order_confirmed.aspx
Disallow: /order_history.aspx
Disallow: /printorder.aspx
Disallow: /receipt.aspx
Disallow: /register.aspx
Disallow: /subscrible_email.aspx
Disallow: /subscription.aspx
Disallow: /view_bigcover.aspx
Disallow: /view_photo.aspx
Disallow: /ppv/addclips.aspx
Disallow: /ppv/download.aspx
Disallow: /ppv/mydownload.aspx
Disallow: /ppv/ppv_addlist.aspx
Disallow: /ppv/ppv_cartsignin.aspx
Disallow: /ppv/ppv_checkout.aspx
Disallow: /ppv/ppv_orderhistory.aspx
Disallow: /ppv/ppv_printorder.aspx
Disallow: /ppv/ppv_receipt.aspx
Disallow: /ppv/thankyou.aspx
|
