1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
#robots_txt_lt_english.php
User-agent: Twitterbot
Disallow:
Allow: /work/book/*
User-agent: 008
Disallow: /
User-agent: Sosospider
Disallow: /
User-agent: *
Crawl-delay: 2
Disallow: /ajax_*
Disallow: /ajaxinc_*
Disallow: /api_*
Disallow: /api-*
Disallow: /api-*
Disallow: /api/
Disallow: /api/*
Disallow: /catalog_bottom
Disallow: /catalog_bottom*
Disallow: /catalog_top
Disallow: /catalog_top*
Disallow: /catalog/
Disallow: /catalog.php
Disallow: /combine.php*
Disallow: /combine.php
Disallow: /author_split.php
Disallow: /author_split.php*
Disallow: /author_claim.php
Disallow: /author_cloud.php
Disallow: /authormerge.php
Disallow: /tagcloud.php
Disallow: /dotimer.php
Disallow: /files/
Disallow: /inc_*
Disallow: /ajaxinc_*
Disallow: /ajax_*
Disallow: /isbn/
Disallow: /ISBN/
Disallow: /ltw/
Disallow: /ltw*
Disallow: /m/
Disallow: /MDS/
Disallow: /readmessage.php*
Disallow: /search_works*
Disallow: /search*
Disallow: /title/
Disallow: /work/book/
Disallow: /work/*/book
Disallow: /work/*/edit
Disallow: /work/*/covers
Disallow: /work/*/workdetails
Disallow: /work/*/details
Disallow: /work/*/reviews
Disallow: /work/*/recommendations
Disallow: /work/*/members
Disallow: /work/*/descriptions
Disallow: /work/*/members
Disallow: /work/*/summary
Disallow: /work/*/conversations
Disallow: /work/*/commonknowledge
Disallow: /work/*/popularity
Disallow: /work/*/editions
Disallow: /work/book/*
Disallow: /work/*/getlocal
Disallow: /work/*/quicklinks
Disallow: /work/*/get
Disallow: /work/*/stats
Disallow: /author/*/stats
Disallow: /author/*/readalikes
Disallow: /author/*/names
Disallow: /profile/*/recommendations
Disallow: /profile/*/readalikes
Disallow: /profile/*/recommendations/member
Disallow: /profile/*/recommendations/memberbooks
Disallow: /profile/*/recommendations/bymember
Disallow: /profile/*/recommendations/unsuggester
Disallow: /profile_reviews.php
Disallow: /profile/*/stats/library
Disallow: /profile/*/stats/*
Disallow: /profile/tagcloud/*
Disallow: /profile/authorgallery/*
Disallow: /profile/tagmirror/*
Disallow: /review/
Disallow: /connect
Disallow: /widget
Disallow: /tagcloud/*
Disallow: /authorgallery.php
Disallow: /home/*
Disallow: /work-info.php
Disallow: /work-info/
Disallow: /rss/tags/
Disallow: /commonknowledge/search.php*
Disallow: /matrix/*
Disallow: /la/*
Disallow: /forlibraries/*
Disallow: /profile/gravlady
Disallow: /work/12275901
Disallow: /work/15929672
Disallow: /author/caitlinradano
Disallow: /signup_twitter.php
Disallow: /signup_fb.php
Disallow: /signup_verify.php
User-agent: bingbot
Crawl-delay: 0
Sitemap: http://www.librarything.com/siteindex.xml
|
