1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
# If you add a new exclusion to this file please ensure that route is removed
# from the sitemap.xml.
# The songkick.com sitemaps are generated by this project through cron
# regularly, and stored in S3 via media-service whence they can be served
# directly by apache. see the README for more
User-agent: 008
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: BeetleBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: CrystalSemanticsBot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: rogerbot
Disallow: /
User-agent: EasouSpider
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: grapeshot
Disallow: /
User-agent: import.io
Disallow: /
User-agent: MaxPointCrawler
Disallow: /
User-agent: netseer
Disallow: /
User-agent: SEOkicks-Robot
Disallow: /
User-agent: Sogou web spider
Disallow: /
User-agent: YoudaoBot
Disallow: /
User-agent: Blekkobot
Crawl-delay: 10
User-agent: SeznamBot
Crawl-delay: 10
User-agent: YandexBot
Crawl-delay: 10
User-agent: *
Disallow: /44522520/*
Disallow: /announcements/seen
Disallow: /artists/*/images/new
Disallow: /artists/*/videos/new
Disallow: /behaviour
Disallow: /concerts/*/edit
Disallow: /concerts/*/images/new
Disallow: /concerts/*/posters/new
Disallow: /concerts/*/setlists/*
Disallow: /concerts/*/videos/new
Disallow: /concerts/new
Disallow: /festival_instances/new
Disallow: /festivals/*/edit
Disallow: /festivals/*/id/*/images/new
Disallow: /festivals/*/id/*/posters/new
Disallow: /festivals/*/id/*/setlists/*
Disallow: /festivals/*/id/*/videos/new
Disallow: /images/new
Disallow: /images/api/
Disallow: /iphone/
Disallow: /light_signups/new
Disallow: /password_reset_requests
Disallow: /posters/new
Disallow: /problems/new
Disallow: /record-share
Disallow: /rx
Disallow: /search
Disallow: /session/create
Disallow: /session/filter_metro_area/edit
Disallow: /session/new
Disallow: /store/events/*/unavailable_allocations
Disallow: /store/reservation_orders*
Disallow: /store/payments*
Disallow: /store/confirmation
Disallow: /tickets/
Disallow: /tracker
Disallow: /trackings
Disallow: /venues/*/edit
Disallow: /venues/*/images/new
Allow: /users/*/calendars.ics
Disallow: /venues/*/videos/new
Disallow: /venues/new
Disallow: /videos/new
Disallow: /harming/humans
Disallow: /ignoring/human/orders
Disallow: /harm/to/self
Disallow: /artists/9115824
Disallow: /artists/9115824-songkickloadtesting1
Disallow: /events/30168404
Disallow: /events/30168429
Disallow: /events/30168489
Disallow: /events/30168499
Disallow: /events/30168504
Disallow: /events/30168539
Disallow: /events/30168549
Disallow: /events/30168554
Disallow: /events/30168559
Disallow: /events/30168564
Disallow: /events/30181959
Disallow: /events/30181969
Disallow: /events/30181984
Disallow: /events/30181994
Disallow: /events/30182004
Disallow: /events/30182014
Disallow: /events/30182029
Disallow: /events/30182039
Disallow: /events/30182044
Disallow: /events/30182049
Disallow: /events/30181949
Disallow: /events/30181964
Disallow: /events/30181989
Disallow: /events/30181999
Disallow: /events/30182009
Disallow: /events/30182034
Disallow: /events/30182249
Disallow: /events/30182254
Disallow: /events/30182264
Disallow: /events/30182274
Disallow: /events/30182284
Disallow: /events/30182289
Disallow: /events/30182294
Disallow: /events/30182559
Disallow: /events/30182584
Disallow: /events/30182594
Disallow: /events/30182644
Disallow: /events/30182559
Disallow: /events/30182694
Disallow: /events/30182774
Sitemap: https://www.songkick.com/sitemap.xml
|
