1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
|
#
# robots.txt for flightaware.com hosted by wlmor.hou.flightaware.com
#
#
# Specific unwanted clients
#
User-agent: UbiCrawler
Disallow: /
User-agent: DOC
Disallow: /
User-agent: Zao
Disallow: /
User-agent: sitecheck.internetseer.com
Disallow: /
User-agent: Zealbot
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: Fetch
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: linko
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: Microsoft.URL.Control
Disallow: /
User-agent: Xenu
Disallow: /
User-agent: larbin
Disallow: /
User-agent: libwww
Disallow: /
User-agent: ZyBORG
Disallow: /
User-agent: Download Ninja
Disallow: /
User-agent: grub-client
Disallow: /
User-agent: k2spider
Disallow: /
User-agent: NPBot
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: Irvine
Disallow: /
User-agent: 360Spider
Disallow: /
#
# Command line recursive requests as well as automated fetching from the non-
# exportable data is not acceptable.
#
# See:
# http://flightaware.com/about/termsofuse
# http://flightaware.com/commercial/flightxml/
#
User-agent: wget
Disallow: /account/
Disallow: /analysis/
Disallow: /include/
Disallow: /live/
Disallow: /resources/
Disallow: /map/
Disallow: /me/
Disallow: /photos/
#
# General robot rules
#
User-agent: *
Disallow: /account/
Disallow: /ajax/
Disallow: /bait/
Disallow: /errors/
Disallow: /include/
Disallow: /live/flight/id/
Disallow: /mapi/
Disallow: /me/
Disallow: /photos/upload.rvt
Disallow: /photos/upload
Disallow: /mp/
Disallow: /live/report.rvt
Disallow: /adsb/register
Disallow: /adsb/request
Disallow: /adsb/flightfeeder/terms
Disallow: /photos/crowdsource/
Disallow: /photos/crowdsource.rvt
#
# Stop Applebot from beating the crap out of ajax endpoints (specifically the
# static flight map one)
#
User-agent: Applebot
Disallow: /ajax/
User-agent: Cliqzbot
Disallow: /
# Allow Twitter to grab article blobs
User-agent: Twitterbot
Allow: /news/
|
