1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
|
User-agent: *
Disallow: /textpattern/
Disallow: /forum/adm/
Disallow: /forum/download/
Disallow: /forum/images/
Disallow: /forum/includes/
Disallow: /forum/language/
Disallow: /forum/styles/
Disallow: /forum/common.php
Disallow: /forum/config.php
Disallow: /forum/cron.php
Disallow: /forum/faq.php
Disallow: /forum/mcp.php
Disallow: /forum/memberlist.php
Disallow: /forum/posting.php
Disallow: /forum/report.php
Disallow: /forum/search.php
Disallow: /forum/ucp.php
Disallow: /forum/viewonline.php
Disallow: /openid
Disallow: /babelzilla.php
Disallow: /getSubscription
Disallow: /403.html
Disallow: /_include/
User-agent: 008
Disallow: /
User-agent: Linguee
Disallow: /
User-agent: mxbot
Disallow: /
User-agent: ip-web-crawler.com
Disallow: /
|
