1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
|
User-agent: Googlebot # google
User-agent: Mediapartners-Google*
User-agent: ia_archiver # web.archive.org
User-agent: mozDex # open source :D
User-agent: bingbot
Crawl-Delay: 3600
Disallow: /action/
Disallow: /read_notes.php
Disallow: /account_settings.php
Disallow: /api/
Disallow: /chat/
Disallow: /fchat/
Disallow: /character_create.php
Disallow: /character_edit.php
Disallow: /character_list.php
Disallow: /character_new.php
Disallow: /login_details.php
Disallow: /character_note.php
Disallow: /character_search.php
Disallow: /forum.php
Disallow: /c/
Disallow: /xml/
Disallow: /json/
Disallow: /doc/
Disallow: /experimental/
Disallow: /static/
Disallow: /newspost/
Disallow: /group.php
Disallow: /group_browse.php
Disallow: /group_create.php
Disallow: /edit_post.php
Disallow: /group_forums.php
Disallow: /group_joined.php
Disallow: /group_manage.php
Disallow: /group_members.php
Disallow: /group_users.php
Disallow: /group_settings.php
Disallow: /password_reset.php
Disallow: /thread.php
Disallow: /warning.php
User-agent: *
Disallow: /
|
