1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
|
# Robots.txt file for https://9gag.com
# All robots will spider the domain
User-agent: Mediapartners-Google
Disallow:
User-agent: *
Disallow: /u/
Disallow: /read/
Disallow: /gag-in-app/*
Disallow: /views?*
Disallow: /search*
Disallow: /search?query=*
Disallow: /pref/*
Disallow: /settings*
Disallow: /notifications*
Disallow: /views*
Disallow: /connect/*
Disallow: /others/*
Disallow: /static/*
Disallow: /static/ads/*
Disallow: /l.php
Disallow: /?
Crawl-delay: 1
|
