1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
|
User-agent: Mediapartners-Google
Disallow: /cgi-bin
Disallow: /livefeed
Disallow: /A
Disallow: /cgi/NGoto
Disallow: /brand-new-look.html
Disallow: /classifieds/data/*
Disallow: /classifieds/admanagement/*
Disallow: /classifieds/Controls/*
Disallow: /classifieds/Click
Disallow: /classifieds/*ajax.svc*
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
# All robots all dirs
User-agent: *
Disallow: /cgi-bin
Disallow: /livefeed
Disallow: /A
Disallow: /share
Disallow: /cgi/NGoto
Disallow: /brand-new-look.html
Allow: /classifieds/*.js
Allow: /classifieds/*.css
Disallow: /classifieds/data/*
Disallow: /classifieds/admanagement/*
Disallow: /classifieds/Controls/*
Disallow: /classifieds/Click
Disallow: /classifieds/*.aspx
Disallow: /classifieds/*itemindex*
Disallow: /classifieds/*ajax.svc*
Disallow: /classifieds/*?*IsReduced=
Disallow: /classifieds/get-filter-results/
Disallow: /classifieds/get-top-box-results/
Disallow: /classifieds/*?*RubrikkScore=
Disallow: /classifieds/*--super-price
Disallow: /classifieds/*--good-price
Disallow: /classifieds/*--fair-price
Disallow: /classifieds/*--a-bit-pricy
Disallow: /classifieds/*--expensive
Disallow: /classifieds/*--no-price-rating
Disallow: /classifieds/*super-price--
Disallow: /classifieds/*good-price--
Disallow: /classifieds/*fair-price--
Disallow: /classifieds/*a-bit-pricy--
Disallow: /classifieds/*expensive--
Disallow: /classifieds/*no-price-rating--
Disallow: /classifieds/*ei=
Disallow: /classifieds/*usp=
Disallow: /classifieds/*nif=
Disallow: /classifieds/*p=
Disallow: /classifieds/*searchcategory=
Disallow: /classifieds/*search?
Disallow: /classifieds/*?s=
Disallow: /classifieds/*&s=
Disallow: /classifieds/*--*.html
Disallow: /classifieds/get-results/*
Disallow: /classifieds/*real_estate__number_of_bedrooms_from=
Disallow: /classifieds/*real_estate__number_of_bedrooms_to=
Disallow: /test-please-ignore/
|
