File: www.bankrate.com

package info (click to toggle)
python-protego 0.6.0%2Bdfsg-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 30,048 kB
  • sloc: python: 1,492; perl: 190; cpp: 33; sh: 4; makefile: 3
file content (55 lines) | stat: -rw-r--r-- 1,282 bytes parent folder | download | duplicates (4)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
# directed to all spiders
User-agent: *  
Disallow: /system/swf/
Disallow: /acc_random*
Disallow: /ads/
Disallow: /affiliates/
Disallow: /applets/
Disallow: /budget/
Disallow: /calc-pop-up/
Disallow: /cgi-bin/
Disallow: /doubleclick/
Disallow: /hlink_redirects/
Disallow: /include/
Disallow: /java/
Disallow: /javacalcs/
Disallow: /jsfeeds/
Disallow: /partners/
Disallow: /sitespeed/
Disallow: /funnel/checking-account/
Disallow: /funnel/checking-savings/
Disallow: /funnel/college-finance/
Disallow: /*/ybir/
Disallow: /apps/sdk/
Disallow: /widgets/
Disallow: /rtrs/
Disallow: /pdfs/mortgage-guide/
Disallow: /rss_trk/
Disallow: /IndustryBrains/
Disallow: /nltrack/
Disallow: /page-analytics/
Disallow: /images/
Allow: /landing/cd-rates-d/
Allow: /landing/cd-rates-f/
Allow: /landing/cd-rates-g/
Disallow: /landing/
Disallow: /search/
Disallow: /credit-card-offers/transfer-page/
Disallow: /amp/
Disallow: /404.aspx
Disallow: /customerror.aspx
Disallow: /zeta/
Noindex: /credit-cards/reviews/*/offer/
Disallow: /credit-cards/reviews/*/offer/

Allow: *.js
Allow: *.css
 
User-agent: Mediapartners-Google
Disallow:

User-agent: Googlebot-Image
Disallow:

Sitemap: https://www.bankrate.com/sitemap/dynamic-sitemap-index.xml
Sitemap: https://www.bankrate.com/sitemap/sitemap-index.xml