1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
|
User-agent: Nutch-Viacom
Allow: /content/bet/app-content-map.html
Allow: /content/bet/roku-app-content-map.html
Allow: /content/bet/appletv-app-content-map.html
Disallow: /content/betcomprivate/
Disallow: /content/betcom/
Disallow: /content/betprivate/
Disallow: /etc/designs/
Disallow: /test/
Disallow: /content/bet/apps/betx-app
Disallow: /content/bet/shows/bet-awards/nominees-mobile
User-agent: Googlebot-news
Disallow: /shows/
Disallow: /video/
Disallow: /topics/
Disallow: */photos/
Disallow: /schedule.html
Disallow: /video.html
User-agent: Twitterbot
Allow: /
Crawl-delay: 5
User-agent: Pinterestbot
Allow: /
Crawl-delay: 5
User-agent: SearchmetricsBot
Crawl-delay: 5
Disallow: /content/betcomprivate/
Disallow: /content/betcom/
Disallow: /content/betprivate/
Disallow: /test/
User-agent: AhrefsBot
Crawl-delay: 5
Disallow: /content/betcomprivate/
Disallow: /content/betcom/
Disallow: /content/betprivate/
Disallow: /test/
User-agent: Mail.RU_Bot
Disallow: /
User-agent: Mail.RU_Bot/Img
Disallow: /
User-agent: Vegi bot (we follow your robots.txt settings before crawling, you can slow down the bot by change the Crawl-Delay parameter in the settings.if you have an enquiry, please email to: abuse-report@terrykyleseoagency.com)
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: Yandex
Disallow: /
User-agent: *
Disallow: /content/bet/app-content-map.html
Disallow: /test/
Disallow: /content/betcomprivate/
Disallow: /content/betcom/
Disallow: /content/betprivate/
Disallow: /content/bet/apps/betx-app/
Disallow: /content/bet/shows/bet-awards/nominees-mobile/
Disallow: /content/bet/searchresults.html
Disallow: /content/bet/app-content-map/
Disallow: /app-content-map/
Disallow: /content/bet/roku-app-content-map/
Disallow: /roku-app-content-map/
Disallow: /content/bet/appletv-app-content-map/
Disallow: /content/bet/news/national/2013/03/09/essence-s-former-eic-says-she-was-fired.html
Disallow: /news/national/2013/03/09/essence-s-former-eic-says-she-was-fired.html
Disallow: /appletv-app-content-map/
#google tries to crawl video ad location paths that are present in html, so denying it here; it is not valid URLs for BET
Disallow: /1125/
#google keeps in index bunch of outdated URLs from previous bet.com, this needs to be disallowed
Disallow: /*.ugc.html$
Disallow: /*.share.responsive-true.html*
Disallow: /etc/designs/bet/amp/stats
Sitemap: https://www.bet.com/sitemap.xml
|
