1User-agent: Nutch-Viacom
2Allow: /content/bet/app-content-map.html
3Allow: /content/bet/roku-app-content-map.html
4Allow: /content/bet/appletv-app-content-map.html
5Disallow: /content/betcomprivate/
6Disallow: /content/betcom/
7Disallow: /content/betprivate/
8Disallow: /etc/designs/
9Disallow: /test/
10Disallow: /content/bet/apps/betx-app
11Disallow: /content/bet/shows/bet-awards/nominees-mobile
12
13User-agent: Googlebot-news
14Disallow: /shows/
15Disallow: /video/
16Disallow: /topics/
17Disallow: */photos/
18Disallow: /schedule.html
19Disallow: /video.html
20
21User-agent: Twitterbot
22Allow: /
23Crawl-delay: 5
24
25User-agent: Pinterestbot
26Allow: /
27Crawl-delay: 5
28
29User-agent: SearchmetricsBot
30Crawl-delay: 5
31Disallow: /content/betcomprivate/
32Disallow: /content/betcom/
33Disallow: /content/betprivate/
34Disallow: /test/
35
36User-agent: AhrefsBot
37Crawl-delay: 5
38Disallow: /content/betcomprivate/
39Disallow: /content/betcom/
40Disallow: /content/betprivate/
41Disallow: /test/
42
43
44User-agent: Mail.RU_Bot
45Disallow: /
46
47User-agent: Mail.RU_Bot/Img
48Disallow: /
49
50User-agent: Vegi bot (we follow your robots.txt settings before crawling, you can slow down the bot by change the Crawl-Delay parameter in the settings.if you have an enquiry, please email to: abuse-report@terrykyleseoagency.com)
51Disallow: /
52
53User-agent: SemrushBot
54Disallow: /
55
56User-agent: Yandex
57Disallow: /
58
59User-agent: *
60Disallow: /content/bet/app-content-map.html
61Disallow: /test/
62Disallow: /content/betcomprivate/
63Disallow: /content/betcom/
64Disallow: /content/betprivate/
65Disallow: /content/bet/apps/betx-app/
66Disallow: /content/bet/shows/bet-awards/nominees-mobile/
67Disallow: /content/bet/searchresults.html
68Disallow: /content/bet/app-content-map/
69Disallow: /app-content-map/
70Disallow: /content/bet/roku-app-content-map/
71Disallow: /roku-app-content-map/
72Disallow: /content/bet/appletv-app-content-map/
73Disallow: /content/bet/news/national/2013/03/09/essence-s-former-eic-says-she-was-fired.html
74Disallow: /news/national/2013/03/09/essence-s-former-eic-says-she-was-fired.html
75Disallow: /appletv-app-content-map/
76#google tries to crawl video ad location paths that are present in html, so denying it here; it is not valid URLs for BET
77Disallow: /1125/
78#google keeps in index bunch of outdated URLs from previous bet.com, this needs to be disallowed
79Disallow: /*.ugc.html$
80Disallow: /*.share.responsive-true.html*
81Disallow: /etc/designs/bet/amp/stats
82
83Sitemap: https://www.bet.com/sitemap.xml
84