1# Optimised Robots 2# Updated: 6/11/2017 3 4################################################## 5# Sitemap 6Sitemap: http://www.kotaku.com.au/sitemap.xml 7 8 9################################################## 10# 3rd Party Integrations 11 12################################################## 13 14 15 16################################################## 17# Global settings 18 19User-agent: * 20Disallow: /wp-admin/ 21Allow: /wp-admin/admin-ajax.php 22Disallow: /wp-login.php 23Disallow: /activate/ 24Disallow: /cgi-bin/ 25Disallow: /mshots/v1/ 26Disallow: /next/ 27Disallow: /public.api/ 28Disallow: /search/ 29Disallow: /*? # Don't index anything with a query string 30Disallow: /tags/*/page/ # Explicitly block pagination on Tag pages 31Disallow: /author/*/page/ # Explicitly block pagination on Author pages 32Disallow: /user/ # Don't index GMG social pages and personal pages 33 34User-agent: Mediapartners-Google 35Disallow: 36 37 38################################################## 39# Bot-specific handling 40 41User-agent: bingbot 42Crawl-delay: 1 43 44# MSNBot works on a scale of 0 = normal, 1-10 from slow to extremely slow. 45User-agent: msnbot 46Crawl-delay: 1 47 48User-agent: Slurp 49Crawl-delay: 1