1# Optimised Robots
2# Updated: 6/11/2017
3
4##################################################
5# Sitemap
6Sitemap: http://www.kotaku.com.au/sitemap.xml
7
8
9##################################################
10# 3rd Party Integrations
11
12##################################################
13
14
15
16##################################################
17# Global settings
18
19User-agent: *
20Disallow: /wp-admin/
21Allow: /wp-admin/admin-ajax.php
22Disallow: /wp-login.php
23Disallow: /activate/
24Disallow: /cgi-bin/
25Disallow: /mshots/v1/
26Disallow: /next/
27Disallow: /public.api/
28Disallow: /search/
29Disallow: /*?                           # Don't index anything with a query string
30Disallow: /tags/*/page/                 # Explicitly block pagination on Tag pages
31Disallow: /author/*/page/               # Explicitly block pagination on Author pages
32Disallow: /user/                        # Don't index GMG social pages and personal pages
33
34User-agent: Mediapartners-Google
35Disallow:
36
37
38##################################################
39# Bot-specific handling
40
41User-agent: bingbot
42Crawl-delay: 1
43
44# MSNBot works on a scale of 0 = normal, 1-10 from slow to extremely slow.
45User-agent: msnbot
46Crawl-delay: 1
47
48User-agent: Slurp
49Crawl-delay: 1