1# 2# robots.txt 3# 4# This file is to prevent the crawling and indexing of certain parts 5# of your site by web crawlers and spiders run by sites like Yahoo! 6# and Google. By telling these "robots" where not to go on your site, 7# you save bandwidth and server resources. 8# 9# This file will be ignored unless it is at the root of your host: 10# Used: http://example.com/robots.txt 11# Ignored: http://example.com/site/robots.txt 12# 13# For more information about the robots.txt standard, see: 14# http://www.robotstxt.org/wc/robots.html 15# 16# For syntax checking, see: 17# http://www.sxw.org.uk/computing/robots/check.html 18 19User-agent: * 20Crawl-delay: 10 21# Directories 22Disallow: /includes/ 23Disallow: /misc/ 24Disallow: /modules/ 25Disallow: /profiles/ 26Disallow: /scripts/ 27Disallow: /themes/ 28# Files 29Disallow: /CHANGELOG.txt 30Disallow: /cron.php 31Disallow: /INSTALL.mysql.txt 32Disallow: /INSTALL.pgsql.txt 33Disallow: /INSTALL.sqlite.txt 34Disallow: /install.php 35Disallow: /INSTALL.txt 36Disallow: /LICENSE.txt 37Disallow: /MAINTAINERS.txt 38Disallow: /update.php 39Disallow: /UPGRADE.txt 40Disallow: /xmlrpc.php 41# Paths (clean URLs) 42Disallow: /admin/ 43Disallow: /comment/reply/ 44Disallow: /filter/tips/ 45Disallow: /node/add/ 46Disallow: /search/ 47Disallow: /user/register/ 48Disallow: /user/password/ 49Disallow: /user/login/ 50Disallow: /user/logout/ 51# Paths (no clean URLs) 52Disallow: /?q=admin/ 53Disallow: /?q=comment/reply/ 54Disallow: /?q=filter/tips/ 55Disallow: /?q=node/add/ 56Disallow: /?q=search/ 57Disallow: /?q=user/password/ 58Disallow: /?q=user/register/ 59Disallow: /?q=user/login/ 60Disallow: /?q=user/logout/ 61# Stop access to videos used in overlays 62Disallow: /content/ 63Sitemap: http://www.ox.ac.uk/sitemap.xml