1# 2# robots.txt 3# 4# This file is to prevent the crawling and indexing of certain parts 5# of your site by web crawlers and spiders run by sites like Yahoo! 6# and Google. By telling these "robots" where not to go on your site, 7# you save bandwidth and server resources. 8# 9# This file will be ignored unless it is at the root of your host: 10# Used: http://example.com/robots.txt 11# Ignored: http://example.com/site/robots.txt 12# 13# For more information about the robots.txt standard, see: 14# http://www.robotstxt.org/robotstxt.html 15# 16# For syntax checking, see: 17# http://www.frobee.com/robots-txt-check 18 19User-agent: * 20Crawl-delay: 10 21# Directories 22Disallow: /includes/ 23Disallow: /misc/ 24Disallow: /modules/ 25Disallow: /profiles/ 26Disallow: /scripts/ 27Disallow: /themes/ 28# Files 29Disallow: /CHANGELOG.txt 30Disallow: /cron.php 31Disallow: /INSTALL.mysql.txt 32Disallow: /INSTALL.pgsql.txt 33Disallow: /INSTALL.sqlite.txt 34Disallow: /install.php 35Disallow: /INSTALL.txt 36Disallow: /LICENSE.txt 37Disallow: /MAINTAINERS.txt 38Disallow: /update.php 39Disallow: /UPGRADE.txt 40Disallow: /xmlrpc.php 41# Paths (clean URLs) 42Disallow: /admin/ 43Disallow: /comment/reply/ 44Disallow: /filter/tips/ 45Disallow: /user/logout/ 46Disallow: /node/add/ 47Disallow: /search/ 48Disallow: /user/register/ 49Disallow: /user/password/ 50Disallow: /user/login/ 51# Paths (no clean URLs) 52Disallow: /?q=admin/ 53Disallow: /?q=comment/reply/ 54Disallow: /?q=filter/tips/ 55Disallow: /?q=user/logout/ 56Disallow: /?q=node/add/ 57Disallow: /?q=search/ 58Disallow: /?q=user/password/ 59Disallow: /?q=user/register/ 60Disallow: /?q=user/login/ 61Disallow: /?q=search/ 62 63# Extras on drupal.org 64# no access for table sorting paths or any paths that have parameters. 65Disallow: /*?sort* 66Disallow: /*&sort* 67 68Disallow: /*?solrsort* 69Disallow: /*&solrsort* 70 71# no access to profile listings, they are costly to generate and don't provide useful information. 72Disallow: /profile 73Disallow: /profile/* 74Disallow: /?q=profile 75Disallow: /?q=profile/* 76 77# Disallow bogus aggregator pages 78Disallow: /aggregator/* 79 80# Disallow project search 81Disallow: /project/issues/search 82Disallow: /project/issues/search/* 83Disallow: /project/issues/*/search 84 85# Disallow book export 86Disallow: /book/export/* 87 88# Disallow pift tests 89Disallow: /pift/retest/* 90 91# Disallow project subscription 92Disallow: /project/issues/subscribe-mail/* 93 94# Disallow URLs with destination parameter 95Disallow: /user/login?destination=* 96Disallow: /user/register?destination=* 97Disallow: /user?destination=* 98 99# Disallow individual user trackers 100Disallow: /user/*/track 101Disallow: /user/*/track?page=* 102 103 104# Googlebot picked up strange homepage URLs somewhere 105Disallow: /home/* 106