1# 2# robots.txt 3# 4# This file is to prevent the crawling and indexing of certain parts 5# of your site by web crawlers and spiders run by sites like Yahoo! 6# and Google. By telling these "robots" where not to go on your site, 7# you save bandwidth and server resources. 8# 9# This file will be ignored unless it is at the root of your host: 10# Used: http://example.com/robots.txt 11# Ignored: http://example.com/site/robots.txt 12# 13# For more information about the robots.txt standard, see: 14# http://www.robotstxt.org/robotstxt.html 15# 16# For syntax checking, see: 17# http://www.frobee.com/robots-txt-check 18 19User-agent: usasearch 20Crawl-delay: 2 21 22User-agent: * 23Crawl-delay: 10 24# Directories 25Disallow: /includes/ 26Disallow: /misc/ 27Disallow: /modules/ 28Disallow: /profiles/ 29Disallow: /scripts/ 30Disallow: /themes/ 31# Files 32Disallow: /CHANGELOG.txt 33Disallow: /cron.php 34Disallow: /INSTALL.mysql.txt 35Disallow: /INSTALL.pgsql.txt 36Disallow: /INSTALL.sqlite.txt 37Disallow: /install.php 38Disallow: /INSTALL.txt 39Disallow: /LICENSE.txt 40Disallow: /MAINTAINERS.txt 41Disallow: /update.php 42Disallow: /UPGRADE.txt 43Disallow: /xmlrpc.php 44# Paths (clean URLs) 45Disallow: /admin/ 46Disallow: /comment/reply/ 47Disallow: /filter/tips/ 48Disallow: /node/add/ 49Disallow: /search/ 50Disallow: /user/register/ 51Disallow: /user/password/ 52Disallow: /user/login/ 53Disallow: /user/logout/ 54# Paths (no clean URLs) 55Disallow: /?q=admin/ 56Disallow: /?q=comment/reply/ 57Disallow: /?q=filter/tips/ 58Disallow: /?q=node/add/ 59Disallow: /?q=search/ 60Disallow: /?q=user/password/ 61Disallow: /?q=user/register/ 62Disallow: /?q=user/login/ 63Disallow: /?q=user/logout/ 64# Paths from legacy origin 65Disallow: /Admin/ 66Disallow: /cgi-bin/ 67Disallow: /help/ 68Disallow: /img/ 69Disallow: /gif/ 70Disallow: /ins/ 71Disallow: /gopherdata/ 72Disallow: /ojp/ 73Disallow: /wusage/ 74Disallow: /opa/pr/support/ 75Disallow: /civil/frauds/ 76Sitemap: https://www.justice.gov/sitemap.xml