1# 2# robots.txt 3# 4# This file is to prevent the crawling and indexing of certain parts 5# of your site by web crawlers and spiders run by sites like Yahoo! 6# and Google. By telling these "robots" where not to go on your site, 7# you save bandwidth and server resources. 8# 9# This file will be ignored unless it is at the root of your host: 10# Used: http://example.com/robots.txt 11# Ignored: http://example.com/site/robots.txt 12# 13# For more information about the robots.txt standard, see: 14# http://www.robotstxt.org/robotstxt.html 15 16User-agent: * 17Crawl-delay: 10 18# HHS.gov Directories 19Disallow: /mental-health-and-addiction-insurance-help/unknown-insurance/ 20Disallow: /search/ 21Disallow: /test/ 22Disallow: /healthit/ 23Disallow: /healthinformationtechnology/ 24Disallow: /pharmacy/ 25Disallow: /about/bios/asstsechealth.html 26Disallow: /about/bios/asphep.html 27Disallow: /digitalstrategy/comment/ 28Disallow: /idealab/calendar/ 29Disallow: /hhs-etc/ 30Disallow: /sites/default/files/hhs-etc/ 31# CSS, JS, Images 32Allow: /misc/*.css$ 33Allow: /misc/*.css? 34Allow: /misc/*.js$ 35Allow: /misc/*.js? 36Allow: /misc/*.gif 37Allow: /misc/*.jpg 38Allow: /misc/*.jpeg 39Allow: /misc/*.png 40Allow: /modules/*.css$ 41Allow: /modules/*.css? 42Allow: /modules/*.js$ 43Allow: /modules/*.js? 44Allow: /modules/*.gif 45Allow: /modules/*.jpg 46Allow: /modules/*.jpeg 47Allow: /modules/*.png 48Allow: /profiles/*.css$ 49Allow: /profiles/*.css? 50Allow: /profiles/*.js$ 51Allow: /profiles/*.js? 52Allow: /profiles/*.gif 53Allow: /profiles/*.jpg 54Allow: /profiles/*.jpeg 55Allow: /profiles/*.png 56Allow: /themes/*.css$ 57Allow: /themes/*.css? 58Allow: /themes/*.js$ 59Allow: /themes/*.js? 60Allow: /themes/*.gif 61Allow: /themes/*.jpg 62Allow: /themes/*.jpeg 63Allow: /themes/*.png 64# Directories 65Disallow: /includes/ 66Disallow: /misc/ 67Disallow: /modules/ 68Disallow: /profiles/ 69Disallow: /scripts/ 70Disallow: /themes/ 71# Files 72Disallow: /CHANGELOG.txt 73Disallow: /cron.php 74Disallow: /INSTALL.mysql.txt 75Disallow: /INSTALL.pgsql.txt 76Disallow: /INSTALL.sqlite.txt 77Disallow: /install.php 78Disallow: /INSTALL.txt 79Disallow: /LICENSE.txt 80Disallow: /MAINTAINERS.txt 81Disallow: /update.php 82Disallow: /UPGRADE.txt 83Disallow: /xmlrpc.php 84# Paths (clean URLs) 85Disallow: /admin/ 86Disallow: /comment/reply/ 87Disallow: /filter/tips/ 88Disallow: /node/add/ 89Disallow: /search/ 90Disallow: /user/register/ 91Disallow: /user/password/ 92Disallow: /user/login/ 93Disallow: /user/logout/ 94# Paths (no clean URLs) 95Disallow: /?q=admin/ 96Disallow: /?q=comment/reply/ 97Disallow: /?q=filter/tips/ 98Disallow: /?q=node/add/ 99Disallow: /?q=search/ 100Disallow: /?q=user/password/ 101Disallow: /?q=user/register/ 102Disallow: /?q=user/login/ 103Disallow: /?q=user/logout/ 104# Paths (language) 105Disallow: /*language=*