1#
2# robots.txt
3#
4# This file is to prevent the crawling and indexing of certain parts
5# of your site by web crawlers and spiders run by sites like Yahoo!
6# and Google. By telling these "robots" where not to go on your site,
7# you save bandwidth and server resources.
8#
9# This file will be ignored unless it is at the root of your host:
10# Used:    http://example.com/robots.txt
11# Ignored: http://example.com/site/robots.txt
12#
13# For more information about the robots.txt standard, see:
14# http://www.robotstxt.org/robotstxt.html
15
16User-agent: *
17Crawl-delay: 10
18# HHS.gov Directories
19Disallow: /mental-health-and-addiction-insurance-help/unknown-insurance/
20Disallow: /search/
21Disallow: /test/
22Disallow: /healthit/
23Disallow: /healthinformationtechnology/
24Disallow: /pharmacy/
25Disallow: /about/bios/asstsechealth.html
26Disallow: /about/bios/asphep.html
27Disallow: /digitalstrategy/comment/
28Disallow: /idealab/calendar/
29Disallow: /hhs-etc/
30Disallow: /sites/default/files/hhs-etc/
31# CSS, JS, Images
32Allow: /misc/*.css$
33Allow: /misc/*.css?
34Allow: /misc/*.js$
35Allow: /misc/*.js?
36Allow: /misc/*.gif
37Allow: /misc/*.jpg
38Allow: /misc/*.jpeg
39Allow: /misc/*.png
40Allow: /modules/*.css$
41Allow: /modules/*.css?
42Allow: /modules/*.js$
43Allow: /modules/*.js?
44Allow: /modules/*.gif
45Allow: /modules/*.jpg
46Allow: /modules/*.jpeg
47Allow: /modules/*.png
48Allow: /profiles/*.css$
49Allow: /profiles/*.css?
50Allow: /profiles/*.js$
51Allow: /profiles/*.js?
52Allow: /profiles/*.gif
53Allow: /profiles/*.jpg
54Allow: /profiles/*.jpeg
55Allow: /profiles/*.png
56Allow: /themes/*.css$
57Allow: /themes/*.css?
58Allow: /themes/*.js$
59Allow: /themes/*.js?
60Allow: /themes/*.gif
61Allow: /themes/*.jpg
62Allow: /themes/*.jpeg
63Allow: /themes/*.png
64# Directories
65Disallow: /includes/
66Disallow: /misc/
67Disallow: /modules/
68Disallow: /profiles/
69Disallow: /scripts/
70Disallow: /themes/
71# Files
72Disallow: /CHANGELOG.txt
73Disallow: /cron.php
74Disallow: /INSTALL.mysql.txt
75Disallow: /INSTALL.pgsql.txt
76Disallow: /INSTALL.sqlite.txt
77Disallow: /install.php
78Disallow: /INSTALL.txt
79Disallow: /LICENSE.txt
80Disallow: /MAINTAINERS.txt
81Disallow: /update.php
82Disallow: /UPGRADE.txt
83Disallow: /xmlrpc.php
84# Paths (clean URLs)
85Disallow: /admin/
86Disallow: /comment/reply/
87Disallow: /filter/tips/
88Disallow: /node/add/
89Disallow: /search/
90Disallow: /user/register/
91Disallow: /user/password/
92Disallow: /user/login/
93Disallow: /user/logout/
94# Paths (no clean URLs)
95Disallow: /?q=admin/
96Disallow: /?q=comment/reply/
97Disallow: /?q=filter/tips/
98Disallow: /?q=node/add/
99Disallow: /?q=search/
100Disallow: /?q=user/password/
101Disallow: /?q=user/register/
102Disallow: /?q=user/login/
103Disallow: /?q=user/logout/
104# Paths (language)
105Disallow: /*language=*