1#
2# robots.txt
3#
4# This file is to prevent the crawling and indexing of certain parts
5# of your site by web crawlers and spiders run by sites like Yahoo!
6# and Google. By telling these "robots" where not to go on your site,
7# you save bandwidth and server resources.
8#
9# This file will be ignored unless it is at the root of your host:
10# Used:    http://example.com/robots.txt
11# Ignored: http://example.com/site/robots.txt
12#
13# For more information about the robots.txt standard, see:
14# http://www.robotstxt.org/robotstxt.html
15#
16# For syntax checking, see:
17# http://www.frobee.com/robots-txt-check
18
19User-agent: usasearch
20Crawl-delay: 2
21
22User-agent: *
23Crawl-delay: 10
24# Directories
25Disallow: /includes/
26Disallow: /misc/
27Disallow: /modules/
28Disallow: /profiles/
29Disallow: /scripts/
30Disallow: /themes/
31# Files
32Disallow: /CHANGELOG.txt
33Disallow: /cron.php
34Disallow: /INSTALL.mysql.txt
35Disallow: /INSTALL.pgsql.txt
36Disallow: /INSTALL.sqlite.txt
37Disallow: /install.php
38Disallow: /INSTALL.txt
39Disallow: /LICENSE.txt
40Disallow: /MAINTAINERS.txt
41Disallow: /update.php
42Disallow: /UPGRADE.txt
43Disallow: /xmlrpc.php
44# Paths (clean URLs)
45Disallow: /admin/
46Disallow: /comment/reply/
47Disallow: /filter/tips/
48Disallow: /node/add/
49Disallow: /search/
50Disallow: /user/register/
51Disallow: /user/password/
52Disallow: /user/login/
53Disallow: /user/logout/
54# Paths (no clean URLs)
55Disallow: /?q=admin/
56Disallow: /?q=comment/reply/
57Disallow: /?q=filter/tips/
58Disallow: /?q=node/add/
59Disallow: /?q=search/
60Disallow: /?q=user/password/
61Disallow: /?q=user/register/
62Disallow: /?q=user/login/
63Disallow: /?q=user/logout/
64# Paths from legacy origin
65Disallow: /Admin/
66Disallow: /cgi-bin/
67Disallow: /help/
68Disallow: /img/
69Disallow: /gif/
70Disallow: /ins/
71Disallow: /gopherdata/
72Disallow: /ojp/
73Disallow: /wusage/
74Disallow: /opa/pr/support/
75Disallow: /civil/frauds/
76Sitemap: https://www.justice.gov/sitemap.xml