1#
2# robots.txt
3#
4# This file is to prevent the crawling and indexing of certain parts
5# of your site by web crawlers and spiders run by sites like Yahoo!
6# and Google. By telling these "robots" where not to go on your site,
7# you save bandwidth and server resources.
8#
9# This file will be ignored unless it is at the root of your host:
10# Used:    http://example.com/robots.txt
11# Ignored: http://example.com/site/robots.txt
12#
13# For more information about the robots.txt standard, see:
14# http://www.robotstxt.org/robotstxt.html
15#
16# For syntax checking, see:
17# http://www.frobee.com/robots-txt-check
18#
19
20Sitemap: https://www.uscis.gov/sitemap.xml
21
22User-agent: *
23Crawl-delay: 1
24# Custom
25Disallow: /rule-list
26# Directories
27Disallow: /includes/
28Disallow: /misc/
29Disallow: /modules/
30Disallow: /profiles/
31Disallow: /scripts/
32Disallow: /themes/
33Disallow: /portal/
34Disallow: /vgn-ext-templating/
35Disallow: /taxonomy/
36Disallow: /uscis-tags/
37Disallow: /general-keywords/
38Disallow: /site-hierarchy/
39Disallow: /FAQ-page/
40Disallow: /rule/
41# Files
42Disallow: /CHANGELOG.txt
43Disallow: /cron.php
44Disallow: /INSTALL.mysql.txt
45Disallow: /INSTALL.pgsql.txt
46Disallow: /INSTALL.sqlite.txt
47Disallow: /install.php
48Disallow: /INSTALL.txt
49Disallow: /LICENSE.txt
50Disallow: /MAINTAINERS.txt
51Disallow: /update.php
52Disallow: /UPGRADE.txt
53Disallow: /xmlrpc.php
54# Paths (clean URLs)
55Disallow: /admin/
56Disallow: /comment/reply/
57Disallow: /filter/tips/
58Disallow: /node/add/
59Disallow: /search/
60Disallow: /user/register/
61Disallow: /user/password/
62Disallow: /user/login/
63Disallow: /user/logout/
64# Paths (no clean URLs)
65Disallow: /?q=admin/
66Disallow: /?q=comment/reply/
67Disallow: /?q=filter/tips/
68Disallow: /?q=node/add/
69Disallow: /?q=search/
70Disallow: /?q=user/password/
71Disallow: /?q=user/register/
72Disallow: /?q=user/login/
73Disallow: /?q=user/logout/
74
75User-agent: Googlebot
76# Custom
77Disallow: /rule-list
78# Directories
79Disallow: /includes/
80Disallow: /misc/
81Disallow: /modules/
82Disallow: /profiles/
83Disallow: /scripts/
84Disallow: /themes/
85Disallow: /portal/
86Disallow: /vgn-ext-templating/
87Disallow: /taxonomy/
88Disallow: /uscis-tags/
89Disallow: /general-keywords/
90Disallow: /site-hierarchy/
91Disallow: /FAQ-page/
92# Files
93Disallow: /CHANGELOG.txt
94Disallow: /cron.php
95Disallow: /INSTALL.mysql.txt
96Disallow: /INSTALL.pgsql.txt
97Disallow: /INSTALL.sqlite.txt
98Disallow: /install.php
99Disallow: /INSTALL.txt
100Disallow: /LICENSE.txt
101Disallow: /MAINTAINERS.txt
102Disallow: /update.php
103Disallow: /UPGRADE.txt
104Disallow: /xmlrpc.php
105# Paths (clean URLs)
106Disallow: /admin/
107Disallow: /comment/reply/
108Disallow: /filter/tips/
109Disallow: /node/add/
110Disallow: /search/
111Disallow: /user/register/
112Disallow: /user/password/
113Disallow: /user/login/
114Disallow: /user/logout/
115# Paths (no clean URLs)
116Disallow: /?q=admin/
117Disallow: /?q=comment/reply/
118Disallow: /?q=filter/tips/
119Disallow: /?q=node/add/
120Disallow: /?q=search/
121Disallow: /?q=user/password/
122Disallow: /?q=user/register/
123Disallow: /?q=user/login/
124Disallow: /?q=user/logout/
125