1# 2# robots.txt 3# 4# This file is to prevent the crawling and indexing of certain parts 5# of your site by web crawlers and spiders run by sites like Yahoo! 6# and Google. By telling these "robots" where not to go on your site, 7# you save bandwidth and server resources. 8# 9# This file will be ignored unless it is at the root of your host: 10# Used: http://example.com/robots.txt 11# Ignored: http://example.com/site/robots.txt 12# 13# For more information about the robots.txt standard, see: 14# http://www.robotstxt.org/robotstxt.html 15# 16# For syntax checking, see: 17# http://www.frobee.com/robots-txt-check 18# 19 20Sitemap: https://www.uscis.gov/sitemap.xml 21 22User-agent: * 23Crawl-delay: 1 24# Custom 25Disallow: /rule-list 26# Directories 27Disallow: /includes/ 28Disallow: /misc/ 29Disallow: /modules/ 30Disallow: /profiles/ 31Disallow: /scripts/ 32Disallow: /themes/ 33Disallow: /portal/ 34Disallow: /vgn-ext-templating/ 35Disallow: /taxonomy/ 36Disallow: /uscis-tags/ 37Disallow: /general-keywords/ 38Disallow: /site-hierarchy/ 39Disallow: /FAQ-page/ 40Disallow: /rule/ 41# Files 42Disallow: /CHANGELOG.txt 43Disallow: /cron.php 44Disallow: /INSTALL.mysql.txt 45Disallow: /INSTALL.pgsql.txt 46Disallow: /INSTALL.sqlite.txt 47Disallow: /install.php 48Disallow: /INSTALL.txt 49Disallow: /LICENSE.txt 50Disallow: /MAINTAINERS.txt 51Disallow: /update.php 52Disallow: /UPGRADE.txt 53Disallow: /xmlrpc.php 54# Paths (clean URLs) 55Disallow: /admin/ 56Disallow: /comment/reply/ 57Disallow: /filter/tips/ 58Disallow: /node/add/ 59Disallow: /search/ 60Disallow: /user/register/ 61Disallow: /user/password/ 62Disallow: /user/login/ 63Disallow: /user/logout/ 64# Paths (no clean URLs) 65Disallow: /?q=admin/ 66Disallow: /?q=comment/reply/ 67Disallow: /?q=filter/tips/ 68Disallow: /?q=node/add/ 69Disallow: /?q=search/ 70Disallow: /?q=user/password/ 71Disallow: /?q=user/register/ 72Disallow: /?q=user/login/ 73Disallow: /?q=user/logout/ 74 75User-agent: Googlebot 76# Custom 77Disallow: /rule-list 78# Directories 79Disallow: /includes/ 80Disallow: /misc/ 81Disallow: /modules/ 82Disallow: /profiles/ 83Disallow: /scripts/ 84Disallow: /themes/ 85Disallow: /portal/ 86Disallow: /vgn-ext-templating/ 87Disallow: /taxonomy/ 88Disallow: /uscis-tags/ 89Disallow: /general-keywords/ 90Disallow: /site-hierarchy/ 91Disallow: /FAQ-page/ 92# Files 93Disallow: /CHANGELOG.txt 94Disallow: /cron.php 95Disallow: /INSTALL.mysql.txt 96Disallow: /INSTALL.pgsql.txt 97Disallow: /INSTALL.sqlite.txt 98Disallow: /install.php 99Disallow: /INSTALL.txt 100Disallow: /LICENSE.txt 101Disallow: /MAINTAINERS.txt 102Disallow: /update.php 103Disallow: /UPGRADE.txt 104Disallow: /xmlrpc.php 105# Paths (clean URLs) 106Disallow: /admin/ 107Disallow: /comment/reply/ 108Disallow: /filter/tips/ 109Disallow: /node/add/ 110Disallow: /search/ 111Disallow: /user/register/ 112Disallow: /user/password/ 113Disallow: /user/login/ 114Disallow: /user/logout/ 115# Paths (no clean URLs) 116Disallow: /?q=admin/ 117Disallow: /?q=comment/reply/ 118Disallow: /?q=filter/tips/ 119Disallow: /?q=node/add/ 120Disallow: /?q=search/ 121Disallow: /?q=user/password/ 122Disallow: /?q=user/register/ 123Disallow: /?q=user/login/ 124Disallow: /?q=user/logout/ 125