1#
2# robots.txt
3#
4# This file is to prevent the crawling and indexing of certain parts
5# of your site by web crawlers and spiders run by sites like Yahoo!
6# and Google. By telling these "robots" where not to go on your site,
7# you save bandwidth and server resources.
8#
9# This file will be ignored unless it is at the root of your host:
10# Used:    http://example.com/robots.txt
11# Ignored: http://example.com/site/robots.txt
12#
13# For more information about the robots.txt standard, see:
14# http://www.robotstxt.org/robotstxt.html
15#
16# For syntax checking, see:
17# http://www.frobee.com/robots-txt-check
18
19User-agent: *
20Crawl-delay: 10
21# Directories
22Disallow: /includes/
23Disallow: /misc/
24Disallow: /modules/
25Disallow: /profiles/
26Disallow: /scripts/
27Disallow: /themes/
28# Files
29Disallow: /CHANGELOG.txt
30Disallow: /cron.php
31Disallow: /INSTALL.mysql.txt
32Disallow: /INSTALL.pgsql.txt
33Disallow: /INSTALL.sqlite.txt
34Disallow: /install.php
35Disallow: /INSTALL.txt
36Disallow: /LICENSE.txt
37Disallow: /MAINTAINERS.txt
38Disallow: /update.php
39Disallow: /UPGRADE.txt
40Disallow: /xmlrpc.php
41# Paths (clean URLs)
42Disallow: /admin/
43Disallow: /comment/reply/
44Disallow: /filter/tips/
45Disallow: /user/logout/
46Disallow: /node/add/
47Disallow: /search/
48Disallow: /user/register/
49Disallow: /user/password/
50Disallow: /user/login/
51# Paths (no clean URLs)
52Disallow: /?q=admin/
53Disallow: /?q=comment/reply/
54Disallow: /?q=filter/tips/
55Disallow: /?q=user/logout/
56Disallow: /?q=node/add/
57Disallow: /?q=search/
58Disallow: /?q=user/password/
59Disallow: /?q=user/register/
60Disallow: /?q=user/login/
61Disallow: /?q=search/
62
63# Extras on drupal.org
64# no access for table sorting paths or any paths that have parameters.
65Disallow: /*?sort*
66Disallow: /*&sort*
67
68Disallow: /*?solrsort*
69Disallow: /*&solrsort*
70
71# no access to profile listings, they are costly to generate and don't provide useful information.
72Disallow: /profile
73Disallow: /profile/*
74Disallow: /?q=profile
75Disallow: /?q=profile/*
76
77# Disallow bogus aggregator pages
78Disallow: /aggregator/*
79
80# Disallow project search
81Disallow: /project/issues/search
82Disallow: /project/issues/search/*
83Disallow: /project/issues/*/search
84
85# Disallow book export
86Disallow: /book/export/*
87
88# Disallow pift tests
89Disallow: /pift/retest/*
90
91# Disallow project subscription
92Disallow: /project/issues/subscribe-mail/*
93
94# Disallow URLs with destination parameter
95Disallow: /user/login?destination=*
96Disallow: /user/register?destination=*
97Disallow: /user?destination=*
98
99# Disallow individual user trackers
100Disallow: /user/*/track
101Disallow: /user/*/track?page=*
102
103
104# Googlebot picked up strange homepage URLs somewhere
105Disallow: /home/*
106