1# 2# robots.txt 3# 4# This file is to prevent the crawling and indexing of certain parts 5# of your site by web crawlers and spiders run by sites like Yahoo! 6# and Google. By telling these "robots" where not to go on your site, 7# you save bandwidth and server resources. 8# 9# This file will be ignored unless it is at the root of your host: 10# Used: http://example.com/robots.txt 11# Ignored: http://example.com/site/robots.txt 12# 13# For more information about the robots.txt standard, see: 14# http://www.robotstxt.org/robotstxt.html 15 16User-agent: * 17Crawl-delay: 10 18# CSS, JS, Images 19Allow: /misc/*.css$ 20Allow: /misc/*.css? 21Allow: /misc/*.js$ 22Allow: /misc/*.js? 23Allow: /misc/*.gif 24Allow: /misc/*.jpg 25Allow: /misc/*.jpeg 26Allow: /misc/*.png 27Allow: /modules/*.css$ 28Allow: /modules/*.css? 29Allow: /modules/*.js$ 30Allow: /modules/*.js? 31Allow: /modules/*.gif 32Allow: /modules/*.jpg 33Allow: /modules/*.jpeg 34Allow: /modules/*.png 35Allow: /profiles/*.css$ 36Allow: /profiles/*.css? 37Allow: /profiles/*.js$ 38Allow: /profiles/*.js? 39Allow: /profiles/*.gif 40Allow: /profiles/*.jpg 41Allow: /profiles/*.jpeg 42Allow: /profiles/*.png 43Allow: /themes/*.css$ 44Allow: /themes/*.css? 45Allow: /themes/*.js$ 46Allow: /themes/*.js? 47Allow: /themes/*.gif 48Allow: /themes/*.jpg 49Allow: /themes/*.jpeg 50Allow: /themes/*.png 51# Directories 52Disallow: /includes/ 53Disallow: /misc/ 54Disallow: /modules/ 55Disallow: /profiles/ 56Disallow: /scripts/ 57Disallow: /themes/ 58# Files 59Disallow: /CHANGELOG.txt 60Disallow: /cron.php 61Disallow: /INSTALL.mysql.txt 62Disallow: /INSTALL.pgsql.txt 63Disallow: /INSTALL.sqlite.txt 64Disallow: /install.php 65Disallow: /INSTALL.txt 66Disallow: /LICENSE.txt 67Disallow: /MAINTAINERS.txt 68Disallow: /update.php 69Disallow: /UPGRADE.txt 70Disallow: /xmlrpc.php 71# Paths (clean URLs) 72Disallow: /admin/ 73Disallow: /comment/reply/ 74Disallow: /filter/tips/ 75Disallow: */node/* 76Disallow: /search/ 77Disallow: /user/register/ 78Disallow: /user/password/ 79Disallow: /user/login/ 80Disallow: /user/logout/ 81# Paths (no clean URLs) 82Disallow: /*login/* 83Disallow: /*file/* 84Disallow: /?q=admin/ 85Disallow: /?q=comment/reply/ 86Disallow: /?q=filter/tips/ 87Disallow: /?q=node/add/ 88Disallow: /?q=search/ 89Disallow: /?q=user/password/ 90Disallow: /?q=user/register/ 91Disallow: /?q=user/login/ 92Disallow: /?q=user/logout/ 93#Forum 94Disallow: /forums/admin/ 95Disallow: /forums/cache/ 96Disallow: /forums/converge_local/ 97Disallow: /forums/hooks/ 98Disallow: /forums/ips_kernel/ 99Disallow: /forums/retail/ 100Disallow: /forums/public/style_captcha/ 101Disallow: /forums/index.php?app=core&module=task 102Disallow: /forums/index.php?app=forums&module=moderate§ion=moderate 103Disallow: /forums/index.php?app=forums&module=extras§ion=forward 104Disallow: /forums/index.php?app=members&module=chat 105Disallow: /forums/index.php?&unlockUserAgent=1 106Disallow: /forums/*&module=search 107Disallow: /forums/*app=core&module=global§ion=reputation 108Disallow: /forums/*app=core&module=usercp 109Disallow: /forums/*app=members&module=messaging 110Disallow: /forums/*&p= 111Disallow: /forums/*&pid= 112Disallow: /forums/*&hl= 113Disallow: /forums/*&start= 114Disallow: /forums/*view__getnewpost$ 115Disallow: /forums/*view__getlastpost$ 116Disallow: /forums/*view__old$ 117Disallow: /forums/*view__new$ 118Disallow: /forums/*view__getfirst$ 119Disallow: /forums/*view__getprevious$ 120Disallow: /forums/*view__getnext$ 121Disallow: /forums/*view__getlast$ 122Disallow: /forums/*&view=getnewpost$ 123Disallow: /forums/*&view=getlastpost$ 124Disallow: /forums/*&view=old$ 125Disallow: /forums/*&view=new$ 126Disallow: /forums/*&view=getfirst$ 127Disallow: /forums/*&view=getprevious$ 128Disallow: /forums/*&view=getnext$ 129Disallow: /forums/*&view=getlast$ 130Disallow: /forums/*?s= 131Disallow: /forums/*&s= 132Disallow: /forums/index.php?app=core&module=global§ion=login&do=deleteCookies 133Disallow: /forums/index.php?app=forums&module=extras§ion=rating 134Disallow: /forums/index.php?app=forums&module=forums§ion=markasread 135Disallow: /forums/*&do=who 136Disallow: /forums/applications/core/interface/file/attachment.php 137Disallow: /forums/*§ion=dname 138Disallow: /forums/tags/ 139Disallow: */profile/*/reputation/ 140Disallow: /forums/*?do= 141Disallow: /forums/*&do= 142Disallow: /forums/*?sortby= 143Disallow: /forums/*&sortby= 144