1# 2# robots.txt 3# 4# This file is to prevent the crawling and indexing of certain parts 5# of your site by web crawlers and spiders run by sites like Yahoo! 6# and Google. By telling these "robots" where not to go on your site, 7# you save bandwidth and server resources. 8# 9# This file will be ignored unless it is at the root of your host: 10# Used: http://example.com/robots.txt 11# Ignored: http://example.com/site/robots.txt 12# 13# For more information about the robots.txt standard, see: 14# http://www.robotstxt.org/robotstxt.html 15 16User-agent: * 17# CSS, JS, Images 18Allow: /core/*.css$ 19Allow: /core/*.css? 20Allow: /core/*.js$ 21Allow: /core/*.js? 22Allow: /core/*.gif 23Allow: /core/*.jpg 24Allow: /core/*.jpeg 25Allow: /core/*.png 26Allow: /core/*.svg 27Allow: /profiles/*.css$ 28Allow: /profiles/*.css? 29Allow: /profiles/*.js$ 30Allow: /profiles/*.js? 31Allow: /profiles/*.gif 32Allow: /profiles/*.jpg 33Allow: /profiles/*.jpeg 34Allow: /profiles/*.png 35Allow: /profiles/*.svg 36# Directories 37Disallow: /core/ 38Disallow: /profiles/ 39# Files 40Disallow: /README.txt 41Disallow: /web.config 42# Paths (clean URLs) 43Disallow: /admin/ 44Disallow: /comment/reply/ 45Disallow: /filter/tips 46Disallow: /node/add/ 47Disallow: /search/ 48Disallow: /user/register/ 49Disallow: /user/password/ 50Disallow: /user/login/ 51Disallow: /user/logout/ 52# Paths (no clean URLs) 53Disallow: /index.php/admin/ 54Disallow: /index.php/comment/reply/ 55Disallow: /index.php/filter/tips 56Disallow: /index.php/node/add/ 57Disallow: /index.php/search/ 58Disallow: /index.php/user/password/ 59Disallow: /index.php/user/register/ 60Disallow: /index.php/user/login/ 61Disallow: /index.php/user/logout/ 62Disallow: /en-en/app-privacy-policy 63 64Disallow: /it-en/unica-privacy-policy 65Disallow: /en-en/unica-privacy-policy 66Disallow: /cn-en/unica-privacy-policy 67Disallow: /jp-en/unica-privacy-policy 68Disallow: /de-en/unica-privacy-policy 69Disallow: /es-en/unica-privacy-policy 70Disallow: /fr-en/unica-privacy-policy 71Disallow: /ru-en/unica-privacy-policy 72 73