1User-agent: 008 2Disallow: / 3 4User-agent: * 5Disallow: /cgi-bin/ 6Disallow: /aids/e-newsletter/ 7Disallow: /pubtest/ 8Disallow: /spanish/aids/e-newsletter/ 9Disallow: /french/aids/e-newsletter/ 10Disallow: /hiv-aids-sessions/ 11Disallow: /2005may/ 12Disallow: /int/ 13Disallow: /earlychildhood/parenting_cd/ 14Disallow: /search/ 15Disallow: /french/search/ 16Disallow: /spanish/search/ 17Disallow: /arabic/search/ 18Disallow: /chinese/search/ 19Disallow: /emailarticle.php 20Disallow: /voy/discussions/ 21Disallow: /*search* 22Disallow: /appeals/tajikistan.html 23Disallow: /appeals/lesotho.html 24Disallow: /appeals/madagascar.html 25Disallow: /appeals/zimbabwe.html 26Disallow: /appeals/pakistan.html 27Disallow: /appeals/burkina_faso.html 28Disallow: /french/eu/crtoolkit/ 29Disallow: /spanish/eu/crtoolkit/ 30Disallow: /search/devprosearch_education.php 31Disallow: /search/devprosearch_aids.php 32Disallow: /*q=printme 33Disallow: /node/ 34 35# Drupal sites 36# For Main site 37 38# CSS, JS, Images 39Allow: /core/*.css$ 40Allow: /core/*.css? 41Allow: /core/*.js$ 42Allow: /core/*.js? 43Allow: /core/*.gif 44Allow: /core/*.jpg 45Allow: /core/*.jpeg 46Allow: /core/*.png 47Allow: /core/*.svg 48Allow: /profiles/*.css$ 49Allow: /profiles/*.css? 50Allow: /profiles/*.js$ 51Allow: /profiles/*.js? 52Allow: /profiles/*.gif 53Allow: /profiles/*.jpg 54Allow: /profiles/*.jpeg 55Allow: /profiles/*.png 56Allow: /profiles/*.svg 57# Directories 58Disallow: /core/ 59Disallow: /profiles/ 60Disallow: /api/country/* 61Disallow: /node/* 62# Files 63Disallow: /README.txt 64Disallow: /web.config 65# Paths (clean URLs) 66Disallow: /admin/ 67Disallow: /comment/reply/ 68Disallow: /filter/tips/ 69Disallow: /node/add/ 70Disallow: /search/ 71Disallow: /user/register/ 72Disallow: /user/password/ 73Disallow: /user/login/ 74Disallow: /user/logout/ 75# Paths (no clean URLs) 76Disallow: /index.php/admin/ 77Disallow: /index.php/comment/reply/ 78Disallow: /index.php/filter/tips/ 79Disallow: /index.php/node/add/ 80Disallow: /index.php/search/ 81Disallow: /index.php/user/password/ 82Disallow: /index.php/user/register/ 83Disallow: /index.php/user/login/ 84Disallow: /index.php/user/logout/ 85 86# For ROCO sites 87# CSS, JS, Images 88Allow: /*/core/*.css$ 89Allow: /*/core/*.css? 90Allow: /*/core/*.js$ 91Allow: /*/core/*.js? 92Allow: /*/core/*.gif 93Allow: /*/core/*.jpg 94Allow: /*/core/*.jpeg 95Allow: /*/core/*.png 96Allow: /*/core/*.svg 97Allow: /*/profiles/*.css$ 98Allow: /*/profiles/*.css? 99Allow: /*/profiles/*.js$ 100Allow: /*/profiles/*.js? 101Allow: /*/profiles/*.gif 102Allow: /*/profiles/*.jpg 103Allow: /*/profiles/*.jpeg 104Allow: /*/profiles/*.png 105Allow: /*/profiles/*.svg 106# Directories 107Disallow: /*/core/ 108Disallow: /*/profiles/ 109# Files 110Disallow: /*/README.txt 111Disallow: /*/web.config 112# Paths (clean URLs) 113Disallow: /*/admin/ 114Disallow: /*/comment/reply/ 115Disallow: /*/filter/tips/ 116Disallow: /*/node/add/ 117Disallow: /*/search/ 118Disallow: /*/user/register/ 119Disallow: /*/user/password/ 120Disallow: /*/user/login/ 121Disallow: /*/user/logout/ 122# Paths (no clean URLs) 123Disallow: /*/index.php/admin/ 124Disallow: /*/index.php/comment/reply/ 125Disallow: /*/index.php/filter/tips/ 126Disallow: /*/index.php/node/add/ 127Disallow: /*/index.php/search/ 128Disallow: /*/index.php/user/password/ 129Disallow: /*/index.php/user/register/ 130Disallow: /*/index.php/user/login/ 131Disallow: /*/index.php/user/logout/ 132 133 134Crawl-delay: 10 135 136User-agent: Yandex 137Crawl-delay: 30 138 139User-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 140Crawl-delay: 3 141 142User-agent: reddot-scraper.unicef.org 143Crawl-delay: 1 144 145