1# robots.txt for Slashdot.org 2# $Id$ 3# "Any empty [Disallow] value, indicates that all URLs can be retrieved. 4# At least one Disallow field needs to be present in a record." 5 6User-agent: Mediapartners-Google 7Disallow: 8 9User-agent: Googlebot 10Disallow: /firehose.pl 11Disallow: /submit.pl 12Disallow: /comments.pl 13Disallow: /users.pl 14Disallow: /zoo.pl 15Disallow: firehose.pl 16Disallow: submit.pl 17Disallow: comments.pl 18Disallow: users.pl 19Disallow: zoo.pl 20Disallow: /~ 21Disallow: ~ 22 23User-agent: Slurp 24Disallow: 25 26User-agent: Yahoo-NewsCrawler 27Disallow: 28 29User-Agent: msnbot 30Disallow: 31 32User-agent: * 33Disallow: /authors.pl 34Disallow: /index.pl 35Disallow: /comments.pl 36Disallow: /firehose.pl 37Disallow: /journal.pl 38Disallow: /messages.pl 39Disallow: /metamod.pl 40Disallow: /users.pl 41Disallow: /search.pl 42Disallow: /submit.pl 43Disallow: /pollBooth.pl 44Disallow: /pubkey.pl 45Disallow: /topics.pl 46Disallow: /zoo.pl 47Disallow: /palm 48Disallow: /slashdot-it.pl 49Disallow: slashdot-it.pl 50Disallow: authors.pl 51Disallow: index.pl 52Disallow: comments.pl 53Disallow: firehose.pl 54Disallow: journal.pl 55Disallow: messages.pl 56Disallow: metamod.pl 57Disallow: users.pl 58Disallow: search.pl 59Disallow: submit.pl 60Disallow: pollBooth.pl 61Disallow: pubkey.pl 62Disallow: topics.pl 63Disallow: zoo.pl 64Disallow: /~ 65Disallow: ~ 66