1User-Agent: * 2Disallow: /admin/errors/ # skip the errors directory 3Disallow: /admin/intro/history/webstats/ # not reliable 4Disallow: /admin/intro/history/marxists/traffic # not reliable 5Disallow: /admin/new-archives/ # skip news archive directory 6Disallow: /archive/justo/ # mirror 7Disallow: /chinese/update/ # skip chinese news 8Disallow: /espanol/admin/ # skip spanish admin directory 9Disallow: /espanol/justo/ # mirror 10Disallow: /espanol/trotsky/ceip/ # it's not linked to, only goofs up link check 11Disallow: /history/canada/socialisthistory/ # mirror 12Disallow: /korean/trotsky/ # bad links 13Disallow: /cgi-bin/ # scripts 14Disallow: /webstats/ # not reliable 15Crawl-delay: 1 16User-Agent: W3C-checklink 17Disallow: 18#User-Agent: LinkChecker 19#Disallow: 20