1User-Agent: *
2Disallow: /admin/errors/ # skip the errors directory
3Disallow: /admin/intro/history/webstats/ # not reliable
4Disallow: /admin/intro/history/marxists/traffic # not reliable
5Disallow: /admin/new-archives/ # skip news archive directory
6Disallow: /archive/justo/ # mirror
7Disallow: /chinese/update/ # skip chinese news
8Disallow: /espanol/admin/ # skip spanish admin directory
9Disallow: /espanol/justo/ # mirror
10Disallow: /espanol/trotsky/ceip/ # it's not linked to, only goofs up link check
11Disallow: /history/canada/socialisthistory/ # mirror
12Disallow: /korean/trotsky/ # bad links
13Disallow: /cgi-bin/ # scripts
14Disallow: /webstats/ # not reliable
15Crawl-delay: 1
16User-Agent: W3C-checklink
17Disallow:
18#User-Agent: LinkChecker
19#Disallow:
20