1## robots.txt for http://www.ams.org/ ## 2## To block pages from search results, use methods such as password protection or noindex tags or directives ## 3## https://developers.google.com/search/reference/robots_txt ## 4 5## SITEMAPS ## 6Sitemap: http://www.ams.org/sitemap.xml 7Sitemap: http://www.ams.org/sitemap_news.xml 8Sitemap: http://www.ams.org/sitemapindex.xml 9 10## SPECIFICALLY ALLOW ## 11 12## Allow Google to get our news RSS feed ## 13User-agent: Googlebot 14Allow: /cgi-bin/content/news_items.cgi?rss=1 15 16## Allow so Google can get lists of issues ## 17Allow: /jourcgi 18 19## SPECIFICALLY DISALLOW THESE AGENTS ## 20 21User-agent: Slurp 22Disallow: /mathscinet-getitem/ 23 24User-agent: Baiduspider 25Disallow: / 26 27## SPECIFICALLY DISALLOW ALL AGENTS FROM THE FOLLOWING AREAS ## 28 29User-agent: * 30 31## JOURNALS ## 32Disallow: /distribution/ 33Disallow: /editflow/ 34Disallow: /cgi-bin/peerdemo/ 35Disallow: /peertrack/ 36Disallow: /ams/web/peer_review/bull/ 37Disallow: /jour-getitem/ 38Disallow: /journal-getitem/ 39Disallow: /joursearch/ 40Disallow: /peer-review/ 41Disallow: /submit-book-journal/ 42Disallow: /peerdomo/ 43 44## MATHSCINET ## 45Disallow: /leavingmsn/ 46Disallow: /msnpdf/ 47Disallow: /mathscinet-docdelivery/ 48Disallow: /mathscinet-relay/ 49Disallow: /mathscinet/ 50Disallow: /mathscinet/addclip.html 51Disallow: /msnhtml/ 52Disallow: /mathscinet-getauth 53Disallow: /mref/ 54Disallow: /batchmref/ 55Disallow: /msdemo/ 56Disallow: /cgi-bin/msdemo/ 57Disallow: /msnmain 58Disallow: /mrlookup/ 59Disallow: /batchmrlookup/ 60Disallow: /mathscinet-getitem/ 61Disallow: /mresub/ 62 63## BOOKS AND BOOKSTORE ## 64Disallow: /online_bks/ 65Disallow: /bookexpress 66Disallow: /amsbookstore 67Disallow: /amsshop 68 69## PRIVATE ## 70Disallow: /ecbt/ 71Disallow: /treas-reports/ 72Disallow: /ams/staffdirectory.html 73Disallow: /investcom/ 74Disallow: /prof-services/nsa/ 75Disallow: /ebc-private/bboard2002/ 76Disallow: /vendor-info/ 77Disallow: /bboards/ 78 79## CML ## 80Disallow: /cml/ 81Disallow: /cml-lookup/ 82Disallow: /cml* 83Disallow: /member-getitem* 84Disallow: /member-directory 85 86## STIX ## 87Disallow: /STIX/ 88Disallow: /STIX/images/ 89Disallow: /STIX/ion/Univ/ 90 91## PAO ## 92Disallow: /ams/press/ 93 94## MEETINGS ## 95Disallow: /amsmtgs/2124_abstracts/1056-30-23.pdf 96Disallow: /meetings/national/jmm/2124_abstracts/1056-30-23.pdf 97 98## ANNUAL SURVEY ## 99Disallow: /profession/data/annual-survey/AS-Committee/ 100 101## PROFESSIONAL SERVICES ## 102Disallow: /prof-services/ 103 104## MISCELLANEOUS ## 105Disallow: /ftp/ 106Disallow: /delete/ 107Disallow: /cgi-bin 108Disallow: /testing/ 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124