1# www.robotstxt.org/ 2# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449 3 4User-agent: * 5 6Disallow: /user/* 7Disallow: /members/ 8Disallow: /members/newsletters/ 9Disallow: /members/alerts/add/ 10 11Disallow: /search/?* 12 13Disallow: *Xhr* 14Disallow: */xhr* 15Disallow: */ajax/* 16 17Disallow: */fly/*/bundles/flyjs/* 18Disallow: */libs/* 19Disallow: */version!libs/* 20Disallow: /.well-known/* 21 22Disallow: /index.php/* 23 24Disallow: *?beta=* 25 26# News 27Sitemap: https://www.zdnet.com/sitemaps/news.xml 28 29# Topics 30Sitemap: https://www.zdnet.com/sitemaps/topics.xml 31 32# Article 33Sitemap: https://www.zdnet.com/sitemaps/article/index.xml 34 35# Gallery 36Sitemap: https://www.zdnet.com/sitemaps/gallery/index.xml 37 38# Video 39Sitemap: https://www.zdnet.com/sitemaps/video/index.xml 40 41# Downloads 42Sitemap: https://www.zdnet.com/sitemaps/downloads/index.xml 43 44# Review 45Sitemap: https://www.zdnet.com/sitemaps/review/index.xml 46 47User-agent: msnbot 48Crawl-delay: 1 49