1# allowed spiders
2User-agent: googlebot
3User-agent: Googlebot-Mobile
4User-agent: Mediapartners-Google
5User-agent: bingbot
6User-agent: msnbot
7User-agent: slurp
8User-agent: duckduckbot
9User-agent: baiduspider
10User-agent: ia_archiver
11User-agent: Applebot
12Disallow: /admin
13Disallow: /admin-bc
14Disallow: /api
15Disallow: /application
16Disallow: /profile
17Disallow: /resource
18Disallow: /whim
19Disallow:
20
21# all other spiders disallowed
22User-agent: *
23Disallow: /