1User-agent: 008
2Disallow: /
3
4User-agent: *
5Disallow: /cgi-bin/
6Disallow: /aids/e-newsletter/
7Disallow: /pubtest/
8Disallow: /spanish/aids/e-newsletter/
9Disallow: /french/aids/e-newsletter/
10Disallow: /hiv-aids-sessions/
11Disallow: /2005may/
12Disallow: /int/
13Disallow: /earlychildhood/parenting_cd/
14Disallow: /search/
15Disallow: /french/search/
16Disallow: /spanish/search/
17Disallow: /arabic/search/
18Disallow: /chinese/search/
19Disallow: /emailarticle.php
20Disallow: /voy/discussions/
21Disallow: /*search*
22Disallow: /appeals/tajikistan.html
23Disallow: /appeals/lesotho.html
24Disallow: /appeals/madagascar.html
25Disallow: /appeals/zimbabwe.html
26Disallow: /appeals/pakistan.html
27Disallow: /appeals/burkina_faso.html
28Disallow: /french/eu/crtoolkit/
29Disallow: /spanish/eu/crtoolkit/
30Disallow: /search/devprosearch_education.php
31Disallow: /search/devprosearch_aids.php
32Disallow: /*q=printme
33Disallow: /node/
34
35# Drupal sites
36# For Main site
37
38# CSS, JS, Images
39Allow: /core/*.css$
40Allow: /core/*.css?
41Allow: /core/*.js$
42Allow: /core/*.js?
43Allow: /core/*.gif
44Allow: /core/*.jpg
45Allow: /core/*.jpeg
46Allow: /core/*.png
47Allow: /core/*.svg
48Allow: /profiles/*.css$
49Allow: /profiles/*.css?
50Allow: /profiles/*.js$
51Allow: /profiles/*.js?
52Allow: /profiles/*.gif
53Allow: /profiles/*.jpg
54Allow: /profiles/*.jpeg
55Allow: /profiles/*.png
56Allow: /profiles/*.svg
57# Directories
58Disallow: /core/
59Disallow: /profiles/
60Disallow: /api/country/*
61Disallow: /node/*
62# Files
63Disallow: /README.txt
64Disallow: /web.config
65# Paths (clean URLs)
66Disallow: /admin/
67Disallow: /comment/reply/
68Disallow: /filter/tips/
69Disallow: /node/add/
70Disallow: /search/
71Disallow: /user/register/
72Disallow: /user/password/
73Disallow: /user/login/
74Disallow: /user/logout/
75# Paths (no clean URLs)
76Disallow: /index.php/admin/
77Disallow: /index.php/comment/reply/
78Disallow: /index.php/filter/tips/
79Disallow: /index.php/node/add/
80Disallow: /index.php/search/
81Disallow: /index.php/user/password/
82Disallow: /index.php/user/register/
83Disallow: /index.php/user/login/
84Disallow: /index.php/user/logout/
85
86# For ROCO sites
87# CSS, JS, Images
88Allow: /*/core/*.css$
89Allow: /*/core/*.css?
90Allow: /*/core/*.js$
91Allow: /*/core/*.js?
92Allow: /*/core/*.gif
93Allow: /*/core/*.jpg
94Allow: /*/core/*.jpeg
95Allow: /*/core/*.png
96Allow: /*/core/*.svg
97Allow: /*/profiles/*.css$
98Allow: /*/profiles/*.css?
99Allow: /*/profiles/*.js$
100Allow: /*/profiles/*.js?
101Allow: /*/profiles/*.gif
102Allow: /*/profiles/*.jpg
103Allow: /*/profiles/*.jpeg
104Allow: /*/profiles/*.png
105Allow: /*/profiles/*.svg
106# Directories
107Disallow: /*/core/
108Disallow: /*/profiles/
109# Files
110Disallow: /*/README.txt
111Disallow: /*/web.config
112# Paths (clean URLs)
113Disallow: /*/admin/
114Disallow: /*/comment/reply/
115Disallow: /*/filter/tips/
116Disallow: /*/node/add/
117Disallow: /*/search/
118Disallow: /*/user/register/
119Disallow: /*/user/password/
120Disallow: /*/user/login/
121Disallow: /*/user/logout/
122# Paths (no clean URLs)
123Disallow: /*/index.php/admin/
124Disallow: /*/index.php/comment/reply/
125Disallow: /*/index.php/filter/tips/
126Disallow: /*/index.php/node/add/
127Disallow: /*/index.php/search/
128Disallow: /*/index.php/user/password/
129Disallow: /*/index.php/user/register/
130Disallow: /*/index.php/user/login/
131Disallow: /*/index.php/user/logout/
132
133
134Crawl-delay: 10
135
136User-agent: Yandex
137Crawl-delay: 30
138
139User-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36
140Crawl-delay: 3
141
142User-agent: reddot-scraper.unicef.org
143Crawl-delay: 1
144
145