1
2User-agent: Mediapartners-Google
3Disallow:
4
5User-agent: SemrushBot
6Disallow: /
7
8User-agent: SemrushBot-SA
9Disallow: /
10
11User-Agent: AhrefsBot
12Disallow: /
13
14User-Agent: cis455crawler
15Disallow:/
16
17User-Agent: netseer
18Disallow:/
19
20User-Agent: coccoc
21Disallow:/
22
23User-agent: CrazyWebCrawler-Spider
24Disallow: /
25
26User-agent: Mail.Ru
27Disallow: /
28User-agent: Mail.RU_Bot*
29Disallow: /
30
31User-Agent: ParadigmCrawler*
32Disallow:/
33
34User-Agent: DotBot*
35Disallow:/
36
37User-Agent: Clickagy*
38Disallow:/
39
40User-Agent: Pu_iN*
41Disallow:/
42
43User-Agent: CRAZYWEBCRAWLER*
44Disallow:/
45
46User-Agent: FAST*
47Disallow:/
48
49User-Agent: Bender
50Disallow:/
51
52User-agent: MJ12bot
53Disallow: /
54
55User-Agent: MSIECrawler
56Disallow:/
57
58User-agent: NPBot
59Disallow: /
60
61User-agent: msrbot
62Disallow: /
63
64User-Agent: Open*
65Disallow: /
66
67User-Agent: Twiceler
68Disallow: /
69
70User-Agent: Scooter*
71Disallow: /
72
73User-Agent: WebSearch*
74Disallow: /
75
76User-Agent: ZyBorg*
77Disallow: /
78
79User-Agent: Wget*
80Disallow: /
81
82User-Agent: TurnitinBot*
83Disallow: /
84
85User-Agent: SlySearch*
86Disallow: /
87
88User-Agent: Seekbot*
89Disallow: /
90
91User-Agent: NetResearchServer*
92Disallow: /
93
94User-Agent: Seekbot*
95Disallow: /
96
97User-Agent: mozDex*
98Disallow: /
99
100User-Agent: Link*
101Disallow: /
102
103User-Agent: larbin*
104Disallow: /
105
106User-Agent: ia_archiver
107Disallow: /
108
109User-Agent: Charlotte
110Disallow: /
111
112User-agent: discobot
113Disallow: /
114
115User-agent: 008
116Disallow: /
117
118User-agent: *
119Disallow: /p/
120Disallow: /d/
121Disallow: /_/misc/
122Disallow: /_/search/
123Disallow: /_/cite.htm
124Disallow: /_/gr.aspx
125Disallow: /_/hp/Controls/printer-friendly.aspx
126Disallow: /unscramble/
127Sitemap: https://www.thefreedictionary.com/www.thefreedictionary.com.xml.gz