1import unittest
2from unittest import mock
3
4from scrapy.settings import (BaseSettings, Settings, SettingsAttribute,
5                             SETTINGS_PRIORITIES, get_settings_priority)
6from . import default_settings
7
8
9class SettingsGlobalFuncsTest(unittest.TestCase):
10
11    def test_get_settings_priority(self):
12        for prio_str, prio_num in SETTINGS_PRIORITIES.items():
13            self.assertEqual(get_settings_priority(prio_str), prio_num)
14        self.assertEqual(get_settings_priority(99), 99)
15
16
17class SettingsAttributeTest(unittest.TestCase):
18
19    def setUp(self):
20        self.attribute = SettingsAttribute('value', 10)
21
22    def test_set_greater_priority(self):
23        self.attribute.set('value2', 20)
24        self.assertEqual(self.attribute.value, 'value2')
25        self.assertEqual(self.attribute.priority, 20)
26
27    def test_set_equal_priority(self):
28        self.attribute.set('value2', 10)
29        self.assertEqual(self.attribute.value, 'value2')
30        self.assertEqual(self.attribute.priority, 10)
31
32    def test_set_less_priority(self):
33        self.attribute.set('value2', 0)
34        self.assertEqual(self.attribute.value, 'value')
35        self.assertEqual(self.attribute.priority, 10)
36
37    def test_overwrite_basesettings(self):
38        original_dict = {'one': 10, 'two': 20}
39        original_settings = BaseSettings(original_dict, 0)
40        attribute = SettingsAttribute(original_settings, 0)
41
42        new_dict = {'three': 11, 'four': 21}
43        attribute.set(new_dict, 10)
44        self.assertIsInstance(attribute.value, BaseSettings)
45        self.assertCountEqual(attribute.value, new_dict)
46        self.assertCountEqual(original_settings, original_dict)
47
48        new_settings = BaseSettings({'five': 12}, 0)
49        attribute.set(new_settings, 0)  # Insufficient priority
50        self.assertCountEqual(attribute.value, new_dict)
51        attribute.set(new_settings, 10)
52        self.assertCountEqual(attribute.value, new_settings)
53
54    def test_repr(self):
55        self.assertEqual(repr(self.attribute),
56                         "<SettingsAttribute value='value' priority=10>")
57
58
59class BaseSettingsTest(unittest.TestCase):
60
61    def setUp(self):
62        self.settings = BaseSettings()
63
64    def test_set_new_attribute(self):
65        self.settings.set('TEST_OPTION', 'value', 0)
66        self.assertIn('TEST_OPTION', self.settings.attributes)
67
68        attr = self.settings.attributes['TEST_OPTION']
69        self.assertIsInstance(attr, SettingsAttribute)
70        self.assertEqual(attr.value, 'value')
71        self.assertEqual(attr.priority, 0)
72
73    def test_set_settingsattribute(self):
74        myattr = SettingsAttribute(0, 30)  # Note priority 30
75        self.settings.set('TEST_ATTR', myattr, 10)
76        self.assertEqual(self.settings.get('TEST_ATTR'), 0)
77        self.assertEqual(self.settings.getpriority('TEST_ATTR'), 30)
78
79    def test_set_instance_identity_on_update(self):
80        attr = SettingsAttribute('value', 0)
81        self.settings.attributes = {'TEST_OPTION': attr}
82        self.settings.set('TEST_OPTION', 'othervalue', 10)
83
84        self.assertIn('TEST_OPTION', self.settings.attributes)
85        self.assertIs(attr, self.settings.attributes['TEST_OPTION'])
86
87    def test_set_calls_settings_attributes_methods_on_update(self):
88        attr = SettingsAttribute('value', 10)
89        with mock.patch.object(attr, '__setattr__') as mock_setattr, mock.patch.object(attr, 'set') as mock_set:
90
91            self.settings.attributes = {'TEST_OPTION': attr}
92
93            for priority in (0, 10, 20):
94                self.settings.set('TEST_OPTION', 'othervalue', priority)
95                mock_set.assert_called_once_with('othervalue', priority)
96                self.assertFalse(mock_setattr.called)
97                mock_set.reset_mock()
98                mock_setattr.reset_mock()
99
100    def test_setitem(self):
101        settings = BaseSettings()
102        settings.set('key', 'a', 'default')
103        settings['key'] = 'b'
104        self.assertEqual(settings['key'], 'b')
105        self.assertEqual(settings.getpriority('key'), 20)
106        settings['key'] = 'c'
107        self.assertEqual(settings['key'], 'c')
108        settings['key2'] = 'x'
109        self.assertIn('key2', settings)
110        self.assertEqual(settings['key2'], 'x')
111        self.assertEqual(settings.getpriority('key2'), 20)
112
113    def test_setdict_alias(self):
114        with mock.patch.object(self.settings, 'set') as mock_set:
115            self.settings.setdict({'TEST_1': 'value1', 'TEST_2': 'value2'}, 10)
116            self.assertEqual(mock_set.call_count, 2)
117            calls = [mock.call('TEST_1', 'value1', 10),
118                     mock.call('TEST_2', 'value2', 10)]
119            mock_set.assert_has_calls(calls, any_order=True)
120
121    def test_setmodule_only_load_uppercase_vars(self):
122        class ModuleMock():
123            UPPERCASE_VAR = 'value'
124            MIXEDcase_VAR = 'othervalue'
125            lowercase_var = 'anothervalue'
126
127        self.settings.attributes = {}
128        self.settings.setmodule(ModuleMock(), 10)
129        self.assertIn('UPPERCASE_VAR', self.settings.attributes)
130        self.assertNotIn('MIXEDcase_VAR', self.settings.attributes)
131        self.assertNotIn('lowercase_var', self.settings.attributes)
132        self.assertEqual(len(self.settings.attributes), 1)
133
134    def test_setmodule_alias(self):
135        with mock.patch.object(self.settings, 'set') as mock_set:
136            self.settings.setmodule(default_settings, 10)
137            mock_set.assert_any_call('TEST_DEFAULT', 'defvalue', 10)
138            mock_set.assert_any_call('TEST_DICT', {'key': 'val'}, 10)
139
140    def test_setmodule_by_path(self):
141        self.settings.attributes = {}
142        self.settings.setmodule(default_settings, 10)
143        ctrl_attributes = self.settings.attributes.copy()
144
145        self.settings.attributes = {}
146        self.settings.setmodule(
147            'tests.test_settings.default_settings', 10)
148
149        self.assertCountEqual(self.settings.attributes.keys(),
150                              ctrl_attributes.keys())
151
152        for key in ctrl_attributes.keys():
153            attr = self.settings.attributes[key]
154            ctrl_attr = ctrl_attributes[key]
155            self.assertEqual(attr.value, ctrl_attr.value)
156            self.assertEqual(attr.priority, ctrl_attr.priority)
157
158    def test_update(self):
159        settings = BaseSettings({'key_lowprio': 0}, priority=0)
160        settings.set('key_highprio', 10, priority=50)
161        custom_settings = BaseSettings({'key_lowprio': 1, 'key_highprio': 11},
162                                       priority=30)
163        custom_settings.set('newkey_one', None, priority=50)
164        custom_dict = {'key_lowprio': 2, 'key_highprio': 12, 'newkey_two': None}
165
166        settings.update(custom_dict, priority=20)
167        self.assertEqual(settings['key_lowprio'], 2)
168        self.assertEqual(settings.getpriority('key_lowprio'), 20)
169        self.assertEqual(settings['key_highprio'], 10)
170        self.assertIn('newkey_two', settings)
171        self.assertEqual(settings.getpriority('newkey_two'), 20)
172
173        settings.update(custom_settings)
174        self.assertEqual(settings['key_lowprio'], 1)
175        self.assertEqual(settings.getpriority('key_lowprio'), 30)
176        self.assertEqual(settings['key_highprio'], 10)
177        self.assertIn('newkey_one', settings)
178        self.assertEqual(settings.getpriority('newkey_one'), 50)
179
180        settings.update({'key_lowprio': 3}, priority=20)
181        self.assertEqual(settings['key_lowprio'], 1)
182
183    def test_update_jsonstring(self):
184        settings = BaseSettings({'number': 0, 'dict': BaseSettings({'key': 'val'})})
185        settings.update('{"number": 1, "newnumber": 2}')
186        self.assertEqual(settings['number'], 1)
187        self.assertEqual(settings['newnumber'], 2)
188        settings.set("dict", '{"key": "newval", "newkey": "newval2"}')
189        self.assertEqual(settings['dict']['key'], "newval")
190        self.assertEqual(settings['dict']['newkey'], "newval2")
191
192    def test_delete(self):
193        settings = BaseSettings({'key': None})
194        settings.set('key_highprio', None, priority=50)
195        settings.delete('key')
196        settings.delete('key_highprio')
197        self.assertNotIn('key', settings)
198        self.assertIn('key_highprio', settings)
199        del settings['key_highprio']
200        self.assertNotIn('key_highprio', settings)
201
202    def test_get(self):
203        test_configuration = {
204            'TEST_ENABLED1': '1',
205            'TEST_ENABLED2': True,
206            'TEST_ENABLED3': 1,
207            'TEST_ENABLED4': 'True',
208            'TEST_ENABLED5': 'true',
209            'TEST_ENABLED_WRONG': 'on',
210            'TEST_DISABLED1': '0',
211            'TEST_DISABLED2': False,
212            'TEST_DISABLED3': 0,
213            'TEST_DISABLED4': 'False',
214            'TEST_DISABLED5': 'false',
215            'TEST_DISABLED_WRONG': 'off',
216            'TEST_INT1': 123,
217            'TEST_INT2': '123',
218            'TEST_FLOAT1': 123.45,
219            'TEST_FLOAT2': '123.45',
220            'TEST_LIST1': ['one', 'two'],
221            'TEST_LIST2': 'one,two',
222            'TEST_STR': 'value',
223            'TEST_DICT1': {'key1': 'val1', 'ke2': 3},
224            'TEST_DICT2': '{"key1": "val1", "ke2": 3}',
225        }
226        settings = self.settings
227        settings.attributes = {key: SettingsAttribute(value, 0) for key, value
228                               in test_configuration.items()}
229
230        self.assertTrue(settings.getbool('TEST_ENABLED1'))
231        self.assertTrue(settings.getbool('TEST_ENABLED2'))
232        self.assertTrue(settings.getbool('TEST_ENABLED3'))
233        self.assertTrue(settings.getbool('TEST_ENABLED4'))
234        self.assertTrue(settings.getbool('TEST_ENABLED5'))
235        self.assertFalse(settings.getbool('TEST_ENABLEDx'))
236        self.assertTrue(settings.getbool('TEST_ENABLEDx', True))
237        self.assertFalse(settings.getbool('TEST_DISABLED1'))
238        self.assertFalse(settings.getbool('TEST_DISABLED2'))
239        self.assertFalse(settings.getbool('TEST_DISABLED3'))
240        self.assertFalse(settings.getbool('TEST_DISABLED4'))
241        self.assertFalse(settings.getbool('TEST_DISABLED5'))
242        self.assertEqual(settings.getint('TEST_INT1'), 123)
243        self.assertEqual(settings.getint('TEST_INT2'), 123)
244        self.assertEqual(settings.getint('TEST_INTx'), 0)
245        self.assertEqual(settings.getint('TEST_INTx', 45), 45)
246        self.assertEqual(settings.getfloat('TEST_FLOAT1'), 123.45)
247        self.assertEqual(settings.getfloat('TEST_FLOAT2'), 123.45)
248        self.assertEqual(settings.getfloat('TEST_FLOATx'), 0.0)
249        self.assertEqual(settings.getfloat('TEST_FLOATx', 55.0), 55.0)
250        self.assertEqual(settings.getlist('TEST_LIST1'), ['one', 'two'])
251        self.assertEqual(settings.getlist('TEST_LIST2'), ['one', 'two'])
252        self.assertEqual(settings.getlist('TEST_LISTx'), [])
253        self.assertEqual(settings.getlist('TEST_LISTx', ['default']), ['default'])
254        self.assertEqual(settings['TEST_STR'], 'value')
255        self.assertEqual(settings.get('TEST_STR'), 'value')
256        self.assertEqual(settings['TEST_STRx'], None)
257        self.assertEqual(settings.get('TEST_STRx'), None)
258        self.assertEqual(settings.get('TEST_STRx', 'default'), 'default')
259        self.assertEqual(settings.getdict('TEST_DICT1'), {'key1': 'val1', 'ke2': 3})
260        self.assertEqual(settings.getdict('TEST_DICT2'), {'key1': 'val1', 'ke2': 3})
261        self.assertEqual(settings.getdict('TEST_DICT3'), {})
262        self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
263        self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
264        self.assertRaises(ValueError, settings.getbool, 'TEST_ENABLED_WRONG')
265        self.assertRaises(ValueError, settings.getbool, 'TEST_DISABLED_WRONG')
266
267    def test_getpriority(self):
268        settings = BaseSettings({'key': 'value'}, priority=99)
269        self.assertEqual(settings.getpriority('key'), 99)
270        self.assertEqual(settings.getpriority('nonexistentkey'), None)
271
272    def test_getwithbase(self):
273        s = BaseSettings({'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
274                          'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
275                          'HASNOBASE': BaseSettings({3: 3000}, 'default')})
276        s['TEST'].set(2, 200, 'cmdline')
277        self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30})
278        self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE'])
279        self.assertEqual(s.getwithbase('NONEXISTENT'), {})
280
281    def test_maxpriority(self):
282        # Empty settings should return 'default'
283        self.assertEqual(self.settings.maxpriority(), 0)
284        self.settings.set('A', 0, 10)
285        self.settings.set('B', 0, 30)
286        self.assertEqual(self.settings.maxpriority(), 30)
287
288    def test_copy(self):
289        values = {
290            'TEST_BOOL': True,
291            'TEST_LIST': ['one', 'two'],
292            'TEST_LIST_OF_LISTS': [['first_one', 'first_two'],
293                                   ['second_one', 'second_two']]
294        }
295        self.settings.setdict(values)
296        copy = self.settings.copy()
297        self.settings.set('TEST_BOOL', False)
298        self.assertTrue(copy.get('TEST_BOOL'))
299
300        test_list = self.settings.get('TEST_LIST')
301        test_list.append('three')
302        self.assertListEqual(copy.get('TEST_LIST'), ['one', 'two'])
303
304        test_list_of_lists = self.settings.get('TEST_LIST_OF_LISTS')
305        test_list_of_lists[0].append('first_three')
306        self.assertListEqual(copy.get('TEST_LIST_OF_LISTS')[0],
307                             ['first_one', 'first_two'])
308
309    def test_copy_to_dict(self):
310        s = BaseSettings({'TEST_STRING': 'a string',
311                          'TEST_LIST': [1, 2],
312                          'TEST_BOOLEAN': False,
313                          'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
314                          'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
315                          'HASNOBASE': BaseSettings({3: 3000}, 'default')})
316        self.assertDictEqual(
317            s.copy_to_dict(),
318            {
319                'HASNOBASE': {3: 3000},
320                'TEST': {1: 10, 3: 30},
321                'TEST_BASE': {1: 1, 2: 2},
322                'TEST_LIST': [1, 2],
323                'TEST_BOOLEAN': False,
324                'TEST_STRING': 'a string',
325            }
326        )
327
328    def test_freeze(self):
329        self.settings.freeze()
330        with self.assertRaises(TypeError) as cm:
331            self.settings.set('TEST_BOOL', False)
332            self.assertEqual(str(cm.exception),
333                             "Trying to modify an immutable Settings object")
334
335    def test_frozencopy(self):
336        frozencopy = self.settings.frozencopy()
337        self.assertTrue(frozencopy.frozen)
338        self.assertIsNot(frozencopy, self.settings)
339
340
341class SettingsTest(unittest.TestCase):
342
343    def setUp(self):
344        self.settings = Settings()
345
346    @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {'default': 10})
347    @mock.patch('scrapy.settings.default_settings', default_settings)
348    def test_initial_defaults(self):
349        settings = Settings()
350        self.assertEqual(len(settings.attributes), 2)
351        self.assertIn('TEST_DEFAULT', settings.attributes)
352
353        attr = settings.attributes['TEST_DEFAULT']
354        self.assertIsInstance(attr, SettingsAttribute)
355        self.assertEqual(attr.value, 'defvalue')
356        self.assertEqual(attr.priority, 10)
357
358    @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {})
359    @mock.patch('scrapy.settings.default_settings', {})
360    def test_initial_values(self):
361        settings = Settings({'TEST_OPTION': 'value'}, 10)
362        self.assertEqual(len(settings.attributes), 1)
363        self.assertIn('TEST_OPTION', settings.attributes)
364
365        attr = settings.attributes['TEST_OPTION']
366        self.assertIsInstance(attr, SettingsAttribute)
367        self.assertEqual(attr.value, 'value')
368        self.assertEqual(attr.priority, 10)
369
370    @mock.patch('scrapy.settings.default_settings', default_settings)
371    def test_autopromote_dicts(self):
372        settings = Settings()
373        mydict = settings.get('TEST_DICT')
374        self.assertIsInstance(mydict, BaseSettings)
375        self.assertIn('key', mydict)
376        self.assertEqual(mydict['key'], 'val')
377        self.assertEqual(mydict.getpriority('key'), 0)
378
379    @mock.patch('scrapy.settings.default_settings', default_settings)
380    def test_getdict_autodegrade_basesettings(self):
381        settings = Settings()
382        mydict = settings.getdict('TEST_DICT')
383        self.assertIsInstance(mydict, dict)
384        self.assertEqual(len(mydict), 1)
385        self.assertIn('key', mydict)
386        self.assertEqual(mydict['key'], 'val')
387
388    def test_passing_objects_as_values(self):
389        from scrapy.core.downloader.handlers.file import FileDownloadHandler
390        from scrapy.utils.misc import create_instance
391        from scrapy.utils.test import get_crawler
392
393        class TestPipeline():
394            def process_item(self, i, s):
395                return i
396
397        settings = Settings({
398            'ITEM_PIPELINES': {
399                TestPipeline: 800,
400            },
401            'DOWNLOAD_HANDLERS': {
402                'ftp': FileDownloadHandler,
403            },
404        })
405
406        self.assertIn('ITEM_PIPELINES', settings.attributes)
407
408        mypipeline, priority = settings.getdict('ITEM_PIPELINES').popitem()
409        self.assertEqual(priority, 800)
410        self.assertEqual(mypipeline, TestPipeline)
411        self.assertIsInstance(mypipeline(), TestPipeline)
412        self.assertEqual(mypipeline().process_item('item', None), 'item')
413
414        myhandler = settings.getdict('DOWNLOAD_HANDLERS').pop('ftp')
415        self.assertEqual(myhandler, FileDownloadHandler)
416        myhandler_instance = create_instance(myhandler, None, get_crawler())
417        self.assertIsInstance(myhandler_instance, FileDownloadHandler)
418        self.assertTrue(hasattr(myhandler_instance, 'download_request'))
419
420
421if __name__ == "__main__":
422    unittest.main()
423