1import os
2import warnings
3
4from importlib import import_module
5from os.path import join, dirname, abspath, isabs, exists
6
7from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
8from scrapy.settings import Settings
9from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
10
11
12ENVVAR = 'SCRAPY_SETTINGS_MODULE'
13DATADIR_CFG_SECTION = 'datadir'
14
15
16def inside_project():
17    scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE')
18    if scrapy_module is not None:
19        try:
20            import_module(scrapy_module)
21        except ImportError as exc:
22            warnings.warn(f"Cannot import scrapy settings module {scrapy_module}: {exc}")
23        else:
24            return True
25    return bool(closest_scrapy_cfg())
26
27
28def project_data_dir(project='default'):
29    """Return the current project data dir, creating it if it doesn't exist"""
30    if not inside_project():
31        raise NotConfigured("Not inside a project")
32    cfg = get_config()
33    if cfg.has_option(DATADIR_CFG_SECTION, project):
34        d = cfg.get(DATADIR_CFG_SECTION, project)
35    else:
36        scrapy_cfg = closest_scrapy_cfg()
37        if not scrapy_cfg:
38            raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir")
39        d = abspath(join(dirname(scrapy_cfg), '.scrapy'))
40    if not exists(d):
41        os.makedirs(d)
42    return d
43
44
45def data_path(path, createdir=False):
46    """
47    Return the given path joined with the .scrapy data directory.
48    If given an absolute path, return it unmodified.
49    """
50    if not isabs(path):
51        if inside_project():
52            path = join(project_data_dir(), path)
53        else:
54            path = join('.scrapy', path)
55    if createdir and not exists(path):
56        os.makedirs(path)
57    return path
58
59
60def get_project_settings():
61    if ENVVAR not in os.environ:
62        project = os.environ.get('SCRAPY_PROJECT', 'default')
63        init_env(project)
64
65    settings = Settings()
66    settings_module_path = os.environ.get(ENVVAR)
67    if settings_module_path:
68        settings.setmodule(settings_module_path, priority='project')
69
70    scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if
71                      k.startswith('SCRAPY_')}
72    valid_envvars = {
73        'CHECK',
74        'PROJECT',
75        'PYTHON_SHELL',
76        'SETTINGS_MODULE',
77    }
78    setting_envvars = {k for k in scrapy_envvars if k not in valid_envvars}
79    if setting_envvars:
80        setting_envvar_list = ', '.join(sorted(setting_envvars))
81        warnings.warn(
82            'Use of environment variables prefixed with SCRAPY_ to override '
83            'settings is deprecated. The following environment variables are '
84            f'currently defined: {setting_envvar_list}',
85            ScrapyDeprecationWarning
86        )
87    settings.setdict(scrapy_envvars, priority='project')
88
89    return settings
90