1import os 2import warnings 3 4from importlib import import_module 5from os.path import join, dirname, abspath, isabs, exists 6 7from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env 8from scrapy.settings import Settings 9from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning 10 11 12ENVVAR = 'SCRAPY_SETTINGS_MODULE' 13DATADIR_CFG_SECTION = 'datadir' 14 15 16def inside_project(): 17 scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE') 18 if scrapy_module is not None: 19 try: 20 import_module(scrapy_module) 21 except ImportError as exc: 22 warnings.warn(f"Cannot import scrapy settings module {scrapy_module}: {exc}") 23 else: 24 return True 25 return bool(closest_scrapy_cfg()) 26 27 28def project_data_dir(project='default'): 29 """Return the current project data dir, creating it if it doesn't exist""" 30 if not inside_project(): 31 raise NotConfigured("Not inside a project") 32 cfg = get_config() 33 if cfg.has_option(DATADIR_CFG_SECTION, project): 34 d = cfg.get(DATADIR_CFG_SECTION, project) 35 else: 36 scrapy_cfg = closest_scrapy_cfg() 37 if not scrapy_cfg: 38 raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir") 39 d = abspath(join(dirname(scrapy_cfg), '.scrapy')) 40 if not exists(d): 41 os.makedirs(d) 42 return d 43 44 45def data_path(path, createdir=False): 46 """ 47 Return the given path joined with the .scrapy data directory. 48 If given an absolute path, return it unmodified. 49 """ 50 if not isabs(path): 51 if inside_project(): 52 path = join(project_data_dir(), path) 53 else: 54 path = join('.scrapy', path) 55 if createdir and not exists(path): 56 os.makedirs(path) 57 return path 58 59 60def get_project_settings(): 61 if ENVVAR not in os.environ: 62 project = os.environ.get('SCRAPY_PROJECT', 'default') 63 init_env(project) 64 65 settings = Settings() 66 settings_module_path = os.environ.get(ENVVAR) 67 if settings_module_path: 68 settings.setmodule(settings_module_path, priority='project') 69 70 scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if 71 k.startswith('SCRAPY_')} 72 valid_envvars = { 73 'CHECK', 74 'PROJECT', 75 'PYTHON_SHELL', 76 'SETTINGS_MODULE', 77 } 78 setting_envvars = {k for k in scrapy_envvars if k not in valid_envvars} 79 if setting_envvars: 80 setting_envvar_list = ', '.join(sorted(setting_envvars)) 81 warnings.warn( 82 'Use of environment variables prefixed with SCRAPY_ to override ' 83 'settings is deprecated. The following environment variables are ' 84 f'currently defined: {setting_envvar_list}', 85 ScrapyDeprecationWarning 86 ) 87 settings.setdict(scrapy_envvars, priority='project') 88 89 return settings 90