1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
|
"""
Scrapy settings manager
See documentation in docs/topics/settings.rst
"""
import os
import cPickle as pickle
from scrapy.conf import default_settings
import_ = lambda x: __import__(x, {}, {}, [''])
class Settings(object):
def __init__(self, values=None):
self.values = values.copy() if values else {}
self.global_defaults = default_settings
def __getitem__(self, opt_name):
if opt_name in self.values:
return self.values[opt_name]
return getattr(self.global_defaults, opt_name, None)
def get(self, name, default=None):
return self[name] if self[name] is not None else default
def getbool(self, name, default=False):
"""
True is: 1, '1', True
False is: 0, '0', False, None
"""
return bool(int(self.get(name, default)))
def getint(self, name, default=0):
return int(self.get(name, default))
def getfloat(self, name, default=0.0):
return float(self.get(name, default))
def getlist(self, name, default=None):
value = self.get(name)
if value is None:
return default or []
elif hasattr(value, '__iter__'):
return value
else:
return str(value).split(',')
class EnvironmentSettings(Settings):
def __init__(self):
super(EnvironmentSettings, self).__init__()
self.defaults = {}
self.disabled = os.environ.get('SCRAPY_SETTINGS_DISABLED', False)
settings_module_path = os.environ.get('SCRAPY_SETTINGS_MODULE', \
'scrapy_settings')
self.set_settings_module(settings_module_path)
# XXX: find a better solution for this hack
pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
self.overrides = pickle.loads(pickled_settings) if pickled_settings else {}
def set_settings_module(self, settings_module_path):
self.settings_module_path = settings_module_path
try:
self.settings_module = import_(settings_module_path)
except ImportError:
self.settings_module = None
def __getitem__(self, opt_name):
if not self.disabled:
if opt_name in self.overrides:
return self.overrides[opt_name]
if 'SCRAPY_' + opt_name in os.environ:
return os.environ['SCRAPY_' + opt_name]
if hasattr(self.settings_module, opt_name):
return getattr(self.settings_module, opt_name)
if opt_name in self.defaults:
return self.defaults[opt_name]
return super(EnvironmentSettings, self).__getitem__(opt_name)
def __str__(self):
return "<Settings %r>" % self.settings_module_path
settings = EnvironmentSettings()
|
