File: defaultheaders.py

package info (click to toggle)
python-scrapy 0.8-3
  • links: PTS, VCS
  • area: main
  • in suites: squeeze
  • size: 2,904 kB
  • ctags: 2,981
  • sloc: python: 15,349; xml: 199; makefile: 68; sql: 64; sh: 34
file content (17 lines) | stat: -rw-r--r-- 446 bytes parent folder | download 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
 
"""
DefaultHeaders downloader middleware

See documentation in docs/topics/downloader-middleware.rst
"""

from scrapy.conf import settings

class DefaultHeadersMiddleware(object):

    def __init__(self):
        self.default_headers = settings.get('DEFAULT_REQUEST_HEADERS')

    def process_request(self, request, spider):
        for k, v in self.default_headers.iteritems():
            if v:
                request.headers.setdefault(k, v)