File: referer.py

package info (click to toggle)
python-scrapy 0.14.4-1
  • links: PTS, VCS
  • area: main
  • in suites: wheezy
  • size: 3,064 kB
  • sloc: python: 19,468; xml: 199; sh: 134; makefile: 67
file content (15 lines) | stat: -rw-r--r-- 440 bytes parent folder | download | duplicates (2) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
 
"""
RefererMiddleware: populates Request referer field, based on the Response which
originated it.
"""

from scrapy.http import Request

class RefererMiddleware(object):
    def process_spider_output(self, response, result, spider):
        def _set_referer(r):
            if isinstance(r, Request):
                r.headers.setdefault('Referer', response.url)
            return r
        return (_set_referer(r) for r in result or ())