File: useragent.py

package info (click to toggle)
python-scrapy 0.8-3
  • links: PTS, VCS
  • area: main
  • in suites: squeeze
  • size: 2,904 kB
  • ctags: 2,981
  • sloc: python: 15,349; xml: 199; makefile: 68; sql: 64; sh: 34
file content (14 lines) | stat: -rw-r--r-- 463 bytes parent folder | download 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
 
"""Set User-Agent header per spider or use a default value from settings"""

from scrapy.conf import settings


class UserAgentMiddleware(object):
    """This middleware allows spiders to override the user_agent"""

    default_useragent = settings.get('USER_AGENT')

    def process_request(self, request, spider):
        ua = getattr(spider, 'user_agent', None) or self.default_useragent
        if ua:
            request.headers.setdefault('User-Agent', ua)