import logging import pprint import signal import warnings from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement try: # zope >= 5.0 only supports MultipleInvalid from zope.interface.exceptions import MultipleInvalid except ImportError: MultipleInvalid = None from zope.interface.verify import verifyClass from scrapy import signals, Spider from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.settings import overridden_settings, Settings from scrapy.signalmanager import SignalManager from scrapy.utils.log import ( configure_logging, get_scrapy_root_handler, install_scrapy_root_handler, log_scrapy_info, LogCounterHandler, ) from scrapy.utils.misc import create_instance, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.reactor import install_reactor, verify_installed_reactor logger = logging.getLogger(__name__) class Crawler: def __init__(self, spidercls, settings=None): if isinstance(spidercls, Spider): raise ValueError('The spidercls argument must be a class, not an object') if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.spidercls = spidercls self.settings = settings.copy() self.spidercls.update_settings(self.settings) self.signals = SignalManager(self) self.stats = load_object(self.settings['STATS_CLASS'])(self) handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL')) logging.root.addHandler(handler) d = dict(overridden_settings(self.settings)) logger.info("Overridden settings:\n%(settings)s", {'settings': pprint.pformat(d)}) if get_scrapy_root_handler() is not None: # scrapy root handler already installed: update it with new settings install_scrapy_root_handler(self.settings) # lambda is assigned to Crawler attribute because this way it is not # garbage collected after leaving __init__ scope self.__remove_handler = lambda: logging.root.removeHandler(handler) self.signals.connect(self.__remove_handler, signals.engine_stopped) lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() self.crawling = False self.spider = None self.engine = None @defer.inlineCallbacks def crawl(self, *args, **kwargs): if self.crawling: raise RuntimeError("Crawling already taking place") self.crawling = True try: self.spider = self._create_spider(*args, **kwargs) self.engine = self._create_engine() start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) yield defer.maybeDeferred(self.engine.start) except Exception: self.crawling = False if self.engine is not None: yield self.engine.close() raise def _create_spider(self, *args, **kwargs): return self.spidercls.from_crawler(self, *args, **kwargs) def _create_engine(self): return ExecutionEngine(self, lambda _: self.stop()) @defer.inlineCallbacks def stop(self): """Starts a graceful stop of the crawler and returns a deferred that is fired when the crawler is stopped.""" if self.crawling: self.crawling = False yield defer.maybeDeferred(self.engine.stop) class CrawlerRunner: """ This is a convenient helper class that keeps track of, manages and runs crawlers inside an already setup :mod:`~twisted.internet.reactor`. The CrawlerRunner object must be instantiated with a :class:`~scrapy.settings.Settings` object. This class shouldn't be needed (since Scrapy is responsible of using it accordingly) unless writing scripts that manually handle the crawling process. See :ref:`run-from-script` for an example. """ crawlers = property( lambda self: self._crawlers, doc="Set of :class:`crawlers ` started by " ":meth:`crawl` and managed by this class." ) @staticmethod def _get_spider_loader(settings): """ Get SpiderLoader instance from settings """ cls_path = settings.get('SPIDER_LOADER_CLASS') loader_cls = load_object(cls_path) excs = (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement try: verifyClass(ISpiderLoader, loader_cls) except excs: warnings.warn( 'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does ' 'not fully implement scrapy.interfaces.ISpiderLoader interface. ' 'Please add all missing methods to avoid unexpected runtime errors.', category=ScrapyDeprecationWarning, stacklevel=2 ) return loader_cls.from_settings(settings.frozencopy()) def __init__(self, settings=None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings self.spider_loader = self._get_spider_loader(settings) self._crawlers = set() self._active = set() self.bootstrap_failed = False self._handle_twisted_reactor() @property def spiders(self): warnings.warn("CrawlerRunner.spiders attribute is renamed to " "CrawlerRunner.spider_loader.", category=ScrapyDeprecationWarning, stacklevel=2) return self.spider_loader def crawl(self, crawler_or_spidercls, *args, **kwargs): """ Run a crawler with the provided arguments. It will call the given Crawler's :meth:`~Crawler.crawl` method, while keeping track of it so it can be stopped later. If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler` instance, this method will try to create one using this parameter as the spider class given to it. Returns a deferred that is fired when the crawling is finished. :param crawler_or_spidercls: already created crawler, or a spider class or spider's name inside the project to create it :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance, :class:`~scrapy.spiders.Spider` subclass or string :param args: arguments to initialize the spider :param kwargs: keyword arguments to initialize the spider """ if isinstance(crawler_or_spidercls, Spider): raise ValueError( 'The crawler_or_spidercls argument cannot be a spider object, ' 'it must be a spider class (or a Crawler object)') crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) def _crawl(self, crawler, *args, **kwargs): self.crawlers.add(crawler) d = crawler.crawl(*args, **kwargs) self._active.add(d) def _done(result): self.crawlers.discard(crawler) self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, 'spider', None) return result return d.addBoth(_done) def create_crawler(self, crawler_or_spidercls): """ Return a :class:`~scrapy.crawler.Crawler` object. * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler is constructed for it. * If ``crawler_or_spidercls`` is a string, this function finds a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it. """ if isinstance(crawler_or_spidercls, Spider): raise ValueError( 'The crawler_or_spidercls argument cannot be a spider object, ' 'it must be a spider class (or a Crawler object)') if isinstance(crawler_or_spidercls, Crawler): return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) def _create_crawler(self, spidercls): if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) def stop(self): """ Stops simultaneously all the crawling jobs taking place. Returns a deferred that is fired when they all have ended. """ return defer.DeferredList([c.stop() for c in list(self.crawlers)]) @defer.inlineCallbacks def join(self): """ join() Returns a deferred that is fired when all managed :attr:`crawlers` have completed their executions. """ while self._active: yield defer.DeferredList(self._active) def _handle_twisted_reactor(self): if self.settings.get("TWISTED_REACTOR"): verify_installed_reactor(self.settings["TWISTED_REACTOR"]) class CrawlerProcess(CrawlerRunner): """ A class to run multiple scrapy crawlers in a process simultaneously. This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support for starting a :mod:`~twisted.internet.reactor` and handling shutdown signals, like the keyboard interrupt command Ctrl-C. It also configures top-level logging. This utility should be a better fit than :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another :mod:`~twisted.internet.reactor` within your application. The CrawlerProcess object must be instantiated with a :class:`~scrapy.settings.Settings` object. :param install_root_handler: whether to install root logging handler (default: True) This class shouldn't be needed (since Scrapy is responsible of using it accordingly) unless writing scripts that manually handle the crawling process. See :ref:`run-from-script` for an example. """ def __init__(self, settings=None, install_root_handler=True): super().__init__(settings) install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) def _signal_shutdown(self, signum, _): from twisted.internet import reactor install_shutdown_handlers(self._signal_kill) signame = signal_names[signum] logger.info("Received %(signame)s, shutting down gracefully. Send again to force ", {'signame': signame}) reactor.callFromThread(self._graceful_stop_reactor) def _signal_kill(self, signum, _): from twisted.internet import reactor install_shutdown_handlers(signal.SIG_IGN) signame = signal_names[signum] logger.info('Received %(signame)s twice, forcing unclean shutdown', {'signame': signame}) reactor.callFromThread(self._stop_reactor) def start(self, stop_after_crawl=True): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished """ from twisted.internet import reactor if stop_after_crawl: d = self.join() # Don't start the reactor if the deferreds are already fired if d.called: return d.addBoth(self._stop_reactor) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE')) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call def _graceful_stop_reactor(self): d = self.stop() d.addBoth(self._stop_reactor) return d def _stop_reactor(self, _=None): from twisted.internet import reactor try: reactor.stop() except RuntimeError: # raised if already stopped or in shutdown stage pass def _handle_twisted_reactor(self): if self.settings.get("TWISTED_REACTOR"): install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) super()._handle_twisted_reactor()