1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
|
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from scrapy import Spider, signals
from scrapy.utils.log import LogCounterHandler
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
class LogCount:
"""Install a log handler that counts log messages by level.
The handler installed is :class:`scrapy.utils.log.LogCounterHandler`.
The counts are stored in stats as ``log_count/<level>``.
.. versionadded:: 2.14
"""
def __init__(self, crawler: Crawler):
self.crawler: Crawler = crawler
self.handler: LogCounterHandler | None = None
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
o = cls(crawler)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_opened(self, spider: Spider) -> None:
self.handler = LogCounterHandler(
self.crawler, level=self.crawler.settings.get("LOG_LEVEL")
)
logging.root.addHandler(self.handler)
def spider_closed(self, spider: Spider, reason: str) -> None:
if self.handler:
logging.root.removeHandler(self.handler)
self.handler = None
|
