1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
|
import asyncio
import sys
from twisted.internet import asyncioreactor
from twisted.python import log
if sys.platform == "win32":
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
asyncioreactor.install(asyncio.get_event_loop())
import scrapy # noqa: E402
from scrapy.crawler import CrawlerProcess # noqa: E402
class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def start(self):
return
yield
process = CrawlerProcess(
settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
"ASYNCIO_EVENT_LOOP": "uvloop.Loop",
}
)
d = process.crawl(NoRequestsSpider)
d.addErrback(log.err)
process.start()
|
