1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
|
import scrapy
from scrapy.crawler import CrawlerProcess
class CachingHostnameResolverSpider(scrapy.Spider):
"""
Finishes without a twisted.internet.error.DNSLookupError exception
"""
name = "caching_hostname_resolver_spider"
start_urls = ["http://[::1]"]
if __name__ == "__main__":
process = CrawlerProcess(
settings={
"RETRY_ENABLED": False,
"DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
}
)
process.crawl(CachingHostnameResolverSpider)
process.start()
|
