1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
|
from collections.abc import Iterable
from typing import NamedTuple
__all__ = ["RobotFileParser"]
class RequestRate(NamedTuple):
requests: int
seconds: int
class RobotFileParser:
def __init__(self, url: str = "") -> None: ...
def set_url(self, url: str) -> None: ...
def read(self) -> None: ...
def parse(self, lines: Iterable[str]) -> None: ...
def can_fetch(self, useragent: str, url: str) -> bool: ...
def mtime(self) -> int: ...
def modified(self) -> None: ...
def crawl_delay(self, useragent: str) -> str | None: ...
def request_rate(self, useragent: str) -> RequestRate | None: ...
def site_maps(self) -> list[str] | None: ...
|
