1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
|
import abc
from typing import Generic, Protocol, TypeVar
from urllib.parse import urljoin
import parsel
from w3lib.html import get_base_url
from web_poet.page_inputs.url import RequestUrl, ResponseUrl
class _ResponseLike(Protocol):
"""Protocol for response objects."""
url: ResponseUrl | str
text: str
ResponseT = TypeVar("ResponseT", bound=_ResponseLike)
class SelectorShortcutsMixin:
def xpath(self, query, **kwargs) -> parsel.SelectorList:
"""A shortcut to ``.selector.xpath()``."""
return self.selector.xpath(query, **kwargs) # type: ignore[attr-defined]
def css(self, query) -> parsel.SelectorList:
"""A shortcut to ``.selector.css()``."""
return self.selector.css(query) # type: ignore[attr-defined]
def jmespath(self, query: str, **kwargs) -> parsel.SelectorList:
"""A shortcut to ``.selector.jmespath()``."""
if not hasattr(self.selector, "jmespath"): # type: ignore[attr-defined]
raise AttributeError(
"Please install parsel >= 1.8.1 to get jmespath support"
)
return self.selector.jmespath(query, **kwargs) # type: ignore[attr-defined]
class SelectableMixin(abc.ABC, SelectorShortcutsMixin):
"""
Inherit from this mixin, implement ``._selector_input`` method,
get ``.selector`` property and ``.xpath`` / ``.css`` / ``.jmespath``
methods.
"""
__cached_selector = None
@abc.abstractmethod
def _selector_input(self) -> str:
raise NotImplementedError # pragma: nocover
@property
def selector(self) -> parsel.Selector:
"""Cached instance of :external:class:`parsel.selector.Selector`."""
# caching is implemented in a manual way to avoid issues with
# non-hashable classes, where memoizemethod_noargs doesn't work
if self.__cached_selector is not None:
return self.__cached_selector
base_url = str(self.url) if hasattr(self, "url") else None
sel = parsel.Selector(text=self._selector_input(), base_url=base_url)
self.__cached_selector = sel
return sel
class UrlShortcutsMixin:
_cached_base_url = None
def _url_shortcuts_input(self) -> str:
return self._selector_input() # type: ignore[attr-defined]
@property
def _base_url(self) -> str:
if self._cached_base_url is None:
text = self._url_shortcuts_input()[:4096]
self._cached_base_url = get_base_url(text, str(self.url)) # type: ignore[attr-defined]
return self._cached_base_url
def urljoin(self, url: str | RequestUrl | ResponseUrl) -> RequestUrl:
"""Return *url* as an absolute URL.
If *url* is relative, it is made absolute relative to the base URL of
*self*."""
return RequestUrl(urljoin(self._base_url, str(url)))
class ResponseShortcutsMixin(Generic[ResponseT], SelectableMixin, UrlShortcutsMixin): # noqa: PYI059
"""Common shortcut methods for working with HTML responses.
This mixin could be used with Page Object base classes.
It requires "response" attribute to be present.
"""
response: ResponseT
_cached_base_url = None
@property
def url(self) -> str:
"""Shortcut to HTML Response's URL, as a string."""
return str(self.response.url)
@property
def html(self) -> str:
"""Shortcut to HTML Response's content."""
return self.response.text
def _selector_input(self) -> str:
return self.html
@property
def base_url(self) -> str:
"""Return the base url of the given response"""
return self._base_url
def urljoin(self, url: str) -> str: # type: ignore[override]
"""Convert url to absolute, taking in account
url and baseurl of the response"""
return str(super().urljoin(url))
|
