1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
|
"""
The tool to check the availability or syntax of domain, IP or URL.
::
██████╗ ██╗ ██╗███████╗██╗ ██╗███╗ ██╗ ██████╗███████╗██████╗ ██╗ ███████╗
██╔══██╗╚██╗ ██╔╝██╔════╝██║ ██║████╗ ██║██╔════╝██╔════╝██╔══██╗██║ ██╔════╝
██████╔╝ ╚████╔╝ █████╗ ██║ ██║██╔██╗ ██║██║ █████╗ ██████╔╝██║ █████╗
██╔═══╝ ╚██╔╝ ██╔══╝ ██║ ██║██║╚██╗██║██║ ██╔══╝ ██╔══██╗██║ ██╔══╝
██║ ██║ ██║ ╚██████╔╝██║ ╚████║╚██████╗███████╗██████╔╝███████╗███████╗
╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═══╝ ╚═════╝╚══════╝╚═════╝ ╚══════╝╚══════╝
Provides the URL availability checker.
Author:
Nissar Chababy, @funilrys, contactTATAfunilrysTODTODcom
Special thanks:
https://pyfunceble.github.io/#/special-thanks
Contributors:
https://pyfunceble.github.io/#/contributors
Project link:
https://github.com/funilrys/PyFunceble
Project documentation:
https://docs.pyfunceble.com
Project homepage:
https://pyfunceble.github.io/
License:
::
Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import PyFunceble.facility
import PyFunceble.factory
import PyFunceble.storage
from PyFunceble.checker.availability.base import AvailabilityCheckerBase
from PyFunceble.checker.availability.status import AvailabilityCheckerStatus
from PyFunceble.checker.reputation.url import URLReputationChecker
from PyFunceble.checker.syntax.url import URLSyntaxChecker
class URLAvailabilityChecker(AvailabilityCheckerBase):
"""
Provides the interface for checking the availability of a given URL.
:param str subject:
Optional, The subject to work with.
:param bool use_extra_rules:
Optional, Activates/Disables the usage of our own set of extra rules.
:param bool use_whois_lookup:
Optional, Activates/Disables the usage of the WHOIS lookup to gather
the status of the given :code:`subject`.
:param bool use_dns_lookup:
Optional, Activates/Disables the usage of the DNS lookup to gather the
status of the given :code:`subject`.
:param bool use_netinfo_lookup:
Optional, Activates/Disables the usage of the network information
lookup module to gather the status of the given :code:`subject`.
:param bool use_http_code_lookup:
Optional, Activates/Disables the usage of the HTTP status code lookup
to gather the status of the given :code:`subject`.
:param bool use_reputation_lookup:
Optional, Activates/Disables the usage of the reputation dataset
lookup to gather the status of the given :code:`subject`.
:param bool do_syntax_check_first:
Optional, Activates/Disables the check of the status before the actual
status gathering.
:param bool use_whois_db:
Optional, Activates/Disable the usage of a local database to store the
WHOIS datasets.
"""
def subject_propagator(self) -> "URLAvailabilityChecker":
"""
Propagate the currently set subject.
.. warning::
You are not invited to run this method directly.
.. versionchanged:: 4.1.0b7.dev
DNS Lookup capability.
"""
self.http_status_code_query_tool.set_subject(self.idna_subject)
self.dns_query_tool.set_subject(
URLSyntaxChecker.get_hostname_from_url(self.idna_subject)
or self.idna_subject
)
self.domain_syntax_checker.subject = self.idna_subject
self.ip_syntax_checker.subject = self.idna_subject
self.url_syntax_checker.subject = self.idna_subject
self.status = AvailabilityCheckerStatus()
self.status.params = self.params
self.status.dns_lookup_record = self.dns_query_tool.lookup_record
self.status.whois_lookup_record = None
self.status.subject = self.subject
self.status.idna_subject = self.idna_subject
self.status.netloc = self.url2netloc.set_data_to_convert(
self.idna_subject
).get_converted()
self.status.status = None
self.query_common_checker()
return self
def try_to_query_status_from_http_status_code(
self, *, from_domain_test: bool = False
) -> "URLAvailabilityChecker":
"""
Tries to query the status from the network information.
:param bool from_domain_test:
Whether we wanted to test a test - actually.
Setting this argument to :py:class:`True` will exit the http_status_code
test if the given subject is already a URL.
"""
PyFunceble.facility.Logger.info(
"Started to try to query the status of %r from: HTTP Status code Lookup",
self.status.idna_subject,
)
if from_domain_test and self.status.url_syntax:
return self
lookup_result = self.http_status_code_query_tool.get_status_code()
if (
lookup_result
and lookup_result
!= self.http_status_code_query_tool.STD_UNKNOWN_STATUS_CODE
):
self.status.http_status_code = lookup_result
if (
PyFunceble.facility.ConfigLoader.is_already_loaded()
): # pragma: no cover ## Special behavior
dataset = PyFunceble.storage.HTTP_CODES
else:
dataset = PyFunceble.storage.STD_HTTP_CODES
if (
self.status.http_status_code in dataset.list.up
or self.status.http_status_code in dataset.list.potentially_up
):
self.status.status = PyFunceble.storage.STATUS.up
self.status.status_source = "HTTP CODE"
PyFunceble.facility.Logger.info(
"Could define the status of %r from: HTTP Status code Lookup",
self.status.idna_subject,
)
else:
self.status.http_status_code = None
PyFunceble.facility.Logger.info(
"Finished to try to query the status of %r from: HTTP Status code Lookup",
self.status.idna_subject,
)
return self
def try_to_query_status_from_reputation(self) -> "URLAvailabilityChecker":
"""
Tries to query the status from the reputation lookup.
"""
PyFunceble.facility.Logger.info(
"Started to try to query the status of %r from: Reputation Lookup",
self.status.idna_subject,
)
lookup_result = URLReputationChecker(self.status.idna_subject).get_status()
# pylint: disable=no-member
if lookup_result and lookup_result.is_malicious():
self.status.status = PyFunceble.storage.STATUS.up
self.status.status_source = "REPUTATION"
PyFunceble.facility.Logger.info(
"Could define the status of %r from: Reputation Lookup",
self.status.idna_subject,
)
PyFunceble.facility.Logger.info(
"Started to try to query the status of %r from: Reputation Lookup",
self.status.idna_subject,
)
return self
def try_to_query_status_from_dns(self) -> "AvailabilityCheckerBase":
"""
Tries to query the status from the DNS lookup after switching the
idna subject to the url base.
.. warning::
This method does not answer as you may expect.
Indeed, if a DNS lookup failed, this method will overwrite the
standard response by setting the status to :code:`INACTIVE` and the
status source to :code:`DNSLOOKUP`.
.. versionadded:: 4.1.0b7
DNS Lookup as a "down" switcher.
"""
result = super().try_to_query_status_from_dns()
if self.status.status == PyFunceble.storage.STATUS.up:
# DNS should only be use to take subject down.
# Therefore, switching back as if nothing happened.
self.status.status = None
self.status.status_source = None
else:
self.status.status = PyFunceble.storage.STATUS.down
self.status.status_source = "DNSLOOKUP"
return result
@AvailabilityCheckerBase.ensure_subject_is_given
@AvailabilityCheckerBase.update_status_date_after_query
def query_status(
self,
) -> "URLAvailabilityChecker": # pragma: no cover
"""
Queries the result without anything more.
.. versionchanged:: 4.1.0b7.dev
DNS Query - first.
"""
## Test Methods are more important.
status_post_syntax_checker = None
if not self.status.status and self.do_syntax_check_first:
self.try_to_query_status_from_syntax_lookup(from_url_test=True)
if self.status.status:
status_post_syntax_checker = self.status.status
if self.use_reputation_lookup and self.should_we_continue_test(
status_post_syntax_checker
):
self.try_to_query_status_from_reputation()
if (
self.should_we_continue_test(status_post_syntax_checker)
and self.status.url_syntax
):
self.try_to_query_status_from_dns()
if self.should_we_continue_test(status_post_syntax_checker):
self.try_to_query_status_from_http_status_code()
if not self.status.status:
self.status.status = PyFunceble.storage.STATUS.down
self.status.status_source = "STDLOOKUP"
PyFunceble.facility.Logger.info(
"Could not define status the status of %r. Setting to %r",
self.status.idna_subject,
self.status.status,
)
if self.use_extra_rules:
self.try_to_query_status_from_extra_rules()
return self
@staticmethod
def is_valid() -> bool: # pylint: disable=arguments-differ
raise NotImplementedError()
|
