1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
# Copyright (C) 2000-2014 Bastian Kleineidam
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
"""
A failures logger.
"""
import os
from . import _Logger
from .. import log, LOG_CHECK
from ..configuration import get_user_data
class FailuresLogger(_Logger):
"""
Updates a list of failed links. If a link already on the list
is found to be working, it is removed. After n days
we only have links on the list which failed within those n days.
"""
LoggerName = "failures"
LoggerArgs = {
"filename": os.path.join(get_user_data(), "failures"),
}
def __init__(self, **kwargs):
"""Initialize with old failures data (if found)."""
blacklist = os.path.join(get_user_data(), "blacklist")
if os.path.isfile(blacklist):
self.LoggerArgs["filename"] = blacklist
log.warn(
LOG_CHECK,
_("%(blacklist)s file is deprecated please rename to failures")
% {"blacklist": blacklist}
)
args = self.get_args(kwargs)
super().__init__(**args)
self.init_fileoutput(args)
self.failures = {}
if self.filename is not None and os.path.exists(self.filename):
self.read_failures()
def comment(self, s, **args):
"""
Write nothing.
"""
pass
def log_url(self, url_data):
"""
Add invalid url to failures, delete valid url from failures.
"""
key = (url_data.parent_url, url_data.cache_url)
key = repr(key)
if key in self.failures:
if url_data.valid:
del self.failures[key]
else:
self.failures[key] += 1
else:
if not url_data.valid:
self.failures[key] = 1
def end_output(self, **kwargs):
"""
Write failures file.
"""
self.write_failures()
def read_failures(self):
"""
Read a previously stored failures from file fd.
"""
with open(self.filename, encoding=self.output_encoding,
errors=self.codec_errors) as fd:
for line in fd:
line = line.rstrip()
if line.startswith('#') or not line:
continue
value, key = line.split(None, 1)
key = key.strip('"')
if not key.startswith('('):
log.critical(
LOG_CHECK,
_("invalid line starting with '%(linestart)s' in %(failures)s")
% {"linestart": line[:12], "failures": self.filename}
)
raise SystemExit(2)
self.failures[key] = int(value)
def write_failures(self):
"""
Write the failures file.
"""
oldmask = os.umask(0o077)
for key, value in self.failures.items():
self.write("%d %s%s" % (value, repr(key), os.linesep))
self.close_fileoutput()
# restore umask
os.umask(oldmask)
|
