1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
|
# Copyright (C) 2000-2016 Bastian Kleineidam
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
"""
Configure linkchecker using command-line options and configuration.
"""
import codecs
import getpass
from .. import fileutil
from .. import i18n
from .. import logger
from .. import LOG_CMDLINE
from .. import get_link_pat, log
from ..cmdline import print_version, print_usage, print_plugins
def has_encoding(encoding):
"""Detect if Python can encode in a certain encoding."""
try:
codecs.lookup(encoding)
return True
except LookupError:
return False
def setup_config(config, options):
"""Set up linkchecker based on command-line options and configuration"""
_username = None
_password = None
# test if running with -O
if options.debug and not __debug__:
log.warn(LOG_CMDLINE, _("Running with python -O disables debugging."))
# apply commandline options and arguments to configuration
constructauth = False
if options.version:
print_version()
if not options.warnings:
config["warnings"] = options.warnings
if options.externstrict:
pats = [get_link_pat(arg, strict=True) for arg in options.externstrict]
config["externlinks"].extend(pats)
if options.extern:
pats = [get_link_pat(arg) for arg in options.extern]
config["externlinks"].extend(pats)
if options.norobotstxt is not None:
config["robotstxt"] = options.norobotstxt
if options.checkextern:
config["checkextern"] = True
elif not config["checkextern"]:
log.info(
LOG_CMDLINE,
"Checking intern URLs only; use --check-extern to check extern URLs.",
)
if options.output:
if "/" in options.output:
logtype, encoding = options.output.split("/", 1)
else:
logtype, encoding = options.output, i18n.default_encoding
logtype = logtype.lower()
if logtype == "blacklist":
log.warn(
LOG_CMDLINE,
_("blacklist is deprecated for option %(option)s, "
"using failures instead") % {"option": "'-o, --output'"}
)
logtype = "failures"
if logtype not in logger.LoggerNames:
print_usage(
_("Unknown logger type %(type)r in %(output)r for option %(option)s")
% {"type": logtype,
"output": options.output,
"option": "'-o, --output'"}
)
if logtype != "none" and not has_encoding(encoding):
print_usage(
_("Unknown encoding %(encoding)r in %(output)r for option %(option)s")
% {
"encoding": encoding,
"output": options.output,
"option": "'-o, --output'",
}
)
config["output"] = logtype
config["logger"] = config.logger_new(logtype, encoding=encoding)
if options.fileoutput:
ns = {"fileoutput": 1}
for arg in options.fileoutput:
ftype = arg
# look for (optional) filename and encoding
if "/" in ftype:
ftype, suffix = ftype.split("/", 1)
if suffix:
if has_encoding(suffix):
# it was an encoding
ns["encoding"] = suffix
elif "/" in suffix:
# look for (optional) encoding
encoding, filename = suffix.split("/", 1)
if has_encoding(encoding):
ns["encoding"] = encoding
ns["filename"] = filename
else:
ns["filename"] = suffix
else:
ns["filename"] = suffix
if ftype == "blacklist":
log.warn(
LOG_CMDLINE,
_("blacklist logger is deprecated for option %(option)s, "
"using failures instead") % {"option": "'-F, --file-output'"}
)
ftype = "failures"
if ftype not in logger.LoggerNames:
print_usage(
_("Unknown logger type %(type)r in %(output)r"
" for option %(option)s")
% {
"type": ftype,
"output": options.fileoutput,
"option": "'-F, --file-output'",
}
)
if ftype != "none" and "encoding" in ns \
and not has_encoding(ns["encoding"]):
print_usage(
_("Unknown encoding %(encoding)r in %(output)r"
" for option %(option)s")
% {
"encoding": ns["encoding"],
"output": options.fileoutput,
"option": "'-F, --file-output'",
}
)
new_logger = config.logger_new(ftype, **ns)
config["fileoutput"].append(new_logger)
if options.username:
_username = options.username
constructauth = True
if options.password:
if _username:
msg = _("Enter LinkChecker HTTP/FTP password for user %(user)s:") % {
"user": _username
}
else:
msg = _("Enter LinkChecker HTTP/FTP password:")
_password = getpass.getpass(msg)
constructauth = True
if options.quiet:
config["logger"] = config.logger_new("none")
if options.recursionlevel is not None:
config["recursionlevel"] = options.recursionlevel
if options.status is not None:
config["status"] = options.status
if options.threads is not None:
if options.threads < 1:
options.threads = 0
config["threads"] = options.threads
if options.timeout is not None:
if options.timeout > 0:
config["timeout"] = options.timeout
else:
print_usage(
_("Illegal argument %(arg)r for option %(option)s")
% {"arg": options.timeout, "option": "'--timeout'"}
)
if options.listplugins:
print_plugins(config["pluginfolders"])
if options.verbose:
if options.verbose:
config["verbose"] = True
config["warnings"] = True
if constructauth:
config.add_auth(pattern=".+", user=_username, password=_password)
# read missing passwords
for entry in config["authentication"]:
if entry["password"] is None:
attrs = entry.copy()
attrs["strpattern"] = attrs["pattern"].pattern
msg = (
_("Enter LinkChecker password for user %(user)s at %(strpattern)s:")
% attrs
)
entry["password"] = getpass.getpass(msg)
if options.useragent is not None:
config["useragent"] = options.useragent
if options.cookiefile is not None:
if not fileutil.is_valid_config_source(options.cookiefile):
print_usage(
_("Cookie file %s does not exist.") % options.cookiefile)
elif not fileutil.is_readable(options.cookiefile):
print_usage(
_("Could not read cookie file %s") % options.cookiefile)
else:
config["cookiefile"] = options.cookiefile
|
