#!/usr/bin/env python3
from __future__ import print_function
from collections import defaultdict
from functools import cmp_to_key
import sys
import argparse
sys.path.append("/usr/share/botch")
from util import write_plain, read_yaml_file, cmp, parse_dose_yaml_mc
from util import read_tag_file
import os
def print_package(arg, bin2src, source=None, color=True):
pkgname, arch, version, vpkg = arg
sn, _ = src_of_pkg(arg, bin2src)
if source:
highlight = sn == source[0]
else:
highlight = False
if pkgname == "":
pkgstring = "(*)"
bugs = False
elif pkgname.startswith("src:"):
pkgname = pkgname[4:]
bugs = srcpkgbugs.get(pkgname)
if version:
pkgstring = 'src:%s' % (version, pkgname)
else:
pkgstring = "src:%s" % pkgname
else:
bugs = binpkgbugs.get(pkgname)
if version:
pkgstring = '%s' % (
pkgname,
arch,
version,
pkgname,
)
else:
pkgstring = "%s:%s" % (pkgname, arch)
if highlight:
pkgstring = '' + pkgstring
pkgstring += ""
if bugs:
bugstring = " ".join(
'%s' % (b, b) for b in bugs
)
if color:
pkgstring = '%s (%s)' % (
pkgstring,
bugstring,
)
else:
pkgstring = "%s (%s)" % (pkgstring, bugstring)
if sn:
wnpp = wnpp_stats.get(sn)
if wnpp:
t, i = wnpp
pkgstring = '%s (%s: #%s)' % (
pkgstring,
t,
i,
i,
)
if vpkg:
pkgstring += ' → ' % vpkg
return pkgstring
def by_srcname_chain(a, b):
num_src = cmp(len(b[1]), len(a[1]))
if num_src:
return num_src
cmp_src = cmp(min(a[1]), min(b[1]))
if cmp_src:
return cmp_src
return cmp(a[0], b[0])
def by_affected_sources_missing(a, b):
a1 = sum([len(v) for v in a[1].values()])
b1 = sum([len(v) for v in b[1].values()])
num_src = cmp(b1, a1)
if num_src:
return num_src
else:
return cmp(a[0], b[0])
def by_affected_sources_conflict(a, b):
a1 = sum([len(v) for v in a[1].values()])
b1 = sum([len(v) for v in b[1].values()])
num_src = cmp(b1, a1)
if num_src:
return num_src
else:
return cmp(a[0], b[0])
def print_header(outfile, description, btsuser, btstag):
print(
"""
""",
file=outfile,
)
print(description, file=outfile)
if btsuser and btstag:
print(
"Bugs are associated with packages on this page if they "
+ 'carry the usertag "%s" of the user ' % btstag
+ '"%s".
' % btsuser,
file=outfile,
)
print(
"You can get an overview of all bugs tagged like that in "
+ 'the Debian bts
",
file=outfile,
)
print(
"Hover over a package name with your cursor for architecture and "
"version information. Hovering over the arrows in the depchain "
"columns will show the dependency that led from one package in the "
"chain to the next.
",
file=outfile,
)
def print_missing(outfile, missing, bin2src, source=None):
print("missing
", file=outfile)
print(
"The packages in the third column cannot satisfy their (possibly "
"transitive) dependencies because of the unsatisfied dependency in "
"the last column. This is mostly because the binary package "
"providing the dependency in the last column is Multi-Arch:no. Some "
"of these packages need to be Multi-Arch:foreign instead. In some "
"other cases, Build-Depends can be annotated with :native. The "
"depchains column shows the dependency chain(s) from the packages "
"in the third column to the unsatisfied dependency in the last "
'column. The "(*)" placeholder in the depchains column represents '
"any package in the third column. Hovering over the arrows in the "
"depchains column with your cursor will show the dependency that "
"led from one package in the chain to the next.
",
file=outfile,
)
print(
"The output is first grouped by the shared unsatisfied "
+ "dependency (last column) and then by shared dependency chain "
+ "(fourth column). The groups are sorted by the number of "
+ "packages missing the dependency in the last column. Within each "
+ "group, the output is sorted by the number of packages "
+ "sharing the same dependency chain.
",
file=outfile,
)
print(
"| # of packages per missing | # of packages per"
" depchain | packages with missing (possibly transitive) depe"
"ndencies | Depchains | " + "Unsatisfied dependency |
|---|
",
file=outfile,
)
for missing_vpkg, v in sorted(
list(missing.items()), key=cmp_to_key(by_affected_sources_missing)
):
rows = len(v)
for c in list(v.keys()):
if c:
rows += len(c) - 1
print(
'| %d | ' % (rows, sum([len(s) for s in v.values()])),
file=outfile,
)
first = True
for chain, v in sorted(list(v.items()), key=cmp_to_key(by_srcname_chain)):
if chain is None:
maxnumchains = 1
else:
maxnumchains = len(chain)
if chain:
c = "".join([print_package(t, bin2src, source) for t in chain[0]])
else:
c = ""
if not first:
print("
", file=outfile)
print(
'| %d | ' % (maxnumchains, len(v))
+ '%s | %s | '
% (
maxnumchains,
" ".join([print_package(t, bin2src, source) for t in sorted(v)]),
c,
),
file=outfile,
)
if first:
print(
'%s |
' % (rows, missing_vpkg), file=outfile
)
else:
print("", file=outfile)
if chain:
for i in range(1, len(chain)):
print(
"| %s |
"
% (
"".join(
[print_package(t, bin2src, source) for t in chain[i]]
)
),
file=outfile,
)
first = False
print("
", file=outfile)
def print_conflict(outfile, conflict, bin2src, source=None):
print("conflict
", file=outfile)
print(
"The packages in the third column cannot satisfy their (possibly "
"transitive) dependencies because the last package(s) in the first "
"depchain have an unsatisfied conflict which is shown in the last "
"column. The second depchain column shows the dependency chain(s) "
"to the package which the last package(s) in the first depchain "
"conflict with. Sometimes, multiple dependency chains sharing the "
"same conflict exist. Hovering over the arrows in the depchains "
"column with your cursor will show the dependency that led from one "
"package in the chain to the next.
",
file=outfile,
)
print(
"The output is first grouped by the shared conflicting "
+ "dependency (last column) and then by the shared dependency "
+ "chains (fourth and fifth column). The groups are sorted by the "
+ "number of packages sharing the conflict in the last "
+ "column. Within each group, the output is sorted by the number of "
+ "packages sharing the same dependency chains.
",
file=outfile,
)
print(
"| # of packages per conflict | # of packages per d"
"epchain | packages with (possibly transitive) conflicting depen"
"dencies | Depchain " + "1 | Depchain2 | Conflict |
|---|
",
file=outfile,
)
for conflict_vpkg, v in sorted(
list(conflict.items()), key=cmp_to_key(by_affected_sources_conflict)
):
rows = len(v)
for c1, c2 in list(v.keys()):
if c1 is not None:
c1 = len(c1)
else:
c1 = 0
if c2 is not None:
c2 = len(c2)
else:
c2 = 0
if max(c1, c2) > 1:
rows += max(c1, c2) - 1
print(
'| %d | ' % (rows, sum([len(s) for s in v.values()])),
file=outfile,
)
first = True
for (chain1, chain2), v in sorted(
list(v.items()), key=cmp_to_key(by_srcname_chain)
):
if chain1 is None:
maxnumchains = len(chain2)
elif chain2 is None:
maxnumchains = len(chain1)
else:
maxnumchains = max(len(chain1), len(chain2))
if chain1:
c1 = "".join([print_package(t, bin2src, source) for t in chain1[0]])
else:
c1 = ""
if chain2:
c2 = "".join([print_package(t, bin2src, source) for t in chain2[0]])
else:
c2 = ""
if not first:
print("
", file=outfile)
print(
'| %d | ' % (maxnumchains, len(v))
+ '%s | %s | %s | '
% (
maxnumchains,
" ".join([print_package(t, bin2src, source) for t in sorted(v)]),
c1,
c2,
),
file=outfile,
)
if first:
print(
'%s |
' % (rows, conflict_vpkg),
file=outfile,
)
else:
print("", file=outfile)
for i in range(1, maxnumchains):
if chain1 is not None and len(chain1) >= i + 1:
c1 = "".join([print_package(t, bin2src, source) for t in chain1[i]])
else:
c1 = ""
if chain2 is not None and len(chain2) >= i + 1:
c2 = "".join([print_package(t, bin2src, source) for t in chain2[i]])
else:
c2 = ""
print("| %s | %s |
" % (c1, c2), file=outfile)
first = False
print("
", file=outfile)
def print_footer(outfile, timestamp, srcs=[], srcsdir="", wwwroot=""):
if srcs:
print("Affected source packages:
", file=outfile)
srcsdir += "/"
if srcsdir.startswith(wwwroot):
srcsdir = srcsdir[len(wwwroot) : -1]
for src in sorted([s for s in srcs if s is not None]):
print('%s' % (srcsdir, src, src), file=outfile)
url = "https://salsa.debian.org/debian-bootstrap-team/boott"
footer = """
The JSON data used to generate these pages was computed using botch, the
bootstrap/build ordering tool chain. The source code of botch can be
redistributed under the terms of the LGPL3+ with an OCaml linking
exception. The source code can be retrieved from
https://salsa.debian.org/debian-bootstrap-team/botch
The html pages were generated by code which can be retrieved from %s and which can be redistributed under the terms of the
AGPL3+
For questions and bugreports please contact j [dot] schauer [at] email
[dot] de.
""" % (
url,
url,
)
if timestamp:
print("generated: %s
" % timestamp, file=outfile)
print("%s" % footer, file=outfile)
def src_of_pkg(t, bin2src):
n, a, v, _ = t
if n.startswith("src:"):
return n[4:], v
if n == "":
return (None, None)
res = bin2src.get((n, a, v))
if res:
return res
# this can happen for a dependency on a package that is not in the archive
print("cannot find source for %s %s" % (n, v), file=sys.stderr)
return (None, None)
def print_top10summary(outfile, missing, conflict, bin2src, source=None):
if not missing and not conflict:
return
print("Top 10 summary
", file=outfile)
print(
'The following is a summary of the full "missing" and '
+ '"conflict" tables below. It only shows the first and last '
+ "columns of the full tables and only displays the top 10 rows.
",
file=outfile,
)
if missing:
print("Missing
", file=outfile)
print(
"| # of packages per missing | Unsatisfied "
+ "dependency |
|---|
",
file=outfile,
)
for missing_vpkg, v in sorted(
list(missing.items()), key=cmp_to_key(by_affected_sources_missing)
)[:10]:
rows = len(v)
for c in list(v.keys()):
if c:
rows += len(c) - 1
print(
"| %d | %s |
"
% (sum([len(s) for s in v.values()]), missing_vpkg),
file=outfile,
)
print("
", file=outfile)
if conflict:
print("Conflict
", file=outfile)
print(
"| # of packages per conflict | " + "Conflict |
|---|
",
file=outfile,
)
for conflict_vpkg, v in sorted(
list(conflict.items()), key=cmp_to_key(by_affected_sources_conflict)
)[:10]:
print(
"| %d | %s |
"
% (sum([len(s) for s in v.values()]), conflict_vpkg),
file=outfile,
)
print("
", file=outfile)
def create_srcpkg_pages(
missing, conflict, bin2src, srcsdir, description, btsuser, btstag, timestamp
):
srcsdict = defaultdict(
lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(dict)))
)
# now go through all missing and conflict and copy them to the source
# packages that associate with them
# it is not possible to find a source package for a vpkg (especially if
# it's missing)
for vpkg, v in missing.items():
for chain, v in v.items():
for c in chain:
for p in c:
sn, sv = src_of_pkg(p, bin2src)
if not sn:
continue
srcsdict[sn][sv]["missing"][vpkg][chain] = v
for p in v:
sn, sv = src_of_pkg(p, bin2src)
srcsdict[sn][sv]["missing"][vpkg][chain] = v
for vpkg, v in conflict.items():
for (chain1, chain2), v in v.items():
for c in chain1:
for p in c:
sn, sv = src_of_pkg(p, bin2src)
if not sn:
continue
srcsdict[sn][sv]["conflict"][vpkg][(chain1, chain2)] = v
for c in chain2:
for p in c:
sn, sv = src_of_pkg(p, bin2src)
if not sn:
continue
srcsdict[sn][sv]["conflict"][vpkg][(chain1, chain2)] = v
for p in v:
sn, sv = src_of_pkg(p, bin2src)
srcsdict[sn][sv]["conflict"][vpkg][(chain1, chain2)] = v
if not os.path.isdir(srcsdir):
os.mkdir(srcsdir)
for srcpkg, versions in srcsdict.items():
if not srcpkg:
continue
fname = os.path.join(srcsdir, srcpkg) + ".html"
with open(fname, "w", encoding="utf8") as srcpkgpage:
print_header(srcpkgpage, description, btsuser, btstag)
for ver in versions.keys():
print(
"%s
"
% print_package(
("src:" + srcpkg, None, ver, None), bin2src, color=False
),
file=srcpkgpage,
)
print_top10summary(
srcpkgpage,
versions[ver].get("missing"),
versions[ver].get("conflict"),
bin2src,
(srcpkg, ver),
)
if versions[ver].get("missing"):
print_missing(
srcpkgpage, versions[ver]["missing"], bin2src, (srcpkg, ver)
)
if versions[ver].get("conflict"):
print_conflict(
srcpkgpage, versions[ver]["conflict"], bin2src, (srcpkg, ver)
)
print_footer(srcpkgpage, timestamp)
return srcsdict.keys()
def dose2html(
yamlin,
outfile,
description="",
srcpkgbugs=set(),
binpkgbugs=set(),
btsuser=None,
btstag=None,
srcsdir=None,
packages=[],
timestamp=None,
wwwroot=None,
verbose=False,
):
bin2src = dict()
for pkg in packages:
name = pkg["Package"]
arch = pkg["Architecture"]
ver = pkg["Version"]
src = pkg.get("Source")
if src:
if " " in src:
srcname, srcver = src.split(" ", 1)
srcver = srcver[1:-1]
else:
srcname = src
srcver = ver
else:
srcname = name
srcver = ver
bin2src[(name, arch, ver)] = (srcname, srcver)
bin2src[(name, arch, None)] = (srcname, None)
provides = pkg.get("Provides")
if not provides:
continue
for pname in provides.split(","):
pname = pname.strip()
bin2src[(pname, arch, ver)] = (srcname, srcver)
bin2src[(pname, arch, None)] = (srcname, None)
missing, conflict = parse_dose_yaml_mc(yamlin)
print_header(outfile, description, btsuser, btstag)
print_top10summary(outfile, missing, conflict, bin2src)
print_missing(outfile, missing, bin2src)
print_conflict(outfile, conflict, bin2src)
if srcsdir and packages:
srcpkgs = create_srcpkg_pages(
missing, conflict, bin2src, srcsdir, description, btsuser, btstag, timestamp
)
print_footer(outfile, timestamp, srcpkgs, srcsdir, wwwroot)
else:
print_footer(outfile, timestamp)
class _ExtendAction(argparse.Action):
def __init__(self, option_strings, dest, nargs=None, const=None, **kwargs):
if nargs == 0:
raise ValueError("nargs for extend actions must be > 0")
if const is not None and nargs != argparse.OPTIONAL:
raise ValueError("nargs must be %r to supply const" % argparse.OPTIONAL)
super(_ExtendAction, self).__init__(
option_strings=option_strings, dest=dest, nargs=nargs, const=const, **kwargs
)
def __call__(self, parser, namespace, values, option_string=None):
items = getattr(namespace, self.dest, None)
items = argparse._copy_items(items)
items.extend(values)
setattr(namespace, self.dest, items)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="given a buildcheck result, create a html overview"
)
parser.register("action", "extend", _ExtendAction)
parser.add_argument(
"yamlin", type=read_yaml_file, help="input in yaml format (dose3 output)"
)
parser.add_argument("htmlout", type=write_plain, help="output in html format")
parser.add_argument("--desc", default="", help="descriptive HTML snippet")
parser.add_argument("--btsuser", help="bts user to associate packages with")
parser.add_argument("--btstag", help="bts tag to associate packages with")
parser.add_argument(
"--srcsdir",
help="output directory for " + "individual source package overviews",
)
parser.add_argument(
"--wnpp",
action="store_true",
help="retrieve and print wnpp status for packages",
)
parser.add_argument(
"--packages",
type=read_tag_file,
action="extend",
default=[],
help="Packages file to create a mapping from binary " + "to source packages",
)
parser.add_argument(
"--timestamp",
help="put a freeform timestamp string at the bottom " + "of the generated HTML",
)
parser.add_argument(
"--wwwroot",
help="The HTML hyperlink to the source package "
"overview by default will begin with the string given "
'in the "--srcsdir" option. The string given by this '
"option will be removed from the beginning of that "
"path in the HTML href attribute. If this option ends "
"with a slash, then the resulting hyperlinks will "
"become relative.",
)
parser.add_argument("-v", "--verbose", action="store_true", help="be verbose")
args = parser.parse_args()
srcpkgbugs = defaultdict(list)
binpkgbugs = defaultdict(list)
if args.btsuser and args.btstag:
# this only corks with python2 because SOAPpy is not available for
# python3
# import SOAPpy
# server = SOAPpy.SOAPProxy('http://bugs.debian.org/cgi-bin/soap.cgi',
# 'Debbugs/SOAP')
# bugnrs = server.get_usertag('debian-cross@lists.debian.org',
# 'cross-satisfiability')
# buginfos = server.get_status(bugnrs['cross-satisfiability'])
# for bug in buginfos['item']:
# k = bug['key']
# v = bug['value']
# if v['package']:
# binpkgbugs[v['package']].append(k)
# else:
# for s in v['source'].split(','):
# s = s.strip()
# srcpkgbugs[s].append(k)
import subprocess
res = subprocess.check_output(
"bts select users:%s " % args.btsuser
+ "tag:%s status:open | bts status " % args.btstag
+ "fields:source,package,bug_num file:-",
shell=True,
)
for b in res.decode().split("\n\n"):
if not b:
continue
d = {k: v for k, v in [L.split("\t") for L in b.split("\n") if L.strip()]}
binpkgbugs[d["package"]].append(d["bug_num"])
for s in d["source"].split(","):
s = s.strip()
srcpkgbugs[s].append(d["bug_num"])
wnpp_stats = dict()
if args.wnpp and args.packages:
import urllib.request
url = "https://qa.debian.org/data/bts/wnpp_rm"
with urllib.request.urlopen(url) as f:
# the following snippet is from
# distro_tracker/vendor/debian/tracker_tasks.py
for line in f:
line = line.decode().strip()
try:
package_name, wnpp_type, bug_id = line.split("|")[0].split()
bug_id = int(bug_id)
except ValueError:
# Badly formatted
continue
# Strip the colon from the end of the package name
package_name = package_name[:-1]
wnpp_stats[package_name] = (wnpp_type, bug_id)
with args.htmlout as f:
dose2html(
args.yamlin,
f,
args.desc,
srcpkgbugs,
binpkgbugs,
args.btsuser,
args.btstag,
args.srcsdir,
args.packages,
args.timestamp,
args.wwwroot,
args.verbose,
)