""" Copyright 2012 NetApp, Inc. All Rights Reserved, contribution by Weston Andros Adamson This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. """ import os from math import sqrt, pow import time from . import graph from .collection import * from .selector import Selector, SELECTOR_ORDER from .config import * from .workloads import * ENABLE_PIE_GRAPHS=False def strip_key_prefix(k): s = k.split(':', 1) if len(s) == 2: return s[1] return k # handle: has data, not reference point def pct_f(x, y): if y == 0.0: return 0.0 pct = (float(x) / float(y)) * 100.0 return pct def get_legend_html(r, color_idx, hatch_idx, classes): color = None if color_idx != None: color_idx = color_idx % len(COLORS) color = COLORS[color_idx] graph_attrs = { 'width': 0.18, 'height': 0.18, 'color': color, 'hatch_idx': hatch_idx, 'classes': classes, } return r.graphs.make_graph('legend', graph_attrs) def fmt_float(f, precision=4): if f != None: w_fmt = "%%.%uf" % precision w_fmt = w_fmt % f seen_dot = False while len(w_fmt): if not seen_dot and w_fmt[-1] == '0': w_fmt = w_fmt[:-1] elif w_fmt[-1] == '.': w_fmt = w_fmt[:-1] seen_dot = True else: break return w_fmt or '0' return f def html_fmt_group(g, report_selector): assert isinstance(g, Selector) def _html_selector_thing(sel, thing): return '%s' % (thing, getattr(sel, thing)) # always display mountopt descr = _html_selector_thing(g, 'mountopt') if g.detect: descr += _html_selector_thing(g, 'detect') if g.tag: descr += _html_selector_thing(g, 'tag') if (len(report_selector.kernels) > 1 or len(report_selector.clients) > 1 or len(report_selector.servers) > 1 or len(report_selector.paths) > 1): descr += '
' # only display kernel, server, etc if there are more than one in this # report's view if len(report_selector.kernels) > 1: descr += _html_selector_thing(g, 'kernel') if len(report_selector.clients) > 1: descr += _html_selector_thing(g, 'client') if len(report_selector.servers) > 1: descr += _html_selector_thing(g, 'server') if len(report_selector.paths) > 1: descr += _html_selector_thing(g, 'path') return """
%s
""" % \ (descr, g.html()) def html_fmt_value(mean, std, units=None): if units: scale, units = fmt_scale_units(mean, units) mean = mean / scale std = std / scale fmt_mean = fmt_float(mean, 2) fmt_std = fmt_float(std, 2) if fmt_std != '0': fmt_mean += ' %s%s' % \ (HTML_PLUSMINUS, fmt_std) if units: fmt_mean += ' %s' % (units,) return fmt_mean def html_stat_info_id(sel, key=None): r = [sel,] if key != None: r.append(key) r = repr(tuple(r)).replace(',', '_').replace("'", '') r = r.replace('(', '').replace(')', '') r = r.replace(' ', '') return r class Table: """ Basic Table """ def __init__(self, report, values, groups, keys, units, nolegend=False, fmt_key=None, fmt_group=None, fmt_cell=None, noheader=False, nolabels=False, index_offset=0, classes=None): assert isinstance(values, dict) self.report = report self.values = values def _empty(x): if isinstance(x, str): if x == HTML_NO_DATA or x == HTML_COMPARISON_ZERO: return True if x: return False return True self.seen_data = False for group in groups: for k in keys: if not _empty(values[group].get(k, None)): self.seen_data = True break self.groups = groups self.keys = keys self.units = units self.nolegend = nolegend self.fmt_key = fmt_key self.fmt_cell = fmt_cell self.fmt_group = fmt_group self.noheader = noheader self.nolabels = nolabels self.index_offset = index_offset self._classes = [] if classes: self._classes.extend(classes) formatted_keys = {} for k in self.keys: formatted_keys[k] = self.html_key(k) self.formatted_keys = formatted_keys formatted_groups = {} for g in self.groups: formatted_groups[g] = self.html_group(g) self.formatted_groups = formatted_groups formatted_cells = {} for g in self.groups: for k in self.keys: formatted_cells[(g, k)] = self.html_cell(g, k) self.formatted_cells = formatted_cells def classes(self): return ' '.join(self._classes) def html_key(self, k): """ return key formatted for html """ if self.fmt_key: return self.fmt_key(k) return k def html_group(self, g): """ return group formatted for html """ # add legend legend = '' % \ (html_stat_info_id(g),) if not self.nolegend: group_idx = self.groups.index(g) cidx = color_idx(group_idx) + self.index_offset classes = ('color_box', 'color_box_link') legend += get_legend_html(self.report, cidx, None, classes) # format group text if self.fmt_group: g = self.fmt_group(g, self.report.selection) return '
%s%s
' % (legend, str(g)) def html_cell(self, g, k): cell = None no_data = False val = self.values.get(g, {}).get(k, None) if isinstance(val, str): cell = val elif isinstance(val, (Stat, Bucket)): cell = html_fmt_value(val.mean(), val.std(), units=self.units) elif type(val).__name__ == "Bucket" or type(val).__name__ == 'Stat': cell = html_fmt_value(val.mean(), val.std(), units=self.units) else: assert val == None, "Not a string, Stat or Bucket: %r\ng = %s, k = %s" % (val, g, k) if cell == None: cell = HTML_NO_DATA if cell in (HTML_NO_DATA, HTML_COMPARISON_ZERO): no_data = True if self.fmt_cell: cell = self.fmt_cell(g, k, self.values, cell) other = '' if no_data: other = 'class="no_data"' return '%s' % (other, cell) def empty(self): return not self.seen_data def html(self): template = html_template(TEMPLATE_TABLE) return template.render(table=self) class WideTable: """ A collection of tables, where groups are split out by nfs proto version """ def __init__(self, report, values, groups, gmap, keys, units, nolegend=False, fmt_key=None, fmt_group=None, fmt_cell=None): self.tables = [] num = len(groups) cur = 0 for vers in NFS_VERSIONS: if vers not in gmap: continue assert cur < num new = Table(report, values, gmap[vers], keys, units, nolegend=nolegend, fmt_key=fmt_key, fmt_group=fmt_group, fmt_cell=fmt_cell, index_offset=cur, classes=['data', 'data_table_%s' % vers]) cur += len(gmap[vers]) self.tables.append(new) assert cur == num def html(self): r = ''.join([ x.html() for x in self.tables]) r = """
%s
""" % r return r def empty(self): for t in self.tables: if not t.empty(): return False return True class TocNode: """ Table of Contents """ def __init__(self, text, section, parent): self.text = text self.section = section self.parent = parent self.children = [] def title_list(self): tl = [] node = self while node: if node.text: tl.insert(0, node.text) node = node.parent return tl def title(self): tl = self.title_list() return ' : '.join(tl) def anchor(self): return '' % self.section def num(self): return '%s' % self.section def add(self, text): if self.section: section = '%s.%s' % (self.section, len(self.children) + 1) else: section = '%s' % (len(self.children) + 1,) new_node = TocNode(text, section, self) self.children.append(new_node) return new_node def unlink(self): if self.parent: self.parent.children.remove(self) def html(self): if not self.text: template = html_template(TEMPLATE_TOC) else: template = html_template(TEMPLATE_TOCNODE) return template.render(node=self) class Dataset: """ Dataset - title, description, image, tables """ def __init__(self, selection, widget, title, units, groups, key, vals, toc, report, no_graph=False, no_title=False, tall_cell=False, subtitle='', anchor=None, fmt_key=None, fmt_cell=None, fmt_group=None, bucket_def=None): """ generate the "dataset" - a graph and a table """ self.key = key self.subtitle = subtitle self.anchor = anchor self.selection = selection self.report = report self.fmt_key = fmt_key self.fmt_cell = fmt_cell self.fmt_group = fmt_group self.bucket_def = bucket_def if not self.fmt_key: self.fmt_key = lambda x: x self.hatch_map, bucket_to_value, total_value = \ self.make_hatch_map(vals, groups, key) # XXX ugly self.all_buckets = None value_map = {} for i, g in enumerate(groups): value_map[g] = {} v = vals.get(g, {}).get(key, None) if v != None: if isinstance(v, Bucket) or type(v).__name__ == 'Bucket': self.all_buckets = True for stat in v.foreach(): value_map[g][stat.name] = stat.mean() else: self.all_buckets = False value_map[g][key] = v.mean() break self.bucket_legend = '' self.bucket_pie = '' # does ordering matter here? bk_order = [ (k,v) for k, v in self.hatch_map.items() ] if bk_order: bk_order.sort(key=lambda x: x[1]) table_values = {} bucket_names = [] for bucket_name, hatch_idx in bk_order: display_key = bucket_name if self.bucket_def: display_key = self.bucket_def.key_display(display_key) display_key = self.fmt_key(display_key) table_values[bucket_name] = { 'description': self.report.collection.stat_description(bucket_name), 'legend': get_legend_html(self.report, None, hatch_idx, ('cmp_ref',)), 'pct': '%0.1f' % pct_f(bucket_to_value[bucket_name], total_value), 'display_key': display_key, } bucket_names.append(bucket_name) tbl = Table(self.report, table_values, bucket_names, ('legend', 'display_key', 'description', 'pct'), '', nolegend=True, noheader=True, nolabels=True) self.bucket_legend = tbl.html() if ENABLE_PIE_GRAPHS and len(bk_order) > 1: pie_values = [] pie_colors = [] pie_hatches = [] for bucket_name, hatch_idx in bk_order: total = 0.0 for g in groups: total += value_map[g].get(bucket_name, 0.0) pie_values.append(total) pie_colors.append('#ffffff') pie_hatches.append(get_hatch(hatch_idx)) pie_scale = 0.3 * float(len(pie_values)) pie_attrs = { 'graph_width': pie_scale, 'graph_height': pie_scale, 'slice_labels': [''] * len(pie_values), 'slice_colors': pie_colors, 'slice_values': pie_values, 'slice_explode': [0.0] * len(pie_values), 'slice_hatches': pie_hatches, 'classes': ('legend_pie',), } self.bucket_pie = report.graphs.make_graph('pie', pie_attrs) self.toc = toc.add(title) no_ylabel = False # make comparison values self.comparison_vals_map = {} select_order = ('mountopt', 'detect', 'tag') self.comparison_vals_map['config'] = \ self.make_comparison_vals(vals, key, groups, select_order) select_order = ('mountopt',) self.comparison_vals_map['mountopt'] = \ self.make_comparison_vals(vals, key, groups, select_order) select_order = ('detect',) self.comparison_vals_map['detect'] = \ self.make_comparison_vals(vals, key, groups, select_order) select_order = ('tag',) self.comparison_vals_map['tag'] = \ self.make_comparison_vals(vals, key, groups, select_order) self.gmap = groups_by_nfsvers(groups) self.nfs_versions = [ v for v in NFS_VERSIONS if v in self.gmap ] # ensure the order of groups is in nfs_version order groups = [] for v in self.nfs_versions: groups.extend(self.gmap[v]) self.color_map = {} for i, g in enumerate(groups): self.color_map[g] = COLORS[color_idx(i)] self.tab = WideTable(report, vals, groups, self.gmap, [key], units, fmt_key=fmt_key, fmt_cell=self.fmt_cell_modes, fmt_group=fmt_group) graph_height = 2.0 graph_width = 8.0 # Graph section self.graph_html = '' if not self.empty() and not no_graph: graph_attrs = { 'units': units, 'key': key, 'groups': groups, 'gmap': self.gmap, 'no_ylabel': no_ylabel, 'graph_width': graph_width, 'graph_height': graph_height, 'classes': ('data_graph',), 'selection': selection, 'hatch_map': self.hatch_map, 'color_map': self.color_map, } self.graph_html = report.graphs.make_graph('bar_and_nfsvers', graph_attrs) binfo = self.report.collection.get_better_info(selection, key) self.better_sym = binfo[0] self.better_str = binfo[1] self.better_more = binfo[2] self.description = self.report.collection.stat_description(key) if not self.description: self.description = '' def fmt_cell_modes(self, g, k, v, c): if self.fmt_cell: c = self.fmt_cell(g, k, v, c) hits = self.fmt_cell_hits(v[g].get(k, None)) c = '
%s
' % (c,) if hits: c += '' % (hits,) for compare, compvals in self.comparison_vals_map.items(): if compvals: c += '
' \ '
%s
' % (compare, compvals[g][k]) stat = v[g].get(k, None) info_html = '' if stat: table_hdrs = [] table_rows = [] color_idx = COLORS.index(self.color_map[g]) if isinstance(stat, Bucket) or type(stat).__name__ == 'Bucket': table_hdrs.append('run') for x in stat.foreach(): hidx = self.hatch_map[x.name] hdr = get_legend_html(self.report, color_idx, hidx, classes=('data_info_hatch',)) hdr += '
%s' % self.fmt_key(self.bucket_def.key_display(x.name)) table_hdrs.append(hdr) table_hdrs.append('total') else: table_hdrs.append('run') hdr = get_legend_html(self.report, color_idx, 0, classes=('data_info_hatch',)) hdr += '
' + stat.name table_hdrs.append(hdr) for run, tracedir in enumerate(stat.tracedirs()): row = [] row.append('%s' % (tracedir, run)) if isinstance(stat, Bucket) or type(stat).__name__ == 'Bucket': for x in stat.foreach(): row.append('%s' % (tracedir, stat.filename(), fmt_float(x.run_value(tracedir, None)))) row.append(fmt_float(stat.run_total(tracedir))) else: row.append('%s' % (tracedir, stat.filename(), fmt_float(stat.run_value(tracedir, None)))) table_rows.append(row) info_html = html_template(TEMPLATE_DATAINFOPANE).render( table_hdrs=table_hdrs, table_rows=table_rows, avg=fmt_float(stat.mean()), std=fmt_float(stat.std())) c += '' % info_html return c def empty(self): return self.tab.empty() def make_hatch_map(self, values, groups, key): # calc magnitude of each key across ALL groups for ordering key2val = {} total_val = 0.0 for g in groups: stat = values[g].get(key, None) if stat == None: continue if isinstance(stat, Bucket) or type(stat).__name__ == 'Bucket': for sub in stat.foreach(): if sub.name not in key2val: key2val[sub.name] = 0.0 key2val[sub.name] += sub.mean() total_val += sub.mean() else: # a basic Stat - makes hatch map with one entry if stat.name not in key2val: key2val[stat.name] = 0.0 key2val[stat.name] += stat.mean() total_val += stat.mean() ordered = [ (k, v) for k, v in key2val.items() ] if ordered: ordered.sort(key=lambda x: x[1], reverse=True) k2h = {} for i, kv in enumerate(ordered): assert kv[0] not in k2h k2h[kv[0]] = i return k2h, key2val, total_val def html(self): template = html_template(TEMPLATE_DATASET) return template.render(dataset=self) def make_comparison_vals(self, vals, key, groups, select_order): newvals = {} #select_order = ('mountopt', 'detect', 'tag') compare_groups = [] for g in groups: idx = None for i, cg in enumerate(compare_groups): if g.match_order(cg[0], select_order): # found a group! idx = i break if idx != None: compare_groups[idx].append(g) # new group else: compare_groups.append([g,]) for cg in compare_groups: ref_val = None ref_g = None for g in cg: if g not in newvals: newvals[g] = {} # handle no data v = vals[g].get(key, None) if v == None: newvals[g][key] = HTML_NO_DATA continue # handle zero if v.empty(): newvals[g][key] = HTML_COMPARISON_ZERO continue # find reference point if not ref_val: ref_val = v ref_g = g vstr = get_legend_html(self.report, groups.index(g), None, ('cmp_ref',)) newvals[g][key] = vstr continue diff_mean = pct_f(v.mean() - ref_val.mean(), ref_val.mean()) diff_std = sqrt((pow(ref_val.std(),2) + pow(v.std(),2))/2.0) diff_std = pct_f(diff_std, ref_val.mean()) operator = '-' if diff_mean >= 0.0: operator = '+' diff_val_str = '%0.2f' % abs(diff_mean) diff_std_str = '%0.2f' % abs(diff_std) ref_idx = groups.index(ref_g) if diff_val_str == '0.00' and diff_std_str == '0.00': operator = '=' vstr = '
%s
%s' % \ (operator, get_legend_html(self.report, ref_idx, None, ('cmp_ref',))) else: cell = html_fmt_value(abs(diff_mean), abs(diff_std)) vstr = """%s
%s
%s%%
""" % \ (get_legend_html(self.report, ref_idx, None, ('cmp_ref',)), operator, cell) newvals[g][key] = vstr return newvals def fmt_cell_hits(self, value): classes = ('hatch_hit',) if isinstance(value, Bucket) or type(value).__name__ == 'Bucket': stat_list = [ x for x in value.foreach() ] stat_list.sort(key=lambda x: x.mean(), reverse=True) units = self.report.collection.stat_units(value.name) pie_html = '' if ENABLE_PIE_GRAPHS and len(stat_list) > 1: pie_values = [ x.mean() for x in stat_list ] pie_attrs = { 'graph_width': 0.8, 'graph_height': 0.8, 'slice_labels': [''] * len(stat_list), 'slice_colors': ['#ffffff'] * len(stat_list), 'slice_values': pie_values, 'slice_explode': [0.0] * len(stat_list), 'slice_hatches': [ get_hatch(self.hatch_map[x.name]) for x in stat_list ], 'classes': ('breakdown_pie',), } pie_html = self.report.graphs.make_graph('pie', pie_attrs) out = [pie_html + ''] total = sum([ x.mean() for x in stat_list ]) for stat in stat_list: legend = get_legend_html(self.report, None, self.hatch_map[stat.name], classes) fmt = html_fmt_value(stat.mean(), stat.std(), units=units) if value.mean(): pct = (stat.mean() / total) * 100.0 else: pct = 0.0 out.append('' % \ (pct, legend, fmt)) fmt = html_fmt_value(total, 0, units=units) out.append('' % \ (fmt,)) out.append('
%0.2f%%%s%s
total%s
') return '
%s
' % ('\n'.join(out)) return '' # # WIDGETS # class Widget: def __init__(self, collection, selection, report, toc): assert self.widget assert self.desc self.collection = collection self.selection = selection self.report = report self.toc = toc.add(self.widget) self.datasets = [] self.statnote_mesgs = [ f(selection) for f in getattr(self, 'statnotes', []) ] self.statnote_mesgs = [ m for m in self.statnote_mesgs if m ] self.setup() def setup(self): raise NotImplemented def new_dataset(self, selection, title, groups, key, vals, **kwargs): collection = self.report.collection units = kwargs.get('units', collection.stat_units(key)) try: del kwargs['units'] except: pass if not 'fmt_group' in kwargs: kwargs['fmt_group'] = html_fmt_group kwargs['fmt_key'] = strip_key_prefix new = Dataset(selection, self.widget, title, units, groups, key, vals, self.toc, self.report, **kwargs) if not new.empty(): self.datasets.append(new) else: new.toc.unlink() def empty(self): return len(self.datasets) == 0 def html(self): template = html_template(TEMPLATE_WIDGET) return template.render(widget=self, dataset_class=Dataset) class SimpleWidget(Widget): ds_info = None bucket_table_html = '' bucket_pie_html = '' def __init__(self, collection, selection, report, toc): assert len(self.ds_info) self.keys = set() for t, key in self.ds_info: if key in self.keys: raise ValueError("key %s already used in ds_info" % k) self.keys.add(key) self.keys = tuple(self.keys) Widget.__init__(self, collection, selection, report, toc) def setup(self): groups, vals = self.collection.gather_data(self.keys, self.selection) for t, key in self.ds_info: self.new_dataset(self.selection, t, groups, key, vals) class Widget_RunTimes(SimpleWidget): widget = 'Times' desc = 'Run times of workload as measured by time(1)' statnotes = (statnote_filebench_times,) ds_info = ( ('Trace Time', 'times:time_trace'), ('Other Time', 'times:time_other'), ('Real Time', 'times:time_real'), ('Sys Time', 'times:time_sys'), ('User Time', 'times:time_user'), ) class Widget_Filebench(SimpleWidget): widget = 'Filebench' desc = 'Stats collected from the Filebench test suite output.' ds_info = ( ('Operation count', 'filebench:op_count'), ('Operations/second', 'filebench:ops_per_second'), ('MB/second', 'filebench:mb_per_second'), ('CPU/Operation', 'filebench:cpu_per_op'), ('Latency', 'filebench:latency_ms'), ) class Widget_NfsBytes(SimpleWidget): widget = 'Bytes' desc = 'Bytes read and written by syscalls (normal and O_DIRECT) ' \ 'and NFS operations.' ds_info = ( ('Read Syscalls', 'mountstats:read_normal'), ('Write Syscalls', 'mountstats:write_normal'), ('O_DIRECT Read Syscalls', 'mountstats:read_odirect'), ('O_DIRECT Write Syscalls', 'mountstats:write_odirect'), ('Read NFS calls', 'mountstats:read_nfs'), ('Write NFS calls', 'mountstats:write_nfs'), ) class Widget_RpcCounts(SimpleWidget): widget = 'RPC' desc = 'RPC message counts' ds_info = ( ('Calls', 'nfsstats:rpc_calls'), ) # XXX not used? class Widget_RpcStats(SimpleWidget): widget = 'RPC' desc = 'RPC message counts' statnotes = (statnote_v3_no_lock, statnote_v41_pnfs_no_ds) #'xid_not_found', #'backlog_queue_avg', ds_info = ( ('RPC Requests', 'mountstats:rpc_requests'), ('RPC Replies', 'mountstats:rpc_requests'), ) class Widget_MaxSlots(SimpleWidget): widget = 'Max Slots' desc = 'Max slots used for rpc transport' ds_info = ( ('Max Slots', 'proc_mountstats:xprt_max_slots'), ) class Widget_Nfsiostat(SimpleWidget): widget = 'Throughput' desc = 'Throughput statistics' statnotes = [statnote_v41_pnfs_no_ds] ds_info = ( ('Read KB/s', 'nfsiostat:read_kb_per_sec'), ('Write KB/s', 'nfsiostat:write_kb_per_sec'), ('Read Operations/s', 'nfsiostat:read_ops_per_sec'), ('Write operations/s', 'nfsiostat:write_ops_per_sec'), ('Read Average KB per Operation', 'nfsiostat:read_kb_per_op'), ('Write Average KB per Operation', 'nfsiostat:write_kb_per_op'), ('Read Average RTT', 'nfsiostat:read_avg_rtt_ms'), ('Write Average RTT', 'nfsiostat:write_avg_rtt_ms'), ) class Widget_VfsEvents(SimpleWidget): widget = 'VFS Events' desc = 'Event counters from the VFS layer' ds_info = ( ('Open', 'proc_mountstats:vfs_open'), ('Lookup', 'proc_mountstats:vfs_lookup'), ('Access', 'proc_mountstats:vfs_access'), ('Updatepage', 'proc_mountstats:vfs_updatepage'), ('Readpage', 'proc_mountstats:vfs_readpage'), ('Readpages', 'proc_mountstats:vfs_readpages'), ('Writepage', 'proc_mountstats:vfs_writepage'), ('Writepages', 'proc_mountstats:vfs_writepages'), ('Getdents', 'proc_mountstats:vfs_getdents'), ('Setattr', 'proc_mountstats:vfs_setattr'), ('Flush', 'proc_mountstats:vfs_flush'), ('Fsync', 'proc_mountstats:vfs_fsync'), ('Lock', 'proc_mountstats:vfs_lock'), ('Release', 'proc_mountstats:vfs_release'), ) class Widget_InvalidateEvents(SimpleWidget): widget = 'Validation Events' desc = 'Counters for validation events' ds_info = ( ('Inode Revalidate', 'proc_mountstats:inode_revalidate'), ('Dentry Revalidate', 'proc_mountstats:dentry_revalidate'), ('Data Invalidate', 'proc_mountstats:data_invalidate'), ('Attr Invalidate', 'proc_mountstats:attr_invalidate'), ) class Widget_PnfsReadWrite(SimpleWidget): widget = 'pNFS Events' # XXX counts or bytes?? desc = 'Counters for pNFS reads and writes' ds_info = ( ('Reads', 'proc_mountstats:pnfs_read'), ('Writes', 'proc_mountstats:pnfs_write'), ) class Widget_OtherEvents(SimpleWidget): widget = 'NFS Events' desc = 'Counters for NFS events' ds_info = ( ('Short Read', 'proc_mountstats:short_read'), ('Short Write', 'proc_mountstats:short_write'), ('Congestion Wait', 'proc_mountstats:congestion_wait'), ('Extend Write', 'proc_mountstats:extend_write'), ('Setattr Truncate', 'proc_mountstats:setattr_trunc'), ('Delay', 'proc_mountstats:delay'), ('Silly Rename', 'proc_mountstats:silly_rename'), ) class BucketWidget(Widget): bucket_def = None bucket_table_html = '' bucket_pie_html = '' def __init__(self, collection, selection, report, toc): assert self.bucket_def assert self.desc Widget.__init__(self, collection, selection, report, toc) def setup(self): bucket_names = self.bucket_def.bucket_names() groups, vals = self.collection.gather_data(bucket_names, self.selection) bucket_totals = {} bucket_hits = {} total = 0.0 for bucket_name in bucket_names: bucket_totals[bucket_name] = 0.0 bucket_hits[bucket_name] = set() # find all Buckets across all groups this_bucket = [] # TODO: move unit gathering to Bucket? units = None for g in groups: bucket = vals[g].get(bucket_name, None) if bucket != None: this_bucket.append(bucket) for stat in bucket.foreach(): m = stat.mean() total += m bucket_totals[bucket_name] += m bucket_hits[bucket_name].add(stat.name) u = self.collection.stat_units(stat.name) if not units: units = u else: assert u == units # skip empty datasets if not this_bucket or all([ x.empty() for x in this_bucket]): continue self.new_dataset(self.selection, bucket_name, groups, bucket_name, vals, tall_cell=True, bucket_def=self.bucket_def, units=units) bucket_info = [ (k, v) for k, v in bucket_totals.items() ] if bucket_info: bucket_info.sort(key=lambda x: x[1], reverse=True) bucket_info = [ x for x in bucket_info if x[1] ] vals = {} for name, btotal in bucket_info: pct = (btotal / total) * 100.0 hits = list(bucket_hits[name]) hits = [ self.bucket_def.key_display(h) for h in bucket_hits[name] ] hits.sort() vals[name] = {'pct': "%.01f%%" % pct, 'hits': ', '.join(hits)} if len(bucket_info) > 1: tbl = Table(self.report, vals, [ x[0] for x in bucket_info if x[1] ], ['pct', 'hits'], '', noheader=True) self.bucket_table_html = tbl.html() if ENABLE_PIE_GRAPHS: pie_size = float(len(bucket_info)) / 2.0 pie_attrs = { 'graph_width': pie_size, 'graph_height': pie_size, 'slice_labels': [''] * len(bucket_info), 'slice_colors': COLORS, 'slice_values': [ x[1] for x in bucket_info ], 'slice_explode': [0.0] * len(bucket_info), 'classes': ('bucket_pie',), } self.bucket_pie_html = \ self.report.graphs.make_graph('pie', pie_attrs) class Widget_Iozone(BucketWidget): widget = 'Iozone' desc = 'Iozone Averages' bucket_def = parse.iozone_bucket_def class Widget_WallTimes(BucketWidget): widget = 'Wall Times' desc = 'Wall-clock times of workloads' bucket_def = parse.wall_times_bucket_def class Widget_ExecTimes(BucketWidget): widget = 'Exec Times' desc = 'Execution times of workloads' bucket_def = parse.exec_times_bucket_def class Widget_NfsOpsCount(BucketWidget): widget = 'NFS Operations' desc = 'Count of NFS operations' statnotes = (statnote_v3_no_lock,) bucket_def = parse.nfsstat_bucket_def class Widget_NfsOpsExec(BucketWidget): widget = 'Exec Time' desc = 'Execution time of NFS operations' statnotes = (statnote_v3_no_lock, statnote_v41_pnfs_no_ds) bucket_def = parse.mountstat_exec_time_bucket_def class Widget_NfsOpsRtt(BucketWidget): widget = 'RTT by Operation Group' desc = 'Round trip time of NFS operations' statnotes = (statnote_v3_no_lock, statnote_v41_pnfs_no_ds) bucket_def = parse.mountstat_rtt_bucket_def class Widget_NfsOpsBytesSent(BucketWidget): widget = 'Bytes Sent' desc = 'Average bytes sent for NFS operations' statnotes = (statnote_v3_no_lock, statnote_v41_pnfs_no_ds) bucket_def = parse.mountstat_bytes_sent_bucket_def class Widget_NfsOpsBytesReceived(BucketWidget): widget = 'Bytes Received' desc = 'Average bytes received for NFS operations' statnotes = (statnote_v3_no_lock, statnote_v41_pnfs_no_ds) bucket_def = parse.mountstat_bytes_received_bucket_def class Info: def __init__(self, collection, selection, report, toc): if toc: self.toc = toc.add('Info') else: self.toc = None self.collection = collection self.selection = selection self.report = report def html(self): self.selector_infos = [] def _join_lists(x): if isinstance(x, (list, tuple)): return ', '.join(x) return x def _htmlize_list(x): if isinstance(x, (list, tuple)): return '\n
'.join([ str(y).replace(',', ' ') for y in x]) return x last_info = None total_runs = 0 start_min = None stop_max = None seen_mdts = [] mount_options = {} workload_info = [] if isinstance(self.report, Report): topsel = self.report.selection else: topsel = self.collection.selection for wsel in topsel.foreach('workload'): workload_name = wsel.fmt('workload') workload_command = '' workload_description = '' # XXX 0? col = self.collection atr = col.get_attr(wsel, 'workload_command') if atr: workload_command = atr[0] atr = col.get_attr(wsel, 'workload_description') if atr: workload_description = atr[0] wdesc = '%s' \ '%s' % \ (workload_name, workload_description) command = _htmlize_list(workload_command.split('\n')) if isinstance(self.report, ReportSet): title = _make_report_title(self.collection, wsel) path = _make_report_path(title) rpts = '%s' % (path, title) else: rpts = '' workload_info.append((wdesc, command, rpts)) for sel in wsel.foreach(): if self.collection.has_traces(sel): trace = self.collection.get_trace(sel) mdt = sel.mountopt if sel.detect: mdt += ' ' + _join_lists(sel.detect) if sel.tags: mdt += ' ' + _join_lists(sel.tags) real_info = { 'workload': sel.workload, 'kernel': sel.kernel, 'mdt': mdt, 'mountopt': sel.mountopt, 'detect': sel.detect, 'tags': sel.tags, 'client': sel.client, 'server': sel.server, 'path': sel.path, 'runs': trace.num_runs(), 'starttime': min(trace.get_attr('starttime')), 'stoptime': max(trace.get_attr('stoptime')), 'mount_options': trace.get_attr('mount_options'), } total_runs += real_info['runs'] if not start_min: start_min = real_info['starttime'] else: start_min = min(start_min, real_info['starttime']) if not stop_max: stop_max = real_info['stoptime'] else: stop_max = max(stop_max, real_info['stoptime']) if not mdt in seen_mdts: seen_mdts.append(mdt) if mdt not in mount_options: mount_options[mdt] = set() mount_options[mdt] = \ mount_options[mdt].union(real_info['mount_options']) # lowlite (opposite of hilite) values same as prev row. info = {} ignore = ('runs',) for k in list(real_info.keys()): if not k in ignore and last_info and \ real_info[k] == last_info[k]: info[k] = '%s' % \ (real_info[k],) else: info[k] = real_info[k] # recompute mdt info['mdt'] = info['mountopt'] if info['detect']: info['mdt'] += ' ' + _join_lists(info['detect']) if info['tags']: info['mdt'] += ' ' + _join_lists(info['tags']) self.selector_infos.append(info) last_info = real_info for mdt in seen_mdts: tmp = list(mount_options[mdt]) tmp.sort() mount_options[mdt] = [ x.replace(',', ', ') for x in tmp ] self.total_runs = total_runs self.mount_options = mount_options self.seen_mdts = seen_mdts self.times = 'ran between %s and %s' % \ (time.ctime(start_min), time.ctime(stop_max)) if isinstance(self.report, Report): self.workload = workload_info[0][0] self.command = workload_info[0][1] else: self.workload = None self.command = None # gather warnings self.warnings = self.collection.warnings() if self.warnings: self.warnings = \ [ '%s:' % (d, '
  • '.join(w)) for d, w in self.warnings ] self.warnings = '
  • %s
    • ' % '
  • '.join(self.warnings) # user notes self.usernotes = self.collection.notes_get() self.more = [] if isinstance(self.report, ReportSet): self.more = [ """ %s
    command:
    %s
    reports:
    %s """ % x for x in workload_info ] template = html_template(TEMPLATE_REPORTINFO) return template.render(info=self, fmt_class=self._fmt_class) def _fmt_class(self, key): """ format class based on 'key' """ k = key.replace(' ', '_') # get rid of trailing 's' so plural and singular can share css if k.endswith('s'): k = k[:-1] return k _WIDGET_ORDER = ( #Widget_Iozone, Widget_Filebench, Widget_WallTimes, Widget_ExecTimes, Widget_NfsBytes, Widget_Nfsiostat, Widget_RpcCounts, Widget_MaxSlots, Widget_NfsOpsCount, Widget_NfsOpsExec, Widget_NfsOpsRtt, Widget_NfsOpsBytesSent, Widget_NfsOpsBytesReceived, Widget_VfsEvents, Widget_InvalidateEvents, Widget_PnfsReadWrite, Widget_OtherEvents, ) def _make_report_path(title): path = title path = path.replace(' ', '_') # erase certain chars for c in ['/', ':', ',']: path = path.replace(c, '') path = path.lower() + '.html' return path.replace('_report', '') def _make_report_title(collection, selection): title = "Report" info = selection.display_info(collection.selection) if info: out = [] for x in info: if x[0].startswith('workload'): out.append(str(x[1])) else: out.append('%s: %s' % x) title += ': ' + ', '.join(out) return title class Report: show_zeros = False widget_classes = _WIDGET_ORDER def __init__(self, rptset, collection, selection, reportdir, graphs, cssfile): self.rptset = rptset self.collection = collection self.selection = selection self.reportdir = reportdir self.graphs = graphs self.cssfile = cssfile self.toc = TocNode(None, None, None) self.title = _make_report_title(collection, selection) self.path = _make_report_path(self.title) self.report_info = Info(collection, selection, self, self.toc) self.widgets = [] for widget_class in self.widget_classes: w = widget_class(self.collection, selection, self, self.toc) if not w.empty(): self.widgets.append(w) else: w.toc.unlink() def empty(self): return len(self.widgets) == 0 def html(self): template = html_template(TEMPLATE_REPORT) return template.render(report=self) class ReportSet: def __init__(self, collection, serial_graph_gen): self.collection = collection self.reportdir = collection.resultsdir self.imagedir = os.path.join(self.reportdir, 'images') self.graphs = graph.GraphFactory(self.collection, self.imagedir, serial_gen=serial_graph_gen) self.cssfilepath = CSSFILEPATH self.cssfile = os.path.split(self.cssfilepath)[-1] self.jqueryurl = JQUERY_URL self.jsfilepath = JSFILEPATH self.jsfile = os.path.split(self.jsfilepath)[-1] self._clear_files() self._write_extrafiles() self.reportset_info = Info(collection, collection.selection, self, None) def _clear_files(self): os.system("rm %s/*.html 2> /dev/null" % self.reportdir) os.system("rm %s/*.css 2> /dev/null" % self.reportdir) # add legend to styles: def _write_extrafiles(self): os.system('cp %s %s' % (self.jsfilepath, self.reportdir)) os.system('cp %s %s' % (self.cssfilepath, self.reportdir)) def _write_report(self, r): abs_path = os.path.join(self.reportdir, r.path) with open(abs_path, 'w+') as f: f.write(r.html()) def _write_index(self): path = 'index.html' abs_path = os.path.join(self.reportdir, path) with open(abs_path, 'w+') as f: f.write(self.html_index()) def _step_through_reports(self, cb_f): for x in self.collection.selection.foreach('workload'): cb_f(x) def generate_report(self, selection): if not self.collection.has_traces(selection): return r = Report(self, self.collection, selection, self.reportdir, self.graphs, self.cssfile) self._write_report(r) def generate_reports(self): check_mpl_version() print inform("Generating Reports") print self._write_index() self._step_through_reports(self.generate_report) print self.graphs.wait_for_graphs() def html_index(self): """ generate an index file """ template = html_template(TEMPLATE_INDEX) return template.render(index=self) def html_toc(self, current_title=''): """ generate a
    tag with the index in it """ template = html_template(TEMPLATE_REPORTLIST) return template.render(index=self, current_title=current_title)