You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1569 lines
47 KiB
1569 lines
47 KiB
# -*- coding: utf-8 -*-
|
|
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""Table generating, analyzing and printing functions.
|
|
|
|
This defines several classes that are used to generate, analyze and print
|
|
tables.
|
|
|
|
Example usage:
|
|
|
|
from cros_utils import tabulator
|
|
|
|
data = [["benchmark1", "33", "44"],["benchmark2", "44", "33"]]
|
|
tabulator.GetSimpleTable(data)
|
|
|
|
You could also use it to generate more complex tables with analysis such as
|
|
p-values, custom colors, etc. Tables are generated by TableGenerator and
|
|
analyzed/formatted by TableFormatter. TableFormatter can take in a list of
|
|
columns with custom result computation and coloring, and will compare values in
|
|
each row according to taht scheme. Here is a complex example on printing a
|
|
table:
|
|
|
|
from cros_utils import tabulator
|
|
|
|
runs = [[{"k1": "10", "k2": "12", "k5": "40", "k6": "40",
|
|
"ms_1": "20", "k7": "FAIL", "k8": "PASS", "k9": "PASS",
|
|
"k10": "0"},
|
|
{"k1": "13", "k2": "14", "k3": "15", "ms_1": "10", "k8": "PASS",
|
|
"k9": "FAIL", "k10": "0"}],
|
|
[{"k1": "50", "k2": "51", "k3": "52", "k4": "53", "k5": "35", "k6":
|
|
"45", "ms_1": "200", "ms_2": "20", "k7": "FAIL", "k8": "PASS", "k9":
|
|
"PASS"}]]
|
|
labels = ["vanilla", "modified"]
|
|
tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC)
|
|
table = tg.GetTable()
|
|
columns = [Column(LiteralResult(),
|
|
Format(),
|
|
"Literal"),
|
|
Column(AmeanResult(),
|
|
Format()),
|
|
Column(StdResult(),
|
|
Format()),
|
|
Column(CoeffVarResult(),
|
|
CoeffVarFormat()),
|
|
Column(NonEmptyCountResult(),
|
|
Format()),
|
|
Column(AmeanRatioResult(),
|
|
PercentFormat()),
|
|
Column(AmeanRatioResult(),
|
|
RatioFormat()),
|
|
Column(GmeanRatioResult(),
|
|
RatioFormat()),
|
|
Column(PValueResult(),
|
|
PValueFormat()),
|
|
]
|
|
tf = TableFormatter(table, columns)
|
|
cell_table = tf.GetCellTable()
|
|
tp = TablePrinter(cell_table, out_to)
|
|
print tp.Print()
|
|
"""
|
|
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
import collections
|
|
import getpass
|
|
import math
|
|
import statistics
|
|
import sys
|
|
# TODO(crbug.com/980719): Drop scipy in the future.
|
|
# pylint: disable=import-error
|
|
import scipy
|
|
|
|
from cros_utils.email_sender import EmailSender
|
|
from cros_utils import misc
|
|
|
|
|
|
def _AllFloat(values):
|
|
return all([misc.IsFloat(v) for v in values])
|
|
|
|
|
|
def _GetFloats(values):
|
|
return [float(v) for v in values]
|
|
|
|
|
|
def _StripNone(results):
|
|
res = []
|
|
for result in results:
|
|
if result is not None:
|
|
res.append(result)
|
|
return res
|
|
|
|
|
|
def _RemoveMinMax(cell, values):
|
|
if len(values) < 3:
|
|
print('WARNING: Values count is less than 3, not ignoring min/max values')
|
|
print('WARNING: Cell name:', cell.name, 'Values:', values)
|
|
return values
|
|
|
|
values.remove(min(values))
|
|
values.remove(max(values))
|
|
return values
|
|
|
|
|
|
class TableGenerator(object):
|
|
"""Creates a table from a list of list of dicts.
|
|
|
|
The main public function is called GetTable().
|
|
"""
|
|
SORT_BY_KEYS = 0
|
|
SORT_BY_KEYS_DESC = 1
|
|
SORT_BY_VALUES = 2
|
|
SORT_BY_VALUES_DESC = 3
|
|
NO_SORT = 4
|
|
|
|
MISSING_VALUE = 'x'
|
|
|
|
def __init__(self, d, l, sort=NO_SORT, key_name='keys'):
|
|
self._runs = d
|
|
self._labels = l
|
|
self._sort = sort
|
|
self._key_name = key_name
|
|
|
|
def _AggregateKeys(self):
|
|
keys = collections.OrderedDict()
|
|
for run_list in self._runs:
|
|
for run in run_list:
|
|
keys.update(dict.fromkeys(run.keys()))
|
|
return list(keys.keys())
|
|
|
|
def _GetHighestValue(self, key):
|
|
values = []
|
|
for run_list in self._runs:
|
|
for run in run_list:
|
|
if key in run:
|
|
values.append(run[key])
|
|
values = _StripNone(values)
|
|
if _AllFloat(values):
|
|
values = _GetFloats(values)
|
|
return max(values)
|
|
|
|
def _GetLowestValue(self, key):
|
|
values = []
|
|
for run_list in self._runs:
|
|
for run in run_list:
|
|
if key in run:
|
|
values.append(run[key])
|
|
values = _StripNone(values)
|
|
if _AllFloat(values):
|
|
values = _GetFloats(values)
|
|
return min(values)
|
|
|
|
def _SortKeys(self, keys):
|
|
if self._sort == self.SORT_BY_KEYS:
|
|
return sorted(keys)
|
|
elif self._sort == self.SORT_BY_VALUES:
|
|
# pylint: disable=unnecessary-lambda
|
|
return sorted(keys, key=lambda x: self._GetLowestValue(x))
|
|
elif self._sort == self.SORT_BY_VALUES_DESC:
|
|
# pylint: disable=unnecessary-lambda
|
|
return sorted(keys, key=lambda x: self._GetHighestValue(x), reverse=True)
|
|
elif self._sort == self.NO_SORT:
|
|
return keys
|
|
else:
|
|
assert 0, 'Unimplemented sort %s' % self._sort
|
|
|
|
def _GetKeys(self):
|
|
keys = self._AggregateKeys()
|
|
return self._SortKeys(keys)
|
|
|
|
def GetTable(self, number_of_rows=sys.maxsize):
|
|
"""Returns a table from a list of list of dicts.
|
|
|
|
Examples:
|
|
We have the following runs:
|
|
[[{"k1": "v1", "k2": "v2"}, {"k1": "v3"}],
|
|
[{"k1": "v4", "k4": "v5"}]]
|
|
and the following labels:
|
|
["vanilla", "modified"]
|
|
it will return:
|
|
[["Key", "vanilla", "modified"]
|
|
["k1", ["v1", "v3"], ["v4"]]
|
|
["k2", ["v2"], []]
|
|
["k4", [], ["v5"]]]
|
|
The returned table can then be processed further by other classes in this
|
|
module.
|
|
|
|
The list of list of dicts is passed into the constructor of TableGenerator.
|
|
This method converts that into a canonical list of lists which represents a
|
|
table of values.
|
|
|
|
Args:
|
|
number_of_rows: Maximum number of rows to return from the table.
|
|
|
|
Returns:
|
|
A list of lists which is the table.
|
|
"""
|
|
keys = self._GetKeys()
|
|
header = [self._key_name] + self._labels
|
|
table = [header]
|
|
rows = 0
|
|
for k in keys:
|
|
row = [k]
|
|
unit = None
|
|
for run_list in self._runs:
|
|
v = []
|
|
for run in run_list:
|
|
if k in run:
|
|
if isinstance(run[k], list):
|
|
val = run[k][0]
|
|
unit = run[k][1]
|
|
else:
|
|
val = run[k]
|
|
v.append(val)
|
|
else:
|
|
v.append(None)
|
|
row.append(v)
|
|
# If we got a 'unit' value, append the units name to the key name.
|
|
if unit:
|
|
keyname = row[0] + ' (%s) ' % unit
|
|
row[0] = keyname
|
|
table.append(row)
|
|
rows += 1
|
|
if rows == number_of_rows:
|
|
break
|
|
return table
|
|
|
|
|
|
class SamplesTableGenerator(TableGenerator):
|
|
"""Creates a table with only samples from the results
|
|
|
|
The main public function is called GetTable().
|
|
|
|
Different than TableGenerator, self._runs is now a dict of {benchmark: runs}
|
|
We are expecting there is 'samples' in `runs`.
|
|
"""
|
|
|
|
def __init__(self, run_keyvals, label_list, iter_counts, weights):
|
|
TableGenerator.__init__(
|
|
self, run_keyvals, label_list, key_name='Benchmarks')
|
|
self._iter_counts = iter_counts
|
|
self._weights = weights
|
|
|
|
def _GetKeys(self):
|
|
keys = self._runs.keys()
|
|
return self._SortKeys(keys)
|
|
|
|
def GetTable(self, number_of_rows=sys.maxsize):
|
|
"""Returns a tuple, which contains three args:
|
|
|
|
1) a table from a list of list of dicts.
|
|
2) updated benchmark_results run_keyvals with composite benchmark
|
|
3) updated benchmark_results iter_count with composite benchmark
|
|
|
|
The dict of list of list of dicts is passed into the constructor of
|
|
SamplesTableGenerator.
|
|
This method converts that into a canonical list of lists which
|
|
represents a table of values.
|
|
|
|
Examples:
|
|
We have the following runs:
|
|
{bench1: [[{"samples": "v1"}, {"samples": "v2"}],
|
|
[{"samples": "v3"}, {"samples": "v4"}]]
|
|
bench2: [[{"samples": "v21"}, None],
|
|
[{"samples": "v22"}, {"samples": "v23"}]]}
|
|
and weights of benchmarks:
|
|
{bench1: w1, bench2: w2}
|
|
and the following labels:
|
|
["vanilla", "modified"]
|
|
it will return:
|
|
[["Benchmark", "Weights", "vanilla", "modified"]
|
|
["bench1", w1,
|
|
((2, 0), ["v1*w1", "v2*w1"]), ((2, 0), ["v3*w1", "v4*w1"])]
|
|
["bench2", w2,
|
|
((1, 1), ["v21*w2", None]), ((2, 0), ["v22*w2", "v23*w2"])]
|
|
["Composite Benchmark", N/A,
|
|
((1, 1), ["v1*w1+v21*w2", None]),
|
|
((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]]
|
|
The returned table can then be processed further by other classes in this
|
|
module.
|
|
|
|
Args:
|
|
number_of_rows: Maximum number of rows to return from the table.
|
|
|
|
Returns:
|
|
A list of lists which is the table.
|
|
"""
|
|
keys = self._GetKeys()
|
|
header = [self._key_name, 'Weights'] + self._labels
|
|
table = [header]
|
|
rows = 0
|
|
iterations = 0
|
|
|
|
for k in keys:
|
|
bench_runs = self._runs[k]
|
|
unit = None
|
|
all_runs_empty = all(not dict for label in bench_runs for dict in label)
|
|
if all_runs_empty:
|
|
cell = Cell()
|
|
cell.string_value = ('Benchmark %s contains no result.'
|
|
' Is the benchmark name valid?' % k)
|
|
table.append([cell])
|
|
else:
|
|
row = [k]
|
|
row.append(self._weights[k])
|
|
for run_list in bench_runs:
|
|
run_pass = 0
|
|
run_fail = 0
|
|
v = []
|
|
for run in run_list:
|
|
if 'samples' in run:
|
|
if isinstance(run['samples'], list):
|
|
val = run['samples'][0] * self._weights[k]
|
|
unit = run['samples'][1]
|
|
else:
|
|
val = run['samples'] * self._weights[k]
|
|
v.append(val)
|
|
run_pass += 1
|
|
else:
|
|
v.append(None)
|
|
run_fail += 1
|
|
one_tuple = ((run_pass, run_fail), v)
|
|
if iterations not in (0, run_pass + run_fail):
|
|
raise ValueError('Iterations of each benchmark run ' \
|
|
'are not the same')
|
|
iterations = run_pass + run_fail
|
|
row.append(one_tuple)
|
|
if unit:
|
|
keyname = row[0] + ' (%s) ' % unit
|
|
row[0] = keyname
|
|
table.append(row)
|
|
rows += 1
|
|
if rows == number_of_rows:
|
|
break
|
|
|
|
k = 'Composite Benchmark'
|
|
if k in keys:
|
|
raise RuntimeError('Composite benchmark already exists in results')
|
|
|
|
# Create a new composite benchmark row at the bottom of the summary table
|
|
# The new row will be like the format in example:
|
|
# ["Composite Benchmark", N/A,
|
|
# ((1, 1), ["v1*w1+v21*w2", None]),
|
|
# ((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]]
|
|
# First we will create a row of [key, weight, [[0] * iterations] * labels]
|
|
row = [None] * len(header)
|
|
row[0] = '%s (samples)' % k
|
|
row[1] = 'N/A'
|
|
for label_index in range(2, len(row)):
|
|
row[label_index] = [0] * iterations
|
|
|
|
for cur_row in table[1:]:
|
|
# Iterate through each benchmark
|
|
if len(cur_row) > 1:
|
|
for label_index in range(2, len(cur_row)):
|
|
# Iterate through each run in a single benchmark
|
|
# each result should look like ((pass, fail), [values_list])
|
|
bench_runs = cur_row[label_index][1]
|
|
for index in range(iterations):
|
|
# Accumulate each run result to composite benchmark run
|
|
# If any run fails, then we set this run for composite benchmark
|
|
# to None so that we know it fails.
|
|
if bench_runs[index] and row[label_index][index] is not None:
|
|
row[label_index][index] += bench_runs[index]
|
|
else:
|
|
row[label_index][index] = None
|
|
else:
|
|
# One benchmark totally fails, no valid data will be in final result
|
|
for label_index in range(2, len(row)):
|
|
row[label_index] = [None] * iterations
|
|
break
|
|
# Calculate pass and fail count for composite benchmark
|
|
for label_index in range(2, len(row)):
|
|
run_pass = 0
|
|
run_fail = 0
|
|
for run in row[label_index]:
|
|
if run:
|
|
run_pass += 1
|
|
else:
|
|
run_fail += 1
|
|
row[label_index] = ((run_pass, run_fail), row[label_index])
|
|
table.append(row)
|
|
|
|
# Now that we have the table genearted, we want to store this new composite
|
|
# benchmark into the benchmark_result in ResultReport object.
|
|
# This will be used to generate a full table which contains our composite
|
|
# benchmark.
|
|
# We need to create composite benchmark result and add it to keyvals in
|
|
# benchmark_results.
|
|
v = []
|
|
for label in row[2:]:
|
|
# each label's result looks like ((pass, fail), [values])
|
|
benchmark_runs = label[1]
|
|
# List of values of each label
|
|
single_run_list = []
|
|
for run in benchmark_runs:
|
|
# Result of each run under the same label is a dict of keys.
|
|
# Here the only key we will add for composite benchmark is the
|
|
# weighted_samples we added up.
|
|
one_dict = {}
|
|
if run:
|
|
one_dict[u'weighted_samples'] = [run, u'samples']
|
|
one_dict['retval'] = 0
|
|
else:
|
|
one_dict['retval'] = 1
|
|
single_run_list.append(one_dict)
|
|
v.append(single_run_list)
|
|
|
|
self._runs[k] = v
|
|
self._iter_counts[k] = iterations
|
|
|
|
return (table, self._runs, self._iter_counts)
|
|
|
|
|
|
class Result(object):
|
|
"""A class that respresents a single result.
|
|
|
|
This single result is obtained by condensing the information from a list of
|
|
runs and a list of baseline runs.
|
|
"""
|
|
|
|
def __init__(self):
|
|
pass
|
|
|
|
def _AllStringsSame(self, values):
|
|
values_set = set(values)
|
|
return len(values_set) == 1
|
|
|
|
def NeedsBaseline(self):
|
|
return False
|
|
|
|
# pylint: disable=unused-argument
|
|
def _Literal(self, cell, values, baseline_values):
|
|
cell.value = ' '.join([str(v) for v in values])
|
|
|
|
def _ComputeFloat(self, cell, values, baseline_values):
|
|
self._Literal(cell, values, baseline_values)
|
|
|
|
def _ComputeString(self, cell, values, baseline_values):
|
|
self._Literal(cell, values, baseline_values)
|
|
|
|
def _InvertIfLowerIsBetter(self, cell):
|
|
pass
|
|
|
|
def _GetGmean(self, values):
|
|
if not values:
|
|
return float('nan')
|
|
if any([v < 0 for v in values]):
|
|
return float('nan')
|
|
if any([v == 0 for v in values]):
|
|
return 0.0
|
|
log_list = [math.log(v) for v in values]
|
|
gmean_log = sum(log_list) / len(log_list)
|
|
return math.exp(gmean_log)
|
|
|
|
def Compute(self, cell, values, baseline_values):
|
|
"""Compute the result given a list of values and baseline values.
|
|
|
|
Args:
|
|
cell: A cell data structure to populate.
|
|
values: List of values.
|
|
baseline_values: List of baseline values. Can be none if this is the
|
|
baseline itself.
|
|
"""
|
|
all_floats = True
|
|
values = _StripNone(values)
|
|
if not values:
|
|
cell.value = ''
|
|
return
|
|
if _AllFloat(values):
|
|
float_values = _GetFloats(values)
|
|
else:
|
|
all_floats = False
|
|
if baseline_values:
|
|
baseline_values = _StripNone(baseline_values)
|
|
if baseline_values:
|
|
if _AllFloat(baseline_values):
|
|
float_baseline_values = _GetFloats(baseline_values)
|
|
else:
|
|
all_floats = False
|
|
else:
|
|
if self.NeedsBaseline():
|
|
cell.value = ''
|
|
return
|
|
float_baseline_values = None
|
|
if all_floats:
|
|
self._ComputeFloat(cell, float_values, float_baseline_values)
|
|
self._InvertIfLowerIsBetter(cell)
|
|
else:
|
|
self._ComputeString(cell, values, baseline_values)
|
|
|
|
|
|
class LiteralResult(Result):
|
|
"""A literal result."""
|
|
|
|
def __init__(self, iteration=0):
|
|
super(LiteralResult, self).__init__()
|
|
self.iteration = iteration
|
|
|
|
def Compute(self, cell, values, baseline_values):
|
|
try:
|
|
cell.value = values[self.iteration]
|
|
except IndexError:
|
|
cell.value = '-'
|
|
|
|
|
|
class NonEmptyCountResult(Result):
|
|
"""A class that counts the number of non-empty results.
|
|
|
|
The number of non-empty values will be stored in the cell.
|
|
"""
|
|
|
|
def Compute(self, cell, values, baseline_values):
|
|
"""Put the number of non-empty values in the cell result.
|
|
|
|
Args:
|
|
cell: Put the result in cell.value.
|
|
values: A list of values for the row.
|
|
baseline_values: A list of baseline values for the row.
|
|
"""
|
|
cell.value = len(_StripNone(values))
|
|
if not baseline_values:
|
|
return
|
|
base_value = len(_StripNone(baseline_values))
|
|
if cell.value == base_value:
|
|
return
|
|
f = ColorBoxFormat()
|
|
len_values = len(values)
|
|
len_baseline_values = len(baseline_values)
|
|
tmp_cell = Cell()
|
|
tmp_cell.value = 1.0 + (
|
|
float(cell.value - base_value) / (max(len_values, len_baseline_values)))
|
|
f.Compute(tmp_cell)
|
|
cell.bgcolor = tmp_cell.bgcolor
|
|
|
|
|
|
class StringMeanResult(Result):
|
|
"""Mean of string values."""
|
|
|
|
def _ComputeString(self, cell, values, baseline_values):
|
|
if self._AllStringsSame(values):
|
|
cell.value = str(values[0])
|
|
else:
|
|
cell.value = '?'
|
|
|
|
|
|
class AmeanResult(StringMeanResult):
|
|
"""Arithmetic mean."""
|
|
|
|
def __init__(self, ignore_min_max=False):
|
|
super(AmeanResult, self).__init__()
|
|
self.ignore_min_max = ignore_min_max
|
|
|
|
def _ComputeFloat(self, cell, values, baseline_values):
|
|
if self.ignore_min_max:
|
|
values = _RemoveMinMax(cell, values)
|
|
cell.value = statistics.mean(values)
|
|
|
|
|
|
class RawResult(Result):
|
|
"""Raw result."""
|
|
|
|
|
|
class IterationResult(Result):
|
|
"""Iteration result."""
|
|
|
|
|
|
class MinResult(Result):
|
|
"""Minimum."""
|
|
|
|
def _ComputeFloat(self, cell, values, baseline_values):
|
|
cell.value = min(values)
|
|
|
|
def _ComputeString(self, cell, values, baseline_values):
|
|
if values:
|
|
cell.value = min(values)
|
|
else:
|
|
cell.value = ''
|
|
|
|
|
|
class MaxResult(Result):
|
|
"""Maximum."""
|
|
|
|
def _ComputeFloat(self, cell, values, baseline_values):
|
|
cell.value = max(values)
|
|
|
|
def _ComputeString(self, cell, values, baseline_values):
|
|
if values:
|
|
cell.value = max(values)
|
|
else:
|
|
cell.value = ''
|
|
|
|
|
|
class NumericalResult(Result):
|
|
"""Numerical result."""
|
|
|
|
def _ComputeString(self, cell, values, baseline_values):
|
|
cell.value = '?'
|
|
|
|
|
|
class StdResult(NumericalResult):
|
|
"""Standard deviation."""
|
|
|
|
def __init__(self, ignore_min_max=False):
|
|
super(StdResult, self).__init__()
|
|
self.ignore_min_max = ignore_min_max
|
|
|
|
def _ComputeFloat(self, cell, values, baseline_values):
|
|
if self.ignore_min_max:
|
|
values = _RemoveMinMax(cell, values)
|
|
cell.value = statistics.pstdev(values)
|
|
|
|
|
|
class CoeffVarResult(NumericalResult):
|
|
"""Standard deviation / Mean"""
|
|
|
|
def __init__(self, ignore_min_max=False):
|
|
super(CoeffVarResult, self).__init__()
|
|
self.ignore_min_max = ignore_min_max
|
|
|
|
def _ComputeFloat(self, cell, values, baseline_values):
|
|
if self.ignore_min_max:
|
|
values = _RemoveMinMax(cell, values)
|
|
if statistics.mean(values) != 0.0:
|
|
noise = abs(statistics.pstdev(values) / statistics.mean(values))
|
|
else:
|
|
noise = 0.0
|
|
cell.value = noise
|
|
|
|
|
|
class ComparisonResult(Result):
|
|
"""Same or Different."""
|
|
|
|
def NeedsBaseline(self):
|
|
return True
|
|
|
|
def _ComputeString(self, cell, values, baseline_values):
|
|
value = None
|
|
baseline_value = None
|
|
if self._AllStringsSame(values):
|
|
value = values[0]
|
|
if self._AllStringsSame(baseline_values):
|
|
baseline_value = baseline_values[0]
|
|
if value is not None and baseline_value is not None:
|
|
if value == baseline_value:
|
|
cell.value = 'SAME'
|
|
else:
|
|
cell.value = 'DIFFERENT'
|
|
else:
|
|
cell.value = '?'
|
|
|
|
|
|
class PValueResult(ComparisonResult):
|
|
"""P-value."""
|
|
|
|
def __init__(self, ignore_min_max=False):
|
|
super(PValueResult, self).__init__()
|
|
self.ignore_min_max = ignore_min_max
|
|
|
|
def _ComputeFloat(self, cell, values, baseline_values):
|
|
if self.ignore_min_max:
|
|
values = _RemoveMinMax(cell, values)
|
|
baseline_values = _RemoveMinMax(cell, baseline_values)
|
|
if len(values) < 2 or len(baseline_values) < 2:
|
|
cell.value = float('nan')
|
|
return
|
|
_, cell.value = scipy.stats.ttest_ind(values, baseline_values)
|
|
|
|
def _ComputeString(self, cell, values, baseline_values):
|
|
return float('nan')
|
|
|
|
|
|
class KeyAwareComparisonResult(ComparisonResult):
|
|
"""Automatic key aware comparison."""
|
|
|
|
def _IsLowerBetter(self, key):
|
|
# Units in histograms should include directions
|
|
if 'smallerIsBetter' in key:
|
|
return True
|
|
if 'biggerIsBetter' in key:
|
|
return False
|
|
|
|
# For units in chartjson:
|
|
# TODO(llozano): Trying to guess direction by looking at the name of the
|
|
# test does not seem like a good idea. Test frameworks should provide this
|
|
# info explicitly. I believe Telemetry has this info. Need to find it out.
|
|
#
|
|
# Below are some test names for which we are not sure what the
|
|
# direction is.
|
|
#
|
|
# For these we dont know what the direction is. But, since we dont
|
|
# specify anything, crosperf will assume higher is better:
|
|
# --percent_impl_scrolled--percent_impl_scrolled--percent
|
|
# --solid_color_tiles_analyzed--solid_color_tiles_analyzed--count
|
|
# --total_image_cache_hit_count--total_image_cache_hit_count--count
|
|
# --total_texture_upload_time_by_url
|
|
#
|
|
# About these we are doubtful but we made a guess:
|
|
# --average_num_missing_tiles_by_url--*--units (low is good)
|
|
# --experimental_mean_frame_time_by_url--*--units (low is good)
|
|
# --experimental_median_frame_time_by_url--*--units (low is good)
|
|
# --texture_upload_count--texture_upload_count--count (high is good)
|
|
# --total_deferred_image_decode_count--count (low is good)
|
|
# --total_tiles_analyzed--total_tiles_analyzed--count (high is good)
|
|
lower_is_better_keys = [
|
|
'milliseconds', 'ms_', 'seconds_', 'KB', 'rdbytes', 'wrbytes',
|
|
'dropped_percent', '(ms)', '(seconds)', '--ms',
|
|
'--average_num_missing_tiles', '--experimental_jank',
|
|
'--experimental_mean_frame', '--experimental_median_frame_time',
|
|
'--total_deferred_image_decode_count', '--seconds', 'samples', 'bytes'
|
|
]
|
|
|
|
return any([l in key for l in lower_is_better_keys])
|
|
|
|
def _InvertIfLowerIsBetter(self, cell):
|
|
if self._IsLowerBetter(cell.name):
|
|
if cell.value:
|
|
cell.value = 1.0 / cell.value
|
|
|
|
|
|
class AmeanRatioResult(KeyAwareComparisonResult):
|
|
"""Ratio of arithmetic means of values vs. baseline values."""
|
|
|
|
def __init__(self, ignore_min_max=False):
|
|
super(AmeanRatioResult, self).__init__()
|
|
self.ignore_min_max = ignore_min_max
|
|
|
|
def _ComputeFloat(self, cell, values, baseline_values):
|
|
if self.ignore_min_max:
|
|
values = _RemoveMinMax(cell, values)
|
|
baseline_values = _RemoveMinMax(cell, baseline_values)
|
|
|
|
baseline_mean = statistics.mean(baseline_values)
|
|
values_mean = statistics.mean(values)
|
|
if baseline_mean != 0:
|
|
cell.value = values_mean / baseline_mean
|
|
elif values_mean != 0:
|
|
cell.value = 0.00
|
|
# cell.value = 0 means the values and baseline_values have big difference
|
|
else:
|
|
cell.value = 1.00
|
|
# no difference if both values and baseline_values are 0
|
|
|
|
|
|
class GmeanRatioResult(KeyAwareComparisonResult):
|
|
"""Ratio of geometric means of values vs. baseline values."""
|
|
|
|
def __init__(self, ignore_min_max=False):
|
|
super(GmeanRatioResult, self).__init__()
|
|
self.ignore_min_max = ignore_min_max
|
|
|
|
def _ComputeFloat(self, cell, values, baseline_values):
|
|
if self.ignore_min_max:
|
|
values = _RemoveMinMax(cell, values)
|
|
baseline_values = _RemoveMinMax(cell, baseline_values)
|
|
if self._GetGmean(baseline_values) != 0:
|
|
cell.value = self._GetGmean(values) / self._GetGmean(baseline_values)
|
|
elif self._GetGmean(values) != 0:
|
|
cell.value = 0.00
|
|
else:
|
|
cell.value = 1.00
|
|
|
|
|
|
class Color(object):
|
|
"""Class that represents color in RGBA format."""
|
|
|
|
def __init__(self, r=0, g=0, b=0, a=0):
|
|
self.r = r
|
|
self.g = g
|
|
self.b = b
|
|
self.a = a
|
|
|
|
def __str__(self):
|
|
return 'r: %s g: %s: b: %s: a: %s' % (self.r, self.g, self.b, self.a)
|
|
|
|
def Round(self):
|
|
"""Round RGBA values to the nearest integer."""
|
|
self.r = int(self.r)
|
|
self.g = int(self.g)
|
|
self.b = int(self.b)
|
|
self.a = int(self.a)
|
|
|
|
def GetRGB(self):
|
|
"""Get a hex representation of the color."""
|
|
return '%02x%02x%02x' % (self.r, self.g, self.b)
|
|
|
|
@classmethod
|
|
def Lerp(cls, ratio, a, b):
|
|
"""Perform linear interpolation between two colors.
|
|
|
|
Args:
|
|
ratio: The ratio to use for linear polation.
|
|
a: The first color object (used when ratio is 0).
|
|
b: The second color object (used when ratio is 1).
|
|
|
|
Returns:
|
|
Linearly interpolated color.
|
|
"""
|
|
ret = cls()
|
|
ret.r = (b.r - a.r) * ratio + a.r
|
|
ret.g = (b.g - a.g) * ratio + a.g
|
|
ret.b = (b.b - a.b) * ratio + a.b
|
|
ret.a = (b.a - a.a) * ratio + a.a
|
|
return ret
|
|
|
|
|
|
class Format(object):
|
|
"""A class that represents the format of a column."""
|
|
|
|
def __init__(self):
|
|
pass
|
|
|
|
def Compute(self, cell):
|
|
"""Computes the attributes of a cell based on its value.
|
|
|
|
Attributes typically are color, width, etc.
|
|
|
|
Args:
|
|
cell: The cell whose attributes are to be populated.
|
|
"""
|
|
if cell.value is None:
|
|
cell.string_value = ''
|
|
if isinstance(cell.value, float):
|
|
self._ComputeFloat(cell)
|
|
else:
|
|
self._ComputeString(cell)
|
|
|
|
def _ComputeFloat(self, cell):
|
|
cell.string_value = '{0:.2f}'.format(cell.value)
|
|
|
|
def _ComputeString(self, cell):
|
|
cell.string_value = str(cell.value)
|
|
|
|
def _GetColor(self, value, low, mid, high, power=6, mid_value=1.0):
|
|
min_value = 0.0
|
|
max_value = 2.0
|
|
if math.isnan(value):
|
|
return mid
|
|
if value > mid_value:
|
|
value = max_value - mid_value / value
|
|
|
|
return self._GetColorBetweenRange(value, min_value, mid_value, max_value,
|
|
low, mid, high, power)
|
|
|
|
def _GetColorBetweenRange(self, value, min_value, mid_value, max_value,
|
|
low_color, mid_color, high_color, power):
|
|
assert value <= max_value
|
|
assert value >= min_value
|
|
if value > mid_value:
|
|
value = (max_value - value) / (max_value - mid_value)
|
|
value **= power
|
|
ret = Color.Lerp(value, high_color, mid_color)
|
|
else:
|
|
value = (value - min_value) / (mid_value - min_value)
|
|
value **= power
|
|
ret = Color.Lerp(value, low_color, mid_color)
|
|
ret.Round()
|
|
return ret
|
|
|
|
|
|
class PValueFormat(Format):
|
|
"""Formatting for p-value."""
|
|
|
|
def _ComputeFloat(self, cell):
|
|
cell.string_value = '%0.2f' % float(cell.value)
|
|
if float(cell.value) < 0.05:
|
|
cell.bgcolor = self._GetColor(
|
|
cell.value,
|
|
Color(255, 255, 0, 0),
|
|
Color(255, 255, 255, 0),
|
|
Color(255, 255, 255, 0),
|
|
mid_value=0.05,
|
|
power=1)
|
|
|
|
|
|
class WeightFormat(Format):
|
|
"""Formatting for weight in cwp mode."""
|
|
|
|
def _ComputeFloat(self, cell):
|
|
cell.string_value = '%0.4f' % float(cell.value)
|
|
|
|
|
|
class StorageFormat(Format):
|
|
"""Format the cell as a storage number.
|
|
|
|
Examples:
|
|
If the cell contains a value of 1024, the string_value will be 1.0K.
|
|
"""
|
|
|
|
def _ComputeFloat(self, cell):
|
|
base = 1024
|
|
suffices = ['K', 'M', 'G']
|
|
v = float(cell.value)
|
|
current = 0
|
|
while v >= base**(current + 1) and current < len(suffices):
|
|
current += 1
|
|
|
|
if current:
|
|
divisor = base**current
|
|
cell.string_value = '%1.1f%s' % ((v / divisor), suffices[current - 1])
|
|
else:
|
|
cell.string_value = str(cell.value)
|
|
|
|
|
|
class CoeffVarFormat(Format):
|
|
"""Format the cell as a percent.
|
|
|
|
Examples:
|
|
If the cell contains a value of 1.5, the string_value will be +150%.
|
|
"""
|
|
|
|
def _ComputeFloat(self, cell):
|
|
cell.string_value = '%1.1f%%' % (float(cell.value) * 100)
|
|
cell.color = self._GetColor(
|
|
cell.value,
|
|
Color(0, 255, 0, 0),
|
|
Color(0, 0, 0, 0),
|
|
Color(255, 0, 0, 0),
|
|
mid_value=0.02,
|
|
power=1)
|
|
|
|
|
|
class PercentFormat(Format):
|
|
"""Format the cell as a percent.
|
|
|
|
Examples:
|
|
If the cell contains a value of 1.5, the string_value will be +50%.
|
|
"""
|
|
|
|
def _ComputeFloat(self, cell):
|
|
cell.string_value = '%+1.1f%%' % ((float(cell.value) - 1) * 100)
|
|
cell.color = self._GetColor(cell.value, Color(255, 0, 0, 0),
|
|
Color(0, 0, 0, 0), Color(0, 255, 0, 0))
|
|
|
|
|
|
class RatioFormat(Format):
|
|
"""Format the cell as a ratio.
|
|
|
|
Examples:
|
|
If the cell contains a value of 1.5642, the string_value will be 1.56.
|
|
"""
|
|
|
|
def _ComputeFloat(self, cell):
|
|
cell.string_value = '%+1.1f%%' % ((cell.value - 1) * 100)
|
|
cell.color = self._GetColor(cell.value, Color(255, 0, 0, 0),
|
|
Color(0, 0, 0, 0), Color(0, 255, 0, 0))
|
|
|
|
|
|
class ColorBoxFormat(Format):
|
|
"""Format the cell as a color box.
|
|
|
|
Examples:
|
|
If the cell contains a value of 1.5, it will get a green color.
|
|
If the cell contains a value of 0.5, it will get a red color.
|
|
The intensity of the green/red will be determined by how much above or below
|
|
1.0 the value is.
|
|
"""
|
|
|
|
def _ComputeFloat(self, cell):
|
|
cell.string_value = '--'
|
|
bgcolor = self._GetColor(cell.value, Color(255, 0, 0, 0),
|
|
Color(255, 255, 255, 0), Color(0, 255, 0, 0))
|
|
cell.bgcolor = bgcolor
|
|
cell.color = bgcolor
|
|
|
|
|
|
class Cell(object):
|
|
"""A class to represent a cell in a table.
|
|
|
|
Attributes:
|
|
value: The raw value of the cell.
|
|
color: The color of the cell.
|
|
bgcolor: The background color of the cell.
|
|
string_value: The string value of the cell.
|
|
suffix: A string suffix to be attached to the value when displaying.
|
|
prefix: A string prefix to be attached to the value when displaying.
|
|
color_row: Indicates whether the whole row is to inherit this cell's color.
|
|
bgcolor_row: Indicates whether the whole row is to inherit this cell's
|
|
bgcolor.
|
|
width: Optional specifier to make a column narrower than the usual width.
|
|
The usual width of a column is the max of all its cells widths.
|
|
colspan: Set the colspan of the cell in the HTML table, this is used for
|
|
table headers. Default value is 1.
|
|
name: the test name of the cell.
|
|
header: Whether this is a header in html.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.value = None
|
|
self.color = None
|
|
self.bgcolor = None
|
|
self.string_value = None
|
|
self.suffix = None
|
|
self.prefix = None
|
|
# Entire row inherits this color.
|
|
self.color_row = False
|
|
self.bgcolor_row = False
|
|
self.width = 0
|
|
self.colspan = 1
|
|
self.name = None
|
|
self.header = False
|
|
|
|
def __str__(self):
|
|
l = []
|
|
l.append('value: %s' % self.value)
|
|
l.append('string_value: %s' % self.string_value)
|
|
return ' '.join(l)
|
|
|
|
|
|
class Column(object):
|
|
"""Class representing a column in a table.
|
|
|
|
Attributes:
|
|
result: an object of the Result class.
|
|
fmt: an object of the Format class.
|
|
"""
|
|
|
|
def __init__(self, result, fmt, name=''):
|
|
self.result = result
|
|
self.fmt = fmt
|
|
self.name = name
|
|
|
|
|
|
# Takes in:
|
|
# ["Key", "Label1", "Label2"]
|
|
# ["k", ["v", "v2"], [v3]]
|
|
# etc.
|
|
# Also takes in a format string.
|
|
# Returns a table like:
|
|
# ["Key", "Label1", "Label2"]
|
|
# ["k", avg("v", "v2"), stddev("v", "v2"), etc.]]
|
|
# according to format string
|
|
class TableFormatter(object):
|
|
"""Class to convert a plain table into a cell-table.
|
|
|
|
This class takes in a table generated by TableGenerator and a list of column
|
|
formats to apply to the table and returns a table of cells.
|
|
"""
|
|
|
|
def __init__(self, table, columns, samples_table=False):
|
|
"""The constructor takes in a table and a list of columns.
|
|
|
|
Args:
|
|
table: A list of lists of values.
|
|
columns: A list of column containing what to produce and how to format
|
|
it.
|
|
samples_table: A flag to check whether we are generating a table of
|
|
samples in CWP apporximation mode.
|
|
"""
|
|
self._table = table
|
|
self._columns = columns
|
|
self._samples_table = samples_table
|
|
self._table_columns = []
|
|
self._out_table = []
|
|
|
|
def GenerateCellTable(self, table_type):
|
|
row_index = 0
|
|
all_failed = False
|
|
|
|
for row in self._table[1:]:
|
|
# If we are generating samples_table, the second value will be weight
|
|
# rather than values.
|
|
start_col = 2 if self._samples_table else 1
|
|
# It does not make sense to put retval in the summary table.
|
|
if str(row[0]) == 'retval' and table_type == 'summary':
|
|
# Check to see if any runs passed, and update all_failed.
|
|
all_failed = True
|
|
for values in row[start_col:]:
|
|
if 0 in values:
|
|
all_failed = False
|
|
continue
|
|
key = Cell()
|
|
key.string_value = str(row[0])
|
|
out_row = [key]
|
|
if self._samples_table:
|
|
# Add one column for weight if in samples_table mode
|
|
weight = Cell()
|
|
weight.value = row[1]
|
|
f = WeightFormat()
|
|
f.Compute(weight)
|
|
out_row.append(weight)
|
|
baseline = None
|
|
for results in row[start_col:]:
|
|
column_start = 0
|
|
values = None
|
|
# If generating sample table, we will split a tuple of iterations info
|
|
# from the results
|
|
if isinstance(results, tuple):
|
|
it, values = results
|
|
column_start = 1
|
|
cell = Cell()
|
|
cell.string_value = '[%d: %d]' % (it[0], it[1])
|
|
out_row.append(cell)
|
|
if not row_index:
|
|
self._table_columns.append(self._columns[0])
|
|
else:
|
|
values = results
|
|
# Parse each column
|
|
for column in self._columns[column_start:]:
|
|
cell = Cell()
|
|
cell.name = key.string_value
|
|
if not column.result.NeedsBaseline() or baseline is not None:
|
|
column.result.Compute(cell, values, baseline)
|
|
column.fmt.Compute(cell)
|
|
out_row.append(cell)
|
|
if not row_index:
|
|
self._table_columns.append(column)
|
|
|
|
if baseline is None:
|
|
baseline = values
|
|
self._out_table.append(out_row)
|
|
row_index += 1
|
|
|
|
# If this is a summary table, and the only row in it is 'retval', and
|
|
# all the test runs failed, we need to a 'Results' row to the output
|
|
# table.
|
|
if table_type == 'summary' and all_failed and len(self._table) == 2:
|
|
labels_row = self._table[0]
|
|
key = Cell()
|
|
key.string_value = 'Results'
|
|
out_row = [key]
|
|
baseline = None
|
|
for _ in labels_row[1:]:
|
|
for column in self._columns:
|
|
cell = Cell()
|
|
cell.name = key.string_value
|
|
column.result.Compute(cell, ['Fail'], baseline)
|
|
column.fmt.Compute(cell)
|
|
out_row.append(cell)
|
|
if not row_index:
|
|
self._table_columns.append(column)
|
|
self._out_table.append(out_row)
|
|
|
|
def AddColumnName(self):
|
|
"""Generate Column name at the top of table."""
|
|
key = Cell()
|
|
key.header = True
|
|
key.string_value = 'Keys' if not self._samples_table else 'Benchmarks'
|
|
header = [key]
|
|
if self._samples_table:
|
|
weight = Cell()
|
|
weight.header = True
|
|
weight.string_value = 'Weights'
|
|
header.append(weight)
|
|
for column in self._table_columns:
|
|
cell = Cell()
|
|
cell.header = True
|
|
if column.name:
|
|
cell.string_value = column.name
|
|
else:
|
|
result_name = column.result.__class__.__name__
|
|
format_name = column.fmt.__class__.__name__
|
|
|
|
cell.string_value = '%s %s' % (
|
|
result_name.replace('Result', ''),
|
|
format_name.replace('Format', ''),
|
|
)
|
|
|
|
header.append(cell)
|
|
|
|
self._out_table = [header] + self._out_table
|
|
|
|
def AddHeader(self, s):
|
|
"""Put additional string on the top of the table."""
|
|
cell = Cell()
|
|
cell.header = True
|
|
cell.string_value = str(s)
|
|
header = [cell]
|
|
colspan = max(1, max(len(row) for row in self._table))
|
|
cell.colspan = colspan
|
|
self._out_table = [header] + self._out_table
|
|
|
|
def GetPassesAndFails(self, values):
|
|
passes = 0
|
|
fails = 0
|
|
for val in values:
|
|
if val == 0:
|
|
passes = passes + 1
|
|
else:
|
|
fails = fails + 1
|
|
return passes, fails
|
|
|
|
def AddLabelName(self):
|
|
"""Put label on the top of the table."""
|
|
top_header = []
|
|
base_colspan = len(
|
|
[c for c in self._columns if not c.result.NeedsBaseline()])
|
|
compare_colspan = len(self._columns)
|
|
# Find the row with the key 'retval', if it exists. This
|
|
# will be used to calculate the number of iterations that passed and
|
|
# failed for each image label.
|
|
retval_row = None
|
|
for row in self._table:
|
|
if row[0] == 'retval':
|
|
retval_row = row
|
|
# The label is organized as follows
|
|
# "keys" label_base, label_comparison1, label_comparison2
|
|
# The first cell has colspan 1, the second is base_colspan
|
|
# The others are compare_colspan
|
|
column_position = 0
|
|
for label in self._table[0]:
|
|
cell = Cell()
|
|
cell.header = True
|
|
# Put the number of pass/fail iterations in the image label header.
|
|
if column_position > 0 and retval_row:
|
|
retval_values = retval_row[column_position]
|
|
if isinstance(retval_values, list):
|
|
passes, fails = self.GetPassesAndFails(retval_values)
|
|
cell.string_value = str(label) + ' (pass:%d fail:%d)' % (passes,
|
|
fails)
|
|
else:
|
|
cell.string_value = str(label)
|
|
else:
|
|
cell.string_value = str(label)
|
|
if top_header:
|
|
if not self._samples_table or (self._samples_table and
|
|
len(top_header) == 2):
|
|
cell.colspan = base_colspan
|
|
if len(top_header) > 1:
|
|
if not self._samples_table or (self._samples_table and
|
|
len(top_header) > 2):
|
|
cell.colspan = compare_colspan
|
|
top_header.append(cell)
|
|
column_position = column_position + 1
|
|
self._out_table = [top_header] + self._out_table
|
|
|
|
def _PrintOutTable(self):
|
|
o = ''
|
|
for row in self._out_table:
|
|
for cell in row:
|
|
o += str(cell) + ' '
|
|
o += '\n'
|
|
print(o)
|
|
|
|
def GetCellTable(self, table_type='full', headers=True):
|
|
"""Function to return a table of cells.
|
|
|
|
The table (list of lists) is converted into a table of cells by this
|
|
function.
|
|
|
|
Args:
|
|
table_type: Can be 'full' or 'summary'
|
|
headers: A boolean saying whether we want default headers
|
|
|
|
Returns:
|
|
A table of cells with each cell having the properties and string values as
|
|
requiested by the columns passed in the constructor.
|
|
"""
|
|
# Generate the cell table, creating a list of dynamic columns on the fly.
|
|
if not self._out_table:
|
|
self.GenerateCellTable(table_type)
|
|
if headers:
|
|
self.AddColumnName()
|
|
self.AddLabelName()
|
|
return self._out_table
|
|
|
|
|
|
class TablePrinter(object):
|
|
"""Class to print a cell table to the console, file or html."""
|
|
PLAIN = 0
|
|
CONSOLE = 1
|
|
HTML = 2
|
|
TSV = 3
|
|
EMAIL = 4
|
|
|
|
def __init__(self, table, output_type):
|
|
"""Constructor that stores the cell table and output type."""
|
|
self._table = table
|
|
self._output_type = output_type
|
|
self._row_styles = []
|
|
self._column_styles = []
|
|
|
|
# Compute whole-table properties like max-size, etc.
|
|
def _ComputeStyle(self):
|
|
self._row_styles = []
|
|
for row in self._table:
|
|
row_style = Cell()
|
|
for cell in row:
|
|
if cell.color_row:
|
|
assert cell.color, 'Cell color not set but color_row set!'
|
|
assert not row_style.color, 'Multiple row_style.colors found!'
|
|
row_style.color = cell.color
|
|
if cell.bgcolor_row:
|
|
assert cell.bgcolor, 'Cell bgcolor not set but bgcolor_row set!'
|
|
assert not row_style.bgcolor, 'Multiple row_style.bgcolors found!'
|
|
row_style.bgcolor = cell.bgcolor
|
|
self._row_styles.append(row_style)
|
|
|
|
self._column_styles = []
|
|
if len(self._table) < 2:
|
|
return
|
|
|
|
for i in range(max(len(row) for row in self._table)):
|
|
column_style = Cell()
|
|
for row in self._table:
|
|
if not any([cell.colspan != 1 for cell in row]):
|
|
column_style.width = max(column_style.width, len(row[i].string_value))
|
|
self._column_styles.append(column_style)
|
|
|
|
def _GetBGColorFix(self, color):
|
|
if self._output_type == self.CONSOLE:
|
|
prefix = misc.rgb2short(color.r, color.g, color.b)
|
|
# pylint: disable=anomalous-backslash-in-string
|
|
prefix = '\033[48;5;%sm' % prefix
|
|
suffix = '\033[0m'
|
|
elif self._output_type in [self.EMAIL, self.HTML]:
|
|
rgb = color.GetRGB()
|
|
prefix = ('<FONT style="BACKGROUND-COLOR:#{0}">'.format(rgb))
|
|
suffix = '</FONT>'
|
|
elif self._output_type in [self.PLAIN, self.TSV]:
|
|
prefix = ''
|
|
suffix = ''
|
|
return prefix, suffix
|
|
|
|
def _GetColorFix(self, color):
|
|
if self._output_type == self.CONSOLE:
|
|
prefix = misc.rgb2short(color.r, color.g, color.b)
|
|
# pylint: disable=anomalous-backslash-in-string
|
|
prefix = '\033[38;5;%sm' % prefix
|
|
suffix = '\033[0m'
|
|
elif self._output_type in [self.EMAIL, self.HTML]:
|
|
rgb = color.GetRGB()
|
|
prefix = '<FONT COLOR=#{0}>'.format(rgb)
|
|
suffix = '</FONT>'
|
|
elif self._output_type in [self.PLAIN, self.TSV]:
|
|
prefix = ''
|
|
suffix = ''
|
|
return prefix, suffix
|
|
|
|
def Print(self):
|
|
"""Print the table to a console, html, etc.
|
|
|
|
Returns:
|
|
A string that contains the desired representation of the table.
|
|
"""
|
|
self._ComputeStyle()
|
|
return self._GetStringValue()
|
|
|
|
def _GetCellValue(self, i, j):
|
|
cell = self._table[i][j]
|
|
out = cell.string_value
|
|
raw_width = len(out)
|
|
|
|
if cell.color:
|
|
p, s = self._GetColorFix(cell.color)
|
|
out = '%s%s%s' % (p, out, s)
|
|
|
|
if cell.bgcolor:
|
|
p, s = self._GetBGColorFix(cell.bgcolor)
|
|
out = '%s%s%s' % (p, out, s)
|
|
|
|
if self._output_type in [self.PLAIN, self.CONSOLE, self.EMAIL]:
|
|
if cell.width:
|
|
width = cell.width
|
|
else:
|
|
if self._column_styles:
|
|
width = self._column_styles[j].width
|
|
else:
|
|
width = len(cell.string_value)
|
|
if cell.colspan > 1:
|
|
width = 0
|
|
start = 0
|
|
for k in range(j):
|
|
start += self._table[i][k].colspan
|
|
for k in range(cell.colspan):
|
|
width += self._column_styles[start + k].width
|
|
if width > raw_width:
|
|
padding = ('%' + str(width - raw_width) + 's') % ''
|
|
out = padding + out
|
|
|
|
if self._output_type == self.HTML:
|
|
if cell.header:
|
|
tag = 'th'
|
|
else:
|
|
tag = 'td'
|
|
out = '<{0} colspan = "{2}"> {1} </{0}>'.format(tag, out, cell.colspan)
|
|
|
|
return out
|
|
|
|
def _GetHorizontalSeparator(self):
|
|
if self._output_type in [self.CONSOLE, self.PLAIN, self.EMAIL]:
|
|
return ' '
|
|
if self._output_type == self.HTML:
|
|
return ''
|
|
if self._output_type == self.TSV:
|
|
return '\t'
|
|
|
|
def _GetVerticalSeparator(self):
|
|
if self._output_type in [self.PLAIN, self.CONSOLE, self.TSV, self.EMAIL]:
|
|
return '\n'
|
|
if self._output_type == self.HTML:
|
|
return '</tr>\n<tr>'
|
|
|
|
def _GetPrefix(self):
|
|
if self._output_type in [self.PLAIN, self.CONSOLE, self.TSV, self.EMAIL]:
|
|
return ''
|
|
if self._output_type == self.HTML:
|
|
return '<p></p><table id="box-table-a">\n<tr>'
|
|
|
|
def _GetSuffix(self):
|
|
if self._output_type in [self.PLAIN, self.CONSOLE, self.TSV, self.EMAIL]:
|
|
return ''
|
|
if self._output_type == self.HTML:
|
|
return '</tr>\n</table>'
|
|
|
|
def _GetStringValue(self):
|
|
o = ''
|
|
o += self._GetPrefix()
|
|
for i in range(len(self._table)):
|
|
row = self._table[i]
|
|
# Apply row color and bgcolor.
|
|
p = s = bgp = bgs = ''
|
|
if self._row_styles[i].bgcolor:
|
|
bgp, bgs = self._GetBGColorFix(self._row_styles[i].bgcolor)
|
|
if self._row_styles[i].color:
|
|
p, s = self._GetColorFix(self._row_styles[i].color)
|
|
o += p + bgp
|
|
for j in range(len(row)):
|
|
out = self._GetCellValue(i, j)
|
|
o += out + self._GetHorizontalSeparator()
|
|
o += s + bgs
|
|
o += self._GetVerticalSeparator()
|
|
o += self._GetSuffix()
|
|
return o
|
|
|
|
|
|
# Some common drivers
|
|
def GetSimpleTable(table, out_to=TablePrinter.CONSOLE):
|
|
"""Prints a simple table.
|
|
|
|
This is used by code that has a very simple list-of-lists and wants to
|
|
produce a table with ameans, a percentage ratio of ameans and a colorbox.
|
|
|
|
Examples:
|
|
GetSimpleConsoleTable([["binary", "b1", "b2"],["size", "300", "400"]])
|
|
will produce a colored table that can be printed to the console.
|
|
|
|
Args:
|
|
table: a list of lists.
|
|
out_to: specify the fomat of output. Currently it supports HTML and CONSOLE.
|
|
|
|
Returns:
|
|
A string version of the table that can be printed to the console.
|
|
"""
|
|
columns = [
|
|
Column(AmeanResult(), Format()),
|
|
Column(AmeanRatioResult(), PercentFormat()),
|
|
Column(AmeanRatioResult(), ColorBoxFormat()),
|
|
]
|
|
our_table = [table[0]]
|
|
for row in table[1:]:
|
|
our_row = [row[0]]
|
|
for v in row[1:]:
|
|
our_row.append([v])
|
|
our_table.append(our_row)
|
|
|
|
tf = TableFormatter(our_table, columns)
|
|
cell_table = tf.GetCellTable()
|
|
tp = TablePrinter(cell_table, out_to)
|
|
return tp.Print()
|
|
|
|
|
|
# pylint: disable=redefined-outer-name
|
|
def GetComplexTable(runs, labels, out_to=TablePrinter.CONSOLE):
|
|
"""Prints a complex table.
|
|
|
|
This can be used to generate a table with arithmetic mean, standard deviation,
|
|
coefficient of variation, p-values, etc.
|
|
|
|
Args:
|
|
runs: A list of lists with data to tabulate.
|
|
labels: A list of labels that correspond to the runs.
|
|
out_to: specifies the format of the table (example CONSOLE or HTML).
|
|
|
|
Returns:
|
|
A string table that can be printed to the console or put in an HTML file.
|
|
"""
|
|
tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC)
|
|
table = tg.GetTable()
|
|
columns = [
|
|
Column(LiteralResult(), Format(), 'Literal'),
|
|
Column(AmeanResult(), Format()),
|
|
Column(StdResult(), Format()),
|
|
Column(CoeffVarResult(), CoeffVarFormat()),
|
|
Column(NonEmptyCountResult(), Format()),
|
|
Column(AmeanRatioResult(), PercentFormat()),
|
|
Column(AmeanRatioResult(), RatioFormat()),
|
|
Column(GmeanRatioResult(), RatioFormat()),
|
|
Column(PValueResult(), PValueFormat())
|
|
]
|
|
tf = TableFormatter(table, columns)
|
|
cell_table = tf.GetCellTable()
|
|
tp = TablePrinter(cell_table, out_to)
|
|
return tp.Print()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
# Run a few small tests here.
|
|
run1 = {
|
|
'k1': '10',
|
|
'k2': '12',
|
|
'k5': '40',
|
|
'k6': '40',
|
|
'ms_1': '20',
|
|
'k7': 'FAIL',
|
|
'k8': 'PASS',
|
|
'k9': 'PASS',
|
|
'k10': '0'
|
|
}
|
|
run2 = {
|
|
'k1': '13',
|
|
'k2': '14',
|
|
'k3': '15',
|
|
'ms_1': '10',
|
|
'k8': 'PASS',
|
|
'k9': 'FAIL',
|
|
'k10': '0'
|
|
}
|
|
run3 = {
|
|
'k1': '50',
|
|
'k2': '51',
|
|
'k3': '52',
|
|
'k4': '53',
|
|
'k5': '35',
|
|
'k6': '45',
|
|
'ms_1': '200',
|
|
'ms_2': '20',
|
|
'k7': 'FAIL',
|
|
'k8': 'PASS',
|
|
'k9': 'PASS'
|
|
}
|
|
runs = [[run1, run2], [run3]]
|
|
labels = ['vanilla', 'modified']
|
|
t = GetComplexTable(runs, labels, TablePrinter.CONSOLE)
|
|
print(t)
|
|
email = GetComplexTable(runs, labels, TablePrinter.EMAIL)
|
|
|
|
runs = [[{
|
|
'k1': '1'
|
|
}, {
|
|
'k1': '1.1'
|
|
}, {
|
|
'k1': '1.2'
|
|
}], [{
|
|
'k1': '5'
|
|
}, {
|
|
'k1': '5.1'
|
|
}, {
|
|
'k1': '5.2'
|
|
}]]
|
|
t = GetComplexTable(runs, labels, TablePrinter.CONSOLE)
|
|
print(t)
|
|
|
|
simple_table = [
|
|
['binary', 'b1', 'b2', 'b3'],
|
|
['size', 100, 105, 108],
|
|
['rodata', 100, 80, 70],
|
|
['data', 100, 100, 100],
|
|
['debug', 100, 140, 60],
|
|
]
|
|
t = GetSimpleTable(simple_table)
|
|
print(t)
|
|
email += GetSimpleTable(simple_table, TablePrinter.HTML)
|
|
email_to = [getpass.getuser()]
|
|
email = "<pre style='font-size: 13px'>%s</pre>" % email
|
|
EmailSender().SendEmail(email_to, 'SimpleTableTest', email, msg_type='html')
|