You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
88 lines
2.8 KiB
88 lines
2.8 KiB
4 months ago
|
# -*- coding: utf-8 -*-
|
||
|
|
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
|
||
|
|
# Use of this source code is governed by a BSD-style license that can be
|
||
|
|
# found in the LICENSE file.
|
||
|
|
|
||
|
|
"""Define a type that wraps a Benchmark instance."""
|
||
|
|
|
||
|
|
from __future__ import division
|
||
|
|
from __future__ import print_function
|
||
|
|
|
||
|
|
import math
|
||
|
|
# FIXME(denik): Fix the import in chroot.
|
||
|
|
# pylint: disable=import-error
|
||
|
|
from scipy import stats
|
||
|
|
|
||
|
|
# See crbug.com/673558 for how these are estimated.
|
||
|
|
_estimated_stddev = {
|
||
|
|
'octane': 0.015,
|
||
|
|
'kraken': 0.019,
|
||
|
|
'speedometer': 0.007,
|
||
|
|
'speedometer2': 0.006,
|
||
|
|
'dromaeo.domcoreattr': 0.023,
|
||
|
|
'dromaeo.domcoremodify': 0.011,
|
||
|
|
'graphics_WebGLAquarium': 0.008,
|
||
|
|
'page_cycler_v2.typical_25': 0.021,
|
||
|
|
'loading.desktop': 0.021, # Copied from page_cycler initially
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
# Get #samples needed to guarantee a given confidence interval, assuming the
|
||
|
|
# samples follow normal distribution.
|
||
|
|
def _samples(b):
|
||
|
|
# TODO: Make this an option
|
||
|
|
# CI = (0.9, 0.02), i.e., 90% chance that |sample mean - true mean| < 2%.
|
||
|
|
p = 0.9
|
||
|
|
e = 0.02
|
||
|
|
if b not in _estimated_stddev:
|
||
|
|
return 1
|
||
|
|
d = _estimated_stddev[b]
|
||
|
|
# Get at least 2 samples so as to calculate standard deviation, which is
|
||
|
|
# needed in T-test for p-value.
|
||
|
|
n = int(math.ceil((stats.norm.isf((1 - p) / 2) * d / e)**2))
|
||
|
|
return n if n > 1 else 2
|
||
|
|
|
||
|
|
|
||
|
|
class Benchmark(object):
|
||
|
|
"""Class representing a benchmark to be run.
|
||
|
|
|
||
|
|
Contains details of the benchmark suite, arguments to pass to the suite,
|
||
|
|
iterations to run the benchmark suite and so on. Note that the benchmark name
|
||
|
|
can be different to the test suite name. For example, you may want to have
|
||
|
|
two different benchmarks which run the same test_name with different
|
||
|
|
arguments.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self,
|
||
|
|
name,
|
||
|
|
test_name,
|
||
|
|
test_args,
|
||
|
|
iterations,
|
||
|
|
rm_chroot_tmp,
|
||
|
|
perf_args,
|
||
|
|
suite='',
|
||
|
|
show_all_results=False,
|
||
|
|
retries=0,
|
||
|
|
run_local=False,
|
||
|
|
cwp_dso='',
|
||
|
|
weight=0):
|
||
|
|
self.name = name
|
||
|
|
# For telemetry, this is the benchmark name.
|
||
|
|
self.test_name = test_name
|
||
|
|
# For telemetry, this is the data.
|
||
|
|
self.test_args = test_args
|
||
|
|
self.iterations = iterations if iterations > 0 else _samples(name)
|
||
|
|
self.perf_args = perf_args
|
||
|
|
self.rm_chroot_tmp = rm_chroot_tmp
|
||
|
|
self.iteration_adjusted = False
|
||
|
|
self.suite = suite
|
||
|
|
self.show_all_results = show_all_results
|
||
|
|
self.retries = retries
|
||
|
|
if self.suite == 'telemetry':
|
||
|
|
self.show_all_results = True
|
||
|
|
if run_local and self.suite != 'telemetry_Crosperf':
|
||
|
|
raise RuntimeError('run_local is only supported by telemetry_Crosperf.')
|
||
|
|
self.run_local = run_local
|
||
|
|
self.cwp_dso = cwp_dso
|
||
|
|
self.weight = weight
|