You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
461 lines
16 KiB
461 lines
16 KiB
# -*- coding: utf-8 -*-
|
|
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""Setting files for global, benchmark and labels."""
|
|
|
|
from __future__ import print_function
|
|
|
|
from field import BooleanField
|
|
from field import EnumField
|
|
from field import FloatField
|
|
from field import IntegerField
|
|
from field import ListField
|
|
from field import TextField
|
|
from settings import Settings
|
|
|
|
|
|
class BenchmarkSettings(Settings):
|
|
"""Settings used to configure individual benchmarks."""
|
|
|
|
def __init__(self, name):
|
|
super(BenchmarkSettings, self).__init__(name, 'benchmark')
|
|
self.AddField(
|
|
TextField(
|
|
'test_name',
|
|
description='The name of the test to run. '
|
|
'Defaults to the name of the benchmark.'))
|
|
self.AddField(
|
|
TextField(
|
|
'test_args', description='Arguments to be passed to the '
|
|
'test.'))
|
|
self.AddField(
|
|
IntegerField(
|
|
'iterations',
|
|
required=False,
|
|
default=0,
|
|
description='Number of iterations to run the test. '
|
|
'If not set, will run each benchmark test the optimum number of '
|
|
'times to get a stable result.'))
|
|
self.AddField(
|
|
TextField(
|
|
'suite',
|
|
default='test_that',
|
|
description='The type of the benchmark.'))
|
|
self.AddField(
|
|
IntegerField(
|
|
'retries',
|
|
default=0,
|
|
description='Number of times to retry a '
|
|
'benchmark run.'))
|
|
self.AddField(
|
|
BooleanField(
|
|
'run_local',
|
|
description='Run benchmark harness on the DUT. '
|
|
'Currently only compatible with the suite: '
|
|
'telemetry_Crosperf.',
|
|
required=False,
|
|
default=True))
|
|
self.AddField(
|
|
FloatField(
|
|
'weight',
|
|
default=0.0,
|
|
description='Weight of the benchmark for CWP approximation'))
|
|
|
|
|
|
class LabelSettings(Settings):
|
|
"""Settings for each label."""
|
|
|
|
def __init__(self, name):
|
|
super(LabelSettings, self).__init__(name, 'label')
|
|
self.AddField(
|
|
TextField(
|
|
'chromeos_image',
|
|
required=False,
|
|
description='The path to the image to run tests '
|
|
'on, for local/custom-built images. See the '
|
|
"'build' option for official or trybot images."))
|
|
self.AddField(
|
|
TextField(
|
|
'autotest_path',
|
|
required=False,
|
|
description='Autotest directory path relative to chroot which '
|
|
'has autotest files for the image to run tests requiring autotest '
|
|
'files.'))
|
|
self.AddField(
|
|
TextField(
|
|
'debug_path',
|
|
required=False,
|
|
description='Debug info directory relative to chroot which has '
|
|
'symbols and vmlinux that can be used by perf tool.'))
|
|
self.AddField(
|
|
TextField(
|
|
'chromeos_root',
|
|
description='The path to a chromeos checkout which '
|
|
'contains a src/scripts directory. Defaults to '
|
|
'the chromeos checkout which contains the '
|
|
'chromeos_image.'))
|
|
self.AddField(
|
|
ListField(
|
|
'remote',
|
|
description='A comma-separated list of IPs of chromeos'
|
|
'devices to run experiments on.'))
|
|
self.AddField(
|
|
TextField(
|
|
'image_args',
|
|
required=False,
|
|
default='',
|
|
description='Extra arguments to pass to '
|
|
'image_chromeos.py.'))
|
|
self.AddField(
|
|
TextField(
|
|
'cache_dir',
|
|
default='',
|
|
description='The cache dir for this image.'))
|
|
self.AddField(
|
|
TextField(
|
|
'compiler',
|
|
default='gcc',
|
|
description='The compiler used to build the '
|
|
'ChromeOS image (gcc or llvm).'))
|
|
self.AddField(
|
|
TextField(
|
|
'chrome_src',
|
|
description='The path to the source of chrome. '
|
|
'This is used to run telemetry benchmarks. '
|
|
'The default one is the src inside chroot.',
|
|
required=False,
|
|
default=''))
|
|
self.AddField(
|
|
TextField(
|
|
'build',
|
|
description='The xbuddy specification for an '
|
|
'official or trybot image to use for tests. '
|
|
"'/remote' is assumed, and the board is given "
|
|
"elsewhere, so omit the '/remote/<board>/' xbuddy "
|
|
'prefix.',
|
|
required=False,
|
|
default=''))
|
|
|
|
|
|
class GlobalSettings(Settings):
|
|
"""Settings that apply per-experiment."""
|
|
|
|
def __init__(self, name):
|
|
super(GlobalSettings, self).__init__(name, 'global')
|
|
self.AddField(
|
|
TextField(
|
|
'name',
|
|
description='The name of the experiment. Just an '
|
|
'identifier.'))
|
|
self.AddField(
|
|
TextField(
|
|
'board',
|
|
description='The target board for running '
|
|
'experiments on, e.g. x86-alex.'))
|
|
self.AddField(
|
|
BooleanField(
|
|
'skylab',
|
|
description='Whether to run experiments via skylab.',
|
|
default=False))
|
|
self.AddField(
|
|
ListField(
|
|
'remote',
|
|
description='A comma-separated list of IPs of '
|
|
'chromeos devices to run experiments on.'))
|
|
self.AddField(
|
|
BooleanField(
|
|
'rerun_if_failed',
|
|
description='Whether to re-run failed test runs '
|
|
'or not.',
|
|
default=False))
|
|
self.AddField(
|
|
BooleanField(
|
|
'rm_chroot_tmp',
|
|
default=False,
|
|
description='Whether to remove the test_that '
|
|
'result in the chroot.'))
|
|
self.AddField(
|
|
ListField(
|
|
'email',
|
|
description='Space-separated list of email '
|
|
'addresses to send email to.'))
|
|
self.AddField(
|
|
BooleanField(
|
|
'rerun',
|
|
description='Whether to ignore the cache and '
|
|
'for tests to be re-run.',
|
|
default=False))
|
|
self.AddField(
|
|
BooleanField(
|
|
'same_specs',
|
|
default=True,
|
|
description='Ensure cached runs are run on the '
|
|
'same kind of devices which are specified as a '
|
|
'remote.'))
|
|
self.AddField(
|
|
BooleanField(
|
|
'same_machine',
|
|
default=False,
|
|
description='Ensure cached runs are run on the '
|
|
'same remote.'))
|
|
self.AddField(
|
|
BooleanField(
|
|
'use_file_locks',
|
|
default=False,
|
|
description='DEPRECATED: Whether to use the file locks '
|
|
'or AFE server lock mechanism.'))
|
|
self.AddField(
|
|
IntegerField(
|
|
'iterations',
|
|
required=False,
|
|
default=0,
|
|
description='Number of iterations to run all tests. '
|
|
'If not set, will run each benchmark test the optimum number of '
|
|
'times to get a stable result.'))
|
|
self.AddField(
|
|
TextField(
|
|
'chromeos_root',
|
|
description='The path to a chromeos checkout which '
|
|
'contains a src/scripts directory. Defaults to '
|
|
'the chromeos checkout which contains the '
|
|
'chromeos_image.'))
|
|
self.AddField(
|
|
TextField(
|
|
'logging_level',
|
|
default='average',
|
|
description='The level of logging desired. '
|
|
"Options are 'quiet', 'average', and 'verbose'."))
|
|
self.AddField(
|
|
IntegerField(
|
|
'acquire_timeout',
|
|
default=0,
|
|
description='Number of seconds to wait for '
|
|
'machine before exit if all the machines in '
|
|
'the experiment file are busy. Default is 0.'))
|
|
self.AddField(
|
|
TextField(
|
|
'perf_args',
|
|
default='',
|
|
description='The optional profile command. It '
|
|
'enables perf commands to record perforamance '
|
|
'related counters. It must start with perf '
|
|
'command record or stat followed by arguments.'))
|
|
self.AddField(
|
|
BooleanField(
|
|
'download_debug',
|
|
default=True,
|
|
description='Download compressed debug symbols alongwith '
|
|
'image. This can provide more info matching symbols for'
|
|
'profiles, but takes larger space. By default, download'
|
|
'it only when perf_args is specified.'))
|
|
self.AddField(
|
|
TextField(
|
|
'cache_dir',
|
|
default='',
|
|
description='The abs path of cache dir. '
|
|
'Default is /home/$(whoami)/cros_scratch.'))
|
|
self.AddField(
|
|
BooleanField(
|
|
'cache_only',
|
|
default=False,
|
|
description='Whether to use only cached '
|
|
'results (do not rerun failed tests).'))
|
|
self.AddField(
|
|
BooleanField(
|
|
'no_email',
|
|
default=False,
|
|
description='Whether to disable the email to '
|
|
'user after crosperf finishes.'))
|
|
self.AddField(
|
|
BooleanField(
|
|
'json_report',
|
|
default=False,
|
|
description='Whether to generate a json version '
|
|
'of the report, for archiving.'))
|
|
self.AddField(
|
|
BooleanField(
|
|
'show_all_results',
|
|
default=False,
|
|
description='When running Telemetry tests, '
|
|
'whether to all the results, instead of just '
|
|
'the default (summary) results.'))
|
|
self.AddField(
|
|
TextField(
|
|
'share_cache',
|
|
default='',
|
|
description='Path to alternate cache whose data '
|
|
'you want to use. It accepts multiple directories '
|
|
'separated by a ",".'))
|
|
self.AddField(
|
|
TextField('results_dir', default='', description='The results dir.'))
|
|
self.AddField(
|
|
BooleanField(
|
|
'compress_results',
|
|
default=True,
|
|
description='Whether to compress all test results other than '
|
|
'reports into a tarball to save disk space.'))
|
|
self.AddField(
|
|
TextField(
|
|
'locks_dir',
|
|
default='',
|
|
description='An alternate directory to use for '
|
|
'storing/checking machine file locks for local machines. '
|
|
'By default the file locks directory is '
|
|
'/google/data/rw/users/mo/mobiletc-prebuild/locks.\n'
|
|
'WARNING: If you use your own locks directory, '
|
|
'there is no guarantee that someone else might not '
|
|
'hold a lock on the same machine in a different '
|
|
'locks directory.'))
|
|
self.AddField(
|
|
TextField(
|
|
'chrome_src',
|
|
description='The path to the source of chrome. '
|
|
'This is used to run telemetry benchmarks. '
|
|
'The default one is the src inside chroot.',
|
|
required=False,
|
|
default=''))
|
|
self.AddField(
|
|
IntegerField(
|
|
'retries',
|
|
default=0,
|
|
description='Number of times to retry a '
|
|
'benchmark run.'))
|
|
self.AddField(
|
|
TextField(
|
|
'cwp_dso',
|
|
description='The DSO type that we want to use for '
|
|
'CWP approximation. This is used to run telemetry '
|
|
'benchmarks. Valid DSO types can be found from dso_list '
|
|
'in experiment_factory.py. The default value is set to '
|
|
'be empty.',
|
|
required=False,
|
|
default=''))
|
|
self.AddField(
|
|
BooleanField(
|
|
'enable_aslr',
|
|
description='Enable ASLR on the machine to run the '
|
|
'benchmarks. ASLR is disabled by default',
|
|
required=False,
|
|
default=False))
|
|
self.AddField(
|
|
BooleanField(
|
|
'ignore_min_max',
|
|
description='When doing math for the raw results, '
|
|
'ignore min and max values to reduce noise.',
|
|
required=False,
|
|
default=False))
|
|
self.AddField(
|
|
TextField(
|
|
'intel_pstate',
|
|
description='Intel Pstate mode.\n'
|
|
'Supported modes: "active", "passive", "no_hwp".\n'
|
|
'Default is "no_hwp" which disables hardware pstates to avoid '
|
|
'noise in benchmarks.',
|
|
required=False,
|
|
default='no_hwp'))
|
|
self.AddField(
|
|
BooleanField(
|
|
'turbostat',
|
|
description='Run turbostat process in the background'
|
|
' of a benchmark. Enabled by default.',
|
|
required=False,
|
|
default=True))
|
|
self.AddField(
|
|
FloatField(
|
|
'top_interval',
|
|
description='Run top command in the background of a benchmark with'
|
|
' interval of sampling specified in seconds.\n'
|
|
'Recommended values 1-5. Lower number provides more accurate'
|
|
' data.\n'
|
|
'With 0 - do not run top.\n'
|
|
'NOTE: Running top with interval 1-5 sec has insignificant'
|
|
' performance impact (performance degradation does not exceed'
|
|
' 0.3%%, measured on x86_64, ARM32, and ARM64). '
|
|
'The default value is 1.',
|
|
required=False,
|
|
default=1))
|
|
self.AddField(
|
|
IntegerField(
|
|
'cooldown_temp',
|
|
required=False,
|
|
default=40,
|
|
description='Wait until CPU temperature goes down below'
|
|
' specified temperature in Celsius'
|
|
' prior starting a benchmark. '
|
|
'By default the value is set to 40 degrees.'))
|
|
self.AddField(
|
|
IntegerField(
|
|
'cooldown_time',
|
|
required=False,
|
|
default=10,
|
|
description='Wait specified time in minutes allowing'
|
|
' CPU to cool down. Zero value disables cooldown. '
|
|
'The default value is 10 minutes.'))
|
|
self.AddField(
|
|
EnumField(
|
|
'governor',
|
|
options=[
|
|
'performance',
|
|
'powersave',
|
|
'userspace',
|
|
'ondemand',
|
|
'conservative',
|
|
'schedutils',
|
|
'sched',
|
|
'interactive',
|
|
],
|
|
default='performance',
|
|
required=False,
|
|
description='Setup CPU governor for all cores.\n'
|
|
'For more details refer to:\n'
|
|
'https://www.kernel.org/doc/Documentation/cpu-freq/governors.txt. '
|
|
'Default is "performance" governor.'))
|
|
self.AddField(
|
|
EnumField(
|
|
'cpu_usage',
|
|
options=[
|
|
'all',
|
|
'big_only',
|
|
'little_only',
|
|
'exclusive_cores',
|
|
],
|
|
default='all',
|
|
required=False,
|
|
description='Restrict usage of CPUs to decrease CPU interference.\n'
|
|
'"all" - no restrictions;\n'
|
|
'"big-only", "little-only" - enable only big/little cores,'
|
|
' applicable only on ARM;\n'
|
|
'"exclusive-cores" - (for future use)'
|
|
' isolate cores for exclusive use of benchmark processes. '
|
|
'By default use all CPUs.'))
|
|
self.AddField(
|
|
IntegerField(
|
|
'cpu_freq_pct',
|
|
required=False,
|
|
default=95,
|
|
description='Setup CPU frequency to a supported value less than'
|
|
' or equal to a percent of max_freq. '
|
|
'CPU frequency is reduced to 95%% by default to reduce thermal '
|
|
'throttling.'))
|
|
|
|
|
|
class SettingsFactory(object):
|
|
"""Factory class for building different types of Settings objects.
|
|
|
|
This factory is currently hardcoded to produce settings for ChromeOS
|
|
experiment files. The idea is that in the future, other types
|
|
of settings could be produced.
|
|
"""
|
|
|
|
def GetSettings(self, name, settings_type):
|
|
if settings_type == 'label' or not settings_type:
|
|
return LabelSettings(name)
|
|
if settings_type == 'global':
|
|
return GlobalSettings(name)
|
|
if settings_type == 'benchmark':
|
|
return BenchmarkSettings(name)
|
|
|
|
raise TypeError("Invalid settings type: '%s'." % settings_type)
|