You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
490 lines
17 KiB
490 lines
17 KiB
#!/usr/bin/env python2
|
|
#
|
|
# Copyright 2017 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
#
|
|
# pylint: disable=cros-logging-import
|
|
|
|
# This is the script to run specified benchmark with different toolchain
|
|
# settings. It includes the process of building benchmark locally and running
|
|
# benchmark on DUT.
|
|
|
|
"""Main script to run the benchmark suite from building to testing."""
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import config
|
|
import ConfigParser
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
def _parse_arguments(argv):
|
|
parser = argparse.ArgumentParser(description='Build and run specific '
|
|
'benchamrk')
|
|
parser.add_argument(
|
|
'-b',
|
|
'--bench',
|
|
action='append',
|
|
default=[],
|
|
help='Select which benchmark to run')
|
|
|
|
# Only one of compiler directory and llvm prebuilts version can be indicated
|
|
# at the beginning, so set -c and -l into a exclusive group.
|
|
group = parser.add_mutually_exclusive_group()
|
|
|
|
# The toolchain setting arguments has action of 'append', so that users
|
|
# could compare performance with several toolchain settings together.
|
|
group.add_argument(
|
|
'-c',
|
|
'--compiler_dir',
|
|
metavar='DIR',
|
|
action='append',
|
|
default=[],
|
|
help='Specify path to the compiler\'s bin directory. '
|
|
'You shall give several paths, each with a -c, to '
|
|
'compare performance differences in '
|
|
'each compiler.')
|
|
|
|
parser.add_argument(
|
|
'-o',
|
|
'--build_os',
|
|
action='append',
|
|
default=[],
|
|
help='Specify the host OS to build the benchmark.')
|
|
|
|
group.add_argument(
|
|
'-l',
|
|
'--llvm_prebuilts_version',
|
|
action='append',
|
|
default=[],
|
|
help='Specify the version of prebuilt LLVM. When '
|
|
'specific prebuilt version of LLVM already '
|
|
'exists, no need to pass the path to compiler '
|
|
'directory.')
|
|
|
|
parser.add_argument(
|
|
'-f',
|
|
'--cflags',
|
|
action='append',
|
|
default=[],
|
|
help='Specify the cflags options for the toolchain. '
|
|
'Be sure to quote all the cflags with quotation '
|
|
'mark("") or use equal(=).')
|
|
parser.add_argument(
|
|
'--ldflags',
|
|
action='append',
|
|
default=[],
|
|
help='Specify linker flags for the toolchain.')
|
|
|
|
parser.add_argument(
|
|
'-i',
|
|
'--iterations',
|
|
type=int,
|
|
default=1,
|
|
help='Specify how many iterations does the test '
|
|
'take.')
|
|
|
|
# Arguments -s and -r are for connecting to DUT.
|
|
parser.add_argument(
|
|
'-s',
|
|
'--serials',
|
|
help='Comma separate list of device serials under '
|
|
'test.')
|
|
|
|
parser.add_argument(
|
|
'-r',
|
|
'--remote',
|
|
default='localhost',
|
|
help='hostname[:port] if the ADB device is connected '
|
|
'to a remote machine. Ensure this workstation '
|
|
'is configured for passwordless ssh access as '
|
|
'users "root" or "adb"')
|
|
|
|
# Arguments -frequency and -m are for device settings
|
|
parser.add_argument(
|
|
'--frequency',
|
|
type=int,
|
|
default=979200,
|
|
help='Specify the CPU frequency of the device. The '
|
|
'unit is KHZ. The available value is defined in'
|
|
'cpufreq/scaling_available_frequency file in '
|
|
'device\'s each core directory. '
|
|
'The default value is 979200, which shows a '
|
|
'balance in noise and performance. Lower '
|
|
'frequency will slow down the performance but '
|
|
'reduce noise.')
|
|
|
|
parser.add_argument(
|
|
'-m',
|
|
'--mode',
|
|
default='little',
|
|
help='User can specify whether \'little\' or \'big\' '
|
|
'mode to use. The default one is little mode. '
|
|
'The little mode runs on a single core of '
|
|
'Cortex-A53, while big mode runs on single core '
|
|
'of Cortex-A57.')
|
|
|
|
# Configure file for benchmark test
|
|
parser.add_argument(
|
|
'-t',
|
|
'--test',
|
|
help='Specify the test settings with configuration '
|
|
'file.')
|
|
|
|
# Whether to keep old json result or not
|
|
parser.add_argument(
|
|
'-k',
|
|
'--keep',
|
|
default='False',
|
|
help='User can specify whether to keep the old json '
|
|
'results from last run. This can be useful if you '
|
|
'want to compare performance differences in two or '
|
|
'more different runs. Default is False(off).')
|
|
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
# Clear old log files in bench suite directory
|
|
def clear_logs():
|
|
logging.info('Removing old logfiles...')
|
|
for f in ['build_log', 'device_log', 'test_log']:
|
|
logfile = os.path.join(config.bench_suite_dir, f)
|
|
try:
|
|
os.remove(logfile)
|
|
except OSError:
|
|
logging.info('No logfile %s need to be removed. Ignored.', f)
|
|
logging.info('Old logfiles been removed.')
|
|
|
|
|
|
# Clear old json files in bench suite directory
|
|
def clear_results():
|
|
logging.info('Clearing old json results...')
|
|
for bench in config.bench_list:
|
|
result = os.path.join(config.bench_suite_dir, bench + '.json')
|
|
try:
|
|
os.remove(result)
|
|
except OSError:
|
|
logging.info('no %s json file need to be removed. Ignored.', bench)
|
|
logging.info('Old json results been removed.')
|
|
|
|
|
|
# Use subprocess.check_call to run other script, and put logs to files
|
|
def check_call_with_log(cmd, log_file):
|
|
log_file = os.path.join(config.bench_suite_dir, log_file)
|
|
with open(log_file, 'a') as logfile:
|
|
log_header = 'Log for command: %s\n' % (cmd)
|
|
logfile.write(log_header)
|
|
try:
|
|
subprocess.check_call(cmd, stdout=logfile)
|
|
except subprocess.CalledProcessError:
|
|
logging.error('Error running %s, please check %s for more info.',
|
|
cmd, log_file)
|
|
raise
|
|
logging.info('Logs for %s are written to %s.', cmd, log_file)
|
|
|
|
|
|
def set_device(serials, remote, frequency):
|
|
setting_cmd = [
|
|
os.path.join(
|
|
os.path.join(config.android_home, config.autotest_dir),
|
|
'site_utils/set_device.py')
|
|
]
|
|
setting_cmd.append('-r=' + remote)
|
|
setting_cmd.append('-q=' + str(frequency))
|
|
|
|
# Deal with serials.
|
|
# If there is no serails specified, try to run test on the only device.
|
|
# If specified, split the serials into a list and run test on each device.
|
|
if serials:
|
|
for serial in serials.split(','):
|
|
setting_cmd.append('-s=' + serial)
|
|
check_call_with_log(setting_cmd, 'device_log')
|
|
setting_cmd.pop()
|
|
else:
|
|
check_call_with_log(setting_cmd, 'device_log')
|
|
|
|
logging.info('CPU mode and frequency set successfully!')
|
|
|
|
|
|
def log_ambiguous_args():
|
|
logging.error('The count of arguments does not match!')
|
|
raise ValueError('The count of arguments does not match.')
|
|
|
|
|
|
# Check if the count of building arguments are log_ambiguous or not. The
|
|
# number of -c/-l, -f, and -os should be either all 0s or all the same.
|
|
def check_count(compiler, llvm_version, build_os, cflags, ldflags):
|
|
# Count will be set to 0 if no compiler or llvm_version specified.
|
|
# Otherwise, one of these two args length should be 0 and count will be
|
|
# the other one.
|
|
count = max(len(compiler), len(llvm_version))
|
|
|
|
# Check if number of cflags is 0 or the same with before.
|
|
if len(cflags) != 0:
|
|
if count != 0 and len(cflags) != count:
|
|
log_ambiguous_args()
|
|
count = len(cflags)
|
|
|
|
if len(ldflags) != 0:
|
|
if count != 0 and len(ldflags) != count:
|
|
log_ambiguous_args()
|
|
count = len(ldflags)
|
|
|
|
if len(build_os) != 0:
|
|
if count != 0 and len(build_os) != count:
|
|
log_ambiguous_args()
|
|
count = len(build_os)
|
|
|
|
# If no settings are passed, only run default once.
|
|
return max(1, count)
|
|
|
|
|
|
# Build benchmark binary with toolchain settings
|
|
def build_bench(setting_no, bench, compiler, llvm_version, build_os, cflags,
|
|
ldflags):
|
|
# Build benchmark locally
|
|
build_cmd = ['./build_bench.py', '-b=' + bench]
|
|
if compiler:
|
|
build_cmd.append('-c=' + compiler[setting_no])
|
|
if llvm_version:
|
|
build_cmd.append('-l=' + llvm_version[setting_no])
|
|
if build_os:
|
|
build_cmd.append('-o=' + build_os[setting_no])
|
|
if cflags:
|
|
build_cmd.append('-f=' + cflags[setting_no])
|
|
if ldflags:
|
|
build_cmd.append('--ldflags=' + ldflags[setting_no])
|
|
|
|
logging.info('Building benchmark for toolchain setting No.%d...',
|
|
setting_no)
|
|
logging.info('Command: %s', build_cmd)
|
|
|
|
try:
|
|
subprocess.check_call(build_cmd)
|
|
except:
|
|
logging.error('Error while building benchmark!')
|
|
raise
|
|
|
|
|
|
def run_and_collect_result(test_cmd, setting_no, i, bench, serial='default'):
|
|
|
|
# Run autotest script for benchmark on DUT
|
|
check_call_with_log(test_cmd, 'test_log')
|
|
|
|
logging.info('Benchmark with setting No.%d, iter.%d finished testing on '
|
|
'device %s.', setting_no, i, serial)
|
|
|
|
# Rename results from the bench_result generated in autotest
|
|
bench_result = os.path.join(config.bench_suite_dir, 'bench_result')
|
|
if not os.path.exists(bench_result):
|
|
logging.error('No result found at %s, '
|
|
'please check test_log for details.', bench_result)
|
|
raise OSError('Result file %s not found.' % bench_result)
|
|
|
|
new_bench_result = 'bench_result_%s_%s_%d_%d' % (bench, serial,
|
|
setting_no, i)
|
|
new_bench_result_path = os.path.join(config.bench_suite_dir,
|
|
new_bench_result)
|
|
try:
|
|
os.rename(bench_result, new_bench_result_path)
|
|
except OSError:
|
|
logging.error('Error while renaming raw result %s to %s',
|
|
bench_result, new_bench_result_path)
|
|
raise
|
|
|
|
logging.info('Benchmark result saved at %s.', new_bench_result_path)
|
|
|
|
|
|
def test_bench(bench, setting_no, iterations, serials, remote, mode):
|
|
logging.info('Start running benchmark on device...')
|
|
|
|
# Run benchmark and tests on DUT
|
|
for i in xrange(iterations):
|
|
logging.info('Iteration No.%d:', i)
|
|
test_cmd = [
|
|
os.path.join(
|
|
os.path.join(config.android_home, config.autotest_dir),
|
|
'site_utils/test_bench.py')
|
|
]
|
|
test_cmd.append('-b=' + bench)
|
|
test_cmd.append('-r=' + remote)
|
|
test_cmd.append('-m=' + mode)
|
|
|
|
# Deal with serials. If there is no serails specified, try to run test
|
|
# on the only device. If specified, split the serials into a list and
|
|
# run test on each device.
|
|
if serials:
|
|
for serial in serials.split(','):
|
|
test_cmd.append('-s=' + serial)
|
|
|
|
run_and_collect_result(test_cmd, setting_no, i, bench, serial)
|
|
test_cmd.pop()
|
|
else:
|
|
run_and_collect_result(test_cmd, setting_no, i, bench)
|
|
|
|
|
|
def gen_json(bench, setting_no, iterations, serials):
|
|
bench_result = os.path.join(config.bench_suite_dir, 'bench_result')
|
|
|
|
logging.info('Generating JSON file for Crosperf...')
|
|
|
|
if not serials:
|
|
serials = 'default'
|
|
|
|
for serial in serials.split(','):
|
|
|
|
# Platform will be used as device lunch combo instead
|
|
#experiment = '_'.join([serial, str(setting_no)])
|
|
experiment = config.product_combo
|
|
|
|
# Input format: bench_result_{bench}_{serial}_{setting_no}_
|
|
input_file = '_'.join([bench_result, bench,
|
|
serial, str(setting_no), ''])
|
|
gen_json_cmd = [
|
|
'./gen_json.py', '--input=' + input_file,
|
|
'--output=%s.json' % os.path.join(config.bench_suite_dir, bench),
|
|
'--bench=' + bench, '--platform=' + experiment,
|
|
'--iterations=' + str(iterations)
|
|
]
|
|
|
|
logging.info('Command: %s', gen_json_cmd)
|
|
if subprocess.call(gen_json_cmd):
|
|
logging.error('Error while generating JSON file, please check raw'
|
|
' data of the results at %s.', input_file)
|
|
|
|
|
|
def gen_crosperf(infile, outfile):
|
|
# Set environment variable for crosperf
|
|
os.environ['PYTHONPATH'] = os.path.dirname(config.toolchain_utils)
|
|
|
|
logging.info('Generating Crosperf Report...')
|
|
crosperf_cmd = [
|
|
os.path.join(config.toolchain_utils, 'generate_report.py'),
|
|
'-i=' + infile, '-o=' + outfile, '-f'
|
|
]
|
|
|
|
# Run crosperf generate_report.py
|
|
logging.info('Command: %s', crosperf_cmd)
|
|
subprocess.call(crosperf_cmd)
|
|
|
|
logging.info('Report generated successfully!')
|
|
logging.info('Report Location: ' + outfile + '.html at bench'
|
|
'suite directory.')
|
|
|
|
|
|
def main(argv):
|
|
# Set environment variable for the local loacation of benchmark suite.
|
|
# This is for collecting testing results to benchmark suite directory.
|
|
os.environ['BENCH_SUITE_DIR'] = config.bench_suite_dir
|
|
|
|
# Set Android type, used for the difference part between aosp and internal.
|
|
os.environ['ANDROID_TYPE'] = config.android_type
|
|
|
|
# Set ANDROID_HOME for both building and testing.
|
|
os.environ['ANDROID_HOME'] = config.android_home
|
|
|
|
# Set environment variable for architecture, this will be used in
|
|
# autotest.
|
|
os.environ['PRODUCT'] = config.product
|
|
|
|
arguments = _parse_arguments(argv)
|
|
|
|
bench_list = arguments.bench
|
|
if not bench_list:
|
|
bench_list = config.bench_list
|
|
|
|
compiler = arguments.compiler_dir
|
|
build_os = arguments.build_os
|
|
llvm_version = arguments.llvm_prebuilts_version
|
|
cflags = arguments.cflags
|
|
ldflags = arguments.ldflags
|
|
iterations = arguments.iterations
|
|
serials = arguments.serials
|
|
remote = arguments.remote
|
|
frequency = arguments.frequency
|
|
mode = arguments.mode
|
|
keep = arguments.keep
|
|
|
|
# Clear old logs every time before run script
|
|
clear_logs()
|
|
|
|
if keep == 'False':
|
|
clear_results()
|
|
|
|
# Set test mode and frequency of CPU on the DUT
|
|
set_device(serials, remote, frequency)
|
|
|
|
test = arguments.test
|
|
# if test configuration file has been given, use the build settings
|
|
# in the configuration file and run the test.
|
|
if test:
|
|
test_config = ConfigParser.ConfigParser(allow_no_value=True)
|
|
if not test_config.read(test):
|
|
logging.error('Error while reading from building '
|
|
'configuration file %s.', test)
|
|
raise RuntimeError('Error while reading configuration file %s.'
|
|
% test)
|
|
|
|
for setting_no, section in enumerate(test_config.sections()):
|
|
bench = test_config.get(section, 'bench')
|
|
compiler = [test_config.get(section, 'compiler')]
|
|
build_os = [test_config.get(section, 'build_os')]
|
|
llvm_version = [test_config.get(section, 'llvm_version')]
|
|
cflags = [test_config.get(section, 'cflags')]
|
|
ldflags = [test_config.get(section, 'ldflags')]
|
|
|
|
# Set iterations from test_config file, if not exist, use the one
|
|
# from command line.
|
|
it = test_config.get(section, 'iterations')
|
|
if not it:
|
|
it = iterations
|
|
it = int(it)
|
|
|
|
# Build benchmark for each single test configuration
|
|
build_bench(0, bench, compiler, llvm_version,
|
|
build_os, cflags, ldflags)
|
|
|
|
test_bench(bench, setting_no, it, serials, remote, mode)
|
|
|
|
gen_json(bench, setting_no, it, serials)
|
|
|
|
for bench in config.bench_list:
|
|
infile = os.path.join(config.bench_suite_dir, bench + '.json')
|
|
if os.path.exists(infile):
|
|
outfile = os.path.join(config.bench_suite_dir,
|
|
bench + '_report')
|
|
gen_crosperf(infile, outfile)
|
|
|
|
# Stop script if there is only config file provided
|
|
return 0
|
|
|
|
# If no configuration file specified, continue running.
|
|
# Check if the count of the setting arguments are log_ambiguous.
|
|
setting_count = check_count(compiler, llvm_version, build_os,
|
|
cflags, ldflags)
|
|
|
|
for bench in bench_list:
|
|
logging.info('Start building and running benchmark: [%s]', bench)
|
|
# Run script for each toolchain settings
|
|
for setting_no in xrange(setting_count):
|
|
build_bench(setting_no, bench, compiler, llvm_version,
|
|
build_os, cflags, ldflags)
|
|
|
|
# Run autotest script for benchmark test on device
|
|
test_bench(bench, setting_no, iterations, serials, remote, mode)
|
|
|
|
gen_json(bench, setting_no, iterations, serials)
|
|
|
|
infile = os.path.join(config.bench_suite_dir, bench + '.json')
|
|
outfile = os.path.join(config.bench_suite_dir, bench + '_report')
|
|
gen_crosperf(infile, outfile)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main(sys.argv[1:])
|