#!/usr/bin/env python2 # # Copyright 2017 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # # pylint: disable=cros-logging-import # This is the script to run specified benchmark with different toolchain # settings. It includes the process of building benchmark locally and running # benchmark on DUT. """Main script to run the benchmark suite from building to testing.""" from __future__ import print_function import argparse import config import ConfigParser import logging import os import subprocess import sys logging.basicConfig(level=logging.INFO) def _parse_arguments(argv): parser = argparse.ArgumentParser(description='Build and run specific ' 'benchamrk') parser.add_argument( '-b', '--bench', action='append', default=[], help='Select which benchmark to run') # Only one of compiler directory and llvm prebuilts version can be indicated # at the beginning, so set -c and -l into a exclusive group. group = parser.add_mutually_exclusive_group() # The toolchain setting arguments has action of 'append', so that users # could compare performance with several toolchain settings together. group.add_argument( '-c', '--compiler_dir', metavar='DIR', action='append', default=[], help='Specify path to the compiler\'s bin directory. ' 'You shall give several paths, each with a -c, to ' 'compare performance differences in ' 'each compiler.') parser.add_argument( '-o', '--build_os', action='append', default=[], help='Specify the host OS to build the benchmark.') group.add_argument( '-l', '--llvm_prebuilts_version', action='append', default=[], help='Specify the version of prebuilt LLVM. When ' 'specific prebuilt version of LLVM already ' 'exists, no need to pass the path to compiler ' 'directory.') parser.add_argument( '-f', '--cflags', action='append', default=[], help='Specify the cflags options for the toolchain. ' 'Be sure to quote all the cflags with quotation ' 'mark("") or use equal(=).') parser.add_argument( '--ldflags', action='append', default=[], help='Specify linker flags for the toolchain.') parser.add_argument( '-i', '--iterations', type=int, default=1, help='Specify how many iterations does the test ' 'take.') # Arguments -s and -r are for connecting to DUT. parser.add_argument( '-s', '--serials', help='Comma separate list of device serials under ' 'test.') parser.add_argument( '-r', '--remote', default='localhost', help='hostname[:port] if the ADB device is connected ' 'to a remote machine. Ensure this workstation ' 'is configured for passwordless ssh access as ' 'users "root" or "adb"') # Arguments -frequency and -m are for device settings parser.add_argument( '--frequency', type=int, default=979200, help='Specify the CPU frequency of the device. The ' 'unit is KHZ. The available value is defined in' 'cpufreq/scaling_available_frequency file in ' 'device\'s each core directory. ' 'The default value is 979200, which shows a ' 'balance in noise and performance. Lower ' 'frequency will slow down the performance but ' 'reduce noise.') parser.add_argument( '-m', '--mode', default='little', help='User can specify whether \'little\' or \'big\' ' 'mode to use. The default one is little mode. ' 'The little mode runs on a single core of ' 'Cortex-A53, while big mode runs on single core ' 'of Cortex-A57.') # Configure file for benchmark test parser.add_argument( '-t', '--test', help='Specify the test settings with configuration ' 'file.') # Whether to keep old json result or not parser.add_argument( '-k', '--keep', default='False', help='User can specify whether to keep the old json ' 'results from last run. This can be useful if you ' 'want to compare performance differences in two or ' 'more different runs. Default is False(off).') return parser.parse_args(argv) # Clear old log files in bench suite directory def clear_logs(): logging.info('Removing old logfiles...') for f in ['build_log', 'device_log', 'test_log']: logfile = os.path.join(config.bench_suite_dir, f) try: os.remove(logfile) except OSError: logging.info('No logfile %s need to be removed. Ignored.', f) logging.info('Old logfiles been removed.') # Clear old json files in bench suite directory def clear_results(): logging.info('Clearing old json results...') for bench in config.bench_list: result = os.path.join(config.bench_suite_dir, bench + '.json') try: os.remove(result) except OSError: logging.info('no %s json file need to be removed. Ignored.', bench) logging.info('Old json results been removed.') # Use subprocess.check_call to run other script, and put logs to files def check_call_with_log(cmd, log_file): log_file = os.path.join(config.bench_suite_dir, log_file) with open(log_file, 'a') as logfile: log_header = 'Log for command: %s\n' % (cmd) logfile.write(log_header) try: subprocess.check_call(cmd, stdout=logfile) except subprocess.CalledProcessError: logging.error('Error running %s, please check %s for more info.', cmd, log_file) raise logging.info('Logs for %s are written to %s.', cmd, log_file) def set_device(serials, remote, frequency): setting_cmd = [ os.path.join( os.path.join(config.android_home, config.autotest_dir), 'site_utils/set_device.py') ] setting_cmd.append('-r=' + remote) setting_cmd.append('-q=' + str(frequency)) # Deal with serials. # If there is no serails specified, try to run test on the only device. # If specified, split the serials into a list and run test on each device. if serials: for serial in serials.split(','): setting_cmd.append('-s=' + serial) check_call_with_log(setting_cmd, 'device_log') setting_cmd.pop() else: check_call_with_log(setting_cmd, 'device_log') logging.info('CPU mode and frequency set successfully!') def log_ambiguous_args(): logging.error('The count of arguments does not match!') raise ValueError('The count of arguments does not match.') # Check if the count of building arguments are log_ambiguous or not. The # number of -c/-l, -f, and -os should be either all 0s or all the same. def check_count(compiler, llvm_version, build_os, cflags, ldflags): # Count will be set to 0 if no compiler or llvm_version specified. # Otherwise, one of these two args length should be 0 and count will be # the other one. count = max(len(compiler), len(llvm_version)) # Check if number of cflags is 0 or the same with before. if len(cflags) != 0: if count != 0 and len(cflags) != count: log_ambiguous_args() count = len(cflags) if len(ldflags) != 0: if count != 0 and len(ldflags) != count: log_ambiguous_args() count = len(ldflags) if len(build_os) != 0: if count != 0 and len(build_os) != count: log_ambiguous_args() count = len(build_os) # If no settings are passed, only run default once. return max(1, count) # Build benchmark binary with toolchain settings def build_bench(setting_no, bench, compiler, llvm_version, build_os, cflags, ldflags): # Build benchmark locally build_cmd = ['./build_bench.py', '-b=' + bench] if compiler: build_cmd.append('-c=' + compiler[setting_no]) if llvm_version: build_cmd.append('-l=' + llvm_version[setting_no]) if build_os: build_cmd.append('-o=' + build_os[setting_no]) if cflags: build_cmd.append('-f=' + cflags[setting_no]) if ldflags: build_cmd.append('--ldflags=' + ldflags[setting_no]) logging.info('Building benchmark for toolchain setting No.%d...', setting_no) logging.info('Command: %s', build_cmd) try: subprocess.check_call(build_cmd) except: logging.error('Error while building benchmark!') raise def run_and_collect_result(test_cmd, setting_no, i, bench, serial='default'): # Run autotest script for benchmark on DUT check_call_with_log(test_cmd, 'test_log') logging.info('Benchmark with setting No.%d, iter.%d finished testing on ' 'device %s.', setting_no, i, serial) # Rename results from the bench_result generated in autotest bench_result = os.path.join(config.bench_suite_dir, 'bench_result') if not os.path.exists(bench_result): logging.error('No result found at %s, ' 'please check test_log for details.', bench_result) raise OSError('Result file %s not found.' % bench_result) new_bench_result = 'bench_result_%s_%s_%d_%d' % (bench, serial, setting_no, i) new_bench_result_path = os.path.join(config.bench_suite_dir, new_bench_result) try: os.rename(bench_result, new_bench_result_path) except OSError: logging.error('Error while renaming raw result %s to %s', bench_result, new_bench_result_path) raise logging.info('Benchmark result saved at %s.', new_bench_result_path) def test_bench(bench, setting_no, iterations, serials, remote, mode): logging.info('Start running benchmark on device...') # Run benchmark and tests on DUT for i in xrange(iterations): logging.info('Iteration No.%d:', i) test_cmd = [ os.path.join( os.path.join(config.android_home, config.autotest_dir), 'site_utils/test_bench.py') ] test_cmd.append('-b=' + bench) test_cmd.append('-r=' + remote) test_cmd.append('-m=' + mode) # Deal with serials. If there is no serails specified, try to run test # on the only device. If specified, split the serials into a list and # run test on each device. if serials: for serial in serials.split(','): test_cmd.append('-s=' + serial) run_and_collect_result(test_cmd, setting_no, i, bench, serial) test_cmd.pop() else: run_and_collect_result(test_cmd, setting_no, i, bench) def gen_json(bench, setting_no, iterations, serials): bench_result = os.path.join(config.bench_suite_dir, 'bench_result') logging.info('Generating JSON file for Crosperf...') if not serials: serials = 'default' for serial in serials.split(','): # Platform will be used as device lunch combo instead #experiment = '_'.join([serial, str(setting_no)]) experiment = config.product_combo # Input format: bench_result_{bench}_{serial}_{setting_no}_ input_file = '_'.join([bench_result, bench, serial, str(setting_no), '']) gen_json_cmd = [ './gen_json.py', '--input=' + input_file, '--output=%s.json' % os.path.join(config.bench_suite_dir, bench), '--bench=' + bench, '--platform=' + experiment, '--iterations=' + str(iterations) ] logging.info('Command: %s', gen_json_cmd) if subprocess.call(gen_json_cmd): logging.error('Error while generating JSON file, please check raw' ' data of the results at %s.', input_file) def gen_crosperf(infile, outfile): # Set environment variable for crosperf os.environ['PYTHONPATH'] = os.path.dirname(config.toolchain_utils) logging.info('Generating Crosperf Report...') crosperf_cmd = [ os.path.join(config.toolchain_utils, 'generate_report.py'), '-i=' + infile, '-o=' + outfile, '-f' ] # Run crosperf generate_report.py logging.info('Command: %s', crosperf_cmd) subprocess.call(crosperf_cmd) logging.info('Report generated successfully!') logging.info('Report Location: ' + outfile + '.html at bench' 'suite directory.') def main(argv): # Set environment variable for the local loacation of benchmark suite. # This is for collecting testing results to benchmark suite directory. os.environ['BENCH_SUITE_DIR'] = config.bench_suite_dir # Set Android type, used for the difference part between aosp and internal. os.environ['ANDROID_TYPE'] = config.android_type # Set ANDROID_HOME for both building and testing. os.environ['ANDROID_HOME'] = config.android_home # Set environment variable for architecture, this will be used in # autotest. os.environ['PRODUCT'] = config.product arguments = _parse_arguments(argv) bench_list = arguments.bench if not bench_list: bench_list = config.bench_list compiler = arguments.compiler_dir build_os = arguments.build_os llvm_version = arguments.llvm_prebuilts_version cflags = arguments.cflags ldflags = arguments.ldflags iterations = arguments.iterations serials = arguments.serials remote = arguments.remote frequency = arguments.frequency mode = arguments.mode keep = arguments.keep # Clear old logs every time before run script clear_logs() if keep == 'False': clear_results() # Set test mode and frequency of CPU on the DUT set_device(serials, remote, frequency) test = arguments.test # if test configuration file has been given, use the build settings # in the configuration file and run the test. if test: test_config = ConfigParser.ConfigParser(allow_no_value=True) if not test_config.read(test): logging.error('Error while reading from building ' 'configuration file %s.', test) raise RuntimeError('Error while reading configuration file %s.' % test) for setting_no, section in enumerate(test_config.sections()): bench = test_config.get(section, 'bench') compiler = [test_config.get(section, 'compiler')] build_os = [test_config.get(section, 'build_os')] llvm_version = [test_config.get(section, 'llvm_version')] cflags = [test_config.get(section, 'cflags')] ldflags = [test_config.get(section, 'ldflags')] # Set iterations from test_config file, if not exist, use the one # from command line. it = test_config.get(section, 'iterations') if not it: it = iterations it = int(it) # Build benchmark for each single test configuration build_bench(0, bench, compiler, llvm_version, build_os, cflags, ldflags) test_bench(bench, setting_no, it, serials, remote, mode) gen_json(bench, setting_no, it, serials) for bench in config.bench_list: infile = os.path.join(config.bench_suite_dir, bench + '.json') if os.path.exists(infile): outfile = os.path.join(config.bench_suite_dir, bench + '_report') gen_crosperf(infile, outfile) # Stop script if there is only config file provided return 0 # If no configuration file specified, continue running. # Check if the count of the setting arguments are log_ambiguous. setting_count = check_count(compiler, llvm_version, build_os, cflags, ldflags) for bench in bench_list: logging.info('Start building and running benchmark: [%s]', bench) # Run script for each toolchain settings for setting_no in xrange(setting_count): build_bench(setting_no, bench, compiler, llvm_version, build_os, cflags, ldflags) # Run autotest script for benchmark test on device test_bench(bench, setting_no, iterations, serials, remote, mode) gen_json(bench, setting_no, iterations, serials) infile = os.path.join(config.bench_suite_dir, bench + '.json') outfile = os.path.join(config.bench_suite_dir, bench + '_report') gen_crosperf(infile, outfile) if __name__ == '__main__': main(sys.argv[1:])