You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
445 lines
16 KiB
445 lines
16 KiB
#!/usr/bin/env python2.7
|
|
|
|
# Copyright 2015, VIXL authors
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions are met:
|
|
#
|
|
# * Redistributions of source code must retain the above copyright notice,
|
|
# this list of conditions and the following disclaimer.
|
|
# * Redistributions in binary form must reproduce the above copyright notice,
|
|
# this list of conditions and the following disclaimer in the documentation
|
|
# and/or other materials provided with the distribution.
|
|
# * Neither the name of ARM Limited nor the names of its contributors may be
|
|
# used to endorse or promote products derived from this software without
|
|
# specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
|
|
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
|
|
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
import argparse
|
|
import fcntl
|
|
import git
|
|
import itertools
|
|
import multiprocessing
|
|
import os
|
|
from os.path import join
|
|
import platform
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
import config
|
|
import clang_format
|
|
import clang_tidy
|
|
import lint
|
|
import printer
|
|
import test
|
|
import test_runner
|
|
import util
|
|
|
|
|
|
dir_root = config.dir_root
|
|
|
|
|
|
# Remove duplicates from a list
|
|
def RemoveDuplicates(values):
|
|
# Convert the list into a set and back to list
|
|
# as sets guarantee items are unique.
|
|
return list(set(values))
|
|
|
|
|
|
# Custom argparse.Action to automatically add and handle an 'all' option.
|
|
# If no 'default' value is set, it will default to 'all.
|
|
# If accepted options are set using 'choices' then only these values will be
|
|
# allowed.
|
|
# If they're set using 'soft_choices' then 'all' will default to these values,
|
|
# but other values will also be accepted.
|
|
class AllChoiceAction(argparse.Action):
|
|
|
|
# At least one option was set by the user.
|
|
WasSetByUser = False
|
|
|
|
def __init__(self, **kwargs):
|
|
if 'choices' in kwargs:
|
|
assert 'soft_choices' not in kwargs,\
|
|
"Can't have both 'choices' and 'soft_choices' options"
|
|
self.all_choices = list(kwargs['choices'])
|
|
kwargs['choices'].append('all')
|
|
else:
|
|
self.all_choices = kwargs['soft_choices']
|
|
kwargs['help'] += ' Supported values: {' + ','.join(
|
|
['all'] + self.all_choices) + '}'
|
|
del kwargs['soft_choices']
|
|
if 'default' not in kwargs:
|
|
kwargs['default'] = self.all_choices
|
|
super(AllChoiceAction, self).__init__(**kwargs)
|
|
|
|
def __call__(self, parser, namespace, values, option_string=None):
|
|
AllChoiceAction.WasSetByUser = True
|
|
if 'all' in values:
|
|
# Substitute 'all' by the actual values.
|
|
values = self.all_choices + [value for value in values if value != 'all']
|
|
|
|
setattr(namespace, self.dest, RemoveDuplicates(values))
|
|
|
|
|
|
def BuildOptions():
|
|
args = argparse.ArgumentParser(
|
|
description =
|
|
'''This tool runs all tests matching the specified filters for multiple
|
|
environment, build options, and runtime options configurations.''',
|
|
# Print default values.
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
|
|
args.add_argument('filters', metavar='filter', nargs='*',
|
|
help='Run tests matching all of the (regexp) filters.')
|
|
|
|
# We automatically build the script options from the options to be tested.
|
|
test_arguments = args.add_argument_group(
|
|
'Test options',
|
|
'These options indicate what should be tested')
|
|
test_arguments.add_argument(
|
|
'--negative_testing',
|
|
help='Tests with negative testing enabled.',
|
|
action='store_const',
|
|
const='on',
|
|
default='off')
|
|
test_arguments.add_argument(
|
|
'--compiler',
|
|
help='Test for the specified compilers.',
|
|
soft_choices=config.tested_compilers,
|
|
action=AllChoiceAction,
|
|
nargs="+")
|
|
test_arguments.add_argument(
|
|
'--mode',
|
|
help='Test with the specified build modes.',
|
|
choices=config.build_options_modes,
|
|
action=AllChoiceAction,
|
|
nargs="+")
|
|
test_arguments.add_argument(
|
|
'--std',
|
|
help='Test with the specified C++ standard.',
|
|
soft_choices=config.tested_cpp_standards,
|
|
action=AllChoiceAction,
|
|
nargs="+")
|
|
test_arguments.add_argument(
|
|
'--target',
|
|
help='Test with the specified isa enabled.',
|
|
soft_choices=config.build_options_target,
|
|
action=AllChoiceAction,
|
|
nargs="+")
|
|
|
|
general_arguments = args.add_argument_group('General options')
|
|
general_arguments.add_argument('--dry-run', action='store_true',
|
|
help='''Don't actually build or run anything,
|
|
but print the configurations that would be
|
|
tested.''')
|
|
general_arguments.add_argument('--verbose', action='store_true',
|
|
help='''Print extra information.''')
|
|
general_arguments.add_argument(
|
|
'--jobs', '-j', metavar='N', type=int, nargs='?',
|
|
default=multiprocessing.cpu_count(),
|
|
const=multiprocessing.cpu_count(),
|
|
help='''Runs the tests using N jobs. If the option is set but no value is
|
|
provided, the script will use as many jobs as it thinks useful.''')
|
|
general_arguments.add_argument('--clang-format',
|
|
default=clang_format.DEFAULT_CLANG_FORMAT,
|
|
help='Path to clang-format.')
|
|
general_arguments.add_argument('--clang-tidy',
|
|
default=clang_tidy.DEFAULT_CLANG_TIDY,
|
|
help='Path to clang-tidy.')
|
|
general_arguments.add_argument('--nobench', action='store_true',
|
|
help='Do not run benchmarks.')
|
|
general_arguments.add_argument('--nolint', action='store_true',
|
|
help='Do not run the linter.')
|
|
general_arguments.add_argument('--noclang-format', action='store_true',
|
|
help='Do not run clang-format.')
|
|
general_arguments.add_argument('--noclang-tidy', action='store_true',
|
|
help='Do not run clang-tidy.')
|
|
general_arguments.add_argument('--notest', action='store_true',
|
|
help='Do not run tests.')
|
|
general_arguments.add_argument('--fail-early', action='store_true',
|
|
help='Exit as soon as a test fails.')
|
|
general_arguments.add_argument(
|
|
'--under_valgrind', action='store_true',
|
|
help='''Run the test-runner commands under Valgrind.
|
|
Note that a few tests are known to fail because of
|
|
issues in Valgrind''')
|
|
return args.parse_args()
|
|
|
|
|
|
def RunCommand(command, environment_options = None):
|
|
# Create a copy of the environment. We do not want to pollute the environment
|
|
# of future commands run.
|
|
environment = os.environ.copy()
|
|
|
|
printable_command = ''
|
|
if environment_options:
|
|
# Add the environment options to the environment:
|
|
environment.update(environment_options)
|
|
printable_command += ' ' + DictToString(environment_options) + ' '
|
|
printable_command += ' '.join(command)
|
|
|
|
printable_command_orange = \
|
|
printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR
|
|
printer.PrintOverwritableLine(printable_command_orange)
|
|
sys.stdout.flush()
|
|
|
|
# Start a process for the command.
|
|
# Interleave `stderr` and `stdout`.
|
|
p = subprocess.Popen(command,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
env=environment)
|
|
|
|
# We want to be able to display a continuously updated 'work indicator' while
|
|
# the process is running. Since the process can hang if the `stdout` pipe is
|
|
# full, we need to pull from it regularly. We cannot do so via the
|
|
# `readline()` function because it is blocking, and would thus cause the
|
|
# indicator to not be updated properly. So use file control mechanisms
|
|
# instead.
|
|
indicator = ' (still working: %d seconds elapsed)'
|
|
|
|
# Mark the process output as non-blocking.
|
|
flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL)
|
|
fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)
|
|
|
|
t_start = time.time()
|
|
t_current = t_start
|
|
t_last_indication = t_start
|
|
t_current = t_start
|
|
process_output = ''
|
|
|
|
# Keep looping as long as the process is running.
|
|
while p.poll() is None:
|
|
# Avoid polling too often.
|
|
time.sleep(0.1)
|
|
# Update the progress indicator.
|
|
t_current = time.time()
|
|
if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1):
|
|
printer.PrintOverwritableLine(
|
|
printable_command_orange + indicator % int(t_current - t_start))
|
|
sys.stdout.flush()
|
|
t_last_indication = t_current
|
|
# Pull from the process output.
|
|
while True:
|
|
try:
|
|
line = os.read(p.stdout.fileno(), 1024)
|
|
except OSError:
|
|
line = ''
|
|
break
|
|
if line == '': break
|
|
process_output += line
|
|
|
|
# The process has exited. Don't forget to retrieve the rest of its output.
|
|
out, err = p.communicate()
|
|
rc = p.poll()
|
|
process_output += out
|
|
|
|
printable_command += ' (took %d seconds)' % int(t_current - t_start)
|
|
if rc == 0:
|
|
printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR)
|
|
else:
|
|
printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR)
|
|
printer.Print(process_output)
|
|
return rc
|
|
|
|
|
|
def RunLinter(jobs):
|
|
return lint.RunLinter(map(lambda x: join(dir_root, x),
|
|
util.get_source_files()),
|
|
jobs = args.jobs, progress_prefix = 'cpp lint: ')
|
|
|
|
|
|
def RunClangFormat(clang_path, jobs):
|
|
return clang_format.ClangFormatFiles(util.get_source_files(),
|
|
clang_path,
|
|
jobs = jobs,
|
|
progress_prefix = 'clang-format: ')
|
|
|
|
def RunClangTidy(clang_path, jobs):
|
|
return clang_tidy.ClangTidyFiles(util.get_source_files(),
|
|
clang_path,
|
|
jobs = jobs,
|
|
progress_prefix = 'clang-tidy: ')
|
|
|
|
def BuildAll(build_options, jobs, environment_options):
|
|
scons_command = ['scons', '-C', dir_root, 'all', '-j', str(jobs)]
|
|
if util.IsCommandAvailable('ccache'):
|
|
scons_command += ['compiler_wrapper=ccache']
|
|
# Fixes warnings for ccache 3.3.1 and lower:
|
|
environment_options = environment_options.copy()
|
|
environment_options["CCACHE_CPP2"] = 'yes'
|
|
scons_command += DictToString(build_options).split()
|
|
return RunCommand(scons_command, environment_options)
|
|
|
|
|
|
def CanRunAarch64(options, args):
|
|
for target in options['target']:
|
|
if target in ['aarch64', 'a64']:
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
def CanRunAarch32(options, args):
|
|
for target in options['target']:
|
|
if target in ['aarch32', 'a32', 't32']:
|
|
return True
|
|
return False
|
|
|
|
|
|
def RunBenchmarks(options, args):
|
|
rc = 0
|
|
if CanRunAarch32(options, args):
|
|
benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch32_benchmarks)
|
|
for bench in benchmark_names:
|
|
rc |= RunCommand(
|
|
[os.path.realpath(
|
|
join(config.dir_build_latest, 'benchmarks/aarch32', bench)), '10'])
|
|
if CanRunAarch64(options, args):
|
|
benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch64_benchmarks)
|
|
for bench in benchmark_names:
|
|
rc |= RunCommand(
|
|
[util.relrealpath(
|
|
join(config.dir_build_latest,
|
|
'benchmarks/aarch64', bench)), '10'])
|
|
return rc
|
|
|
|
|
|
|
|
# It is a precommit run if the user did not specify any of the
|
|
# options that would affect the automatically generated combinations.
|
|
def IsPrecommitRun(args):
|
|
return args.negative_testing == "off" and not AllChoiceAction.WasSetByUser
|
|
|
|
# Generate a list of all the possible combinations of the passed list:
|
|
# ListCombinations( a = [a0, a1], b = [b0, b1] ) will return
|
|
# [ {a : a0, b : b0}, {a : a0, b : b1}, {a: a1, b : b0}, {a : a1, b : b1}]
|
|
def ListCombinations(**kwargs):
|
|
# End of recursion: no options passed
|
|
if not kwargs:
|
|
return [{}]
|
|
option, values = kwargs.popitem()
|
|
configs = ListCombinations(**kwargs)
|
|
retval = []
|
|
if not isinstance(values, list):
|
|
values = [values]
|
|
for value in values:
|
|
for config in configs:
|
|
new_config = config.copy()
|
|
new_config[option] = value
|
|
retval.append(new_config)
|
|
return retval
|
|
|
|
# Convert a dictionary into a space separated string
|
|
# {a : a0, b : b0} --> "a=a0 b=b0"
|
|
def DictToString(options):
|
|
return " ".join(
|
|
["{}={}".format(option, value) for option, value in options.items()])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
util.require_program('scons')
|
|
|
|
args = BuildOptions()
|
|
|
|
rc = util.ReturnCode(args.fail_early, printer.Print)
|
|
|
|
if args.under_valgrind:
|
|
util.require_program('valgrind')
|
|
|
|
tests = test_runner.TestQueue()
|
|
if not args.nolint and not args.dry_run:
|
|
rc.Combine(RunLinter(args.jobs))
|
|
|
|
if not args.noclang_format and not args.dry_run:
|
|
rc.Combine(RunClangFormat(args.clang_format, args.jobs))
|
|
|
|
if not args.noclang_tidy and not args.dry_run:
|
|
rc.Combine(RunClangTidy(args.clang_tidy, args.jobs))
|
|
|
|
list_options = []
|
|
if IsPrecommitRun(args):
|
|
# Maximize the coverage for precommit testing.
|
|
|
|
# Debug builds with negative testing and all targets enabled.
|
|
list_options += ListCombinations(
|
|
compiler = args.compiler,
|
|
negative_testing = 'on',
|
|
std = args.std,
|
|
mode = 'debug',
|
|
target = 'a64,a32,t32')
|
|
|
|
# Release builds with all targets enabled.
|
|
list_options += ListCombinations(
|
|
compiler = args.compiler,
|
|
negative_testing = 'off',
|
|
std = args.std,
|
|
mode = 'release',
|
|
target = 'a64,a32,t32')
|
|
|
|
# Debug builds for individual targets.
|
|
list_options += ListCombinations(
|
|
compiler = args.compiler[0],
|
|
negative_testing = 'off',
|
|
std = args.std,
|
|
mode = 'debug',
|
|
target = ['a32', 't32', 'a64'])
|
|
else:
|
|
list_options = ListCombinations(
|
|
compiler = args.compiler,
|
|
negative_testing = args.negative_testing,
|
|
std = args.std,
|
|
mode = args.mode,
|
|
target = args.target)
|
|
|
|
for options in list_options:
|
|
if (args.dry_run):
|
|
print(DictToString(options))
|
|
continue
|
|
# Convert 'compiler' into an environment variable:
|
|
environment_options = {'CXX': options['compiler']}
|
|
del options['compiler']
|
|
|
|
# Avoid going through the build stage if we are not using the build
|
|
# result.
|
|
if not (args.notest and args.nobench):
|
|
build_rc = BuildAll(options, args.jobs, environment_options)
|
|
# Don't run the tests for this configuration if the build failed.
|
|
if build_rc != 0:
|
|
rc.Combine(build_rc)
|
|
continue
|
|
|
|
# Use the realpath of the test executable so that the commands printed
|
|
# can be copy-pasted and run.
|
|
test_executable = util.relrealpath(
|
|
join(config.dir_build_latest, 'test', 'test-runner'))
|
|
|
|
if not args.notest:
|
|
printer.Print(test_executable)
|
|
tests.AddTests(
|
|
test_executable,
|
|
args.filters,
|
|
list(),
|
|
args.under_valgrind)
|
|
|
|
if not args.nobench:
|
|
rc.Combine(RunBenchmarks(options, args))
|
|
|
|
rc.Combine(tests.Run(args.jobs, args.verbose))
|
|
if not args.dry_run:
|
|
rc.PrintStatus()
|
|
|
|
sys.exit(rc.Value)
|