You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
101 lines
3.5 KiB
101 lines
3.5 KiB
# Copyright 2020, The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import inspect
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
import utils
|
|
|
|
# Must assign TITLE so that smoke_tests is able to print it.
|
|
TITLE = "Iteration Runs"
|
|
ATEST = "atest"
|
|
ARGS = "-c {}={} {}"
|
|
ITERS = "--iterations"
|
|
RERUN = "--rerun-until-failure"
|
|
RETRY = "--retry-any-failure"
|
|
RUNS = 5
|
|
PASS_TEST_NAME = "atest_will_pass_tests"
|
|
FAIL_TEST_NAME = "atest_will_fail_tests"
|
|
|
|
|
|
class Iterations():
|
|
"""Class of testing --iterations, --rerun-until-failure,
|
|
--retry-any-failure arguments."""
|
|
|
|
def run_test(self, test_arg, expected_pass):
|
|
"""Define pass_tests/fail_tests from expect_pass.
|
|
|
|
The test result will be like running:
|
|
$ atest -c --retry-any-failure=5 atest_will_fail_tests
|
|
|
|
Args:
|
|
test_arg: A string of "--iterations", etc.
|
|
expected_pass: A boolean of expected test result.
|
|
"""
|
|
if expected_pass:
|
|
args = ARGS.format(test_arg, RUNS, PASS_TEST_NAME)
|
|
pass_tests = 3
|
|
fail_tests = 0
|
|
else:
|
|
args = ARGS.format(test_arg, RUNS, FAIL_TEST_NAME)
|
|
pass_tests = 1
|
|
fail_tests = 2
|
|
# Define expected_threads from the argument.
|
|
if test_arg == ITERS:
|
|
expected_threads = (pass_tests+fail_tests) * RUNS
|
|
elif test_arg == RERUN:
|
|
expected_threads = (pass_tests+fail_tests) * 1
|
|
elif test_arg == RETRY:
|
|
expected_threads = pass_tests + (fail_tests*RUNS)
|
|
|
|
print('test{}'.format(test_arg).replace('-', '_'))
|
|
cmd = '{} {}'.format(ATEST, args)
|
|
print('Running: {}'.format(cmd))
|
|
# 1. Check return code
|
|
if expected_pass:
|
|
if subprocess.call(cmd, shell=True) != 0:
|
|
print('Failed testing:\n {}'.format(cmd))
|
|
sys.exit(1)
|
|
else:
|
|
if subprocess.call(cmd, shell=True) == 0:
|
|
print('Testing:\n{}\nis expected failure,'
|
|
' but it passed.'.format(cmd))
|
|
sys.exit(1)
|
|
|
|
# 2. Should observe same threads in host_log.
|
|
if not utils.is_identical(expected_threads,
|
|
utils.get_test_threads(),
|
|
"Test threads"):
|
|
sys.exit(1)
|
|
|
|
# 3. Should observe same amount of passed/failed test in end_host_log.
|
|
if not utils.has_correct_passed_failed_counts(pass_tests, fail_tests):
|
|
sys.exit(1)
|
|
|
|
|
|
def main():
|
|
utils.print_banner(TITLE)
|
|
iters = Iterations()
|
|
# There are 3 test arguments:
|
|
# "--iterations": runs atest_will_pass_test and expect pass.
|
|
# "--rerun-until-failure": runs atest_will_fail_test and expect fail.
|
|
# "--retry-any-failure": runs atest_will_fail_test and expect fail.
|
|
test_args = [(ITERS, True), (RERUN, False), (RETRY, False)]
|
|
for i, _ in enumerate(test_args):
|
|
print('\n[{}/{}] '.format(i+1, len(test_args)), end='')
|
|
utils.func_wrapper(iters.run_test, test_args[i])
|
|
utils.print_banner(TITLE, True)
|