You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
236 lines
9.6 KiB
236 lines
9.6 KiB
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
# Copyright 2019 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""Tests for auto bisection of LLVM."""
|
|
|
|
from __future__ import print_function
|
|
|
|
import os
|
|
import subprocess
|
|
import time
|
|
import traceback
|
|
import unittest
|
|
import unittest.mock as mock
|
|
|
|
import auto_llvm_bisection
|
|
import chroot
|
|
import llvm_bisection
|
|
import test_helpers
|
|
|
|
|
|
class AutoLLVMBisectionTest(unittest.TestCase):
|
|
"""Unittests for auto bisection of LLVM."""
|
|
|
|
# Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking
|
|
# the script outside of the chroot.
|
|
@mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
|
|
# Simulate behavior of `time.sleep()` when waiting for errors to settle caused
|
|
# by `llvm_bisection.main()` (e.g. network issue, etc.).
|
|
@mock.patch.object(time, 'sleep')
|
|
# Simulate behavior of `traceback.print_exc()` when an exception happened in
|
|
# `llvm_bisection.main()`.
|
|
@mock.patch.object(traceback, 'print_exc')
|
|
# Simulate behavior of `llvm_bisection.main()` when failed to launch tryjobs
|
|
# (exception happened along the way, etc.).
|
|
@mock.patch.object(llvm_bisection, 'main')
|
|
# Simulate behavior of `os.path.isfile()` when starting a new bisection.
|
|
@mock.patch.object(os.path, 'isfile', return_value=False)
|
|
# Simulate behavior of `GetPathToUpdateAllTryjobsWithAutoScript()` when
|
|
# returning the absolute path to that script that updates all 'pending'
|
|
# tryjobs to the result of `cros buildresult`.
|
|
@mock.patch.object(
|
|
auto_llvm_bisection,
|
|
'GetPathToUpdateAllTryjobsWithAutoScript',
|
|
return_value='/abs/path/to/update_tryjob.py')
|
|
# Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line
|
|
# arguments required by the bisection script.
|
|
@mock.patch.object(
|
|
llvm_bisection,
|
|
'GetCommandLineArgs',
|
|
return_value=test_helpers.ArgsOutputTest())
|
|
def testFailedToStartBisection(
|
|
self, mock_get_args, mock_get_auto_script, mock_is_file,
|
|
mock_llvm_bisection, mock_traceback, mock_sleep, mock_outside_chroot):
|
|
|
|
def MockLLVMBisectionRaisesException(_args_output):
|
|
raise ValueError('Failed to launch more tryjobs.')
|
|
|
|
# Use the test function to simulate the behavior of an exception happening
|
|
# when launching more tryjobs.
|
|
mock_llvm_bisection.side_effect = MockLLVMBisectionRaisesException
|
|
|
|
# Verify the exception is raised when the number of attempts to launched
|
|
# more tryjobs is exceeded, so unable to continue
|
|
# bisection.
|
|
with self.assertRaises(SystemExit) as err:
|
|
auto_llvm_bisection.main()
|
|
|
|
self.assertEqual(err.exception.code, 1)
|
|
|
|
mock_outside_chroot.assert_called_once()
|
|
mock_get_args.assert_called_once()
|
|
mock_get_auto_script.assert_called_once()
|
|
self.assertEqual(mock_is_file.call_count, 2)
|
|
self.assertEqual(mock_llvm_bisection.call_count, 3)
|
|
self.assertEqual(mock_traceback.call_count, 3)
|
|
self.assertEqual(mock_sleep.call_count, 2)
|
|
|
|
# Simulate the behavior of `subprocess.call()` when successfully updated all
|
|
# tryjobs whose 'status' value is 'pending'.
|
|
@mock.patch.object(subprocess, 'call', return_value=0)
|
|
# Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking
|
|
# the script outside of the chroot.
|
|
@mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
|
|
# Simulate behavior of `time.sleep()` when waiting for errors to settle caused
|
|
# by `llvm_bisection.main()` (e.g. network issue, etc.).
|
|
@mock.patch.object(time, 'sleep')
|
|
# Simulate behavior of `traceback.print_exc()` when an exception happened in
|
|
# `llvm_bisection.main()`.
|
|
@mock.patch.object(traceback, 'print_exc')
|
|
# Simulate behavior of `llvm_bisection.main()` when failed to launch tryjobs
|
|
# (exception happened along the way, etc.).
|
|
@mock.patch.object(llvm_bisection, 'main')
|
|
# Simulate behavior of `os.path.isfile()` when starting a new bisection.
|
|
@mock.patch.object(os.path, 'isfile')
|
|
# Simulate behavior of `GetPathToUpdateAllTryjobsWithAutoScript()` when
|
|
# returning the absolute path to that script that updates all 'pending'
|
|
# tryjobs to the result of `cros buildresult`.
|
|
@mock.patch.object(
|
|
auto_llvm_bisection,
|
|
'GetPathToUpdateAllTryjobsWithAutoScript',
|
|
return_value='/abs/path/to/update_tryjob.py')
|
|
# Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line
|
|
# arguments required by the bisection script.
|
|
@mock.patch.object(
|
|
llvm_bisection,
|
|
'GetCommandLineArgs',
|
|
return_value=test_helpers.ArgsOutputTest())
|
|
def testSuccessfullyBisectedLLVMRevision(
|
|
self, mock_get_args, mock_get_auto_script, mock_is_file,
|
|
mock_llvm_bisection, mock_traceback, mock_sleep, mock_outside_chroot,
|
|
mock_update_tryjobs):
|
|
|
|
# Simulate the behavior of `os.path.isfile()` when checking whether the
|
|
# status file provided exists.
|
|
@test_helpers.CallCountsToMockFunctions
|
|
def MockStatusFileCheck(call_count, _last_tested):
|
|
# Simulate that the status file does not exist, so the LLVM bisection
|
|
# script would create the status file and launch tryjobs.
|
|
if call_count < 2:
|
|
return False
|
|
|
|
# Simulate when the status file exists and `subprocess.call()` executes
|
|
# the script that updates all the 'pending' tryjobs to the result of `cros
|
|
# buildresult`.
|
|
if call_count == 2:
|
|
return True
|
|
|
|
assert False, 'os.path.isfile() called more times than expected.'
|
|
|
|
# Simulate behavior of `llvm_bisection.main()` when successfully bisected
|
|
# between the good and bad LLVM revision.
|
|
@test_helpers.CallCountsToMockFunctions
|
|
def MockLLVMBisectionReturnValue(call_count, _args_output):
|
|
# Simulate that successfully launched more tryjobs.
|
|
if call_count == 0:
|
|
return 0
|
|
|
|
# Simulate that failed to launch more tryjobs.
|
|
if call_count == 1:
|
|
raise ValueError('Failed to launch more tryjobs.')
|
|
|
|
# Simulate that the bad revision has been found.
|
|
if call_count == 2:
|
|
return llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value
|
|
|
|
assert False, 'Called `llvm_bisection.main()` more than expected.'
|
|
|
|
# Use the test function to simulate the behavior of `llvm_bisection.main()`.
|
|
mock_llvm_bisection.side_effect = MockLLVMBisectionReturnValue
|
|
|
|
# Use the test function to simulate the behavior of `os.path.isfile()`.
|
|
mock_is_file.side_effect = MockStatusFileCheck
|
|
|
|
# Verify the excpetion is raised when successfully found the bad revision.
|
|
# Uses `sys.exit(0)` to indicate success.
|
|
with self.assertRaises(SystemExit) as err:
|
|
auto_llvm_bisection.main()
|
|
|
|
self.assertEqual(err.exception.code, 0)
|
|
|
|
mock_outside_chroot.assert_called_once()
|
|
mock_get_args.assert_called_once()
|
|
mock_get_auto_script.assert_called_once()
|
|
self.assertEqual(mock_is_file.call_count, 3)
|
|
self.assertEqual(mock_llvm_bisection.call_count, 3)
|
|
mock_traceback.assert_called_once()
|
|
mock_sleep.assert_called_once()
|
|
mock_update_tryjobs.assert_called_once()
|
|
|
|
# Simulate behavior of `subprocess.call()` when failed to update tryjobs to
|
|
# `cros buildresult` (script failed).
|
|
@mock.patch.object(subprocess, 'call', return_value=1)
|
|
# Simulate behavior of `time.time()` when determining the time passed when
|
|
# updating tryjobs whose 'status' is 'pending'.
|
|
@mock.patch.object(time, 'time')
|
|
# Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking
|
|
# the script outside of the chroot.
|
|
@mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True)
|
|
# Simulate behavior of `time.sleep()` when waiting for errors to settle caused
|
|
# by `llvm_bisection.main()` (e.g. network issue, etc.).
|
|
@mock.patch.object(time, 'sleep')
|
|
# Simulate behavior of `traceback.print_exc()` when resuming bisection.
|
|
@mock.patch.object(os.path, 'isfile', return_value=True)
|
|
# Simulate behavior of `GetPathToUpdateAllTryjobsWithAutoScript()` when
|
|
# returning the absolute path to that script that updates all 'pending'
|
|
# tryjobs to the result of `cros buildresult`.
|
|
@mock.patch.object(
|
|
auto_llvm_bisection,
|
|
'GetPathToUpdateAllTryjobsWithAutoScript',
|
|
return_value='/abs/path/to/update_tryjob.py')
|
|
# Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line
|
|
# arguments required by the bisection script.
|
|
@mock.patch.object(
|
|
llvm_bisection,
|
|
'GetCommandLineArgs',
|
|
return_value=test_helpers.ArgsOutputTest())
|
|
def testFailedToUpdatePendingTryJobs(
|
|
self, mock_get_args, mock_get_auto_script, mock_is_file, mock_sleep,
|
|
mock_outside_chroot, mock_time, mock_update_tryjobs):
|
|
|
|
# Simulate behavior of `time.time()` for time passed.
|
|
@test_helpers.CallCountsToMockFunctions
|
|
def MockTimePassed(call_count):
|
|
if call_count < 3:
|
|
return call_count
|
|
|
|
assert False, 'Called `time.time()` more than expected.'
|
|
|
|
# Use the test function to simulate the behavior of `time.time()`.
|
|
mock_time.side_effect = MockTimePassed
|
|
|
|
# Reduce the polling limit for the test case to terminate faster.
|
|
auto_llvm_bisection.POLLING_LIMIT_SECS = 1
|
|
|
|
# Verify the exception is raised when unable to update tryjobs whose
|
|
# 'status' value is 'pending'.
|
|
with self.assertRaises(SystemExit) as err:
|
|
auto_llvm_bisection.main()
|
|
|
|
self.assertEqual(err.exception.code, 1)
|
|
|
|
mock_outside_chroot.assert_called_once()
|
|
mock_get_args.assert_called_once()
|
|
mock_get_auto_script.assert_called_once()
|
|
self.assertEqual(mock_is_file.call_count, 2)
|
|
mock_sleep.assert_called_once()
|
|
self.assertEqual(mock_time.call_count, 3)
|
|
self.assertEqual(mock_update_tryjobs.call_count, 2)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|