You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
293 lines
10 KiB
293 lines
10 KiB
# -*- coding: utf-8 -*-
|
|
# Copyright 2017 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""Utilities for launching and accessing ChromeOS buildbots."""
|
|
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
import ast
|
|
import json
|
|
import os
|
|
import re
|
|
import time
|
|
|
|
from cros_utils import command_executer
|
|
from cros_utils import logger
|
|
|
|
INITIAL_SLEEP_TIME = 7200 # 2 hours; wait time before polling buildbot.
|
|
SLEEP_TIME = 600 # 10 minutes; time between polling of buildbot.
|
|
|
|
# Some of our slower builders (llvm-next) are taking more
|
|
# than 12 hours. So, increase this TIME_OUT to 15 hours.
|
|
TIME_OUT = 15 * 60 * 60 # Decide the build is dead or will never finish
|
|
|
|
|
|
class BuildbotTimeout(Exception):
|
|
"""Exception to throw when a buildbot operation timesout."""
|
|
|
|
|
|
def RunCommandInPath(path, cmd):
|
|
ce = command_executer.GetCommandExecuter()
|
|
cwd = os.getcwd()
|
|
os.chdir(path)
|
|
status, stdout, stderr = ce.RunCommandWOutput(cmd, print_to_console=False)
|
|
os.chdir(cwd)
|
|
return status, stdout, stderr
|
|
|
|
|
|
def PeekTrybotImage(chromeos_root, buildbucket_id):
|
|
"""Get the artifact URL of a given tryjob.
|
|
|
|
Args:
|
|
buildbucket_id: buildbucket-id
|
|
chromeos_root: root dir of chrome os checkout
|
|
|
|
Returns:
|
|
(status, url) where status can be 'pass', 'fail', 'running',
|
|
and url looks like:
|
|
gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789
|
|
"""
|
|
command = ('cros buildresult --report json --buildbucket-id %s' %
|
|
buildbucket_id)
|
|
rc, out, _ = RunCommandInPath(chromeos_root, command)
|
|
|
|
# Current implementation of cros buildresult returns fail when a job is still
|
|
# running.
|
|
if rc != 0:
|
|
return ('running', None)
|
|
|
|
results = json.loads(out)[buildbucket_id]
|
|
|
|
# Handle the case where the tryjob failed to launch correctly.
|
|
if results['artifacts_url'] is None:
|
|
return (results['status'], '')
|
|
|
|
return (results['status'], results['artifacts_url'].rstrip('/'))
|
|
|
|
|
|
def ParseTryjobBuildbucketId(msg):
|
|
"""Find the buildbucket-id in the messages from `cros tryjob`.
|
|
|
|
Args:
|
|
msg: messages from `cros tryjob`
|
|
|
|
Returns:
|
|
buildbucket-id, which will be passed to `cros buildresult`
|
|
"""
|
|
output_list = ast.literal_eval(msg)
|
|
output_dict = output_list[0]
|
|
if 'buildbucket_id' in output_dict:
|
|
return output_dict['buildbucket_id']
|
|
return None
|
|
|
|
|
|
def SubmitTryjob(chromeos_root,
|
|
buildbot_name,
|
|
patch_list,
|
|
tryjob_flags=None,
|
|
build_toolchain=False):
|
|
"""Calls `cros tryjob ...`
|
|
|
|
Args:
|
|
chromeos_root: the path to the ChromeOS root, needed for finding chromite
|
|
and launching the buildbot.
|
|
buildbot_name: the name of the buildbot queue, such as lumpy-release or
|
|
daisy-paladin.
|
|
patch_list: a python list of the patches, if any, for the buildbot to use.
|
|
tryjob_flags: See cros tryjob --help for available options.
|
|
build_toolchain: builds and uses the latest toolchain, rather than the
|
|
prebuilt one in SDK.
|
|
|
|
Returns:
|
|
buildbucket id
|
|
"""
|
|
patch_arg = ''
|
|
if patch_list:
|
|
for p in patch_list:
|
|
patch_arg = patch_arg + ' -g ' + repr(p)
|
|
if not tryjob_flags:
|
|
tryjob_flags = []
|
|
if build_toolchain:
|
|
tryjob_flags.append('--latest-toolchain')
|
|
tryjob_flags = ' '.join(tryjob_flags)
|
|
|
|
# Launch buildbot with appropriate flags.
|
|
build = buildbot_name
|
|
command = ('cros_sdk -- cros tryjob --yes --json --nochromesdk %s %s %s' %
|
|
(tryjob_flags, patch_arg, build))
|
|
print('CMD: %s' % command)
|
|
_, out, _ = RunCommandInPath(chromeos_root, command)
|
|
buildbucket_id = ParseTryjobBuildbucketId(out)
|
|
print('buildbucket_id: %s' % repr(buildbucket_id))
|
|
if not buildbucket_id:
|
|
logger.GetLogger().LogFatal('Error occurred while launching trybot job: '
|
|
'%s' % command)
|
|
return buildbucket_id
|
|
|
|
|
|
def GetTrybotImage(chromeos_root,
|
|
buildbot_name,
|
|
patch_list,
|
|
tryjob_flags=None,
|
|
build_toolchain=False,
|
|
asynchronous=False):
|
|
"""Launch buildbot and get resulting trybot artifact name.
|
|
|
|
This function launches a buildbot with the appropriate flags to
|
|
build the test ChromeOS image, with the current ToT mobile compiler. It
|
|
checks every 10 minutes to see if the trybot has finished. When the trybot
|
|
has finished, it parses the resulting report logs to find the trybot
|
|
artifact (if one was created), and returns that artifact name.
|
|
|
|
Args:
|
|
chromeos_root: the path to the ChromeOS root, needed for finding chromite
|
|
and launching the buildbot.
|
|
buildbot_name: the name of the buildbot queue, such as lumpy-release or
|
|
daisy-paladin.
|
|
patch_list: a python list of the patches, if any, for the buildbot to use.
|
|
tryjob_flags: See cros tryjob --help for available options.
|
|
build_toolchain: builds and uses the latest toolchain, rather than the
|
|
prebuilt one in SDK.
|
|
asynchronous: don't wait for artifacts; just return the buildbucket id
|
|
|
|
Returns:
|
|
(buildbucket id, partial image url) e.g.
|
|
(8952271933586980528, trybot-elm-release-tryjob/R67-10480.0.0-b2373596)
|
|
"""
|
|
buildbucket_id = SubmitTryjob(chromeos_root, buildbot_name, patch_list,
|
|
tryjob_flags, build_toolchain)
|
|
if asynchronous:
|
|
return buildbucket_id, ' '
|
|
|
|
# The trybot generally takes more than 2 hours to finish.
|
|
# Wait two hours before polling the status.
|
|
time.sleep(INITIAL_SLEEP_TIME)
|
|
elapsed = INITIAL_SLEEP_TIME
|
|
status = 'running'
|
|
image = ''
|
|
while True:
|
|
status, image = PeekTrybotImage(chromeos_root, buildbucket_id)
|
|
if status == 'running':
|
|
if elapsed > TIME_OUT:
|
|
logger.GetLogger().LogFatal(
|
|
'Unable to get build result for target %s.' % buildbot_name)
|
|
else:
|
|
wait_msg = 'Unable to find build result; job may be running.'
|
|
logger.GetLogger().LogOutput(wait_msg)
|
|
logger.GetLogger().LogOutput('{0} minutes elapsed.'.format(elapsed / 60))
|
|
logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME))
|
|
time.sleep(SLEEP_TIME)
|
|
elapsed += SLEEP_TIME
|
|
else:
|
|
break
|
|
|
|
if not buildbot_name.endswith('-toolchain') and status == 'fail':
|
|
# For rotating testers, we don't care about their status
|
|
# result, because if any HWTest failed it will be non-zero.
|
|
#
|
|
# The nightly performance tests do not run HWTests, so if
|
|
# their status is non-zero, we do care. In this case
|
|
# non-zero means the image itself probably did not build.
|
|
image = ''
|
|
|
|
if not image:
|
|
logger.GetLogger().LogError('Trybot job (buildbucket id: %s) failed with'
|
|
'status %s; no trybot image generated. ' %
|
|
(buildbucket_id, status))
|
|
else:
|
|
# Convert full gs path to what crosperf expects. For example, convert
|
|
# gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789
|
|
# to
|
|
# trybot-elm-release-tryjob/R67-10468.0.0-b20789
|
|
image = '/'.join(image.split('/')[-2:])
|
|
|
|
logger.GetLogger().LogOutput("image is '%s'" % image)
|
|
logger.GetLogger().LogOutput('status is %s' % status)
|
|
return buildbucket_id, image
|
|
|
|
|
|
def DoesImageExist(chromeos_root, build):
|
|
"""Check if the image for the given build exists."""
|
|
|
|
ce = command_executer.GetCommandExecuter()
|
|
command = ('gsutil ls gs://chromeos-image-archive/%s'
|
|
'/chromiumos_test_image.tar.xz' % (build))
|
|
ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False)
|
|
return not ret
|
|
|
|
|
|
def WaitForImage(chromeos_root, build):
|
|
"""Wait for an image to be ready."""
|
|
|
|
elapsed_time = 0
|
|
while elapsed_time < TIME_OUT:
|
|
if DoesImageExist(chromeos_root, build):
|
|
return
|
|
logger.GetLogger().LogOutput('Image %s not ready, waiting for 10 minutes' %
|
|
build)
|
|
time.sleep(SLEEP_TIME)
|
|
elapsed_time += SLEEP_TIME
|
|
|
|
logger.GetLogger().LogOutput('Image %s not found, waited for %d hours' %
|
|
(build, (TIME_OUT / 3600)))
|
|
raise BuildbotTimeout('Timeout while waiting for image %s' % build)
|
|
|
|
|
|
def GetLatestImage(chromeos_root, path):
|
|
"""Get latest image"""
|
|
|
|
fmt = re.compile(r'R([0-9]+)-([0-9]+).([0-9]+).([0-9]+)')
|
|
|
|
ce = command_executer.GetCommandExecuter()
|
|
command = ('gsutil ls gs://chromeos-image-archive/%s' % path)
|
|
ret, out, _ = ce.ChrootRunCommandWOutput(
|
|
chromeos_root, command, print_to_console=False)
|
|
if ret != 0:
|
|
raise RuntimeError('Failed to list buckets with command: %s.' % command)
|
|
candidates = [l.split('/')[-2] for l in out.split()]
|
|
candidates = [fmt.match(c) for c in candidates]
|
|
candidates = [[int(r) for r in m.group(1, 2, 3, 4)] for m in candidates if m]
|
|
candidates.sort(reverse=True)
|
|
for c in candidates:
|
|
build = '%s/R%d-%d.%d.%d' % (path, c[0], c[1], c[2], c[3])
|
|
# Denylist "R79-12384.0.0" image released by mistake.
|
|
# TODO(crbug.com/992242): Remove the filter by 2019-09-05.
|
|
if c == [79, 12384, 0, 0]:
|
|
continue
|
|
|
|
if DoesImageExist(chromeos_root, build):
|
|
return build
|
|
|
|
|
|
def GetLatestRecipeImage(chromeos_root, path):
|
|
"""Get latest nightly test image from recipe bucket.
|
|
|
|
Image location example:
|
|
$ARCHIVE/lulu-llvm-next-nightly/R84-13037.0.0-31011-8883172717979984032
|
|
"""
|
|
|
|
fmt = re.compile(r'R([0-9]+)-([0-9]+).([0-9]+).([0-9]+)-([0-9]+)')
|
|
|
|
ce = command_executer.GetCommandExecuter()
|
|
command = ('gsutil ls gs://chromeos-image-archive/%s' % path)
|
|
ret, out, _ = ce.ChrootRunCommandWOutput(
|
|
chromeos_root, command, print_to_console=False)
|
|
if ret != 0:
|
|
raise RuntimeError('Failed to list buckets with command: %s.' % command)
|
|
candidates = [l.split('/')[-2] for l in out.split()]
|
|
candidates = [(fmt.match(c), c) for c in candidates]
|
|
candidates = [([int(r)
|
|
for r in m[0].group(1, 2, 3, 4, 5)], m[1])
|
|
for m in candidates
|
|
if m]
|
|
candidates.sort(key=lambda x: x[0], reverse=True)
|
|
# Try to get ony last two days of images since nightly tests are run once
|
|
# another day.
|
|
for c in candidates[:2]:
|
|
build = '%s/%s' % (path, c[1])
|
|
if DoesImageExist(chromeos_root, build):
|
|
return build
|