# Copyright (c) 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This file provides core logic for labstation verify/repair process."""
import logging
from autotest_lib.client.common_lib import error
from autotest_lib.server import afe_utils
from autotest_lib.server.hosts import base_label
from autotest_lib.server.hosts import cros_label
from autotest_lib.server.hosts import labstation_repair
from autotest_lib.server.cros import provision
from autotest_lib.server.hosts import base_servohost
from autotest_lib.server.cros.dynamic_suite import constants as ds_constants
from autotest_lib.server.cros.dynamic_suite import tools
from autotest_lib.client.common_lib.cros import dev_server
from autotest_lib.server import utils as server_utils
from autotest_lib.site_utils.rpm_control_system import rpm_client
class LabstationHost(base_servohost.BaseServoHost):
"""Labstation specific host class"""
# Threshold we decide to ignore a in_use file lock. In minutes
IN_USE_FILE_EXPIRE_MINS = 90
# Uptime threshold to perform a labstation reboot, this is to prevent a
# broken DUT keep trying to reboot a labstation. In hours
UP_TIME_THRESH_HOLD_HOURS = 6
VERSION_PREFIX = provision.CROS_VERSION_PREFIX
@staticmethod
def check_host(host, timeout=10):
"""
Check if the given host is a labstation host.
@param host: An ssh host representing a device.
@param timeout: The timeout for the run command.
@return: True if the host device is labstation.
@raises AutoservRunError: If the command failed.
@raises AutoservSSHTimeout: Ssh connection has timed out.
"""
try:
result = host.run(
'grep -q labstation /etc/lsb-release',
ignore_status=True, timeout=timeout)
except (error.AutoservRunError, error.AutoservSSHTimeout):
return False
return result.exit_status == 0
def _initialize(self, hostname, *args, **dargs):
super(LabstationHost, self)._initialize(hostname=hostname,
*args, **dargs)
self._repair_strategy = (
labstation_repair.create_labstation_repair_strategy())
self.labels = base_label.LabelRetriever(cros_label.LABSTATION_LABELS)
def is_reboot_requested(self):
"""Check if a reboot is requested for this labstation, the reboot can
either be requested from labstation or DUTs. For request from DUTs we
only process it if uptime longer than a threshold because we want
to prevent a broken servo keep its labstation in reboot cycle.
@returns True if a reboot is required, otherwise False
"""
if self._check_update_status() == self.UPDATE_STATE.PENDING_REBOOT:
logging.info('Labstation reboot requested from labstation for'
' update image')
return True
if not self._validate_uptime():
logging.info('Ignoring DUTs reboot request because %s was'
' rebooted in last %d hours.',
self.hostname, self.UP_TIME_THRESH_HOLD_HOURS)
return False
cmd = 'find %s*%s' % (self.TEMP_FILE_DIR, self.REBOOT_FILE_POSTFIX)
output = self.run(cmd, ignore_status=True).stdout
if output:
in_use_file_list = output.strip().split('\n')
logging.info('%s DUT(s) are currently requesting to'
' reboot labstation.', len(in_use_file_list))
return True
else:
return False
def try_reboot(self):
"""Try to reboot the labstation if it's safe to do(no servo in use,
and not processing updates), and cleanup reboot control file.
"""
if self._is_servo_in_use():
logging.info('Aborting reboot action because some DUT(s) are'
' currently using servo(s).')
return
update_state = self._check_update_status()
if update_state == self.UPDATE_STATE.RUNNING:
logging.info('Aborting reboot action because an update process'
' is running.')
return
if update_state == self.UPDATE_STATE.PENDING_REBOOT:
self._post_update_reboot()
else:
self._servo_host_reboot()
self.update_cros_version_label()
logging.info('Cleaning up reboot control files.')
self._cleanup_post_reboot()
def get_labels(self):
"""Return the detected labels on the host."""
return self.labels.get_labels(self)
def get_os_type(self):
return 'labstation'
def verify_job_repo_url(self, tag=''):
"""
Make sure job_repo_url of this host is valid.
Eg: The job_repo_url "http://lmn.cd.ab.xyx:8080/static/\
lumpy-release/R29-4279.0.0/autotest/packages" claims to have the
autotest package for lumpy-release/R29-4279.0.0. If this isn't the case,
download and extract it. If the devserver embedded in the url is
unresponsive, update the job_repo_url of the host after staging it on
another devserver.
@param job_repo_url: A url pointing to the devserver where the autotest
package for this build should be staged.
@param tag: The tag from the server job, in the format
<job_id>-<user>/<hostname>, or <hostless> for a server job.
@raises DevServerException: If we could not resolve a devserver.
@raises AutoservError: If we're unable to save the new job_repo_url as
a result of choosing a new devserver because the old one failed to
respond to a health check.
@raises urllib2.URLError: If the devserver embedded in job_repo_url
doesn't respond within the timeout.
"""
info = self.host_info_store.get()
job_repo_url = info.attributes.get(ds_constants.JOB_REPO_URL, '')
if not job_repo_url:
logging.warning('No job repo url set on host %s', self.hostname)
return
logging.info('Verifying job repo url %s', job_repo_url)
devserver_url, image_name = tools.get_devserver_build_from_package_url(
job_repo_url)
ds = dev_server.ImageServer(devserver_url)
logging.info('Staging autotest artifacts for %s on devserver %s',
image_name, ds.url())
ds.stage_artifacts(image_name, ['autotest_packages'])
def host_version_prefix(self, image):
"""Return version label prefix.
In case the CrOS provisioning version is something other than the
standard CrOS version e.g. CrOS TH version, this function will
find the prefix from provision.py.
@param image: The image name to find its version prefix.
@returns: A prefix string for the image type.
"""
return provision.get_version_label_prefix(image)
def stage_server_side_package(self, image=None):
"""Stage autotest server-side package on devserver.
@param image: Full path of an OS image to install or a build name.
@return: A url to the autotest server-side package.
@raise: error.AutoservError if fail to locate the build to test with, or
fail to stage server-side package.
"""
# If enable_drone_in_restricted_subnet is False, do not set hostname
# in devserver.resolve call, so a devserver in non-restricted subnet
# is picked to stage autotest server package for drone to download.
hostname = self.hostname
if not server_utils.ENABLE_DRONE_IN_RESTRICTED_SUBNET:
hostname = None
if image:
image_name = tools.get_build_from_image(image)
if not image_name:
raise error.AutoservError(
'Failed to parse build name from %s' % image)
ds = dev_server.ImageServer.resolve(image_name, hostname)
else:
info = self.host_info_store.get()
job_repo_url = info.attributes.get(ds_constants.JOB_REPO_URL, '')
if job_repo_url:
devserver_url, image_name = (
tools.get_devserver_build_from_package_url(job_repo_url))
# If enable_drone_in_restricted_subnet is True, use the
# existing devserver. Otherwise, resolve a new one in
# non-restricted subnet.
if server_utils.ENABLE_DRONE_IN_RESTRICTED_SUBNET:
ds = dev_server.ImageServer(devserver_url)
else:
ds = dev_server.ImageServer.resolve(image_name)
elif info.build is not None:
ds = dev_server.ImageServer.resolve(info.build, hostname)
image_name = info.build
else:
raise error.AutoservError(
'Failed to stage server-side package. The host has '
'no job_repo_url attribute or cros-version label.')
ds.stage_artifacts(image_name, ['autotest_server_package'])
return '%s/static/%s/%s' % (ds.url(), image_name,
'autotest_server_package.tar.bz2')
def repair(self):
"""Attempt to repair a labstation."""
message = 'Beginning repair for host %s board %s model %s'
info = self.host_info_store.get()
message %= (self.hostname, info.board, info.model)
self.record('INFO', None, None, message)
self._repair_strategy.repair(self)
def update_cros_version_label(self):
"""Update cros-version label on labstation"""
image_name = self.get_full_release_path()
if not image_name:
logging.info('Could not get labstation version, it could be'
' the labstation is running a customized image.')
info = self.host_info_store.get()
info.clear_version_labels(version_prefix=self.VERSION_PREFIX)
self.host_info_store.commit(info)
return
afe_utils.add_provision_labels(self, self.VERSION_PREFIX, image_name)
def _validate_uptime(self):
return (float(self.check_uptime()) >
self.UP_TIME_THRESH_HOLD_HOURS * 3600)
def _is_servo_in_use(self):
"""Determine if there are any DUTs currently running task that uses
servo, only files that has been touched within pre-set threshold of
minutes counts.
@returns True if any DUTs is using servos, otherwise False.
"""
cmd = 'find %s*%s -mmin -%s' % (self.TEMP_FILE_DIR,
self.LOCK_FILE_POSTFIX,
self.IN_USE_FILE_EXPIRE_MINS)
result = self.run(cmd, ignore_status=True)
return bool(result.stdout)
def _cleanup_post_reboot(self):
"""Clean up all xxxx_reboot file after reboot."""
cmd = 'rm %s*%s' % (self.TEMP_FILE_DIR, self.REBOOT_FILE_POSTFIX)
self.run(cmd, ignore_status=True)
def rpm_power_on_and_wait(self, _rpm_client=None):
"""Power on a labstation through RPM and wait for it to come up"""
return self.change_rpm_state_and_wait("ON", _rpm_client=_rpm_client)
def rpm_power_off_and_wait(self, _rpm_client=None):
"""Power off a labstation through RPM and wait for it to shut down"""
return self.change_rpm_state_and_wait("OFF", _rpm_client=_rpm_client)
def change_rpm_state_and_wait(self, state, _rpm_client=None):
"""Change the state of a labstation
@param state: on or off
@param _rpm_client: rpm_client module, to support testing
"""
_rpm_client = _rpm_client or rpm_client
wait = {
"ON": self.wait_up,
"OFF": self.wait_down,
}[state]
timeout = {
"ON": self.BOOT_TIMEOUT,
"OFF": self.WAIT_DOWN_REBOOT_TIMEOUT,
}[state]
_rpm_client.set_power(self, state)
if not wait(timeout=timeout):
msg = "%s didn't enter %s state in %s seconds" % (
getattr(self, 'hostname', None),
state,
timeout,
)
raise Exception(msg)