You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1054 lines
44 KiB
1054 lines
44 KiB
# Lint as: python2, python3
|
|
# Copyright (c) 2008 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
import os, time, socket, shutil, glob, logging, tempfile, re
|
|
import shlex
|
|
import subprocess
|
|
|
|
from autotest_lib.client.bin.result_tools import runner as result_tools_runner
|
|
from autotest_lib.client.common_lib import error
|
|
from autotest_lib.client.common_lib import utils
|
|
from autotest_lib.client.common_lib.cros.network import ping_runner
|
|
from autotest_lib.client.common_lib.global_config import global_config
|
|
from autotest_lib.server import utils, autotest
|
|
from autotest_lib.server.hosts import host_info
|
|
from autotest_lib.server.hosts import remote
|
|
from autotest_lib.server.hosts import rpc_server_tracker
|
|
from autotest_lib.server.hosts import ssh_multiplex
|
|
import six
|
|
from six.moves import filter
|
|
|
|
try:
|
|
from chromite.lib import metrics
|
|
except ImportError:
|
|
metrics = utils.metrics_mock
|
|
|
|
# pylint: disable=C0111
|
|
|
|
get_value = global_config.get_config_value
|
|
enable_master_ssh = get_value('AUTOSERV', 'enable_master_ssh', type=bool,
|
|
default=False)
|
|
|
|
# Number of seconds to use the cached up status.
|
|
_DEFAULT_UP_STATUS_EXPIRATION_SECONDS = 300
|
|
_DEFAULT_SSH_PORT = 22
|
|
|
|
# Number of seconds to wait for the host to shut down in wait_down().
|
|
_DEFAULT_WAIT_DOWN_TIME_SECONDS = 120
|
|
|
|
# Number of seconds to wait for the host to boot up in wait_up().
|
|
_DEFAULT_WAIT_UP_TIME_SECONDS = 120
|
|
|
|
# Timeout in seconds for a single call of get_boot_id() in wait_down()
|
|
# and a single ssh ping in wait_up().
|
|
_DEFAULT_MAX_PING_TIMEOUT = 10
|
|
|
|
class AbstractSSHHost(remote.RemoteHost):
|
|
"""
|
|
This class represents a generic implementation of most of the
|
|
framework necessary for controlling a host via ssh. It implements
|
|
almost all of the abstract Host methods, except for the core
|
|
Host.run method.
|
|
"""
|
|
VERSION_PREFIX = ''
|
|
# Timeout for master ssh connection setup, in seconds.
|
|
DEFAULT_START_MASTER_SSH_TIMEOUT_S = 5
|
|
|
|
def _initialize(self, hostname, user="root", port=_DEFAULT_SSH_PORT,
|
|
password="", is_client_install_supported=True,
|
|
afe_host=None, host_info_store=None, connection_pool=None,
|
|
*args, **dargs):
|
|
super(AbstractSSHHost, self)._initialize(hostname=hostname,
|
|
*args, **dargs)
|
|
"""
|
|
@param hostname: The hostname of the host.
|
|
@param user: The username to use when ssh'ing into the host.
|
|
@param password: The password to use when ssh'ing into the host.
|
|
@param port: The port to use for ssh.
|
|
@param is_client_install_supported: Boolean to indicate if we can
|
|
install autotest on the host.
|
|
@param afe_host: The host object attained from the AFE (get_hosts).
|
|
@param host_info_store: Optional host_info.CachingHostInfoStore object
|
|
to obtain / update host information.
|
|
@param connection_pool: ssh_multiplex.ConnectionPool instance to share
|
|
the master ssh connection across control scripts.
|
|
"""
|
|
self._track_class_usage()
|
|
# IP address is retrieved only on demand. Otherwise the host
|
|
# initialization will fail for host is not online.
|
|
self._ip = None
|
|
self.user = user
|
|
self.port = port
|
|
self.password = password
|
|
self._is_client_install_supported = is_client_install_supported
|
|
self._use_rsync = None
|
|
self.known_hosts_file = tempfile.mkstemp()[1]
|
|
self._rpc_server_tracker = rpc_server_tracker.RpcServerTracker(self);
|
|
|
|
"""
|
|
Master SSH connection background job, socket temp directory and socket
|
|
control path option. If master-SSH is enabled, these fields will be
|
|
initialized by start_master_ssh when a new SSH connection is initiated.
|
|
"""
|
|
self._connection_pool = connection_pool
|
|
if connection_pool:
|
|
self._master_ssh = connection_pool.get(hostname, user, port)
|
|
else:
|
|
self._master_ssh = ssh_multiplex.MasterSsh(hostname, user, port)
|
|
|
|
self._afe_host = afe_host or utils.EmptyAFEHost()
|
|
self.host_info_store = (host_info_store or
|
|
host_info.InMemoryHostInfoStore())
|
|
|
|
# The cached status of whether the DUT responded to ping.
|
|
self._cached_up_status = None
|
|
# The timestamp when the value of _cached_up_status is set.
|
|
self._cached_up_status_updated = None
|
|
|
|
|
|
@property
|
|
def ip(self):
|
|
"""@return IP address of the host.
|
|
"""
|
|
if not self._ip:
|
|
self._ip = socket.getaddrinfo(self.hostname, None)[0][4][0]
|
|
return self._ip
|
|
|
|
|
|
@property
|
|
def is_client_install_supported(self):
|
|
""""
|
|
Returns True if the host supports autotest client installs, False
|
|
otherwise.
|
|
"""
|
|
return self._is_client_install_supported
|
|
|
|
|
|
@property
|
|
def rpc_server_tracker(self):
|
|
""""
|
|
@return The RPC server tracker associated with this host.
|
|
"""
|
|
return self._rpc_server_tracker
|
|
|
|
|
|
@property
|
|
def is_default_port(self):
|
|
"""Returns True if its port is default SSH port."""
|
|
return self.port == _DEFAULT_SSH_PORT
|
|
|
|
@property
|
|
def host_port(self):
|
|
"""Returns hostname if port is default. Otherwise, hostname:port.
|
|
"""
|
|
if self.is_default_port:
|
|
return self.hostname
|
|
else:
|
|
return '%s:%d' % (self.hostname, self.port)
|
|
|
|
|
|
# Though it doesn't use self here, it is not declared as staticmethod
|
|
# because its subclass may use self to access member variables.
|
|
def make_ssh_command(self, user="root", port=_DEFAULT_SSH_PORT, opts='',
|
|
hosts_file='/dev/null', connect_timeout=30,
|
|
alive_interval=300, alive_count_max=3,
|
|
connection_attempts=1):
|
|
ssh_options = " ".join([
|
|
opts,
|
|
self.make_ssh_options(
|
|
hosts_file=hosts_file, connect_timeout=connect_timeout,
|
|
alive_interval=alive_interval, alive_count_max=alive_count_max,
|
|
connection_attempts=connection_attempts)])
|
|
return "/usr/bin/ssh -a -x %s -l %s -p %d" % (ssh_options, user, port)
|
|
|
|
|
|
@staticmethod
|
|
def make_ssh_options(hosts_file='/dev/null', connect_timeout=30,
|
|
alive_interval=300, alive_count_max=3,
|
|
connection_attempts=1):
|
|
"""Composes SSH -o options."""
|
|
assert isinstance(connect_timeout, six.integer_types)
|
|
assert connect_timeout > 0 # can't disable the timeout
|
|
|
|
options = [("StrictHostKeyChecking", "no"),
|
|
("UserKnownHostsFile", hosts_file),
|
|
("BatchMode", "yes"),
|
|
("ConnectTimeout", str(connect_timeout)),
|
|
("ServerAliveInterval", str(alive_interval)),
|
|
("ServerAliveCountMax", str(alive_count_max)),
|
|
("ConnectionAttempts", str(connection_attempts))]
|
|
return " ".join("-o %s=%s" % kv for kv in options)
|
|
|
|
|
|
def use_rsync(self):
|
|
if self._use_rsync is not None:
|
|
return self._use_rsync
|
|
|
|
# Check if rsync is available on the remote host. If it's not,
|
|
# don't try to use it for any future file transfers.
|
|
self._use_rsync = self.check_rsync()
|
|
if not self._use_rsync:
|
|
logging.warning("rsync not available on remote host %s -- disabled",
|
|
self.host_port)
|
|
return self._use_rsync
|
|
|
|
|
|
def check_rsync(self):
|
|
"""
|
|
Check if rsync is available on the remote host.
|
|
"""
|
|
try:
|
|
self.run("rsync --version", stdout_tee=None, stderr_tee=None)
|
|
except error.AutoservRunError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _encode_remote_paths(self, paths, escape=True, use_scp=False):
|
|
"""
|
|
Given a list of file paths, encodes it as a single remote path, in
|
|
the style used by rsync and scp.
|
|
escape: add \\ to protect special characters.
|
|
use_scp: encode for scp if true, rsync if false.
|
|
"""
|
|
if escape:
|
|
paths = [utils.scp_remote_escape(path) for path in paths]
|
|
|
|
remote = self.hostname
|
|
|
|
# rsync and scp require IPv6 brackets, even when there isn't any
|
|
# trailing port number (ssh doesn't support IPv6 brackets).
|
|
# In the Python >= 3.3 future, 'import ipaddress' will parse addresses.
|
|
if re.search(r':.*:', remote):
|
|
remote = '[%s]' % remote
|
|
|
|
if use_scp:
|
|
return '%s@%s:"%s"' % (self.user, remote, " ".join(paths))
|
|
else:
|
|
return '%s@%s:%s' % (
|
|
self.user, remote,
|
|
" :".join('"%s"' % p for p in paths))
|
|
|
|
def _encode_local_paths(self, paths, escape=True):
|
|
"""
|
|
Given a list of file paths, encodes it as a single local path.
|
|
escape: add \\ to protect special characters.
|
|
"""
|
|
if escape:
|
|
paths = [utils.sh_escape(path) for path in paths]
|
|
|
|
return " ".join('"%s"' % p for p in paths)
|
|
|
|
|
|
def rsync_options(self, delete_dest=False, preserve_symlinks=False,
|
|
safe_symlinks=False, excludes=None):
|
|
"""Obtains rsync options for the remote."""
|
|
ssh_cmd = self.make_ssh_command(user=self.user, port=self.port,
|
|
opts=self._master_ssh.ssh_option,
|
|
hosts_file=self.known_hosts_file)
|
|
if delete_dest:
|
|
delete_flag = "--delete"
|
|
else:
|
|
delete_flag = ""
|
|
if safe_symlinks:
|
|
symlink_flag = "-l --safe-links"
|
|
elif preserve_symlinks:
|
|
symlink_flag = "-l"
|
|
else:
|
|
symlink_flag = "-L"
|
|
exclude_args = ''
|
|
if excludes:
|
|
exclude_args = ' '.join(
|
|
["--exclude '%s'" % exclude for exclude in excludes])
|
|
return "%s %s --timeout=1800 --rsh='%s' -az --no-o --no-g %s" % (
|
|
symlink_flag, delete_flag, ssh_cmd, exclude_args)
|
|
|
|
|
|
def _make_rsync_cmd(self, sources, dest, delete_dest,
|
|
preserve_symlinks, safe_symlinks, excludes=None):
|
|
"""
|
|
Given a string of source paths and a destination path, produces the
|
|
appropriate rsync command for copying them. Remote paths must be
|
|
pre-encoded.
|
|
"""
|
|
rsync_options = self.rsync_options(
|
|
delete_dest=delete_dest, preserve_symlinks=preserve_symlinks,
|
|
safe_symlinks=safe_symlinks, excludes=excludes)
|
|
return 'rsync %s %s "%s"' % (rsync_options, sources, dest)
|
|
|
|
|
|
def _make_ssh_cmd(self, cmd):
|
|
"""
|
|
Create a base ssh command string for the host which can be used
|
|
to run commands directly on the machine
|
|
"""
|
|
base_cmd = self.make_ssh_command(user=self.user, port=self.port,
|
|
opts=self._master_ssh.ssh_option,
|
|
hosts_file=self.known_hosts_file)
|
|
|
|
return '%s %s "%s"' % (base_cmd, self.hostname, utils.sh_escape(cmd))
|
|
|
|
def _make_scp_cmd(self, sources, dest):
|
|
"""
|
|
Given a string of source paths and a destination path, produces the
|
|
appropriate scp command for encoding it. Remote paths must be
|
|
pre-encoded.
|
|
"""
|
|
command = ("scp -rq %s -o StrictHostKeyChecking=no "
|
|
"-o UserKnownHostsFile=%s -P %d %s '%s'")
|
|
return command % (self._master_ssh.ssh_option, self.known_hosts_file,
|
|
self.port, sources, dest)
|
|
|
|
|
|
def _make_rsync_compatible_globs(self, path, is_local):
|
|
"""
|
|
Given an rsync-style path, returns a list of globbed paths
|
|
that will hopefully provide equivalent behaviour for scp. Does not
|
|
support the full range of rsync pattern matching behaviour, only that
|
|
exposed in the get/send_file interface (trailing slashes).
|
|
|
|
The is_local param is flag indicating if the paths should be
|
|
interpreted as local or remote paths.
|
|
"""
|
|
|
|
# non-trailing slash paths should just work
|
|
if len(path) == 0 or path[-1] != "/":
|
|
return [path]
|
|
|
|
# make a function to test if a pattern matches any files
|
|
if is_local:
|
|
def glob_matches_files(path, pattern):
|
|
return len(glob.glob(path + pattern)) > 0
|
|
else:
|
|
def glob_matches_files(path, pattern):
|
|
result = self.run("ls \"%s\"%s" % (utils.sh_escape(path),
|
|
pattern),
|
|
stdout_tee=None, ignore_status=True)
|
|
return result.exit_status == 0
|
|
|
|
# take a set of globs that cover all files, and see which are needed
|
|
patterns = ["*", ".[!.]*"]
|
|
patterns = [p for p in patterns if glob_matches_files(path, p)]
|
|
|
|
# convert them into a set of paths suitable for the commandline
|
|
if is_local:
|
|
return ["\"%s\"%s" % (utils.sh_escape(path), pattern)
|
|
for pattern in patterns]
|
|
else:
|
|
return [utils.scp_remote_escape(path) + pattern
|
|
for pattern in patterns]
|
|
|
|
|
|
def _make_rsync_compatible_source(self, source, is_local):
|
|
"""
|
|
Applies the same logic as _make_rsync_compatible_globs, but
|
|
applies it to an entire list of sources, producing a new list of
|
|
sources, properly quoted.
|
|
"""
|
|
return sum((self._make_rsync_compatible_globs(path, is_local)
|
|
for path in source), [])
|
|
|
|
|
|
def _set_umask_perms(self, dest):
|
|
"""
|
|
Given a destination file/dir (recursively) set the permissions on
|
|
all the files and directories to the max allowed by running umask.
|
|
"""
|
|
|
|
# now this looks strange but I haven't found a way in Python to _just_
|
|
# get the umask, apparently the only option is to try to set it
|
|
umask = os.umask(0)
|
|
os.umask(umask)
|
|
|
|
max_privs = 0o777 & ~umask
|
|
|
|
def set_file_privs(filename):
|
|
"""Sets mode of |filename|. Assumes |filename| exists."""
|
|
file_stat = os.stat(filename)
|
|
|
|
file_privs = max_privs
|
|
# if the original file permissions do not have at least one
|
|
# executable bit then do not set it anywhere
|
|
if not file_stat.st_mode & 0o111:
|
|
file_privs &= ~0o111
|
|
|
|
os.chmod(filename, file_privs)
|
|
|
|
# try a bottom-up walk so changes on directory permissions won't cut
|
|
# our access to the files/directories inside it
|
|
for root, dirs, files in os.walk(dest, topdown=False):
|
|
# when setting the privileges we emulate the chmod "X" behaviour
|
|
# that sets to execute only if it is a directory or any of the
|
|
# owner/group/other already has execute right
|
|
for dirname in dirs:
|
|
os.chmod(os.path.join(root, dirname), max_privs)
|
|
|
|
# Filter out broken symlinks as we go.
|
|
for filename in filter(os.path.exists, files):
|
|
set_file_privs(os.path.join(root, filename))
|
|
|
|
|
|
# now set privs for the dest itself
|
|
if os.path.isdir(dest):
|
|
os.chmod(dest, max_privs)
|
|
else:
|
|
set_file_privs(dest)
|
|
|
|
|
|
def get_file(self, source, dest, delete_dest=False, preserve_perm=True,
|
|
preserve_symlinks=False, retry=True, safe_symlinks=False,
|
|
try_rsync=True):
|
|
"""
|
|
Copy files from the remote host to a local path.
|
|
|
|
Directories will be copied recursively.
|
|
If a source component is a directory with a trailing slash,
|
|
the content of the directory will be copied, otherwise, the
|
|
directory itself and its content will be copied. This
|
|
behavior is similar to that of the program 'rsync'.
|
|
|
|
Args:
|
|
source: either
|
|
1) a single file or directory, as a string
|
|
2) a list of one or more (possibly mixed)
|
|
files or directories
|
|
dest: a file or a directory (if source contains a
|
|
directory or more than one element, you must
|
|
supply a directory dest)
|
|
delete_dest: if this is true, the command will also clear
|
|
out any old files at dest that are not in the
|
|
source
|
|
preserve_perm: tells get_file() to try to preserve the sources
|
|
permissions on files and dirs
|
|
preserve_symlinks: try to preserve symlinks instead of
|
|
transforming them into files/dirs on copy
|
|
safe_symlinks: same as preserve_symlinks, but discard links
|
|
that may point outside the copied tree
|
|
try_rsync: set to False to skip directly to using scp
|
|
Raises:
|
|
AutoservRunError: the scp command failed
|
|
"""
|
|
logging.debug('get_file. source: %s, dest: %s, delete_dest: %s,'
|
|
'preserve_perm: %s, preserve_symlinks:%s', source, dest,
|
|
delete_dest, preserve_perm, preserve_symlinks)
|
|
|
|
# Start a master SSH connection if necessary.
|
|
self.start_master_ssh()
|
|
|
|
if isinstance(source, six.string_types):
|
|
source = [source]
|
|
dest = os.path.abspath(dest)
|
|
|
|
# If rsync is disabled or fails, try scp.
|
|
try_scp = True
|
|
if try_rsync and self.use_rsync():
|
|
logging.debug('Using Rsync.')
|
|
try:
|
|
remote_source = self._encode_remote_paths(source)
|
|
local_dest = utils.sh_escape(dest)
|
|
rsync = self._make_rsync_cmd(remote_source, local_dest,
|
|
delete_dest, preserve_symlinks,
|
|
safe_symlinks)
|
|
utils.run(rsync)
|
|
try_scp = False
|
|
except error.CmdError as e:
|
|
# retry on rsync exit values which may be caused by transient
|
|
# network problems:
|
|
#
|
|
# rc 10: Error in socket I/O
|
|
# rc 12: Error in rsync protocol data stream
|
|
# rc 23: Partial transfer due to error
|
|
# rc 255: Ssh error
|
|
#
|
|
# Note that rc 23 includes dangling symlinks. In this case
|
|
# retrying is useless, but not very damaging since rsync checks
|
|
# for those before starting the transfer (scp does not).
|
|
status = e.result_obj.exit_status
|
|
if status in [10, 12, 23, 255] and retry:
|
|
logging.warning('rsync status %d, retrying', status)
|
|
self.get_file(source, dest, delete_dest, preserve_perm,
|
|
preserve_symlinks, retry=False)
|
|
# The nested get_file() does all that's needed.
|
|
return
|
|
else:
|
|
logging.warning("trying scp, rsync failed: %s (%d)",
|
|
e, status)
|
|
|
|
if try_scp:
|
|
logging.debug('Trying scp.')
|
|
# scp has no equivalent to --delete, just drop the entire dest dir
|
|
if delete_dest and os.path.isdir(dest):
|
|
shutil.rmtree(dest)
|
|
os.mkdir(dest)
|
|
|
|
remote_source = self._make_rsync_compatible_source(source, False)
|
|
if remote_source:
|
|
# _make_rsync_compatible_source() already did the escaping
|
|
remote_source = self._encode_remote_paths(
|
|
remote_source, escape=False, use_scp=True)
|
|
local_dest = utils.sh_escape(dest)
|
|
scp = self._make_scp_cmd(remote_source, local_dest)
|
|
try:
|
|
utils.run(scp)
|
|
except error.CmdError as e:
|
|
logging.debug('scp failed: %s', e)
|
|
raise error.AutoservRunError(e.args[0], e.args[1])
|
|
|
|
if not preserve_perm:
|
|
# we have no way to tell scp to not try to preserve the
|
|
# permissions so set them after copy instead.
|
|
# for rsync we could use "--no-p --chmod=ugo=rwX" but those
|
|
# options are only in very recent rsync versions
|
|
self._set_umask_perms(dest)
|
|
|
|
|
|
def send_file(self, source, dest, delete_dest=False,
|
|
preserve_symlinks=False, excludes=None):
|
|
"""
|
|
Copy files from a local path to the remote host.
|
|
|
|
Directories will be copied recursively.
|
|
If a source component is a directory with a trailing slash,
|
|
the content of the directory will be copied, otherwise, the
|
|
directory itself and its content will be copied. This
|
|
behavior is similar to that of the program 'rsync'.
|
|
|
|
Args:
|
|
source: either
|
|
1) a single file or directory, as a string
|
|
2) a list of one or more (possibly mixed)
|
|
files or directories
|
|
dest: a file or a directory (if source contains a
|
|
directory or more than one element, you must
|
|
supply a directory dest)
|
|
delete_dest: if this is true, the command will also clear
|
|
out any old files at dest that are not in the
|
|
source
|
|
preserve_symlinks: controls if symlinks on the source will be
|
|
copied as such on the destination or transformed into the
|
|
referenced file/directory
|
|
excludes: A list of file pattern that matches files not to be
|
|
sent. `send_file` will fail if exclude is set, since
|
|
local copy does not support --exclude, e.g., when
|
|
using scp to copy file.
|
|
|
|
Raises:
|
|
AutoservRunError: the scp command failed
|
|
"""
|
|
logging.debug('send_file. source: %s, dest: %s, delete_dest: %s,'
|
|
'preserve_symlinks:%s', source, dest,
|
|
delete_dest, preserve_symlinks)
|
|
# Start a master SSH connection if necessary.
|
|
self.start_master_ssh()
|
|
|
|
if isinstance(source, six.string_types):
|
|
source = [source]
|
|
|
|
local_sources = self._encode_local_paths(source)
|
|
if not local_sources:
|
|
raise error.TestError('source |%s| yielded an empty string' % (
|
|
source))
|
|
if local_sources.find('\x00') != -1:
|
|
raise error.TestError('one or more sources include NUL char')
|
|
|
|
# If rsync is disabled or fails, try scp.
|
|
try_scp = True
|
|
if self.use_rsync():
|
|
logging.debug('Using Rsync.')
|
|
remote_dest = self._encode_remote_paths([dest])
|
|
try:
|
|
rsync = self._make_rsync_cmd(local_sources, remote_dest,
|
|
delete_dest, preserve_symlinks,
|
|
False, excludes=excludes)
|
|
utils.run(rsync)
|
|
try_scp = False
|
|
except error.CmdError as e:
|
|
logging.warning("trying scp, rsync failed: %s", e)
|
|
|
|
if try_scp:
|
|
logging.debug('Trying scp.')
|
|
if excludes:
|
|
raise error.AutotestHostRunError(
|
|
'--exclude is not supported in scp, try to use rsync. '
|
|
'excludes: %s' % ','.join(excludes), None)
|
|
# scp has no equivalent to --delete, just drop the entire dest dir
|
|
if delete_dest:
|
|
is_dir = self.run("ls -d %s/" % dest,
|
|
ignore_status=True).exit_status == 0
|
|
if is_dir:
|
|
cmd = "rm -rf %s && mkdir %s"
|
|
cmd %= (dest, dest)
|
|
self.run(cmd)
|
|
|
|
remote_dest = self._encode_remote_paths([dest], use_scp=True)
|
|
local_sources = self._make_rsync_compatible_source(source, True)
|
|
if local_sources:
|
|
sources = self._encode_local_paths(local_sources, escape=False)
|
|
scp = self._make_scp_cmd(sources, remote_dest)
|
|
try:
|
|
utils.run(scp)
|
|
except error.CmdError as e:
|
|
logging.debug('scp failed: %s', e)
|
|
raise error.AutoservRunError(e.args[0], e.args[1])
|
|
else:
|
|
logging.debug('skipping scp for empty source list')
|
|
|
|
def verify_ssh_user_access(self):
|
|
"""Verify ssh access to this host.
|
|
|
|
@returns False if ssh_ping fails due to Permissions error, True
|
|
otherwise.
|
|
"""
|
|
try:
|
|
self.ssh_ping()
|
|
except (error.AutoservSshPermissionDeniedError,
|
|
error.AutoservSshPingHostError):
|
|
return False
|
|
return True
|
|
|
|
|
|
def ssh_ping(self, timeout=60, connect_timeout=None, base_cmd='true'):
|
|
"""
|
|
Pings remote host via ssh.
|
|
|
|
@param timeout: Command execution timeout in seconds.
|
|
Defaults to 60 seconds.
|
|
@param connect_timeout: ssh connection timeout in seconds.
|
|
@param base_cmd: The base command to run with the ssh ping.
|
|
Defaults to true.
|
|
@raise AutoservSSHTimeout: If the ssh ping times out.
|
|
@raise AutoservSshPermissionDeniedError: If ssh ping fails due to
|
|
permissions.
|
|
@raise AutoservSshPingHostError: For other AutoservRunErrors.
|
|
"""
|
|
ctimeout = min(timeout, connect_timeout or timeout)
|
|
try:
|
|
self.run(base_cmd, timeout=timeout, connect_timeout=ctimeout,
|
|
ssh_failure_retry_ok=True)
|
|
except error.AutoservSSHTimeout:
|
|
msg = "Host (ssh) verify timed out (timeout = %d)" % timeout
|
|
raise error.AutoservSSHTimeout(msg)
|
|
except error.AutoservSshPermissionDeniedError:
|
|
#let AutoservSshPermissionDeniedError be visible to the callers
|
|
raise
|
|
except error.AutoservRunError as e:
|
|
# convert the generic AutoservRunError into something more
|
|
# specific for this context
|
|
raise error.AutoservSshPingHostError(e.description + '\n' +
|
|
repr(e.result_obj))
|
|
|
|
|
|
def is_up(self, timeout=60, connect_timeout=None, base_cmd='true'):
|
|
"""
|
|
Check if the remote host is up by ssh-ing and running a base command.
|
|
|
|
@param timeout: command execution timeout in seconds.
|
|
@param connect_timeout: ssh connection timeout in seconds.
|
|
@param base_cmd: a base command to run with ssh. The default is 'true'.
|
|
@returns True if the remote host is up before the timeout expires,
|
|
False otherwise.
|
|
"""
|
|
try:
|
|
self.ssh_ping(timeout=timeout,
|
|
connect_timeout=connect_timeout,
|
|
base_cmd=base_cmd)
|
|
except error.AutoservError:
|
|
return False
|
|
else:
|
|
return True
|
|
|
|
|
|
def is_up_fast(self, count=1):
|
|
"""Return True if the host can be pinged.
|
|
|
|
@param count How many time try to ping before decide that host is not
|
|
reachable by ping.
|
|
"""
|
|
ping_config = ping_runner.PingConfig(self.hostname,
|
|
count=count,
|
|
ignore_result=True,
|
|
ignore_status=True)
|
|
return ping_runner.PingRunner().ping(ping_config).received > 0
|
|
|
|
|
|
def wait_up(self, timeout=_DEFAULT_WAIT_UP_TIME_SECONDS):
|
|
"""
|
|
Wait until the remote host is up or the timeout expires.
|
|
|
|
In fact, it will wait until an ssh connection to the remote
|
|
host can be established, and getty is running.
|
|
|
|
@param timeout time limit in seconds before returning even
|
|
if the host is not up.
|
|
|
|
@returns True if the host was found to be up before the timeout expires,
|
|
False otherwise
|
|
"""
|
|
current_time = int(time.time())
|
|
end_time = current_time + timeout
|
|
|
|
autoserv_error_logged = False
|
|
while current_time < end_time:
|
|
ping_timeout = min(_DEFAULT_MAX_PING_TIMEOUT,
|
|
end_time - current_time)
|
|
if self.is_up(timeout=ping_timeout, connect_timeout=ping_timeout):
|
|
try:
|
|
if self.are_wait_up_processes_up():
|
|
logging.debug('Host %s is now up', self.host_port)
|
|
return True
|
|
except error.AutoservError as e:
|
|
if not autoserv_error_logged:
|
|
logging.debug('Ignoring failure to reach %s: %s %s',
|
|
self.host_port, e,
|
|
'(and further similar failures)')
|
|
autoserv_error_logged = True
|
|
time.sleep(1)
|
|
current_time = int(time.time())
|
|
|
|
logging.debug('Host %s is still down after waiting %d seconds',
|
|
self.host_port, int(timeout + time.time() - end_time))
|
|
return False
|
|
|
|
|
|
def wait_down(self, timeout=_DEFAULT_WAIT_DOWN_TIME_SECONDS,
|
|
warning_timer=None, old_boot_id=None,
|
|
max_ping_timeout=_DEFAULT_MAX_PING_TIMEOUT):
|
|
"""
|
|
Wait until the remote host is down or the timeout expires.
|
|
|
|
If old_boot_id is provided, waits until either the machine is
|
|
unpingable or self.get_boot_id() returns a value different from
|
|
old_boot_id. If the boot_id value has changed then the function
|
|
returns True under the assumption that the machine has shut down
|
|
and has now already come back up.
|
|
|
|
If old_boot_id is None then until the machine becomes unreachable the
|
|
method assumes the machine has not yet shut down.
|
|
|
|
@param timeout Time limit in seconds before returning even if the host
|
|
is still up.
|
|
@param warning_timer Time limit in seconds that will generate a warning
|
|
if the host is not down yet. Can be None for no warning.
|
|
@param old_boot_id A string containing the result of self.get_boot_id()
|
|
prior to the host being told to shut down. Can be None if this is
|
|
not available.
|
|
@param max_ping_timeout Maximum timeout in seconds for each
|
|
self.get_boot_id() call. If this timeout is hit, it is assumed that
|
|
the host went down and became unreachable.
|
|
|
|
@returns True if the host was found to be down (max_ping_timeout timeout
|
|
expired or boot_id changed if provided) and False if timeout
|
|
expired.
|
|
"""
|
|
#TODO: there is currently no way to distinguish between knowing
|
|
#TODO: boot_id was unsupported and not knowing the boot_id.
|
|
current_time = int(time.time())
|
|
end_time = current_time + timeout
|
|
|
|
if warning_timer:
|
|
warn_time = current_time + warning_timer
|
|
|
|
if old_boot_id is not None:
|
|
logging.debug('Host %s pre-shutdown boot_id is %s',
|
|
self.host_port, old_boot_id)
|
|
|
|
# Impose semi real-time deadline constraints, since some clients
|
|
# (eg: watchdog timer tests) expect strict checking of time elapsed.
|
|
# Each iteration of this loop is treated as though it atomically
|
|
# completes within current_time, this is needed because if we used
|
|
# inline time.time() calls instead then the following could happen:
|
|
#
|
|
# while time.time() < end_time: [23 < 30]
|
|
# some code. [takes 10 secs]
|
|
# try:
|
|
# new_boot_id = self.get_boot_id(timeout=end_time - time.time())
|
|
# [30 - 33]
|
|
# The last step will lead to a return True, when in fact the machine
|
|
# went down at 32 seconds (>30). Hence we need to pass get_boot_id
|
|
# the same time that allowed us into that iteration of the loop.
|
|
while current_time < end_time:
|
|
ping_timeout = min(end_time - current_time, max_ping_timeout)
|
|
try:
|
|
new_boot_id = self.get_boot_id(timeout=ping_timeout)
|
|
except error.AutoservError:
|
|
logging.debug('Host %s is now unreachable over ssh, is down',
|
|
self.host_port)
|
|
return True
|
|
else:
|
|
# if the machine is up but the boot_id value has changed from
|
|
# old boot id, then we can assume the machine has gone down
|
|
# and then already come back up
|
|
if old_boot_id is not None and old_boot_id != new_boot_id:
|
|
logging.debug('Host %s now has boot_id %s and so must '
|
|
'have rebooted', self.host_port, new_boot_id)
|
|
return True
|
|
|
|
if warning_timer and current_time > warn_time:
|
|
self.record("INFO", None, "shutdown",
|
|
"Shutdown took longer than %ds" % warning_timer)
|
|
# Print the warning only once.
|
|
warning_timer = None
|
|
# If a machine is stuck switching runlevels
|
|
# This may cause the machine to reboot.
|
|
self.run('kill -HUP 1', ignore_status=True)
|
|
|
|
time.sleep(1)
|
|
current_time = int(time.time())
|
|
|
|
return False
|
|
|
|
|
|
# tunable constants for the verify & repair code
|
|
AUTOTEST_GB_DISKSPACE_REQUIRED = get_value("SERVER",
|
|
"gb_diskspace_required",
|
|
type=float,
|
|
default=20.0)
|
|
|
|
|
|
def verify_connectivity(self):
|
|
super(AbstractSSHHost, self).verify_connectivity()
|
|
|
|
logging.info('Pinging host ' + self.host_port)
|
|
self.ssh_ping()
|
|
logging.info("Host (ssh) %s is alive", self.host_port)
|
|
|
|
if self.is_shutting_down():
|
|
raise error.AutoservHostIsShuttingDownError("Host is shutting down")
|
|
|
|
|
|
def verify_software(self):
|
|
super(AbstractSSHHost, self).verify_software()
|
|
try:
|
|
self.check_diskspace(autotest.Autotest.get_install_dir(self),
|
|
self.AUTOTEST_GB_DISKSPACE_REQUIRED)
|
|
except error.AutoservDiskFullHostError:
|
|
# only want to raise if it's a space issue
|
|
raise
|
|
except (error.AutoservHostError, autotest.AutodirNotFoundError):
|
|
logging.exception('autodir space check exception, this is probably '
|
|
'safe to ignore\n')
|
|
|
|
|
|
def close(self):
|
|
super(AbstractSSHHost, self).close()
|
|
self.rpc_server_tracker.disconnect_all()
|
|
if not self._connection_pool:
|
|
self._master_ssh.close()
|
|
if os.path.exists(self.known_hosts_file):
|
|
os.remove(self.known_hosts_file)
|
|
|
|
|
|
def restart_master_ssh(self):
|
|
"""
|
|
Stop and restart the ssh master connection. This is meant as a last
|
|
resort when ssh commands fail and we don't understand why.
|
|
"""
|
|
logging.debug('Restarting master ssh connection')
|
|
self._master_ssh.close()
|
|
self._master_ssh.maybe_start(timeout=30)
|
|
|
|
|
|
|
|
def start_master_ssh(self, timeout=DEFAULT_START_MASTER_SSH_TIMEOUT_S):
|
|
"""
|
|
Called whenever a slave SSH connection needs to be initiated (e.g., by
|
|
run, rsync, scp). If master SSH support is enabled and a master SSH
|
|
connection is not active already, start a new one in the background.
|
|
Also, cleanup any zombie master SSH connections (e.g., dead due to
|
|
reboot).
|
|
|
|
timeout: timeout in seconds (default 5) to wait for master ssh
|
|
connection to be established. If timeout is reached, a
|
|
warning message is logged, but no other action is taken.
|
|
"""
|
|
if not enable_master_ssh:
|
|
return
|
|
self._master_ssh.maybe_start(timeout=timeout)
|
|
|
|
|
|
def clear_known_hosts(self):
|
|
"""Clears out the temporary ssh known_hosts file.
|
|
|
|
This is useful if the test SSHes to the machine, then reinstalls it,
|
|
then SSHes to it again. It can be called after the reinstall to
|
|
reduce the spam in the logs.
|
|
"""
|
|
logging.info("Clearing known hosts for host '%s', file '%s'.",
|
|
self.host_port, self.known_hosts_file)
|
|
# Clear out the file by opening it for writing and then closing.
|
|
fh = open(self.known_hosts_file, "w")
|
|
fh.close()
|
|
|
|
|
|
def collect_logs(self, remote_src_dir, local_dest_dir, ignore_errors=True):
|
|
"""Copy log directories from a host to a local directory.
|
|
|
|
@param remote_src_dir: A destination directory on the host.
|
|
@param local_dest_dir: A path to a local destination directory.
|
|
If it doesn't exist it will be created.
|
|
@param ignore_errors: If True, ignore exceptions.
|
|
|
|
@raises OSError: If there were problems creating the local_dest_dir and
|
|
ignore_errors is False.
|
|
@raises AutoservRunError, AutotestRunError: If something goes wrong
|
|
while copying the directories and ignore_errors is False.
|
|
"""
|
|
if not self.check_cached_up_status():
|
|
logging.warning('Host %s did not answer to ping, skip collecting '
|
|
'logs.', self.host_port)
|
|
return
|
|
|
|
locally_created_dest = False
|
|
if (not os.path.exists(local_dest_dir)
|
|
or not os.path.isdir(local_dest_dir)):
|
|
try:
|
|
os.makedirs(local_dest_dir)
|
|
locally_created_dest = True
|
|
except OSError as e:
|
|
logging.warning('Unable to collect logs from host '
|
|
'%s: %s', self.host_port, e)
|
|
if not ignore_errors:
|
|
raise
|
|
return
|
|
|
|
# Build test result directory summary
|
|
try:
|
|
result_tools_runner.run_on_client(self, remote_src_dir)
|
|
except (error.AutotestRunError, error.AutoservRunError,
|
|
error.AutoservSSHTimeout) as e:
|
|
logging.exception(
|
|
'Non-critical failure: Failed to collect and throttle '
|
|
'results at %s from host %s', remote_src_dir,
|
|
self.host_port)
|
|
|
|
try:
|
|
self.get_file(remote_src_dir, local_dest_dir, safe_symlinks=True)
|
|
except (error.AutotestRunError, error.AutoservRunError,
|
|
error.AutoservSSHTimeout) as e:
|
|
logging.warning('Collection of %s to local dir %s from host %s '
|
|
'failed: %s', remote_src_dir, local_dest_dir,
|
|
self.host_port, e)
|
|
if locally_created_dest:
|
|
shutil.rmtree(local_dest_dir, ignore_errors=ignore_errors)
|
|
if not ignore_errors:
|
|
raise
|
|
|
|
# Clean up directory summary file on the client side.
|
|
try:
|
|
result_tools_runner.run_on_client(self, remote_src_dir,
|
|
cleanup_only=True)
|
|
except (error.AutotestRunError, error.AutoservRunError,
|
|
error.AutoservSSHTimeout) as e:
|
|
logging.exception(
|
|
'Non-critical failure: Failed to cleanup result summary '
|
|
'files at %s in host %s', remote_src_dir, self.hostname)
|
|
|
|
|
|
def create_ssh_tunnel(self, port, local_port):
|
|
"""Create an ssh tunnel from local_port to port.
|
|
|
|
This is used to forward a port securely through a tunnel process from
|
|
the server to the DUT for RPC server connection.
|
|
|
|
@param port: remote port on the host.
|
|
@param local_port: local forwarding port.
|
|
|
|
@return: the tunnel process.
|
|
"""
|
|
tunnel_options = '-n -N -q -L %d:localhost:%d' % (local_port, port)
|
|
ssh_cmd = self.make_ssh_command(opts=tunnel_options, port=self.port)
|
|
tunnel_cmd = '%s %s' % (ssh_cmd, self.hostname)
|
|
logging.debug('Full tunnel command: %s', tunnel_cmd)
|
|
# Exec the ssh process directly here rather than using a shell.
|
|
# Using a shell leaves a dangling ssh process, because we deliver
|
|
# signals to the shell wrapping ssh, not the ssh process itself.
|
|
args = shlex.split(tunnel_cmd)
|
|
with open('/dev/null', 'w') as devnull:
|
|
tunnel_proc = subprocess.Popen(args, stdout=devnull, stderr=devnull,
|
|
close_fds=True)
|
|
logging.debug('Started ssh tunnel, local = %d'
|
|
' remote = %d, pid = %d',
|
|
local_port, port, tunnel_proc.pid)
|
|
return tunnel_proc
|
|
|
|
|
|
def disconnect_ssh_tunnel(self, tunnel_proc):
|
|
"""
|
|
Disconnects a previously forwarded port from the server to the DUT for
|
|
RPC server connection.
|
|
|
|
@param tunnel_proc: a tunnel process returned from |create_ssh_tunnel|.
|
|
"""
|
|
if tunnel_proc.poll() is None:
|
|
tunnel_proc.terminate()
|
|
logging.debug('Terminated tunnel, pid %d', tunnel_proc.pid)
|
|
else:
|
|
logging.debug('Tunnel pid %d terminated early, status %d',
|
|
tunnel_proc.pid, tunnel_proc.returncode)
|
|
|
|
|
|
def get_os_type(self):
|
|
"""Returns the host OS descriptor (to be implemented in subclasses).
|
|
|
|
@return A string describing the OS type.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
|
|
def check_cached_up_status(
|
|
self, expiration_seconds=_DEFAULT_UP_STATUS_EXPIRATION_SECONDS):
|
|
"""Check if the DUT responded to ping in the past `expiration_seconds`.
|
|
|
|
@param expiration_seconds: The number of seconds to keep the cached
|
|
status of whether the DUT responded to ping.
|
|
@return: True if the DUT has responded to ping during the past
|
|
`expiration_seconds`.
|
|
"""
|
|
# Refresh the up status if any of following conditions is true:
|
|
# * cached status is never set
|
|
# * cached status is False, so the method can check if the host is up
|
|
# again.
|
|
# * If the cached status is older than `expiration_seconds`
|
|
expire_time = time.time() - expiration_seconds
|
|
if (self._cached_up_status_updated is None or
|
|
not self._cached_up_status or
|
|
self._cached_up_status_updated < expire_time):
|
|
self._cached_up_status = self.is_up_fast()
|
|
self._cached_up_status_updated = time.time()
|
|
return self._cached_up_status
|
|
|
|
|
|
def _track_class_usage(self):
|
|
"""Tracking which class was used.
|
|
|
|
The idea to identify unused classes to be able clean them up.
|
|
We skip names with dynamic created classes where the name is
|
|
hostname of the device.
|
|
"""
|
|
class_name = None
|
|
if 'chrome' not in self.__class__.__name__:
|
|
class_name = self.__class__.__name__
|
|
else:
|
|
for base in self.__class__.__bases__:
|
|
if 'chrome' not in base.__name__:
|
|
class_name = base.__name__
|
|
break
|
|
if class_name:
|
|
data = {'host_class': class_name}
|
|
metrics.Counter(
|
|
'chromeos/autotest/used_hosts').increment(fields=data)
|
|
|
|
def is_file_exists(self, file_path):
|
|
"""Check whether a given file is exist on the host.
|
|
"""
|
|
result = self.run('test -f ' + file_path,
|
|
timeout=30,
|
|
ignore_status=True)
|
|
return result.exit_status == 0
|