You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
338 lines
9.6 KiB
338 lines
9.6 KiB
# Lint as: python2, python3
|
|
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
|
|
#pylint: disable-msg=C0111
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
import six
|
|
from six.moves import filter
|
|
|
|
def order_by_complexity(host_spec_list):
|
|
"""
|
|
Returns a new list of HostSpecs, ordered from most to least complex.
|
|
|
|
Currently, 'complex' means that the spec contains more labels.
|
|
We may want to get smarter about this.
|
|
|
|
@param host_spec_list: a list of HostSpec objects.
|
|
@return a new list of HostSpec, ordered from most to least complex.
|
|
"""
|
|
def extract_label_list_len(host_spec):
|
|
return len(host_spec.labels)
|
|
return sorted(host_spec_list, key=extract_label_list_len, reverse=True)
|
|
|
|
|
|
def is_simple_list(host_spec_list):
|
|
"""
|
|
Returns true if this is a 'simple' list of HostSpec objects.
|
|
|
|
A 'simple' list of HostSpec objects is defined as a list of one HostSpec.
|
|
|
|
@param host_spec_list: a list of HostSpec objects.
|
|
@return True if this is a list of size 1, False otherwise.
|
|
"""
|
|
return len(host_spec_list) == 1
|
|
|
|
|
|
def simple_get_spec_and_hosts(host_specs, hosts_per_spec):
|
|
"""Given a simple list of HostSpec, extract hosts from hosts_per_spec.
|
|
|
|
Given a simple list of HostSpec objects, pull out the spec and use it to
|
|
get the associated hosts out of hosts_per_spec. Return the spec and the
|
|
host list as a pair.
|
|
|
|
@param host_specs: an iterable of HostSpec objects.
|
|
@param hosts_per_spec: map of {HostSpec: [list, of, hosts]}
|
|
@return (HostSpec, [list, of, hosts]}
|
|
"""
|
|
spec = host_specs.pop()
|
|
return spec, hosts_per_spec[spec]
|
|
|
|
|
|
class HostGroup(object):
|
|
"""A high-level specification of a group of hosts.
|
|
|
|
A HostGroup represents a group of hosts against which a job can be
|
|
scheduled. An instance is capable of returning arguments that can specify
|
|
this group in a call to AFE.create_job().
|
|
"""
|
|
def __init__(self):
|
|
pass
|
|
|
|
|
|
def as_args(self):
|
|
"""Return args suitable for passing to AFE.create_job()."""
|
|
raise NotImplementedError()
|
|
|
|
|
|
def size(self):
|
|
"""Returns the number of hosts specified by the group."""
|
|
raise NotImplementedError()
|
|
|
|
|
|
def mark_host_success(self, hostname):
|
|
"""Marks the provided host as successfully reimaged.
|
|
|
|
@param hostname: the name of the host that was reimaged.
|
|
"""
|
|
raise NotImplementedError()
|
|
|
|
|
|
def enough_hosts_succeeded(self):
|
|
"""Returns True if enough hosts in the group were reimaged for use."""
|
|
raise NotImplementedError()
|
|
|
|
|
|
#pylint: disable-msg=C0111
|
|
@property
|
|
def unsatisfied_specs(self):
|
|
return []
|
|
|
|
|
|
#pylint: disable-msg=C0111
|
|
@property
|
|
def doomed_specs(self):
|
|
return []
|
|
|
|
|
|
class ExplicitHostGroup(HostGroup):
|
|
"""A group of hosts, specified by name, to be reimaged for use.
|
|
|
|
@var _hostname_data_dict: {hostname: HostData()}.
|
|
"""
|
|
|
|
class HostData(object):
|
|
"""A HostSpec of a given host, and whether it reimaged successfully."""
|
|
def __init__(self, spec):
|
|
self.spec = spec
|
|
self.image_success = False
|
|
|
|
|
|
def __init__(self, hosts_per_spec={}):
|
|
"""Constructor.
|
|
|
|
@param hosts_per_spec: {HostSpec: [list, of, hosts]}.
|
|
Each host can appear only once.
|
|
"""
|
|
self._hostname_data_dict = {}
|
|
self._potentially_unsatisfied_specs = []
|
|
for spec, host_list in six.iteritems(hosts_per_spec):
|
|
for host in host_list:
|
|
self.add_host_for_spec(spec, host)
|
|
|
|
|
|
def _get_host_datas(self):
|
|
return six.itervalues(self._hostname_data_dict)
|
|
|
|
|
|
def as_args(self):
|
|
return {'hosts': list(self._hostname_data_dict.keys())}
|
|
|
|
|
|
def size(self):
|
|
return len(self._hostname_data_dict)
|
|
|
|
|
|
def mark_host_success(self, hostname):
|
|
self._hostname_data_dict[hostname].image_success = True
|
|
|
|
|
|
def enough_hosts_succeeded(self):
|
|
"""If _any_ hosts were reimaged, that's enough."""
|
|
return True in [d.image_success for d in self._get_host_datas()]
|
|
|
|
|
|
def add_host_for_spec(self, spec, host):
|
|
"""Add a new host for the given HostSpec to the group.
|
|
|
|
@param spec: HostSpec to associate host with.
|
|
@param host: a Host object; each host can appear only once.
|
|
If None, this spec will be relegated to the list of
|
|
potentially unsatisfied specs.
|
|
"""
|
|
if not host:
|
|
if spec not in [d.spec for d in self._get_host_datas()]:
|
|
self._potentially_unsatisfied_specs.append(spec)
|
|
return
|
|
|
|
if self.contains_host(host):
|
|
raise ValueError('A Host can appear in an '
|
|
'ExplicitHostGroup only once.')
|
|
if spec in self._potentially_unsatisfied_specs:
|
|
self._potentially_unsatisfied_specs.remove(spec)
|
|
self._hostname_data_dict[host.hostname] = self.HostData(spec)
|
|
|
|
|
|
def contains_host(self, host):
|
|
"""Whether host is already part of this HostGroup
|
|
|
|
@param host: a Host object.
|
|
@return True if the host is already tracked; False otherwise.
|
|
"""
|
|
return host.hostname in self._hostname_data_dict
|
|
|
|
|
|
@property
|
|
def unsatisfied_specs(self):
|
|
unsatisfied = []
|
|
for spec in self._potentially_unsatisfied_specs:
|
|
# If a spec in _potentially_unsatisfied_specs is a subset of some
|
|
# satisfied spec, then it's not unsatisfied.
|
|
if [d for d in self._get_host_datas() if spec.is_subset(d.spec)]:
|
|
continue
|
|
unsatisfied.append(spec)
|
|
return unsatisfied
|
|
|
|
|
|
@property
|
|
def doomed_specs(self):
|
|
ok = set()
|
|
possibly_doomed = set()
|
|
for data in self._get_host_datas():
|
|
# If imaging succeeded for any host that satisfies a spec,
|
|
# it's definitely not doomed.
|
|
if data.image_success:
|
|
ok.add(data.spec)
|
|
else:
|
|
possibly_doomed.add(data.spec)
|
|
# If a spec is not a subset of any ok spec, it's doomed.
|
|
return set([s for s in possibly_doomed
|
|
if not list(filter(s.is_subset, ok))])
|
|
|
|
|
|
class MetaHostGroup(HostGroup):
|
|
"""A group of hosts, specified by a meta_host and deps, to be reimaged.
|
|
|
|
@var _meta_hosts: a meta_host, as expected by AFE.create_job()
|
|
@var _dependencies: list of dependencies that all hosts to be used
|
|
must satisfy
|
|
@var _successful_hosts: set of successful hosts.
|
|
"""
|
|
def __init__(self, labels, num):
|
|
"""Constructor.
|
|
|
|
Given a set of labels specifying what kind of hosts we need,
|
|
and the num of hosts we need, build a meta_host and dependency list
|
|
that represent this group of hosts.
|
|
|
|
@param labels: list of labels indicating what kind of hosts need
|
|
to be reimaged.
|
|
@param num: how many hosts we'd like to reimage.
|
|
"""
|
|
self._spec = HostSpec(labels)
|
|
self._meta_hosts = labels[:1]*num
|
|
self._dependencies = labels[1:]
|
|
self._successful_hosts = set()
|
|
|
|
|
|
def as_args(self):
|
|
return {'meta_hosts': self._meta_hosts,
|
|
'dependencies': self._dependencies}
|
|
|
|
|
|
def size(self):
|
|
return len(self._meta_hosts)
|
|
|
|
|
|
def mark_host_success(self, hostname):
|
|
self._successful_hosts.add(hostname)
|
|
|
|
|
|
def enough_hosts_succeeded(self):
|
|
return self._successful_hosts
|
|
|
|
|
|
@property
|
|
def doomed_specs(self):
|
|
if self._successful_hosts:
|
|
return []
|
|
return [self._spec]
|
|
|
|
|
|
def _safeunion(iter_a, iter_b):
|
|
"""Returns an immutable set that contains the union of two iterables.
|
|
|
|
This function returns a frozen set containing the all the elements of
|
|
two iterables, regardless of whether those iterables are lists, sets,
|
|
or whatever.
|
|
|
|
@param iter_a: The first iterable.
|
|
@param iter_b: The second iterable.
|
|
@returns: An immutable union of the contents of iter_a and iter_b.
|
|
"""
|
|
return frozenset({a for a in iter_a} | {b for b in iter_b})
|
|
|
|
|
|
|
|
class HostSpec(object):
|
|
"""Specifies a kind of host on which dependency-having tests can be run.
|
|
|
|
Wraps a list of labels, for the purposes of specifying a set of hosts
|
|
on which a test with matching dependencies can be run.
|
|
"""
|
|
|
|
def __init__(self, base, extended=[]):
|
|
self._labels = _safeunion(base, extended)
|
|
# To amortize cost of __hash__()
|
|
self._str = 'HostSpec %r' % sorted(self._labels)
|
|
self._trivial = extended == []
|
|
|
|
|
|
#pylint: disable-msg=C0111
|
|
@property
|
|
def labels(self):
|
|
# Can I just do this as a set? Inquiring minds want to know.
|
|
return sorted(self._labels)
|
|
|
|
|
|
#pylint: disable-msg=C0111
|
|
@property
|
|
def is_trivial(self):
|
|
return self._trivial
|
|
|
|
|
|
#pylint: disable-msg=C0111
|
|
def is_subset(self, other):
|
|
return self._labels <= other._labels
|
|
|
|
|
|
def __str__(self):
|
|
return self._str
|
|
|
|
|
|
def __repr__(self):
|
|
return self._str
|
|
|
|
|
|
def __lt__(self, other):
|
|
return str(self) < str(other)
|
|
|
|
|
|
def __le__(self, other):
|
|
return str(self) <= str(other)
|
|
|
|
|
|
def __eq__(self, other):
|
|
return str(self) == str(other)
|
|
|
|
|
|
def __ne__(self, other):
|
|
return str(self) != str(other)
|
|
|
|
|
|
def __gt__(self, other):
|
|
return str(self) > str(other)
|
|
|
|
|
|
def __ge__(self, other):
|
|
return str(self) >= str(other)
|
|
|
|
|
|
def __hash__(self):
|
|
"""Allows instances to be correctly deduped when used in a set."""
|
|
return hash(str(self))
|