You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
307 lines
9.7 KiB
307 lines
9.7 KiB
import os, re, db, sys, datetime
|
|
import common
|
|
from autotest_lib.client.common_lib import kernel_versions
|
|
|
|
MAX_RECORDS = 50000L
|
|
MAX_CELLS = 500000L
|
|
|
|
tko = os.path.dirname(os.path.realpath(os.path.abspath(__file__)))
|
|
root_url_file = os.path.join(tko, '.root_url')
|
|
if os.path.exists(root_url_file):
|
|
html_root = open(root_url_file, 'r').readline().rstrip()
|
|
else:
|
|
html_root = '/results/'
|
|
|
|
|
|
class status_cell:
|
|
# One cell in the matrix of status data.
|
|
def __init__(self):
|
|
# Count is a dictionary: status -> count of tests with status
|
|
self.status_count = {}
|
|
self.reasons_list = []
|
|
self.job_tag = None
|
|
self.job_tag_count = 0
|
|
|
|
|
|
def add(self, status, count, job_tags, reasons = None):
|
|
assert count > 0
|
|
|
|
self.job_tag = job_tags
|
|
self.job_tag_count += count
|
|
if self.job_tag_count > 1:
|
|
self.job_tag = None
|
|
|
|
self.status_count[status] = count
|
|
### status == 6 means 'GOOD'
|
|
if status != 6:
|
|
## None implies sorting problems and extra CRs in a cell
|
|
if reasons:
|
|
self.reasons_list.append(reasons)
|
|
|
|
|
|
class status_data:
|
|
def __init__(self, sql_rows, x_field, y_field, query_reasons = False):
|
|
data = {}
|
|
y_values = set()
|
|
|
|
# Walk through the query, filing all results by x, y info
|
|
for row in sql_rows:
|
|
if query_reasons:
|
|
(x,y, status, count, job_tags, reasons) = row
|
|
else:
|
|
(x,y, status, count, job_tags) = row
|
|
reasons = None
|
|
if not data.has_key(x):
|
|
data[x] = {}
|
|
if not data[x].has_key(y):
|
|
y_values.add(y)
|
|
data[x][y] = status_cell()
|
|
data[x][y].add(status, count, job_tags, reasons)
|
|
|
|
# 2-d hash of data - [x-value][y-value]
|
|
self.data = data
|
|
# List of possible columns (x-values)
|
|
self.x_values = smart_sort(data.keys(), x_field)
|
|
# List of rows columns (y-values)
|
|
self.y_values = smart_sort(list(y_values), y_field)
|
|
nCells = len(self.y_values)*len(self.x_values)
|
|
if nCells > MAX_CELLS:
|
|
msg = 'Exceeded allowed number of cells in a table'
|
|
raise db.MySQLTooManyRows(msg)
|
|
|
|
|
|
def get_matrix_data(db_obj, x_axis, y_axis, where = None,
|
|
query_reasons = False):
|
|
# Searches on the test_view table - x_axis and y_axis must both be
|
|
# column names in that table.
|
|
x_field = test_view_field_dict[x_axis]
|
|
y_field = test_view_field_dict[y_axis]
|
|
query_fields_list = [x_field, y_field, 'status','COUNT(status)']
|
|
query_fields_list.append("LEFT(GROUP_CONCAT(job_tag),100)")
|
|
if query_reasons:
|
|
query_fields_list.append(
|
|
"LEFT(GROUP_CONCAT(DISTINCT reason SEPARATOR '|'),500)"
|
|
)
|
|
fields = ','.join(query_fields_list)
|
|
|
|
group_by = '%s, %s, status' % (x_field, y_field)
|
|
rows = db_obj.select(fields, 'tko_test_view',
|
|
where=where, group_by=group_by, max_rows = MAX_RECORDS)
|
|
return status_data(rows, x_field, y_field, query_reasons)
|
|
|
|
|
|
# Dictionary used simply for fast lookups from short reference names for users
|
|
# to fieldnames in test_view
|
|
test_view_field_dict = {
|
|
'kernel' : 'kernel_printable',
|
|
'hostname' : 'machine_hostname',
|
|
'test' : 'test',
|
|
'label' : 'job_label',
|
|
'machine_group' : 'machine_group',
|
|
'reason' : 'reason',
|
|
'tag' : 'job_tag',
|
|
'user' : 'job_username',
|
|
'status' : 'status_word',
|
|
'time' : 'test_finished_time',
|
|
'start_time' : 'test_started_time',
|
|
'time_daily' : 'DATE(test_finished_time)'
|
|
}
|
|
|
|
|
|
def smart_sort(list, field):
|
|
if field == 'kernel_printable':
|
|
def kernel_encode(kernel):
|
|
return kernel_versions.version_encode(kernel)
|
|
list.sort(key = kernel_encode, reverse = True)
|
|
return list
|
|
## old records may contain time=None
|
|
## make None comparable with timestamp datetime or date
|
|
elif field == 'test_finished_time':
|
|
def convert_None_to_datetime(date_time):
|
|
if not date_time:
|
|
return datetime.datetime(1970, 1, 1, 0, 0, 0)
|
|
else:
|
|
return date_time
|
|
list = map(convert_None_to_datetime, list)
|
|
elif field == 'DATE(test_finished_time)':
|
|
def convert_None_to_date(date):
|
|
if not date:
|
|
return datetime.date(1970, 1, 1)
|
|
else:
|
|
return date
|
|
list = map(convert_None_to_date, list)
|
|
list.sort()
|
|
return list
|
|
|
|
|
|
class group:
|
|
@classmethod
|
|
def select(klass, db):
|
|
"""Return all possible machine groups"""
|
|
rows = db.select('distinct machine_group', 'tko_machines',
|
|
'machine_group is not null')
|
|
groupnames = sorted([row[0] for row in rows])
|
|
return [klass(db, groupname) for groupname in groupnames]
|
|
|
|
|
|
def __init__(self, db, name):
|
|
self.name = name
|
|
self.db = db
|
|
|
|
|
|
def machines(self):
|
|
return machine.select(self.db, { 'machine_group' : self.name })
|
|
|
|
|
|
def tests(self, where = {}):
|
|
values = [self.name]
|
|
sql = 't inner join tko_machines m on m.machine_idx=t.machine_idx'
|
|
sql += ' where m.machine_group=%s'
|
|
for key in where.keys():
|
|
sql += ' and %s=%%s' % key
|
|
values.append(where[key])
|
|
return test.select_sql(self.db, sql, values)
|
|
|
|
|
|
class machine:
|
|
@classmethod
|
|
def select(klass, db, where = {}):
|
|
fields = ['machine_idx', 'hostname', 'machine_group', 'owner']
|
|
machines = []
|
|
for row in db.select(','.join(fields), 'tko_machines', where):
|
|
machines.append(klass(db, *row))
|
|
return machines
|
|
|
|
|
|
def __init__(self, db, idx, hostname, group, owner):
|
|
self.db = db
|
|
self.idx = idx
|
|
self.hostname = hostname
|
|
self.group = group
|
|
self.owner = owner
|
|
|
|
|
|
class kernel:
|
|
@classmethod
|
|
def select(klass, db, where = {}):
|
|
fields = ['kernel_idx', 'kernel_hash', 'base', 'printable']
|
|
rows = db.select(','.join(fields), 'tko_kernels', where)
|
|
return [klass(db, *row) for row in rows]
|
|
|
|
|
|
def __init__(self, db, idx, hash, base, printable):
|
|
self.db = db
|
|
self.idx = idx
|
|
self.hash = hash
|
|
self.base = base
|
|
self.printable = printable
|
|
self.patches = [] # THIS SHOULD PULL IN PATCHES!
|
|
|
|
|
|
class test:
|
|
@classmethod
|
|
def select(klass, db, where={}, distinct=False):
|
|
fields = ['test_idx', 'job_idx', 'test', 'subdir',
|
|
'kernel_idx', 'status', 'reason', 'machine_idx']
|
|
tests = []
|
|
for row in db.select(','.join(fields), 'tko_tests', where,
|
|
distinct):
|
|
tests.append(klass(db, *row))
|
|
return tests
|
|
|
|
|
|
@classmethod
|
|
def select_sql(klass, db, sql, values):
|
|
fields = ['test_idx', 'job_idx', 'test', 'subdir',
|
|
'kernel_idx', 'status', 'reason', 'machine_idx']
|
|
fields = ['t.'+field for field in fields]
|
|
rows = db.select_sql(','.join(fields), 'tko_tests', sql, values)
|
|
return [klass(db, *row) for row in rows]
|
|
|
|
|
|
def __init__(self, db, test_idx, job_idx, testname, subdir, kernel_idx,
|
|
status_num, reason, machine_idx):
|
|
self.idx = test_idx
|
|
self.job = job(db, job_idx)
|
|
self.testname = testname
|
|
self.subdir = subdir
|
|
self.kernel_idx = kernel_idx
|
|
self.__kernel = None
|
|
self.__iterations = None
|
|
self.machine_idx = machine_idx
|
|
self.__machine = None
|
|
self.status_num = status_num
|
|
self.status_word = db.status_word[status_num]
|
|
self.reason = reason
|
|
self.db = db
|
|
if self.subdir:
|
|
self.url = html_root + self.job.tag + '/' + self.subdir
|
|
else:
|
|
self.url = None
|
|
|
|
|
|
def iterations(self):
|
|
"""
|
|
Caching function for iterations
|
|
"""
|
|
if not self.__iterations:
|
|
self.__iterations = {}
|
|
# A dictionary - dict{key} = [value1, value2, ....]
|
|
where = {'test_idx' : self.idx}
|
|
for i in iteration.select(self.db, where):
|
|
if self.__iterations.has_key(i.key):
|
|
self.__iterations[i.key].append(i.value)
|
|
else:
|
|
self.__iterations[i.key] = [i.value]
|
|
return self.__iterations
|
|
|
|
|
|
def kernel(self):
|
|
"""
|
|
Caching function for kernels
|
|
"""
|
|
if not self.__kernel:
|
|
where = {'kernel_idx' : self.kernel_idx}
|
|
self.__kernel = kernel.select(self.db, where)[0]
|
|
return self.__kernel
|
|
|
|
|
|
def machine(self):
|
|
"""
|
|
Caching function for kernels
|
|
"""
|
|
if not self.__machine:
|
|
where = {'machine_idx' : self.machine_idx}
|
|
self.__machine = machine.select(self.db, where)[0]
|
|
return self.__machine
|
|
|
|
|
|
class job:
|
|
def __init__(self, db, job_idx):
|
|
where = {'job_idx' : job_idx}
|
|
rows = db.select('tag, machine_idx', 'tko_jobs', where)
|
|
if rows:
|
|
self.tag, self.machine_idx = rows[0]
|
|
self.job_idx = job_idx
|
|
|
|
|
|
class iteration:
|
|
@classmethod
|
|
def select(klass, db, where):
|
|
fields = ['iteration', 'attribute', 'value']
|
|
iterations = []
|
|
rows = db.select(','.join(fields), 'tko_iteration_result', where)
|
|
for row in rows:
|
|
iterations.append(klass(*row))
|
|
return iterations
|
|
|
|
|
|
def __init__(self, iteration, key, value):
|
|
self.iteration = iteration
|
|
self.key = key
|
|
self.value = value
|
|
|
|
# class patch:
|
|
# def __init__(self):
|
|
# self.spec = None
|