You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
274 lines
9.9 KiB
274 lines
9.9 KiB
#!/usr/bin/env python2
|
|
|
|
# Copyright 2016 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""
|
|
This module is used to upload csv files generated by performance related tests
|
|
to cns. More details about the implementation can be found in crbug.com/598504.
|
|
|
|
The overall work flow is as follows.
|
|
1. Query tko_test_attributes table for perf_csv_folder attribute. The attribute
|
|
contains a path to csv files need to be uploaded to cns.
|
|
2. Filter the perf_csv_folder attributes only for test jobs have finished an
|
|
hour before. This is to make sure the results have already being uploaded to GS.
|
|
3. Locate the csv files in GS, and upload them to desired cns location.
|
|
|
|
After every run, the script saves the maximum test idx to a local file, and
|
|
repeats the workflow.
|
|
|
|
"""
|
|
|
|
import argparse
|
|
import datetime
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
import time
|
|
|
|
import common
|
|
from autotest_lib.client.bin import utils
|
|
from autotest_lib.client.common_lib import logging_config
|
|
from autotest_lib.client.common_lib.cros import retry
|
|
from autotest_lib.frontend import setup_django_environment
|
|
from autotest_lib.frontend.tko import models as tko_models
|
|
|
|
|
|
# Number of hours that a test has to be finished for the script to process.
|
|
# This allows gs_offloader to have enough time to upload the results to GS.
|
|
CUTOFF_TIME_HOURS = 1
|
|
|
|
# Default wait time in seconds after each run.
|
|
DEFAULT_INTERVAL_SEC = 60
|
|
|
|
# Timeout in minutes for upload attempts for a given folder.
|
|
UPLOAD_TIMEOUT_MINS = 5
|
|
|
|
class CsvNonexistenceException(Exception):
|
|
"""Exception raised when csv files not found in GS."""
|
|
|
|
|
|
class CsvFolder(object):
|
|
"""A class contains the information of a folder storing csv files to be
|
|
uploaded, and logic to upload the csv files.
|
|
"""
|
|
|
|
# A class variable whose value is the GoogleStorage path to the test
|
|
# results.
|
|
gs_path = None
|
|
|
|
# A class variable whose value is the cns path to upload the csv files to.
|
|
cns_path = None
|
|
|
|
def __init__(self, test_attribute_id, perf_csv_folder, test_view):
|
|
"""Initialize a CsvFolder object.
|
|
|
|
@param test_attribute_id: ID of test attribute record.
|
|
@param perf_csv_folder: Path of the folder contains csv files in test
|
|
results. It's the value of perf_csv_folder attribute from
|
|
tko_test_attributes table.
|
|
@param test_view: A db object from querying tko_test_view_2 for the
|
|
related tko_test_attributes.
|
|
"""
|
|
self.test_attribute_id = test_attribute_id
|
|
self.perf_csv_folder = perf_csv_folder
|
|
self.test_view = test_view
|
|
|
|
|
|
def __str__(self):
|
|
return '%s:%s:%s' % (self.test_view.job_name, self.test_view.job_tag,
|
|
self.perf_csv_folder)
|
|
|
|
|
|
def _get_url(self):
|
|
"""Get the url to the folder storing csv files in GS.
|
|
|
|
The url can be formulated based on csv folder, test_name and hostname.
|
|
For example:
|
|
gs://chromeos-autotest-results/123-chromeos-test/host1/
|
|
gsutil is used to download the csv files with this gs url.
|
|
"""
|
|
return os.path.join(self.gs_path, self.test_view.job_tag)
|
|
|
|
|
|
def _download(self, dest_dir):
|
|
"""Download the folder containing csv files to the given dest_dir.
|
|
|
|
@param dest_dir: A directory to store the downloaded csv files.
|
|
|
|
@return: A list of strings, each is a path to a csv file in the
|
|
downloaded folder.
|
|
@raise CsvNonexistenceException: If no csv file found in the GS.
|
|
"""
|
|
gs_url = self._get_url()
|
|
# Find all csv files in given GS url recursively
|
|
files = utils.run('gsutil ls -r %s | grep -e .*\\\\.csv$' %
|
|
gs_url, ignore_status=True).stdout.strip().split('\n')
|
|
if not files or files == ['']:
|
|
raise CsvNonexistenceException('No csv file found in %s', gs_url)
|
|
|
|
# Copy files from GS to temp_dir
|
|
for f in files:
|
|
utils.run('gsutil cp %s %s' % (f, dest_dir))
|
|
|
|
|
|
@retry.retry(Exception, raiselist=[CsvNonexistenceException],
|
|
timeout_min=UPLOAD_TIMEOUT_MINS)
|
|
def upload(self):
|
|
"""Upload the folder to cns.
|
|
"""
|
|
temp_dir = tempfile.mkdtemp(suffix='perf_csv')
|
|
try:
|
|
self._download(temp_dir)
|
|
files = os.listdir(temp_dir)
|
|
# File in cns is stored under folder with format of:
|
|
# <test_name>/<host_name>/YYYY/mm/dd/hh/mm
|
|
path_in_cns = os.path.join(
|
|
self.cns_path,
|
|
self.test_view.test_name, self.test_view.hostname,
|
|
str(self.test_view.job_finished_time.year),
|
|
str(self.test_view.job_finished_time.month).zfill(2),
|
|
str(self.test_view.job_finished_time.day).zfill(2),
|
|
str(self.test_view.job_finished_time.hour).zfill(2),
|
|
str(self.test_view.job_finished_time.minute).zfill(2))
|
|
utils.run('fileutil mkdir -p %s' % path_in_cns)
|
|
for f in files:
|
|
utils.run('fileutil copytodir -f %s %s' %
|
|
(os.path.join(temp_dir, f), path_in_cns))
|
|
finally:
|
|
shutil.rmtree(temp_dir)
|
|
|
|
|
|
class DBScanner(object):
|
|
"""Class contains the logic to query tko_test_attributes table for
|
|
new perf_csv_folder attributes and create CsvFolder object for each
|
|
new perf_csv_folder attribute.
|
|
"""
|
|
|
|
# Minimum test_attribute id for querying tko_test_attributes table.
|
|
min_test_attribute_id = -1
|
|
|
|
@classmethod
|
|
def get_perf_csv_folders(cls):
|
|
"""Query tko_test_attributes table for new entries of perf_csv_folder.
|
|
|
|
@return: A list of CsvFolder objects for each new entry of
|
|
perf_csv_folder attribute in tko_test_attributes table.
|
|
"""
|
|
attributes = tko_models.TestAttribute.objects.filter(
|
|
attribute='perf_csv_folder', id__gte=cls.min_test_attribute_id)
|
|
folders = []
|
|
|
|
cutoff_time = (datetime.datetime.now() -
|
|
datetime.timedelta(hours=CUTOFF_TIME_HOURS))
|
|
for attribute in attributes:
|
|
test_views = tko_models.TestView.objects.filter(
|
|
test_idx=attribute.test_id)
|
|
if test_views[0].job_finished_time > cutoff_time:
|
|
continue
|
|
folders.append(CsvFolder(attribute.id, attribute.value,
|
|
test_views[0]))
|
|
return folders
|
|
|
|
|
|
def setup_logging(log_dir):
|
|
"""Setup logging information.
|
|
|
|
@param log_dir: Path to the directory storing logs of this script.
|
|
"""
|
|
config = logging_config.LoggingConfig()
|
|
logfile = os.path.join(os.path.abspath(log_dir), 'perf_csv_uploader.log')
|
|
config.add_file_handler(file_path=logfile, level=logging.DEBUG)
|
|
|
|
|
|
def save_min_test_attribute_id(test_attribute_id_file):
|
|
"""Save the minimum test attribute id to a cached file.
|
|
|
|
@param test_attribute_id_file: Path to the file storing the value of
|
|
min_test_attribute_id.
|
|
"""
|
|
with open(test_attribute_id_file, 'w') as f:
|
|
return f.write(str(DBScanner.min_test_attribute_id))
|
|
|
|
|
|
def get_min_test_attribute_id(test_attribute_id_file):
|
|
"""Get the minimum test attribute id from a cached file.
|
|
|
|
@param test_attribute_id_file: Path to the file storing the value of
|
|
min_test_attribute_id.
|
|
"""
|
|
try:
|
|
with open(test_attribute_id_file, 'r') as f:
|
|
return int(f.read())
|
|
except IOError:
|
|
# min_test_attribute_id has not been set, default to -1.
|
|
return -1
|
|
|
|
|
|
def get_options():
|
|
"""Get the command line options.
|
|
|
|
@return: Command line options of the script.
|
|
"""
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--gs_path', type=str, dest='gs_path',
|
|
help='GoogleStorage path that stores test results.')
|
|
parser.add_argument('--cns_path', type=str, dest='cns_path',
|
|
help='cns path to where csv files are uploaded to.')
|
|
parser.add_argument('--log_dir', type=str, dest='log_dir',
|
|
help='Directory used to store logs.')
|
|
|
|
options = parser.parse_args()
|
|
CsvFolder.gs_path = options.gs_path
|
|
CsvFolder.cns_path = options.cns_path
|
|
|
|
return options
|
|
|
|
|
|
def main():
|
|
"""Main process to repeat the workflow of searching/uploading csv files.
|
|
"""
|
|
options = get_options()
|
|
setup_logging(options.log_dir)
|
|
test_attribute_id_file = os.path.join(options.log_dir,
|
|
'perf_csv_uploader_test_attr_id')
|
|
DBScanner.min_test_attribute_id = get_min_test_attribute_id(
|
|
test_attribute_id_file)
|
|
|
|
while True:
|
|
folders = DBScanner.get_perf_csv_folders()
|
|
if not folders:
|
|
logging.info('No new folders found. Wait...')
|
|
time.sleep(DEFAULT_INTERVAL_SEC)
|
|
continue
|
|
|
|
failed_folders = []
|
|
for folder in folders:
|
|
try:
|
|
logging.info('Uploading folder: %s', folder)
|
|
folder.upload()
|
|
except CsvNonexistenceException:
|
|
# Ignore the failure if CSV files are not found in GS.
|
|
pass
|
|
except Exception as e:
|
|
failed_folders.append(folder)
|
|
logging.error('Failed to upload folder %s, error: %s',
|
|
folder, e)
|
|
if failed_folders:
|
|
# Set the min_test_attribute_id to be the smallest one that failed
|
|
# to upload.
|
|
min_test_attribute_id = min([folder.test_attribute_id for folder in
|
|
failed_folders])
|
|
else:
|
|
min_test_attribute_id = max([folder.test_attribute_id for folder in
|
|
folders]) + 1
|
|
if DBScanner.min_test_attribute_id != min_test_attribute_id:
|
|
DBScanner.min_test_attribute_id = min_test_attribute_id
|
|
save_min_test_attribute_id(test_attribute_id_file)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|