You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
246 lines
7.3 KiB
246 lines
7.3 KiB
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (C) 2019 The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
# Sample Usage:
|
|
# $ python update_profiles.py 500000 ALL --profdata-suffix 2019-04-15
|
|
#
|
|
# Additional/frequently-used arguments:
|
|
# -b BUG adds a 'Bug: <BUG>' to the commit message when adding the profiles.
|
|
# --do-not-merge adds a 'DO NOT MERGE' tag to the commit message to restrict
|
|
# automerge of profiles from release branches.
|
|
#
|
|
# Try '-h' for a full list of command line arguments.
|
|
|
|
import argparse
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import zipfile
|
|
|
|
import utils
|
|
|
|
from android_build_client import AndroidBuildClient
|
|
|
|
|
|
class Benchmark(object):
|
|
|
|
def __init__(self, name):
|
|
self.name = name
|
|
|
|
def apct_test_tag(self):
|
|
raise NotImplementedError()
|
|
|
|
def profdata_file(self, suffix=''):
|
|
profdata = os.path.join(self.name, '{}.profdata'.format(self.name))
|
|
if suffix:
|
|
profdata += '.' + suffix
|
|
return profdata
|
|
|
|
def profraw_files(self):
|
|
raise NotImplementedError()
|
|
|
|
def merge_profraws(self, profile_dir, output):
|
|
profraws = [
|
|
os.path.join(profile_dir, p)
|
|
for p in self.profraw_files(profile_dir)
|
|
]
|
|
utils.run_llvm_profdata(profraws, output)
|
|
|
|
|
|
class NativeExeBenchmark(Benchmark):
|
|
|
|
def apct_test_tag(self):
|
|
return 'apct/perf/pgo/profile-collector'
|
|
|
|
def profraw_files(self, profile_dir):
|
|
if self.name == 'hwui':
|
|
return [
|
|
'hwuimacro.profraw', 'hwuimacro_64.profraw',
|
|
'hwuimicro.profraw', 'hwuimicro_64.profraw',
|
|
'skia_nanobench.profraw', 'skia_nanobench_64.profraw'
|
|
]
|
|
elif self.name == 'hwbinder':
|
|
return ['hwbinder.profraw', 'hwbinder_64.profraw']
|
|
|
|
|
|
class APKBenchmark(Benchmark):
|
|
|
|
def apct_test_tag(self):
|
|
return 'apct/perf/pgo/apk-profile-collector'
|
|
|
|
def profdata_file(self, suffix=''):
|
|
profdata = os.path.join('art',
|
|
'{}_arm_arm64.profdata'.format(self.name))
|
|
if suffix:
|
|
profdata += '.' + suffix
|
|
return profdata
|
|
|
|
def profraw_files(self, profile_dir):
|
|
return os.listdir(profile_dir)
|
|
|
|
|
|
def BenchmarkFactory(benchmark_name):
|
|
if benchmark_name == 'dex2oat':
|
|
return APKBenchmark(benchmark_name)
|
|
elif benchmark_name in ['hwui', 'hwbinder']:
|
|
return NativeExeBenchmark(benchmark_name)
|
|
else:
|
|
raise RuntimeError('Unknown benchmark ' + benchmark_name)
|
|
|
|
|
|
def extract_profiles(build, test_tag, build_client, output_dir):
|
|
pgo_zip = build_client.download_pgo_zip(build, test_tag, output_dir)
|
|
|
|
zipfile_name = os.path.join(pgo_zip)
|
|
zip_ref = zipfile.ZipFile(zipfile_name)
|
|
zip_ref.extractall(output_dir)
|
|
zip_ref.close()
|
|
|
|
|
|
KNOWN_BENCHMARKS = ['ALL', 'dex2oat', 'hwui', 'hwbinder']
|
|
|
|
|
|
def parse_args():
|
|
"""Parses and returns command line arguments."""
|
|
parser = argparse.ArgumentParser()
|
|
|
|
parser.add_argument(
|
|
'build',
|
|
metavar='BUILD',
|
|
help='Build number to pull from the build server.')
|
|
|
|
parser.add_argument(
|
|
'-b', '--bug', type=int, help='Bug to reference in commit message.')
|
|
|
|
parser.add_argument(
|
|
'--use-current-branch',
|
|
action='store_true',
|
|
help='Do not repo start a new branch for the update.')
|
|
|
|
parser.add_argument(
|
|
'--add-do-not-merge',
|
|
action='store_true',
|
|
help='Add \'DO NOT MERGE\' to the commit message.')
|
|
|
|
parser.add_argument(
|
|
'--profdata-suffix',
|
|
type=str,
|
|
default='',
|
|
help='Suffix to append to merged profdata file')
|
|
|
|
parser.add_argument(
|
|
'benchmark',
|
|
metavar='BENCHMARK',
|
|
help='Update profiles for BENCHMARK. Choices are {}'.format(
|
|
KNOWN_BENCHMARKS))
|
|
|
|
parser.add_argument(
|
|
'--skip-cleanup',
|
|
'-sc',
|
|
action='store_true',
|
|
default=False,
|
|
help='Skip the cleanup, and leave intermediate files (in /tmp/pgo-profiles-*)'
|
|
)
|
|
|
|
return parser.parse_args()
|
|
|
|
|
|
def get_current_profile(benchmark):
|
|
profile = benchmark.profdata_file()
|
|
dirname, basename = os.path.split(profile)
|
|
|
|
old_profiles = [f for f in os.listdir(dirname) if f.startswith(basename)]
|
|
if len(old_profiles) == 0:
|
|
return ''
|
|
return os.path.join(dirname, old_profiles[0])
|
|
|
|
|
|
def main():
|
|
args = parse_args()
|
|
utils.check_gcertstatus()
|
|
|
|
if args.benchmark == 'ALL':
|
|
worklist = KNOWN_BENCHMARKS[1:]
|
|
else:
|
|
worklist = [args.benchmark]
|
|
|
|
profiles_project = os.path.join(utils.android_build_top(), 'toolchain',
|
|
'pgo-profiles')
|
|
os.chdir(profiles_project)
|
|
|
|
if not args.use_current_branch:
|
|
branch_name = 'update-profiles-' + args.build
|
|
utils.check_call(['repo', 'start', branch_name, '.'])
|
|
|
|
build_client = AndroidBuildClient()
|
|
|
|
for benchmark_name in worklist:
|
|
benchmark = BenchmarkFactory(benchmark_name)
|
|
|
|
# Existing profile file, which gets 'rm'-ed from 'git' down below.
|
|
current_profile = get_current_profile(benchmark)
|
|
|
|
# Extract profiles to a temporary directory. After extraction, we
|
|
# expect to find one subdirectory with profraw files under the temporary
|
|
# directory.
|
|
extract_dir = tempfile.mkdtemp(prefix='pgo-profiles-' + benchmark_name)
|
|
extract_profiles(args.build, benchmark.apct_test_tag(), build_client,
|
|
extract_dir)
|
|
|
|
extract_subdirs = [
|
|
os.path.join(extract_dir, sub)
|
|
for sub in os.listdir(extract_dir)
|
|
if os.path.isdir(os.path.join(extract_dir, sub))
|
|
]
|
|
if len(extract_subdirs) != 1:
|
|
raise RuntimeError(
|
|
'Expected one subdir under {}'.format(extract_dir))
|
|
|
|
# Merge profiles.
|
|
profdata = benchmark.profdata_file(args.profdata_suffix)
|
|
benchmark.merge_profraws(extract_subdirs[0], profdata)
|
|
|
|
# Construct 'git' commit message.
|
|
message_lines = [
|
|
'Update PGO profiles for {}'.format(benchmark_name), '',
|
|
'The profiles are from build {}.'.format(args.build), ''
|
|
]
|
|
|
|
if args.add_do_not_merge:
|
|
message_lines[0] = '[DO NOT MERGE] ' + message_lines[0]
|
|
|
|
if args.bug:
|
|
message_lines.append('')
|
|
message_lines.append('Bug: http://b/{}'.format(args.bug))
|
|
message_lines.append('Test: Build (TH)')
|
|
message = '\n'.join(message_lines)
|
|
|
|
# Invoke git: Delete current profile, add new profile and commit these
|
|
# changes.
|
|
if current_profile:
|
|
utils.check_call(['git', 'rm', current_profile])
|
|
utils.check_call(['git', 'add', profdata])
|
|
utils.check_call(['git', 'commit', '-m', message])
|
|
|
|
if not args.skip_cleanup:
|
|
shutil.rmtree(extract_dir)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|