You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
559 lines
21 KiB
559 lines
21 KiB
#!/usr/bin/env python
|
|
#
|
|
# Copyright 2016 - The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
# Parses the output of parse_ltp_{make,make_install} and generates a
|
|
# corresponding Android.mk.
|
|
#
|
|
# This process is split into two steps so this second step can later be replaced
|
|
# with an Android.bp generator.
|
|
|
|
import argparse
|
|
import fileinput
|
|
import json
|
|
import os.path
|
|
import re
|
|
|
|
import make_parser
|
|
import make_install_parser
|
|
|
|
MAKE_DRY_RUN_FILE_NAME = 'make_dry_run.dump'
|
|
MAKE_INSTALL_DRY_RUN_FILE_NAME = 'make_install_dry_run.dump'
|
|
DISABLED_TESTS_FILE_NAME = 'disabled_tests.txt'
|
|
DISABLED_LIBS_FILE_NAME = 'disabled_libs.txt'
|
|
DISABLED_CFLAGS_FILE_NAME = 'disabled_cflags.txt'
|
|
|
|
|
|
class BuildGenerator(object):
|
|
'''A class to parse make output and convert the result to Android.ltp.mk modules.
|
|
|
|
Attributes:
|
|
_bp_result: directory of list of strings for blueprint file keyed by target name
|
|
_mk_result: directory of list of strings for makefile keyed by target name
|
|
_custom_cflags: dict of string (module name) to lists of strings (cflags
|
|
to add for said module)
|
|
_unused_custom_cflags: set of strings; tracks the modules with custom
|
|
cflags that we haven't yet seen
|
|
_packages: list of strings of packages for package list file
|
|
'''
|
|
|
|
def __init__(self, custom_cflags):
|
|
self._bp_result = {}
|
|
self._mk_result = {}
|
|
self._custom_cflags = custom_cflags
|
|
self._unused_custom_cflags = set(custom_cflags)
|
|
self._packages = []
|
|
|
|
def UniqueKeepOrder(self, sequence):
|
|
'''Get a copy of list where items are unique and order is preserved.
|
|
|
|
Args:
|
|
sequence: a sequence, can be a list, tuple, or other iterable
|
|
|
|
Returns:
|
|
a list where items copied from input sequence are unique
|
|
and order is preserved.
|
|
'''
|
|
seen = set()
|
|
return [x for x in sequence if not (x in seen or seen.add(x))]
|
|
|
|
def ReadCommentedText(self, file_path):
|
|
'''Read pound commented text file into a list of lines.
|
|
|
|
Comments or empty lines will be excluded
|
|
|
|
Args:
|
|
file_path: string
|
|
'''
|
|
ret = set()
|
|
with open(file_path, 'r') as f:
|
|
lines = [line.strip() for line in f.readlines()]
|
|
ret = set([s for s in lines if s and not s.startswith('#')])
|
|
|
|
return ret
|
|
|
|
def ArTargetToLibraryName(self, ar_target):
|
|
'''Convert ar target to library name.
|
|
|
|
Args:
|
|
ar_target: string
|
|
'''
|
|
return os.path.basename(ar_target)[len('lib'):-len('.a')]
|
|
|
|
def BuildExecutable(self, cc_target, local_src_files, local_cflags,
|
|
local_c_includes, local_libraries, ltp_libs,
|
|
ltp_libs_used, ltp_names_used):
|
|
'''Build a test module.
|
|
|
|
Args:
|
|
cc_target: string
|
|
local_src_files: list of string
|
|
local_cflags: list of string
|
|
local_c_includes: list of string
|
|
local_libraries: list of string
|
|
ltp_libs: list of string
|
|
ltp_libs_used: set of string
|
|
ltp_names_used: set of string, set of already used cc_target basenames
|
|
'''
|
|
base_name = os.path.basename(cc_target)
|
|
if base_name in ltp_names_used:
|
|
print 'ERROR: base name {} of cc_target {} already used. Skipping...'.format(
|
|
base_name, cc_target)
|
|
return
|
|
ltp_names_used.add(base_name)
|
|
|
|
if cc_target in self._custom_cflags:
|
|
local_cflags.extend(self._custom_cflags[cc_target])
|
|
self._unused_custom_cflags.remove(cc_target)
|
|
|
|
# ltp_defaults already adds the include directory
|
|
local_c_includes = [i for i in local_c_includes if i != 'include']
|
|
target_name = 'ltp_%s' % base_name
|
|
target_bp = []
|
|
|
|
self._packages.append(target_name)
|
|
|
|
target_bp.append('cc_test {')
|
|
target_bp.append(' name: "%s",' % target_name)
|
|
target_bp.append(' stem: "%s",' % base_name)
|
|
target_bp.append(' defaults: ["ltp_test_defaults"],')
|
|
|
|
if len(local_src_files) == 1:
|
|
target_bp.append(' srcs: ["%s"],' % list(local_src_files)[0])
|
|
else:
|
|
target_bp.append(' srcs: [')
|
|
for src in local_src_files:
|
|
target_bp.append(' "%s",' % src)
|
|
target_bp.append(' ],')
|
|
|
|
if len(local_cflags) == 1:
|
|
target_bp.append(' cflags: ["%s"],' % list(local_cflags)[0])
|
|
elif len(local_cflags) > 1:
|
|
target_bp.append(' cflags: [')
|
|
for cflag in local_cflags:
|
|
target_bp.append(' "%s",' % cflag)
|
|
target_bp.append(' ],')
|
|
|
|
if len(local_c_includes) == 1:
|
|
target_bp.append(' local_include_dirs: ["%s"],' % list(local_c_includes)[0])
|
|
elif len(local_c_includes) > 1:
|
|
target_bp.append(' local_include_dirs: [')
|
|
for d in local_c_includes:
|
|
target_bp.append(' "%s",' % d)
|
|
target_bp.append(' ],')
|
|
|
|
bionic_builtin_libs = set(['m', 'rt', 'pthread', 'util'])
|
|
filtered_libs = set(local_libraries).difference(bionic_builtin_libs)
|
|
|
|
static_libraries = set(i for i in local_libraries if i in ltp_libs)
|
|
if len(static_libraries) == 1:
|
|
target_bp.append(' static_libs: ["libltp_%s"],' % list(static_libraries)[0])
|
|
elif len(static_libraries) > 1:
|
|
target_bp.append(' static_libs: [')
|
|
for lib in static_libraries:
|
|
target_bp.append(' "libltp_%s",' % lib)
|
|
target_bp.append(' ],')
|
|
|
|
for lib in static_libraries:
|
|
ltp_libs_used.add(lib)
|
|
|
|
shared_libraries = set(i for i in filtered_libs if i not in ltp_libs)
|
|
if len(shared_libraries) == 1:
|
|
target_bp.append(' shared_libs: ["lib%s"],' % list(shared_libraries)[0])
|
|
elif len(shared_libraries) > 1:
|
|
target_bp.append(' shared_libs: [')
|
|
for lib in shared_libraries:
|
|
target_bp.append(' "lib%s",' % lib)
|
|
target_bp.append(' ],')
|
|
|
|
target_bp.append('}')
|
|
target_bp.append('')
|
|
self._bp_result[target_name] = target_bp
|
|
|
|
def BuildStaticLibrary(self, ar_target, local_src_files, local_cflags,
|
|
local_c_includes):
|
|
'''Build a library module.
|
|
|
|
Args:
|
|
ar_target: string
|
|
local_src_files: list of string
|
|
local_cflags: list of string
|
|
local_c_includes: list of string
|
|
'''
|
|
target_name = 'libltp_%s' % self.ArTargetToLibraryName(ar_target)
|
|
target_bp = []
|
|
target_bp.append('cc_library_static {')
|
|
target_bp.append(' name: "%s",' % target_name)
|
|
target_bp.append(' defaults: ["ltp_defaults"],')
|
|
|
|
if len(local_c_includes):
|
|
target_bp.append(' local_include_dirs: [')
|
|
for d in local_c_includes:
|
|
target_bp.append(' "%s",' % d)
|
|
target_bp.append(' ],')
|
|
|
|
if len(local_cflags):
|
|
target_bp.append(' cflags: [')
|
|
for cflag in local_cflags:
|
|
target_bp.append(' "%s",' % cflag)
|
|
target_bp.append(' ],')
|
|
|
|
target_bp.append(' srcs: [')
|
|
for src in local_src_files:
|
|
target_bp.append(' "%s",' % src)
|
|
target_bp.append(' ],')
|
|
|
|
target_bp.append('}')
|
|
target_bp.append('')
|
|
self._bp_result[target_name] = target_bp
|
|
|
|
def BuildShellScript(self, install_target, local_src_file):
|
|
'''Build a shell script.
|
|
|
|
Args:
|
|
install_target: string
|
|
local_src_file: string
|
|
'''
|
|
base_name = os.path.basename(install_target)
|
|
bp_result = []
|
|
|
|
module = 'ltp_%s' % install_target.replace('/', '_')
|
|
self._packages.append(module)
|
|
|
|
module_dir = os.path.dirname(install_target)
|
|
module_stem = os.path.basename(install_target)
|
|
|
|
bp_result.append('sh_test {')
|
|
bp_result.append(' name: "%s",' % module)
|
|
bp_result.append(' src: "%s",' % local_src_file)
|
|
bp_result.append(' sub_dir: "ltp/%s",' % module_dir)
|
|
bp_result.append(' filename: "%s",' % module_stem)
|
|
bp_result.append(' compile_multilib: "both",')
|
|
bp_result.append('}')
|
|
bp_result.append('')
|
|
|
|
self._bp_result[module] = bp_result
|
|
|
|
def BuildPrebuilt(self, install_target, local_src_file):
|
|
'''Build a prebuild module.
|
|
|
|
Args:
|
|
install_target: string
|
|
local_src_file: string
|
|
'''
|
|
base_name = os.path.basename(install_target)
|
|
mk_result = []
|
|
mk_result.append('module_prebuilt := %s' % install_target)
|
|
mk_result.append('module_src_files := %s' % local_src_file)
|
|
module_dir = os.path.dirname(install_target)
|
|
module_stem = os.path.basename(install_target)
|
|
module = 'ltp_%s' % install_target.replace('/', '_')
|
|
self._packages.append(module)
|
|
|
|
mk_result.append('include $(ltp_build_prebuilt)')
|
|
mk_result.append('')
|
|
self._mk_result[base_name] = mk_result
|
|
|
|
def HandleParsedRule(self, line, rules):
|
|
'''Prepare parse rules.
|
|
|
|
Args:
|
|
line: string
|
|
rules: dictionary {string, dictionary}
|
|
'''
|
|
groups = re.match(r'(.*)\[\'(.*)\'\] = \[(.*)\]', line).groups()
|
|
rule = groups[0]
|
|
rule_key = groups[1]
|
|
if groups[2] == '':
|
|
rule_value = []
|
|
else:
|
|
rule_value = list(i.strip()[1:-1] for i in groups[2].split(','))
|
|
|
|
rule_value = self.UniqueKeepOrder(rule_value)
|
|
rules.setdefault(rule, {})[rule_key] = rule_value
|
|
|
|
def ParseInput(self, input_list, ltp_root):
|
|
'''Parse a interpreted make output and produce Android.ltp.mk module.
|
|
|
|
Args:
|
|
input_list: list of string
|
|
'''
|
|
disabled_tests = self.ReadCommentedText(DISABLED_TESTS_FILE_NAME)
|
|
disabled_libs = self.ReadCommentedText(DISABLED_LIBS_FILE_NAME)
|
|
disabled_cflags = self.ReadCommentedText(DISABLED_CFLAGS_FILE_NAME)
|
|
|
|
rules = {}
|
|
for line in input_list:
|
|
self.HandleParsedRule(line.strip(), rules)
|
|
|
|
# .a target -> .o files
|
|
ar = rules.get('ar', {})
|
|
# executable target -> .o files
|
|
cc_link = rules.get('cc_link', {})
|
|
# .o target -> .c file
|
|
cc_compile = rules.get('cc_compile', {})
|
|
# executable target -> .c files
|
|
cc_compilelink = rules.get('cc_compilelink', {})
|
|
# Target name -> CFLAGS passed to gcc
|
|
cc_flags = rules.get('cc_flags', {})
|
|
# Target name -> -I paths passed to gcc
|
|
cc_includes = rules.get('cc_includes', {})
|
|
# Target name -> -l paths passed to gcc
|
|
cc_libraries = rules.get('cc_libraries', {})
|
|
# target -> prebuilt source
|
|
install = rules.get('install', {})
|
|
|
|
# All libraries used by any LTP test (built or not)
|
|
ltp_libs = set(self.ArTargetToLibraryName(i) for i in ar.keys())
|
|
# All libraries used by the LTP tests we actually build
|
|
ltp_libs_used = set()
|
|
ltp_names_used = set()
|
|
|
|
# Remove -Wno-error from cflags, we don't want to print warnings.
|
|
# Silence individual warnings in ltp_defaults or fix them.
|
|
for target in cc_flags:
|
|
if '-Wno-error' in cc_flags[target]:
|
|
cc_flags[target].remove('-Wno-error')
|
|
|
|
print(
|
|
"Disabled lib tests: Test cases listed here are"
|
|
"suggested to be disabled since they require a disabled library. "
|
|
"Please copy and paste them into disabled_tests.txt\n")
|
|
for i in cc_libraries:
|
|
if len(set(cc_libraries[i]).intersection(disabled_libs)) > 0:
|
|
if not os.path.basename(i) in disabled_tests:
|
|
print os.path.basename(i)
|
|
|
|
print("Disabled_cflag tests: Test cases listed here are"
|
|
"suggested to be disabled since they require a disabled cflag. "
|
|
"Please copy and paste them into disabled_tests.txt\n")
|
|
for i in cc_flags:
|
|
if len(set(cc_flags[i]).intersection(disabled_cflags)) > 0:
|
|
module_name = os.path.basename(i)
|
|
idx = module_name.find('_')
|
|
if idx > 0:
|
|
module_name = module_name[:idx]
|
|
print module_name
|
|
|
|
# Remove include directories that don't exist. They're an error in
|
|
# Soong.
|
|
for target in cc_includes:
|
|
cc_includes[target] = [i for i in cc_includes[target] if os.path.isdir(os.path.join(ltp_root, i))]
|
|
|
|
for target in cc_compilelink:
|
|
module_name = os.path.basename(target)
|
|
if module_name in disabled_tests:
|
|
continue
|
|
local_src_files = []
|
|
src_files = cc_compilelink[target]
|
|
for i in src_files:
|
|
# some targets may have a mix of .c and .o files in srcs
|
|
# find the .c files to build those .o from cc_compile targets
|
|
if i.endswith('.o'):
|
|
local_src_files.extend(cc_compile[i])
|
|
else:
|
|
local_src_files.append(i)
|
|
local_cflags = cc_flags[target]
|
|
local_c_includes = cc_includes[target]
|
|
local_libraries = cc_libraries[target]
|
|
if len(set(local_libraries).intersection(disabled_libs)) > 0:
|
|
continue
|
|
if len(set(local_cflags).intersection(disabled_cflags)) > 0:
|
|
continue
|
|
self.BuildExecutable(target, local_src_files, local_cflags,
|
|
local_c_includes, local_libraries, ltp_libs,
|
|
ltp_libs_used, ltp_names_used)
|
|
|
|
for target in cc_link:
|
|
if os.path.basename(target) in disabled_tests:
|
|
continue
|
|
local_src_files = set()
|
|
local_cflags = set()
|
|
local_c_includes = set()
|
|
local_libraries = cc_libraries[target]
|
|
# Accumulate flags for all .c files needed to build the .o files.
|
|
# (Android.mk requires a consistent set of flags across a given target.
|
|
# Thankfully using the superset of all flags in the target works fine
|
|
# with LTP tests.)
|
|
for obj in cc_link[target]:
|
|
for i in cc_compile[obj]:
|
|
local_src_files.add(i)
|
|
for i in cc_flags[obj]:
|
|
local_cflags.add(i)
|
|
for i in cc_includes[obj]:
|
|
local_c_includes.add(i)
|
|
if len(set(local_libraries).intersection(disabled_libs)) > 0:
|
|
continue
|
|
if len(set(local_cflags).intersection(disabled_cflags)) > 0:
|
|
continue
|
|
|
|
self.BuildExecutable(target, local_src_files, local_cflags,
|
|
local_c_includes, local_libraries, ltp_libs,
|
|
ltp_libs_used, ltp_names_used)
|
|
|
|
for target in ar:
|
|
# Disabled ltp library is already excluded
|
|
# since it won't be in ltp_libs_used
|
|
if not self.ArTargetToLibraryName(target) in ltp_libs_used:
|
|
continue
|
|
|
|
local_src_files = set()
|
|
local_cflags = set()
|
|
local_c_includes = set()
|
|
|
|
# TODO: disabled cflags
|
|
|
|
for obj in ar[target]:
|
|
for i in cc_compile[obj]:
|
|
local_src_files.add(i)
|
|
for i in cc_flags[obj]:
|
|
local_cflags.add(i)
|
|
for i in cc_includes[obj]:
|
|
local_c_includes.add(i)
|
|
|
|
if len(set(local_cflags).intersection(disabled_cflags)) > 0:
|
|
continue
|
|
|
|
local_src_files = sorted(local_src_files)
|
|
local_cflags = sorted(local_cflags)
|
|
local_c_includes = sorted(local_c_includes)
|
|
|
|
self.BuildStaticLibrary(target, local_src_files, local_cflags,
|
|
local_c_includes)
|
|
|
|
for target in install:
|
|
# Check if the absolute path to the prebuilt (relative to LTP_ROOT)
|
|
# is disabled. This is helpful in case there are duplicates with basename
|
|
# of the prebuilt.
|
|
# e.g.
|
|
# ./ testcases / kernel / fs / fs_bind / move / test01
|
|
# ./ testcases / kernel / fs / fs_bind / cloneNS / test01
|
|
# ./ testcases / kernel / fs / fs_bind / regression / test01
|
|
# ./ testcases / kernel / fs / fs_bind / rbind / test01
|
|
# ./ testcases / kernel / fs / fs_bind / bind / test01
|
|
if target in disabled_tests:
|
|
continue
|
|
if os.path.basename(target) in disabled_tests:
|
|
continue
|
|
local_src_files = install[target]
|
|
assert len(local_src_files) == 1
|
|
|
|
if target.startswith("testcases/bin/"):
|
|
self.BuildShellScript(target, local_src_files[0])
|
|
else:
|
|
self.BuildPrebuilt(target, local_src_files[0])
|
|
|
|
def WriteAndroidBp(self, output_path):
|
|
'''Write parse result to blueprint file.
|
|
|
|
Args:
|
|
output_path: string
|
|
'''
|
|
with open(output_path, 'a') as f:
|
|
for k in sorted(self._bp_result.iterkeys()):
|
|
f.write('\n'.join(self._bp_result[k]))
|
|
f.write('\n')
|
|
self._bp_result = {}
|
|
|
|
def WriteAndroidMk(self, output_path):
|
|
'''Write parse result to make file.
|
|
|
|
Args:
|
|
output_path: string
|
|
'''
|
|
with open(output_path, 'a') as f:
|
|
for k in sorted(self._mk_result.iterkeys()):
|
|
f.write('\n'.join(self._mk_result[k]))
|
|
f.write('\n')
|
|
self._mk_result = {}
|
|
|
|
def WritePackageList(self, output_path):
|
|
'''Write parse result to package list file.
|
|
|
|
Args:
|
|
output_path: string
|
|
'''
|
|
with open(output_path, 'a') as f:
|
|
f.write('ltp_packages := \\\n ')
|
|
f.write(' \\\n '.join(sorted(self._packages)))
|
|
self._packages = []
|
|
|
|
def ParseAll(self, ltp_root):
|
|
'''Parse outputs from both 'make' and 'make install'.
|
|
|
|
Args:
|
|
ltp_root: string
|
|
'''
|
|
parser = make_parser.MakeParser(ltp_root)
|
|
self.ParseInput(parser.ParseFile(MAKE_DRY_RUN_FILE_NAME), ltp_root)
|
|
parser = make_install_parser.MakeInstallParser(ltp_root)
|
|
self.ParseInput(parser.ParseFile(MAKE_INSTALL_DRY_RUN_FILE_NAME), ltp_root)
|
|
|
|
def GetUnusedCustomCFlagsTargets(self):
|
|
'''Get targets that have custom cflags, but that weren't built.'''
|
|
return list(self._unused_custom_cflags)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description='Generate Android.mk from parsed LTP make output')
|
|
parser.add_argument(
|
|
'--ltp_root', dest='ltp_root', required=True, help='LTP root dir')
|
|
parser.add_argument(
|
|
'--output_mk_path',
|
|
dest='output_mk_path',
|
|
required=True,
|
|
help='output makefile path')
|
|
parser.add_argument(
|
|
'--output_bp_path',
|
|
dest='output_bp_path',
|
|
required=True,
|
|
help='output blueprint path')
|
|
parser.add_argument(
|
|
'--output_plist_path',
|
|
required=True,
|
|
help='output package list path')
|
|
parser.add_argument(
|
|
'--custom_cflags_file',
|
|
dest='custom_cflags_file',
|
|
required=True,
|
|
help='file with custom per-module cflags. empty means no file.')
|
|
args = parser.parse_args()
|
|
|
|
custom_cflags = {}
|
|
if args.custom_cflags_file:
|
|
# The file is expected to just be a JSON map of string -> [string], e.g.
|
|
# {"testcases/kernel/syscalls/getcwd/getcwd02": ["-DFOO", "-O3"]}
|
|
with open(args.custom_cflags_file) as f:
|
|
custom_cflags = json.load(f)
|
|
|
|
generator = BuildGenerator(custom_cflags)
|
|
generator.ParseAll(args.ltp_root)
|
|
generator.WriteAndroidMk(args.output_mk_path)
|
|
generator.WriteAndroidBp(args.output_bp_path)
|
|
generator.WritePackageList(args.output_plist_path)
|
|
|
|
unused_cflags_targs = generator.GetUnusedCustomCFlagsTargets()
|
|
if unused_cflags_targs:
|
|
print 'NOTE: Tests had custom cflags, but were never seen: {}'.format(
|
|
', '.join(unused_cflags_targs))
|
|
|
|
print 'Finished!'
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|