You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
601 lines
23 KiB
601 lines
23 KiB
# Copyright 2018, The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""
|
|
Module Info class used to hold cached module-info.json.
|
|
"""
|
|
|
|
# pylint: disable=line-too-long
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
import atest_utils
|
|
import constants
|
|
|
|
from metrics import metrics
|
|
|
|
# JSON file generated by build system that lists all buildable targets.
|
|
_MODULE_INFO = 'module-info.json'
|
|
# JSON file generated by build system that lists dependencies for java.
|
|
_JAVA_DEP_INFO = 'module_bp_java_deps.json'
|
|
# JSON file generated by build system that lists dependencies for cc.
|
|
_CC_DEP_INFO = 'module_bp_cc_deps.json'
|
|
# JSON file generated by atest merged the content from module-info,
|
|
# module_bp_java_deps.json, and module_bp_cc_deps.
|
|
_MERGED_INFO = 'atest_merged_dep.json'
|
|
|
|
class ModuleInfo:
|
|
"""Class that offers fast/easy lookup for Module related details."""
|
|
|
|
def __init__(self, force_build=False, module_file=None):
|
|
"""Initialize the ModuleInfo object.
|
|
|
|
Load up the module-info.json file and initialize the helper vars.
|
|
|
|
Args:
|
|
force_build: Boolean to indicate if we should rebuild the
|
|
module_info file regardless if it's created or not.
|
|
module_file: String of path to file to load up. Used for testing.
|
|
"""
|
|
module_info_target, name_to_module_info = self._load_module_info_file(
|
|
force_build, module_file)
|
|
self.name_to_module_info = name_to_module_info
|
|
self.module_info_target = module_info_target
|
|
self.path_to_module_info = self._get_path_to_module_info(
|
|
self.name_to_module_info)
|
|
self.root_dir = os.environ.get(constants.ANDROID_BUILD_TOP)
|
|
|
|
@staticmethod
|
|
def _discover_mod_file_and_target(force_build):
|
|
"""Find the module file.
|
|
|
|
Args:
|
|
force_build: Boolean to indicate if we should rebuild the
|
|
module_info file regardless if it's created or not.
|
|
|
|
Returns:
|
|
Tuple of module_info_target and path to module file.
|
|
"""
|
|
logging.debug('Probing and validating module info...')
|
|
module_info_target = None
|
|
root_dir = os.environ.get(constants.ANDROID_BUILD_TOP, '/')
|
|
out_dir = os.environ.get(constants.ANDROID_PRODUCT_OUT, root_dir)
|
|
module_file_path = os.path.join(out_dir, _MODULE_INFO)
|
|
|
|
# Check if the user set a custom out directory by comparing the out_dir
|
|
# to the root_dir.
|
|
if out_dir.find(root_dir) == 0:
|
|
# Make target is simply file path no-absolute to root
|
|
module_info_target = os.path.relpath(module_file_path, root_dir)
|
|
else:
|
|
# If the user has set a custom out directory, generate an absolute
|
|
# path for module info targets.
|
|
logging.debug('User customized out dir!')
|
|
module_file_path = os.path.join(
|
|
os.environ.get(constants.ANDROID_PRODUCT_OUT), _MODULE_INFO)
|
|
module_info_target = module_file_path
|
|
# Make sure module-info exist and could be load properly.
|
|
if not atest_utils.is_valid_json_file(module_file_path) or force_build:
|
|
logging.debug('Generating %s - this is required for '
|
|
'initial runs or forced rebuilds.', _MODULE_INFO)
|
|
build_env = dict(constants.ATEST_BUILD_ENV)
|
|
build_start = time.time()
|
|
if not atest_utils.build([module_info_target],
|
|
verbose=logging.getLogger().isEnabledFor(
|
|
logging.DEBUG), env_vars=build_env):
|
|
sys.exit(constants.EXIT_CODE_BUILD_FAILURE)
|
|
build_duration = time.time() - build_start
|
|
metrics.LocalDetectEvent(
|
|
detect_type=constants.DETECT_TYPE_ONLY_BUILD_MODULE_INFO,
|
|
result=int(build_duration))
|
|
return module_info_target, module_file_path
|
|
|
|
def _load_module_info_file(self, force_build, module_file):
|
|
"""Load the module file.
|
|
|
|
Args:
|
|
force_build: Boolean to indicate if we should rebuild the
|
|
module_info file regardless if it's created or not.
|
|
module_file: String of path to file to load up. Used for testing.
|
|
|
|
Returns:
|
|
Tuple of module_info_target and dict of json.
|
|
"""
|
|
# If module_file is specified, we're testing so we don't care if
|
|
# module_info_target stays None.
|
|
module_info_target = None
|
|
file_path = module_file
|
|
if not file_path:
|
|
module_info_target, file_path = self._discover_mod_file_and_target(
|
|
force_build)
|
|
merged_file_path = self.get_atest_merged_info_path()
|
|
if (not self.need_update_merged_file(force_build)
|
|
and os.path.exists(merged_file_path)):
|
|
file_path = merged_file_path
|
|
logging.debug('Loading %s as module-info.', file_path)
|
|
with open(file_path) as json_file:
|
|
mod_info = json.load(json_file)
|
|
if self.need_update_merged_file(force_build):
|
|
mod_info = self._merge_build_system_infos(mod_info)
|
|
return module_info_target, mod_info
|
|
|
|
@staticmethod
|
|
def _get_path_to_module_info(name_to_module_info):
|
|
"""Return the path_to_module_info dict.
|
|
|
|
Args:
|
|
name_to_module_info: Dict of module name to module info dict.
|
|
|
|
Returns:
|
|
Dict of module path to module info dict.
|
|
"""
|
|
path_to_module_info = {}
|
|
for mod_name, mod_info in name_to_module_info.items():
|
|
# Cross-compiled and multi-arch modules actually all belong to
|
|
# a single target so filter out these extra modules.
|
|
if mod_name != mod_info.get(constants.MODULE_NAME, ''):
|
|
continue
|
|
for path in mod_info.get(constants.MODULE_PATH, []):
|
|
mod_info[constants.MODULE_NAME] = mod_name
|
|
# There could be multiple modules in a path.
|
|
if path in path_to_module_info:
|
|
path_to_module_info[path].append(mod_info)
|
|
else:
|
|
path_to_module_info[path] = [mod_info]
|
|
return path_to_module_info
|
|
|
|
def is_module(self, name):
|
|
"""Return True if name is a module, False otherwise."""
|
|
if self.get_module_info(name):
|
|
return True
|
|
return False
|
|
|
|
def get_paths(self, name):
|
|
"""Return paths of supplied module name, Empty list if non-existent."""
|
|
info = self.get_module_info(name)
|
|
if info:
|
|
return info.get(constants.MODULE_PATH, [])
|
|
return []
|
|
|
|
def get_module_names(self, rel_module_path):
|
|
"""Get the modules that all have module_path.
|
|
|
|
Args:
|
|
rel_module_path: path of module in module-info.json
|
|
|
|
Returns:
|
|
List of module names.
|
|
"""
|
|
return [m.get(constants.MODULE_NAME)
|
|
for m in self.path_to_module_info.get(rel_module_path, [])]
|
|
|
|
def get_module_info(self, mod_name):
|
|
"""Return dict of info for given module name, None if non-existence."""
|
|
module_info = self.name_to_module_info.get(mod_name)
|
|
# Android's build system will automatically adding 2nd arch bitness
|
|
# string at the end of the module name which will make atest could not
|
|
# find the matched module. Rescan the module-info with the matched module
|
|
# name without bitness.
|
|
if not module_info:
|
|
for _, mod_info in self.name_to_module_info.items():
|
|
if mod_name == mod_info.get(constants.MODULE_NAME, ''):
|
|
return mod_info
|
|
return module_info
|
|
|
|
def is_suite_in_compatibility_suites(self, suite, mod_info):
|
|
"""Check if suite exists in the compatibility_suites of module-info.
|
|
|
|
Args:
|
|
suite: A string of suite name.
|
|
mod_info: Dict of module info to check.
|
|
|
|
Returns:
|
|
True if it exists in mod_info, False otherwise.
|
|
"""
|
|
return suite in mod_info.get(constants.MODULE_COMPATIBILITY_SUITES, [])
|
|
|
|
def get_testable_modules(self, suite=None):
|
|
"""Return the testable modules of the given suite name.
|
|
|
|
Args:
|
|
suite: A string of suite name. Set to None to return all testable
|
|
modules.
|
|
|
|
Returns:
|
|
List of testable modules. Empty list if non-existent.
|
|
If suite is None, return all the testable modules in module-info.
|
|
"""
|
|
modules = set()
|
|
for _, info in self.name_to_module_info.items():
|
|
if self.is_testable_module(info):
|
|
if suite:
|
|
if self.is_suite_in_compatibility_suites(suite, info):
|
|
modules.add(info.get(constants.MODULE_NAME))
|
|
else:
|
|
modules.add(info.get(constants.MODULE_NAME))
|
|
return modules
|
|
|
|
def is_testable_module(self, mod_info):
|
|
"""Check if module is something we can test.
|
|
|
|
A module is testable if:
|
|
- it's installed, or
|
|
- it's a robolectric module (or shares path with one).
|
|
|
|
Args:
|
|
mod_info: Dict of module info to check.
|
|
|
|
Returns:
|
|
True if we can test this module, False otherwise.
|
|
"""
|
|
if not mod_info:
|
|
return False
|
|
if mod_info.get(constants.MODULE_INSTALLED) and self.has_test_config(mod_info):
|
|
return True
|
|
if self.is_robolectric_test(mod_info.get(constants.MODULE_NAME)):
|
|
return True
|
|
return False
|
|
|
|
def has_test_config(self, mod_info):
|
|
"""Validate if this module has a test config.
|
|
|
|
A module can have a test config in the following manner:
|
|
- AndroidTest.xml at the module path.
|
|
- test_config be set in module-info.json.
|
|
- Auto-generated config via the auto_test_config key
|
|
in module-info.json.
|
|
|
|
Args:
|
|
mod_info: Dict of module info to check.
|
|
|
|
Returns:
|
|
True if this module has a test config, False otherwise.
|
|
"""
|
|
# Check if test_config in module-info is set.
|
|
for test_config in mod_info.get(constants.MODULE_TEST_CONFIG, []):
|
|
if os.path.isfile(os.path.join(self.root_dir, test_config)):
|
|
return True
|
|
# Check for AndroidTest.xml at the module path.
|
|
for path in mod_info.get(constants.MODULE_PATH, []):
|
|
if os.path.isfile(os.path.join(self.root_dir, path,
|
|
constants.MODULE_CONFIG)):
|
|
return True
|
|
# Check if the module has an auto-generated config.
|
|
return self.is_auto_gen_test_config(mod_info.get(constants.MODULE_NAME))
|
|
|
|
def get_robolectric_test_name(self, module_name):
|
|
"""Returns runnable robolectric module name.
|
|
|
|
There are at least 2 modules in every robolectric module path, return
|
|
the module that we can run as a build target.
|
|
|
|
Arg:
|
|
module_name: String of module.
|
|
|
|
Returns:
|
|
String of module that is the runnable robolectric module, None if
|
|
none could be found.
|
|
"""
|
|
module_name_info = self.get_module_info(module_name)
|
|
if not module_name_info:
|
|
return None
|
|
module_paths = module_name_info.get(constants.MODULE_PATH, [])
|
|
if module_paths:
|
|
for mod in self.get_module_names(module_paths[0]):
|
|
mod_info = self.get_module_info(mod)
|
|
if self.is_robolectric_module(mod_info):
|
|
return mod
|
|
return None
|
|
|
|
def is_robolectric_test(self, module_name):
|
|
"""Check if module is a robolectric test.
|
|
|
|
A module can be a robolectric test if the specified module has their
|
|
class set as ROBOLECTRIC (or shares their path with a module that does).
|
|
|
|
Args:
|
|
module_name: String of module to check.
|
|
|
|
Returns:
|
|
True if the module is a robolectric module, else False.
|
|
"""
|
|
# Check 1, module class is ROBOLECTRIC
|
|
mod_info = self.get_module_info(module_name)
|
|
if self.is_robolectric_module(mod_info):
|
|
return True
|
|
# Check 2, shared modules in the path have class ROBOLECTRIC_CLASS.
|
|
if self.get_robolectric_test_name(module_name):
|
|
return True
|
|
return False
|
|
|
|
def is_auto_gen_test_config(self, module_name):
|
|
"""Check if the test config file will be generated automatically.
|
|
|
|
Args:
|
|
module_name: A string of the module name.
|
|
|
|
Returns:
|
|
True if the test config file will be generated automatically.
|
|
"""
|
|
if self.is_module(module_name):
|
|
mod_info = self.get_module_info(module_name)
|
|
auto_test_config = mod_info.get('auto_test_config', [])
|
|
return auto_test_config and auto_test_config[0]
|
|
return False
|
|
|
|
def is_robolectric_module(self, mod_info):
|
|
"""Check if a module is a robolectric module.
|
|
|
|
Args:
|
|
mod_info: ModuleInfo to check.
|
|
|
|
Returns:
|
|
True if module is a robolectric module, False otherwise.
|
|
"""
|
|
if mod_info:
|
|
return (mod_info.get(constants.MODULE_CLASS, [None])[0] ==
|
|
constants.MODULE_CLASS_ROBOLECTRIC)
|
|
return False
|
|
|
|
def is_native_test(self, module_name):
|
|
"""Check if the input module is a native test.
|
|
|
|
Args:
|
|
module_name: A string of the module name.
|
|
|
|
Returns:
|
|
True if the test is a native test, False otherwise.
|
|
"""
|
|
mod_info = self.get_module_info(module_name)
|
|
return constants.MODULE_CLASS_NATIVE_TESTS in mod_info.get(
|
|
constants.MODULE_CLASS, [])
|
|
|
|
def has_mainline_modules(self, module_name, mainline_modules):
|
|
"""Check if the mainline modules are in module-info.
|
|
|
|
Args:
|
|
module_name: A string of the module name.
|
|
mainline_modules: A list of mainline modules.
|
|
|
|
Returns:
|
|
True if mainline_modules is in module-info, False otherwise.
|
|
"""
|
|
# TODO: (b/165425972)Check AndroidTest.xml or specific test config.
|
|
mod_info = self.get_module_info(module_name)
|
|
if mainline_modules in mod_info.get(constants.MODULE_MAINLINE_MODULES,
|
|
[]):
|
|
return True
|
|
return False
|
|
|
|
def generate_atest_merged_dep_file(self):
|
|
"""Method for generating atest_merged_dep.json."""
|
|
self._merge_build_system_infos(self.name_to_module_info,
|
|
self.get_java_dep_info_path(),
|
|
self.get_cc_dep_info_path())
|
|
|
|
def _merge_build_system_infos(self, name_to_module_info,
|
|
java_bp_info_path=None, cc_bp_info_path=None):
|
|
"""Merge the full build system's info to name_to_module_info.
|
|
|
|
Args:
|
|
name_to_module_info: Dict of module name to module info dict.
|
|
java_bp_info_path: String of path to java dep file to load up.
|
|
Used for testing.
|
|
cc_bp_info_path: String of path to cc dep file to load up.
|
|
Used for testing.
|
|
|
|
Returns:
|
|
Dict of merged json of input def_file_path and name_to_module_info.
|
|
"""
|
|
# Merge _JAVA_DEP_INFO
|
|
if not java_bp_info_path:
|
|
java_bp_info_path = self.get_java_dep_info_path()
|
|
if atest_utils.is_valid_json_file(java_bp_info_path):
|
|
with open(java_bp_info_path) as json_file:
|
|
java_bp_infos = json.load(json_file)
|
|
logging.debug('Merging Java build info: %s', java_bp_info_path)
|
|
name_to_module_info = self._merge_soong_info(
|
|
name_to_module_info, java_bp_infos)
|
|
# Merge _CC_DEP_INFO
|
|
if not cc_bp_info_path:
|
|
cc_bp_info_path = self.get_cc_dep_info_path()
|
|
if atest_utils.is_valid_json_file(cc_bp_info_path):
|
|
with open(cc_bp_info_path) as json_file:
|
|
cc_bp_infos = json.load(json_file)
|
|
logging.debug('Merging CC build info: %s', cc_bp_info_path)
|
|
# CC's dep json format is different with java.
|
|
# Below is the example content:
|
|
# {
|
|
# "clang": "${ANDROID_ROOT}/bin/clang",
|
|
# "clang++": "${ANDROID_ROOT}/bin/clang++",
|
|
# "modules": {
|
|
# "ACameraNdkVendorTest": {
|
|
# "path": [
|
|
# "frameworks/av/camera/ndk"
|
|
# ],
|
|
# "srcs": [
|
|
# "frameworks/tests/AImageVendorTest.cpp",
|
|
# "frameworks/tests/ACameraManagerTest.cpp"
|
|
# ],
|
|
name_to_module_info = self._merge_soong_info(
|
|
name_to_module_info, cc_bp_infos.get('modules', {}))
|
|
return name_to_module_info
|
|
|
|
def _merge_soong_info(self, name_to_module_info, mod_bp_infos):
|
|
"""Merge the dependency and srcs in mod_bp_infos to name_to_module_info.
|
|
|
|
Args:
|
|
name_to_module_info: Dict of module name to module info dict.
|
|
mod_bp_infos: Dict of module name to bp's module info dict.
|
|
|
|
Returns:
|
|
Dict of merged json of input def_file_path and name_to_module_info.
|
|
"""
|
|
merge_items = [constants.MODULE_DEPENDENCIES, constants.MODULE_SRCS]
|
|
for module_name, dep_info in mod_bp_infos.items():
|
|
if name_to_module_info.get(module_name, None):
|
|
mod_info = name_to_module_info.get(module_name)
|
|
for merge_item in merge_items:
|
|
dep_info_values = dep_info.get(merge_item, [])
|
|
mod_info_values = mod_info.get(merge_item, [])
|
|
for dep_info_value in dep_info_values:
|
|
if dep_info_value not in mod_info_values:
|
|
mod_info_values.append(dep_info_value)
|
|
mod_info_values.sort()
|
|
name_to_module_info[
|
|
module_name][merge_item] = mod_info_values
|
|
output_file = self.get_atest_merged_info_path()
|
|
if not os.path.isdir(os.path.dirname(output_file)):
|
|
os.makedirs(os.path.dirname(output_file))
|
|
# b/178559543 saving merged module info in a temp file and copying it to
|
|
# atest_merged_dep.json can eliminate the possibility of accessing it
|
|
# concurrently and resulting in invalid JSON format.
|
|
temp_file = tempfile.NamedTemporaryFile()
|
|
with open(temp_file.name, 'w') as _temp:
|
|
json.dump(name_to_module_info, _temp, indent=0)
|
|
shutil.copy(temp_file.name, output_file)
|
|
temp_file.close()
|
|
return name_to_module_info
|
|
|
|
def get_module_dependency(self, module_name, depend_on=None):
|
|
"""Get the dependency sets for input module.
|
|
|
|
Recursively find all the dependencies of the input module.
|
|
|
|
Args:
|
|
module_name: String of module to check.
|
|
depend_on: The list of parent dependencies.
|
|
|
|
Returns:
|
|
Set of dependency modules.
|
|
"""
|
|
if not depend_on:
|
|
depend_on = set()
|
|
deps = set()
|
|
mod_info = self.get_module_info(module_name)
|
|
if not mod_info:
|
|
return deps
|
|
mod_deps = set(mod_info.get(constants.MODULE_DEPENDENCIES, []))
|
|
# Remove item in deps if it already in depend_on:
|
|
mod_deps = mod_deps - depend_on
|
|
deps = deps.union(mod_deps)
|
|
for mod_dep in mod_deps:
|
|
deps = deps.union(set(self.get_module_dependency(
|
|
mod_dep, depend_on=depend_on.union(deps))))
|
|
return deps
|
|
|
|
def get_install_module_dependency(self, module_name, depend_on=None):
|
|
"""Get the dependency set for the given modules with installed path.
|
|
|
|
Args:
|
|
module_name: String of module to check.
|
|
depend_on: The list of parent dependencies.
|
|
|
|
Returns:
|
|
Set of dependency modules which has installed path.
|
|
"""
|
|
install_deps = set()
|
|
deps = self.get_module_dependency(module_name, depend_on)
|
|
logging.debug('%s depends on: %s', module_name, deps)
|
|
for module in deps:
|
|
mod_info = self.get_module_info(module)
|
|
if mod_info and mod_info.get(constants.MODULE_INSTALLED, []):
|
|
install_deps.add(module)
|
|
logging.debug('modules %s required by %s were not installed',
|
|
install_deps, module_name)
|
|
return install_deps
|
|
|
|
@staticmethod
|
|
def get_atest_merged_info_path():
|
|
"""Returns the path for atest_merged_dep.json.
|
|
|
|
Returns:
|
|
String for atest_merged_dep.json.
|
|
"""
|
|
return os.path.join(atest_utils.get_build_out_dir(),
|
|
'soong', _MERGED_INFO)
|
|
|
|
@staticmethod
|
|
def get_java_dep_info_path():
|
|
"""Returns the path for atest_merged_dep.json.
|
|
|
|
Returns:
|
|
String for atest_merged_dep.json.
|
|
"""
|
|
return os.path.join(atest_utils.get_build_out_dir(),
|
|
'soong', _JAVA_DEP_INFO)
|
|
|
|
@staticmethod
|
|
def get_cc_dep_info_path():
|
|
"""Returns the path for atest_merged_dep.json.
|
|
|
|
Returns:
|
|
String for atest_merged_dep.json.
|
|
"""
|
|
return os.path.join(atest_utils.get_build_out_dir(),
|
|
'soong', _CC_DEP_INFO)
|
|
|
|
def has_soong_info(self):
|
|
"""Ensure the existence of soong info files.
|
|
|
|
Returns:
|
|
True if soong info need to merge, false otherwise.
|
|
"""
|
|
return (os.path.isfile(self.get_java_dep_info_path()) and
|
|
os.path.isfile(self.get_cc_dep_info_path()))
|
|
|
|
def need_update_merged_file(self, force_build=False):
|
|
"""Check if need to update/generated atest_merged_dep.
|
|
|
|
If force_build, always update merged info.
|
|
If not force build, if soong info exist but merged inforamtion not exist,
|
|
need to update merged file.
|
|
|
|
Args:
|
|
force_build: Boolean to indicate that if user want to rebuild
|
|
module_info file regardless if it's created or not.
|
|
|
|
Returns:
|
|
True if atest_merged_dep should be updated, false otherwise.
|
|
"""
|
|
return (force_build or
|
|
(self.has_soong_info() and
|
|
not os.path.exists(self.get_atest_merged_info_path())))
|
|
|
|
def is_unit_test(self, mod_info):
|
|
"""Return True if input module is unit test, False otherwise.
|
|
|
|
Args:
|
|
mod_info: ModuleInfo to check.
|
|
|
|
Returns:
|
|
True if if input module is unit test, False otherwise.
|
|
"""
|
|
return mod_info.get(constants.MODULE_IS_UNIT_TEST, '') == 'true'
|
|
|
|
def get_all_unit_tests(self):
|
|
"""Get a list of all the module names which are unit tests."""
|
|
unit_tests = []
|
|
for mod_name, mod_info in self.name_to_module_info.items():
|
|
if mod_info.get(constants.MODULE_NAME, '') == mod_name:
|
|
if self.is_unit_test(mod_info):
|
|
unit_tests.append(mod_name)
|
|
return unit_tests
|