# Copyright 2018, The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module Info class used to hold cached module-info.json. """ # pylint: disable=line-too-long import json import logging import os import shutil import sys import tempfile import time import atest_utils import constants from metrics import metrics # JSON file generated by build system that lists all buildable targets. _MODULE_INFO = 'module-info.json' # JSON file generated by build system that lists dependencies for java. _JAVA_DEP_INFO = 'module_bp_java_deps.json' # JSON file generated by build system that lists dependencies for cc. _CC_DEP_INFO = 'module_bp_cc_deps.json' # JSON file generated by atest merged the content from module-info, # module_bp_java_deps.json, and module_bp_cc_deps. _MERGED_INFO = 'atest_merged_dep.json' class ModuleInfo: """Class that offers fast/easy lookup for Module related details.""" def __init__(self, force_build=False, module_file=None): """Initialize the ModuleInfo object. Load up the module-info.json file and initialize the helper vars. Args: force_build: Boolean to indicate if we should rebuild the module_info file regardless if it's created or not. module_file: String of path to file to load up. Used for testing. """ module_info_target, name_to_module_info = self._load_module_info_file( force_build, module_file) self.name_to_module_info = name_to_module_info self.module_info_target = module_info_target self.path_to_module_info = self._get_path_to_module_info( self.name_to_module_info) self.root_dir = os.environ.get(constants.ANDROID_BUILD_TOP) @staticmethod def _discover_mod_file_and_target(force_build): """Find the module file. Args: force_build: Boolean to indicate if we should rebuild the module_info file regardless if it's created or not. Returns: Tuple of module_info_target and path to module file. """ logging.debug('Probing and validating module info...') module_info_target = None root_dir = os.environ.get(constants.ANDROID_BUILD_TOP, '/') out_dir = os.environ.get(constants.ANDROID_PRODUCT_OUT, root_dir) module_file_path = os.path.join(out_dir, _MODULE_INFO) # Check if the user set a custom out directory by comparing the out_dir # to the root_dir. if out_dir.find(root_dir) == 0: # Make target is simply file path no-absolute to root module_info_target = os.path.relpath(module_file_path, root_dir) else: # If the user has set a custom out directory, generate an absolute # path for module info targets. logging.debug('User customized out dir!') module_file_path = os.path.join( os.environ.get(constants.ANDROID_PRODUCT_OUT), _MODULE_INFO) module_info_target = module_file_path # Make sure module-info exist and could be load properly. if not atest_utils.is_valid_json_file(module_file_path) or force_build: logging.debug('Generating %s - this is required for ' 'initial runs or forced rebuilds.', _MODULE_INFO) build_env = dict(constants.ATEST_BUILD_ENV) build_start = time.time() if not atest_utils.build([module_info_target], verbose=logging.getLogger().isEnabledFor( logging.DEBUG), env_vars=build_env): sys.exit(constants.EXIT_CODE_BUILD_FAILURE) build_duration = time.time() - build_start metrics.LocalDetectEvent( detect_type=constants.DETECT_TYPE_ONLY_BUILD_MODULE_INFO, result=int(build_duration)) return module_info_target, module_file_path def _load_module_info_file(self, force_build, module_file): """Load the module file. Args: force_build: Boolean to indicate if we should rebuild the module_info file regardless if it's created or not. module_file: String of path to file to load up. Used for testing. Returns: Tuple of module_info_target and dict of json. """ # If module_file is specified, we're testing so we don't care if # module_info_target stays None. module_info_target = None file_path = module_file if not file_path: module_info_target, file_path = self._discover_mod_file_and_target( force_build) merged_file_path = self.get_atest_merged_info_path() if (not self.need_update_merged_file(force_build) and os.path.exists(merged_file_path)): file_path = merged_file_path logging.debug('Loading %s as module-info.', file_path) with open(file_path) as json_file: mod_info = json.load(json_file) if self.need_update_merged_file(force_build): mod_info = self._merge_build_system_infos(mod_info) return module_info_target, mod_info @staticmethod def _get_path_to_module_info(name_to_module_info): """Return the path_to_module_info dict. Args: name_to_module_info: Dict of module name to module info dict. Returns: Dict of module path to module info dict. """ path_to_module_info = {} for mod_name, mod_info in name_to_module_info.items(): # Cross-compiled and multi-arch modules actually all belong to # a single target so filter out these extra modules. if mod_name != mod_info.get(constants.MODULE_NAME, ''): continue for path in mod_info.get(constants.MODULE_PATH, []): mod_info[constants.MODULE_NAME] = mod_name # There could be multiple modules in a path. if path in path_to_module_info: path_to_module_info[path].append(mod_info) else: path_to_module_info[path] = [mod_info] return path_to_module_info def is_module(self, name): """Return True if name is a module, False otherwise.""" if self.get_module_info(name): return True return False def get_paths(self, name): """Return paths of supplied module name, Empty list if non-existent.""" info = self.get_module_info(name) if info: return info.get(constants.MODULE_PATH, []) return [] def get_module_names(self, rel_module_path): """Get the modules that all have module_path. Args: rel_module_path: path of module in module-info.json Returns: List of module names. """ return [m.get(constants.MODULE_NAME) for m in self.path_to_module_info.get(rel_module_path, [])] def get_module_info(self, mod_name): """Return dict of info for given module name, None if non-existence.""" module_info = self.name_to_module_info.get(mod_name) # Android's build system will automatically adding 2nd arch bitness # string at the end of the module name which will make atest could not # find the matched module. Rescan the module-info with the matched module # name without bitness. if not module_info: for _, mod_info in self.name_to_module_info.items(): if mod_name == mod_info.get(constants.MODULE_NAME, ''): return mod_info return module_info def is_suite_in_compatibility_suites(self, suite, mod_info): """Check if suite exists in the compatibility_suites of module-info. Args: suite: A string of suite name. mod_info: Dict of module info to check. Returns: True if it exists in mod_info, False otherwise. """ return suite in mod_info.get(constants.MODULE_COMPATIBILITY_SUITES, []) def get_testable_modules(self, suite=None): """Return the testable modules of the given suite name. Args: suite: A string of suite name. Set to None to return all testable modules. Returns: List of testable modules. Empty list if non-existent. If suite is None, return all the testable modules in module-info. """ modules = set() for _, info in self.name_to_module_info.items(): if self.is_testable_module(info): if suite: if self.is_suite_in_compatibility_suites(suite, info): modules.add(info.get(constants.MODULE_NAME)) else: modules.add(info.get(constants.MODULE_NAME)) return modules def is_testable_module(self, mod_info): """Check if module is something we can test. A module is testable if: - it's installed, or - it's a robolectric module (or shares path with one). Args: mod_info: Dict of module info to check. Returns: True if we can test this module, False otherwise. """ if not mod_info: return False if mod_info.get(constants.MODULE_INSTALLED) and self.has_test_config(mod_info): return True if self.is_robolectric_test(mod_info.get(constants.MODULE_NAME)): return True return False def has_test_config(self, mod_info): """Validate if this module has a test config. A module can have a test config in the following manner: - AndroidTest.xml at the module path. - test_config be set in module-info.json. - Auto-generated config via the auto_test_config key in module-info.json. Args: mod_info: Dict of module info to check. Returns: True if this module has a test config, False otherwise. """ # Check if test_config in module-info is set. for test_config in mod_info.get(constants.MODULE_TEST_CONFIG, []): if os.path.isfile(os.path.join(self.root_dir, test_config)): return True # Check for AndroidTest.xml at the module path. for path in mod_info.get(constants.MODULE_PATH, []): if os.path.isfile(os.path.join(self.root_dir, path, constants.MODULE_CONFIG)): return True # Check if the module has an auto-generated config. return self.is_auto_gen_test_config(mod_info.get(constants.MODULE_NAME)) def get_robolectric_test_name(self, module_name): """Returns runnable robolectric module name. There are at least 2 modules in every robolectric module path, return the module that we can run as a build target. Arg: module_name: String of module. Returns: String of module that is the runnable robolectric module, None if none could be found. """ module_name_info = self.get_module_info(module_name) if not module_name_info: return None module_paths = module_name_info.get(constants.MODULE_PATH, []) if module_paths: for mod in self.get_module_names(module_paths[0]): mod_info = self.get_module_info(mod) if self.is_robolectric_module(mod_info): return mod return None def is_robolectric_test(self, module_name): """Check if module is a robolectric test. A module can be a robolectric test if the specified module has their class set as ROBOLECTRIC (or shares their path with a module that does). Args: module_name: String of module to check. Returns: True if the module is a robolectric module, else False. """ # Check 1, module class is ROBOLECTRIC mod_info = self.get_module_info(module_name) if self.is_robolectric_module(mod_info): return True # Check 2, shared modules in the path have class ROBOLECTRIC_CLASS. if self.get_robolectric_test_name(module_name): return True return False def is_auto_gen_test_config(self, module_name): """Check if the test config file will be generated automatically. Args: module_name: A string of the module name. Returns: True if the test config file will be generated automatically. """ if self.is_module(module_name): mod_info = self.get_module_info(module_name) auto_test_config = mod_info.get('auto_test_config', []) return auto_test_config and auto_test_config[0] return False def is_robolectric_module(self, mod_info): """Check if a module is a robolectric module. Args: mod_info: ModuleInfo to check. Returns: True if module is a robolectric module, False otherwise. """ if mod_info: return (mod_info.get(constants.MODULE_CLASS, [None])[0] == constants.MODULE_CLASS_ROBOLECTRIC) return False def is_native_test(self, module_name): """Check if the input module is a native test. Args: module_name: A string of the module name. Returns: True if the test is a native test, False otherwise. """ mod_info = self.get_module_info(module_name) return constants.MODULE_CLASS_NATIVE_TESTS in mod_info.get( constants.MODULE_CLASS, []) def has_mainline_modules(self, module_name, mainline_modules): """Check if the mainline modules are in module-info. Args: module_name: A string of the module name. mainline_modules: A list of mainline modules. Returns: True if mainline_modules is in module-info, False otherwise. """ # TODO: (b/165425972)Check AndroidTest.xml or specific test config. mod_info = self.get_module_info(module_name) if mainline_modules in mod_info.get(constants.MODULE_MAINLINE_MODULES, []): return True return False def generate_atest_merged_dep_file(self): """Method for generating atest_merged_dep.json.""" self._merge_build_system_infos(self.name_to_module_info, self.get_java_dep_info_path(), self.get_cc_dep_info_path()) def _merge_build_system_infos(self, name_to_module_info, java_bp_info_path=None, cc_bp_info_path=None): """Merge the full build system's info to name_to_module_info. Args: name_to_module_info: Dict of module name to module info dict. java_bp_info_path: String of path to java dep file to load up. Used for testing. cc_bp_info_path: String of path to cc dep file to load up. Used for testing. Returns: Dict of merged json of input def_file_path and name_to_module_info. """ # Merge _JAVA_DEP_INFO if not java_bp_info_path: java_bp_info_path = self.get_java_dep_info_path() if atest_utils.is_valid_json_file(java_bp_info_path): with open(java_bp_info_path) as json_file: java_bp_infos = json.load(json_file) logging.debug('Merging Java build info: %s', java_bp_info_path) name_to_module_info = self._merge_soong_info( name_to_module_info, java_bp_infos) # Merge _CC_DEP_INFO if not cc_bp_info_path: cc_bp_info_path = self.get_cc_dep_info_path() if atest_utils.is_valid_json_file(cc_bp_info_path): with open(cc_bp_info_path) as json_file: cc_bp_infos = json.load(json_file) logging.debug('Merging CC build info: %s', cc_bp_info_path) # CC's dep json format is different with java. # Below is the example content: # { # "clang": "${ANDROID_ROOT}/bin/clang", # "clang++": "${ANDROID_ROOT}/bin/clang++", # "modules": { # "ACameraNdkVendorTest": { # "path": [ # "frameworks/av/camera/ndk" # ], # "srcs": [ # "frameworks/tests/AImageVendorTest.cpp", # "frameworks/tests/ACameraManagerTest.cpp" # ], name_to_module_info = self._merge_soong_info( name_to_module_info, cc_bp_infos.get('modules', {})) return name_to_module_info def _merge_soong_info(self, name_to_module_info, mod_bp_infos): """Merge the dependency and srcs in mod_bp_infos to name_to_module_info. Args: name_to_module_info: Dict of module name to module info dict. mod_bp_infos: Dict of module name to bp's module info dict. Returns: Dict of merged json of input def_file_path and name_to_module_info. """ merge_items = [constants.MODULE_DEPENDENCIES, constants.MODULE_SRCS] for module_name, dep_info in mod_bp_infos.items(): if name_to_module_info.get(module_name, None): mod_info = name_to_module_info.get(module_name) for merge_item in merge_items: dep_info_values = dep_info.get(merge_item, []) mod_info_values = mod_info.get(merge_item, []) for dep_info_value in dep_info_values: if dep_info_value not in mod_info_values: mod_info_values.append(dep_info_value) mod_info_values.sort() name_to_module_info[ module_name][merge_item] = mod_info_values output_file = self.get_atest_merged_info_path() if not os.path.isdir(os.path.dirname(output_file)): os.makedirs(os.path.dirname(output_file)) # b/178559543 saving merged module info in a temp file and copying it to # atest_merged_dep.json can eliminate the possibility of accessing it # concurrently and resulting in invalid JSON format. temp_file = tempfile.NamedTemporaryFile() with open(temp_file.name, 'w') as _temp: json.dump(name_to_module_info, _temp, indent=0) shutil.copy(temp_file.name, output_file) temp_file.close() return name_to_module_info def get_module_dependency(self, module_name, depend_on=None): """Get the dependency sets for input module. Recursively find all the dependencies of the input module. Args: module_name: String of module to check. depend_on: The list of parent dependencies. Returns: Set of dependency modules. """ if not depend_on: depend_on = set() deps = set() mod_info = self.get_module_info(module_name) if not mod_info: return deps mod_deps = set(mod_info.get(constants.MODULE_DEPENDENCIES, [])) # Remove item in deps if it already in depend_on: mod_deps = mod_deps - depend_on deps = deps.union(mod_deps) for mod_dep in mod_deps: deps = deps.union(set(self.get_module_dependency( mod_dep, depend_on=depend_on.union(deps)))) return deps def get_install_module_dependency(self, module_name, depend_on=None): """Get the dependency set for the given modules with installed path. Args: module_name: String of module to check. depend_on: The list of parent dependencies. Returns: Set of dependency modules which has installed path. """ install_deps = set() deps = self.get_module_dependency(module_name, depend_on) logging.debug('%s depends on: %s', module_name, deps) for module in deps: mod_info = self.get_module_info(module) if mod_info and mod_info.get(constants.MODULE_INSTALLED, []): install_deps.add(module) logging.debug('modules %s required by %s were not installed', install_deps, module_name) return install_deps @staticmethod def get_atest_merged_info_path(): """Returns the path for atest_merged_dep.json. Returns: String for atest_merged_dep.json. """ return os.path.join(atest_utils.get_build_out_dir(), 'soong', _MERGED_INFO) @staticmethod def get_java_dep_info_path(): """Returns the path for atest_merged_dep.json. Returns: String for atest_merged_dep.json. """ return os.path.join(atest_utils.get_build_out_dir(), 'soong', _JAVA_DEP_INFO) @staticmethod def get_cc_dep_info_path(): """Returns the path for atest_merged_dep.json. Returns: String for atest_merged_dep.json. """ return os.path.join(atest_utils.get_build_out_dir(), 'soong', _CC_DEP_INFO) def has_soong_info(self): """Ensure the existence of soong info files. Returns: True if soong info need to merge, false otherwise. """ return (os.path.isfile(self.get_java_dep_info_path()) and os.path.isfile(self.get_cc_dep_info_path())) def need_update_merged_file(self, force_build=False): """Check if need to update/generated atest_merged_dep. If force_build, always update merged info. If not force build, if soong info exist but merged inforamtion not exist, need to update merged file. Args: force_build: Boolean to indicate that if user want to rebuild module_info file regardless if it's created or not. Returns: True if atest_merged_dep should be updated, false otherwise. """ return (force_build or (self.has_soong_info() and not os.path.exists(self.get_atest_merged_info_path()))) def is_unit_test(self, mod_info): """Return True if input module is unit test, False otherwise. Args: mod_info: ModuleInfo to check. Returns: True if if input module is unit test, False otherwise. """ return mod_info.get(constants.MODULE_IS_UNIT_TEST, '') == 'true' def get_all_unit_tests(self): """Get a list of all the module names which are unit tests.""" unit_tests = [] for mod_name, mod_info in self.name_to_module_info.items(): if mod_info.get(constants.MODULE_NAME, '') == mod_name: if self.is_unit_test(mod_info): unit_tests.append(mod_name) return unit_tests