You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

658 lines
27 KiB

#!/usr/bin/env python3
#
# Copyright 2018 - The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Project information."""
from __future__ import absolute_import
import logging
import os
import time
from aidegen import constant
from aidegen.lib import aidegen_metrics
from aidegen.lib import common_util
from aidegen.lib import errors
from aidegen.lib import module_info
from aidegen.lib import project_config
from aidegen.lib import source_locator
from aidegen.idea import iml
from atest import atest_utils
_CONVERT_MK_URL = ('https://android.googlesource.com/platform/build/soong/'
'#convert-android_mk-files')
_ROBOLECTRIC_MODULE = 'Robolectric_all'
_NOT_TARGET = ('The module %s does not contain any Java or Kotlin file, '
'therefore we skip this module in the project.')
# The module fake-framework have the same package name with framework but empty
# content. It will impact the dependency for framework when referencing the
# package from fake-framework in IntelliJ.
_EXCLUDE_MODULES = ['fake-framework']
# When we use atest_utils.build(), there is a command length limit on
# soong_ui.bash. We reserve 5000 characters for rewriting the command line
# in soong_ui.bash.
_CMD_LENGTH_BUFFER = 5000
# For each argument, it need a space to separate following argument.
_BLANK_SIZE = 1
_CORE_MODULES = [constant.FRAMEWORK_ALL, constant.CORE_ALL,
'org.apache.http.legacy.stubs.system']
class ProjectInfo:
"""Project information.
Users should call config_project first before starting using ProjectInfo.
Class attributes:
modules_info: An AidegenModuleInfo instance whose name_to_module_info is
combining module-info.json with module_bp_java_deps.json.
projects: A list of instances of ProjectInfo that are generated in an
AIDEGen command.
Attributes:
project_absolute_path: The absolute path of the project.
project_relative_path: The relative path of the project to
common_util.get_android_root_dir().
project_module_names: A set of module names under project_absolute_path
directory or it's subdirectories.
dep_modules: A dict has recursively dependent modules of
project_module_names.
iml_path: The project's iml file path.
source_path: A dictionary to keep following data:
source_folder_path: A set contains the source folder
relative paths.
test_folder_path: A set contains the test folder relative
paths.
jar_path: A set contains the jar file paths.
jar_module_path: A dictionary contains the jar file and
the module's path mapping, only used in
Eclipse.
r_java_path: A set contains the relative path to the
R.java files, only used in Eclipse.
srcjar_path: A source content descriptor only used in
IntelliJ.
e.g. out/.../aapt2.srcjar!/
The "!/" is a content descriptor for
compressed files in IntelliJ.
is_main_project: A boolean to verify the project is main project.
dependencies: A list of dependency projects' iml file names, e.g. base,
framework-all.
iml_name: The iml project file name of this project.
rel_out_soong_jar_path: A string of relative project path in the
'out/soong/.intermediates' directory, e.g., if
self.project_relative_path = 'frameworks/base'
the rel_out_soong_jar_path should be
'out/soong/.intermediates/frameworks/base/'.
"""
modules_info = None
def __init__(self, target=None, is_main_project=False):
"""ProjectInfo initialize.
Args:
target: Includes target module or project path from user input, when
locating the target, project with matching module name of
the given target has a higher priority than project path.
is_main_project: A boolean, default is False. True if the target is
the main project, otherwise False.
"""
rel_path, abs_path = common_util.get_related_paths(
self.modules_info, target)
self.module_name = self.get_target_name(target, abs_path)
self.is_main_project = is_main_project
self.project_module_names = set(
self.modules_info.get_module_names(rel_path))
self.project_relative_path = rel_path
self.project_absolute_path = abs_path
self.iml_path = ''
self._set_default_modues()
self._init_source_path()
if target == constant.FRAMEWORK_ALL:
self.dep_modules = self.get_dep_modules([target])
else:
self.dep_modules = self.get_dep_modules()
self._filter_out_modules()
self.dependencies = []
self.iml_name = iml.IMLGenerator.get_unique_iml_name(abs_path)
self.rel_out_soong_jar_path = self._get_rel_project_out_soong_jar_path()
def _set_default_modues(self):
"""Append default hard-code modules, source paths and jar files.
1. framework: Framework module is always needed for dependencies but it
might not always be located by module dependency.
2. org.apache.http.legacy.stubs.system: The module can't be located
through module dependency. Without it, a lot of java files will have
error of "cannot resolve symbol" in IntelliJ since they import
packages android.Manifest and com.android.internal.R.
"""
# Set the default modules framework-all and core-all as the core
# dependency modules.
self.project_module_names.update(_CORE_MODULES)
def _init_source_path(self):
"""Initialize source_path dictionary."""
self.source_path = {
'source_folder_path': set(),
'test_folder_path': set(),
'jar_path': set(),
'jar_module_path': dict(),
'r_java_path': set(),
'srcjar_path': set()
}
def _search_android_make_files(self):
"""Search project and dependency modules contain Android.mk files.
If there is only Android.mk but no Android.bp, we'll show the warning
message, otherwise we won't.
Yields:
A string: the relative path of Android.mk.
"""
if (common_util.exist_android_mk(self.project_absolute_path) and
not common_util.exist_android_bp(self.project_absolute_path)):
yield '\t' + os.path.join(self.project_relative_path,
constant.ANDROID_MK)
for mod_name in self.dep_modules:
rel_path, abs_path = common_util.get_related_paths(
self.modules_info, mod_name)
if rel_path and abs_path:
if (common_util.exist_android_mk(abs_path)
and not common_util.exist_android_bp(abs_path)):
yield '\t' + os.path.join(rel_path, constant.ANDROID_MK)
def _get_modules_under_project_path(self, rel_path):
"""Find qualified modules under the rel_path.
Find modules which contain any Java or Kotlin file as a target module.
If it's the whole source tree project, add all modules into it.
Args:
rel_path: A string, the project's relative path.
Returns:
A set of module names.
"""
logging.info('Find modules contain any Java or Kotlin file under %s.',
rel_path)
if rel_path == '':
return self.modules_info.name_to_module_info.keys()
modules = set()
root_dir = common_util.get_android_root_dir()
for name, data in self.modules_info.name_to_module_info.items():
if module_info.AidegenModuleInfo.is_project_path_relative_module(
data, rel_path):
if common_util.check_java_or_kotlin_file_exists(
os.path.join(root_dir, data[constant.KEY_PATH][0])):
modules.add(name)
else:
logging.debug(_NOT_TARGET, name)
return modules
def _get_robolectric_dep_module(self, modules):
"""Return the robolectric module set as dependency if any module is a
robolectric test.
Args:
modules: A set of modules.
Returns:
A set with a robolectric_all module name if one of the modules
needs the robolectric test module. Otherwise return empty list.
"""
for module in modules:
if self.modules_info.is_robolectric_test(module):
return {_ROBOLECTRIC_MODULE}
return set()
def _filter_out_modules(self):
"""Filter out unnecessary modules."""
for module in _EXCLUDE_MODULES:
self.dep_modules.pop(module, None)
def get_dep_modules(self, module_names=None, depth=0):
"""Recursively find dependent modules of the project.
Find dependent modules by dependencies parameter of each module.
For example:
The module_names is ['m1'].
The modules_info is
{
'm1': {'dependencies': ['m2'], 'path': ['path_to_m1']},
'm2': {'path': ['path_to_m4']},
'm3': {'path': ['path_to_m1']}
'm4': {'path': []}
}
The result dependent modules are:
{
'm1': {'dependencies': ['m2'], 'path': ['path_to_m1']
'depth': 0},
'm2': {'path': ['path_to_m4'], 'depth': 1},
'm3': {'path': ['path_to_m1'], 'depth': 0}
}
Note that:
1. m4 is not in the result as it's not among dependent modules.
2. m3 is in the result as it has the same path to m1.
Args:
module_names: A set of module names.
depth: An integer shows the depth of module dependency referenced by
source. Zero means the max module depth.
Returns:
deps: A dict contains all dependent modules data of given modules.
"""
dep = {}
children = set()
if not module_names:
module_names = self.project_module_names
module_names.update(
self._get_modules_under_project_path(
self.project_relative_path))
module_names.update(self._get_robolectric_dep_module(module_names))
self.project_module_names = set()
for name in module_names:
if (name in self.modules_info.name_to_module_info
and name not in self.project_module_names):
dep[name] = self.modules_info.name_to_module_info[name]
dep[name][constant.KEY_DEPTH] = depth
self.project_module_names.add(name)
if (constant.KEY_DEPENDENCIES in dep[name]
and dep[name][constant.KEY_DEPENDENCIES]):
children.update(dep[name][constant.KEY_DEPENDENCIES])
if children:
dep.update(self.get_dep_modules(children, depth + 1))
return dep
@staticmethod
def generate_projects(targets):
"""Generate a list of projects in one time by a list of module names.
Args:
targets: A list of target modules or project paths from user input,
when locating the target, project with matched module name
of the target has a higher priority than project path.
Returns:
List: A list of ProjectInfo instances.
"""
return [ProjectInfo(target, i == 0) for i, target in enumerate(targets)]
@staticmethod
def get_target_name(target, abs_path):
"""Get target name from target's absolute path.
If the project is for entire Android source tree, change the target to
source tree's root folder name. In this way, we give IDE project file
a more specific name. e.g, master.iml.
Args:
target: Includes target module or project path from user input, when
locating the target, project with matching module name of
the given target has a higher priority than project path.
abs_path: A string, target's absolute path.
Returns:
A string, the target name.
"""
if abs_path == common_util.get_android_root_dir():
return os.path.basename(abs_path)
return target
def locate_source(self, build=True):
"""Locate the paths of dependent source folders and jar files.
Try to reference source folder path as dependent module unless the
dependent module should be referenced to a jar file, such as modules
have jars and jarjar_rules parameter.
For example:
Module: asm-6.0
java_import {
name: 'asm-6.0',
host_supported: true,
jars: ['asm-6.0.jar'],
}
Module: bouncycastle
java_library {
name: 'bouncycastle',
...
target: {
android: {
jarjar_rules: 'jarjar-rules.txt',
},
},
}
Args:
build: A boolean default to true. If false, skip building jar and
srcjar files, otherwise build them.
Example usage:
project.source_path = project.locate_source()
E.g.
project.source_path = {
'source_folder_path': ['path/to/source/folder1',
'path/to/source/folder2', ...],
'test_folder_path': ['path/to/test/folder', ...],
'jar_path': ['path/to/jar/file1', 'path/to/jar/file2', ...]
}
"""
if not hasattr(self, 'dep_modules') or not self.dep_modules:
raise errors.EmptyModuleDependencyError(
'Dependent modules dictionary is empty.')
rebuild_targets = set()
for module_name, module_data in self.dep_modules.items():
module = self._generate_moduledata(module_name, module_data)
module.locate_sources_path()
self.source_path['source_folder_path'].update(set(module.src_dirs))
self.source_path['test_folder_path'].update(set(module.test_dirs))
self.source_path['r_java_path'].update(set(module.r_java_paths))
self.source_path['srcjar_path'].update(set(module.srcjar_paths))
self._append_jars_as_dependencies(module)
rebuild_targets.update(module.build_targets)
config = project_config.ProjectConfig.get_instance()
if config.is_skip_build:
return
if rebuild_targets:
if build:
logging.info('\nThe batch_build_dependencies function is '
'called by ProjectInfo\'s locate_source method.')
batch_build_dependencies(rebuild_targets)
self.locate_source(build=False)
else:
logging.warning('Jar or srcjar files build skipped:\n\t%s.',
'\n\t'.join(rebuild_targets))
def _generate_moduledata(self, module_name, module_data):
"""Generate a module class to collect dependencies in IDE.
The rules of initialize a module data instance: if ide_object isn't None
and its ide_name is 'eclipse', we'll create an EclipseModuleData
instance otherwise create a ModuleData instance.
Args:
module_name: Name of the module.
module_data: A dictionary holding a module information.
Returns:
A ModuleData class.
"""
ide_name = project_config.ProjectConfig.get_instance().ide_name
if ide_name == constant.IDE_ECLIPSE:
return source_locator.EclipseModuleData(
module_name, module_data, self.project_relative_path)
depth = project_config.ProjectConfig.get_instance().depth
return source_locator.ModuleData(module_name, module_data, depth)
def _append_jars_as_dependencies(self, module):
"""Add given module's jar files into dependent_data as dependencies.
Args:
module: A ModuleData instance.
"""
if module.jar_files:
self.source_path['jar_path'].update(module.jar_files)
for jar in list(module.jar_files):
self.source_path['jar_module_path'].update({
jar:
module.module_path
})
# Collecting the jar files of default core modules as dependencies.
if constant.KEY_DEPENDENCIES in module.module_data:
self.source_path['jar_path'].update([
x for x in module.module_data[constant.KEY_DEPENDENCIES]
if common_util.is_target(x, constant.TARGET_LIBS)
])
def _get_rel_project_out_soong_jar_path(self):
"""Gets the projects' jar path in 'out/soong/.intermediates' folder.
Gets the relative project's jar path in the 'out/soong/.intermediates'
directory. For example, if the self.project_relative_path is
'frameworks/base', the returned value should be
'out/soong/.intermediates/frameworks/base/'.
Returns:
A string of relative project path in out/soong/.intermediates/
directory, e.g. 'out/soong/.intermediates/frameworks/base/'.
"""
rdir = os.path.relpath(common_util.get_soong_out_path(),
common_util.get_android_root_dir())
return os.sep.join(
[rdir, constant.INTERMEDIATES, self.project_relative_path]) + os.sep
@classmethod
def multi_projects_locate_source(cls, projects):
"""Locate the paths of dependent source folders and jar files.
Args:
projects: A list of ProjectInfo instances. Information of a project
such as project relative path, project real path, project
dependencies.
"""
cls.projects = projects
for project in projects:
project.locate_source()
_update_iml_dep_modules(project)
class MultiProjectsInfo(ProjectInfo):
"""Multiple projects info.
Usage example:
if folder_base:
project = MultiProjectsInfo(['module_name'])
project.collect_all_dep_modules()
project.gen_folder_base_dependencies()
else:
ProjectInfo.generate_projects(['module_name'])
Attributes:
_targets: A list of module names or project paths.
path_to_sources: A dictionary of modules' sources, the module's path
as key and the sources as value.
e.g.
{
'frameworks/base': {
'src_dirs': [],
'test_dirs': [],
'r_java_paths': [],
'srcjar_paths': [],
'jar_files': [],
'dep_paths': [],
}
}
"""
def __init__(self, targets=None):
"""MultiProjectsInfo initialize.
Args:
targets: A list of module names or project paths from user's input.
"""
super().__init__(targets[0], True)
self._targets = targets
self.path_to_sources = {}
def _clear_srcjar_paths(self, module):
"""Clears the srcjar_paths.
Args:
module: A ModuleData instance.
"""
module.srcjar_paths = []
def _collect_framework_srcjar_info(self, module):
"""Clears the framework's srcjars.
Args:
module: A ModuleData instance.
"""
if module.module_path == constant.FRAMEWORK_PATH:
framework_srcjar_path = os.path.join(constant.FRAMEWORK_PATH,
constant.FRAMEWORK_SRCJARS)
if module.module_name == constant.FRAMEWORK_ALL:
self.path_to_sources[framework_srcjar_path] = {
'src_dirs': [],
'test_dirs': [],
'r_java_paths': [],
'srcjar_paths': module.srcjar_paths,
'jar_files': [],
'dep_paths': [constant.FRAMEWORK_PATH],
}
# In the folder base case, AIDEGen has to ignore all module's srcjar
# files under the frameworks/base except the framework-all. Because
# there are too many duplicate srcjars of modules under the
# frameworks/base. So that AIDEGen keeps the srcjar files only from
# the framework-all module. Other modeuls' srcjar files will be
# removed. However, when users choose the module base case, srcjar
# files will be collected by the ProjectInfo class, so that the
# removing srcjar_paths in this class does not impact the
# srcjar_paths collection of modules in the ProjectInfo class.
self._clear_srcjar_paths(module)
def collect_all_dep_modules(self):
"""Collects all dependency modules for the projects."""
self.project_module_names.clear()
module_names = set(_CORE_MODULES)
for target in self._targets:
relpath, _ = common_util.get_related_paths(self.modules_info,
target)
module_names.update(self._get_modules_under_project_path(relpath))
module_names.update(self._get_robolectric_dep_module(module_names))
self.dep_modules = self.get_dep_modules(module_names)
def gen_folder_base_dependencies(self, module):
"""Generates the folder base dependencies dictionary.
Args:
module: A ModuleData instance.
"""
mod_path = module.module_path
if not mod_path:
logging.debug('The %s\'s path is empty.', module.module_name)
return
self._collect_framework_srcjar_info(module)
if mod_path not in self.path_to_sources:
self.path_to_sources[mod_path] = {
'src_dirs': module.src_dirs,
'test_dirs': module.test_dirs,
'r_java_paths': module.r_java_paths,
'srcjar_paths': module.srcjar_paths,
'jar_files': module.jar_files,
'dep_paths': module.dep_paths,
}
else:
for key, val in self.path_to_sources[mod_path].items():
val.extend([v for v in getattr(module, key) if v not in val])
def batch_build_dependencies(rebuild_targets):
"""Batch build the jar or srcjar files of the modules if they don't exist.
Command line has the max length limit, MAX_ARG_STRLEN, and
MAX_ARG_STRLEN = (PAGE_SIZE * 32).
If the build command is longer than MAX_ARG_STRLEN, this function will
separate the rebuild_targets into chunks with size less or equal to
MAX_ARG_STRLEN to make sure it can be built successfully.
Args:
rebuild_targets: A set of jar or srcjar files which do not exist.
"""
start_time = time.time()
logging.info('Ready to build the jar or srcjar files. Files count = %s',
str(len(rebuild_targets)))
arg_max = os.sysconf('SC_PAGE_SIZE') * 32 - _CMD_LENGTH_BUFFER
rebuild_targets = list(rebuild_targets)
for start, end in iter(_separate_build_targets(rebuild_targets, arg_max)):
_build_target(rebuild_targets[start:end])
duration = time.time() - start_time
logging.debug('Build Time, duration = %s', str(duration))
aidegen_metrics.performance_metrics(constant.TYPE_AIDEGEN_BUILD_TIME,
duration)
def _build_target(targets):
"""Build the jar or srcjar files.
Use -k to keep going when some targets can't be built or build failed.
Use -j to speed up building.
Args:
targets: A list of jar or srcjar files which need to build.
"""
build_cmd = ['-k', '-j']
build_cmd.extend(list(targets))
verbose = True
if not atest_utils.build(build_cmd, verbose):
message = ('Build failed!\n{}\nAIDEGen will proceed but dependency '
'correctness is not guaranteed if not all targets being '
'built successfully.'.format('\n'.join(targets)))
print('\n{} {}\n'.format(common_util.COLORED_INFO('Warning:'), message))
def _separate_build_targets(build_targets, max_length):
"""Separate the build_targets by limit the command size to max command
length.
Args:
build_targets: A list to be separated.
max_length: The max number of each build command length.
Yields:
The start index and end index of build_targets.
"""
arg_len = 0
first_item_index = 0
for i, item in enumerate(build_targets):
arg_len = arg_len + len(item) + _BLANK_SIZE
if arg_len > max_length:
yield first_item_index, i
first_item_index = i
arg_len = len(item) + _BLANK_SIZE
if first_item_index < len(build_targets):
yield first_item_index, len(build_targets)
def _update_iml_dep_modules(project):
"""Gets the dependent modules in the project's iml file.
The jar files which have the same source codes as cls.projects' source files
should be removed from the dependencies.iml file's jar paths. The codes are
written in aidegen.project.project_splitter.py.
We should also add the jar project's unique iml name into self.dependencies
which later will be written into its own iml project file. If we don't
remove these files in dependencies.iml, it will cause the duplicated codes
in IDE and raise issues. For example, when users do 'refactor' and rename a
class in the IDE, it will search all sources and dependencies' jar paths and
lead to the error.
"""
keys = ('source_folder_path', 'test_folder_path', 'r_java_path',
'srcjar_path', 'jar_path')
for key in keys:
for jar in project.source_path[key]:
for prj in ProjectInfo.projects:
if prj is project:
continue
if (prj.rel_out_soong_jar_path in jar and
jar.endswith(constant.JAR_EXT)):
if prj.iml_name not in project.dependencies:
project.dependencies.append(prj.iml_name)
break