You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

644 lines
24 KiB

# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Core logic for generating, syncing, and cleaning up a Bazel environment."""
import abc
import datetime
import logging
import os
import pathlib
import re
import shutil
import subprocess
from typing import Set, Dict, List
# Regex for BUILD files used to identify them since they can be named
# BUILD or BUILD.bazel.
BUILD_FILENAME_REGEX = re.compile("(BUILD|BUILD.bazel)")
class Error(Exception):
"""Base Error that all other errors the system throws are descendants of."""
pass
class SoongExecutionError(Error):
"""Raised when Soong fails to build provided targets."""
pass
class Soong:
"""Interface for the Soong build system.
Attributes:
soong_workspace: the top of the Android workspace that Soong
will be operating in.
soong_executable: the path to the executable for the soong_ui.bash
launcher.
"""
soong_workspace: pathlib.Path
soong_executable: pathlib.Path
def __init__(self, soong_workspace: pathlib.Path):
self.soong_workspace = soong_workspace
self.soong_executable = self.soong_workspace.joinpath(
"build/soong/soong_ui.bash")
if not self.soong_executable.exists():
raise SoongExecutionError(
"Unable to find Soong executable, expected location: %s" %
self.soong_executable)
def build(self, build_targets: Set[str]) -> None:
"""Builds the provided set of targets with Soong.
Of note, there is no verification for the targets that get passed in,
rather that responsibility passes to Soong which will fail to build if a
target is invalid.
Args:
build_targets: a set of targets to build with Soong.
"""
cmd_args = [
str(self.soong_executable), "--build-mode", "--all-modules",
f"--dir={self.soong_workspace}"
]
cmd_args.extend(build_targets)
logging.info("Building targets with Soong: %s", build_targets)
logging.info("Please be patient, this may take a while...")
logging.debug("Soong Command is: %s", " ".join(cmd_args))
try:
subprocess.run(cmd_args,
cwd=self.soong_workspace,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as cpe:
raise SoongExecutionError(
"There was an error during the Soong build process. Please "
"correct the error with Soong, and try running this script "
"again. Soong output follows:\n\n"
f"{cpe.stdout.decode('utf-8')}") from cpe
logging.info("Soong command completed successfully.")
class Resource(abc.ABC):
"""
Represents a resource file that is used for scaffolding the Bazel env.
Resource is an abstract class. Inheriting classes must provide the following
attributes which are used by the default method implementations below.
Attributes:
stage_path: the path where this resource should be written when staged.
workspace_path: the path where this resource should be synced to in the
workspace.
"""
stage_path: pathlib.Path
workspace_path: pathlib.Path
def build_targets(self) -> set:
return set()
@abc.abstractmethod
def stage(self, _):
"""Writes a resource to its stage location."""
pass
@abc.abstractmethod
def sync(self):
"""Syncs a resource to its workspace location."""
# Overwrite any existing file in the workspace when synced.
self.workspace_path.unlink(missing_ok=True)
self.workspace_path.symlink_to(self.stage_path)
@abc.abstractmethod
def clean(self):
"""Cleans a resource from its workspace location."""
self.workspace_path.unlink(missing_ok=True)
class StaticResource(Resource):
"""Resource representing a static file to be copied.
Attributes:
resource_path: path to the resource on disk.
"""
def __init__(self, stage_path: pathlib.Path, workspace_path: pathlib.Path,
resource_path: pathlib.Path):
self.stage_path = stage_path
self.workspace_path = workspace_path
self._resource_path = resource_path
def stage(self, _):
_verify_directory(self.stage_path.parent)
shutil.copy(self._resource_path, self.stage_path)
_make_executable_if_script(self.stage_path)
def sync(self):
super().sync()
def clean(self):
super().sync()
def __repr__(self):
return (f"StaticResource(stage_path={self.stage_path}, "
f"workspace_path={self.workspace_path}, "
f"resource_path={self._resource_path})")
class TemplateError(Error):
"""Raised when there is an issue while templating a template file."""
pass
class TemplateResource(Resource):
"""Resource that represents a file to be templated.
When staged, the resource is templated using the "mapping" provided
to the stage function.
Attributes:
resource_path: path to the resource on disk.
"""
resource_path: pathlib.Path
# Key within templates that identifies a Soong target to build for a
# provided template.
SOONG_TARGET_KEY = "SOONG_TARGET"
_KEY_VALUE_SEP = ":"
# For a provided template, lines matching this regex are ignored when
# loading the template.
#
# This enables to contain metadata that, while visible to the script, is
# not visible in the generated templates. This is currently used with the
# SOONG_TARGET key/value pairs in templates.
_IGNORE_LINE_REGEX = re.compile(f"({SOONG_TARGET_KEY})")
def __init__(self, stage_path: pathlib.Path, workspace_path: pathlib.Path,
resource_path: pathlib.Path):
self.stage_path = stage_path
self.workspace_path = workspace_path
self.resource_path = resource_path
def __repr__(self):
return (f"TemplateResource(stage_path={self.stage_path}, "
f"workspace_path={self.workspace_path}, "
f"resource_path={self.resource_path})")
def stage(self, mapping: Dict[str, str]):
_verify_directory(self.stage_path.parent)
lines = self.resource_path.open().readlines()
lines = [
line for line in lines if not self._IGNORE_LINE_REGEX.search(line)
]
try:
output = "".join(lines).format_map(mapping)
except KeyError as ke:
raise TemplateError(
f"Malformed template file: {self.resource_path}") from ke
with self.stage_path.open("w") as output_file:
output_file.write(output)
_make_executable_if_script(self.stage_path)
def sync(self):
super().sync()
def clean(self):
super().clean()
@classmethod
def read_value_from_template_var(cls, line: str) -> str:
value = line.split(cls._KEY_VALUE_SEP)[-1]
return value.strip()
class BuildTemplateResource(Resource):
"""Resource that represents a BUILD file.
This operates in the same way as a TemplateResource, however also
sets up the prebuilts directory needed by BUILD files.
Attributes:
_template_resource: underlying TemplateResource for this resource.
global_prebuilts_dir: the directory in the filesystem where prebuilts
live, this directory is symlinked to by a directory adjacent
to the BUILD file represented by this Resource.
prebuilts_stage_path: the path of the staged prebuilts directory
which is a symlink to the global_prebuilts_dir.
prebuilts_workspace_path: the path of the workspace prebuilts directory,
which is a symlink to the global_prebuilts_dir.
"""
_template_resource: TemplateResource
global_prebuilts_dir: pathlib.Path
prebuilts_stage_path: pathlib.Path
prebuilts_workspace_path: pathlib.Path
def __init__(self, stage_path: pathlib.Path, workspace_path: pathlib.Path,
resource_path: pathlib.Path,
global_prebuilts_dir: pathlib.Path, prebuilts_dir_name: str):
self.stage_path = stage_path
self.workspace_path = workspace_path
self._template_resource = TemplateResource(stage_path, workspace_path,
resource_path)
self.global_prebuilts_dir = global_prebuilts_dir
self.prebuilts_stage_path = self.stage_path.parent.joinpath(
prebuilts_dir_name)
self.prebuilts_workspace_path = self.workspace_path.parent.joinpath(
prebuilts_dir_name)
def __repr__(self):
return ("BuildTemplateResource("
f"template_resource={self._template_resource}, "
f"global_prebuilts_dir={self.global_prebuilts_dir}, "
f"prebuilts_stage_path={self.prebuilts_stage_path}, "
f"prebuilts_workspace_path={self.prebuilts_workspace_path})")
def build_targets(self) -> set:
"""Overrides build_targets() to read targets from the BUILD template."""
targets = set()
with self._template_resource.resource_path.open() as build_file:
while line := build_file.readline():
if self._template_resource.SOONG_TARGET_KEY in line:
targets.add(
self._template_resource.read_value_from_template_var(
line))
return targets
def stage(self, mapping: Dict[str, str]):
"""Overrides stage() to stage a BUILD resource.
Delegates most actions to the _template_resource, while ensuring that
a generated prebuilts directory is also staged.
"""
self._template_resource.stage(mapping)
self.prebuilts_stage_path.symlink_to(self.global_prebuilts_dir,
target_is_directory=True)
def sync(self):
"""Overrides sync() to sync a BUILD resource with prebuilts.
Delegates to the _template_resource while also ensuring that the
generated prebuilts directory is written to the workspace.
"""
self._template_resource.sync()
# Overwrite the existing prebuilts directory, if it exists.
self.prebuilts_workspace_path.unlink(missing_ok=True)
self.prebuilts_workspace_path.symlink_to(self.global_prebuilts_dir,
target_is_directory=True)
def clean(self):
"""Overrides clean() to clean a BUILD resource.
Delegates most actions to the _template_resource, while also ensuring
that the generated prebuilts directory is removed from the workspace.
"""
self._template_resource.clean()
self.prebuilts_workspace_path.unlink(missing_ok=True)
def _make_executable_if_script(path: pathlib.Path) -> None:
"""Makes the provided path executable if it is a script.
Args:
path: the path to check and conditionally make executable.
"""
if path.name.endswith(".sh"):
# Grant full permissions (read/write/execute) for current user and
# read/write permissions for group.
path.chmod(mode=0o750)
class ResourcesNotFoundError(Error):
"""Raised when the required resources are not found."""
pass
class Resources:
"""Manages and loads resources from disk.
Attributes:
workspace_base_path: the path to the root of the workspace where
staged files will be synced to.
gendir_base_path: the path to the root of the staging directory.
global_prebuilts_dir_path: the path to the root of the global
prebuilts directory to which all generated prebuilts directories will
symlink to.
prebuilts_dir_name: the name to use for generated prebuilts
directories.
static_path: the path to the static resources directory.
template_path: the path to the template resources directory.
"""
workspace_base_path: pathlib.Path
gendir_base_path: pathlib.Path
global_prebuilts_dir_path: pathlib.Path
prebuilts_dir_name: str
static_path: pathlib.Path
template_path: pathlib.Path
# Name of the directory where script runfiles are located.
_DATA_DIRNAME = "data"
# Name of the directory where the templates should be located.
_TEMPLATES_DIRNAME = "templates"
# Name of the directory where static files (to be copied to the environment)
# should be located.
_STATIC_DIRNAME = "static"
# If the script is executed from the root of the runfiles directory, this is
# where the data should be.
_RESOURCES_RUNFILES_BASEPATH = pathlib.Path(
"build/pesto/experiments/prepare_bazel_test_env", _DATA_DIRNAME)
# File extension for templates, determining whether or not a given file is a
# template.
_TEMPLATE_FILE_EXT = ".template"
def __init__(self,
workspace_base_path: pathlib.Path,
gendir_base_path: pathlib.Path,
global_prebuilts_dir_path: pathlib.Path,
prebuilts_dir_name: str,
path: pathlib.Path = _RESOURCES_RUNFILES_BASEPATH):
logging.debug("Resources(path=%s)", path)
self.workspace_base_path = workspace_base_path
self.gendir_base_path = gendir_base_path
self.global_prebuilts_dir_path = global_prebuilts_dir_path
self.prebuilts_dir_name = prebuilts_dir_name
path = path.resolve()
self.template_path = path.joinpath(Resources._TEMPLATES_DIRNAME)
self.static_path = path.joinpath(Resources._STATIC_DIRNAME)
if not self.template_path.exists() or not self.static_path.exists():
raise ResourcesNotFoundError("Unable to find resources at path "
f"{path}, expected the following "
"directories: "
f"{Resources._TEMPLATES_DIRNAME}, "
f"{Resources._STATIC_DIRNAME}")
def __repr__(self):
return ("Resources("
f"template_path={self.template_path}"
f"static_path={self.static_path})")
def stage(self, mapping: Dict[str, str]) -> None:
for resource in self.load():
logging.debug("Staging resource: %s", resource)
resource.stage(mapping)
def sync(self) -> None:
for resource in self.load():
logging.debug("Syncing resource: %s", resource)
resource.sync()
def clean(self) -> None:
for resource in self.load():
logging.debug("Cleaning resource: %s", resource)
resource.clean()
def build_targets(self) -> Set[str]:
return {
t
for resource in self.load() for t in resource.build_targets()
}
def load(self) -> List[Resource]:
"""Loads the Resources used to scaffold the Bazel env.
Returns:
a list of Resource objects representing the files
that should be used to template the environment.
"""
resources = []
# Add all templates.
for p in self._template_resource_paths():
template_relpath = Resources._strip_template_identifier_from_path(
p.relative_to(self.template_path))
stage_path = self.gendir_base_path.joinpath(template_relpath)
workspace_path = self.workspace_base_path.joinpath(
template_relpath)
if BUILD_FILENAME_REGEX.match(template_relpath.name):
resources.append(
BuildTemplateResource(
stage_path=stage_path,
workspace_path=workspace_path,
resource_path=p,
global_prebuilts_dir=self.global_prebuilts_dir_path,
prebuilts_dir_name=self.prebuilts_dir_name))
else:
resources.append(
TemplateResource(stage_path=stage_path,
workspace_path=workspace_path,
resource_path=p))
# Add all static files.
for p in self._static_resource_paths():
static_relpath = p.relative_to(self.static_path)
stage_path = self.gendir_base_path.joinpath(static_relpath)
workspace_path = self.workspace_base_path.joinpath(static_relpath)
resources.append(
StaticResource(stage_path=stage_path,
workspace_path=workspace_path,
resource_path=p))
return resources
def _static_resource_paths(self) -> List[pathlib.Path]:
return [p for p in self.static_path.glob("**/*") if p.is_file()]
def _template_resource_paths(self) -> List[pathlib.Path]:
return [p for p in self.template_path.glob("**/*") if p.is_file()]
@staticmethod
def _strip_template_identifier_from_path(p: pathlib.Path):
"""Strips the template file extension from a provided path."""
if p.name.endswith(Resources._TEMPLATE_FILE_EXT):
p = p.with_name(p.name[:len(p.name) -
len(Resources._TEMPLATE_FILE_EXT)])
return p
class AndroidBuildEnvironmentError(Error):
"""Raised when the Android Build Environment is not properly set."""
pass
class BazelTestEnvGenerator:
"""Context for the Bazel environment generation.
This class provides access to locations within the filesystem pertinent to the
current execution as members that can be accessed by users of the class.
Attributes:
workspace_top: the top of the codebase, which also serves as the top of
the Bazel WORKSPACE.
host_out: Host artifact staging directory location.
host_testcases: Host testcase staging directory location.
product_out: Product/Target artifact staging directory location.
target_testcases: Product/Target testcase staging directory location.
staging_dir: Staging directory for generated Bazel artifacts.
prebuilts_dir_name: Name of the directory that should be used for the
directories that prebuilts are placed into when synced into the source
tree.
global_prebuilts_dir_path: The absolute path to the global prebuilts
directory.
year: the current calendar year.
gen_dir: Subdirectory of the staging dir where the Bazel environment
should be generated to.
"""
workspace_top: pathlib.Path
host_out: pathlib.Path
host_testcases: pathlib.Path
product_out: pathlib.Path
target_testcases: pathlib.Path
staging_dir: pathlib.Path
prebuilts_dir_name: str = ".soong_prebuilts"
prebuilts_dir_path: pathlib.Path
gen_dir: pathlib.Path
year: str
_resources: Resources
_soong: Soong
# Name of the subdirectory, within the output directory, where the
# prebuilts directory is scaffolded.
#
# This directory then serves as the target of symlinks from across the
# source tree that gives any Bazel target access to the Soong staging
# directories.
GLOBAL_PREBUILTS_DIR_PATH = "prebuilts"
def __init__(self, env_dict: Dict[str, str] = os.environ):
try:
self.workspace_top = pathlib.Path(env_dict["ANDROID_BUILD_TOP"])
self.host_out = pathlib.Path(env_dict["ANDROID_HOST_OUT"])
self.host_testcases = pathlib.Path(
env_dict["ANDROID_HOST_OUT_TESTCASES"])
self.product_out = pathlib.Path(env_dict["ANDROID_PRODUCT_OUT"])
self.target_testcases = pathlib.Path(
env_dict["ANDROID_TARGET_OUT_TESTCASES"])
except KeyError as e:
raise AndroidBuildEnvironmentError(
"Missing expected environment variable.") from e
self.staging_dir = pathlib.Path(self.workspace_top,
"out/pesto-environment")
self.prebuilts_dir_name = BazelTestEnvGenerator.prebuilts_dir_name
self.global_prebuilts_dir_path = self.staging_dir.joinpath(
self.GLOBAL_PREBUILTS_DIR_PATH)
self.year = str(datetime.date.today().year)
self.gen_dir = pathlib.Path(self.staging_dir, "gen")
self._resources = Resources(self.workspace_top, self.gen_dir,
self.global_prebuilts_dir_path,
self.prebuilts_dir_name)
self._soong = Soong(self.workspace_top)
def __repr__(self):
return "GenerationContext(%s)" % vars(self)
def generate(self):
logging.info("Starting generation of Bazel environment.")
self._soong.build(self._resources.build_targets())
logging.debug("Creating fresh staging dir at: %s", self.staging_dir)
_verify_directory(self.staging_dir, clean=True)
logging.debug("Creating fresh gen dir at: %s", self.gen_dir)
_verify_directory(self.gen_dir, clean=True)
logging.debug("Creating global prebuilts directory at: %s",
self.global_prebuilts_dir_path)
_verify_directory(self.global_prebuilts_dir_path, clean=True)
# Symlink the build system provided staging directories to the
# global prebuilts directory.
self.global_prebuilts_dir_path.joinpath("host").symlink_to(
self.host_out)
self.global_prebuilts_dir_path.joinpath("host_testcases").symlink_to(
self.host_testcases)
self.global_prebuilts_dir_path.joinpath("product").symlink_to(
self.product_out)
self.global_prebuilts_dir_path.joinpath("target_testcases").symlink_to(
self.target_testcases)
# Load and process each resource into the gen directory.
self._resources.stage(mapping=vars(self))
logging.info(
"Generation of Bazel environment to staging directory "
"(%s) completed successfully.", self.staging_dir)
def sync(self):
logging.info(
"Starting synchronization of generated environment to source tree."
)
if not self.staging_dir.exists():
raise FileNotFoundError("Staging directory does not exist, "
"the generate function should be called "
" to create this directory.")
self._resources.sync()
logging.info(
"Successfully synchronized generated Bazel environment from "
"%s to %s", self.gen_dir, self.workspace_top)
def clean(self):
logging.info("Starting clean of generated environment.")
logging.info("Cleaning up synchronized files from the source tree.")
# For all of our configured templates, attempt to find the corresponding
# location in the source tree and remove them.
self._resources.clean()
logging.info("Cleaning up staging directory: %s", self.staging_dir)
try:
shutil.rmtree(self.staging_dir)
except FileNotFoundError:
logging.debug("Staging directory not found during cleanup "
"and may have already been removed.")
logging.info("Successfully cleaned up generated environment.")
def _verify_directory(directory: pathlib.Path, clean: bool = False) -> None:
"""Verifies that the provided directory exists, creating it if it does not.
Args:
directory: path to the directory to create.
clean: whether or not the existing directory should be removed if found or
reused.
"""
if directory.exists() and clean:
logging.debug("Cleaning existing directory %s", directory)
shutil.rmtree(directory)
logging.debug("Verifying directory exists at %s", directory)
directory.mkdir(parents=True, exist_ok=True)