You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
389 lines
15 KiB
389 lines
15 KiB
#!/usr/bin/env python
|
|
#
|
|
# Copyright 2016 - The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""A client that talks to Android Build APIs."""
|
|
|
|
import collections
|
|
import io
|
|
import json
|
|
import logging
|
|
import os
|
|
import ssl
|
|
import stat
|
|
|
|
import apiclient
|
|
|
|
from acloud import errors
|
|
from acloud.internal.lib import base_cloud_client
|
|
from acloud.internal.lib import utils
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# The BuildInfo namedtuple data structure.
|
|
# It will be the data structure returned by GetBuildInfo method.
|
|
BuildInfo = collections.namedtuple("BuildInfo", [
|
|
"branch", # The branch name string
|
|
"build_id", # The build id string
|
|
"build_target", # The build target string
|
|
"release_build_id"]) # The release build id string
|
|
_DEFAULT_BRANCH = "aosp-master"
|
|
|
|
|
|
class AndroidBuildClient(base_cloud_client.BaseCloudApiClient):
|
|
"""Client that manages Android Build."""
|
|
|
|
# API settings, used by BaseCloudApiClient.
|
|
API_NAME = "androidbuildinternal"
|
|
API_VERSION = "v2beta1"
|
|
SCOPE = "https://www.googleapis.com/auth/androidbuild.internal"
|
|
|
|
# other variables.
|
|
DEFAULT_RESOURCE_ID = "0"
|
|
# TODO(b/27269552): We should use "latest".
|
|
DEFAULT_ATTEMPT_ID = "0"
|
|
DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024
|
|
NO_ACCESS_ERROR_PATTERN = "does not have storage.objects.create access"
|
|
# LKGB variables.
|
|
BUILD_STATUS_COMPLETE = "complete"
|
|
BUILD_TYPE_SUBMITTED = "submitted"
|
|
ONE_RESULT = 1
|
|
BUILD_SUCCESSFUL = True
|
|
LATEST = "latest"
|
|
# FETCH_CVD variables.
|
|
FETCHER_NAME = "fetch_cvd"
|
|
FETCHER_BUILD_TARGET = "aosp_cf_x86_phone-userdebug"
|
|
MAX_RETRY = 3
|
|
RETRY_SLEEP_SECS = 3
|
|
|
|
# Message constant
|
|
COPY_TO_MSG = ("build artifact (target: %s, build_id: %s, "
|
|
"artifact: %s, attempt_id: %s) to "
|
|
"google storage (bucket: %s, path: %s)")
|
|
# pylint: disable=invalid-name
|
|
def DownloadArtifact(self,
|
|
build_target,
|
|
build_id,
|
|
resource_id,
|
|
local_dest,
|
|
attempt_id=None):
|
|
"""Get Android build attempt information.
|
|
|
|
Args:
|
|
build_target: Target name, e.g. "aosp_cf_x86_phone-userdebug"
|
|
build_id: Build id, a string, e.g. "2263051", "P2804227"
|
|
resource_id: Id of the resource, e.g "avd-system.tar.gz".
|
|
local_dest: A local path where the artifact should be stored.
|
|
e.g. "/tmp/avd-system.tar.gz"
|
|
attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID.
|
|
"""
|
|
attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID
|
|
api = self.service.buildartifact().get_media(
|
|
buildId=build_id,
|
|
target=build_target,
|
|
attemptId=attempt_id,
|
|
resourceId=resource_id)
|
|
logger.info("Downloading artifact: target: %s, build_id: %s, "
|
|
"resource_id: %s, dest: %s", build_target, build_id,
|
|
resource_id, local_dest)
|
|
try:
|
|
with io.FileIO(local_dest, mode="wb") as fh:
|
|
downloader = apiclient.http.MediaIoBaseDownload(
|
|
fh, api, chunksize=self.DEFAULT_CHUNK_SIZE)
|
|
done = False
|
|
while not done:
|
|
_, done = downloader.next_chunk()
|
|
logger.info("Downloaded artifact: %s", local_dest)
|
|
except (OSError, apiclient.errors.HttpError) as e:
|
|
logger.error("Downloading artifact failed: %s", str(e))
|
|
raise errors.DriverError(str(e))
|
|
|
|
def DownloadFetchcvd(self, local_dest, fetch_cvd_version):
|
|
"""Get fetch_cvd from Android Build.
|
|
|
|
Args:
|
|
local_dest: A local path where the artifact should be stored.
|
|
e.g. "/tmp/fetch_cvd"
|
|
fetch_cvd_version: String of fetch_cvd version.
|
|
"""
|
|
utils.RetryExceptionType(
|
|
exception_types=ssl.SSLError,
|
|
max_retries=self.MAX_RETRY,
|
|
functor=self.DownloadArtifact,
|
|
sleep_multiplier=self.RETRY_SLEEP_SECS,
|
|
retry_backoff_factor=utils.DEFAULT_RETRY_BACKOFF_FACTOR,
|
|
build_target=self.FETCHER_BUILD_TARGET,
|
|
build_id=fetch_cvd_version,
|
|
resource_id=self.FETCHER_NAME,
|
|
local_dest=local_dest,
|
|
attempt_id=self.LATEST)
|
|
fetch_cvd_stat = os.stat(local_dest)
|
|
os.chmod(local_dest, fetch_cvd_stat.st_mode | stat.S_IEXEC)
|
|
|
|
@staticmethod
|
|
def ProcessBuild(build_id=None, branch=None, build_target=None):
|
|
"""Create a Cuttlefish fetch_cvd build string.
|
|
|
|
Args:
|
|
build_id: A specific build number to load from. Takes precedence over `branch`.
|
|
branch: A manifest-branch at which to get the latest build.
|
|
build_target: A particular device to load at the desired build.
|
|
|
|
Returns:
|
|
A string, used in the fetch_cvd cmd or None if all args are None.
|
|
"""
|
|
if not build_target:
|
|
return build_id or branch
|
|
|
|
if build_target and not branch:
|
|
branch = _DEFAULT_BRANCH
|
|
return (build_id or branch) + "/" + build_target
|
|
|
|
# pylint: disable=too-many-locals
|
|
def GetFetchBuildArgs(self, build_id, branch, build_target, system_build_id,
|
|
system_branch, system_build_target, kernel_build_id,
|
|
kernel_branch, kernel_build_target, bootloader_build_id,
|
|
bootloader_branch, bootloader_build_target):
|
|
"""Get args from build information for fetch_cvd.
|
|
|
|
Args:
|
|
build_id: String of build id, e.g. "2263051", "P2804227"
|
|
branch: String of branch name, e.g. "aosp-master"
|
|
build_target: String of target name.
|
|
e.g. "aosp_cf_x86_phone-userdebug"
|
|
system_build_id: String of the system image build id.
|
|
system_branch: String of the system image branch name.
|
|
system_build_target: String of the system image target name,
|
|
e.g. "cf_x86_phone-userdebug"
|
|
kernel_build_id: String of the kernel image build id.
|
|
kernel_branch: String of the kernel image branch name.
|
|
kernel_build_target: String of the kernel image target name,
|
|
bootloader_build_id: String of the bootloader build id.
|
|
bootloader_branch: String of the bootloader branch name.
|
|
bootloader_build_target: String of the bootloader target name.
|
|
|
|
Returns:
|
|
List of string args for fetch_cvd.
|
|
"""
|
|
fetch_cvd_args = []
|
|
|
|
default_build = self.ProcessBuild(build_id, branch, build_target)
|
|
if default_build:
|
|
fetch_cvd_args.append("-default_build=" + default_build)
|
|
system_build = self.ProcessBuild(
|
|
system_build_id, system_branch, system_build_target)
|
|
if system_build:
|
|
fetch_cvd_args.append("-system_build=" + system_build)
|
|
bootloader_build = self.ProcessBuild(bootloader_build_id,
|
|
bootloader_branch,
|
|
bootloader_build_target)
|
|
if bootloader_build:
|
|
fetch_cvd_args.append("-bootloader_build=%s" % bootloader_build)
|
|
kernel_build = self.GetKernelBuild(kernel_build_id,
|
|
kernel_branch,
|
|
kernel_build_target)
|
|
if kernel_build:
|
|
fetch_cvd_args.append("-kernel_build=" + kernel_build)
|
|
|
|
return fetch_cvd_args
|
|
|
|
@staticmethod
|
|
# pylint: disable=broad-except
|
|
def GetFetchCertArg(certification_file):
|
|
"""Get cert arg from certification file for fetch_cvd.
|
|
|
|
Parse the certification file to get access token of the latest
|
|
credential data and pass it to fetch_cvd command.
|
|
Example of certification file:
|
|
{
|
|
"data": [
|
|
{
|
|
"credential": {
|
|
"_class": "OAuth2Credentials",
|
|
"_module": "oauth2client.client",
|
|
"access_token": "token_strings",
|
|
"client_id": "179485041932",
|
|
}
|
|
}]
|
|
}
|
|
|
|
|
|
Args:
|
|
certification_file: String of certification file path.
|
|
|
|
Returns:
|
|
String of certificate arg for fetch_cvd. If there is no
|
|
certification file, return empty string for aosp branch.
|
|
"""
|
|
cert_arg = ""
|
|
|
|
try:
|
|
with open(certification_file) as cert_file:
|
|
auth_token = json.load(cert_file).get("data")[-1].get(
|
|
"credential").get("access_token")
|
|
if auth_token:
|
|
cert_arg = "-credential_source=%s" % auth_token
|
|
except Exception as e:
|
|
utils.PrintColorString(
|
|
"Fail to open the certification file(%s): %s" %
|
|
(certification_file, e), utils.TextColors.WARNING)
|
|
return cert_arg
|
|
|
|
def GetKernelBuild(self, kernel_build_id, kernel_branch, kernel_build_target):
|
|
"""Get kernel build args for fetch_cvd.
|
|
|
|
Args:
|
|
kernel_branch: Kernel branch name, e.g. "kernel-common-android-4.14"
|
|
kernel_build_id: Kernel build id, a string, e.g. "223051", "P280427"
|
|
kernel_build_target: String, Kernel build target name.
|
|
|
|
Returns:
|
|
String of kernel build args for fetch_cvd.
|
|
If no kernel build then return None.
|
|
"""
|
|
# kernel_target have default value "kernel". If user provide kernel_build_id
|
|
# or kernel_branch, then start to process kernel image.
|
|
if kernel_build_id or kernel_branch:
|
|
return self.ProcessBuild(kernel_build_id, kernel_branch, kernel_build_target)
|
|
return None
|
|
|
|
def CopyTo(self,
|
|
build_target,
|
|
build_id,
|
|
artifact_name,
|
|
destination_bucket,
|
|
destination_path,
|
|
attempt_id=None):
|
|
"""Copy an Android Build artifact to a storage bucket.
|
|
|
|
Args:
|
|
build_target: Target name, e.g. "aosp_cf_x86_phone-userdebug"
|
|
build_id: Build id, a string, e.g. "2263051", "P2804227"
|
|
artifact_name: Name of the artifact, e.g "avd-system.tar.gz".
|
|
destination_bucket: String, a google storage bucket name.
|
|
destination_path: String, "path/inside/bucket"
|
|
attempt_id: String, attempt id, will default to DEFAULT_ATTEMPT_ID.
|
|
"""
|
|
attempt_id = attempt_id or self.DEFAULT_ATTEMPT_ID
|
|
copy_msg = "Copying %s" % self.COPY_TO_MSG
|
|
logger.info(copy_msg, build_target, build_id, artifact_name,
|
|
attempt_id, destination_bucket, destination_path)
|
|
api = self.service.buildartifact().copyTo(
|
|
buildId=build_id,
|
|
target=build_target,
|
|
attemptId=attempt_id,
|
|
artifactName=artifact_name,
|
|
destinationBucket=destination_bucket,
|
|
destinationPath=destination_path)
|
|
try:
|
|
self.Execute(api)
|
|
finish_msg = "Finished copying %s" % self.COPY_TO_MSG
|
|
logger.info(finish_msg, build_target, build_id, artifact_name,
|
|
attempt_id, destination_bucket, destination_path)
|
|
except errors.HttpError as e:
|
|
if e.code == 503:
|
|
if self.NO_ACCESS_ERROR_PATTERN in str(e):
|
|
error_msg = "Please grant android build team's service account "
|
|
error_msg += "write access to bucket %s. Original error: %s"
|
|
error_msg %= (destination_bucket, str(e))
|
|
raise errors.HttpError(e.code, message=error_msg)
|
|
raise
|
|
|
|
def GetBranch(self, build_target, build_id):
|
|
"""Derives branch name.
|
|
|
|
Args:
|
|
build_target: Target name, e.g. "aosp_cf_x86_phone-userdebug"
|
|
build_id: Build ID, a string, e.g. "2263051", "P2804227"
|
|
|
|
Returns:
|
|
A string, the name of the branch
|
|
"""
|
|
api = self.service.build().get(buildId=build_id, target=build_target)
|
|
build = self.Execute(api)
|
|
return build.get("branch", "")
|
|
|
|
def GetLKGB(self, build_target, build_branch):
|
|
"""Get latest successful build id.
|
|
|
|
From branch and target, we can use api to query latest successful build id.
|
|
e.g. {u'nextPageToken':..., u'builds': [{u'completionTimestamp':u'1534157869286',
|
|
... u'buildId': u'4949805', u'machineName'...}]}
|
|
|
|
Args:
|
|
build_target: String, target name, e.g. "aosp_cf_x86_phone-userdebug"
|
|
build_branch: String, git branch name, e.g. "aosp-master"
|
|
|
|
Returns:
|
|
A string, string of build id number.
|
|
|
|
Raises:
|
|
errors.CreateError: Can't get build id.
|
|
"""
|
|
api = self.service.build().list(
|
|
branch=build_branch,
|
|
target=build_target,
|
|
buildAttemptStatus=self.BUILD_STATUS_COMPLETE,
|
|
buildType=self.BUILD_TYPE_SUBMITTED,
|
|
maxResults=self.ONE_RESULT,
|
|
successful=self.BUILD_SUCCESSFUL)
|
|
build = self.Execute(api)
|
|
logger.info("GetLKGB build API response: %s", build)
|
|
if build:
|
|
return str(build.get("builds")[0].get("buildId"))
|
|
raise errors.GetBuildIDError(
|
|
"No available good builds for branch: %s target: %s"
|
|
% (build_branch, build_target)
|
|
)
|
|
|
|
def GetBuildInfo(self, build_target, build_id, branch):
|
|
"""Get build info namedtuple.
|
|
|
|
Args:
|
|
build_target: Target name.
|
|
build_id: Build id, a string or None, e.g. "2263051", "P2804227"
|
|
If None or latest, the last green build id will be
|
|
returned.
|
|
branch: Branch name, a string or None, e.g. git_master. If None, the
|
|
returned branch will be searched by given build_id.
|
|
|
|
Returns:
|
|
A build info namedtuple with keys build_target, build_id, branch and
|
|
gcs_bucket_build_id
|
|
"""
|
|
if build_id and build_id != self.LATEST:
|
|
# Get build from build_id and build_target
|
|
api = self.service.build().get(buildId=build_id,
|
|
target=build_target)
|
|
build = self.Execute(api) or {}
|
|
elif branch:
|
|
# Get last green build in the branch
|
|
api = self.service.build().list(
|
|
branch=branch,
|
|
target=build_target,
|
|
successful=True,
|
|
maxResults=1,
|
|
buildType="submitted")
|
|
builds = self.Execute(api).get("builds", [])
|
|
build = builds[0] if builds else {}
|
|
else:
|
|
build = {}
|
|
|
|
build_id = build.get("buildId")
|
|
build_target = build_target if build_id else None
|
|
return BuildInfo(build.get("branch"), build_id, build_target,
|
|
build.get("releaseCandidateName"))
|