king
/
v811_spc009
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'main'
${ noResults }
v811_spc009/device/google/cuttlefish/host/commands/fetcher/fetch_cvd.cc

553 lines
23 KiB
Raw Permalink Blame History

//
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <iostream>
#include <iterator>
#include <string>
#include <sys/stat.h>
#include <unistd.h>
#include "android-base/logging.h"
#include "android-base/strings.h"
#include "gflags/gflags.h"
#include "common/libs/fs/shared_fd.h"
#include "common/libs/utils/archive.h"
#include "common/libs/utils/files.h"
#include "common/libs/utils/subprocess.h"
#include "host/libs/config/fetcher_config.h"
#include "build_api.h"
#include "credential_source.h"
#include "install_zip.h"
namespace {
const std::string DEFAULT_BRANCH = "aosp-master";
const std::string DEFAULT_BUILD_TARGET = "aosp_cf_x86_64_phone-userdebug";
}
using cuttlefish::CurrentDirectory;
DEFINE_string(default_build, DEFAULT_BRANCH + "/" + DEFAULT_BUILD_TARGET,
"source for the cuttlefish build to use (vendor.img + host)");
DEFINE_string(system_build, "", "source for system.img and product.img");
DEFINE_string(kernel_build, "", "source for the kernel or gki target");
DEFINE_string(bootloader_build, "", "source for the bootloader target");
DEFINE_string(otatools_build, "", "source for the host ota tools");
DEFINE_bool(download_img_zip, true, "Whether to fetch the -img-*.zip file.");
DEFINE_bool(download_target_files_zip, false, "Whether to fetch the "
"-target_files-*.zip file.");
DEFINE_string(credential_source, "", "Build API credential source");
DEFINE_string(directory, CurrentDirectory(), "Target directory to fetch "
"files into");
DEFINE_bool(run_next_stage, false, "Continue running the device through the next stage.");
DEFINE_string(wait_retry_period, "20", "Retry period for pending builds given "
"in seconds. Set to 0 to not wait.");
namespace cuttlefish {
namespace {
const std::string HOST_TOOLS = "cvd-host_package.tar.gz";
const std::string OTA_TOOLS = "otatools.zip";
const std::string OTA_TOOLS_DIR = "/otatools/";
/** Returns the name of one of the artifact target zip files.
*
* For example, for a target "aosp_cf_x86_phone-userdebug" at a build "5824130",
* the image zip file would be "aosp_cf_x86_phone-img-5824130.zip"
*/
std::string TargetBuildZipFromArtifacts(
const Build& build, const std::string& name,
const std::vector<Artifact>& artifacts) {
std::string product = std::visit([](auto&& arg) { return arg.product; }, build);
auto id = std::visit([](auto&& arg) { return arg.id; }, build);
auto match = product + "-" + name + "-" + id;
for (const auto& artifact : artifacts) {
if (artifact.Name().find(match) != std::string::npos) {
return artifact.Name();
}
}
return "";
}
std::vector<std::string> download_images(BuildApi* build_api,
const Build& build,
const std::string& target_directory,
const std::vector<std::string>& images) {
auto artifacts = build_api->Artifacts(build);
std::string img_zip_name = TargetBuildZipFromArtifacts(build, "img", artifacts);
if (img_zip_name.size() == 0) {
LOG(ERROR) << "Target " << build << " did not have an img zip";
return {};
}
std::string local_path = target_directory + "/" + img_zip_name;
if (!build_api->ArtifactToFile(build, img_zip_name, local_path)) {
LOG(ERROR) << "Unable to download " << build << ":" << img_zip_name << " to "
<< local_path;
return {};
}
std::vector<std::string> files = ExtractImages(local_path, target_directory, images);
if (files.empty()) {
LOG(ERROR) << "Could not extract " << local_path;
return {};
}
if (unlink(local_path.c_str()) != 0) {
LOG(ERROR) << "Could not delete " << local_path;
files.push_back(local_path);
}
return files;
}
std::vector<std::string> download_images(BuildApi* build_api,
const Build& build,
const std::string& target_directory) {
return download_images(build_api, build, target_directory, {});
}
std::vector<std::string> download_target_files(BuildApi* build_api,
const Build& build,
const std::string& target_directory) {
auto artifacts = build_api->Artifacts(build);
std::string target_zip = TargetBuildZipFromArtifacts(build, "target_files", artifacts);
if (target_zip.size() == 0) {
LOG(ERROR) << "Target " << build << " did not have a target files zip";
return {};
}
std::string local_path = target_directory + "/" + target_zip;
if (!build_api->ArtifactToFile(build, target_zip, local_path)) {
LOG(ERROR) << "Unable to download " << build << ":" << target_zip << " to "
<< local_path;
return {};
}
return {local_path};
}
std::vector<std::string> download_host_package(BuildApi* build_api,
const Build& build,
const std::string& target_directory) {
auto artifacts = build_api->Artifacts(build);
bool has_host_package = false;
for (const auto& artifact : artifacts) {
has_host_package |= artifact.Name() == HOST_TOOLS;
}
if (!has_host_package) {
LOG(ERROR) << "Target " << build << " did not have " << HOST_TOOLS;
return {};
}
std::string local_path = target_directory + "/" + HOST_TOOLS;
if (!build_api->ArtifactToFile(build, HOST_TOOLS, local_path)) {
LOG(ERROR) << "Unable to download " << build << ":" << HOST_TOOLS << " to "
<< local_path;
return {};
}
Archive archive(local_path);
if (!archive.ExtractAll(target_directory)) {
LOG(ERROR) << "Could not extract " << local_path;
return {};
}
std::vector<std::string> files = archive.Contents();
for (auto& file : files) {
file = target_directory + "/" + file;
}
if (unlink(local_path.c_str()) != 0) {
LOG(ERROR) << "Could not delete " << local_path;
files.push_back(local_path);
}
return files;
}
std::vector<std::string> download_ota_tools(BuildApi* build_api,
const Build& build,
const std::string& target_directory) {
auto artifacts = build_api->Artifacts(build);
bool has_host_package = false;
for (const auto& artifact : artifacts) {
has_host_package |= artifact.Name() == OTA_TOOLS;
}
if (!has_host_package) {
LOG(ERROR) << "Target " << build << " did not have " << OTA_TOOLS;
return {};
}
std::string local_path = target_directory + "/" + OTA_TOOLS;
if (!build_api->ArtifactToFile(build, OTA_TOOLS, local_path)) {
LOG(ERROR) << "Unable to download " << build << ":" << OTA_TOOLS << " to "
<< local_path;
return {};
}
std::string otatools_dir = target_directory + OTA_TOOLS_DIR;
if (!DirectoryExists(otatools_dir) && mkdir(otatools_dir.c_str(), 0777) != 0) {
LOG(ERROR) << "Could not create " << otatools_dir;
return {};
}
Archive archive(local_path);
if (!archive.ExtractAll(otatools_dir)) {
LOG(ERROR) << "Could not extract " << local_path;
return {};
}
std::vector<std::string> files = archive.Contents();
for (auto& file : files) {
file = target_directory + OTA_TOOLS_DIR + file;
}
files.push_back(local_path);
return files;
}
void AddFilesToConfig(FileSource purpose, const Build& build,
const std::vector<std::string>& paths,
FetcherConfig* config,
const std::string& directory_prefix,
bool override_entry = false) {
for (const std::string& path : paths) {
std::string_view local_path(path);
if (!android::base::ConsumePrefix(&local_path, directory_prefix)) {
LOG(ERROR) << "Failed to remove prefix " << directory_prefix << " from "
<< local_path;
}
while (android::base::StartsWith(local_path, "/")) {
android::base::ConsumePrefix(&local_path, "/");
}
// TODO(schuffelen): Do better for local builds here.
auto id = std::visit([](auto&& arg) { return arg.id; }, build);
auto target = std::visit([](auto&& arg) { return arg.target; }, build);
CvdFile file(purpose, id, target, std::string(local_path));
bool added = config->add_cvd_file(file, override_entry);
if (!added) {
LOG(ERROR) << "Duplicate file " << file;
LOG(ERROR) << "Existing file: " << config->get_cvd_files()[path];
LOG(FATAL) << "Failed to add path " << path;
}
}
}
std::string USAGE_MESSAGE =
"<flags>\n"
"\n"
"\"*_build\" flags accept values in the following format:\n"
"\"branch/build_target\" - latest build of \"branch\" for \"build_target\"\n"
"\"build_id/build_target\" - build \"build_id\" for \"build_target\"\n"
"\"branch\" - latest build of \"branch\" for \"aosp_cf_x86_phone-userdebug\"\n"
"\"build_id\" - build \"build_id\" for \"aosp_cf_x86_phone-userdebug\"\n";
} // namespace
int FetchCvdMain(int argc, char** argv) {
::android::base::InitLogging(argv, android::base::StderrLogger);
gflags::SetUsageMessage(USAGE_MESSAGE);
gflags::ParseCommandLineFlags(&argc, &argv, true);
FetcherConfig config;
config.RecordFlags();
std::string target_dir = AbsolutePath(FLAGS_directory);
if (!DirectoryExists(target_dir) && mkdir(target_dir.c_str(), 0777) != 0) {
LOG(FATAL) << "Could not create " << target_dir;
}
std::string target_dir_slash = target_dir;
std::chrono::seconds retry_period(std::stoi(FLAGS_wait_retry_period));
curl_global_init(CURL_GLOBAL_DEFAULT);
{
std::unique_ptr<CredentialSource> credential_source;
if (FLAGS_credential_source == "gce") {
credential_source = GceMetadataCredentialSource::make();
} else if (FLAGS_credential_source != "") {
credential_source = FixedCredentialSource::make(FLAGS_credential_source);
}
BuildApi build_api(std::move(credential_source));
auto default_build = ArgumentToBuild(&build_api, FLAGS_default_build,
DEFAULT_BUILD_TARGET,
retry_period);
std::vector<std::string> host_package_files =
download_host_package(&build_api, default_build, target_dir);
if (host_package_files.empty()) {
LOG(FATAL) << "Could not download host package for " << default_build;
}
AddFilesToConfig(FileSource::DEFAULT_BUILD, default_build,
host_package_files, &config, target_dir);
if (FLAGS_system_build != "" || FLAGS_kernel_build != "" || FLAGS_otatools_build != "") {
auto ota_build = default_build;
if (FLAGS_otatools_build != "") {
ota_build = ArgumentToBuild(&build_api, FLAGS_otatools_build,
DEFAULT_BUILD_TARGET, retry_period);
}
std::vector<std::string> ota_tools_files =
download_ota_tools(&build_api, ota_build, target_dir);
if (ota_tools_files.empty()) {
LOG(FATAL) << "Could not download ota tools for " << ota_build;
}
AddFilesToConfig(FileSource::DEFAULT_BUILD, default_build,
ota_tools_files, &config, target_dir);
}
if (FLAGS_download_img_zip) {
std::vector<std::string> image_files =
download_images(&build_api, default_build, target_dir);
if (image_files.empty()) {
LOG(FATAL) << "Could not download images for " << default_build;
}
LOG(INFO) << "Adding img-zip files for default build";
for (auto& file : image_files) {
LOG(INFO) << file;
}
AddFilesToConfig(FileSource::DEFAULT_BUILD, default_build, image_files,
&config, target_dir);
}
if (FLAGS_system_build != "" || FLAGS_download_target_files_zip) {
std::string default_target_dir = target_dir + "/default";
if (mkdir(default_target_dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) < 0) {
LOG(FATAL) << "Could not create " << default_target_dir;
}
std::vector<std::string> target_files =
download_target_files(&build_api, default_build, default_target_dir);
if (target_files.empty()) {
LOG(FATAL) << "Could not download target files for " << default_build;
}
LOG(INFO) << "Adding target files for default build";
AddFilesToConfig(FileSource::DEFAULT_BUILD, default_build, target_files,
&config, target_dir);
}
if (FLAGS_system_build != "") {
auto system_build = ArgumentToBuild(&build_api, FLAGS_system_build,
DEFAULT_BUILD_TARGET,
retry_period);
bool system_in_img_zip = true;
if (FLAGS_download_img_zip) {
std::vector<std::string> image_files =
download_images(&build_api, system_build, target_dir,
{"system.img", "product.img"});
if (image_files.empty()) {
LOG(INFO) << "Could not find system image for " << system_build
<< "in the img zip. Assuming a super image build, which will "
<< "get the system image from the target zip.";
system_in_img_zip = false;
} else {
LOG(INFO) << "Adding img-zip files for system build";
AddFilesToConfig(FileSource::SYSTEM_BUILD, system_build, image_files,
&config, target_dir, true);
}
}
std::string system_target_dir = target_dir + "/system";
if (mkdir(system_target_dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) < 0) {
LOG(FATAL) << "Could not create " << system_target_dir;
}
std::vector<std::string> target_files =
download_target_files(&build_api, system_build, system_target_dir);
if (target_files.empty()) {
LOG(FATAL) << "Could not download target files for " << system_build;
return -1;
}
AddFilesToConfig(FileSource::SYSTEM_BUILD, system_build, target_files,
&config, target_dir);
if (!system_in_img_zip) {
if (ExtractImages(target_files[0], target_dir, {"IMAGES/system.img"})
!= std::vector<std::string>{}) {
std::string extracted_system = target_dir + "/IMAGES/system.img";
std::string target_system = target_dir + "/system.img";
if (rename(extracted_system.c_str(), target_system.c_str())) {
int error_num = errno;
LOG(FATAL) << "Could not replace system.img in target directory: "
<< strerror(error_num);
return -1;
}
} else {
LOG(FATAL) << "Could not get system.img from the target zip";
return -1;
}
if (ExtractImages(target_files[0], target_dir, {"IMAGES/product.img"})
!= std::vector<std::string>{}) {
std::string extracted_product = target_dir + "/IMAGES/product.img";
std::string target_product = target_dir + "/product.img";
if (rename(extracted_product.c_str(), target_product.c_str())) {
int error_num = errno;
LOG(FATAL) << "Could not replace product.img in target directory"
<< strerror(error_num);
return -1;
}
}
if (ExtractImages(target_files[0], target_dir, {"IMAGES/system_ext.img"})
!= std::vector<std::string>{}) {
std::string extracted_system_ext = target_dir + "/IMAGES/system_ext.img";
std::string target_system_ext = target_dir + "/system_ext.img";
if (rename(extracted_system_ext.c_str(), target_system_ext.c_str())) {
int error_num = errno;
LOG(FATAL) << "Could not move system_ext.img in target directory: "
<< strerror(error_num);
return -1;
}
}
if (ExtractImages(target_files[0], target_dir, {"IMAGES/vbmeta_system.img"})
!= std::vector<std::string>{}) {
std::string extracted_vbmeta_system = target_dir + "/IMAGES/vbmeta_system.img";
std::string target_vbmeta_system = target_dir + "/vbmeta_system.img";
if (rename(extracted_vbmeta_system.c_str(), target_vbmeta_system.c_str())) {
int error_num = errno;
LOG(FATAL) << "Could not move vbmeta_system.img in target directory: "
<< strerror(error_num);
return -1;
}
}
// This should technically call AddFilesToConfig with the produced files,
// but it will conflict with the ones produced from the default system image
// and pie doesn't care about the produced file list anyway.
}
}
if (FLAGS_kernel_build != "") {
auto kernel_build = ArgumentToBuild(&build_api, FLAGS_kernel_build,
"kernel", retry_period);
std::string local_path = target_dir + "/kernel";
if (build_api.ArtifactToFile(kernel_build, "bzImage", local_path)) {
AddFilesToConfig(FileSource::KERNEL_BUILD, kernel_build, {local_path},
&config, target_dir);
}
// If the kernel is from an arm/aarch64 build, the artifact will be called
// Image.
else if (build_api.ArtifactToFile(kernel_build, "Image", local_path)) {
AddFilesToConfig(FileSource::KERNEL_BUILD, kernel_build, {local_path},
&config, target_dir);
} else {
LOG(FATAL) << "Could not download " << kernel_build << ":bzImage to "
<< local_path;
}
std::vector<Artifact> kernel_artifacts = build_api.Artifacts(kernel_build);
for (const auto& artifact : kernel_artifacts) {
if (artifact.Name() != "initramfs.img") {
continue;
}
bool downloaded = build_api.ArtifactToFile(
kernel_build, "initramfs.img", target_dir + "/initramfs.img");
if (!downloaded) {
LOG(FATAL) << "Could not download " << kernel_build << ":initramfs.img to "
<< target_dir + "/initramfs.img";
}
AddFilesToConfig(FileSource::KERNEL_BUILD, kernel_build,
{target_dir + "/initramfs.img"}, &config, target_dir);
}
}
if (FLAGS_bootloader_build != "") {
auto bootloader_build = ArgumentToBuild(&build_api,
FLAGS_bootloader_build,
"u-boot_crosvm_x86_64",
retry_period);
std::string local_path = target_dir + "/bootloader";
if (build_api.ArtifactToFile(bootloader_build, "u-boot.rom", local_path)) {
AddFilesToConfig(FileSource::BOOTLOADER_BUILD, bootloader_build,
{local_path}, &config, target_dir, true);
}
// If the bootloader is from an arm/aarch64 build, the artifact will be of
// filetype bin.
else if (build_api.ArtifactToFile(bootloader_build, "u-boot.bin",
local_path)) {
AddFilesToConfig(FileSource::BOOTLOADER_BUILD, bootloader_build,
{local_path}, &config, target_dir, true);
} else {
LOG(FATAL) << "Could not download " << bootloader_build << ":u-boot.rom to "
<< local_path;
}
}
}
curl_global_cleanup();
// Due to constraints of the build system, artifacts intentionally cannot determine
// their own build id. So it's unclear which build number fetch_cvd itself was built at.
// https://android.googlesource.com/platform/build/+/979c9f3/Changes.md#build_number
std::string fetcher_path = target_dir + "/fetcher_config.json";
AddFilesToConfig(GENERATED, DeviceBuild("", ""), {fetcher_path}, &config,
target_dir);
config.SaveToFile(fetcher_path);
for (const auto& file : config.get_cvd_files()) {
std::cout << target_dir << "/" << file.second.file_path << "\n";
}
std::cout << std::flush;
if (!FLAGS_run_next_stage) {
return 0;
}
// Ignore return code. We want to make sure there is no running instance,
// and stop_cvd will exit with an error code if there is already no running instance.
Command stop_cmd(target_dir + "/bin/stop_cvd");
stop_cmd.RedirectStdIO(Subprocess::StdIOChannel::kStdOut,
Subprocess::StdIOChannel::kStdErr);
stop_cmd.Start().Wait();
// gflags::ParseCommandLineFlags will remove fetch_cvd's flags from this.
// This depends the remove_flags argument (3rd) is "true".
auto filelist_fd = SharedFD::MemfdCreate("files_list");
if (!filelist_fd->IsOpen()) {
LOG(FATAL) << "Unable to create temp file to write file list. "
<< filelist_fd->StrError() << " (" << filelist_fd->GetErrno() << ")";
}
for (const auto& file : config.get_cvd_files()) {
std::string file_entry = file.second.file_path + "\n";
auto chars_written = filelist_fd->Write(file_entry.c_str(), file_entry.size());
if (chars_written != file_entry.size()) {
LOG(FATAL) << "Unable to write entry to file list. Expected to write "
<< file_entry.size() << " but wrote " << chars_written << ". "
<< filelist_fd->StrError() << " (" << filelist_fd->GetErrno() << ")";
}
}
auto seek_result = filelist_fd->LSeek(0, SEEK_SET);
if (seek_result != 0) {
LOG(FATAL) << "Unable to seek on file list file. Expected 0, received " << seek_result
<< filelist_fd->StrError() << " (" << filelist_fd->GetErrno() << ")";
}
if (filelist_fd->UNMANAGED_Dup2(0) == -1) {
LOG(FATAL) << "Unable to set file list to stdin. "
<< filelist_fd->StrError() << " (" << filelist_fd->GetErrno() << ")";
}
// TODO(b/139199114): Go into assemble_cvd when the interface is stable and implemented.
std::string next_stage = target_dir + "/bin/launch_cvd";
std::vector<const char*> next_stage_argv = {"launch_cvd"};
LOG(INFO) << "Running " << next_stage;
for (int i = 1; i < argc; i++) {
LOG(INFO) << argv[i];
next_stage_argv.push_back(argv[i]);
}
next_stage_argv.push_back(nullptr);
execv(next_stage.c_str(), const_cast<char* const*>(next_stage_argv.data()));
int error = errno;
LOG(FATAL) << "execv returned with errno " << error << ":" << strerror(error);
return -1;
}
} // namespace cuttlefish
int main(int argc, char** argv) {
return cuttlefish::FetchCvdMain(argc, argv);
}
Reference in new issue View Git Blame Copy Permalink