You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
412 lines
13 KiB
412 lines
13 KiB
# Copyright 2019 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Module to build a image from a specific commit, branch or pull request
|
|
|
|
This module is allows each of the OSS Fuzz projects fuzzers to be built
|
|
from a specific point in time. This feature can be used for implementations
|
|
like continuious integration fuzzing and bisection to find errors
|
|
"""
|
|
import argparse
|
|
import bisect
|
|
import datetime
|
|
from distutils import spawn
|
|
import os
|
|
import collections
|
|
import json
|
|
import logging
|
|
import re
|
|
import shutil
|
|
import tempfile
|
|
|
|
import helper
|
|
import repo_manager
|
|
import retry
|
|
import utils
|
|
|
|
BuildData = collections.namedtuple(
|
|
'BuildData', ['project_name', 'engine', 'sanitizer', 'architecture'])
|
|
|
|
_GIT_DIR_MARKER = 'gitdir: '
|
|
_IMAGE_BUILD_TRIES = 3
|
|
|
|
|
|
class BaseBuilderRepo:
|
|
"""Repo of base-builder images."""
|
|
|
|
def __init__(self):
|
|
self.timestamps = []
|
|
self.digests = []
|
|
|
|
def add_digest(self, timestamp, digest):
|
|
"""Add a digest."""
|
|
self.timestamps.append(timestamp)
|
|
self.digests.append(digest)
|
|
|
|
def find_digest(self, timestamp):
|
|
"""Find the latest image before the given timestamp."""
|
|
index = bisect.bisect_right(self.timestamps, timestamp)
|
|
if index > 0:
|
|
return self.digests[index - 1]
|
|
|
|
logging.error('Failed to find suitable base-builder.')
|
|
return None
|
|
|
|
|
|
def _replace_gitdir(src_dir, file_path):
|
|
"""Replace gitdir with a relative path."""
|
|
with open(file_path) as handle:
|
|
lines = handle.readlines()
|
|
|
|
new_lines = []
|
|
for line in lines:
|
|
if line.startswith(_GIT_DIR_MARKER):
|
|
absolute_path = line[len(_GIT_DIR_MARKER):].strip()
|
|
if not os.path.isabs(absolute_path):
|
|
# Already relative.
|
|
return
|
|
|
|
current_dir = os.path.dirname(file_path)
|
|
# Rebase to /src rather than the host src dir.
|
|
base_dir = current_dir.replace(src_dir, '/src')
|
|
relative_path = os.path.relpath(absolute_path, base_dir)
|
|
logging.info('Replacing absolute submodule gitdir from %s to %s',
|
|
absolute_path, relative_path)
|
|
|
|
line = _GIT_DIR_MARKER + relative_path
|
|
|
|
new_lines.append(line)
|
|
|
|
with open(file_path, 'w') as handle:
|
|
handle.write(''.join(new_lines))
|
|
|
|
|
|
def _make_gitdirs_relative(src_dir):
|
|
"""Make gitdirs relative."""
|
|
for root_dir, _, files in os.walk(src_dir):
|
|
for filename in files:
|
|
if filename != '.git':
|
|
continue
|
|
|
|
file_path = os.path.join(root_dir, filename)
|
|
_replace_gitdir(src_dir, file_path)
|
|
|
|
|
|
def _replace_base_builder_digest(dockerfile_path, digest):
|
|
"""Replace the base-builder digest in a Dockerfile."""
|
|
with open(dockerfile_path) as handle:
|
|
lines = handle.readlines()
|
|
|
|
new_lines = []
|
|
for line in lines:
|
|
if line.strip().startswith('FROM'):
|
|
line = 'FROM gcr.io/oss-fuzz-base/base-builder@' + digest + '\n'
|
|
|
|
new_lines.append(line)
|
|
|
|
with open(dockerfile_path, 'w') as handle:
|
|
handle.write(''.join(new_lines))
|
|
|
|
|
|
def copy_src_from_docker(project_name, host_dir):
|
|
"""Copy /src from docker to the host."""
|
|
# Copy /src to host.
|
|
image_name = 'gcr.io/oss-fuzz/' + project_name
|
|
src_dir = os.path.join(host_dir, 'src')
|
|
if os.path.exists(src_dir):
|
|
shutil.rmtree(src_dir, ignore_errors=True)
|
|
|
|
docker_args = [
|
|
'-v',
|
|
host_dir + ':/out',
|
|
image_name,
|
|
'cp',
|
|
'-r',
|
|
'-p',
|
|
'/src',
|
|
'/out',
|
|
]
|
|
helper.docker_run(docker_args)
|
|
|
|
# Submodules can have gitdir entries which point to absolute paths. Make them
|
|
# relative, as otherwise we can't do operations on the checkout on the host.
|
|
_make_gitdirs_relative(src_dir)
|
|
return src_dir
|
|
|
|
|
|
@retry.wrap(_IMAGE_BUILD_TRIES, 2)
|
|
def _build_image_with_retries(project_name):
|
|
"""Build image with retries."""
|
|
return helper.build_image_impl(project_name)
|
|
|
|
|
|
def get_required_post_checkout_steps(dockerfile_path):
|
|
"""Get required post checkout steps (best effort)."""
|
|
|
|
checkout_pattern = re.compile(r'\s*RUN\s*(git|svn|hg)')
|
|
|
|
# If the build.sh is copied from upstream, we need to copy it again after
|
|
# changing the revision to ensure correct building.
|
|
post_run_pattern = re.compile(r'\s*RUN\s*(.*build\.sh.*(\$SRC|/src).*)')
|
|
|
|
with open(dockerfile_path) as handle:
|
|
lines = handle.readlines()
|
|
|
|
subsequent_run_cmds = []
|
|
for i, line in enumerate(lines):
|
|
if checkout_pattern.match(line):
|
|
subsequent_run_cmds = []
|
|
continue
|
|
|
|
match = post_run_pattern.match(line)
|
|
if match:
|
|
workdir = helper.workdir_from_lines(lines[:i])
|
|
command = match.group(1)
|
|
subsequent_run_cmds.append((workdir, command))
|
|
|
|
return subsequent_run_cmds
|
|
|
|
|
|
# pylint: disable=too-many-locals
|
|
def build_fuzzers_from_commit(commit,
|
|
build_repo_manager,
|
|
host_src_path,
|
|
build_data,
|
|
base_builder_repo=None):
|
|
"""Builds a OSS-Fuzz fuzzer at a specific commit SHA.
|
|
|
|
Args:
|
|
commit: The commit SHA to build the fuzzers at.
|
|
build_repo_manager: The OSS-Fuzz project's repo manager to be built at.
|
|
build_data: A struct containing project build information.
|
|
base_builder_repo: A BaseBuilderRepo.
|
|
Returns:
|
|
0 on successful build or error code on failure.
|
|
"""
|
|
oss_fuzz_repo_manager = repo_manager.RepoManager(helper.OSS_FUZZ_DIR)
|
|
num_retry = 1
|
|
|
|
def cleanup():
|
|
# Re-copy /src for a clean checkout every time.
|
|
copy_src_from_docker(build_data.project_name,
|
|
os.path.dirname(host_src_path))
|
|
build_repo_manager.fetch_all_remotes()
|
|
|
|
projects_dir = os.path.join('projects', build_data.project_name)
|
|
dockerfile_path = os.path.join(projects_dir, 'Dockerfile')
|
|
|
|
for i in range(num_retry + 1):
|
|
build_repo_manager.checkout_commit(commit, clean=False)
|
|
|
|
post_checkout_steps = get_required_post_checkout_steps(dockerfile_path)
|
|
for workdir, post_checkout_step in post_checkout_steps:
|
|
logging.info('Running post-checkout step `%s` in %s.', post_checkout_step,
|
|
workdir)
|
|
helper.docker_run([
|
|
'-w',
|
|
workdir,
|
|
'-v',
|
|
host_src_path + ':' + '/src',
|
|
'gcr.io/oss-fuzz/' + build_data.project_name,
|
|
'/bin/bash',
|
|
'-c',
|
|
post_checkout_step,
|
|
])
|
|
|
|
result = helper.build_fuzzers_impl(project_name=build_data.project_name,
|
|
clean=True,
|
|
engine=build_data.engine,
|
|
sanitizer=build_data.sanitizer,
|
|
architecture=build_data.architecture,
|
|
env_to_add=None,
|
|
source_path=host_src_path,
|
|
mount_location='/src')
|
|
if result == 0 or i == num_retry:
|
|
break
|
|
|
|
# Retry with an OSS-Fuzz builder container that's closer to the project
|
|
# commit date.
|
|
commit_date = build_repo_manager.commit_date(commit)
|
|
|
|
# Find first change in the projects/<PROJECT> directory before the project
|
|
# commit date.
|
|
oss_fuzz_commit, _, _ = oss_fuzz_repo_manager.git([
|
|
'log', '--before=' + commit_date.isoformat(), '-n1', '--format=%H',
|
|
projects_dir
|
|
],
|
|
check_result=True)
|
|
oss_fuzz_commit = oss_fuzz_commit.strip()
|
|
if not oss_fuzz_commit:
|
|
logging.info(
|
|
'Could not find first OSS-Fuzz commit prior to upstream commit. '
|
|
'Falling back to oldest integration commit.')
|
|
|
|
# Find the oldest commit.
|
|
oss_fuzz_commit, _, _ = oss_fuzz_repo_manager.git(
|
|
['log', '--reverse', '--format=%H', projects_dir], check_result=True)
|
|
|
|
oss_fuzz_commit = oss_fuzz_commit.splitlines()[0].strip()
|
|
|
|
if not oss_fuzz_commit:
|
|
logging.error('Failed to get oldest integration commit.')
|
|
break
|
|
|
|
logging.info('Build failed. Retrying on earlier OSS-Fuzz commit %s.',
|
|
oss_fuzz_commit)
|
|
|
|
# Check out projects/<PROJECT> dir to the commit that was found.
|
|
oss_fuzz_repo_manager.git(['checkout', oss_fuzz_commit, projects_dir],
|
|
check_result=True)
|
|
|
|
# Also use the closest base-builder we can find.
|
|
if base_builder_repo:
|
|
base_builder_digest = base_builder_repo.find_digest(commit_date)
|
|
if not base_builder_digest:
|
|
return False
|
|
|
|
logging.info('Using base-builder with digest %s.', base_builder_digest)
|
|
_replace_base_builder_digest(dockerfile_path, base_builder_digest)
|
|
|
|
# Rebuild image and re-copy src dir since things in /src could have changed.
|
|
if not _build_image_with_retries(build_data.project_name):
|
|
logging.error('Failed to rebuild image.')
|
|
return False
|
|
|
|
cleanup()
|
|
|
|
cleanup()
|
|
return result == 0
|
|
|
|
|
|
def detect_main_repo(project_name, repo_name=None, commit=None):
|
|
"""Checks a docker image for the main repo of an OSS-Fuzz project.
|
|
|
|
Note: The default is to use the repo name to detect the main repo.
|
|
|
|
Args:
|
|
project_name: The name of the oss-fuzz project.
|
|
repo_name: The name of the main repo in an OSS-Fuzz project.
|
|
commit: A commit SHA that is associated with the main repo.
|
|
src_dir: The location of the projects source on the docker image.
|
|
|
|
Returns:
|
|
The repo's origin, the repo's path.
|
|
"""
|
|
|
|
if not repo_name and not commit:
|
|
logging.error(
|
|
'Error: can not detect main repo without a repo_name or a commit.')
|
|
return None, None
|
|
if repo_name and commit:
|
|
logging.info(
|
|
'Both repo name and commit specific. Using repo name for detection.')
|
|
|
|
# Change to oss-fuzz main directory so helper.py runs correctly.
|
|
utils.chdir_to_root()
|
|
if not _build_image_with_retries(project_name):
|
|
logging.error('Error: building %s image failed.', project_name)
|
|
return None, None
|
|
docker_image_name = 'gcr.io/oss-fuzz/' + project_name
|
|
command_to_run = [
|
|
'docker', 'run', '--rm', '-t', docker_image_name, 'python3',
|
|
os.path.join('/opt', 'cifuzz', 'detect_repo.py')
|
|
]
|
|
if repo_name:
|
|
command_to_run.extend(['--repo_name', repo_name])
|
|
else:
|
|
command_to_run.extend(['--example_commit', commit])
|
|
out, _, _ = utils.execute(command_to_run)
|
|
match = re.search(r'\bDetected repo: ([^ ]+) ([^ ]+)', out.rstrip())
|
|
if match and match.group(1) and match.group(2):
|
|
return match.group(1), match.group(2)
|
|
|
|
logging.error('Failed to detect repo:\n%s', out)
|
|
return None, None
|
|
|
|
|
|
def load_base_builder_repo():
|
|
"""Get base-image digests."""
|
|
gcloud_path = spawn.find_executable('gcloud')
|
|
if not gcloud_path:
|
|
logging.warning('gcloud not found in PATH.')
|
|
return None
|
|
|
|
result, _, _ = utils.execute([
|
|
gcloud_path,
|
|
'container',
|
|
'images',
|
|
'list-tags',
|
|
'gcr.io/oss-fuzz-base/base-builder',
|
|
'--format=json',
|
|
'--sort-by=timestamp',
|
|
],
|
|
check_result=True)
|
|
result = json.loads(result)
|
|
|
|
repo = BaseBuilderRepo()
|
|
for image in result:
|
|
timestamp = datetime.datetime.fromisoformat(
|
|
image['timestamp']['datetime']).astimezone(datetime.timezone.utc)
|
|
repo.add_digest(timestamp, image['digest'])
|
|
|
|
return repo
|
|
|
|
|
|
def main():
|
|
"""Main function."""
|
|
logging.getLogger().setLevel(logging.INFO)
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description='Build fuzzers at a specific commit')
|
|
parser.add_argument('--project_name',
|
|
help='The name of the project where the bug occurred.',
|
|
required=True)
|
|
parser.add_argument('--commit',
|
|
help='The newest commit SHA to be bisected.',
|
|
required=True)
|
|
parser.add_argument('--engine',
|
|
help='The default is "libfuzzer".',
|
|
default='libfuzzer')
|
|
parser.add_argument('--sanitizer',
|
|
default='address',
|
|
help='The default is "address".')
|
|
parser.add_argument('--architecture', default='x86_64')
|
|
|
|
args = parser.parse_args()
|
|
|
|
repo_url, repo_path = detect_main_repo(args.project_name, commit=args.commit)
|
|
|
|
if not repo_url or not repo_path:
|
|
raise ValueError('Main git repo can not be determined.')
|
|
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
host_src_dir = copy_src_from_docker(args.project_name, tmp_dir)
|
|
build_repo_manager = repo_manager.RepoManager(
|
|
os.path.join(host_src_dir, os.path.basename(repo_path)))
|
|
base_builder_repo = load_base_builder_repo()
|
|
|
|
build_data = BuildData(project_name=args.project_name,
|
|
engine=args.engine,
|
|
sanitizer=args.sanitizer,
|
|
architecture=args.architecture)
|
|
if not build_fuzzers_from_commit(args.commit,
|
|
build_repo_manager,
|
|
host_src_dir,
|
|
build_data,
|
|
base_builder_repo=base_builder_repo):
|
|
raise RuntimeError('Failed to build.')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|