You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
352 lines
10 KiB
352 lines
10 KiB
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
# Copyright 2020 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""Checks for various upstream events with the Rust toolchain.
|
|
|
|
Sends an email if something interesting (probably) happened.
|
|
"""
|
|
|
|
# pylint: disable=cros-logging-import
|
|
|
|
import argparse
|
|
import itertools
|
|
import json
|
|
import logging
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from typing import Any, Dict, Iterable, List, Optional, Tuple, NamedTuple
|
|
|
|
from cros_utils import email_sender
|
|
from cros_utils import tiny_render
|
|
|
|
|
|
def gentoo_sha_to_link(sha: str) -> str:
|
|
"""Gets a URL to a webpage that shows the Gentoo commit at `sha`."""
|
|
return f'https://gitweb.gentoo.org/repo/gentoo.git/commit?id={sha}'
|
|
|
|
|
|
def send_email(subject: str, body: List[tiny_render.Piece]) -> None:
|
|
"""Sends an email with the given title and body to... whoever cares."""
|
|
email_sender.EmailSender().SendX20Email(
|
|
subject=subject,
|
|
identifier='rust-watch',
|
|
well_known_recipients=['cros-team'],
|
|
text_body=tiny_render.render_text_pieces(body),
|
|
html_body=tiny_render.render_html_pieces(body),
|
|
)
|
|
|
|
|
|
class RustReleaseVersion(NamedTuple):
|
|
"""Represents a version of Rust's stable compiler."""
|
|
major: int
|
|
minor: int
|
|
patch: int
|
|
|
|
@staticmethod
|
|
def from_string(version_string: str) -> 'RustReleaseVersion':
|
|
m = re.match(r'(\d+)\.(\d+)\.(\d+)', version_string)
|
|
if not m:
|
|
raise ValueError(f"{version_string!r} isn't a valid version string")
|
|
return RustReleaseVersion(*[int(x) for x in m.groups()])
|
|
|
|
def __str__(self) -> str:
|
|
return f'{self.major}.{self.minor}.{self.patch}'
|
|
|
|
def to_json(self) -> str:
|
|
return str(self)
|
|
|
|
@staticmethod
|
|
def from_json(s: str) -> 'RustReleaseVersion':
|
|
return RustReleaseVersion.from_string(s)
|
|
|
|
|
|
class State(NamedTuple):
|
|
"""State that we keep around from run to run."""
|
|
# The last Rust release tag that we've seen.
|
|
last_seen_release: RustReleaseVersion
|
|
|
|
# We track Gentoo's upstream Rust ebuild. This is the last SHA we've seen
|
|
# that updates it.
|
|
last_gentoo_sha: str
|
|
|
|
def to_json(self) -> Dict[str, Any]:
|
|
return {
|
|
'last_seen_release': self.last_seen_release.to_json(),
|
|
'last_gentoo_sha': self.last_gentoo_sha,
|
|
}
|
|
|
|
@staticmethod
|
|
def from_json(s: Dict[str, Any]) -> 'State':
|
|
return State(
|
|
last_seen_release=RustReleaseVersion.from_json(s['last_seen_release']),
|
|
last_gentoo_sha=s['last_gentoo_sha'],
|
|
)
|
|
|
|
|
|
def parse_release_tags(lines: Iterable[str]) -> Iterable[RustReleaseVersion]:
|
|
"""Parses `git ls-remote --tags` output into Rust stable release versions."""
|
|
refs_tags = 'refs/tags/'
|
|
for line in lines:
|
|
_sha, tag = line.split(None, 1)
|
|
tag = tag.strip()
|
|
# Each tag has an associated 'refs/tags/name^{}', which is the actual
|
|
# object that the tag points to. That's irrelevant to us.
|
|
if tag.endswith('^{}'):
|
|
continue
|
|
|
|
if not tag.startswith(refs_tags):
|
|
continue
|
|
|
|
short_tag = tag[len(refs_tags):]
|
|
# There are a few old versioning schemes. Ignore them.
|
|
if short_tag.startswith('0.') or short_tag.startswith('release-'):
|
|
continue
|
|
yield RustReleaseVersion.from_string(short_tag)
|
|
|
|
|
|
def fetch_most_recent_release() -> RustReleaseVersion:
|
|
"""Fetches the most recent stable `rustc` version."""
|
|
result = subprocess.run(
|
|
['git', 'ls-remote', '--tags', 'https://github.com/rust-lang/rust'],
|
|
check=True,
|
|
stdin=None,
|
|
capture_output=True,
|
|
encoding='utf-8',
|
|
)
|
|
tag_lines = result.stdout.strip().splitlines()
|
|
return max(parse_release_tags(tag_lines))
|
|
|
|
|
|
class GitCommit(NamedTuple):
|
|
"""Represents a single git commit."""
|
|
sha: str
|
|
subject: str
|
|
|
|
|
|
def update_git_repo(git_dir: pathlib.Path) -> None:
|
|
"""Updates the repo at `git_dir`, retrying a few times on failure."""
|
|
for i in itertools.count(start=1):
|
|
result = subprocess.run(
|
|
['git', 'fetch', 'origin'],
|
|
check=False,
|
|
cwd=str(git_dir),
|
|
stdin=None,
|
|
)
|
|
|
|
if not result.returncode:
|
|
break
|
|
|
|
if i == 5:
|
|
# 5 attempts is too many. Something else may be wrong.
|
|
result.check_returncode()
|
|
|
|
sleep_time = 60 * i
|
|
logging.error("Failed updating gentoo's repo; will try again in %ds...",
|
|
sleep_time)
|
|
time.sleep(sleep_time)
|
|
|
|
|
|
def get_new_gentoo_commits(git_dir: pathlib.Path,
|
|
most_recent_sha: str) -> List[GitCommit]:
|
|
"""Gets commits to dev-lang/rust since `most_recent_sha`.
|
|
|
|
Older commits come earlier in the returned list.
|
|
"""
|
|
commits = subprocess.run(
|
|
[
|
|
'git',
|
|
'log',
|
|
'--format=%H %s',
|
|
f'{most_recent_sha}..origin/master',
|
|
'--',
|
|
'dev-lang/rust',
|
|
],
|
|
capture_output=True,
|
|
check=False,
|
|
cwd=str(git_dir),
|
|
encoding='utf-8',
|
|
)
|
|
|
|
if commits.returncode:
|
|
logging.error('Error getting new gentoo commits; stderr:\n%s',
|
|
commits.stderr)
|
|
commits.check_returncode()
|
|
|
|
results = []
|
|
for line in commits.stdout.strip().splitlines():
|
|
sha, subject = line.strip().split(None, 1)
|
|
results.append(GitCommit(sha=sha, subject=subject))
|
|
|
|
# `git log` outputs things in newest -> oldest order.
|
|
results.reverse()
|
|
return results
|
|
|
|
|
|
def setup_gentoo_git_repo(git_dir: pathlib.Path) -> str:
|
|
"""Sets up a gentoo git repo at the given directory. Returns HEAD."""
|
|
subprocess.run(
|
|
[
|
|
'git', 'clone', 'https://anongit.gentoo.org/git/repo/gentoo.git',
|
|
str(git_dir)
|
|
],
|
|
stdin=None,
|
|
check=True,
|
|
)
|
|
|
|
head_rev = subprocess.run(
|
|
['git', 'rev-parse', 'HEAD'],
|
|
cwd=str(git_dir),
|
|
check=True,
|
|
stdin=None,
|
|
capture_output=True,
|
|
encoding='utf-8',
|
|
)
|
|
return head_rev.stdout.strip()
|
|
|
|
|
|
def read_state(state_file: pathlib.Path) -> State:
|
|
"""Reads state from the given file."""
|
|
with state_file.open(encoding='utf-8') as f:
|
|
return State.from_json(json.load(f))
|
|
|
|
|
|
def atomically_write_state(state_file: pathlib.Path, state: State) -> None:
|
|
"""Writes state to the given file."""
|
|
temp_file = pathlib.Path(str(state_file) + '.new')
|
|
with temp_file.open('w', encoding='utf-8') as f:
|
|
json.dump(state.to_json(), f)
|
|
temp_file.rename(state_file)
|
|
|
|
|
|
def maybe_compose_email(old_state: State, newest_release: RustReleaseVersion,
|
|
new_gentoo_commits: List[GitCommit]
|
|
) -> Optional[Tuple[str, List[tiny_render.Piece]]]:
|
|
"""Creates an email given our new state, if doing so is appropriate."""
|
|
subject_pieces = []
|
|
body_pieces = []
|
|
|
|
if newest_release > old_state.last_seen_release:
|
|
subject_pieces.append('new rustc release detected')
|
|
body_pieces.append(f'Rustc tag for v{newest_release} was found.')
|
|
|
|
if new_gentoo_commits:
|
|
# Separate the sections a bit for prettier output.
|
|
if body_pieces:
|
|
body_pieces += [tiny_render.line_break, tiny_render.line_break]
|
|
|
|
if len(new_gentoo_commits) == 1:
|
|
subject_pieces.append('new rust ebuild commit detected')
|
|
body_pieces.append('commit:')
|
|
else:
|
|
subject_pieces.append('new rust ebuild commits detected')
|
|
body_pieces.append('commits (newest first):')
|
|
|
|
commit_lines = []
|
|
for commit in new_gentoo_commits:
|
|
commit_lines.append([
|
|
tiny_render.Link(
|
|
gentoo_sha_to_link(commit.sha),
|
|
commit.sha[:12],
|
|
),
|
|
f': {commit.subject}',
|
|
])
|
|
|
|
body_pieces.append(tiny_render.UnorderedList(commit_lines))
|
|
|
|
if not subject_pieces:
|
|
return None
|
|
|
|
subject = '[rust-watch] ' + '; '.join(subject_pieces)
|
|
return subject, body_pieces
|
|
|
|
|
|
def main(argv: List[str]) -> None:
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument(
|
|
'--state_dir', required=True, help='Directory to store state in.')
|
|
parser.add_argument(
|
|
'--skip_email', action='store_true', help="Don't send an email.")
|
|
parser.add_argument(
|
|
'--skip_state_update',
|
|
action='store_true',
|
|
help="Don't update the state file. Doesn't apply to initial setup.")
|
|
opts = parser.parse_args(argv)
|
|
|
|
state_dir = pathlib.Path(opts.state_dir)
|
|
state_file = state_dir / 'state.json'
|
|
gentoo_subdir = state_dir / 'upstream-gentoo'
|
|
if not state_file.exists():
|
|
logging.info("state_dir isn't fully set up; doing that now.")
|
|
|
|
# Could be in a partially set-up state.
|
|
if state_dir.exists():
|
|
logging.info('incomplete state_dir detected; removing.')
|
|
shutil.rmtree(str(state_dir))
|
|
|
|
state_dir.mkdir(parents=True)
|
|
most_recent_release = fetch_most_recent_release()
|
|
most_recent_gentoo_commit = setup_gentoo_git_repo(gentoo_subdir)
|
|
atomically_write_state(
|
|
state_file,
|
|
State(
|
|
last_seen_release=most_recent_release,
|
|
last_gentoo_sha=most_recent_gentoo_commit,
|
|
),
|
|
)
|
|
# Running through this _should_ be a nop, but do it anyway. Should make any
|
|
# bugs more obvious on the first run of the script.
|
|
|
|
prior_state = read_state(state_file)
|
|
logging.info('Last state was %r', prior_state)
|
|
|
|
most_recent_release = fetch_most_recent_release()
|
|
logging.info('Most recent Rust release is %s', most_recent_release)
|
|
|
|
logging.info('Fetching new commits from Gentoo')
|
|
update_git_repo(gentoo_subdir)
|
|
new_commits = get_new_gentoo_commits(gentoo_subdir,
|
|
prior_state.last_gentoo_sha)
|
|
logging.info('New commits: %r', new_commits)
|
|
|
|
maybe_email = maybe_compose_email(prior_state, most_recent_release,
|
|
new_commits)
|
|
|
|
if maybe_email is None:
|
|
logging.info('No updates to send')
|
|
else:
|
|
title, body = maybe_email
|
|
if opts.skip_email:
|
|
logging.info('Skipping sending email with title %r and contents\n%s',
|
|
title, tiny_render.render_html_pieces(body))
|
|
else:
|
|
logging.info('Sending email')
|
|
send_email(title, body)
|
|
|
|
if opts.skip_state_update:
|
|
logging.info('Skipping state update, as requested')
|
|
return
|
|
|
|
newest_sha = (
|
|
new_commits[-1].sha if new_commits else prior_state.last_gentoo_sha)
|
|
atomically_write_state(
|
|
state_file,
|
|
State(
|
|
last_seen_release=most_recent_release,
|
|
last_gentoo_sha=newest_sha,
|
|
),
|
|
)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main(sys.argv[1:]))
|