You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
526 lines
17 KiB
526 lines
17 KiB
#!/usr/bin/env vpython
|
|
|
|
# [VPYTHON:BEGIN]
|
|
# # Third party dependencies. These are only listed because pylint itself needs
|
|
# # them. Feel free to add/remove anything here.
|
|
#
|
|
# wheel: <
|
|
# name: "infra/python/wheels/configparser-py2_py3"
|
|
# version: "version:3.5.0"
|
|
# >
|
|
# wheel: <
|
|
# name: "infra/python/wheels/futures-py2_py3"
|
|
# version: "version:3.1.1"
|
|
# >
|
|
# wheel: <
|
|
# name: "infra/python/wheels/isort-py2_py3"
|
|
# version: "version:4.3.4"
|
|
# >
|
|
# wheel: <
|
|
# name: "infra/python/wheels/wrapt/${vpython_platform}"
|
|
# version: "version:1.10.11"
|
|
# >
|
|
# wheel: <
|
|
# name: "infra/python/wheels/backports_functools_lru_cache-py2_py3"
|
|
# version: "version:1.5"
|
|
# >
|
|
# wheel: <
|
|
# name: "infra/python/wheels/lazy-object-proxy/${vpython_platform}"
|
|
# version: "version:1.3.1"
|
|
# >
|
|
# wheel: <
|
|
# name: "infra/python/wheels/singledispatch-py2_py3"
|
|
# version: "version:3.4.0.3"
|
|
# >
|
|
# wheel: <
|
|
# name: "infra/python/wheels/enum34-py2"
|
|
# version: "version:1.1.6"
|
|
# >
|
|
# wheel: <
|
|
# name: "infra/python/wheels/mccabe-py2_py3"
|
|
# version: "version:0.6.1"
|
|
# >
|
|
# wheel: <
|
|
# name: "infra/python/wheels/six-py2_py3"
|
|
# version: "version:1.10.0"
|
|
# >
|
|
#
|
|
# # Pylint dependencies.
|
|
#
|
|
# wheel: <
|
|
# name: "infra/python/wheels/astroid-py2_py3"
|
|
# version: "version:1.6.6"
|
|
# >
|
|
#
|
|
# wheel: <
|
|
# name: "infra/python/wheels/pylint-py2_py3"
|
|
# version: "version:1.9.5-45a720817e4de1df2f173c7e4029e176"
|
|
# >
|
|
# [VPYTHON:END]
|
|
|
|
"""
|
|
Wrapper to patch pylint library functions to suit autotest.
|
|
|
|
This script is invoked as part of the presubmit checks for autotest python
|
|
files. It runs pylint on a list of files that it obtains either through
|
|
the command line or from an environment variable set in pre-upload.py.
|
|
|
|
Example:
|
|
run_pylint.py filename.py
|
|
"""
|
|
|
|
import fnmatch
|
|
import logging
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
import common
|
|
from autotest_lib.client.common_lib import autotemp, revision_control
|
|
|
|
# Do a basic check to see if pylint is even installed.
|
|
try:
|
|
import pylint
|
|
from pylint.__pkginfo__ import version as pylint_version
|
|
except ImportError:
|
|
print ("Unable to import pylint, it may need to be installed."
|
|
" Run 'sudo aptitude install pylint' if you haven't already.")
|
|
sys.exit(1)
|
|
|
|
pylint_version_parsed = tuple(map(int, pylint_version.split('.')))
|
|
|
|
# some files make pylint blow up, so make sure we ignore them
|
|
SKIPLIST = ['/site-packages/*', '/contrib/*', '/frontend/afe/management.py']
|
|
|
|
import astroid
|
|
import pylint.lint
|
|
from pylint.checkers import base, imports, variables
|
|
|
|
# need to put autotest root dir on sys.path so pylint will be happy
|
|
autotest_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
|
|
sys.path.insert(0, autotest_root)
|
|
|
|
# patch up pylint import checker to handle our importing magic
|
|
ROOT_MODULE = 'autotest_lib.'
|
|
|
|
# A list of modules for pylint to ignore, specifically, these modules
|
|
# are imported for their side-effects and are not meant to be used.
|
|
_IGNORE_MODULES=['common', 'frontend_test_utils',
|
|
'setup_django_environment',
|
|
'setup_django_lite_environment',
|
|
'setup_django_readonly_environment', 'setup_test_environment',]
|
|
|
|
|
|
class pylint_error(Exception):
|
|
"""
|
|
Error raised when pylint complains about a file.
|
|
"""
|
|
|
|
|
|
class run_pylint_error(pylint_error):
|
|
"""
|
|
Error raised when an assumption made in this file is violated.
|
|
"""
|
|
|
|
|
|
def patch_modname(modname):
|
|
"""
|
|
Patches modname so we can make sense of autotest_lib modules.
|
|
|
|
@param modname: name of a module, contains '.'
|
|
@return modified modname string.
|
|
"""
|
|
if modname.startswith(ROOT_MODULE) or modname.startswith(ROOT_MODULE[:-1]):
|
|
modname = modname[len(ROOT_MODULE):]
|
|
return modname
|
|
|
|
|
|
def patch_consumed_list(to_consume=None, consumed=None):
|
|
"""
|
|
Patches the consumed modules list to ignore modules with side effects.
|
|
|
|
Autotest relies on importing certain modules solely for their side
|
|
effects. Pylint doesn't understand this and flags them as unused, since
|
|
they're not referenced anywhere in the code. To overcome this we need
|
|
to transplant said modules into the dictionary of modules pylint has
|
|
already seen, before pylint checks it.
|
|
|
|
@param to_consume: a dictionary of names pylint needs to see referenced.
|
|
@param consumed: a dictionary of names that pylint has seen referenced.
|
|
"""
|
|
ignore_modules = []
|
|
if (to_consume is not None and consumed is not None):
|
|
ignore_modules = [module_name for module_name in _IGNORE_MODULES
|
|
if module_name in to_consume]
|
|
|
|
for module_name in ignore_modules:
|
|
consumed[module_name] = to_consume[module_name]
|
|
del to_consume[module_name]
|
|
|
|
|
|
class CustomImportsChecker(imports.ImportsChecker):
|
|
"""Modifies stock imports checker to suit autotest."""
|
|
def visit_importfrom(self, node):
|
|
"""Patches modnames so pylints understands autotest_lib."""
|
|
node.modname = patch_modname(node.modname)
|
|
return super(CustomImportsChecker, self).visit_importfrom(node)
|
|
|
|
|
|
class CustomVariablesChecker(variables.VariablesChecker):
|
|
"""Modifies stock variables checker to suit autotest."""
|
|
|
|
def visit_module(self, node):
|
|
"""
|
|
Unflag 'import common'.
|
|
|
|
_to_consume eg: [({to reference}, {referenced}, 'scope type')]
|
|
Enteries are appended to this list as we drill deeper in scope.
|
|
If we ever come across a module to ignore, we immediately move it
|
|
to the consumed list.
|
|
|
|
@param node: node of the ast we're currently checking.
|
|
"""
|
|
super(CustomVariablesChecker, self).visit_module(node)
|
|
scoped_names = self._to_consume.pop()
|
|
# The type of the object has changed in pylint 1.8.2
|
|
if pylint_version_parsed >= (1, 8, 2):
|
|
patch_consumed_list(scoped_names.to_consume,scoped_names.consumed)
|
|
else:
|
|
patch_consumed_list(scoped_names[0],scoped_names[1])
|
|
self._to_consume.append(scoped_names)
|
|
|
|
def visit_importfrom(self, node):
|
|
"""Patches modnames so pylints understands autotest_lib."""
|
|
node.modname = patch_modname(node.modname)
|
|
return super(CustomVariablesChecker, self).visit_importfrom(node)
|
|
|
|
def visit_expr(self, node):
|
|
"""
|
|
Flag exceptions instantiated but not used.
|
|
|
|
https://crbug.com/1005893
|
|
"""
|
|
if not isinstance(node.value, astroid.Call):
|
|
return
|
|
func = node.value.func
|
|
try:
|
|
cls = next(func.infer())
|
|
except astroid.InferenceError:
|
|
return
|
|
if not isinstance(cls, astroid.ClassDef):
|
|
return
|
|
if any(x for x in cls.ancestors() if x.name == 'BaseException'):
|
|
self.add_message('W0104', node=node, line=node.fromlineno)
|
|
|
|
|
|
class CustomDocStringChecker(base.DocStringChecker):
|
|
"""Modifies stock docstring checker to suit Autotest doxygen style."""
|
|
|
|
def visit_module(self, node):
|
|
"""
|
|
Don't visit imported modules when checking for docstrings.
|
|
|
|
@param node: the node we're visiting.
|
|
"""
|
|
pass
|
|
|
|
|
|
def visit_functiondef(self, node):
|
|
"""
|
|
Don't request docstrings for commonly overridden autotest functions.
|
|
|
|
@param node: node of the ast we're currently checking.
|
|
"""
|
|
|
|
# Even plain functions will have a parent, which is the
|
|
# module they're in, and a frame, which is the context
|
|
# of said module; They need not however, always have
|
|
# ancestors.
|
|
if (node.name in ('run_once', 'initialize', 'cleanup') and
|
|
hasattr(node.parent.frame(), 'ancestors') and
|
|
any(ancestor.name == 'base_test' for ancestor in
|
|
node.parent.frame().ancestors())):
|
|
return
|
|
|
|
if _is_test_case_method(node):
|
|
return
|
|
|
|
super(CustomDocStringChecker, self).visit_functiondef(node)
|
|
|
|
|
|
@staticmethod
|
|
def _should_skip_arg(arg):
|
|
"""
|
|
@return: True if the argument given by arg is allowlisted, and does
|
|
not require a "@param" docstring.
|
|
"""
|
|
return arg in ('self', 'cls', 'args', 'kwargs', 'dargs')
|
|
|
|
base.DocStringChecker = CustomDocStringChecker
|
|
imports.ImportsChecker = CustomImportsChecker
|
|
variables.VariablesChecker = CustomVariablesChecker
|
|
|
|
|
|
def batch_check_files(file_paths, base_opts):
|
|
"""
|
|
Run pylint on a list of files so we get consolidated errors.
|
|
|
|
@param file_paths: a list of file paths.
|
|
@param base_opts: a list of pylint config options.
|
|
|
|
@returns pylint return code
|
|
|
|
@raises: pylint_error if pylint finds problems with a file
|
|
in this commit.
|
|
"""
|
|
if not file_paths:
|
|
return 0
|
|
|
|
pylint_runner = pylint.lint.Run(list(base_opts) + list(file_paths),
|
|
exit=False)
|
|
return pylint_runner.linter.msg_status
|
|
|
|
|
|
def should_check_file(file_path):
|
|
"""
|
|
Don't check skiplisted or non .py files.
|
|
|
|
@param file_path: abs path of file to check.
|
|
@return: True if this file is a non-skiplisted python file.
|
|
"""
|
|
file_path = os.path.abspath(file_path)
|
|
if file_path.endswith('.py'):
|
|
return all(not fnmatch.fnmatch(file_path, '*' + pattern)
|
|
for pattern in SKIPLIST)
|
|
return False
|
|
|
|
|
|
def check_file(file_path, base_opts):
|
|
"""
|
|
Invokes pylint on files after confirming that they're not black listed.
|
|
|
|
@param base_opts: pylint base options.
|
|
@param file_path: path to the file we need to run pylint on.
|
|
|
|
@returns pylint return code
|
|
"""
|
|
if not isinstance(file_path, basestring):
|
|
raise TypeError('expected a string as filepath, got %s'%
|
|
type(file_path))
|
|
|
|
if should_check_file(file_path):
|
|
pylint_runner = pylint.lint.Run(base_opts + [file_path], exit=False)
|
|
|
|
return pylint_runner.linter.msg_status
|
|
|
|
return 0
|
|
|
|
|
|
def visit(arg, dirname, filenames):
|
|
"""
|
|
Visit function invoked in check_dir.
|
|
|
|
@param arg: arg from os.walk.path
|
|
@param dirname: dir from os.walk.path
|
|
@param filenames: files in dir from os.walk.path
|
|
"""
|
|
for filename in filenames:
|
|
arg.append(os.path.join(dirname, filename))
|
|
|
|
|
|
def check_dir(dir_path, base_opts):
|
|
"""
|
|
Calls visit on files in dir_path.
|
|
|
|
@param base_opts: pylint base options.
|
|
@param dir_path: path to directory.
|
|
|
|
@returns pylint return code
|
|
"""
|
|
files = []
|
|
|
|
os.path.walk(dir_path, visit, files)
|
|
|
|
return batch_check_files(files, base_opts)
|
|
|
|
|
|
def extend_baseopts(base_opts, new_opt):
|
|
"""
|
|
Replaces an argument in base_opts with a cmd line argument.
|
|
|
|
@param base_opts: original pylint_base_opts.
|
|
@param new_opt: new cmd line option.
|
|
"""
|
|
for args in base_opts:
|
|
if new_opt in args:
|
|
base_opts.remove(args)
|
|
base_opts.append(new_opt)
|
|
|
|
|
|
def get_cmdline_options(args_list, pylint_base_opts, rcfile):
|
|
"""
|
|
Parses args_list and extends pylint_base_opts.
|
|
|
|
Command line arguments might include options mixed with files.
|
|
Go through this list and filter out the options, if the options are
|
|
specified in the pylintrc file we cannot replace them and the file
|
|
needs to be edited. If the options are already a part of
|
|
pylint_base_opts we replace them, and if not we append to
|
|
pylint_base_opts.
|
|
|
|
@param args_list: list of files/pylint args passed in through argv.
|
|
@param pylint_base_opts: default pylint options.
|
|
@param rcfile: text from pylint_rc.
|
|
"""
|
|
for args in args_list:
|
|
if args.startswith('--'):
|
|
opt_name = args[2:].split('=')[0]
|
|
extend_baseopts(pylint_base_opts, args)
|
|
args_list.remove(args)
|
|
|
|
|
|
def git_show_to_temp_file(commit, original_file, new_temp_file):
|
|
"""
|
|
'Git shows' the file in original_file to a tmp file with
|
|
the name new_temp_file. We need to preserve the filename
|
|
as it gets reflected in pylints error report.
|
|
|
|
@param commit: commit hash of the commit we're running repo upload on.
|
|
@param original_file: the path to the original file we'd like to run
|
|
'git show' on.
|
|
@param new_temp_file: new_temp_file is the path to a temp file we write the
|
|
output of 'git show' into.
|
|
"""
|
|
git_repo = revision_control.GitRepo(common.autotest_dir, None, None,
|
|
common.autotest_dir)
|
|
|
|
with open(new_temp_file, 'w') as f:
|
|
output = git_repo.gitcmd('show --no-ext-diff %s:%s'
|
|
% (commit, original_file),
|
|
ignore_status=False).stdout
|
|
f.write(output)
|
|
|
|
|
|
def check_committed_files(work_tree_files, commit, pylint_base_opts):
|
|
"""
|
|
Get a list of files corresponding to the commit hash.
|
|
|
|
The contents of a file in the git work tree can differ from the contents
|
|
of a file in the commit we mean to upload. To work around this we run
|
|
pylint on a temp file into which we've 'git show'n the committed version
|
|
of each file.
|
|
|
|
@param work_tree_files: list of files in this commit specified by their
|
|
absolute path.
|
|
@param commit: hash of the commit this upload applies to.
|
|
@param pylint_base_opts: a list of pylint config options.
|
|
|
|
@returns pylint return code
|
|
"""
|
|
files_to_check = filter(should_check_file, work_tree_files)
|
|
|
|
# Map the absolute path of each file so it's relative to the autotest repo.
|
|
# All files that are a part of this commit should have an abs path within
|
|
# the autotest repo, so this regex should never fail.
|
|
work_tree_files = [re.search(r'%s/(.*)' % common.autotest_dir, f).group(1)
|
|
for f in files_to_check]
|
|
|
|
tempdir = None
|
|
try:
|
|
tempdir = autotemp.tempdir()
|
|
temp_files = [os.path.join(tempdir.name, file_path.split('/')[-1:][0])
|
|
for file_path in work_tree_files]
|
|
|
|
for file_tuple in zip(work_tree_files, temp_files):
|
|
git_show_to_temp_file(commit, *file_tuple)
|
|
# Only check if we successfully git showed all files in the commit.
|
|
return batch_check_files(temp_files, pylint_base_opts)
|
|
finally:
|
|
if tempdir:
|
|
tempdir.clean()
|
|
|
|
|
|
def _is_test_case_method(node):
|
|
"""Determine if the given function node is a method of a TestCase.
|
|
|
|
We simply check for 'TestCase' being one of the parent classes in the mro of
|
|
the containing class.
|
|
|
|
@params node: A function node.
|
|
"""
|
|
if not hasattr(node.parent.frame(), 'ancestors'):
|
|
return False
|
|
|
|
parent_class_names = {x.name for x in node.parent.frame().ancestors()}
|
|
return 'TestCase' in parent_class_names
|
|
|
|
|
|
def main():
|
|
"""Main function checks each file in a commit for pylint violations."""
|
|
|
|
# For now all error/warning/refactor/convention exceptions except those in
|
|
# the enable string are disabled.
|
|
# W0611: All imported modules (except common) need to be used.
|
|
# W1201: Logging methods should take the form
|
|
# logging.<loggingmethod>(format_string, format_args...); and not
|
|
# logging.<loggingmethod>(format_string % (format_args...))
|
|
# C0111: Docstring needed. Also checks @param for each arg.
|
|
# C0112: Non-empty Docstring needed.
|
|
# Ideally we would like to enable as much as we can, but if we did so at
|
|
# this stage anyone who makes a tiny change to a file will be tasked with
|
|
# cleaning all the lint in it. See chromium-os:37364.
|
|
|
|
# Note:
|
|
# 1. There are three major sources of E1101/E1103/E1120 false positives:
|
|
# * common_lib.enum.Enum objects
|
|
# * DB model objects (scheduler models are the worst, but Django models
|
|
# also generate some errors)
|
|
# 2. Docstrings are optional on private methods, and any methods that begin
|
|
# with either 'set_' or 'get_'.
|
|
pylint_rc = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|
'pylintrc')
|
|
|
|
no_docstring_rgx = r'((_.*)|(set_.*)|(get_.*))'
|
|
if pylint_version_parsed >= (0, 21):
|
|
pylint_base_opts = ['--rcfile=%s' % pylint_rc,
|
|
'--reports=no',
|
|
'--disable=W,R,E,C,F',
|
|
'--enable=W0104,W0611,W1201,C0111,C0112,E0602,'
|
|
'W0601,E0633',
|
|
'--no-docstring-rgx=%s' % no_docstring_rgx,]
|
|
else:
|
|
all_failures = 'error,warning,refactor,convention'
|
|
pylint_base_opts = ['--disable-msg-cat=%s' % all_failures,
|
|
'--reports=no',
|
|
'--include-ids=y',
|
|
'--ignore-docstrings=n',
|
|
'--no-docstring-rgx=%s' % no_docstring_rgx,]
|
|
|
|
# run_pylint can be invoked directly with command line arguments,
|
|
# or through a presubmit hook which uses the arguments in pylintrc. In the
|
|
# latter case no command line arguments are passed. If it is invoked
|
|
# directly without any arguments, it should check all files in the cwd.
|
|
args_list = sys.argv[1:]
|
|
if args_list:
|
|
get_cmdline_options(args_list,
|
|
pylint_base_opts,
|
|
open(pylint_rc).read())
|
|
return batch_check_files(args_list, pylint_base_opts)
|
|
elif os.environ.get('PRESUBMIT_FILES') is not None:
|
|
return check_committed_files(
|
|
os.environ.get('PRESUBMIT_FILES').split('\n'),
|
|
os.environ.get('PRESUBMIT_COMMIT'),
|
|
pylint_base_opts)
|
|
else:
|
|
return check_dir('.', pylint_base_opts)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
ret = main()
|
|
|
|
sys.exit(ret)
|
|
except pylint_error as e:
|
|
logging.error(e)
|
|
sys.exit(1)
|