You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
64 lines
1.8 KiB
64 lines
1.8 KiB
#!/usr/bin/python2
|
|
#
|
|
# Copyright (c) 2015 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
|
|
"""Script to check the number of long-running processes.
|
|
|
|
This script gets the number of processes for "gsutil" and "autoserv"
|
|
that are running more than 24 hours, and throws the number to stats
|
|
dashboard.
|
|
|
|
This script depends on the "etimes" user-defined format of "ps".
|
|
Goobuntu 14.04 has the version of ps that supports etimes, but not
|
|
Goobuntu 12.04.
|
|
"""
|
|
|
|
|
|
import subprocess
|
|
|
|
from autotest_lib.server import site_utils
|
|
|
|
try:
|
|
from chromite.lib import metrics
|
|
except ImportError:
|
|
metrics = site_utils.metrics_mock
|
|
|
|
|
|
PROGRAM_TO_CHECK_SET = set(['gsutil', 'autoserv'])
|
|
|
|
def check_proc(prog, max_elapsed_sec):
|
|
"""Check the number of long-running processes for a given program.
|
|
|
|
Finds out the number of processes for a given program that have run
|
|
more than a given elapsed time.
|
|
Sends out the number to stats dashboard.
|
|
|
|
@param prog: Program name.
|
|
@param max_elapsed_sec: Max elapsed time in seconds. Processes that
|
|
have run more than this value will be caught.
|
|
"""
|
|
cmd = ('ps -eo etimes,args | grep "%s" | awk \'{if($1 > %d) print $0}\' | '
|
|
'wc -l' % (prog, max_elapsed_sec))
|
|
count = int(subprocess.check_output(cmd, shell = True))
|
|
|
|
if prog not in PROGRAM_TO_CHECK_SET:
|
|
prog = 'unknown'
|
|
|
|
metrics.Gauge('chromeos/autotest/hung_processes').set(
|
|
count, fields={'program': prog}
|
|
)
|
|
|
|
|
|
def main():
|
|
"""Main script. """
|
|
with site_utils.SetupTsMonGlobalState('check_hung_proc', short_lived=True):
|
|
for p in PROGRAM_TO_CHECK_SET:
|
|
check_proc(p, 86400)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|