You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
459 lines
18 KiB
459 lines
18 KiB
#
|
|
# Copyright (C) 2016 The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import logging
|
|
import os
|
|
|
|
from google.protobuf import text_format
|
|
from vts.proto import ComponentSpecificationMessage_pb2 as CompSpecMsg
|
|
from vts.proto import VtsProfilingMessage_pb2 as VtsProfilingMsg
|
|
from vts.proto import VtsReportMessage_pb2 as ReportMsg
|
|
from vts.runners.host import asserts
|
|
from vts.runners.host import const
|
|
from vts.runners.host import keys
|
|
from vts.utils.python.common import cmd_utils
|
|
from vts.utils.python.os import path_utils
|
|
from vts.utils.python.web import feature_utils
|
|
|
|
LOCAL_PROFILING_TRACE_PATH = "/tmp/vts-test-trace"
|
|
TARGET_PROFILING_TRACE_PATH = "/data/local/tmp/"
|
|
HAL_INSTRUMENTATION_LIB_PATH_32 = "/data/local/tmp/32/"
|
|
HAL_INSTRUMENTATION_LIB_PATH_64 = "/data/local/tmp/64/"
|
|
DEFAULT_HAL_ROOT = "android.hardware."
|
|
|
|
_PROFILING_DATA = "profiling_data"
|
|
_HOST_PROFILING_DATA = "host_profiling_data"
|
|
|
|
|
|
class VTSProfilingData(object):
|
|
"""Class to store the VTS profiling data.
|
|
|
|
Attributes:
|
|
values: A dict that stores the profiling data. e.g. latencies of each api.
|
|
options: A set of strings where each string specifies an associated
|
|
option (which is the form of 'key=value').
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.values = {}
|
|
self.options = set()
|
|
|
|
|
|
EVENT_TYPE_DICT = {
|
|
0: "SERVER_API_ENTRY",
|
|
1: "SERVER_API_EXIT",
|
|
2: "CLIENT_API_ENTRY",
|
|
3: "CLIENT_API_EXIT",
|
|
4: "SYNC_CALLBACK_ENTRY",
|
|
5: "SYNC_CALLBACK_EXIT",
|
|
6: "ASYNC_CALLBACK_ENTRY",
|
|
7: "ASYNC_CALLBACK_EXIT",
|
|
8: "PASSTHROUGH_ENTRY",
|
|
9: "PASSTHROUGH_EXIT",
|
|
}
|
|
|
|
|
|
class VTSApiCoverageData(object):
|
|
"""Class to store the API coverage data.
|
|
|
|
Attributes:
|
|
package_name: sting, HAL package name (e.g. android.hardware.foo).
|
|
version_major: string, HAL major version (e.g. 1).
|
|
version_minor: string, HAL minor version (e.g. 0).
|
|
interface_name: string, HAL interface name (e.g. IFoo).
|
|
total_apis: A set of strings, each string represents an API name defined
|
|
in the given HAL.
|
|
covered_apis: A set of strings, each string represents an API name
|
|
covered by the test.
|
|
"""
|
|
|
|
def __init__(self, package_name, version, interface_name):
|
|
self.package_name = package_name
|
|
self.version_major, self.version_minor = version.split(".")
|
|
self.interface_name = interface_name
|
|
self.total_apis = set()
|
|
self.covered_apis = set()
|
|
|
|
|
|
class ProfilingFeature(feature_utils.Feature):
|
|
"""Feature object for profiling functionality.
|
|
|
|
Attributes:
|
|
enabled: boolean, True if profiling is enabled, False otherwise
|
|
web: (optional) WebFeature, object storing web feature util for test run.
|
|
data_file_path: Path to the data directory within vts package.
|
|
api_coverage_data: A dictionary from full HAL interface name
|
|
(e.g. android.hardware.foo@1.0::IFoo) to VtsApiCoverageData.
|
|
"""
|
|
|
|
_TOGGLE_PARAM = keys.ConfigKeys.IKEY_ENABLE_PROFILING
|
|
_REQUIRED_PARAMS = [keys.ConfigKeys.IKEY_DATA_FILE_PATH]
|
|
_OPTIONAL_PARAMS = [
|
|
keys.ConfigKeys.IKEY_PROFILING_TRACING_PATH,
|
|
keys.ConfigKeys.IKEY_TRACE_FILE_TOOL_NAME,
|
|
keys.ConfigKeys.IKEY_SAVE_TRACE_FILE_REMOTE,
|
|
keys.ConfigKeys.IKEY_ABI_BITNESS,
|
|
keys.ConfigKeys.IKEY_PROFILING_ARG_VALUE,
|
|
]
|
|
|
|
def __init__(self, user_params, web=None):
|
|
"""Initializes the profiling feature.
|
|
|
|
Args:
|
|
user_params: A dictionary from parameter name (String) to parameter value.
|
|
web: (optional) WebFeature, object storing web feature util for test run
|
|
"""
|
|
self.ParseParameters(self._TOGGLE_PARAM, self._REQUIRED_PARAMS,
|
|
self._OPTIONAL_PARAMS, user_params)
|
|
self.web = web
|
|
if self.enabled:
|
|
logging.info("Profiling is enabled.")
|
|
else:
|
|
logging.debug("Profiling is disabled.")
|
|
|
|
self.data_file_path = getattr(self,
|
|
keys.ConfigKeys.IKEY_DATA_FILE_PATH, None)
|
|
self.api_coverage_data = {}
|
|
|
|
def _IsEventFromBinderizedHal(self, event_type):
|
|
"""Returns True if the event type is from a binderized HAL."""
|
|
if event_type in [8, 9]:
|
|
return False
|
|
return True
|
|
|
|
def GetTraceFiles(self,
|
|
dut,
|
|
host_profiling_trace_path=None,
|
|
trace_file_tool=None):
|
|
"""Pulls the trace file and save it under the profiling trace path.
|
|
|
|
Args:
|
|
dut: the testing device.
|
|
host_profiling_trace_path: directory that stores trace files on host.
|
|
trace_file_tool: tools that used to store the trace file.
|
|
|
|
Returns:
|
|
Name list of trace files that stored on host.
|
|
"""
|
|
if not os.path.exists(LOCAL_PROFILING_TRACE_PATH):
|
|
os.makedirs(LOCAL_PROFILING_TRACE_PATH)
|
|
|
|
if not host_profiling_trace_path:
|
|
host_profiling_trace_path = LOCAL_PROFILING_TRACE_PATH
|
|
|
|
target_trace_file = path_utils.JoinTargetPath(
|
|
TARGET_PROFILING_TRACE_PATH, "*.vts.trace")
|
|
results = dut.shell.Execute("ls " + target_trace_file)
|
|
asserts.assertTrue(results, "failed to find trace file")
|
|
stdout_lines = results[const.STDOUT][0].split("\n")
|
|
logging.debug("stdout: %s", stdout_lines)
|
|
trace_files = []
|
|
for line in stdout_lines:
|
|
if line:
|
|
temp_file_name = os.path.join(LOCAL_PROFILING_TRACE_PATH,
|
|
os.path.basename(line.strip()))
|
|
dut.adb.pull("%s %s" % (line, temp_file_name))
|
|
trace_file_name = os.path.join(host_profiling_trace_path,
|
|
os.path.basename(line.strip()))
|
|
logging.info("Saving profiling traces: %s" % trace_file_name)
|
|
if temp_file_name != trace_file_name:
|
|
file_cmd = ""
|
|
if trace_file_tool:
|
|
file_cmd += trace_file_tool
|
|
file_cmd += " cp " + temp_file_name + " " + trace_file_name
|
|
results = cmd_utils.ExecuteShellCommand(file_cmd)
|
|
if results[const.EXIT_CODE][0] != 0:
|
|
logging.error(results[const.STDERR][0])
|
|
logging.error("Fail to execute command: %s" % file_cmd)
|
|
trace_files.append(temp_file_name)
|
|
return trace_files
|
|
|
|
def EnableVTSProfiling(self, shell, hal_instrumentation_lib_path=None):
|
|
""" Enable profiling by setting the system property.
|
|
|
|
Args:
|
|
shell: shell to control the testing device.
|
|
hal_instrumentation_lib_path: string, the path of directory that stores
|
|
profiling libraries.
|
|
"""
|
|
hal_instrumentation_lib_path_32 = HAL_INSTRUMENTATION_LIB_PATH_32
|
|
hal_instrumentation_lib_path_64 = HAL_INSTRUMENTATION_LIB_PATH_64
|
|
if hal_instrumentation_lib_path is not None:
|
|
bitness = getattr(self, keys.ConfigKeys.IKEY_ABI_BITNESS, None)
|
|
if bitness == '64':
|
|
hal_instrumentation_lib_path_64 = hal_instrumentation_lib_path
|
|
elif bitness == '32':
|
|
hal_instrumentation_lib_path_32 = hal_instrumentation_lib_path
|
|
else:
|
|
logging.error('Unknown abi bitness "%s". Using 64bit hal '
|
|
'instrumentation lib path.', bitness)
|
|
|
|
# cleanup any existing traces.
|
|
shell.Execute(
|
|
"rm " + os.path.join(TARGET_PROFILING_TRACE_PATH, "*.vts.trace"))
|
|
logging.debug("enabling VTS profiling.")
|
|
|
|
# give permission to write the trace file.
|
|
shell.Execute("chmod 777 " + TARGET_PROFILING_TRACE_PATH)
|
|
|
|
shell.Execute("setprop hal.instrumentation.lib.path.32 " +
|
|
hal_instrumentation_lib_path_32)
|
|
shell.Execute("setprop hal.instrumentation.lib.path.64 " +
|
|
hal_instrumentation_lib_path_64)
|
|
|
|
if getattr(self, keys.ConfigKeys.IKEY_PROFILING_ARG_VALUE, False):
|
|
shell.Execute("setprop hal.instrumentation.profile.args true")
|
|
else:
|
|
shell.Execute("setprop hal.instrumentation.profile.args false")
|
|
shell.Execute("setprop hal.instrumentation.enable true")
|
|
|
|
def DisableVTSProfiling(self, shell):
|
|
""" Disable profiling by resetting the system property.
|
|
|
|
Args:
|
|
shell: shell to control the testing device.
|
|
"""
|
|
shell.Execute("setprop hal.instrumentation.lib.path \"\"")
|
|
shell.Execute("setprop hal.instrumentation.profile.args \"\"")
|
|
shell.Execute("setprop hal.instrumentation.enable false")
|
|
|
|
def _ParseTraceData(self, trace_file, measure_api_coverage):
|
|
"""Parses the data stored in trace_file, calculates the avg/max/min
|
|
latency for each API.
|
|
|
|
Args:
|
|
trace_file: file that stores the trace data.
|
|
measure_api_coverage: whether to measure the api coverage data.
|
|
|
|
Returns:
|
|
VTSProfilingData which contain the list of API names and the avg/max/min
|
|
latency for each API.
|
|
"""
|
|
profiling_data = VTSProfilingData()
|
|
api_timestamps = {}
|
|
api_latencies = {}
|
|
|
|
trace_processor_binary = os.path.join(self.data_file_path, "host",
|
|
"bin", "trace_processor")
|
|
trace_processor_lib = os.path.join(self.data_file_path, "host",
|
|
"lib64")
|
|
trace_processor_cmd = [
|
|
"chmod a+x %s" % trace_processor_binary,
|
|
"LD_LIBRARY_PATH=%s %s -m profiling_trace %s" %
|
|
(trace_processor_lib, trace_processor_binary, trace_file)
|
|
]
|
|
|
|
results = cmd_utils.ExecuteShellCommand(trace_processor_cmd)
|
|
if any(results[cmd_utils.EXIT_CODE]):
|
|
logging.error("Fail to execute command: %s" % trace_processor_cmd)
|
|
logging.error("stdout: %s" % results[const.STDOUT])
|
|
logging.error("stderr: %s" % results[const.STDERR])
|
|
return profiling_data
|
|
|
|
stdout_lines = results[const.STDOUT][1].split("\n")
|
|
first_line = True
|
|
for line in stdout_lines:
|
|
if not line:
|
|
continue
|
|
if first_line:
|
|
_, mode = line.split(":")
|
|
profiling_data.options.add("hidl_hal_mode=%s" % mode)
|
|
first_line = False
|
|
else:
|
|
full_api, latency = line.rsplit(":", 1)
|
|
full_interface, api_name = full_api.rsplit("::", 1)
|
|
if profiling_data.values.get(api_name):
|
|
profiling_data.values[api_name].append(long(latency))
|
|
else:
|
|
profiling_data.values[api_name] = [long(latency)]
|
|
|
|
if measure_api_coverage:
|
|
package, interface_name = full_interface.split("::")
|
|
package_name, version = package.split("@")
|
|
|
|
if full_interface in self.api_coverage_data:
|
|
self.api_coverage_data[
|
|
full_interface].covered_apis.add(api_name)
|
|
else:
|
|
total_apis = self._GetTotalApis(
|
|
package_name, version, interface_name)
|
|
if total_apis:
|
|
vts_api_coverage = VTSApiCoverageData(
|
|
package_name, version, interface_name)
|
|
vts_api_coverage.total_apis = total_apis
|
|
if api_name in total_apis:
|
|
vts_api_coverage.covered_apis.add(api_name)
|
|
else:
|
|
logging.warning("API %s is not supported by %s",
|
|
api_name, full_interface)
|
|
self.api_coverage_data[
|
|
full_interface] = vts_api_coverage
|
|
|
|
return profiling_data
|
|
|
|
def _GetTotalApis(self, package_name, version, interface_name):
|
|
"""Parse the specified vts spec and get all APIs defined in the spec.
|
|
|
|
Args:
|
|
package_name: string, HAL package name.
|
|
version: string, HAL version.
|
|
interface_name: string, HAL interface name.
|
|
|
|
Returns:
|
|
A set of strings, each string represents an API defined in the spec.
|
|
"""
|
|
total_apis = set()
|
|
spec_proto = CompSpecMsg.ComponentSpecificationMessage()
|
|
# TODO: support general package that does not start with android.hardware.
|
|
if not package_name.startswith(DEFAULT_HAL_ROOT):
|
|
logging.warning("Unsupported hal package: %s", package_name)
|
|
return total_apis
|
|
|
|
hal_package_path = package_name[len(DEFAULT_HAL_ROOT):].replace(
|
|
".", "/")
|
|
vts_spec_path = os.path.join(
|
|
self.data_file_path, "spec/hardware/interfaces", hal_package_path,
|
|
version, "vts", interface_name[1:] + ".vts")
|
|
logging.debug("vts_spec_path: %s", vts_spec_path)
|
|
with open(vts_spec_path, 'r') as spec_file:
|
|
spec_string = spec_file.read()
|
|
text_format.Merge(spec_string, spec_proto)
|
|
for api in spec_proto.interface.api:
|
|
if not api.is_inherited:
|
|
total_apis.add(api.name)
|
|
return total_apis
|
|
|
|
def StartHostProfiling(self, name):
|
|
"""Starts a profiling operation.
|
|
|
|
Args:
|
|
name: string, the name of a profiling point
|
|
|
|
Returns:
|
|
True if successful, False otherwise
|
|
"""
|
|
if not self.enabled:
|
|
return False
|
|
|
|
if not hasattr(self, _HOST_PROFILING_DATA):
|
|
setattr(self, _HOST_PROFILING_DATA, {})
|
|
|
|
host_profiling_data = getattr(self, _HOST_PROFILING_DATA)
|
|
|
|
if name in host_profiling_data:
|
|
logging.error("profiling point %s is already active.", name)
|
|
return False
|
|
host_profiling_data[name] = feature_utils.GetTimestamp()
|
|
return True
|
|
|
|
def StopHostProfiling(self, name):
|
|
"""Stops a profiling operation.
|
|
|
|
Args:
|
|
name: string, the name of a profiling point
|
|
"""
|
|
if not self.enabled:
|
|
return
|
|
|
|
if not hasattr(self, _HOST_PROFILING_DATA):
|
|
setattr(self, _HOST_PROFILING_DATA, {})
|
|
|
|
host_profiling_data = getattr(self, _HOST_PROFILING_DATA)
|
|
|
|
if name not in host_profiling_data:
|
|
logging.error("profiling point %s is not active.", name)
|
|
return False
|
|
|
|
start_timestamp = host_profiling_data[name]
|
|
end_timestamp = feature_utils.GetTimestamp()
|
|
if self.web and self.web.enabled:
|
|
self.web.AddProfilingDataTimestamp(name, start_timestamp,
|
|
end_timestamp)
|
|
return True
|
|
|
|
def ProcessTraceDataForTestCase(self, dut, measure_api_coverage=True):
|
|
"""Pulls the generated trace file to the host, parses the trace file to
|
|
get the profiling data (e.g. latency of each API call) and stores these
|
|
data in _profiling_data.
|
|
|
|
Requires the feature to be enabled; no-op otherwise.
|
|
|
|
Args:
|
|
dut: the registered device.
|
|
"""
|
|
if not self.enabled:
|
|
return
|
|
|
|
if not hasattr(self, _PROFILING_DATA):
|
|
setattr(self, _PROFILING_DATA, [])
|
|
|
|
profiling_data = getattr(self, _PROFILING_DATA)
|
|
|
|
trace_files = []
|
|
save_trace_remote = getattr(
|
|
self, keys.ConfigKeys.IKEY_SAVE_TRACE_FILE_REMOTE, False)
|
|
if save_trace_remote:
|
|
trace_files = self.GetTraceFiles(
|
|
dut,
|
|
getattr(self, keys.ConfigKeys.IKEY_PROFILING_TRACING_PATH,
|
|
None),
|
|
getattr(self, keys.ConfigKeys.IKEY_TRACE_FILE_TOOL_NAME, None))
|
|
else:
|
|
trace_files = self.GetTraceFiles(dut)
|
|
|
|
for file in trace_files:
|
|
logging.info("parsing trace file: %s.", file)
|
|
data = self._ParseTraceData(file, measure_api_coverage)
|
|
if data:
|
|
profiling_data.append(data)
|
|
|
|
def ProcessAndUploadTraceData(self, upload_api_coverage=True):
|
|
"""Process and upload profiling trace data.
|
|
|
|
Requires the feature to be enabled; no-op otherwise.
|
|
|
|
Merges the profiling data generated by each test case, calculates the
|
|
aggregated max/min/avg latency for each API and uploads these latency
|
|
metrics to webdb.
|
|
|
|
Args:
|
|
upload_api_coverage: whether to upload the API coverage data.
|
|
"""
|
|
if not self.enabled:
|
|
return
|
|
|
|
merged_profiling_data = VTSProfilingData()
|
|
for data in getattr(self, _PROFILING_DATA, []):
|
|
for item in data.options:
|
|
merged_profiling_data.options.add(item)
|
|
for api, latences in data.values.items():
|
|
if merged_profiling_data.values.get(api):
|
|
merged_profiling_data.values[api].extend(latences)
|
|
else:
|
|
merged_profiling_data.values[api] = latences
|
|
for api, latencies in merged_profiling_data.values.items():
|
|
if not self.web or not self.web.enabled:
|
|
continue
|
|
|
|
self.web.AddProfilingDataUnlabeledVector(
|
|
api,
|
|
latencies,
|
|
merged_profiling_data.options,
|
|
x_axis_label="API processing latency (nano secs)",
|
|
y_axis_label="Frequency")
|
|
|
|
if upload_api_coverage:
|
|
self.web.AddApiCoverageReport(self.api_coverage_data.values())
|