You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
409 lines
21 KiB
409 lines
21 KiB
# Copyright 2017 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
import os
|
|
import unittest
|
|
|
|
import tradefed_utils
|
|
|
|
|
|
def _load_data(filename):
|
|
"""Loads the test data of the given file name."""
|
|
with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
|
|
'tradefed_utils_unittest_data', filename), 'r') as f:
|
|
return f.read()
|
|
|
|
|
|
class TradefedTestTest(unittest.TestCase):
|
|
"""Unittest for tradefed_utils."""
|
|
|
|
def test_parse_tradefed_result(self):
|
|
"""Test for parse_tradefed_result."""
|
|
|
|
waivers = set([
|
|
'android.app.cts.SystemFeaturesTest#testUsbAccessory',
|
|
'android.widget.cts.GridViewTest#testSetNumColumns',
|
|
])
|
|
|
|
# b/35605415 and b/36520623
|
|
# http://pantheon/storage/browser/chromeos-autotest-results/108103986-chromeos-test/
|
|
# CTS: Tradefed may split a module to multiple chunks.
|
|
# Besides, the module name may not end with "TestCases".
|
|
waived, _ = tradefed_utils.parse_tradefed_result(
|
|
_load_data('CtsHostsideNetworkTests.txt'),
|
|
waivers=waivers)
|
|
self.assertEquals(0, len(waived))
|
|
|
|
# b/35530394
|
|
# http://pantheon/storage/browser/chromeos-autotest-results/108291418-chromeos-test/
|
|
# Crashed, but the automatic retry by tradefed executed the rest.
|
|
waived, _ = tradefed_utils.parse_tradefed_result(
|
|
_load_data('CtsMediaTestCases.txt'),
|
|
waivers=waivers)
|
|
self.assertEquals(0, len(waived))
|
|
|
|
# b/35530394
|
|
# http://pantheon/storage/browser/chromeos-autotest-results/106540705-chromeos-test/
|
|
# Crashed in the middle, and the device didn't came back.
|
|
waived, _ = tradefed_utils.parse_tradefed_result(
|
|
_load_data('CtsSecurityTestCases.txt'),
|
|
waivers=waivers)
|
|
self.assertEquals(0, len(waived))
|
|
|
|
# b/36629187
|
|
# http://pantheon/storage/browser/chromeos-autotest-results/108855595-chromeos-test/
|
|
# Crashed in the middle. Tradefed decided not to continue.
|
|
waived, _ = tradefed_utils.parse_tradefed_result(
|
|
_load_data('CtsViewTestCases.txt'),
|
|
waivers=waivers)
|
|
self.assertEquals(0, len(waived))
|
|
|
|
# b/36375690
|
|
# http://pantheon/storage/browser/chromeos-autotest-results/109040174-chromeos-test/
|
|
# Mixture of real failures and waivers.
|
|
waived, _ = tradefed_utils.parse_tradefed_result(
|
|
_load_data('CtsAppTestCases.txt'),
|
|
waivers=waivers)
|
|
self.assertEquals(1, len(waived))
|
|
# ... and the retry of the above failing iteration.
|
|
waived, _ = tradefed_utils.parse_tradefed_result(
|
|
_load_data('CtsAppTestCases-retry.txt'),
|
|
waivers=waivers)
|
|
self.assertEquals(1, len(waived))
|
|
|
|
# http://pantheon/storage/browser/chromeos-autotest-results/116875512-chromeos-test/
|
|
# When a test case crashed during teardown, tradefed prints the "fail"
|
|
# message twice. Tolerate it and still return an (inconsistent) count.
|
|
waived, _ = tradefed_utils.parse_tradefed_result(
|
|
_load_data('CtsWidgetTestCases.txt'),
|
|
waivers=waivers)
|
|
self.assertEquals(1, len(waived))
|
|
|
|
# http://pantheon/storage/browser/chromeos-autotest-results/117914707-chromeos-test/
|
|
# When a test case unrecoverably crashed during teardown, tradefed
|
|
# prints the "fail" and failure summary message twice. Tolerate it.
|
|
waived, _ = tradefed_utils.parse_tradefed_result(
|
|
_load_data('CtsPrintTestCases.txt'),
|
|
waivers=waivers)
|
|
self.assertEquals(0, len(waived))
|
|
|
|
gts_waivers = set([
|
|
('com.google.android.placement.gts.CoreGmsAppsTest#' +
|
|
'testCoreGmsAppsPreloaded'),
|
|
('com.google.android.placement.gts.CoreGmsAppsTest#' +
|
|
'testGoogleDuoPreloaded'),
|
|
'com.google.android.placement.gts.UiPlacementTest#testPlayStore'
|
|
])
|
|
|
|
# crbug.com/748116
|
|
# http://pantheon/storage/browser/chromeos-autotest-results/130080763-chromeos-test/
|
|
# 3 ABIS: x86, x86_64, and armeabi-v7a
|
|
waived, _ = tradefed_utils.parse_tradefed_result(
|
|
_load_data('GtsPlacementTestCases.txt'),
|
|
waivers=gts_waivers)
|
|
self.assertEquals(9, len(waived))
|
|
|
|
# b/64095702
|
|
# http://pantheon/storage/browser/chromeos-autotest-results/130211812-chromeos-test/
|
|
# The result of the last chunk not reported by tradefed.
|
|
# The actual dEQP log is too big, hence the test data here is trimmed.
|
|
waived, _ = tradefed_utils.parse_tradefed_result(
|
|
_load_data('CtsDeqpTestCases-trimmed.txt'),
|
|
waivers=waivers)
|
|
self.assertEquals(0, len(waived))
|
|
|
|
# b/80160772
|
|
# http://pantheon/storage/browser/chromeos-autotest-results/201962931-kkanchi/
|
|
# The newer tradefed requires different parsing to count waivers.
|
|
waived, _ = tradefed_utils.parse_tradefed_result(
|
|
_load_data('CtsAppTestCases_P_simplified.txt'),
|
|
waivers=waivers)
|
|
self.assertEquals(1, len(waived))
|
|
|
|
# b/66899135, tradefed may reported inaccuratly with `list results`.
|
|
# Check if summary section shows that the result is inacurrate.
|
|
_, accurate = tradefed_utils.parse_tradefed_result(
|
|
_load_data('CtsAppTestCases_P_simplified.txt'),
|
|
waivers=waivers)
|
|
self.assertTrue(accurate)
|
|
|
|
_, accurate = tradefed_utils.parse_tradefed_result(
|
|
_load_data('CtsDeqpTestCases-trimmed-inaccurate.txt'),
|
|
waivers=waivers)
|
|
self.assertFalse(accurate)
|
|
|
|
def test_get_test_result_xml_path(self):
|
|
path = tradefed_utils.get_test_result_xml_path(os.path.join(
|
|
os.path.dirname(os.path.realpath(__file__)),
|
|
'tradefed_utils_unittest_data', 'results'))
|
|
self.assertEqual(path, os.path.join(
|
|
os.path.dirname(os.path.realpath(__file__)),
|
|
'tradefed_utils_unittest_data', 'results', '2019.11.07_10.14.55',
|
|
'test_result.xml'))
|
|
|
|
# assertNoRaises
|
|
tradefed_utils.get_test_result_xml_path(os.path.join(
|
|
os.path.dirname(os.path.realpath(__file__)),
|
|
'tradefed_utils_unittest_data', 'not_exist'))
|
|
|
|
def test_parse_tradefed_testresults_xml_no_failure(self):
|
|
waived, accurate = tradefed_utils.parse_tradefed_testresults_xml(
|
|
os.path.join(os.path.dirname(os.path.realpath(__file__)),
|
|
'tradefed_utils_unittest_data', 'test_result.xml'))
|
|
self.assertEquals(0, len(waived))
|
|
self.assertTrue(accurate)
|
|
|
|
def test_parse_tradefed_testresult_xml_waivers(self):
|
|
waived, accurate = tradefed_utils.parse_tradefed_testresults_xml(
|
|
os.path.join(os.path.dirname(os.path.realpath(__file__)),
|
|
'tradefed_utils_unittest_data',
|
|
'gtsplacement_test_result.xml'))
|
|
self.assertEquals(0, len(waived))
|
|
|
|
waivers = set([
|
|
'com.google.android.media.gts.WidevineDashPolicyTests#testL1RenewalDelay5S',
|
|
'com.google.android.media.gts.MediaDrmTest#testWidevineApi28',
|
|
'com.google.android.media.gts.WidevineGenericOpsTests#testL3',
|
|
'com.google.android.media.gts.WidevineDashPolicyTests#testL3RenewalDelay5S',
|
|
'com.google.android.media.gts.WidevineH264PlaybackTests#testCbc1L3WithUHD30',
|
|
'com.google.android.media.gts.WidevineH264PlaybackTests#testCbcsL3WithUHD30',
|
|
'com.google.android.media.gts.WidevineH264PlaybackTests#testCbc1L1WithUHD30',
|
|
'com.google.android.media.gts.WidevineDashPolicyTests#testL3RenewalDelay13S',
|
|
'com.google.android.gts.backup.BackupHostTest#testGmsBackupTransportIsDefault',
|
|
'com.google.android.placement.gts.CoreGmsAppsTest#testGoogleDuoPreloaded',
|
|
'com.google.android.placement.gts.CoreGmsAppsTest#testCoreGmsAppsPreloaded',
|
|
'com.google.android.media.gts.WidevineH264PlaybackTests#testCbcsL1WithUHD30'])
|
|
waived, accurate = tradefed_utils.parse_tradefed_testresults_xml(
|
|
os.path.join(os.path.dirname(os.path.realpath(__file__)),
|
|
'tradefed_utils_unittest_data',
|
|
'gtsplacement_test_result.xml'), waivers=waivers)
|
|
self.assertEquals(4, len(waived))
|
|
|
|
def test_get_perf_metrics_from_test_result_xml(self):
|
|
perf_result = tradefed_utils.get_perf_metrics_from_test_result_xml(
|
|
os.path.join(os.path.dirname(os.path.realpath(__file__)),
|
|
'tradefed_utils_unittest_data', 'test_result.xml'),
|
|
os.path.join('/', 'resultsdir'))
|
|
expected_result = [
|
|
{'units': 'ms',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.AudioRecordTest'
|
|
'#testAudioRecordLocalMono16Bit',
|
|
'value': '7.1688596491228065', 'higher_is_better': False},
|
|
{'units': 'ms',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.AudioRecordTest'
|
|
'#testAudioRecordLocalMono16BitShort',
|
|
'value': '2.5416666666666665', 'higher_is_better': False},
|
|
{'units': 'ms',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.AudioRecordTest'
|
|
'#testAudioRecordLocalNonblockingStereoFloat',
|
|
'value': '1.75', 'higher_is_better': False},
|
|
{'units': 'ms',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.AudioRecordTest'
|
|
'#testAudioRecordMonoFloat',
|
|
'value': '12.958881578947368', 'higher_is_better': False},
|
|
{'units': 'ms',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.AudioRecordTest'
|
|
'#testAudioRecordResamplerMono8Bit',
|
|
'value': '0.0', 'higher_is_better': False},
|
|
{'units': 'ms',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.AudioRecordTest'
|
|
'#testAudioRecordResamplerStereo8Bit',
|
|
'value': '3.5', 'higher_is_better': False},
|
|
{'units': 'ms',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.AudioRecordTest'
|
|
'#testAudioRecordStereo16Bit',
|
|
'value': '3.5', 'higher_is_better': False},
|
|
{'units': 'ms',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.AudioTrackTest'
|
|
'#testFastTimestamp',
|
|
'value': '0.1547618955373764', 'higher_is_better': False},
|
|
{'units': 'ms',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.AudioTrackTest'
|
|
'#testGetTimestamp',
|
|
'value': '0.1490119844675064', 'higher_is_better': False},
|
|
{'units': 'ms',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.AudioTrack_ListenerTest'
|
|
'#testAudioTrackCallback',
|
|
'value': '9.347127739984884', 'higher_is_better': False},
|
|
{'units': 'ms',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.AudioTrack_ListenerTest'
|
|
'#testAudioTrackCallbackWithHandler',
|
|
'value': '7.776177955844914', 'higher_is_better': False},
|
|
{'units': 'ms',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.AudioTrack_ListenerTest'
|
|
'#testStaticAudioTrackCallback',
|
|
'value': '7.776177955844914', 'higher_is_better': False},
|
|
{'units': 'ms',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.AudioTrack_ListenerTest'
|
|
'#testStaticAudioTrackCallbackWithHandler',
|
|
'value': '9.514361300075587', 'higher_is_better': False},
|
|
{'units': 'count',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.DecoderTest'
|
|
'#testH264ColorAspects',
|
|
'value': '1.0', 'higher_is_better': True},
|
|
{'units': 'count',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.DecoderTest'
|
|
'#testH265ColorAspects',
|
|
'value': '1.0', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testAvcGoog0Perf0320x0240',
|
|
'value': '580.1607045151507', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testAvcGoog0Perf0720x0480',
|
|
'value': '244.18184010611358', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testAvcGoog0Perf1280x0720',
|
|
'value': '70.96290491279275', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testAvcGoog0Perf1920x1080',
|
|
'value': '31.299613935451564', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testAvcOther0Perf0320x0240',
|
|
'value': '1079.6843075197307', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testAvcOther0Perf0720x0480',
|
|
'value': '873.7785366761784', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testAvcOther0Perf1280x0720',
|
|
'value': '664.6463289568261', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testAvcOther0Perf1920x1080',
|
|
'value': '382.10811352923474', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testH263Goog0Perf0176x0144',
|
|
'value': '1511.3027429644353', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testHevcGoog0Perf0352x0288',
|
|
'value': '768.8737453173384', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testHevcGoog0Perf0640x0360',
|
|
'value': '353.7226028743237', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testHevcGoog0Perf0720x0480',
|
|
'value': '319.3122874170939', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testHevcGoog0Perf1280x0720',
|
|
'value': '120.89218432028369', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testMpeg4Goog0Perf0176x0144',
|
|
'value': '1851.890822618321', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testVp8Goog0Perf0320x0180',
|
|
'value': '1087.946513466716', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testVp8Goog0Perf0640x0360',
|
|
'value': '410.18461316281423', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testVp8Goog0Perf1920x1080',
|
|
'value': '36.26433070651982', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testVp8Other0Perf0320x0180',
|
|
'value': '1066.7819511702078', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testVp8Other0Perf0640x0360',
|
|
'value': '930.261434505189', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testVp8Other0Perf1280x0720',
|
|
'value': '720.4170603577236', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testVp8Other0Perf1920x1080',
|
|
'value': '377.55742437554915', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testVp9Goog0Perf0320x0180',
|
|
'value': '988.6158776121617', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testVp9Goog0Perf0640x0360',
|
|
'value': '409.8162085338674', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testVp9Goog0Perf1280x0720',
|
|
'value': '147.75847359424512', 'higher_is_better': True},
|
|
{'units': 'fps',
|
|
'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases',
|
|
'description': 'android.media.cts.VideoDecoderPerfTest'
|
|
'#testVp9Goog0Perf1920x1080',
|
|
'value': '83.95677136649255', 'higher_is_better': True}
|
|
]
|
|
self.assertListEqual(list(perf_result), expected_result)
|
|
|
|
perf_result = tradefed_utils.get_perf_metrics_from_test_result_xml(
|
|
os.path.join(os.path.dirname(os.path.realpath(__file__)),
|
|
'tradefed_utils_unittest_data',
|
|
'malformed_test_result.xml'),
|
|
os.path.join('/', 'resultsdir'))
|
|
self.assertListEqual(list(perf_result), [])
|
|
|
|
# assertNoRaises
|
|
tradefed_utils.get_perf_metrics_from_test_result_xml(
|
|
os.path.join(os.path.dirname(os.path.realpath(__file__)),
|
|
'tradefed_utils_unittest_data',
|
|
'not_exist'),
|
|
os.path.join('/', 'resultsdir'))
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|