# Copyright 2021 The ANGLE Project Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # This is a .pyl, or "Python Literal", file. You can treat it just like a # .json file, with the following exceptions: # * all keys must be quoted (use single quotes, please); # * comments are allowed, using '#' syntax; and # * trailing commas are allowed. # # For more info see Chromium's test_suites.pyl in testing/buildbot. { 'basic_suites': { 'angle_deqp_egl_d3d11_gtests': { 'angle_deqp_egl_d3d11_tests': { 'args': [ 'angle_deqp_egl_tests', '--use-angle=d3d11', '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'test': 'angle_deqp_egl_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_egl_gl_gtests': { 'angle_deqp_egl_gl_tests': { 'args': [ 'angle_deqp_egl_tests', '--use-angle=gl', '--bot-mode', # Flaky when run with multiple processes. '--max-processes=1', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'test': 'angle_deqp_egl_tests', 'use_isolated_scripts_api': True, }, }, # Android-only. Temporarily disabled. # 'angle_deqp_egl_gles_gtests': { # 'angle_deqp_egl_gles_tests': { # 'args': [ # 'angle_deqp_egl_tests', # '--use-angle=gles', # ], # 'android_args': [ # '-v', # '--shard-timeout=500', # ], # 'android_swarming': { # 'shards': 4, # }, # 'merge': { # 'script': '//scripts/angle_deqp_test_merge.py', # }, # 'test': 'angle_deqp_egl_tests', # 'use_isolated_scripts_api': True, # }, # }, 'angle_deqp_egl_metal_gtests': { 'angle_deqp_egl_metal_tests': { 'args': [ 'angle_deqp_egl_tests', '--use-angle=metal', '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'test': 'angle_deqp_egl_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_egl_vulkan_gtests': { 'angle_deqp_egl_vulkan_tests': { 'args': [ 'angle_deqp_egl_tests', '--use-angle=vulkan', ], 'android_args': [ '-v', '--shard-timeout=500', ], 'android_swarming': { 'shards': 4, }, 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 2, }, 'test': 'angle_deqp_egl_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles2_d3d11_gtests': { 'angle_deqp_gles2_d3d11_tests': { 'args': [ 'angle_deqp_gles2_tests', '--use-angle=d3d11', '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'test': 'angle_deqp_gles2_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles2_gl_gtests': { 'angle_deqp_gles2_gl_tests': { 'args': [ 'angle_deqp_gles2_tests', '--use-angle=gl', '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'test': 'angle_deqp_gles2_tests', 'use_isolated_scripts_api': True, }, }, # Android-only. Temporarily disabled. # 'angle_deqp_gles2_gles_gtests': { # 'angle_deqp_gles2_gles_tests': { # 'args': [ # 'angle_deqp_gles2_tests', # '--use-angle=gles', # ], # 'android_args': [ # '-v', # '--shard-timeout=500', # ], # 'android_swarming': { # 'shards': 4, # }, # 'desktop_args': [ # '--bot-mode', # ], # 'merge': { # 'script': '//scripts/angle_deqp_test_merge.py', # }, # 'test': 'angle_deqp_gles2_tests', # 'use_isolated_scripts_api': True, # }, # }, 'angle_deqp_gles2_metal_gtests': { 'angle_deqp_gles2_metal_tests': { 'args': [ 'angle_deqp_gles2_tests', '--use-angle=metal', ], 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'test': 'angle_deqp_gles2_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles2_vulkan_gtests': { 'angle_deqp_gles2_vulkan_tests': { 'args': [ 'angle_deqp_gles2_tests', '--use-angle=vulkan', ], 'android_args': [ '-v', '--shard-timeout=500', ], 'android_swarming': { 'shards': 4, }, 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'test': 'angle_deqp_gles2_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles31_d3d11_gtests': { 'angle_deqp_gles31_d3d11_tests': { 'args': [ 'angle_deqp_gles31_tests', '--use-angle=d3d11', '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 2, }, 'test': 'angle_deqp_gles31_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles31_gl_gtests': { 'angle_deqp_gles31_gl_tests': { 'args': [ 'angle_deqp_gles31_tests', '--use-angle=gl', '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 2, }, 'test': 'angle_deqp_gles31_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles31_vulkan_gtests': { 'angle_deqp_gles31_vulkan_tests': { 'args': [ 'angle_deqp_gles31_tests', '--use-angle=vulkan', ], 'android_args': [ '-v', '--shard-timeout=500', ], 'android_swarming': { 'shards': 20, }, 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 2, }, 'test': 'angle_deqp_gles31_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles31_vulkan_rotate180_gtests': { 'angle_deqp_gles31_vulkan_rotate180_tests': { 'args': [ 'angle_deqp_gles31_rotate180_tests', '--use-angle=vulkan', ], 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 1, }, 'test': 'angle_deqp_gles31_rotate180_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles31_vulkan_rotate270_gtests': { 'angle_deqp_gles31_vulkan_rotate270_tests': { 'args': [ 'angle_deqp_gles31_rotate270_tests', '--use-angle=vulkan', ], 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 1, }, 'test': 'angle_deqp_gles31_rotate270_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles31_vulkan_rotate90_gtests': { 'angle_deqp_gles31_vulkan_rotate90_tests': { 'args': [ 'angle_deqp_gles31_rotate90_tests', '--use-angle=vulkan', ], 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 1, }, 'test': 'angle_deqp_gles31_rotate90_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles3_d3d11_gtests': { 'angle_deqp_gles3_d3d11_tests': { 'args': [ 'angle_deqp_gles3_tests', '--use-angle=d3d11', '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 2, }, 'test': 'angle_deqp_gles3_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles3_gl_gtests': { 'angle_deqp_gles3_gl_tests': { 'args': [ 'angle_deqp_gles3_tests', '--use-angle=gl', '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 2, }, 'test': 'angle_deqp_gles3_tests', 'use_isolated_scripts_api': True, }, }, # Android-only. Temporarily disabled. # 'angle_deqp_gles3_gles_gtests': { # 'angle_deqp_gles3_gles_tests': { # 'args': [ # 'angle_deqp_gles3_tests', # '--use-angle=gles', # ], # 'android_args': [ # '-v', # '--shard-timeout=500', # ], # 'android_swarming': { # 'shards': 12, # }, # 'merge': { # 'script': '//scripts/angle_deqp_test_merge.py', # }, # 'test': 'angle_deqp_gles3_tests', # 'use_isolated_scripts_api': True, # }, # }, 'angle_deqp_gles3_metal_gtests': { 'angle_deqp_gles3_metal_tests': { 'args': [ 'angle_deqp_gles3_tests', '--use-angle=metal', # Flakiness on multiple configs. http://anglebug.com/6019 '--flaky-retries=2', ], 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 2, }, 'test': 'angle_deqp_gles3_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles3_vulkan_gtests': { 'angle_deqp_gles3_vulkan_tests': { 'args': [ 'angle_deqp_gles3_tests', '--use-angle=vulkan', ], 'android_args': [ '-v', '--shard-timeout=500', ], 'android_swarming': { 'shards': 12, }, 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 4, }, 'test': 'angle_deqp_gles3_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles3_vulkan_rotate180_gtests': { 'angle_deqp_gles3_vulkan_rotate180_tests': { 'args': [ 'angle_deqp_gles3_rotate180_tests', '--use-angle=vulkan', ], 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 2, }, 'test': 'angle_deqp_gles3_rotate180_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles3_vulkan_rotate270_gtests': { 'angle_deqp_gles3_vulkan_rotate270_tests': { 'args': [ 'angle_deqp_gles3_rotate270_tests', '--use-angle=vulkan', ], 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 2, }, 'test': 'angle_deqp_gles3_rotate270_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_gles3_vulkan_rotate90_gtests': { 'angle_deqp_gles3_vulkan_rotate90_tests': { 'args': [ 'angle_deqp_gles3_rotate90_tests', '--use-angle=vulkan', ], 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'swarming': { 'shards': 2, }, 'test': 'angle_deqp_gles3_rotate90_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_khr_gles2_vulkan_gtests': { 'angle_deqp_khr_gles2_vulkan_tests': { 'args': [ 'angle_deqp_khr_gles2_tests', '--use-angle=vulkan', ], 'android_args': [ '-v', '--shard-timeout=500', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'desktop_args': [ '--bot-mode', ], 'test': 'angle_deqp_khr_gles2_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_khr_gles31_vulkan_gtests': { 'angle_deqp_khr_gles31_vulkan_tests': { 'args': [ 'angle_deqp_khr_gles31_tests', '--use-angle=vulkan', ], 'android_args': [ '-v', '--shard-timeout=500', ], 'android_swarming': { 'shards': 2, }, 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'test': 'angle_deqp_khr_gles31_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_khr_gles32_vulkan_gtests': { 'angle_deqp_khr_gles32_vulkan_tests': { 'args': [ 'angle_deqp_khr_gles32_tests', '--use-angle=vulkan', ], 'android_args': [ '-v', '--shard-timeout=500', ], 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'test': 'angle_deqp_khr_gles32_tests', 'use_isolated_scripts_api': True, }, }, 'angle_deqp_khr_gles3_vulkan_gtests': { 'angle_deqp_khr_gles3_vulkan_tests': { 'args': [ 'angle_deqp_khr_gles3_tests', '--use-angle=vulkan', ], 'android_args': [ '-v', '--shard-timeout=500', ], 'android_swarming': { 'shards': 2, }, 'desktop_args': [ '--bot-mode', ], 'merge': { 'script': '//scripts/angle_deqp_test_merge.py', }, 'test': 'angle_deqp_khr_gles3_tests', 'use_isolated_scripts_api': True, }, }, 'angle_end2end_gtests': { 'angle_end2end_tests': { 'android_args': [ '--shard-timeout=180', '-v', ], 'android_swarming': { 'shards': 4, }, 'args': [ 'angle_end2end_tests', '--gtest_filter=-*Vulkan_SwiftShader*', ], 'desktop_args': [ '--bot-mode', ], # Linux has issues with creating too many windows at once. 'linux_args': [ '--max-processes=4', ], 'swarming': { 'shards': 2, }, 'use_isolated_scripts_api': True, }, }, 'angle_end2end_with_retry_gtests': { 'angle_end2end_tests': { 'args': [ 'angle_end2end_tests', '--gtest_filter=-*Vulkan_SwiftShader*', '--bot-mode', '--flaky-retries=2', ], # Linux has issues with creating too many windows at once. 'linux_args': [ '--max-processes=4', ], 'swarming': { 'shards': 2, }, 'use_isolated_scripts_api': True, }, }, 'angle_gles1_conformance_gtests': { 'angle_gles1_conformance_tests': { 'android_args': [ '--shard-timeout=180', '-v', ], 'args': [ 'angle_gles1_conformance_tests', '--test-timeout=40', ], 'desktop_args': [ '--bot-mode', ], 'use_isolated_scripts_api': True, }, }, # Actually uploads results to the perf dashboard 'angle_perf_isolated_scripts': { 'angle_perftests': { 'args': [ 'angle_perftests', '--non-telemetry=true', '--gtest-benchmark-name=angle_perftests', '-v', '--test-timeout=100', ], 'android_args': [ '--test-launcher-batch-limit=1', ], 'desktop_args': [ '--batch-size=1', '--bot-mode', '--max-processes=1', '--print-test-stdout', ], 'merge': { 'script': '//tools/perf/process_perf_results.py', }, 'android_swarming': { 'shards': 12, }, 'swarming': { 'shards': 10, }, }, }, # The angle perf tests should eventually be run on every platform # ANGLE supports. Currently they run on Windows, Linux and Android. 'angle_perf_smoke_isolated_scripts': { 'angle_perftests': { 'args': [ 'angle_perftests', '--non-telemetry=true', '--gtest-benchmark-name=angle_perftests', '-v', # Tell the tests to exit after one frame for faster iteration. '--one-frame-only', '--test-timeout=100', # Do not run trace tests because they're already run in Gold tests. '--gtest_filter=-TracePerfTest.*', ], 'android_args': [ '--test-launcher-batch-limit=1', ], 'desktop_args': [ '--batch-size=1', '--bot-mode', '--max-processes=1', '--print-test-stdout', ], 'merge': { 'script': '//tools/perf/process_perf_results.py', # Does not upload to the perf dashboard 'args': [ '--smoke-test-mode', ], }, 'swarming': { 'shards': 2, }, 'android_swarming': { 'shards': 6, }, }, }, 'angle_restricted_trace_gold_isolated_scripts': { 'angle_restricted_trace_gold_tests': { 'android_args': [ '-v', ], 'args': [ '--test-machine-name', '${buildername}', ], 'mixins': [ 'angle_skia_gold_test', 'chrome-gpu-gold-service-account', ], 'android_swarming': { 'shards': 6, }, } }, 'angle_unit_gtests': { 'angle_unittests': { 'args': [ 'angle_unittests', ], 'android_args': [ '-v', ], 'use_isolated_scripts_api': True, }, }, 'angle_white_box_gtests': { 'angle_white_box_tests': { 'android_args': [ '--shard-timeout=180', '-v', ], 'args': [ 'angle_white_box_tests', ], 'desktop_args': [ '--bot-mode', ], 'use_isolated_scripts_api': True, }, }, }, 'compound_suites': { 'android_gtests': [ 'angle_deqp_egl_vulkan_gtests', 'angle_deqp_gles2_vulkan_gtests', 'angle_deqp_gles31_vulkan_gtests', 'angle_deqp_gles3_vulkan_gtests', 'angle_deqp_khr_gles2_vulkan_gtests', 'angle_deqp_khr_gles31_vulkan_gtests', 'angle_deqp_khr_gles32_vulkan_gtests', 'angle_deqp_khr_gles3_vulkan_gtests', 'angle_end2end_gtests', 'angle_unit_gtests', ], 'common_isolated_scripts': [ 'angle_perf_smoke_isolated_scripts', 'angle_restricted_trace_gold_isolated_scripts', ], 'linux_intel_gtests': [ 'angle_deqp_gles2_gl_gtests', 'angle_end2end_gtests', 'angle_unit_gtests', 'angle_white_box_gtests', ], 'linux_nvidia_gtests': [ 'angle_deqp_egl_gl_gtests', 'angle_deqp_egl_vulkan_gtests', 'angle_deqp_gles2_gl_gtests', 'angle_deqp_gles2_vulkan_gtests', 'angle_deqp_gles31_gl_gtests', 'angle_deqp_gles31_vulkan_gtests', 'angle_deqp_gles31_vulkan_rotate180_gtests', 'angle_deqp_gles31_vulkan_rotate270_gtests', 'angle_deqp_gles31_vulkan_rotate90_gtests', 'angle_deqp_gles3_gl_gtests', 'angle_deqp_gles3_vulkan_gtests', 'angle_deqp_gles3_vulkan_rotate180_gtests', 'angle_deqp_gles3_vulkan_rotate270_gtests', 'angle_deqp_gles3_vulkan_rotate90_gtests', 'angle_deqp_khr_gles2_vulkan_gtests', 'angle_deqp_khr_gles31_vulkan_gtests', 'angle_deqp_khr_gles32_vulkan_gtests', 'angle_deqp_khr_gles3_vulkan_gtests', 'angle_end2end_gtests', 'angle_unit_gtests', 'angle_white_box_gtests', ], 'mac_amd_and_intel_gtests': [ 'angle_deqp_egl_gl_gtests', 'angle_deqp_egl_metal_gtests', 'angle_deqp_gles2_gl_gtests', 'angle_deqp_gles2_metal_gtests', 'angle_deqp_gles3_gl_gtests', 'angle_deqp_gles3_metal_gtests', 'angle_end2end_gtests', 'angle_unit_gtests', ], 'mac_nvidia_gtests': [ 'angle_end2end_gtests', 'angle_unit_gtests', ], 'win10_intel_gtests': [ 'angle_deqp_gles2_d3d11_gtests', 'angle_end2end_gtests', 'angle_gles1_conformance_gtests', 'angle_unit_gtests', 'angle_white_box_gtests', ], 'win10_nvidia_gtests': [ 'angle_deqp_egl_d3d11_gtests', 'angle_deqp_egl_gl_gtests', 'angle_deqp_egl_vulkan_gtests', 'angle_deqp_gles2_d3d11_gtests', 'angle_deqp_gles2_gl_gtests', 'angle_deqp_gles2_vulkan_gtests', 'angle_deqp_gles31_d3d11_gtests', 'angle_deqp_gles31_gl_gtests', 'angle_deqp_gles31_vulkan_gtests', 'angle_deqp_gles31_vulkan_rotate180_gtests', 'angle_deqp_gles31_vulkan_rotate270_gtests', 'angle_deqp_gles31_vulkan_rotate90_gtests', 'angle_deqp_gles3_d3d11_gtests', 'angle_deqp_gles3_gl_gtests', 'angle_deqp_gles3_vulkan_gtests', 'angle_deqp_gles3_vulkan_rotate180_gtests', 'angle_deqp_gles3_vulkan_rotate270_gtests', 'angle_deqp_gles3_vulkan_rotate90_gtests', 'angle_deqp_khr_gles2_vulkan_gtests', 'angle_deqp_khr_gles31_vulkan_gtests', 'angle_deqp_khr_gles32_vulkan_gtests', 'angle_deqp_khr_gles3_vulkan_gtests', 'angle_end2end_gtests', 'angle_gles1_conformance_gtests', 'angle_unit_gtests', 'angle_white_box_gtests', ], 'win7_amd_gtests': [ 'angle_deqp_gles2_d3d11_gtests', 'angle_deqp_gles2_vulkan_gtests', 'angle_end2end_with_retry_gtests', 'angle_unit_gtests', 'angle_white_box_gtests', ], 'win7_nvidia_gtests': [ 'angle_end2end_gtests', 'angle_gles1_conformance_gtests', 'angle_unit_gtests', 'angle_white_box_gtests', 'angle_deqp_egl_d3d11_gtests', 'angle_deqp_gles2_d3d11_gtests', 'angle_deqp_gles3_d3d11_gtests', ], }, }