From c92091d40e2ac9f165199995d1ecdead898086ca Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 8 Sep 2023 12:51:00 -0600 Subject: [PATCH 1/3] Add new 'perf' test property This will be used to mark suites as performance test suites. --- CIME/get_tests.py | 39 ++++++++++++++++++++++++++++++++++++--- CIME/test_scheduler.py | 7 +++++-- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/CIME/get_tests.py b/CIME/get_tests.py index 04baf328f78..361e034a6e3 100644 --- a/CIME/get_tests.py +++ b/CIME/get_tests.py @@ -23,6 +23,7 @@ # "inherit" : (suite1, suite2, ...), # Optional. Suites to inherit tests from. Default is None. Tuple, list, or str. # "time" : "HH:MM:SS", # Optional. Recommended upper-limit on test time. # "share" : True|False, # Optional. If True, all tests in this suite share a build. Default is False. +# "perf" : True|False, # Optional. If True, all tests in this suite will do performance tracking. Default is False. # "tests" : (test1, test2, ...) # Optional. The list of tests for this suite. See above for format. Tuple, list, or str. This is the ONLY inheritable attribute. # } @@ -89,6 +90,16 @@ "SMS_P16.f19_g16_rx1.X", ), }, + "cime_test_perf": { + "time": "0:10:00", + "perf": True, + "tests": ( + "SMS_P2.f19_g16_rx1.Z", + "SMS_P4.f19_g16_rx1.Z", + "SMS_P8.f19_g16_rx1.Z", + "SMS_P16.f19_g16_rx1.Z", + ), + }, "cime_test_repeat": { "tests": ( "TESTRUNPASS_P1.f19_g16_rx1.A", @@ -159,12 +170,12 @@ def _get_key_data(raw_dict, key, the_type): def get_test_data(suite): ############################################################################### """ - For a given suite, returns (inherit, time, share, tests) + For a given suite, returns (inherit, time, share, perf, tests) """ raw_dict = _ALL_TESTS[suite] for key in raw_dict.keys(): expect( - key in ["inherit", "time", "share", "tests"], + key in ["inherit", "time", "share", "perf", "tests"], "Unexpected test key '{}'".format(key), ) @@ -172,6 +183,7 @@ def get_test_data(suite): _get_key_data(raw_dict, "inherit", tuple), _get_key_data(raw_dict, "time", str), _get_key_data(raw_dict, "share", bool), + _get_key_data(raw_dict, "perf", bool), _get_key_data(raw_dict, "tests", tuple), ) @@ -201,7 +213,7 @@ def get_test_suite( "Compiler {} not valid for machine {}".format(compiler, machine), ) - inherits_from, _, _, tests_raw = get_test_data(suite) + inherits_from, _, _, _, tests_raw = get_test_data(suite) tests = [] for item in tests_raw: expect( @@ -298,6 +310,27 @@ def get_build_groups(tests): return [tuple(item[0]) for item in build_groups] +############################################################################### +def is_perf_test(test): + ############################################################################### + """ + Is the provided test in a suite with perf=True? + + >>> is_perf_test("SMS_P2.f19_g16_rx1.Z.melvin_gnu") + True + >>> is_perf_test("SMS_P2.f19_g16_rx1.X.melvin_gnu") + False + """ + # Get a list of performance suites + suites = get_test_suites() + perf_suites = [] + for suite in suites: + perf = get_test_data(suite)[3] + if perf and suite_has_test(suite, test, skip_inherit=True): + return True + + return False + ############################################################################### def infer_arch_from_tests(testargs): ############################################################################### diff --git a/CIME/test_scheduler.py b/CIME/test_scheduler.py index ef3caedf102..47119a09320 100644 --- a/CIME/test_scheduler.py +++ b/CIME/test_scheduler.py @@ -13,7 +13,7 @@ from collections import OrderedDict from CIME.XML.standard_module_setup import * -from CIME.get_tests import get_recommended_test_time, get_build_groups +from CIME.get_tests import get_recommended_test_time, get_build_groups, is_perf_test from CIME.utils import ( append_status, append_testlog, @@ -967,7 +967,10 @@ def _xml_phase(self, test): ) envtest.set_initial_values(case) case.set_value("TEST", True) - case.set_value("SAVE_TIMING", self._save_timing) + if is_perf_test(test): + case.set_value("SAVE_TIMING", True) + else: + case.set_value("SAVE_TIMING", self._save_timing) # handle single-exe here, all cases will use the EXEROOT from # the first case in the build group From 32f3ea2640ac75de5576b1906c1287ec43dbbbbd Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 8 Sep 2023 13:18:06 -0600 Subject: [PATCH 2/3] Minor fixes so that the cime_test_perf suite is actually runnable --- CIME/get_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CIME/get_tests.py b/CIME/get_tests.py index 361e034a6e3..3efea82e081 100644 --- a/CIME/get_tests.py +++ b/CIME/get_tests.py @@ -94,10 +94,10 @@ "time": "0:10:00", "perf": True, "tests": ( - "SMS_P2.f19_g16_rx1.Z", - "SMS_P4.f19_g16_rx1.Z", - "SMS_P8.f19_g16_rx1.Z", - "SMS_P16.f19_g16_rx1.Z", + "SMS_P2.T42_T42.S", + "SMS_P4.T42_T42.S", + "SMS_P8.T42_T42.S", + "SMS_P16.T42_T42.S", ), }, "cime_test_repeat": { @@ -316,7 +316,7 @@ def is_perf_test(test): """ Is the provided test in a suite with perf=True? - >>> is_perf_test("SMS_P2.f19_g16_rx1.Z.melvin_gnu") + >>> is_perf_test("SMS_P2.T42_T42.S.melvin_gnu") True >>> is_perf_test("SMS_P2.f19_g16_rx1.X.melvin_gnu") False From 45a07941f82a8d562b94ce0efc2026c785b583a9 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 8 Sep 2023 13:39:26 -0600 Subject: [PATCH 3/3] pylint fix --- CIME/get_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/get_tests.py b/CIME/get_tests.py index 3efea82e081..4be4147ef4f 100644 --- a/CIME/get_tests.py +++ b/CIME/get_tests.py @@ -323,7 +323,6 @@ def is_perf_test(test): """ # Get a list of performance suites suites = get_test_suites() - perf_suites = [] for suite in suites: perf = get_test_data(suite)[3] if perf and suite_has_test(suite, test, skip_inherit=True): @@ -331,6 +330,7 @@ def is_perf_test(test): return False + ############################################################################### def infer_arch_from_tests(testargs): ###############################################################################