From 6b48aa24f01a0374b586cec27189455d0a29e36b Mon Sep 17 00:00:00 2001 From: "Ryan Anderson (AWS)" Date: Wed, 7 Aug 2024 19:51:52 -0400 Subject: [PATCH] Add the cluster stack name to the metadata table for quick identification Also add the cw log group name as the stack name for now. Adds a dummy cluster test for quick iteration as well. --- tests/integration-tests/conftest.py | 21 +++++++++------- tests/integration-tests/conftest_utils.py | 6 +++-- .../framework/metadata_table_manager.py | 6 ++--- tests/integration-tests/requirements.txt | 2 ++ .../tests/dummy/test_dummy.py | 16 ++++++++++++ .../test_dummy_cluster/pcluster.config.yaml | 25 +++++++++++++++++++ 6 files changed, 62 insertions(+), 14 deletions(-) create mode 100644 tests/integration-tests/tests/dummy/test_dummy/test_dummy_cluster/pcluster.config.yaml diff --git a/tests/integration-tests/conftest.py b/tests/integration-tests/conftest.py index 948a0575f8..1db53d37e7 100644 --- a/tests/integration-tests/conftest.py +++ b/tests/integration-tests/conftest.py @@ -392,16 +392,19 @@ def clusters_factory(request, region): def _cluster_factory(cluster_config, upper_case_cluster_name=False, custom_cli_credentials=None, **kwargs): cluster_config = _write_config_to_outdir(request, cluster_config, "clusters_configs") + cluster_name = ( + request.config.getoption("cluster") + if request.config.getoption("cluster") + else "integ-tests-{0}{1}{2}".format( + random_alphanumeric().upper() if upper_case_cluster_name else random_alphanumeric(), + "-" if request.config.getoption("stackname_suffix") else "", + request.config.getoption("stackname_suffix"), + ) + ) + request.node.user_properties.append(("cluster_stack_name", f"{cluster_name}")) + request.node.user_properties.append(("cw_log_group_name", f"{cluster_name}")) cluster = Cluster( - name=( - request.config.getoption("cluster") - if request.config.getoption("cluster") - else "integ-tests-{0}{1}{2}".format( - random_alphanumeric().upper() if upper_case_cluster_name else random_alphanumeric(), - "-" if request.config.getoption("stackname_suffix") else "", - request.config.getoption("stackname_suffix"), - ) - ), + name=cluster_name, config_file=cluster_config, ssh_key=request.config.getoption("key_path"), region=region, diff --git a/tests/integration-tests/conftest_utils.py b/tests/integration-tests/conftest_utils.py index c69640d1e6..67d31c2f10 100644 --- a/tests/integration-tests/conftest_utils.py +++ b/tests/integration-tests/conftest_utils.py @@ -192,6 +192,8 @@ def publish_test_metadata(item: pytest.Item, rep: pytest.TestReport): cli_commit=item.config.getoption("--pcluster-git-ref"), cookbook_commit=item.config.getoption("--cookbook-git-ref"), node_commit=item.config.getoption("--node-git-ref"), + cluster_stack_name="none", + cw_log_group_name="none", setup_metadata=PhaseMetadata( rep.when, status=rep.outcome, @@ -199,8 +201,6 @@ def publish_test_metadata(item: pytest.Item, rep: pytest.TestReport): end_time=get_user_prop(item, f"end_time_{rep.when}"), ), ) - # Create the metadata table on the fly if it doesn't exist - metadata_table_mgr.create_metadata_table() if rep.when == "call": # Update the call test data test_metadata = jsonpickle.decode(get_user_prop(item, "metadata")) @@ -210,6 +210,8 @@ def publish_test_metadata(item: pytest.Item, rep: pytest.TestReport): start_time=get_user_prop(item, f"start_time_{rep.when}"), end_time=get_user_prop(item, f"end_time_{rep.when}"), ) + test_metadata.cluster_stack_name = get_user_prop(item, "cluster_stack_name") + test_metadata.cw_log_group_name = get_user_prop(item, "cw_log_group_name") if rep.when == "teardown": # Update the teardown test data test_metadata = jsonpickle.decode(get_user_prop(item, "metadata")) diff --git a/tests/integration-tests/framework/metadata_table_manager.py b/tests/integration-tests/framework/metadata_table_manager.py index 3d2088b695..8713705cc4 100644 --- a/tests/integration-tests/framework/metadata_table_manager.py +++ b/tests/integration-tests/framework/metadata_table_manager.py @@ -41,7 +41,7 @@ class TestMetadata: cli_commit: str = "" cookbook_commit: str = "" node_commit: str = "" - cfn_stack_name: str = "" + cluster_stack_name: str = "" cw_log_group_name: str = "" global_build_number: int = 0 @@ -118,7 +118,7 @@ def publish_metadata(self, metadata: List[TestMetadata]): "cli_commit": {"S": datum.cli_commit if datum.cli_commit else "None"}, "cookbook_commit": {"S": datum.cookbook_commit if datum.cookbook_commit else "None"}, "node_commit": {"S": datum.node_commit if datum.node_commit else "None"}, - "cfn_stack_name": {"S": datum.cfn_stack_name if datum.cfn_stack_name else "None"}, + "cluster_stack_name": {"S": datum.cluster_stack_name if datum.cluster_stack_name else "None"}, "cw_log_group_name": {"S": datum.cw_log_group_name if datum.cw_log_group_name else "None"}, "global_build_number": { "N": str(datum.global_build_number) if datum.global_build_number else "0" @@ -166,7 +166,7 @@ def get_metadata(self, ids: List[str]) -> List[TestMetadata]: cli_commit=response["Item"]["cli_commit"]["S"], cookbook_commit=response["Item"]["cookbook_commit"]["S"], node_commit=response["Item"]["node_commit"]["S"], - cfn_stack_name=response["Item"]["cfn_stack_name"]["S"], + cluster_stack_name=response["Item"]["cluster_stack_name"]["S"], cw_log_group_name=response["Item"]["cw_log_group_name"]["S"], global_build_number=int(response["Item"]["global_build_number"]["N"]), ) diff --git a/tests/integration-tests/requirements.txt b/tests/integration-tests/requirements.txt index c91a838616..5081d11bc6 100644 --- a/tests/integration-tests/requirements.txt +++ b/tests/integration-tests/requirements.txt @@ -4,12 +4,14 @@ assertpy boto3 cfn_flip click +decorator #https://github.com/fabric/fabric/issues/2204 fabric==2.6.0 filelock jinja2 jsonpickle junitparser +lexicon matplotlib pexpect psutil diff --git a/tests/integration-tests/tests/dummy/test_dummy.py b/tests/integration-tests/tests/dummy/test_dummy.py index 53a8ea3ed2..1869df2dad 100644 --- a/tests/integration-tests/tests/dummy/test_dummy.py +++ b/tests/integration-tests/tests/dummy/test_dummy.py @@ -26,3 +26,19 @@ def test_dummy( ): """Do nothing""" logging.info("Do nothing but test hooks") + + +@pytest.mark.usefixtures("os", "scheduler", "instance") +def test_dummy_cluster( + region, + scheduler, + pcluster_config_reader, + vpc_stack, + s3_bucket_factory, + test_datadir, + clusters_factory, +): + """Do nothing""" + logging.info("Do nothing but launch a cluster") + cluster_config = pcluster_config_reader() + cluster = clusters_factory(cluster_config) diff --git a/tests/integration-tests/tests/dummy/test_dummy/test_dummy_cluster/pcluster.config.yaml b/tests/integration-tests/tests/dummy/test_dummy/test_dummy_cluster/pcluster.config.yaml new file mode 100644 index 0000000000..a6977ecfab --- /dev/null +++ b/tests/integration-tests/tests/dummy/test_dummy/test_dummy_cluster/pcluster.config.yaml @@ -0,0 +1,25 @@ +Image: + Os: {{ os }} +HeadNode: + InstanceType: {{ instance }} + Networking: + SubnetId: {{ public_subnet_id }} + Ssh: + KeyName: {{ key_name }} + Imds: + Secured: {{ imds_secured }} +Scheduling: + Scheduler: {{ scheduler }} + {{ scheduler_prefix }}Queues: + - Name: ondemand1 + Networking: + SubnetIds: + - {{ private_subnet_id }} + ComputeResources: + - Name: compute-resource-11 + Instances: + - InstanceType: c5.large + - Name: compute-resource-12 + Instances: + - InstanceType: {{ instance }} + MinCount: 0