From b3b8268452117917afb903343c67f4150a8445dd Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 19 Dec 2023 11:09:15 +0100 Subject: [PATCH] Report test statistics to elastic --- .gitlab/docker/Dockerfile.spack_base | 8 +- .gitlab/includes/clang14_cuda11_pipeline.yml | 5 +- .gitlab/pipeline.yml | 24 ++-- .gitlab/scripts/collect_ctest_metrics.sh | 140 +++++++++++++++++++ 4 files changed, 162 insertions(+), 15 deletions(-) create mode 100755 .gitlab/scripts/collect_ctest_metrics.sh diff --git a/.gitlab/docker/Dockerfile.spack_base b/.gitlab/docker/Dockerfile.spack_base index c429af4ac2..957f28cfe7 100644 --- a/.gitlab/docker/Dockerfile.spack_base +++ b/.gitlab/docker/Dockerfile.spack_base @@ -17,10 +17,13 @@ RUN apt-get update && apt-get -yqq install --no-install-recommends \ gcc \ git \ gnupg2 \ + jq \ libtool \ locales \ m4 \ + moreutils \ openssl \ + pipx \ pkgconf \ python3 \ python3-pip \ @@ -28,7 +31,8 @@ RUN apt-get update && apt-get -yqq install --no-install-recommends \ unzip \ vim \ xz-utils \ - && rm -Rf /var/lib/apt/lists/* + && rm -Rf /var/lib/apt/lists/* && \ + pipx install yq # Install spack ARG SPACK_REPO=https://github.com/spack/spack @@ -38,6 +42,6 @@ RUN mkdir -p $SPACK_ROOT \ && curl -OL $SPACK_REPO/archive/$SPACK_COMMIT.tar.gz \ && tar -xzvf $SPACK_COMMIT.tar.gz -C /opt && rm -f $SPACK_COMMIT.tar.gz -ENV PATH $SPACK_ROOT/bin:$PATH +ENV PATH $SPACK_ROOT/bin:/root/.local/bin:$PATH RUN spack external find --scope site && spack compiler find --scope site diff --git a/.gitlab/includes/clang14_cuda11_pipeline.yml b/.gitlab/includes/clang14_cuda11_pipeline.yml index 0806cf893a..f7e97145ef 100644 --- a/.gitlab/includes/clang14_cuda11_pipeline.yml +++ b/.gitlab/includes/clang14_cuda11_pipeline.yml @@ -73,9 +73,12 @@ clang14_cuda11_debug_test: extends: - .variables_clang14_cuda11_config - .test_common_gpu_clariden_cuda + - .cmake_variables_common needs: - clang14_cuda11_debug_build script: - spack arch - - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(nproc) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + - export CTEST_XML=$PWD/ctest.xml + - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT + - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(nproc) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/pipeline.yml b/.gitlab/pipeline.yml index e33b22ccc0..c7a4ed69b2 100644 --- a/.gitlab/pipeline.yml +++ b/.gitlab/pipeline.yml @@ -5,16 +5,16 @@ # file BOOST_LICENSE_1_0.rst or copy at http://www.boost.org/LICENSE_1_0.txt) include: - - local: '.gitlab/includes/dockerhub_pipeline.yml' - - local: '.gitlab/includes/gcc9_pipeline.yml' - - local: '.gitlab/includes/gcc9_cuda11_pipeline.yml' - - local: '.gitlab/includes/gcc11_pipeline.yml' - - local: '.gitlab/includes/gcc12_pipeline.yml' - - local: '.gitlab/includes/gcc12_cuda12_pipeline.yml' - - local: '.gitlab/includes/gcc13_pipeline.yml' - - local: '.gitlab/includes/clang11_pipeline.yml' - - local: '.gitlab/includes/clang12_pipeline.yml' - - local: '.gitlab/includes/clang13_pipeline.yml' + # - local: '.gitlab/includes/dockerhub_pipeline.yml' + # - local: '.gitlab/includes/gcc9_pipeline.yml' + # - local: '.gitlab/includes/gcc9_cuda11_pipeline.yml' + # - local: '.gitlab/includes/gcc11_pipeline.yml' + # - local: '.gitlab/includes/gcc12_pipeline.yml' + # - local: '.gitlab/includes/gcc12_cuda12_pipeline.yml' + # - local: '.gitlab/includes/gcc13_pipeline.yml' + # - local: '.gitlab/includes/clang11_pipeline.yml' + # - local: '.gitlab/includes/clang12_pipeline.yml' + # - local: '.gitlab/includes/clang13_pipeline.yml' - local: '.gitlab/includes/clang14_cuda11_pipeline.yml' - - local: '.gitlab/includes/clang15_pipeline.yml' - - local: '.gitlab/includes/clang16_pipeline.yml' + # - local: '.gitlab/includes/clang15_pipeline.yml' + # - local: '.gitlab/includes/clang16_pipeline.yml' diff --git a/.gitlab/scripts/collect_ctest_metrics.sh b/.gitlab/scripts/collect_ctest_metrics.sh new file mode 100755 index 0000000000..68b0a48df7 --- /dev/null +++ b/.gitlab/scripts/collect_ctest_metrics.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash + +# Copyright (c) 2023 ETH Zurich +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +set -euo pipefail + +ctest_xml="${1}" + +function submit_logstash { + echo Submitting to logstash: + jq . "${1}" + + curl \ + --request POST \ + --header "Content-Type: application/json" \ + --data "@${1}" \ + "${CSCS_LOGSTASH_URL}" +} + +function json_merge { + # Merge json files according to + # https://stackoverflow.com/questions/19529688/how-to-merge-2-json-objects-from-2-files-using-jq + # + # --slurp adds all the objects from different files into an array, and add merges the objects + # --sort-keys is used only to always have the keys in the same order + echo $(jq --slurp --sort-keys add "${1}" "${2}") > "${3}" +} + +function json_add_value { + file=${1} + key=${2} + value=${3} + + jq --arg value "${value}" ".${key} += \$value" "${file}" | sponge "${file}" +} + +function json_add_value_json { + file=${1} + key=${2} + value=${3} + + jq --argjson value "${value}" ".${key} += \$value" "${file}" | sponge "${file}" +} + +function json_add_from_env { + file=${1} + key=${2} + + for var in ${@:3}; do + jq --arg value "${!var:-}" ".${key}.${var} += \$value" "${file}" | sponge "${file}" + done +} + +function json_add_from_command { + file=${1} + key=${2} + + for cmd in ${@:3}; do + jq --arg value "$(${cmd})" ".${key}.${cmd} += \$value" "${file}" | sponge "${file}" + done +} + +metadata_file=$(mktemp --tmpdir metadata.XXXXXXXXXX.json) +echo '{}' > "${metadata_file}" + +# Logstash data stream metadata section +json_add_value "${metadata_file}" "data_stream.type" "logs" +json_add_value "${metadata_file}" "data_stream.dataset" "telemetry.pika" +json_add_value "${metadata_file}" "data_stream.namespace" "test5" + +# CI/git metadata section +json_add_value "${metadata_file}" "ci.organization" "pika-org" +json_add_value "${metadata_file}" "ci.repository" "pika" +json_add_from_env \ + "${metadata_file}" "ci" \ + CI_COMMIT_AUTHOR \ + CI_COMMIT_BRANCH \ + CI_COMMIT_DESCRIPTION \ + CI_COMMIT_MESSAGE \ + CI_COMMIT_SHA \ + CI_COMMIT_SHORT_SHA \ + CI_COMMIT_TIMESTAMP \ + CI_COMMIT_TITLE \ + CI_JOB_IMAGE + +# System section +json_add_from_command "${metadata_file}" "system" "hostname" + +# Slurm section +json_add_from_env \ + "${metadata_file}" "slurm" \ + SLURM_CLUSTER_NAME \ + SLURM_CPUS_ON_NODE \ + SLURM_CPU_BIND \ + SLURM_JOBID \ + SLURM_JOB_NAME \ + SLURM_JOB_NODELIST \ + SLURM_JOB_NUM_NODES \ + SLURM_JOB_PARTITION \ + SLURM_NODELIST \ + SLURM_NTASKS \ + SLURM_TASKS_PER_NODE + +# Build configuration section +json_add_from_env \ + "${metadata_file}" "build_configuration" \ + ARCH \ + BUILD_TYPE \ + CMAKE_COMMON_FLAGS \ + CMAKE_FLAGS \ + COMPILER \ + SPACK_COMMIT \ + SPACK_SPEC + +# Submit individual test data +num_tests=$(xq . "${ctest_xml}" | jq '.testsuite.testcase | length') +for i in $(seq 1 ${num_tests}); do + result_file=$(mktemp --tmpdir "ctest_${i}.XXXXXXXXXX.json") + echo '{}' > "${result_file}" + + ctest_object=$(xq . "${ctest_xml}" | jq ".testsuite.testcase[$((i - 1))]") + json_add_value_json "${result_file}" "metric.ctest.testcase" "${ctest_object}" + + json_merge "${metadata_file}" "${result_file}" "${result_file}" + submit_logstash "${result_file}" +done + +result_file=$(mktemp --tmpdir "ctest.XXXXXXXXXX.json") +echo '{}' > "${result_file}" + +# Submit overall ctest data +ctest_object=$(xq . "${ctest_xml}" | jq "del(.testsuite.testcase)") +json_add_value_json "${result_file}" "metric.ctest.testsuite" "${ctest_object}" + +json_merge "${metadata_file}" "${result_file}" "${result_file}" +submit_logstash "${result_file}"