From 13c70bde77cc4aa57ba72e5064dfbd00cfdece39 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 19 Dec 2023 11:09:15 +0100 Subject: [PATCH 1/5] Report test statistics to elastic --- .gitlab/docker/Dockerfile.spack_base | 6 +- .gitlab/includes/clang11_pipeline.yml | 13 +- .gitlab/includes/clang12_pipeline.yml | 21 +-- .gitlab/includes/clang13_pipeline.yml | 13 +- .gitlab/includes/clang14_cuda11_pipeline.yml | 17 +- .gitlab/includes/clang15_pipeline.yml | 11 +- .gitlab/includes/clang16_pipeline.yml | 11 +- .gitlab/includes/gcc11_pipeline.yml | 13 +- .gitlab/includes/gcc12_cuda12_pipeline.yml | 17 +- .gitlab/includes/gcc12_pipeline.yml | 11 +- .gitlab/includes/gcc13_pipeline.yml | 13 +- .gitlab/includes/gcc9_cuda11_pipeline.yml | 13 +- .gitlab/includes/gcc9_pipeline.yml | 13 +- .gitlab/includes/hip_pipeline.yml | 13 +- .../includes/performance_gcc13_pipeline.yml | 3 +- .gitlab/scripts/collect_ctest_metrics.sh | 149 ++++++++++++++++++ 16 files changed, 263 insertions(+), 74 deletions(-) create mode 100755 .gitlab/scripts/collect_ctest_metrics.sh diff --git a/.gitlab/docker/Dockerfile.spack_base b/.gitlab/docker/Dockerfile.spack_base index 26b817067..957f28cfe 100644 --- a/.gitlab/docker/Dockerfile.spack_base +++ b/.gitlab/docker/Dockerfile.spack_base @@ -23,6 +23,7 @@ RUN apt-get update && apt-get -yqq install --no-install-recommends \ m4 \ moreutils \ openssl \ + pipx \ pkgconf \ python3 \ python3-pip \ @@ -30,7 +31,8 @@ RUN apt-get update && apt-get -yqq install --no-install-recommends \ unzip \ vim \ xz-utils \ - && rm -Rf /var/lib/apt/lists/* + && rm -Rf /var/lib/apt/lists/* && \ + pipx install yq # Install spack ARG SPACK_REPO=https://github.com/spack/spack @@ -40,6 +42,6 @@ RUN mkdir -p $SPACK_ROOT \ && curl -OL $SPACK_REPO/archive/$SPACK_COMMIT.tar.gz \ && tar -xzvf $SPACK_COMMIT.tar.gz -C /opt && rm -f $SPACK_COMMIT.tar.gz -ENV PATH $SPACK_ROOT/bin:$PATH +ENV PATH $SPACK_ROOT/bin:/root/.local/bin:$PATH RUN spack external find --scope site && spack compiler find --scope site diff --git a/.gitlab/includes/clang11_pipeline.yml b/.gitlab/includes/clang11_pipeline.yml index 7e1e66ec4..5c38a0e4e 100644 --- a/.gitlab/includes/clang11_pipeline.yml +++ b/.gitlab/includes/clang11_pipeline.yml @@ -16,6 +16,9 @@ include: CXXSTD: 17 SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} malloc=system cxxstd=$CXXSTD ^boost@1.72.0 \ ^hwloc@1.11.11" + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ + -DPIKA_WITH_MAX_CPU_COUNT=256 -DPIKA_WITH_MALLOC=system \ + -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" clang11_spack_image: stage: spack_configs @@ -52,12 +55,9 @@ clang11_debug_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ - -DPIKA_WITH_MAX_CPU_COUNT=256 -DPIKA_WITH_MALLOC=system \ - -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: @@ -67,9 +67,12 @@ clang11_debug_test: extends: - .variables_clang11_config - .test_common_mc + - .cmake_variables_common needs: - clang11_debug_build script: - spack arch - - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + - export CTEST_XML=$PWD/ctest.xml + - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT + - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/includes/clang12_pipeline.yml b/.gitlab/includes/clang12_pipeline.yml index e1c818e74..b39e330ce 100644 --- a/.gitlab/includes/clang12_pipeline.yml +++ b/.gitlab/includes/clang12_pipeline.yml @@ -16,6 +16,13 @@ include: CXXSTD: 17 SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} malloc=system cxxstd=$CXXSTD ^boost@1.79.0 \ ^hwloc@2.2.0" + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ + -DPIKA_WITH_MALLOC=system -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON \ + -DPIKA_WITH_UNITY_BUILD=ON -DPIKA_WITH_COROUTINE_COUNTERS=ON \ + -DPIKA_WITH_THREAD_IDLE_RATES=ON \ + -DPIKA_WITH_THREAD_CREATION_AND_CLEANUP_RATES=ON \ + -DPIKA_WITH_THREAD_CUMULATIVE_COUNTS=ON -DPIKA_WITH_THREAD_QUEUE_WAITTIME=ON \ + -DPIKA_WITH_THREAD_STEALING_COUNTS=ON" clang12_spack_image: stage: spack_configs @@ -52,16 +59,9 @@ clang12_debug_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ - -DPIKA_WITH_MALLOC=system -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON \ - -DPIKA_WITH_UNITY_BUILD=ON -DPIKA_WITH_COROUTINE_COUNTERS=ON \ - -DPIKA_WITH_THREAD_IDLE_RATES=ON \ - -DPIKA_WITH_THREAD_CREATION_AND_CLEANUP_RATES=ON \ - -DPIKA_WITH_THREAD_CUMULATIVE_COUNTS=ON -DPIKA_WITH_THREAD_QUEUE_WAITTIME=ON \ - -DPIKA_WITH_THREAD_STEALING_COUNTS=ON" DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: @@ -71,9 +71,12 @@ clang12_debug_test: extends: - .variables_clang12_config - .test_common_mc + - .cmake_variables_common needs: - clang12_debug_build script: - spack arch - - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + - export CTEST_XML=$PWD/ctest.xml + - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT + - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/includes/clang13_pipeline.yml b/.gitlab/includes/clang13_pipeline.yml index 169e74f8d..51179864d 100644 --- a/.gitlab/includes/clang13_pipeline.yml +++ b/.gitlab/includes/clang13_pipeline.yml @@ -17,6 +17,9 @@ include: SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} cxxflags=-stdlib=libc++ malloc=system \ cxxstd=$CXXSTD +stdexec ^boost@1.79.0 ^hwloc@2.6.0 ^fmt cxxflags=-stdlib=libc++ \ ^stdexec@git.nvhpc-23.09.rc4=main" + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ + -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DPIKA_WITH_MALLOC=system \ + -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON -DPIKA_WITH_STDEXEC=ON" clang13_spack_image: stage: spack_configs @@ -53,12 +56,9 @@ clang13_debug_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ - -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DPIKA_WITH_MALLOC=system \ - -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON -DPIKA_WITH_STDEXEC=ON" DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: @@ -68,9 +68,12 @@ clang13_debug_test: extends: - .variables_clang13_config - .test_common_mc + - .cmake_variables_common needs: - clang13_debug_build script: - spack arch - - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + - export CTEST_XML=$PWD/ctest.xml + - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT + - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/includes/clang14_cuda11_pipeline.yml b/.gitlab/includes/clang14_cuda11_pipeline.yml index a5ff47858..dc0def9a8 100644 --- a/.gitlab/includes/clang14_cuda11_pipeline.yml +++ b/.gitlab/includes/clang14_cuda11_pipeline.yml @@ -16,6 +16,11 @@ include: CXXSTD: 17 SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} +cuda malloc=system cxxstd=$CXXSTD \ ^boost@1.79.0 ^cuda@11.5.0 +allow-unsupported-compilers ^hwloc@2.7.0" + # PIKA_WITH_CXX_LAMBDA_CAPTURE_DECLTYPE is OFF to test the fallback implementation of PIKA_FORWARD. + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=20 \ + -DPIKA_WITH_CUDA=ON -DPIKA_WITH_MALLOC=system \ + -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON -DCMAKE_CUDA_COMPILER=c++ \ + -DCMAKE_CUDA_ARCHITECTURES=80 -DPIKA_WITH_CXX_LAMBDA_CAPTURE_DECLTYPE=OFF" clang14_cuda11_spack_image: stage: spack_configs @@ -53,14 +58,9 @@ clang14_cuda11_debug_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - # PIKA_WITH_CXX_LAMBDA_CAPTURE_DECLTYPE is OFF to test the fallback implementation of PIKA_FORWARD. - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=20 \ - -DPIKA_WITH_CUDA=ON -DPIKA_WITH_MALLOC=system \ - -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON -DCMAKE_CUDA_COMPILER=c++ \ - -DCMAKE_CUDA_ARCHITECTURES=80 -DPIKA_WITH_CXX_LAMBDA_CAPTURE_DECLTYPE=OFF" DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: @@ -70,9 +70,12 @@ clang14_cuda11_debug_test: extends: - .variables_clang14_cuda11_config - .test_common_gpu_clariden_cuda + - .cmake_variables_common needs: - clang14_cuda11_debug_build script: - spack arch - - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(nproc) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + - export CTEST_XML=$PWD/ctest.xml + - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT + - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/includes/clang15_pipeline.yml b/.gitlab/includes/clang15_pipeline.yml index a7a703a23..85e936114 100644 --- a/.gitlab/includes/clang15_pipeline.yml +++ b/.gitlab/includes/clang15_pipeline.yml @@ -16,6 +16,8 @@ include: CXXSTD: 20 SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} malloc=system cxxstd=$CXXSTD ^boost@1.80.0 \ ^hwloc@2.8.0" + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ + -DPIKA_WITH_MALLOC=system -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" clang15_spack_image: stage: spack_configs @@ -52,11 +54,9 @@ clang15_debug_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ - -DPIKA_WITH_MALLOC=system -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: @@ -66,9 +66,12 @@ clang15_debug_test: extends: - .variables_clang15_config - .test_common_mc + - .cmake_variables_common needs: - clang15_debug_build script: - spack arch - - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + - export CTEST_XML=$PWD/ctest.xml + - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT + - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/includes/clang16_pipeline.yml b/.gitlab/includes/clang16_pipeline.yml index e6f6e1765..bf568c738 100644 --- a/.gitlab/includes/clang16_pipeline.yml +++ b/.gitlab/includes/clang16_pipeline.yml @@ -16,6 +16,8 @@ include: CXXSTD: 23 SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} malloc=system cxxstd=$CXXSTD ^boost@1.82.0 \ ^hwloc@2.9.1" + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ + -DPIKA_WITH_MALLOC=system -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" clang16_spack_image: stage: spack_configs @@ -52,11 +54,9 @@ clang16_debug_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ - -DPIKA_WITH_MALLOC=system -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: @@ -66,9 +66,12 @@ clang16_debug_test: extends: - .variables_clang16_config - .test_common_mc + - .cmake_variables_common needs: - clang16_debug_build script: - spack arch - - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + - export CTEST_XML=$PWD/ctest.xml + - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT + - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/includes/gcc11_pipeline.yml b/.gitlab/includes/gcc11_pipeline.yml index 5b7a2c379..2b7549843 100644 --- a/.gitlab/includes/gcc11_pipeline.yml +++ b/.gitlab/includes/gcc11_pipeline.yml @@ -16,6 +16,9 @@ include: CXXSTD: 20 SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} malloc=system cxxstd=$CXXSTD ^boost@1.78.0 \ ^hwloc@2.7.0" + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ + -DPIKA_WITH_MAX_CPU_COUNT=256 -DPIKA_WITH_MALLOC=system \ + -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" gcc11_spack_image: stage: spack_configs @@ -52,12 +55,9 @@ gcc11_debug_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ - -DPIKA_WITH_MAX_CPU_COUNT=256 -DPIKA_WITH_MALLOC=system \ - -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: @@ -67,9 +67,12 @@ gcc11_debug_test: extends: - .variables_gcc11_config - .test_common_mc + - .cmake_variables_common needs: - gcc11_debug_build script: - spack arch - - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + - export CTEST_XML=$PWD/ctest.xml + - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT + - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/includes/gcc12_cuda12_pipeline.yml b/.gitlab/includes/gcc12_cuda12_pipeline.yml index 2d7c3a6e0..f9dd131bb 100644 --- a/.gitlab/includes/gcc12_cuda12_pipeline.yml +++ b/.gitlab/includes/gcc12_cuda12_pipeline.yml @@ -17,6 +17,11 @@ include: GPU_TARGET: "80" SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} +cuda cuda_arch=${GPU_TARGET} malloc=system \ cxxstd=$CXXSTD ^boost@1.82.0 ^hwloc@2.9.1 ^cuda@12.0.0 ^fmt@10.0.0" + # PIKA_WITH_CXX_LAMBDA_CAPTURE_DECLTYPE is OFF to test the fallback implementation of PIKA_FORWARD. + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ + -DPIKA_WITH_CUDA=ON -DPIKA_WITH_MALLOC=system \ + -DCMAKE_CUDA_ARCHITECTURES=$GPU_TARGET \ + -DPIKA_WITH_CXX_LAMBDA_CAPTURE_DECLTYPE=OFF" gcc12_cuda12_spack_image: stage: spack_configs @@ -53,14 +58,9 @@ gcc12_cuda12_debug_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ - -DPIKA_WITH_CUDA=ON -DPIKA_WITH_MALLOC=system \ - -DCMAKE_CUDA_ARCHITECTURES=$GPU_TARGET \ - -DPIKA_WITH_CXX_LAMBDA_CAPTURE_DECLTYPE=OFF" -# PIKA_WITH_CXX_LAMBDA_CAPTURE_DECLTYPE is OFF to test the fallback implementation of PIKA_FORWARD. DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: @@ -72,9 +72,12 @@ gcc12_cuda12_debug_build: # extends: # - .variables_gcc12_cuda12_config # - .test_common_gpu_clariden_cuda +# - .cmake_variables_common # needs: # - gcc12_cuda12_debug_build # script: # - spack arch -# - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(nproc) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" +# - export CTEST_XML=$PWD/ctest.xml +# - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT +# - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" # image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/includes/gcc12_pipeline.yml b/.gitlab/includes/gcc12_pipeline.yml index 9be668bd8..b6276325e 100644 --- a/.gitlab/includes/gcc12_pipeline.yml +++ b/.gitlab/includes/gcc12_pipeline.yml @@ -16,6 +16,8 @@ include: CXXSTD: 20 SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} malloc=system cxxstd=$CXXSTD ^boost@1.79.0 \ ^hwloc@2.7.0" + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ + -DPIKA_WITH_MALLOC=system -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" gcc12_spack_image: stage: spack_configs @@ -52,11 +54,9 @@ gcc12_debug_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ - -DPIKA_WITH_MALLOC=system -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: @@ -66,9 +66,12 @@ gcc12_debug_test: extends: - .variables_gcc12_config - .test_common_mc + - .cmake_variables_common needs: - gcc12_debug_build script: - spack arch - - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + - export CTEST_XML=$PWD/ctest.xml + - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT + - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/includes/gcc13_pipeline.yml b/.gitlab/includes/gcc13_pipeline.yml index 77767d02e..f45816257 100644 --- a/.gitlab/includes/gcc13_pipeline.yml +++ b/.gitlab/includes/gcc13_pipeline.yml @@ -17,6 +17,9 @@ include: SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} malloc=system cxxstd=$CXXSTD +stdexec \ ^boost@1.82.0 ^hwloc@2.9.1 \ ^stdexec@git.nvhpc-23.09.rc4=main" + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ + -DPIKA_WITH_MALLOC=system -DPIKA_WITH_STDEXEC=ON \ + -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" gcc13_spack_image: stage: spack_configs @@ -53,12 +56,9 @@ gcc13_debug_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ - -DPIKA_WITH_MALLOC=system -DPIKA_WITH_STDEXEC=ON \ - -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: @@ -68,9 +68,12 @@ gcc13_debug_test: extends: - .variables_gcc13_config - .test_common_mc + - .cmake_variables_common needs: - gcc13_debug_build script: - spack arch - - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + - export CTEST_XML=$PWD/ctest.xml + - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT + - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/includes/gcc9_cuda11_pipeline.yml b/.gitlab/includes/gcc9_cuda11_pipeline.yml index 907f68752..fd154aa10 100644 --- a/.gitlab/includes/gcc9_cuda11_pipeline.yml +++ b/.gitlab/includes/gcc9_cuda11_pipeline.yml @@ -16,6 +16,9 @@ include: CXXSTD: 17 SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} +cuda malloc=system cxxstd=$CXXSTD \ ^boost@1.75.0 ^hwloc@2.0.3 ^cuda@11.2.0 ^fmt@9" + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ + -DPIKA_WITH_CUDA=ON -DPIKA_WITH_MALLOC=system \ + -DPIKA_WITH_EXAMPLES_OPENMP=ON" gcc9_cuda11_spack_image: stage: spack_configs @@ -52,12 +55,9 @@ gcc9_cuda11_debug_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ - -DPIKA_WITH_CUDA=ON -DPIKA_WITH_MALLOC=system \ - -DPIKA_WITH_EXAMPLES_OPENMP=ON" DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: @@ -67,9 +67,12 @@ gcc9_cuda11_debug_test: extends: - .variables_gcc9_cuda11_config - .test_common_gpu_daint_cuda + - .cmake_variables_common needs: - gcc9_cuda11_debug_build script: - spack arch - - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(nproc) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + - export CTEST_XML=$PWD/ctest.xml + - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT + - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/includes/gcc9_pipeline.yml b/.gitlab/includes/gcc9_pipeline.yml index d88491c7c..a84a169dd 100644 --- a/.gitlab/includes/gcc9_pipeline.yml +++ b/.gitlab/includes/gcc9_pipeline.yml @@ -16,6 +16,9 @@ include: CXXSTD: 17 SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} malloc=system cxxstd=$CXXSTD \ +generic_coroutines ^boost@1.71.0 ^hwloc@1.11.5" + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ + -DPIKA_WITH_MAX_CPU_COUNT=256 -DPIKA_WITH_MALLOC=system \ + -DPIKA_WITH_BOOST_CONTEXT=ON -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" gcc9_spack_image: stage: spack_configs @@ -52,12 +55,9 @@ gcc9_debug_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ - -DPIKA_WITH_MAX_CPU_COUNT=256 -DPIKA_WITH_MALLOC=system \ - -DPIKA_WITH_BOOST_CONTEXT=ON -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: @@ -67,9 +67,12 @@ gcc9_debug_test: extends: - .variables_gcc9_config - .test_common_mc + - .cmake_variables_common needs: - gcc9_debug_build script: - spack arch - - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + - export CTEST_XML=$PWD/ctest.xml + - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT + - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/includes/hip_pipeline.yml b/.gitlab/includes/hip_pipeline.yml index f8c79bb6e..09169a188 100644 --- a/.gitlab/includes/hip_pipeline.yml +++ b/.gitlab/includes/hip_pipeline.yml @@ -18,6 +18,9 @@ include: SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} +rocm +stdexec amdgpu_target=${GPU_TARGET} \ malloc=system cxxstd=$CXXSTD ^boost@1.79.0 ^hwloc@2.6.0 ^hip@5.5 ^llvm~gold \ ^fmt@10.0.0 ^stdexec@git.nvhpc-23.09.rc4=main" + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ + -DPIKA_WITH_HIP=ON -DPIKA_WITH_MALLOC=system \ + -DCMAKE_HIP_ARCHITECTURES=$GPU_TARGET -DPIKA_WITH_STDEXEC=ON" gcc12_hip5_spack_image: stage: spack_configs @@ -55,12 +58,9 @@ gcc12_hip5_debug_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ - -DPIKA_WITH_HIP=ON -DPIKA_WITH_MALLOC=system \ - -DCMAKE_HIP_ARCHITECTURES=$GPU_TARGET -DPIKA_WITH_STDEXEC=ON" DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: @@ -70,9 +70,12 @@ gcc12_hip5_debug_test: extends: - .variables_gcc12_hip5_config - .test_common_gpu_clariden_hip + - .cmake_variables_common needs: - gcc12_hip5_debug_build script: - spack arch - - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + - export CTEST_XML=$PWD/ctest.xml + - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT + - spack build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/includes/performance_gcc13_pipeline.yml b/.gitlab/includes/performance_gcc13_pipeline.yml index 393ce1c00..78e713fc2 100644 --- a/.gitlab/includes/performance_gcc13_pipeline.yml +++ b/.gitlab/includes/performance_gcc13_pipeline.yml @@ -16,6 +16,7 @@ include: CXXSTD: 20 SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} malloc=mimalloc cxxstd=$CXXSTD \ ^boost@1.83.0 ^hwloc@2.9.1" + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD" performance_gcc13_spack_image: stage: spack_configs @@ -52,11 +53,9 @@ performance_gcc13_release_build: - configuration=${configuration//-D/} - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - - echo -e "CMAKE_FLAGS=$CMAKE_FLAGS" >> build.env - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env variables: DOCKERFILE: .gitlab/docker/Dockerfile.spack_build_performance - CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD" DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: diff --git a/.gitlab/scripts/collect_ctest_metrics.sh b/.gitlab/scripts/collect_ctest_metrics.sh new file mode 100755 index 000000000..4f6bc03c5 --- /dev/null +++ b/.gitlab/scripts/collect_ctest_metrics.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash + +# Copyright (c) 2023 ETH Zurich +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +set -euo pipefail + +ctest_xml="${1}" + +function submit_logstash { + echo Submitting to logstash: + jq . "${1}" + + curl \ + --request POST \ + --header "Content-Type: application/json" \ + --data "@${1}" \ + "${CSCS_LOGSTASH_URL}" +} + +function json_merge { + # Merge json files according to + # https://stackoverflow.com/questions/19529688/how-to-merge-2-json-objects-from-2-files-using-jq + # + # --slurp adds all the objects from different files into an array, and add merges the objects + # --sort-keys is used only to always have the keys in the same order + echo $(jq --slurp --sort-keys add "${1}" "${2}") >"${3}" +} + +function json_add_value { + file=${1} + key=${2} + value=${3} + + jq --arg value "${value}" ".${key} += \$value" "${file}" | sponge "${file}" +} + +function json_add_value_json { + file=${1} + key=${2} + value=${3} + + jq --argjson value "${value}" ".${key} += \$value" "${file}" | sponge "${file}" +} + +function json_add_from_env { + file=${1} + key=${2} + + for var in ${@:3}; do + jq --arg value "${!var:-}" ".${key}.${var} += \$value" "${file}" | sponge "${file}" + done +} + +function json_add_from_command { + file=${1} + key=${2} + + for cmd in ${@:3}; do + jq --arg value "$(${cmd})" ".${key}.${cmd} += \$value" "${file}" | sponge "${file}" + done +} + +metadata_file=$(mktemp --tmpdir metadata.XXXXXXXXXX.json) +echo '{}' >"${metadata_file}" + +# Logstash data stream metadata section +json_add_value "${metadata_file}" "data_stream.type" "logs" +json_add_value "${metadata_file}" "data_stream.dataset" "service.pika" +json_add_value "${metadata_file}" "data_stream.namespace" "alps" + +# CI/git metadata section +json_add_value "${metadata_file}" "ci.organization" "pika-org" +json_add_value "${metadata_file}" "ci.repository" "pika" +json_add_from_env \ + "${metadata_file}" "ci" \ + CI_COMMIT_AUTHOR \ + CI_COMMIT_BRANCH \ + CI_COMMIT_DESCRIPTION \ + CI_COMMIT_MESSAGE \ + CI_COMMIT_SHA \ + CI_COMMIT_SHORT_SHA \ + CI_COMMIT_TIMESTAMP \ + CI_COMMIT_TITLE \ + CI_JOB_IMAGE + +# System section +json_add_from_command "${metadata_file}" "system" "hostname" + +# Slurm section +json_add_from_env \ + "${metadata_file}" "slurm" \ + SLURM_CLUSTER_NAME \ + SLURM_CPUS_ON_NODE \ + SLURM_CPU_BIND \ + SLURM_JOBID \ + SLURM_JOB_NAME \ + SLURM_JOB_NODELIST \ + SLURM_JOB_NUM_NODES \ + SLURM_JOB_PARTITION \ + SLURM_NODELIST \ + SLURM_NTASKS \ + SLURM_TASKS_PER_NODE + +# Build configuration section +json_add_from_env \ + "${metadata_file}" "build_configuration" \ + ARCH \ + BUILD_TYPE \ + CMAKE_COMMON_FLAGS \ + CMAKE_FLAGS \ + COMPILER \ + SPACK_COMMIT \ + SPACK_SPEC + +# Submit individual test data +num_tests=$(xq . "${ctest_xml}" | jq '.testsuite.testcase | length') +for i in $(seq 1 ${num_tests}); do + result_file=$(mktemp --tmpdir "ctest_${i}.XXXXXXXXXX.json") + echo '{}' >"${result_file}" + + # Extract a single test case from the XML output and convert the time from a string to a number. + ctest_object=$(xq . "${ctest_xml}" | jq ".testsuite.testcase[$((i - 1))]" | jq '."@time" |= tonumber') + json_add_value_json "${result_file}" "ctest.testcase" "${ctest_object}" + + json_merge "${metadata_file}" "${result_file}" "${result_file}" + submit_logstash "${result_file}" +done + +result_file=$(mktemp --tmpdir "ctest.XXXXXXXXXX.json") +echo '{}' >"${result_file}" + +# Submit overall ctest data. Convert numeric fields from strings to numbers. +ctest_object=$( + xq . "${ctest_xml}" | + jq 'del(.testsuite.testcase)' | + jq '.testsuite."@disabled" |= tonumber' | + jq '.testsuite."@failures" |= tonumber' | + jq '.testsuite."@skipped" |= tonumber' | + jq '.testsuite."@tests" |= tonumber' | + jq '.testsuite."@time" |= tonumber' +) +json_add_value_json "${result_file}" "ctest" "${ctest_object}" + +json_merge "${metadata_file}" "${result_file}" "${result_file}" +submit_logstash "${result_file}" From 0c349b281fcb6fc1752df2c9ca7c98787d77997d Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 5 Jan 2024 08:42:02 +0100 Subject: [PATCH 2/5] Move json utilities used for elastic submission to separate file --- .gitlab/scripts/collect_ctest_metrics.sh | 55 +---------------- .gitlab/scripts/json_utilities.sh | 59 +++++++++++++++++++ .gitlab/scripts/run_performance_benchmarks.sh | 58 ++---------------- 3 files changed, 65 insertions(+), 107 deletions(-) create mode 100644 .gitlab/scripts/json_utilities.sh diff --git a/.gitlab/scripts/collect_ctest_metrics.sh b/.gitlab/scripts/collect_ctest_metrics.sh index 4f6bc03c5..f64d10d39 100755 --- a/.gitlab/scripts/collect_ctest_metrics.sh +++ b/.gitlab/scripts/collect_ctest_metrics.sh @@ -9,60 +9,9 @@ set -euo pipefail ctest_xml="${1}" +current_dir=$(dirname -- "${BASH_SOURCE[0]}") -function submit_logstash { - echo Submitting to logstash: - jq . "${1}" - - curl \ - --request POST \ - --header "Content-Type: application/json" \ - --data "@${1}" \ - "${CSCS_LOGSTASH_URL}" -} - -function json_merge { - # Merge json files according to - # https://stackoverflow.com/questions/19529688/how-to-merge-2-json-objects-from-2-files-using-jq - # - # --slurp adds all the objects from different files into an array, and add merges the objects - # --sort-keys is used only to always have the keys in the same order - echo $(jq --slurp --sort-keys add "${1}" "${2}") >"${3}" -} - -function json_add_value { - file=${1} - key=${2} - value=${3} - - jq --arg value "${value}" ".${key} += \$value" "${file}" | sponge "${file}" -} - -function json_add_value_json { - file=${1} - key=${2} - value=${3} - - jq --argjson value "${value}" ".${key} += \$value" "${file}" | sponge "${file}" -} - -function json_add_from_env { - file=${1} - key=${2} - - for var in ${@:3}; do - jq --arg value "${!var:-}" ".${key}.${var} += \$value" "${file}" | sponge "${file}" - done -} - -function json_add_from_command { - file=${1} - key=${2} - - for cmd in ${@:3}; do - jq --arg value "$(${cmd})" ".${key}.${cmd} += \$value" "${file}" | sponge "${file}" - done -} +source "${current_dir}/json_utilities.sh" metadata_file=$(mktemp --tmpdir metadata.XXXXXXXXXX.json) echo '{}' >"${metadata_file}" diff --git a/.gitlab/scripts/json_utilities.sh b/.gitlab/scripts/json_utilities.sh new file mode 100644 index 000000000..45d325dce --- /dev/null +++ b/.gitlab/scripts/json_utilities.sh @@ -0,0 +1,59 @@ +# Copyright (c) 2023 ETH Zurich +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +function submit_logstash { + echo Submitting to logstash: + jq . "${1}" + + curl \ + --request POST \ + --header "Content-Type: application/json" \ + --data "@${1}" \ + "${CSCS_LOGSTASH_URL}" +} + +function json_merge { + # Merge json files according to + # https://stackoverflow.com/questions/19529688/how-to-merge-2-json-objects-from-2-files-using-jq + # + # --slurp adds all the objects from different files into an array, and add merges the objects + # --sort-keys is used only to always have the keys in the same order + echo $(jq --slurp --sort-keys add "${1}" "${2}") >"${3}" +} + +function json_add_value { + file=${1} + key=${2} + value=${3} + + jq --arg value "${value}" ".${key} += \$value" "${file}" | sponge "${file}" +} + +function json_add_value_json { + file=${1} + key=${2} + value=${3} + + jq --argjson value "${value}" ".${key} += \$value" "${file}" | sponge "${file}" +} + +function json_add_from_env { + file=${1} + key=${2} + + for var in ${@:3}; do + jq --arg value "${!var:-}" ".${key}.${var} += \$value" "${file}" | sponge "${file}" + done +} + +function json_add_from_command { + file=${1} + key=${2} + + for cmd in ${@:3}; do + jq --arg value "$(${cmd})" ".${key}.${cmd} += \$value" "${file}" | sponge "${file}" + done +} diff --git a/.gitlab/scripts/run_performance_benchmarks.sh b/.gitlab/scripts/run_performance_benchmarks.sh index 7680f1d5c..5e95bebfd 100755 --- a/.gitlab/scripts/run_performance_benchmarks.sh +++ b/.gitlab/scripts/run_performance_benchmarks.sh @@ -8,59 +8,9 @@ set -euo pipefail -function submit_performance_result { - echo Submitting performance result to logstash: - jq . "${1}" - - curl \ - --request POST \ - --header "Content-Type: application/json" \ - --data "@${1}" \ - "${CSCS_LOGSTASH_URL}" -} - -function json_merge { - # Merge json files according to - # https://stackoverflow.com/questions/19529688/how-to-merge-2-json-objects-from-2-files-using-jq - # - # --slurp adds all the objects from different files into an array, and add merges the objects - # --sort-keys is used only to always have the keys in the same order - echo $(jq --slurp --sort-keys add "${1}" "${2}") > "${3}" -} - -function json_add_value { - file=${1} - key=${2} - value=${3} - - jq --arg value "${value}" ".${key} += \$value" "${file}" | sponge "${file}" -} - -function json_add_value_json { - file=${1} - key=${2} - value=${3} - - jq --argjson value "${value}" ".${key} += \$value" "${file}" | sponge "${file}" -} - -function json_add_from_env { - file=${1} - key=${2} - - for var in ${@:3}; do - jq --arg value "${!var:-}" ".${key}.${var} += \$value" "${file}" | sponge "${file}" - done -} - -function json_add_from_command { - file=${1} - key=${2} - - for cmd in ${@:3}; do - jq --arg value "$(${cmd})" ".${key}.${cmd} += \$value" "${file}" | sponge "${file}" - done -} +current_dir=$(dirname -- "${BASH_SOURCE[0]}") + +source "${current_dir}/json_utilities.sh" metadata_file=$(mktemp --tmpdir metadata.XXXXXXXXXX.json) echo '{}' > "${metadata_file}" @@ -165,7 +115,7 @@ for executable in "${pika_targets[@]}"; do json_add_value_json "${result_file}" "metric.benchmark.series" "${benchmark_series}" json_merge "${metadata_file}" "${result_file}" "${result_file}" - submit_performance_result "${result_file}" + submit_logstash "${result_file}" index=$((index + 1)) done From c407c110b4635e4ac98f222a098d56ec329ae6bd Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 5 Jan 2024 08:50:07 +0100 Subject: [PATCH 3/5] Move more elastic data collection utilities into separate file --- .gitlab/scripts/collect_ctest_metrics.sh | 51 +----------------- .gitlab/scripts/json_utilities.sh | 54 +++++++++++++++++++ .gitlab/scripts/run_performance_benchmarks.sh | 51 +----------------- 3 files changed, 56 insertions(+), 100 deletions(-) diff --git a/.gitlab/scripts/collect_ctest_metrics.sh b/.gitlab/scripts/collect_ctest_metrics.sh index f64d10d39..a703567a2 100755 --- a/.gitlab/scripts/collect_ctest_metrics.sh +++ b/.gitlab/scripts/collect_ctest_metrics.sh @@ -14,56 +14,7 @@ current_dir=$(dirname -- "${BASH_SOURCE[0]}") source "${current_dir}/json_utilities.sh" metadata_file=$(mktemp --tmpdir metadata.XXXXXXXXXX.json) -echo '{}' >"${metadata_file}" - -# Logstash data stream metadata section -json_add_value "${metadata_file}" "data_stream.type" "logs" -json_add_value "${metadata_file}" "data_stream.dataset" "service.pika" -json_add_value "${metadata_file}" "data_stream.namespace" "alps" - -# CI/git metadata section -json_add_value "${metadata_file}" "ci.organization" "pika-org" -json_add_value "${metadata_file}" "ci.repository" "pika" -json_add_from_env \ - "${metadata_file}" "ci" \ - CI_COMMIT_AUTHOR \ - CI_COMMIT_BRANCH \ - CI_COMMIT_DESCRIPTION \ - CI_COMMIT_MESSAGE \ - CI_COMMIT_SHA \ - CI_COMMIT_SHORT_SHA \ - CI_COMMIT_TIMESTAMP \ - CI_COMMIT_TITLE \ - CI_JOB_IMAGE - -# System section -json_add_from_command "${metadata_file}" "system" "hostname" - -# Slurm section -json_add_from_env \ - "${metadata_file}" "slurm" \ - SLURM_CLUSTER_NAME \ - SLURM_CPUS_ON_NODE \ - SLURM_CPU_BIND \ - SLURM_JOBID \ - SLURM_JOB_NAME \ - SLURM_JOB_NODELIST \ - SLURM_JOB_NUM_NODES \ - SLURM_JOB_PARTITION \ - SLURM_NODELIST \ - SLURM_NTASKS \ - SLURM_TASKS_PER_NODE - -# Build configuration section -json_add_from_env \ - "${metadata_file}" "build_configuration" \ - ARCH \ - BUILD_TYPE \ - CMAKE_COMMON_FLAGS \ - CMAKE_FLAGS \ - COMPILER \ - SPACK_COMMIT \ - SPACK_SPEC +create_metadata_file "${metadata_file}" # Submit individual test data num_tests=$(xq . "${ctest_xml}" | jq '.testsuite.testcase | length') diff --git a/.gitlab/scripts/json_utilities.sh b/.gitlab/scripts/json_utilities.sh index 45d325dce..c35c23464 100644 --- a/.gitlab/scripts/json_utilities.sh +++ b/.gitlab/scripts/json_utilities.sh @@ -57,3 +57,57 @@ function json_add_from_command { jq --arg value "$(${cmd})" ".${key}.${cmd} += \$value" "${file}" | sponge "${file}" done } + +function create_metadata_file { + metadata_file=${1} + echo '{}' >"${metadata_file}" + + # Logstash data stream metadata section + json_add_value "${metadata_file}" "data_stream.type" "logs" + json_add_value "${metadata_file}" "data_stream.dataset" "service.pika" + json_add_value "${metadata_file}" "data_stream.namespace" "alps" + + # CI/git metadata section + json_add_value "${metadata_file}" "ci.organization" "pika-org" + json_add_value "${metadata_file}" "ci.repository" "pika" + json_add_from_env \ + "${metadata_file}" "ci" \ + CI_COMMIT_AUTHOR \ + CI_COMMIT_BRANCH \ + CI_COMMIT_DESCRIPTION \ + CI_COMMIT_MESSAGE \ + CI_COMMIT_SHA \ + CI_COMMIT_SHORT_SHA \ + CI_COMMIT_TIMESTAMP \ + CI_COMMIT_TITLE \ + CI_JOB_IMAGE + + # System section + json_add_from_command "${metadata_file}" "system" "hostname" + + # Slurm section + json_add_from_env \ + "${metadata_file}" "slurm" \ + SLURM_CLUSTER_NAME \ + SLURM_CPUS_ON_NODE \ + SLURM_CPU_BIND \ + SLURM_JOBID \ + SLURM_JOB_NAME \ + SLURM_JOB_NODELIST \ + SLURM_JOB_NUM_NODES \ + SLURM_JOB_PARTITION \ + SLURM_NODELIST \ + SLURM_NTASKS \ + SLURM_TASKS_PER_NODE + + # Build configuration section + json_add_from_env \ + "${metadata_file}" "build_configuration" \ + ARCH \ + BUILD_TYPE \ + CMAKE_COMMON_FLAGS \ + CMAKE_FLAGS \ + COMPILER \ + SPACK_COMMIT \ + SPACK_SPEC +} diff --git a/.gitlab/scripts/run_performance_benchmarks.sh b/.gitlab/scripts/run_performance_benchmarks.sh index 5e95bebfd..e9d43234c 100755 --- a/.gitlab/scripts/run_performance_benchmarks.sh +++ b/.gitlab/scripts/run_performance_benchmarks.sh @@ -13,56 +13,7 @@ current_dir=$(dirname -- "${BASH_SOURCE[0]}") source "${current_dir}/json_utilities.sh" metadata_file=$(mktemp --tmpdir metadata.XXXXXXXXXX.json) -echo '{}' > "${metadata_file}" - -# Logstash data stream metadata section -json_add_value "${metadata_file}" "data_stream.type" "logs" -json_add_value "${metadata_file}" "data_stream.dataset" "service.pika" -json_add_value "${metadata_file}" "data_stream.namespace" "alps" - -# CI/git metadata section -json_add_value "${metadata_file}" "ci.organization" "pika-org" -json_add_value "${metadata_file}" "ci.repository" "pika" -json_add_from_env \ - "${metadata_file}" "ci" \ - CI_COMMIT_AUTHOR \ - CI_COMMIT_BRANCH \ - CI_COMMIT_DESCRIPTION \ - CI_COMMIT_MESSAGE \ - CI_COMMIT_SHA \ - CI_COMMIT_SHORT_SHA \ - CI_COMMIT_TIMESTAMP \ - CI_COMMIT_TITLE \ - CI_JOB_IMAGE - -# System section -json_add_from_command "${metadata_file}" "system" "hostname" - -# Slurm section -json_add_from_env \ - "${metadata_file}" "slurm" \ - SLURM_CLUSTER_NAME \ - SLURM_CPUS_ON_NODE \ - SLURM_CPU_BIND \ - SLURM_JOBID \ - SLURM_JOB_NAME \ - SLURM_JOB_NODELIST \ - SLURM_JOB_NUM_NODES \ - SLURM_JOB_PARTITION \ - SLURM_NODELIST \ - SLURM_NTASKS \ - SLURM_TASKS_PER_NODE - -# Build configuration section -json_add_from_env \ - "${metadata_file}" "build_configuration" \ - ARCH \ - BUILD_TYPE \ - CMAKE_COMMON_FLAGS \ - CMAKE_FLAGS \ - COMPILER \ - SPACK_COMMIT \ - SPACK_SPEC +create_metadata_file "${metadata_file}" pika_targets=( "task_overhead_report_test" From ebf4a4ae3cfad692dda2ba2c17d856a158fbcdab Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 5 Jan 2024 08:53:38 +0100 Subject: [PATCH 4/5] Rename elastic utilities file to something more generic --- .gitlab/scripts/collect_ctest_metrics.sh | 2 +- .gitlab/scripts/run_performance_benchmarks.sh | 2 +- .gitlab/scripts/{json_utilities.sh => utilities.sh} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename .gitlab/scripts/{json_utilities.sh => utilities.sh} (100%) diff --git a/.gitlab/scripts/collect_ctest_metrics.sh b/.gitlab/scripts/collect_ctest_metrics.sh index a703567a2..221d06e83 100755 --- a/.gitlab/scripts/collect_ctest_metrics.sh +++ b/.gitlab/scripts/collect_ctest_metrics.sh @@ -11,7 +11,7 @@ set -euo pipefail ctest_xml="${1}" current_dir=$(dirname -- "${BASH_SOURCE[0]}") -source "${current_dir}/json_utilities.sh" +source "${current_dir}/utilities.sh" metadata_file=$(mktemp --tmpdir metadata.XXXXXXXXXX.json) create_metadata_file "${metadata_file}" diff --git a/.gitlab/scripts/run_performance_benchmarks.sh b/.gitlab/scripts/run_performance_benchmarks.sh index e9d43234c..9ed62c0d8 100755 --- a/.gitlab/scripts/run_performance_benchmarks.sh +++ b/.gitlab/scripts/run_performance_benchmarks.sh @@ -10,7 +10,7 @@ set -euo pipefail current_dir=$(dirname -- "${BASH_SOURCE[0]}") -source "${current_dir}/json_utilities.sh" +source "${current_dir}/utilities.sh" metadata_file=$(mktemp --tmpdir metadata.XXXXXXXXXX.json) create_metadata_file "${metadata_file}" diff --git a/.gitlab/scripts/json_utilities.sh b/.gitlab/scripts/utilities.sh similarity index 100% rename from .gitlab/scripts/json_utilities.sh rename to .gitlab/scripts/utilities.sh From 8ae3a849724e0b37803604c06da5bac5339ac643 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 5 Jan 2024 08:55:35 +0100 Subject: [PATCH 5/5] Quote variables in elastic utilities script --- .gitlab/scripts/utilities.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.gitlab/scripts/utilities.sh b/.gitlab/scripts/utilities.sh index c35c23464..06d2076aa 100644 --- a/.gitlab/scripts/utilities.sh +++ b/.gitlab/scripts/utilities.sh @@ -25,24 +25,24 @@ function json_merge { } function json_add_value { - file=${1} - key=${2} - value=${3} + file="${1}" + key="${2}" + value="${3}" jq --arg value "${value}" ".${key} += \$value" "${file}" | sponge "${file}" } function json_add_value_json { - file=${1} - key=${2} - value=${3} + file="${1}" + key="${2}" + value="${3}" jq --argjson value "${value}" ".${key} += \$value" "${file}" | sponge "${file}" } function json_add_from_env { - file=${1} - key=${2} + file="${1}" + key="${2}" for var in ${@:3}; do jq --arg value "${!var:-}" ".${key}.${var} += \$value" "${file}" | sponge "${file}" @@ -50,8 +50,8 @@ function json_add_from_env { } function json_add_from_command { - file=${1} - key=${2} + file="${1}" + key="${2}" for cmd in ${@:3}; do jq --arg value "$(${cmd})" ".${key}.${cmd} += \$value" "${file}" | sponge "${file}" @@ -59,7 +59,7 @@ function json_add_from_command { } function create_metadata_file { - metadata_file=${1} + metadata_file="${1}" echo '{}' >"${metadata_file}" # Logstash data stream metadata section