From 7baf1af6b9b08a2f5844b9346c2cd183c4313bce Mon Sep 17 00:00:00 2001 From: aurianer Date: Fri, 3 Nov 2023 16:51:09 +0100 Subject: [PATCH 1/7] Add hip configuration for CSCS CI --- .gitlab/includes/common_pipeline.yml | 3 +- .gitlab/includes/hip_pipeline.yml | 78 ++++++++++++++++++++++++++++ .gitlab/pipeline.yml | 1 + 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 .gitlab/includes/hip_pipeline.yml diff --git a/.gitlab/includes/common_pipeline.yml b/.gitlab/includes/common_pipeline.yml index 0db22e2f8..f52ff572c 100644 --- a/.gitlab/includes/common_pipeline.yml +++ b/.gitlab/includes/common_pipeline.yml @@ -27,7 +27,6 @@ variables: .test_common: variables: - SLURM_PARTITION: normal SLURM_JOB_NUM_NODES: 1 SLURM_NTASKS: 1 @@ -36,6 +35,8 @@ variables: extends: - .test_common - .container-runner-daint-mc + variables: + SLURM_PARTITION: normal .test_common_gpu_daint_cuda: extends: diff --git a/.gitlab/includes/hip_pipeline.yml b/.gitlab/includes/hip_pipeline.yml new file mode 100644 index 000000000..25a3a21a3 --- /dev/null +++ b/.gitlab/includes/hip_pipeline.yml @@ -0,0 +1,78 @@ +# Copyright (c) 2023 ETH Zurich +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file BOOST_LICENSE_1_0.rst or copy at http://www.boost.org/LICENSE_1_0.txt) + +include: + - local: '.gitlab/includes/common_pipeline.yml' + - local: '.gitlab/includes/common_spack_pipeline.yml' + +.variables_gcc12_hip5_config: + variables: + ARCH: linux-ubuntu22.04-broadwell + BUILD_TYPE: Debug + COMPILER: gcc@12.2.0 + CXXSTD: 20 + GPU_TARGET: 'gfx90a:xnack-' + SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} +rocm +stdexec amdgpu_target=${GPU_TARGET} \ + malloc=system cxxstd=$CXXSTD ^boost@1.79.0 ^hwloc@2.6.0 ^hip@5.3.3 ^fmt@10.0.0 \ + ^whip amdgpu_target=${GPU_TARGET} \ + ^stdexec@git.5e378418d9360a1a53392baae8502c4961f5f4c8=main" + +gcc12_hip5_spack_image: + stage: spack_configs + needs: [base_spack_image] + extends: + - .container-builder + - .variables_gcc12_hip5_config + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER-$SPACK_SPEC | sha256sum - | head -c 16` + - compiler=${COMPILER/@/-} + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG + - echo -e "compiler=$compiler" >> compiler.env + - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler + DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER","SPACK_SPEC"]' + artifacts: + reports: + dotenv: compiler.env + +gcc12_hip5_debug_build: + stage: build + extends: + - .container-builder + - .variables_gcc12_hip5_config + - .cmake_variables_common + needs: + - gcc12_hip5_spack_image + before_script: + - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16` + - build_type=`echo $BUILD_TYPE | tr '[:upper:]' '[:lower:]'` + - configuration=$DOCKERFILE_SHA-$BASE_IMAGE-$BUILD_DIR-$CMAKE_COMMON_FLAGS-$CMAKE_FLAGS-$CI_COMMIT_SHORT_SHA-$SOURCE_DIR + - configuration=${configuration//-D/} + - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` + - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG + - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" > build.env + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_build + CMAKE_FLAGS: "-DCMAKE_BUILD_TYPE=$BUILD_TYPE -DPIKA_WITH_CXX_STANDARD=$CXXSTD \ + -DPIKA_WITH_HIP=ON -DPIKA_WITH_MALLOC=system \ + -DCMAKE_HIP_ARCHITECTURES=$GPU_TARGET -DPIKA_WITH_STDEXEC=ON" + DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' + artifacts: + reports: + dotenv: build.env + +gcc12_hip5_debug_test: + extends: + - .variables_gcc12_hip5_config + - .test_common_gpu_clariden_hip + needs: + - gcc12_hip5_debug_build + script: + - spack arch + - spack build-env $spack_spec -- bash -c "ctest --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error" + image: $PERSIST_IMAGE_NAME diff --git a/.gitlab/pipeline.yml b/.gitlab/pipeline.yml index 0446c6138..e0fd3d06f 100644 --- a/.gitlab/pipeline.yml +++ b/.gitlab/pipeline.yml @@ -19,3 +19,4 @@ include: - local: '.gitlab/includes/clang15_pipeline.yml' - local: '.gitlab/includes/clang16_pipeline.yml' - local: '.gitlab/includes/performance_gcc13_pipeline.yml' + - local: '.gitlab/includes/hip_pipeline.yml' From 6d5033260ef290765d3852ba491da7e886b9340c Mon Sep 17 00:00:00 2001 From: aurianer Date: Wed, 15 Nov 2023 22:33:57 +0100 Subject: [PATCH 2/7] Increase the timeout for building the hip spack dependencies --- .gitlab/includes/hip_pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/includes/hip_pipeline.yml b/.gitlab/includes/hip_pipeline.yml index 25a3a21a3..8f03e97d2 100644 --- a/.gitlab/includes/hip_pipeline.yml +++ b/.gitlab/includes/hip_pipeline.yml @@ -22,6 +22,7 @@ include: gcc12_hip5_spack_image: stage: spack_configs + timeout: 4 hours needs: [base_spack_image] extends: - .container-builder From c67436247d70b8b7ef467fb3f32c2f20bdfc05a3 Mon Sep 17 00:00:00 2001 From: aurianer Date: Mon, 20 Nov 2023 12:44:41 +0100 Subject: [PATCH 3/7] Disable gold variant from hip's llvm dependency --- .gitlab/includes/hip_pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/includes/hip_pipeline.yml b/.gitlab/includes/hip_pipeline.yml index 8f03e97d2..567329c46 100644 --- a/.gitlab/includes/hip_pipeline.yml +++ b/.gitlab/includes/hip_pipeline.yml @@ -16,8 +16,8 @@ include: CXXSTD: 20 GPU_TARGET: 'gfx90a:xnack-' SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} +rocm +stdexec amdgpu_target=${GPU_TARGET} \ - malloc=system cxxstd=$CXXSTD ^boost@1.79.0 ^hwloc@2.6.0 ^hip@5.3.3 ^fmt@10.0.0 \ - ^whip amdgpu_target=${GPU_TARGET} \ + malloc=system cxxstd=$CXXSTD ^boost@1.79.0 ^hwloc@2.6.0 ^hip@5.3.3 ^llvm~gold \ + ^fmt@10.0.0 ^whip amdgpu_target=${GPU_TARGET} \ ^stdexec@git.5e378418d9360a1a53392baae8502c4961f5f4c8=main" gcc12_hip5_spack_image: From d80d9be2a16edc2a964bb59663ba3f2d177a161a Mon Sep 17 00:00:00 2001 From: aurianer Date: Tue, 21 Nov 2023 15:50:36 +0100 Subject: [PATCH 4/7] Update arch and gpu target for clariden --- .gitlab/includes/hip_pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/includes/hip_pipeline.yml b/.gitlab/includes/hip_pipeline.yml index 567329c46..30713b750 100644 --- a/.gitlab/includes/hip_pipeline.yml +++ b/.gitlab/includes/hip_pipeline.yml @@ -10,11 +10,11 @@ include: .variables_gcc12_hip5_config: variables: - ARCH: linux-ubuntu22.04-broadwell + ARCH: linux-ubuntu22.04-zen3 BUILD_TYPE: Debug COMPILER: gcc@12.2.0 CXXSTD: 20 - GPU_TARGET: 'gfx90a:xnack-' + GPU_TARGET: 'gfx90a' SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} +rocm +stdexec amdgpu_target=${GPU_TARGET} \ malloc=system cxxstd=$CXXSTD ^boost@1.79.0 ^hwloc@2.6.0 ^hip@5.3.3 ^llvm~gold \ ^fmt@10.0.0 ^whip amdgpu_target=${GPU_TARGET} \ From 4108923722b1c070af42525a69fea028bc35a506 Mon Sep 17 00:00:00 2001 From: aurianer Date: Thu, 7 Dec 2023 14:22:39 +0100 Subject: [PATCH 5/7] Remove redundant amdgpu_target, enforced by spack already --- .gitlab/includes/hip_pipeline.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab/includes/hip_pipeline.yml b/.gitlab/includes/hip_pipeline.yml index 30713b750..df09ae6c9 100644 --- a/.gitlab/includes/hip_pipeline.yml +++ b/.gitlab/includes/hip_pipeline.yml @@ -17,8 +17,7 @@ include: GPU_TARGET: 'gfx90a' SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} +rocm +stdexec amdgpu_target=${GPU_TARGET} \ malloc=system cxxstd=$CXXSTD ^boost@1.79.0 ^hwloc@2.6.0 ^hip@5.3.3 ^llvm~gold \ - ^fmt@10.0.0 ^whip amdgpu_target=${GPU_TARGET} \ - ^stdexec@git.5e378418d9360a1a53392baae8502c4961f5f4c8=main" + ^fmt@10.0.0 ^stdexec@git.5e378418d9360a1a53392baae8502c4961f5f4c8=main" gcc12_hip5_spack_image: stage: spack_configs From 957626df2b9c135a2a813112aa3babd041f3a947 Mon Sep 17 00:00:00 2001 From: aurianer Date: Mon, 4 Dec 2023 18:42:59 +0100 Subject: [PATCH 6/7] Update to hip@5.5 --- .gitlab/includes/hip_pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/includes/hip_pipeline.yml b/.gitlab/includes/hip_pipeline.yml index df09ae6c9..2f17f9c5f 100644 --- a/.gitlab/includes/hip_pipeline.yml +++ b/.gitlab/includes/hip_pipeline.yml @@ -16,7 +16,7 @@ include: CXXSTD: 20 GPU_TARGET: 'gfx90a' SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} +rocm +stdexec amdgpu_target=${GPU_TARGET} \ - malloc=system cxxstd=$CXXSTD ^boost@1.79.0 ^hwloc@2.6.0 ^hip@5.3.3 ^llvm~gold \ + malloc=system cxxstd=$CXXSTD ^boost@1.79.0 ^hwloc@2.6.0 ^hip@5.5 ^llvm~gold \ ^fmt@10.0.0 ^stdexec@git.5e378418d9360a1a53392baae8502c4961f5f4c8=main" gcc12_hip5_spack_image: From 4ffbe04f7971dbb83dbb5538265bae76fafb0b19 Mon Sep 17 00:00:00 2001 From: aurianer Date: Wed, 13 Dec 2023 17:34:56 +0100 Subject: [PATCH 7/7] Update stdexec to nvhpc-23.09.rc4 --- .gitlab/includes/hip_pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/includes/hip_pipeline.yml b/.gitlab/includes/hip_pipeline.yml index 2f17f9c5f..f8c79bb6e 100644 --- a/.gitlab/includes/hip_pipeline.yml +++ b/.gitlab/includes/hip_pipeline.yml @@ -17,7 +17,7 @@ include: GPU_TARGET: 'gfx90a' SPACK_SPEC: "pika@main arch=$ARCH %${COMPILER} +rocm +stdexec amdgpu_target=${GPU_TARGET} \ malloc=system cxxstd=$CXXSTD ^boost@1.79.0 ^hwloc@2.6.0 ^hip@5.5 ^llvm~gold \ - ^fmt@10.0.0 ^stdexec@git.5e378418d9360a1a53392baae8502c4961f5f4c8=main" + ^fmt@10.0.0 ^stdexec@git.nvhpc-23.09.rc4=main" gcc12_hip5_spack_image: stage: spack_configs