Enable hardening flags on Linux

.github/workflows/cmake.yml

@@ -94,6 +94,8 @@ jobs:
         -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
         -DUR_BUILD_TESTS=ON
         -DUR_FORMAT_CPP_STYLE=ON
+        -DUR_WEXTRA=ON
+        -DUR_HARDEN=ON
         -DUR_DPCXX=${{github.workspace}}/dpcpp_compiler/bin/clang++
         ${{matrix.libbacktrace}}
         ${{matrix.pool_tracking}}
@@ -110,6 +112,8 @@ jobs:
         -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
         -DUR_BUILD_TESTS=ON
         -DUR_FORMAT_CPP_STYLE=ON
+        -DUR_WEXTRA=ON
+        -DUR_HARDEN=OFF
         ${{matrix.libbacktrace}}
         ${{matrix.pool_tracking}}
         ${{matrix.latency_tracking}}
@@ -121,6 +125,9 @@ jobs:
     - name: Verify that each source file contains a license
       run: cmake --build ${{github.workspace}}/build --target verify-licenses
 
+    - name: Verify hardening flags have been set
+      run: cmake --build ${{github.workspace}}/build --target verify-licenses
+
     - name: Build
       run: cmake --build ${{github.workspace}}/build -j $(nproc)
 

CMakeLists.txt

@@ -60,6 +60,8 @@ option(UR_BUILD_XPTI_LIBS "Build the XPTI libraries when tracing is enabled" ON)
 option(UR_STATIC_LOADER "Build loader as a static library" OFF)
 option(UR_FORCE_LIBSTDCXX "Force use of libstdc++ in a build using libc++ on Linux" OFF)
 option(UR_ENABLE_LATENCY_HISTOGRAM "Enable latncy histogram" OFF)
+option(UR_WEXTRA "Enable -Wextra on all build targets" OFF)
+option(UR_HARDEN "Enable additional hardening flags" ON)
 set(UR_DPCXX "" CACHE FILEPATH "Path of the DPC++ compiler executable")
 set(UR_DPCXX_BUILD_FLAGS "" CACHE STRING "Build flags to pass to DPC++ when compiling device programs")
 set(UR_SYCL_LIBRARY_DIR "" CACHE PATH
@@ -160,6 +162,8 @@ if(UR_ENABLE_TRACING)
         set_target_properties(xptifw PROPERTIES
             LIBRARY_OUTPUT_DIRECTORY ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}
         )
+        add_ur_target_compile_options(xptifw)
+        add_ur_target_link_options(xptifw)
 
         if (UR_STATIC_LOADER)
             install(TARGETS xpti xptifw
@@ -269,6 +273,14 @@ add_custom_target(verify-licenses
     COMMENT "Verify all files contain a license."
 )
 
+# Add hardening check
+list(FILTER license_src EXCLUDE REGEX "registry.yml")
+add_custom_target(verify-hardening
+    COMMAND "${PROJECT_SOURCE_DIR}/scripts/check-hardening.sh"
+        ${CMAKE_BINARY_DIR}
+    COMMENT "Check hardening settings on built binaries and libraries"
+)
+
 # Add code formatter target
 add_custom_target(cppformat)
 # ... and all source files to the formatter

cmake/helpers.cmake

@@ -42,6 +42,10 @@ endfunction()
 
 include(CheckCXXCompilerFlag)
 
+if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0)
+    set(OLD_GCC_VERSION ON)
+endif()
+
 macro(add_sanitizer_flag flag)
     set(SAVED_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
     set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES} -fsanitize=${flag}")
@@ -59,6 +63,7 @@ endmacro()
 
 function(add_ur_target_compile_options name)
     if(NOT MSVC)
+        target_compile_definitions(${name} PRIVATE -D_FORTIFY_SOURCE=2)
         target_compile_options(${name} PRIVATE
             -fPIC
             -Wall
@@ -68,9 +73,26 @@ function(add_ur_target_compile_options name)
             $<$<CXX_COMPILER_ID:GNU>:-fdiagnostics-color=always>
             $<$<CXX_COMPILER_ID:Clang,AppleClang>:-fcolor-diagnostics>
         )
+        if (UR_HARDEN)
+            target_compile_options(${name} PRIVATE
+                -fstack-protector
+                -fstack-clash-protection
+                -fcf-protection=none
+                -flto
+                -fstack-protector-strong
+                -fvisibility=hidden # Required for -fsanitize=cfi
+                $<$<CXX_COMPILER_ID:GNU>:-mfunction-return=thunk>
+                $<$<CXX_COMPILER_ID:GNU>:-mindirect-branch=thunk>
+                $<$<CXX_COMPILER_ID:GNU>:-mindirect-branch-register>
+                $<$<CXX_COMPILER_ID:Clang,AppleClang>:-mretpoline>
+                $<$<CXX_COMPILER_ID:Clang,AppleClang>:-fsanitize=cfi>
+            )
+        endif()
         if (CMAKE_BUILD_TYPE STREQUAL "Release")
-            target_compile_definitions(${name} PRIVATE -D_FORTIFY_SOURCE=2)
-            target_compile_options(${name} PRIVATE -fvisibility=hidden)
+            target_compile_options(${name} PRIVATE
+                #-Werror
+                -fvisibility=hidden
+            )
         endif()
         if(UR_DEVELOPER_MODE)
             target_compile_options(${name} PRIVATE
@@ -79,6 +101,19 @@ function(add_ur_target_compile_options name)
                 -fstack-protector-strong
             )
         endif()
+        if(UR_WEXTRA)
+            target_compile_options(${name} PRIVATE
+                # https://github.com/oneapi-src/unified-runtime/issues/2109
+                #-Wextra
+                -Wformat
+                -Wformat-security
+            )
+            if (CMAKE_BUILD_TYPE STREQUAL "Release")
+                target_compile_options(${name} PRIVATE
+                    -Werror=format-security
+                )
+            endif()
+        endif()
     elseif(MSVC)
         target_compile_options(${name} PRIVATE
             $<$<CXX_COMPILER_ID:MSVC>:/MP>  # clang-cl.exe does not support /MP
@@ -102,7 +137,13 @@ endfunction()
 function(add_ur_target_link_options name)
     if(NOT MSVC)
         if (NOT APPLE)
-            target_link_options(${name} PRIVATE "LINKER:-z,relro,-z,now")
+            target_link_options(${name} PRIVATE "LINKER:-z,relro,-z,now,-z,noexecstack")
+            if (CMAKE_BUILD_TYPE STREQUAL "Release")
+                target_link_options(${name} PRIVATE
+                    -Werror
+                    $<$<CXX_COMPILER_ID:GNU>:-pie>
+                )
+            endif()
         endif()
     elseif(MSVC)
         target_link_options(${name} PRIVATE

scripts/check-hardening.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+if [ -z $1 ]; then
+    echo "Usage: $0 builddir" >&2;
+    exit;
+fi
+
+which hardening-check >> /dev/null;
+if [ $? != "0" ]; then
+    echo "hardening-check not found - on Ubuntu it is from the 'devscripts' package." >&2;
+    exit;
+fi
+
+RET=0;
+
+for file in $1/bin/*; do
+    case "$file" in
+        */urtrace)
+            # This is a python script
+            true;;
+        *)
+            hardening-check -q --nocfprotection --nofortify $file;;
+    esac
+    RET=$(($RET + $?))
+done;
+
+for file in $1/lib/*.so; do
+    case "$file" in
+        */libOpenCL*)
+            # This is not built as part of UR
+            true;;
+        */libzeCallMap.so | */libur_mock_headers.so)
+            # Only used in testing, and are too simple for many of the hardening flags to have an effect.
+            true;;
+        *)
+            hardening-check -q --nocfprotection --nofortify $file;;
+    esac
+    RET=$(($RET + $?))
+done;
+
+if [ $RET != "0" ]; then
+    exit 1;
+fi

source/adapters/cuda/enqueue.cpp

@@ -666,6 +666,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueKernelLaunchCustomExp(
   }
   return UR_RESULT_SUCCESS;
 #else
+  [[maybe_unused]] auto _ = launchPropList;
   setErrorMessage("This feature requires cuda 11.8 or later.",
                   UR_RESULT_ERROR_ADAPTER_SPECIFIC);
   return UR_RESULT_ERROR_ADAPTER_SPECIFIC;

source/adapters/cuda/image.cpp

@@ -315,8 +315,8 @@ ur_result_t urTextureCreate(ur_sampler_handle_t hSampler,
 #if CUDA_VERSION >= 11060
       ImageTexDesc.flags |= CU_TRSF_SEAMLESS_CUBEMAP;
 #else
-      setErrorMessage("The " UR_EXP_SAMPLER_CUBEMAP_FILTER_MODE_SEAMLESS
-                      " feature requires cuda 11.6 or later.",
+      setErrorMessage("The EXP_SAMPLER_CUBEMAP_FILTER_MODE_SEAMLESS"
+                      "feature requires cuda 11.6 or later.",
                       UR_RESULT_ERROR_ADAPTER_SPECIFIC);
       return UR_RESULT_ERROR_ADAPTER_SPECIFIC;
 #endif
@@ -657,6 +657,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp(
   UR_ASSERT(pSrcImageFormat->channelOrder == pDstImageFormat->channelOrder,
             UR_RESULT_ERROR_INVALID_ARGUMENT);
 
+  auto as_CUArray = [](const void *ptr) {
+    return static_cast<CUarray>(const_cast<void *>(ptr));
+  };
+
   unsigned int NumChannels = 0;
   size_t PixelSizeBytes = 0;
 
@@ -785,7 +789,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp(
 
         if (isCudaArray) {
           UR_CHECK_ERROR(
-              cuMemcpyAtoHAsync(DstWithOffset, (CUarray)pSrc,
+              cuMemcpyAtoHAsync(DstWithOffset, as_CUArray(pSrc),
                                 PixelSizeBytes * pCopyRegion->srcOffset.x,
                                 CopyExtentBytes, Stream));
         } else if (memType == CU_MEMORYTYPE_DEVICE) {
@@ -809,7 +813,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp(
         cpy_desc.dstHost = pDst;
         if (pSrcImageDesc->rowPitch == 0) {
           cpy_desc.srcMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_ARRAY;
-          cpy_desc.srcArray = (CUarray)pSrc;
+          cpy_desc.srcArray = as_CUArray(pSrc);
         } else {
           // Pitched memory
           cpy_desc.srcMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_DEVICE;
@@ -831,7 +835,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp(
         cpy_desc.dstY = pCopyRegion->dstOffset.y;
         cpy_desc.dstZ = pCopyRegion->dstOffset.z;
         cpy_desc.srcMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_ARRAY;
-        cpy_desc.srcArray = (CUarray)pSrc;
+        cpy_desc.srcArray = as_CUArray(pSrc);
         cpy_desc.dstMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_HOST;
         cpy_desc.dstHost = pDst;
         cpy_desc.dstPitch = pDstImageDesc->width * PixelSizeBytes;
@@ -851,7 +855,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp(
         cpy_desc.dstY = pCopyRegion->dstOffset.y;
         cpy_desc.dstZ = pCopyRegion->dstOffset.z;
         cpy_desc.srcMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_ARRAY;
-        cpy_desc.srcArray = (CUarray)pSrc;
+        cpy_desc.srcArray = as_CUArray(pSrc);
         cpy_desc.dstMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_HOST;
         cpy_desc.dstHost = pDst;
         cpy_desc.dstPitch = pDstImageDesc->width * PixelSizeBytes;
@@ -881,7 +885,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp(
         cpy_desc.dstXInBytes = pCopyRegion->dstOffset.x * PixelSizeBytes;
         cpy_desc.dstY = 0;
         cpy_desc.srcMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_ARRAY;
-        cpy_desc.srcArray = (CUarray)pSrc;
+        cpy_desc.srcArray = as_CUArray(pSrc);
         cpy_desc.dstMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_ARRAY;
         cpy_desc.dstArray = (CUarray)pDst;
         cpy_desc.WidthInBytes = PixelSizeBytes * pCopyRegion->copyExtent.width;
@@ -894,7 +898,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp(
         cpy_desc.dstXInBytes = pCopyRegion->dstOffset.x * PixelSizeBytes;
         cpy_desc.dstY = pCopyRegion->dstOffset.y;
         cpy_desc.srcMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_ARRAY;
-        cpy_desc.srcArray = (CUarray)pSrc;
+        cpy_desc.srcArray = as_CUArray(pSrc);
         cpy_desc.dstMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_ARRAY;
         cpy_desc.dstArray = (CUarray)pDst;
         cpy_desc.WidthInBytes = PixelSizeBytes * pCopyRegion->copyExtent.width;
@@ -909,7 +913,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp(
         cpy_desc.dstY = pCopyRegion->dstOffset.y;
         cpy_desc.dstZ = pCopyRegion->dstOffset.z;
         cpy_desc.srcMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_ARRAY;
-        cpy_desc.srcArray = (CUarray)pSrc;
+        cpy_desc.srcArray = as_CUArray(pSrc);
         cpy_desc.dstMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_ARRAY;
         cpy_desc.dstArray = (CUarray)pDst;
         cpy_desc.WidthInBytes = PixelSizeBytes * pCopyRegion->copyExtent.width;
@@ -927,7 +931,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp(
         cpy_desc.dstY = pCopyRegion->dstOffset.y;
         cpy_desc.dstZ = pCopyRegion->dstOffset.z;
         cpy_desc.srcMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_ARRAY;
-        cpy_desc.srcArray = (CUarray)pSrc;
+        cpy_desc.srcArray = as_CUArray(pSrc);
         cpy_desc.dstMemoryType = CUmemorytype_enum::CU_MEMORYTYPE_ARRAY;
         cpy_desc.dstArray = (CUarray)pDst;
         cpy_desc.WidthInBytes = PixelSizeBytes * pCopyRegion->copyExtent.width;

source/adapters/cuda/kernel.cpp

@@ -13,6 +13,7 @@
 #include "memory.hpp"
 #include "queue.hpp"
 #include "sampler.hpp"
+#include "ur_api.h"
 
 UR_APIEXPORT ur_result_t UR_APICALL
 urKernelCreate(ur_program_handle_t hProgram, const char *pKernelName,
@@ -343,7 +344,8 @@ urKernelSetArgMemObj(ur_kernel_handle_t hKernel, uint32_t argIndex,
   try {
     auto Device = hKernel->getProgram()->getDevice();
     ur_mem_flags_t MemAccess =
-        Properties ? Properties->memoryAccess : UR_MEM_FLAG_READ_WRITE;
+        Properties ? Properties->memoryAccess
+                   : static_cast<ur_mem_flags_t>(UR_MEM_FLAG_READ_WRITE);
     hKernel->Args.addMemObjArg(argIndex, hArgValue, MemAccess);
     if (hArgValue->isImage()) {
       CUDA_ARRAY3D_DESCRIPTOR arrayDesc;

source/adapters/hip/usm.cpp

@@ -218,7 +218,7 @@ urUSMGetMemAllocInfo(ur_context_handle_t hContext, const void *pMem,
           void *Base = nullptr;
           UR_CHECK_ERROR(hipPointerGetAttribute(
               &Base, HIP_POINTER_ATTRIBUTE_RANGE_START_ADDR,
-              (hipDeviceptr_t)pMem));
+              reinterpret_cast<const hipDeviceptr_t *>(pMem)));
           return ReturnValue(Base);
         }
       }

source/adapters/level_zero/CMakeLists.txt

@@ -155,6 +155,17 @@ if(UR_BUILD_ADAPTER_L0)
         )
     endif()
 
+    # Ensure UR flags are propogated to level zero
+    # TODO: https://github.com/oneapi-src/unified-runtime/issues/2105
+    #foreach(TARGET IN ITEMS ze_loader ze_validation_layer ze_tracing_layer ze_null)
+    #    add_ur_target_compile_options(${TARGET})
+    #    add_ur_target_link_options(${TARGET})
+    #    target_compile_options(${TARGET} PRIVATE
+    #        $<$<IN_LIST:$<CXX_COMPILER_ID>,GNU;Clang;Intel;IntelLLVM>:-Wno-error -Wno-unused-parameter>
+    #        $<$<CXX_COMPILER_ID:MSVC>:/WX- /UUNICODE>
+    #    )
+    #endforeach()
+
     if(NOT WIN32)
         target_sources(ur_adapter_level_zero
             PRIVATE

source/loader/ur_lib.cpp

@@ -49,11 +49,10 @@ void context_t::parseEnvEnabledLayers() {
     }
 }
 
-void context_t::initLayers() const {
-    for (auto &[layer, _] : layers) {
-        layer->init((ur_dditable_t *)&urDdiTable, enabledLayerNames,
-                    codelocData);
-    }
+void context_t::initLayers() {
+  for (auto &[layer, _] : layers) {
+    layer->init(&urDdiTable, enabledLayerNames, codelocData);
+  }
 }
 
 void context_t::tearDownLayers() const {

source/loader/ur_lib.hpp

@@ -110,7 +110,7 @@ class __urdlllocal context_t : public AtomicSingleton<context_t> {
     codeloc_data codelocData;
 
     void parseEnvEnabledLayers();
-    void initLayers() const;
+    void initLayers();
     void tearDownLayers() const;
 };
 

source/mock/CMakeLists.txt

@@ -3,7 +3,7 @@
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-add_library (ur_mock_headers SHARED
+add_ur_library (ur_mock_headers SHARED
     "${CMAKE_CURRENT_SOURCE_DIR}/ur_mock_helpers.cpp")
 
 target_include_directories(ur_mock_headers

test/adapters/level_zero/CMakeLists.txt

@@ -43,7 +43,7 @@ if(UR_BUILD_ADAPTER_L0)
     if(NOT WIN32 AND NOT UR_STATIC_ADAPTER_L0)
         # Make L0 use CallMap from a seprate shared lib so that we can access the map
         # from the tests. This only seems to work on linux
-        add_library(zeCallMap SHARED zeCallMap.cpp)
+        add_ur_library(zeCallMap SHARED zeCallMap.cpp)
         target_compile_definitions(ur_adapter_level_zero PRIVATE UR_L0_CALL_COUNT_IN_TESTS)
         # TODO: stop exporting internals like this for tests...
         target_link_libraries(ur_adapter_level_zero PRIVATE zeCallMap)

test/adapters/level_zero/zeCallMap.cpp

@@ -9,4 +9,5 @@
 // Map used by L0 adapter to count the number of calls to each L0 function
 // Lifetime is managed by the adapter, this variable is defined here
 // only so that we can read it from the tests.
-std::map<std::string, int> *ZeCallCount = nullptr;
+__attribute__((visibility("default"))) std::map<std::string, int> *ZeCallCount =
+    nullptr;