Remove OpenCV dependency from C_API mode (#800)

* Remove OpenCV dependency from C_API model

* fix build on Windows

* switch ci build flag

* try to fix the macOS build issue

* more fixing

* fix the macOS build issue

* list jpeg source

* verified on MacOS

* update the pp_api too

* avoid the codecs library conflicts

* Add the unit tests

* move the codec test

* add the missing dl lib for extensions test

* refine the code

* a smaller fixing for Windows Python

.pipelines/ci.yml

@@ -197,7 +197,7 @@ stages:
     # compiled as only one operator selected.
     - bash: |
         set -e -x -u
-        ./build.sh -DOCOS_ENABLE_C_API=ON
+        ./build.sh -DOCOS_ENABLE_C_API=ON -DOCOS_ENABLE_CV2=OFF -DOCOS_ENABLE_VISION=OFF -DOCOS_ENABLE_OPENCV_CODECS=OFF
         cd out/Linux/RelWithDebInfo
         ctest -C RelWithDebInfo --output-on-failure
       displayName: Build ort-extensions with API enabled and run tests
@@ -281,7 +281,7 @@ stages:
     # compiled as only one operator selected.
     - bash: |
         set -e -x -u
-        ./build.sh -DOCOS_ENABLE_C_API=ON
+        ./build.sh -DOCOS_ENABLE_C_API=ON -DOCOS_ENABLE_CV2=OFF -DOCOS_ENABLE_VISION=OFF -DOCOS_ENABLE_OPENCV_CODECS=OFF
         cd out/Darwin/RelWithDebInfo
         ctest -C RelWithDebInfo --output-on-failure
       displayName: Build ort-extensions with API enabled and run tests
@@ -431,7 +431,7 @@ stages:
 
     steps:
     - script: |
-        call .\build.bat -DOCOS_ENABLE_C_API=ON
+        call .\build.bat -DOCOS_ENABLE_C_API=ON -DOCOS_ENABLE_CV2=OFF -DOCOS_ENABLE_VISION=OFF -DOCOS_ENABLE_OPENCV_CODECS=OFF
         cd out\Windows
         ctest -C RelWithDebInfo --output-on-failure
       displayName: Build ort-extensions with API enabled and run tests

.pyproject/cmdclass.py

@@ -212,6 +212,9 @@ def build_cmake(self, extension):
                 '-DOCOS_ENABLE_VISION=OFF']
 
         if self.pp_api:
+            if not self.no_opencv:
+                raise RuntimeError(
+                    "Cannot enable PP C API Python Wrapper without disabling OpenCV.")
             cmake_args += ['-DOCOS_ENABLE_C_API=ON']
 
         if self.no_azure is not None:

CMakeLists.txt

@@ -206,7 +206,6 @@ if(NOT PROJECT_IS_TOP_LEVEL AND ONNXRUNTIME_ROOT)
   set(_ONNXRUNTIME_EMBEDDED TRUE)
 endif()
 
-
 if (OCOS_ENABLE_SELECTED_OPLIST OR OCOS_BUILD_PRESET)
   disable_all_operators()
   if(OCOS_ENABLE_SELECTED_OPLIST)
@@ -737,9 +736,12 @@ if(OCOS_ENABLE_C_API)
     file(GLOB audio_TARGET_SRC "shared/api/c_api_feature_extraction.*" "shared/api/speech_*")
     list(APPEND _TARGET_LIB_SRC ${audio_TARGET_SRC})
   endif()
-  if(OCOS_ENABLE_CV2)
+  if(OCOS_ENABLE_DLIB)
+    include(ext_imgcodecs)
     file(GLOB cv2_TARGET_SRC "shared/api/c_api_processor.*" "shared/api/image_*.*")
     list(APPEND _TARGET_LIB_SRC ${cv2_TARGET_SRC})
+    target_include_directories(ocos_operators PUBLIC ${libPNG_SOURCE_DIR} ${libJPEG_SOURCE_DIR})
+    target_link_libraries(ocos_operators PUBLIC ${PNG_LIBRARY} ${JPEG_LIBRARY} ${ZLIB_LIBRARY})
   endif()
 endif()
 
@@ -852,8 +854,8 @@ target_link_libraries(ortcustomops PUBLIC ocos_operators)
 if(OCOS_BUILD_SHARED_LIB)
   file(GLOB shared_TARGET_SRC "shared/*.cc" "shared/*.h")
   if (OCOS_ENABLE_C_API)
-    if (NOT _HAS_TOKENIZER OR NOT OCOS_ENABLE_CV2 OR NOT OCOS_ENABLE_AUDIO)
-      message(FATAL_ERROR "Shared library build requires GPT2_TOKENIZER, CV2 and AUDIO to be enabled.")
+    if (NOT _HAS_TOKENIZER OR NOT OCOS_ENABLE_AUDIO)
+      message(FATAL_ERROR "Shared library build requires GPT2_TOKENIZER, AUDIO to be enabled.")
     endif()
     list(APPEND shared_TARGET_SRC "shared/extensions_c.def")
   else()

MANIFEST.in

@@ -5,11 +5,3 @@ recursive-include include *.*
 recursive-include operators *.*
 recursive-include pyop *.*
 recursive-include shared *.*
-prune ci_build
-prune docs
-prune test
-prune _subbuild
-prune out
-exclude *.bat
-exclude *.yaml
-exclude *.git*

build.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # The example build script to build the source in Linux-like platform
-set -e -x -u
+set -e -u
 
 cuda_arch=''
 if [[ $@ == *"DOCOS_USE_CUDA=ON"* && $@ != *"DCMAKE_CUDA_ARCHITECTURES"* ]]; then

cmake/ext_imgcodecs.cmake

@@ -0,0 +1,131 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+set(_IMGCODEC_ROOT_DIR ${dlib_SOURCE_DIR}/dlib/external)
+
+# ----------------------------------------------------------------------------
+#  project libpng
+#
+# ----------------------------------------------------------------------------
+set (PNG_LIBRARY "libpng_static_c")
+set (libPNG_SOURCE_DIR ${_IMGCODEC_ROOT_DIR}/libpng)
+set (zlib_SOURCE_DIR ${_IMGCODEC_ROOT_DIR}/zlib)
+
+if(NOT WIN32)
+  find_library(M_LIBRARY
+    NAMES m
+    PATHS /usr/lib /usr/local/lib
+  )
+  if(NOT M_LIBRARY)
+    message(STATUS "math lib 'libm' not found; floating point support disabled")
+  endif()
+else()
+  # not needed on windows
+  set(M_LIBRARY "")
+endif()
+
+set(lib_srcs
+   ${libPNG_SOURCE_DIR}/arm/arm_init.c
+   ${libPNG_SOURCE_DIR}/arm/filter_neon_intrinsics.c
+   ${libPNG_SOURCE_DIR}/arm/palette_neon_intrinsics.c
+   ${libPNG_SOURCE_DIR}//png.c
+   ${libPNG_SOURCE_DIR}//pngerror.c
+   ${libPNG_SOURCE_DIR}//pngget.c
+   ${libPNG_SOURCE_DIR}//pngmem.c
+   ${libPNG_SOURCE_DIR}//pngpread.c
+   ${libPNG_SOURCE_DIR}//pngread.c
+   ${libPNG_SOURCE_DIR}//pngrio.c
+   ${libPNG_SOURCE_DIR}//pngrtran.c
+   ${libPNG_SOURCE_DIR}//pngrutil.c
+   ${libPNG_SOURCE_DIR}//pngset.c
+   ${libPNG_SOURCE_DIR}//pngtrans.c
+   ${libPNG_SOURCE_DIR}//pngwio.c
+   ${libPNG_SOURCE_DIR}//pngwrite.c
+   ${libPNG_SOURCE_DIR}//pngwtran.c
+   ${libPNG_SOURCE_DIR}//pngwutil.c
+   ${zlib_SOURCE_DIR}/adler32.c
+   ${zlib_SOURCE_DIR}/compress.c
+   ${zlib_SOURCE_DIR}/crc32.c
+   ${zlib_SOURCE_DIR}/deflate.c
+   ${zlib_SOURCE_DIR}/gzclose.c
+   ${zlib_SOURCE_DIR}/gzlib.c
+   ${zlib_SOURCE_DIR}/gzread.c
+   ${zlib_SOURCE_DIR}/gzwrite.c
+   ${zlib_SOURCE_DIR}/infback.c
+   ${zlib_SOURCE_DIR}/inffast.c
+   ${zlib_SOURCE_DIR}/inflate.c
+   ${zlib_SOURCE_DIR}/inftrees.c
+   ${zlib_SOURCE_DIR}/trees.c
+   ${zlib_SOURCE_DIR}/uncompr.c
+   ${zlib_SOURCE_DIR}/zutil.c
+)
+
+add_library(${PNG_LIBRARY} STATIC EXCLUDE_FROM_ALL ${lib_srcs})
+target_include_directories(${PNG_LIBRARY} BEFORE PRIVATE ${zlib_SOURCE_DIR})
+
+if(MSVC)
+  target_compile_definitions(${PNG_LIBRARY} PRIVATE -D_CRT_SECURE_NO_DEPRECATE)
+else()
+  target_compile_options(${PNG_LIBRARY} PRIVATE -Wno-deprecated-non-prototype)
+endif()
+
+# ----------------------------------------------------------------------------
+#  project libjpeg
+#
+# ----------------------------------------------------------------------------
+set(JPEG_LIBRARY "libjpeg_static_c")
+set(libJPEG_SOURCE_DIR ${_IMGCODEC_ROOT_DIR}/libjpeg)
+
+set(lib_srcs
+  ${libJPEG_SOURCE_DIR}/jaricom.c
+  ${libJPEG_SOURCE_DIR}/jcapimin.c
+  ${libJPEG_SOURCE_DIR}/jcapistd.c
+  ${libJPEG_SOURCE_DIR}/jcarith.c
+  ${libJPEG_SOURCE_DIR}/jccoefct.c
+  ${libJPEG_SOURCE_DIR}/jccolor.c
+  ${libJPEG_SOURCE_DIR}/jcdctmgr.c
+  ${libJPEG_SOURCE_DIR}/jchuff.c
+  ${libJPEG_SOURCE_DIR}/jcinit.c
+  ${libJPEG_SOURCE_DIR}/jcmainct.c
+  ${libJPEG_SOURCE_DIR}/jcmarker.c
+  ${libJPEG_SOURCE_DIR}/jcmaster.c
+  ${libJPEG_SOURCE_DIR}/jcomapi.c
+  ${libJPEG_SOURCE_DIR}/jcparam.c
+  ${libJPEG_SOURCE_DIR}/jcprepct.c
+  ${libJPEG_SOURCE_DIR}/jcsample.c
+  ${libJPEG_SOURCE_DIR}/jdapimin.c
+  ${libJPEG_SOURCE_DIR}/jdapistd.c
+  ${libJPEG_SOURCE_DIR}/jdarith.c
+  ${libJPEG_SOURCE_DIR}/jdatadst.c
+  ${libJPEG_SOURCE_DIR}/jdatasrc.c
+  ${libJPEG_SOURCE_DIR}/jdcoefct.c
+  ${libJPEG_SOURCE_DIR}/jdcolor.c
+  ${libJPEG_SOURCE_DIR}/jddctmgr.c
+  ${libJPEG_SOURCE_DIR}/jdhuff.c
+  ${libJPEG_SOURCE_DIR}/jdinput.c
+  ${libJPEG_SOURCE_DIR}/jdmainct.c
+  ${libJPEG_SOURCE_DIR}/jdmarker.c
+  ${libJPEG_SOURCE_DIR}/jdmaster.c
+  ${libJPEG_SOURCE_DIR}/jdmerge.c
+  ${libJPEG_SOURCE_DIR}/jdpostct.c
+  ${libJPEG_SOURCE_DIR}/jdsample.c
+  ${libJPEG_SOURCE_DIR}/jerror.c
+  ${libJPEG_SOURCE_DIR}/jfdctflt.c
+  ${libJPEG_SOURCE_DIR}/jfdctfst.c
+  ${libJPEG_SOURCE_DIR}/jfdctint.c
+  ${libJPEG_SOURCE_DIR}/jidctflt.c
+  ${libJPEG_SOURCE_DIR}/jidctfst.c
+  ${libJPEG_SOURCE_DIR}/jidctint.c
+  ${libJPEG_SOURCE_DIR}/jmemmgr.c
+  ${libJPEG_SOURCE_DIR}/jmemnobs.c
+  ${libJPEG_SOURCE_DIR}/jquant1.c
+  ${libJPEG_SOURCE_DIR}/jquant2.c
+  ${libJPEG_SOURCE_DIR}/jutils.c
+  )
+file(GLOB lib_hdrs ${libJPEG_SOURCE_DIR}/*.h)
+add_library(${JPEG_LIBRARY} STATIC EXCLUDE_FROM_ALL ${lib_srcs} ${lib_hdrs})
+
+if(NOT MSVC)
+  set_source_files_properties(jcdctmgr.c PROPERTIES COMPILE_FLAGS "-O1")
+endif()
+target_compile_definitions(${JPEG_LIBRARY} PRIVATE -DNO_MKTEMP)

cmake/ext_tests.cmake

@@ -189,6 +189,10 @@ if (OCOS_BUILD_SHARED_LIB)
         list(APPEND extensions_test_libraries stdc++fs -pthread)
       endif()
 
+      if (NOT MSVC)
+        list(APPEND extensions_test_libraries ${CMAKE_DL_LIBS})
+      endif()
+
       add_test_target(TARGET extensions_test
                       TEST_SOURCES ${shared_TEST_SRC}
                       LIBRARIES ${extensions_test_libraries}

cmake/presets/ort_genai.cmake

@@ -3,8 +3,6 @@
 
 set(OCOS_ENABLE_GPT2_TOKENIZER ON CACHE INTERNAL "" FORCE)
 set(OCOS_ENABLE_C_API ON CACHE INTERNAL "" FORCE)
-set(OCOS_ENABLE_CV2 ON CACHE INTERNAL "" FORCE)
-set(OCOS_ENABLE_OPENCV_CODECS ON CACHE INTERNAL "" FORCE)
 set(OCOS_ENABLE_DLIB ON CACHE INTERNAL "" FORCE)
 set(OCOS_ENABLE_MATH ON CACHE INTERNAL "" FORCE)
 set(OCOS_ENABLE_AUDIO ON CACHE INTERNAL "" FORCE)

include/ortx_utils.h

@@ -105,6 +105,17 @@ extError_t ORTX_API_CALL OrtxTensorResultGetAt(OrtxTensorResult* result, size_t
  */
 extError_t ORTX_API_CALL OrtxGetTensorType(OrtxTensor* tensor, extDataType_t* type);
 
+/**
+ * @brief Retrieves the size of each element in the given tensor.
+ *
+ * This function calculates the size of each element in the specified tensor and stores it in the provided size variable.
+ *
+ * @param tensor A pointer to the OrtxTensor object.
+ * @param size A pointer to a size_t variable to store the size of each element.
+ * @return An extError_t value indicating the success or failure of the operation.
+ */
+extError_t ORTX_API_CALL OrtxGetTensorSizeOfElement(OrtxTensor* tensor, size_t* size);
+
 /** \brief Get the data from the tensor
  *
  * \param tensor The tensor object

onnxruntime_extensions/pp_api.py

@@ -65,8 +65,16 @@ def __init__(self, processor_json):
         self.processor = create_processor(processor_json)
 
     def pre_process(self, images):
+        if isinstance(images, str):
+            images = [images]
+        if isinstance(images, list):
+            images = load_images(images)
         return image_pre_process(self.processor, images)
 
+    @staticmethod
+    def to_numpy(result):
+        return tensor_result_get_at(result, 0)
+
     def __del__(self):
         if delete_object and self.processor:
             delete_object(self.processor)

pyop/py_c_api.cc

@@ -85,15 +85,12 @@ void AddGlobalMethodsCApi(pybind11::module& m) {
         const int64_t* shape{};
         size_t num_dims;
         const void* data{};
-        size_t elem_size = 0;
-        if (tensor_type == extDataType_t::kOrtxInt64 || tensor_type == extDataType_t::kOrtxFloat) {
+        size_t elem_size = 1;
+        if (tensor_type == extDataType_t::kOrtxInt64 ||
+            tensor_type == extDataType_t::kOrtxFloat ||
+            tensor_type == extDataType_t::kOrtxUint8) {
           OrtxGetTensorData(tensor, reinterpret_cast<const void**>(&data), &shape, &num_dims);
-          elem_size = 4;
-          if (tensor_type == extDataType_t::kOrtxInt64) {
-            elem_size = 8;
-          }
-        } else if (tensor_type == extDataType_t::kOrtxUnknownType) {
-          throw std::runtime_error("Failed to get tensor type");
+          OrtxGetTensorSizeOfElement(tensor, &elem_size);
         } else if (tensor_type == extDataType_t::kOrtxUnknownType) {
           throw std::runtime_error("unsupported tensor type");
         }
@@ -108,6 +105,8 @@ void AddGlobalMethodsCApi(pybind11::module& m) {
           obj = py::array_t<float>(npy_dims);
         } else if (tensor_type == extDataType_t::kOrtxInt64) {
           obj = py::array_t<int64_t>(npy_dims);
+        } else if (tensor_type == extDataType_t::kOrtxUint8) {
+          obj = py::array_t<uint8_t>(npy_dims);
         }
 
         void* out_ptr = obj.mutable_data();

shared/api/c_api_utils.cc

@@ -103,7 +103,6 @@ extError_t ORTX_API_CALL OrtxTensorResultGetAt(OrtxTensorResult* result, size_t
 
   auto tensor_ptr = std::make_unique<TensorObject>();
   tensor_ptr->SetTensor(ts);
-  tensor_ptr->SetTensorType(result_ptr->GetTensorType(index));
   *tensor = static_cast<OrtxTensor*>(tensor_ptr.release());
   return extError_t();
 }
@@ -124,6 +123,24 @@ extError_t ORTX_API_CALL OrtxGetTensorType(OrtxTensor* tensor, extDataType_t* ty
   return extError_t();
 }
 
+extError_t ORTX_API_CALL OrtxGetTensorSizeOfElement(OrtxTensor* tensor, size_t* size) {
+  if (tensor == nullptr || size == nullptr) {
+    ReturnableStatus::last_error_message_ = "Invalid argument";
+    return kOrtxErrorInvalidArgument;
+  }
+
+  auto tensor_impl = static_cast<TensorObject*>(tensor);
+  if (tensor_impl->ortx_kind() != extObjectKind_t::kOrtxKindTensor) {
+    ReturnableStatus::last_error_message_ = "Invalid argument";
+    return kOrtxErrorInvalidArgument;
+  }
+
+  auto tb = tensor_impl->GetTensor();
+  assert(tb != nullptr); 
+  *size = tb->SizeInBytes() / tb->NumberOfElement();
+  return extError_t();
+}
+
 extError_t ORTX_API_CALL OrtxGetTensorData(OrtxTensor* tensor, const void** data, const int64_t** shape,
                                            size_t* num_dims) {
   if (tensor == nullptr) {
@@ -158,3 +175,11 @@ extError_t ORTX_API_CALL OrtxGetTensorDataFloat(OrtxTensor* tensor, const float*
   *data = reinterpret_cast<const float*>(data_ptr);  // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
   return err;
 }
+
+extError_t ORTX_API_CALL OrtxGetTensorDataUint8(OrtxTensor* tensor, const uint8_t** data, const int64_t** shape,
+                                                size_t* num_dims) {
+  const void* data_ptr{};
+  auto err = OrtxGetTensorData(tensor, &data_ptr, shape, num_dims);
+  *data = reinterpret_cast<const uint8_t*>(data_ptr);  // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
+  return err;
+}