[Feature] Triton server

csrc/mmdeploy/CMakeLists.txt

@@ -18,4 +18,8 @@ if (MMDEPLOY_BUILD_SDK)
     add_subdirectory(net)
     add_subdirectory(codebase)
     add_subdirectory(apis)
+
+    if (TRITON_MMDEPLOY_BACKEND)
+        add_subdirectory(triton)
+    endif ()
 endif ()

csrc/mmdeploy/apis/cxx/mmdeploy/pipeline.hpp

@@ -11,27 +11,28 @@ namespace mmdeploy {
 
 namespace cxx {
 
-class Pipeline : public NonMovable {
+class Pipeline : public UniqueHandle<mmdeploy_pipeline_t> {
  public:
   Pipeline(const Value& config, const Context& context) {
-    mmdeploy_pipeline_t pipeline{};
-    auto ec = mmdeploy_pipeline_create_v3((mmdeploy_value_t)&config, context, &pipeline);
+    auto ec = mmdeploy_pipeline_create_v3((mmdeploy_value_t)&config, context, &handle_);
     if (ec != MMDEPLOY_SUCCESS) {
       throw_exception(static_cast<ErrorCode>(ec));
     }
-    pipeline_ = pipeline;
   }
 
   ~Pipeline() {
-    if (pipeline_) {
-      mmdeploy_pipeline_destroy(pipeline_);
-      pipeline_ = nullptr;
+    if (handle_) {
+      mmdeploy_pipeline_destroy(handle_);
+      handle_ = nullptr;
     }
   }
 
+  Pipeline(Pipeline&&) noexcept = default;
+  Pipeline& operator=(Pipeline&&) noexcept = default;
+
   Value Apply(const Value& inputs) {
     mmdeploy_value_t tmp{};
-    auto ec = mmdeploy_pipeline_apply(pipeline_, (mmdeploy_value_t)&inputs, &tmp);
+    auto ec = mmdeploy_pipeline_apply(handle_, (mmdeploy_value_t)&inputs, &tmp);
     if (ec != MMDEPLOY_SUCCESS) {
       throw_exception(static_cast<ErrorCode>(ec));
     }
@@ -50,7 +51,7 @@ class Pipeline : public NonMovable {
     if (ec != MMDEPLOY_SUCCESS) {
       throw_exception(static_cast<ErrorCode>(ec));
     }
-    auto outputs = Apply(*reinterpret_cast<Value*>(inputs));
+    auto outputs = this->Apply(*reinterpret_cast<Value*>(inputs));
     mmdeploy_value_destroy(inputs);
 
     return outputs;
@@ -65,9 +66,6 @@ class Pipeline : public NonMovable {
     }
     return rets;
   }
-
- private:
-  mmdeploy_pipeline_t pipeline_{};
 };
 
 }  // namespace cxx

csrc/mmdeploy/codebase/mmdet/rtmdet_head.cpp

@@ -64,9 +64,6 @@ static float sigmoid(float x) { return 1.0 / (1.0 + expf(-x)); }
 Result<Detections> RTMDetSepBNHead::GetBBoxes(const Value& prep_res,
                                               const std::vector<Tensor>& bbox_preds,
                                               const std::vector<Tensor>& cls_scores) const {
-  MMDEPLOY_DEBUG("bbox_pred: {}, {}", bbox_preds[0].shape(), dets[0].data_type());
-  MMDEPLOY_DEBUG("cls_score: {}, {}", scores[0].shape(), scores[0].data_type());
-
   std::vector<float> filter_boxes;
   std::vector<float> obj_probs;
   std::vector<int> class_ids;

csrc/mmdeploy/codebase/mmseg/segment.cpp

@@ -90,6 +90,7 @@ class ResizeMask : public MMSegmentation {
       std::vector<int> axes = {0, 3, 1, 2};
       ::mmdeploy::operation::Context ctx(host, stream_);
       OUTCOME_TRY(permute_.Apply(tensor_score, tensor_score, axes));
+      tensor_score.Squeeze(0);
     }
 
     SegmentorOutput output{tensor_mask, tensor_score, input_height, input_width, classes_};

csrc/mmdeploy/core/logger.h

@@ -47,7 +47,7 @@ MMDEPLOY_API void SetLogger(spdlog::logger *logger);
 #endif
 
 #ifdef SPDLOG_LOGGER_CALL
-#define MMDEPLOY_LOG(level, ...) SPDLOG_LOGGER_CALL(mmdeploy::GetLogger(), level, __VA_ARGS__)
+#define MMDEPLOY_LOG(level, ...) SPDLOG_LOGGER_CALL(::mmdeploy::GetLogger(), level, __VA_ARGS__)
 #else
 #define MMDEPLOY_LOG(level, ...) mmdeploy::GetLogger()->log(level, __VA_ARGS__)
 #endif

csrc/mmdeploy/core/model.h

@@ -30,8 +30,9 @@ struct model_meta_info_t {
 
 struct deploy_meta_info_t {
   std::string version;
+  std::string task;
   std::vector<model_meta_info_t> models;
-  MMDEPLOY_ARCHIVE_MEMBERS(version, models);
+  MMDEPLOY_ARCHIVE_MEMBERS(version, task, models);
 };
 
 class ModelImpl;

csrc/mmdeploy/device/cuda/cuda_device.h

@@ -1,5 +1,8 @@
 // Copyright (c) OpenMMLab. All rights reserved.
 
+#ifndef MMDEPLOY_SRC_DEVICE_CUDA_CUDE_DEVICE_H_
+#define MMDEPLOY_SRC_DEVICE_CUDA_CUDE_DEVICE_H_
+
 #include <any>
 #include <mutex>
 
@@ -196,3 +199,5 @@ class CudaDeviceGuard {
 };
 
 }  // namespace mmdeploy::framework
+
+#endif  // MMDEPLOY_SRC_DEVICE_CUDA_CUDE_DEVICE_H_

csrc/mmdeploy/graph/inference.cpp

@@ -14,24 +14,30 @@ using namespace framework;
 InferenceBuilder::InferenceBuilder(Value config) : Builder(std::move(config)) {}
 
 Result<unique_ptr<Node>> InferenceBuilder::BuildImpl() {
-  auto& model_config = config_["params"]["model"];
-  Model model;
-  if (model_config.is_any<Model>()) {
-    model = model_config.get<Model>();
-  } else {
-    auto model_name = model_config.get<string>();
-    if (auto m = Maybe{config_} / "context" / "model" / model_name / identity<Model>{}) {
-      model = *m;
+  Value pipeline_config;
+  auto context = config_.value("context", Value(ValueType::kObject));
+  const auto& params = config_["params"];
+  if (params.contains("model")) {
+    auto& model_config = params["model"];
+    Model model;
+    if (model_config.is_any<Model>()) {
+      model = model_config.get<Model>();
     } else {
-      model = Model(model_name);
+      auto model_name = model_config.get<string>();
+      if (auto m = Maybe{config_} / "context" / "model" / model_name / identity<Model>{}) {
+        model = *m;
+      } else {
+        model = Model(model_name);
+      }
     }
+    OUTCOME_TRY(pipeline_config, model.ReadConfig("pipeline.json"));
+    context["model"] = std::move(model);
+  } else if (params.contains("pipeline")) {
+    assert(context.contains("model"));
+    auto model = context["model"].get<Model>();
+    OUTCOME_TRY(pipeline_config, model.ReadConfig(params["pipeline"].get<std::string>()));
   }
 
-  OUTCOME_TRY(auto pipeline_config, model.ReadConfig("pipeline.json"));
-
-  auto context = config_.value("context", Value(ValueType::kObject));
-  context["model"] = std::move(model);
-
   if (context.contains("scope")) {
     auto name = config_.value("name", config_["type"].get<std::string>());
     auto scope = context["scope"].get_ref<profiler::Scope*&>()->CreateScope(name);

csrc/mmdeploy/graph/task.cpp

@@ -96,6 +96,7 @@ Result<unique_ptr<Node>> TaskBuilder::BuildImpl() {
     task->is_thread_safe_ = config_.value("is_thread_safe", false);
     return std::move(task);
   } catch (const std::exception& e) {
+    MMDEPLOY_ERROR("unhandled exception: {}", e.what());
     MMDEPLOY_ERROR("error parsing config: {}", config_);
     return nullptr;
   }

csrc/mmdeploy/preprocess/transform_module.cpp

@@ -10,46 +10,112 @@ namespace mmdeploy {
 
 class TransformModule {
  public:
-  ~TransformModule();
-  TransformModule(TransformModule&&) noexcept;
+  ~TransformModule() = default;
+  TransformModule(TransformModule&&) noexcept = default;
 
-  explicit TransformModule(const Value& args);
-  Result<Value> operator()(const Value& input);
+  explicit TransformModule(const Value& args) {
+    const auto type = "Compose";
+    auto creator = gRegistry<transform::Transform>().Get(type);
+    if (!creator) {
+      MMDEPLOY_ERROR("Unable to find Transform creator: {}. Available transforms: {}", type,
+                     gRegistry<transform::Transform>().List());
+      throw_exception(eEntryNotFound);
+    }
+    auto cfg = args;
+    if (cfg.contains("device")) {
+      MMDEPLOY_WARN("force using device: {}", cfg["device"].get<const char*>());
+      auto device = Device(cfg["device"].get<const char*>());
+      cfg["context"]["device"] = device;
+      cfg["context"]["stream"] = Stream::GetDefault(device);
+    }
+    transform_ = creator->Create(cfg);
+  }
+
+  Result<Value> operator()(const Value& input) {
+    auto data = input;
+    OUTCOME_TRY(transform_->Apply(data));
+    return data;
+  }
 
  private:
   std::unique_ptr<transform::Transform> transform_;
 };
 
-TransformModule::~TransformModule() = default;
+MMDEPLOY_REGISTER_FACTORY_FUNC(Module, (Transform, 0), [](const Value& config) {
+  return CreateTask(TransformModule{config});
+});
 
-TransformModule::TransformModule(TransformModule&&) noexcept = default;
+#if 0
+class Preload {
+ public:
+  explicit Preload(const Value& args) {
+    const auto type = "Compose";
+    auto creator = gRegistry<transform::Transform>().Get(type);
+    if (!creator) {
+      MMDEPLOY_ERROR("Unable to find Transform creator: {}. Available transforms: {}", type,
+                     gRegistry<transform::Transform>().List());
+      throw_exception(eEntryNotFound);
+    }
+    auto cfg = args;
+    if (cfg.contains("device")) {
+      MMDEPLOY_WARN("force using device: {}", cfg["device"].get<const char*>());
+      auto device = Device(cfg["device"].get<const char*>());
+      cfg["context"]["device"] = device;
+      cfg["context"]["stream"] = Stream::GetDefault(device);
+    }
+    const auto& ctx = cfg["context"];
+    ctx["device"].get_to(device_);
+    ctx["stream"].get_to(stream_);
+  }
 
-TransformModule::TransformModule(const Value& args) {
-  const auto type = "Compose";
-  auto creator = gRegistry<transform::Transform>().Get(type);
-  if (!creator) {
-    MMDEPLOY_ERROR("Unable to find Transform creator: {}. Available transforms: {}", type,
-                   gRegistry<transform::Transform>().List());
-    throw_exception(eEntryNotFound);
+  Result<Value> operator()(const Value& input) {
+    auto data = input;
+    if (device_.is_device()) {
+      bool need_sync = false;
+      OUTCOME_TRY(Process(data, need_sync));
+      MMDEPLOY_ERROR("need_sync = {}", need_sync);
+      MMDEPLOY_ERROR("{}", data);
+      if (need_sync) {
+        OUTCOME_TRY(stream_.Wait());
+      }
+    }
+    return data;
   }
-  auto cfg = args;
-  if (cfg.contains("device")) {
-    MMDEPLOY_WARN("force using device: {}", cfg["device"].get<const char*>());
-    auto device = Device(cfg["device"].get<const char*>());
-    cfg["context"]["device"] = device;
-    cfg["context"]["stream"] = Stream::GetDefault(device);
+
+  Result<void> Process(Value& item, bool& need_sync) {
+    if (item.is_any<Mat>()) {
+      auto& mat = item.get_ref<Mat&>();
+      if (mat.device().is_host()) {
+        Mat tmp(mat.height(), mat.width(), mat.pixel_format(), mat.type(), device_);
+        OUTCOME_TRY(stream_.Copy(mat.buffer(), tmp.buffer(), mat.byte_size()));
+        mat = tmp;
+        need_sync |= true;
+      }
+    } else if (item.is_any<Tensor>()) {
+      auto& ten = item.get_ref<Tensor&>();
+      if (ten.device().is_host()) {
+        TensorDesc desc = ten.desc();
+        desc.device = device_;
+        Tensor tmp(desc);
+        OUTCOME_TRY(stream_.Copy(ten.buffer(), tmp.buffer(), ten.byte_size()));
+        ten = tmp;
+        need_sync |= true;
+      }
+    } else if (item.is_array() || item.is_object()) {
+      for (auto& child : item) {
+        OUTCOME_TRY(Process(child, need_sync));
+      }
+    }
+    return success();
   }
-  transform_ = creator->Create(cfg);
-}
 
-Result<Value> TransformModule::operator()(const Value& input) {
-  auto data = input;
-  OUTCOME_TRY(transform_->Apply(data));
-  return data;
-}
+ private:
+  Device device_;
+  Stream stream_;
+};
 
-MMDEPLOY_REGISTER_FACTORY_FUNC(Module, (Transform, 0), [](const Value& config) {
-  return CreateTask(TransformModule{config});
-});
+MMDEPLOY_REGISTER_FACTORY_FUNC(Module, (Preload, 0),
+                               [](const Value& config) { return CreateTask(Preload{config}); });
+#endif
 
 }  // namespace mmdeploy