adaptation rockchip image

fujl · fujl · commit d7f781092218 · 2022-12-23T16:03:42.000+08:00
diff --git a/CMake/FindROCKCHIP.cmake b/CMake/FindROCKCHIP.cmake
@@ -9,28 +9,41 @@ else()
 endif()
 
 find_path(ROCKCHIP_RGA_INCLUDE NAMES im2d.h rga.h
-    HINTS ${HINTS_ROCKCHIP_PATH}/rk-rga/include)
+    HINTS ${HINTS_ROCKCHIP_PATH}/rga/include)
 mark_as_advanced(ROCKCHIP_RGA_INCLUDE)
 
 find_path(ROCKCHIP_MPP_INCLUDE NAMES rk_mpi.h rk_type.h
-    HINTS ${HINTS_ROCKCHIP_PATH}/rkmpp/include/rockchip)
+    HINTS ${HINTS_ROCKCHIP_PATH}/rkmpp/include)
 mark_as_advanced(ROCKCHIP_MPP_INCLUDE)
 
-find_path(RKNN_INCLUDE NAMES rknn_api.h
-    HINTS ${HINTS_ROCKCHIP_PATH}/rknn/include)
-mark_as_advanced(RKNN_INCLUDE)
+find_path(RKNPU_INCLUDE NAMES rknn_api.h
+    HINTS ${HINTS_ROCKCHIP_PATH}/rknpu/rknn-api/librknn_api/include)
+mark_as_advanced(RKNPU_INCLUDE)
 
-find_path(RKNPU2_INCLUDE NAMES rknn_api.h
-    HINTS ${HINTS_ROCKCHIP_PATH}/rknnrt/include)
-mark_as_advanced(RKNPU2_INCLUDE)
+find_library(RKNPU_LIBRARY NAMES rknn_api HINTS ${HINTS_ROCKCHIP_PATH}/rknpu/rknn-api/librknn_api/Linux/lib64/)
+mark_as_advanced(RKNPU_LIBRARY)
 
-find_library(RKNN_LIBRARY NAMES rknn_api HINTS ${HINTS_ROCKCHIP_PATH}/rknn/lib)
-mark_as_advanced(RKNN_LIBRARY)
+if(EXISTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK3588)
+    find_path(RKNPU2_INCLUDE NAMES rknn_api.h
+        HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK3588/Linux/librknn_api/include)
+    mark_as_advanced(RKNPU2_INCLUDE)
 
-find_library(RKNPU2_LIBRARY NAMES rknnrt HINTS ${HINTS_ROCKCHIP_PATH}/rknnrt/lib)
-mark_as_advanced(RKNPU2_LIBRARY)
+    find_library(RKNPU2_LIBRARY NAMES rknnrt HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK3588/Linux/librknn_api/aarch64)
+    mark_as_advanced(RKNPU2_LIBRARY)
+    message(STATUS "rk3588 platform")
+elseif(EXISTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK356X)
+    find_path(RKNPU2_INCLUDE NAMES rknn_api.h
+        HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK356X/Linux/librknn_api/include)
+    mark_as_advanced(RKNPU2_INCLUDE)
 
-find_library(RKRGA_LIBRARY NAMES rga HINTS ${HINTS_ROCKCHIP_PATH}/rk-rga/lib)
+    find_library(RKNPU2_LIBRARY NAMES rknnrt HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK356X/Linux/librknn_api/aarch64)
+    mark_as_advanced(RKNPU2_LIBRARY)
+    message(STATUS "rk356x platform")
+else()
+    message(STATUS "other platform")
+endif()
+
+find_library(RKRGA_LIBRARY NAMES rga HINTS ${HINTS_ROCKCHIP_PATH}/rga/libs/Linux/gcc-aarch64)
 mark_as_advanced(RKRGA_LIBRARY)
 
 find_library(RKMPP_LIBRARY NAMES rockchip_mpp HINTS ${HINTS_ROCKCHIP_PATH}/rkmpp/lib)
@@ -42,8 +55,8 @@ FIND_PACKAGE_HANDLE_STANDARD_ARGS(ROCKCHIP
     VERSION_VAR ROCKCHIP_VERSION_STRING)
 
 if(ROCKCHIP_FOUND)
-    set(RKNN_LIBRARIES ${RKNN_LIBRARY})
-    set(RKNN_INCLUDE_DIR ${RKNN_INCLUDE})
+    set(RKNPU_LIBRARIES ${RKNPU_LIBRARY})
+    set(RKNPU_INCLUDE_DIR ${RKNPU_INCLUDE})
     set(RKNPU2_INCLUDE_DIR ${RKNPU2_INCLUDE})
     set(RKNPU2_LIBRARIES ${RKNPU2_LIBRARY})
     set(RKRGA_LIBRARIES ${RKRGA_LIBRARY})
diff --git a/src/drivers/devices/rockchip/core/CMakeLists.txt b/src/drivers/devices/rockchip/core/CMakeLists.txt
@@ -29,7 +29,7 @@ include_directories(${LIBMODELBOX_INCLUDE})
 include_directories(${LIBMODELBOX_BASE_INCLUDE})
 include_directories(${LIBMODELBOX_DEVICE_ROCKCHIP_INCLUDE})
 include_directories(${ROCKCHIP_MPP_INCLUDE})
-include_directories(${RKNN_INCLUDE_DIR})
+include_directories(${RKNPU_INCLUDE_DIR})
 include_directories(${ROCKCHIP_RGA_INCLUDE})
 
 set(HEADER
diff --git a/src/drivers/devices/rockchip/core/rockchip_api.cc b/src/drivers/devices/rockchip/core/rockchip_api.cc
@@ -119,6 +119,16 @@ Status CopyNVMemory(uint8_t *psrc, uint8_t *pdst, int w, int h, int ws,
 
 Status CopyRGBMemory(uint8_t *psrc, uint8_t *pdst, int w, int h, int ws,
                      int hs) {
+  if (psrc == nullptr) {
+    MBLOG_ERROR << "psrc is nullptr";
+    return {STATUS_FAULT, "psrc is nullptr"};
+  }
+
+  if (pdst == nullptr) {
+    MBLOG_ERROR << "pdst is nullptr";
+    return {STATUS_FAULT, "pdst is nullptr"};
+  }
+
   uint8_t *rgbsrc = psrc;
   uint8_t *rgbdst = pdst;
 
diff --git a/src/drivers/devices/rockchip/flowunit/crop/crop_flowunit.cc b/src/drivers/devices/rockchip/flowunit/crop/crop_flowunit.cc
@@ -161,7 +161,6 @@ MODELBOX_FLOWUNIT(RockchipCropFlowUnit, rk_crop_desc) {
   rk_crop_desc.AddFlowUnitInput({IN_REGION, "cpu"});
   rk_crop_desc.AddFlowUnitOutput({OUT_IMG, modelbox::DEVICE_TYPE});
   rk_crop_desc.SetFlowType(modelbox::NORMAL);
-  rk_crop_desc.SetInputContiguous(false);
   rk_crop_desc.SetDescription(FLOWUNIT_DESC);
 }
 
diff --git a/src/drivers/devices/rockchip/flowunit/image_decoder/image_decoder.cc b/src/drivers/devices/rockchip/flowunit/image_decoder/image_decoder.cc
@@ -227,7 +227,6 @@ MODELBOX_FLOWUNIT(ImageDecoderFlowUnit, rk_imgdec_desc) {
       "the output pixel format"));
 
   rk_imgdec_desc.SetFlowType(modelbox::NORMAL);
-  rk_imgdec_desc.SetInputContiguous(false);
   rk_imgdec_desc.SetDescription(FLOWUNIT_DESC);
 }
 
diff --git a/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference.cc b/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference.cc
@@ -152,6 +152,9 @@ modelbox::Status modelbox::RKNPU2Inference::Init(
     return {modelbox::STATUS_FAULT, "query input_output error"};
   }
 
+  MBLOG_INFO << "model input num: " << rknpu2_io_num.n_input
+             << ", output num: " << rknpu2_io_num.n_output << std::endl;
+
   if (npu2model_input_list_.size() != rknpu2_io_num.n_input ||
       npu2model_output_list_.size() != rknpu2_io_num.n_output) {
     MBLOG_ERROR << "model input output num mismatch: input num in graph is "
@@ -285,9 +288,10 @@ size_t modelbox::RKNPU2Inference::GetInputBuffer(
     input_params->in_width_ = in_image->GetBytes();
   }
 
-  if (input_buf_list->GetDevice()->GetType() == "rknpu") {
+  if (input_buf_list->GetDevice()->GetType() == "rockchip") {
     return CopyFromAlignMemory(input_buf_list, input_buf, input_params);
   }
+
   input_buf.reset((uint8_t *)input_buf_list->ConstData(), [](uint8_t *p) {});
   return input_buf_list->GetBytes();
 }
@@ -326,6 +330,7 @@ modelbox::Status modelbox::RKNPU2Inference::Infer(
                   << " " << inputs_size_[i];
       return modelbox::STATUS_FAULT;
     }
+
     one_input.pass_through = false;
     one_input.type = (rknn_tensor_type)inputs_type_[i];
     one_input.fmt = RKNN_TENSOR_NHWC;
diff --git a/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference_flowunit.h b/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference_flowunit.h
@@ -26,7 +26,7 @@
 
 #include "rknpu2_inference.h"
 
-constexpr const char *RKNPU2_FLOWUNIT_TYPE = "rknpu2";
+constexpr const char *RKNPU2_FLOWUNIT_TYPE = "rockchip";
 constexpr const char *RKNPU2_INFERENCE_TYPE = "rknpu2";
 
 class RKNPU2InferenceFlowUnit : public modelbox::FlowUnit {
diff --git a/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h b/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h
@@ -68,9 +68,6 @@ class RockChipLocalCameraFlowUnit : public modelbox::FlowUnit {
   MppFrame SetMppFrameInfo(size_t w, size_t h, MppFrameFormat fmt,
                            MppBuffer mpp_buf);
 
-  MppFrame SetMppFrameInfo(size_t w, size_t h, MppFrameFormat fmt,
-                           MppBuffer mpp_buf);
-
   uint32_t camWidth_{0};
   uint32_t camHeight_{0};
   uint32_t camera_id_{0};
diff --git a/src/drivers/devices/rockchip/flowunit/resize/resize_flowunit.cc b/src/drivers/devices/rockchip/flowunit/resize/resize_flowunit.cc
@@ -135,7 +135,6 @@ MODELBOX_FLOWUNIT(ResizeFlowUnit, rk_resize_desc) {
   rk_resize_desc.AddFlowUnitInput({IN_IMG});
   rk_resize_desc.AddFlowUnitOutput({OUT_IMG});
   rk_resize_desc.SetFlowType(modelbox::NORMAL);
-  rk_resize_desc.SetInputContiguous(false);
   rk_resize_desc.SetDescription(FLOWUNIT_DESC);
   rk_resize_desc.AddFlowUnitOption(modelbox::FlowUnitOption(
       "image_width", "int", true, "0", "the resize width"));
diff --git a/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.cc b/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.cc
@@ -174,7 +174,6 @@ MODELBOX_FLOWUNIT(VideoDecoderFlowUnit, desc) {
   desc.AddFlowUnitInput({VIDEO_PACKET_INPUT, "cpu"});
   desc.AddFlowUnitOutput({FRAME_INFO_OUTPUT, modelbox::DEVICE_TYPE});
   desc.SetFlowType(modelbox::STREAM);
-  desc.SetInputContiguous(false);
   desc.SetResourceNice(false);
   desc.SetDescription(FLOWUNIT_DESC);
   desc.AddFlowUnitOption(modelbox::FlowUnitOption(
diff --git a/src/drivers/devices/rockchip/flowunit/video_out/video_out_flowunit.cc b/src/drivers/devices/rockchip/flowunit/video_out/video_out_flowunit.cc
@@ -317,25 +317,37 @@ modelbox::Status VideoOutFlowUnit::GetDestUrl(
     const std::shared_ptr<modelbox::DataContext> &data_ctx,
     std::string &dest_url) {
   dest_url = default_dest_url_;
-
   Defer {
     std::regex url_auth_pattern("://[^ /]*?:[^ /]*?@");
     auto result = std::regex_replace(dest_url, url_auth_pattern, "://*:*@");
     MBLOG_INFO << "video_out url is " << result;
   };
 
+  if (data_ctx == nullptr) {
+    MBLOG_ERROR << "data ctx is nullptr";
+    return modelbox::STATUS_INVALID;
+  }
+
   // 3种方式获取
   auto stream_meta = data_ctx->GetInputMeta(FRAME_INFO_INPUT);
   if (stream_meta != nullptr) {
+    auto meta_dest_url = stream_meta->GetMeta(DEST_URL);
     auto dest_url_ptr =
-        std::static_pointer_cast<std::string>(stream_meta->GetMeta(DEST_URL));
+        meta_dest_url == nullptr
+            ? nullptr
+            : std::static_pointer_cast<std::string>(meta_dest_url);
     if (dest_url_ptr != nullptr && !(*dest_url_ptr).empty()) {
       dest_url = *dest_url_ptr;
       return modelbox::STATUS_SUCCESS;
     }
   }
 
   auto config = data_ctx->GetSessionConfig();
+  if (config == nullptr) {
+    MBLOG_ERROR << "data ctx session config is empty";
+    return modelbox::STATUS_INVALID;
+  }
+
   auto cfg_str = config->GetString("iva_task_output");
   if (cfg_str.empty()) {
     return modelbox::STATUS_SUCCESS;
@@ -368,7 +380,6 @@ MODELBOX_FLOWUNIT(VideoOutFlowUnit, desc) {
   desc.SetFlowUnitGroupType("Video");
   desc.AddFlowUnitInput({FRAME_INFO_INPUT, "cpu"});
   desc.SetFlowType(modelbox::STREAM);
-  desc.SetInputContiguous(false);
   // 禁止异步执行，编码必须一帧帧的编码
   desc.SetResourceNice(false);
   desc.SetDescription(FLOWUNIT_DESC);

Original file line number	Diff line number	Diff line change
`@@ -161,7 +161,6 @@ MODELBOX_FLOWUNIT(RockchipCropFlowUnit, rk_crop_desc) {`
`161`	`161`	`rk_crop_desc.AddFlowUnitInput({IN_REGION, "cpu"});`
`162`	`162`	`rk_crop_desc.AddFlowUnitOutput({OUT_IMG, modelbox::DEVICE_TYPE});`
`163`	`163`	`rk_crop_desc.SetFlowType(modelbox::NORMAL);`
`164`		`- rk_crop_desc.SetInputContiguous(false);`
`165`	`164`	`rk_crop_desc.SetDescription(FLOWUNIT_DESC);`
`166`	`165`	`}`
`167`	`166`
Original file line number	Diff line number	Diff line change
`@@ -227,7 +227,6 @@ MODELBOX_FLOWUNIT(ImageDecoderFlowUnit, rk_imgdec_desc) {`
`227`	`227`	`"the output pixel format"));`
`228`	`228`
`229`	`229`	`rk_imgdec_desc.SetFlowType(modelbox::NORMAL);`
`230`		`- rk_imgdec_desc.SetInputContiguous(false);`
`231`	`230`	`rk_imgdec_desc.SetDescription(FLOWUNIT_DESC);`
`232`	`231`	`}`
`233`	`232`
Original file line number	Diff line number	Diff line change
`@@ -152,6 +152,9 @@ modelbox::Status modelbox::RKNPU2Inference::Init(`
`152`	`152`	`return {modelbox::STATUS_FAULT, "query input_output error"};`
`153`	`153`	`}`
`154`	`154`
	`155`	`+ MBLOG_INFO << "model input num: " << rknpu2_io_num.n_input`
	`156`	`+ << ", output num: " << rknpu2_io_num.n_output << std::endl;`
	`157`	`+`
`155`	`158`	`if (npu2model_input_list_.size() != rknpu2_io_num.n_input \|\|`
`156`	`159`	`npu2model_output_list_.size() != rknpu2_io_num.n_output) {`
`157`	`160`	`MBLOG_ERROR << "model input output num mismatch: input num in graph is "`
`@@ -285,9 +288,10 @@ size_t modelbox::RKNPU2Inference::GetInputBuffer(`
`285`	`288`	`input_params->in_width_ = in_image->GetBytes();`
`286`	`289`	`}`
`287`	`290`
`288`		`- if (input_buf_list->GetDevice()->GetType() == "rknpu") {`
	`291`	`+ if (input_buf_list->GetDevice()->GetType() == "rockchip") {`
`289`	`292`	`return CopyFromAlignMemory(input_buf_list, input_buf, input_params);`
`290`	`293`	`}`
	`294`	`+`
`291`	`295`	`input_buf.reset((uint8_t )input_buf_list->ConstData(), [](uint8_t p) {});`
`292`	`296`	`return input_buf_list->GetBytes();`
`293`	`297`	`}`
`@@ -326,6 +330,7 @@ modelbox::Status modelbox::RKNPU2Inference::Infer(`
`326`	`330`	`<< " " << inputs_size_[i];`
`327`	`331`	`return modelbox::STATUS_FAULT;`
`328`	`332`	`}`
	`333`	`+`
`329`	`334`	`one_input.pass_through = false;`
`330`	`335`	`one_input.type = (rknn_tensor_type)inputs_type_[i];`
`331`	`336`	`one_input.fmt = RKNN_TENSOR_NHWC;`