diff --git a/CMakeLists.txt b/CMakeLists.txt index c63445b..d68200e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,8 @@ add_definitions(-DAPI_EXPORTS) set(CMAKE_BUILD_TYPE "Release") + + ## JETSON, X86_CUDA, X86_INTEL message(STATUS "System:${CMAKE_HOST_SYSTEM_PROCESSOR}") if(NOT DEFINED PLATFORM) @@ -24,6 +26,7 @@ else() option(USE_FFMPEG "BUILD WITH FFMPEG." ON) elseif(PLATFORM STREQUAL "X86_INTEL") add_definitions(-DPLATFORM_X86_INTEL) + option(USE_INTEL "BUILD WITH INTEL." ON) option(USE_FFMPEG "BUILD WITH FFMPEG." ON) else() message(FATAL_ERROR "UNSUPPORTED PLATFORM!") @@ -38,6 +41,10 @@ if(USE_CUDA) message(STATUS "CUDA: ON") endif() +if(USE_INTEL) + add_definitions(-DWITH_INTEL) + message(STATUS "INTEL: ON") +endif() if(USE_GSTREAMER) add_definitions(-DWITH_GSTREAMER) @@ -72,6 +79,9 @@ include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/gimbal_ctrl/driver/src/GX40 ${CMAKE_CURRENT_SOURCE_DIR}/gimbal_ctrl/driver/src ${CMAKE_CURRENT_SOURCE_DIR}/gimbal_ctrl + ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/common_det/intel + ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/landing_det/intel + ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/veri/intel ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/common_det/cuda ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/landing_det/cuda ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/sot/ocv470 @@ -187,6 +197,15 @@ if(USE_CUDA) list(APPEND spirecv_SRCS ${ALG_SRC_FILES}) endif() +if(USE_INTEL) + file(GLOB ALG_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/common_det/intel/*.cpp) + list(APPEND spirecv_SRCS ${ALG_SRC_FILES}) + file(GLOB ALG_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/landing_det/intel/*.cpp) + list(APPEND spirecv_SRCS ${ALG_SRC_FILES}) + file(GLOB ALG_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/veri/intel/*.cpp) + list(APPEND spirecv_SRCS ${ALG_SRC_FILES}) +endif() + if(USE_FFMPEG) file(GLOB ALG_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/video_io/ffmpeg/*.cpp) list(APPEND spirecv_SRCS ${ALG_SRC_FILES}) @@ -231,10 +250,22 @@ if(USE_CUDA) # PLATFORM_X86_CUDA & PLATFORM_JETSON target_link_libraries(SpireCVSeg sv_world) elseif(PLATFORM STREQUAL "X86_INTEL") # Links to Intel-OpenVINO libraries here + # Intel-Openvino + include_directories( + PUBLIC /opt/intel/openvino_2022/runtime/include/ + PUBLIC /opt/intel/openvino_2022/runtime/include/ie/ + ) + link_directories( + ${InferenceEngine_LIBRARIES} + /opt/intel/openvino_2022/runtime/lib/intel64/libopenvino.so + ) + add_library(sv_world SHARED ${spirecv_SRCS}) target_link_libraries( sv_world ${OpenCV_LIBS} sv_gimbal + ${InferenceEngine_LIBRARIES} + /opt/intel/openvino_2022/runtime/lib/intel64/libopenvino.so ) endif() diff --git a/algorithm/common_det/intel/common_det_intel_impl.cpp b/algorithm/common_det/intel/common_det_intel_impl.cpp new file mode 100644 index 0000000..ee8cd28 --- /dev/null +++ b/algorithm/common_det/intel/common_det_intel_impl.cpp @@ -0,0 +1,447 @@ +#include "common_det_intel_impl.h" +#include +#include + +#define SV_MODEL_DIR "/SpireCV/models/" +#define SV_ROOT_DIR "/SpireCV/" + +namespace sv +{ + +#ifdef WITH_INTEL + using namespace cv; + using namespace std; + using namespace dnn; +#endif + + float sigmoid_function(float a) + { + float b = 1. / (1. + exp(-a)); + return b; + } + + cv::Mat letterbox(cv::Mat &img_, std::vector &paddings) + { + std::vector new_shape = {640, 640}; + + // Get current image shape [height, width] + int img_h = img_.rows; + int img_w = img_.cols; + + // Compute scale ratio(new / old) and target resized shape + float scale = std::min(new_shape[1] * 1.0 / img_h, new_shape[0] * 1.0 / img_w); + int resize_h = int(round(img_h * scale)); + int resize_w = int(round(img_w * scale)); + paddings[0] = scale; + + // Compute padding + int pad_h = new_shape[1] - resize_h; + int pad_w = new_shape[0] - resize_w; + + // Resize and pad image while meeting stride-multiple constraints + cv::Mat resized_img; + cv::resize(img_, resized_img, cv::Size(resize_w, resize_h)); + + // divide padding into 2 sides + float half_h = pad_h * 1.0 / 2; + float half_w = pad_w * 1.0 / 2; + paddings[1] = half_h; + paddings[2] = half_w; + + // Compute padding boarder + int top = int(round(half_h - 0.1)); + int bottom = int(round(half_h + 0.1)); + int left = int(round(half_w - 0.1)); + int right = int(round(half_w + 0.1)); + + // Add border + cv::copyMakeBorder(resized_img, resized_img, top, bottom, left, right, 0, cv::Scalar(114, 114, 114)); + + return resized_img; + } + + CommonObjectDetectorIntelImpl::CommonObjectDetectorIntelImpl() + { + } + + CommonObjectDetectorIntelImpl::~CommonObjectDetectorIntelImpl() + { + } + + void CommonObjectDetectorIntelImpl::intelDetect( + CommonObjectDetectorBase *base_, + cv::Mat img_, + std::vector &boxes_x_, + std::vector &boxes_y_, + std::vector &boxes_w_, + std::vector &boxes_h_, + std::vector &boxes_label_, + std::vector &boxes_score_, + std::vector &boxes_seg_, + bool input_4k_) + { +#ifdef WITH_INTEL + int input_h = base_->getInputH(); + int input_w = base_->getInputW(); + bool with_segmentation = base_->withSegmentation(); + double thrs_conf = base_->getThrsConf(); + double thrs_nms = base_->getThrsNms(); + + if (with_segmentation) + { + std::vector paddings(3); // scale, half_h, half_w + this->preprocess_img_seg(img_, paddings); + + infer_request.start_async(); + infer_request.wait(); + + // Postprocess + this->postprocess_img_seg(img_, paddings, boxes_x_, boxes_y_, boxes_w_, boxes_h_, boxes_label_, boxes_score_, boxes_seg_, thrs_conf, thrs_nms); + } + else + { + // Preprocess + this->preprocess_img(img_); + + // Run inference + infer_request.start_async(); + infer_request.wait(); + + // Postprocess + this->postprocess_img(boxes_x_, boxes_y_, boxes_w_, boxes_h_, boxes_label_, boxes_score_, thrs_conf, thrs_nms); + } + +#endif + } + + bool CommonObjectDetectorIntelImpl::intelSetup(CommonObjectDetectorBase *base_, bool input_4k_) + { +#ifdef WITH_INTEL + ov::Core core; + std::string dataset = base_->getDataset(); + double thrs_conf = base_->getThrsConf(); + double thrs_nms = base_->getThrsNms(); + inpHeight = base_->getInputH(); + inpWidth = base_->getInputW(); + with_segmentation = base_->withSegmentation(); + + std::string openvino_fn = get_home() + SV_MODEL_DIR + dataset + ".onnx"; + if (inpWidth == 1280) + { + openvino_fn = get_home() + SV_MODEL_DIR + dataset + "_HD.onnx"; + } + if (with_segmentation) + { + base_->setInputH(640); + base_->setInputW(640); + openvino_fn = get_home() + SV_MODEL_DIR + dataset + "_SEG.onnx"; + } + std::cout << "Load: " << openvino_fn << std::endl; + if (!is_file_exist(openvino_fn)) + { + throw std::runtime_error("SpireCV (104) Error loading the CommonObject OpenVINO model (File Not Exist)"); + } + + if (input_4k_ && with_segmentation) + { + throw std::runtime_error("SpireCV (106) Resolution 4K DO NOT Support Segmentation!"); + } + + if (with_segmentation) + { + this->compiled_model = core.compile_model(openvino_fn, "GPU"); + this->infer_request = compiled_model.create_infer_request(); + } + else + { + std::shared_ptr model_ = core.read_model(openvino_fn); + ov::preprocess::PrePostProcessor Pre_P = ov::preprocess::PrePostProcessor(model_); + Pre_P.input().tensor().set_element_type(ov::element::u8).set_layout("NHWC").set_color_format(ov::preprocess::ColorFormat::RGB); + Pre_P.input().preprocess().convert_element_type(ov::element::f32).convert_color(ov::preprocess::ColorFormat::RGB).scale({255, 255, 255}); // .scale({ 112, 112, 112 }); + Pre_P.input().model().set_layout("NCHW"); + Pre_P.output().tensor().set_element_type(ov::element::f32); + model_ = Pre_P.build(); + this->compiled_model = core.compile_model(model_, "GPU"); + this->infer_request = compiled_model.create_infer_request(); + } + return true; +#endif + return false; + } + + void CommonObjectDetectorIntelImpl::preprocess_img(cv::Mat &img_) + { +#ifdef WITH_INTEL + float width = img_.cols; + float height = img_.rows; + cv::Size new_shape = cv::Size(inpHeight, inpWidth); + float r = float(new_shape.width / max(width, height)); + int new_unpadW = int(round(width * r)); + int new_unpadH = int(round(height * r)); + + cv::resize(img_, resize.resized_image, cv::Size(new_unpadW, new_unpadH), 0, 0, cv::INTER_AREA); + resize.resized_image = resize.resized_image; + resize.dw = new_shape.width - new_unpadW; + resize.dh = new_shape.height - new_unpadH; + cv::Scalar color = cv::Scalar(100, 100, 100); + cv::copyMakeBorder(resize.resized_image, resize.resized_image, 0, resize.dh, 0, resize.dw, cv::BORDER_CONSTANT, color); + + this->rx = (float)img_.cols / (float)(resize.resized_image.cols - resize.dw); + this->ry = (float)img_.rows / (float)(resize.resized_image.rows - resize.dh); + if (with_segmentation) + { + cv::Mat blob = cv::dnn::blobFromImage(resize.resized_image, 1 / 255.0, cv::Size(640, 640), cv::Scalar(0, 0, 0), true); + auto input_port = compiled_model.input(); + ov::Tensor input_tensor(input_port.get_element_type(), input_port.get_shape(), blob.ptr(0)); + infer_request.set_input_tensor(input_tensor); + } + else + { + float *input_data = (float *)resize.resized_image.data; + input_tensor = ov::Tensor(compiled_model.input().get_element_type(), compiled_model.input().get_shape(), input_data); + infer_request.set_input_tensor(input_tensor); + } +#endif + } + + void CommonObjectDetectorIntelImpl::preprocess_img_seg(cv::Mat &img_, std::vector &paddings) + { +#ifdef WITH_INTEL + cv::Mat masked_img; + cv::Mat resized_img = letterbox(img_, paddings); // resize to (640,640) by letterbox + // BGR->RGB, u8(0-255)->f32(0.0-1.0), HWC->NCHW + cv::Mat blob = cv::dnn::blobFromImage(resized_img, 1 / 255.0, cv::Size(640, 640), cv::Scalar(0, 0, 0), true); + + // Get input port for model with one input + auto input_port = compiled_model.input(); + // Create tensor from external memory + ov::Tensor input_tensor(input_port.get_element_type(), input_port.get_shape(), blob.ptr(0)); + // Set input tensor for model with one input + infer_request.set_input_tensor(input_tensor); +#endif + } + + void CommonObjectDetectorIntelImpl::postprocess_img_seg(cv::Mat &img_, std::vector &paddings, std::vector &boxes_x_, + std::vector &boxes_y_, + std::vector &boxes_w_, + std::vector &boxes_h_, + std::vector &boxes_label_, + std::vector &boxes_score_, + std::vector &boxes_seg_, + double &thrs_conf, + double &thrs_nms) + { +#ifdef WITH_INTEL + const ov::Tensor &detect = infer_request.get_output_tensor(0); + ov::Shape detect_shape = detect.get_shape(); + const ov::Tensor &proto = infer_request.get_output_tensor(1); + ov::Shape proto_shape = proto.get_shape(); + + cv::Mat detect_buffer(detect_shape[1], detect_shape[2], CV_32F, detect.data()); + cv::Mat proto_buffer(proto_shape[1], proto_shape[2] * proto_shape[3], CV_32F, proto.data()); + + cv::RNG rng; + float conf_threshold = thrs_conf; + float nms_threshold = thrs_nms; + std::vector boxes; + std::vector class_ids; + std::vector class_scores; + std::vector confidences; + std::vector masks; + + float scale = paddings[0]; + for (int i = 0; i < detect_buffer.rows; i++) + { + float confidence = detect_buffer.at(i, 4); + if (confidence < conf_threshold) + { + continue; + } + cv::Mat classes_scores = detect_buffer.row(i).colRange(5, 85); + cv::Point class_id; + double score; + cv::minMaxLoc(classes_scores, NULL, &score, NULL, &class_id); + + // class score: 0~1 + if (score > 0.25) + { + cv::Mat mask = detect_buffer.row(i).colRange(85, 117); + float cx = detect_buffer.at(i, 0); + float cy = detect_buffer.at(i, 1); + float w = detect_buffer.at(i, 2); + float h = detect_buffer.at(i, 3); + int left = static_cast((cx - 0.5 * w - paddings[2]) / scale); + int top = static_cast((cy - 0.5 * h - paddings[1]) / scale); + int width = static_cast(w / scale); + int height = static_cast(h / scale); + cv::Rect box; + box.x = left; + box.y = top; + box.width = width; + box.height = height; + + boxes.push_back(box); + class_ids.push_back(class_id.x); + class_scores.push_back(score); + confidences.push_back(confidence); + masks.push_back(mask); + } + } + + // NMS + std::vector indices; + cv::dnn::NMSBoxes(boxes, confidences, thrs_conf, thrs_nms, indices); + // cv::Mat rgb_mask; + cv::Mat rgb_mask = cv::Mat::zeros(img_.size(), img_.type()); + + for (size_t i = 0; i < indices.size(); i++) + { + int index = indices[i]; + int class_id = class_ids[index]; + cv::Rect box = boxes[index]; + int x1 = std::max(0, box.x); + int y1 = std::max(0, box.y); + int x2 = std::max(0, box.br().x); + int y2 = std::max(0, box.br().y); + + cv::Mat m = masks[index] * proto_buffer; + for (int col = 0; col < m.cols; col++) + { + m.at(0, col) = sigmoid_function(m.at(0, col)); + } + cv::Mat m1 = m.reshape(1, 160); // 1x25600 -> 160x160 + int mx1 = std::max(0, int((x1 * scale + paddings[2]) * 0.25)); + int mx2 = std::max(0, int((x2 * scale + paddings[2]) * 0.25)); + int my1 = std::max(0, int((y1 * scale + paddings[1]) * 0.25)); + int my2 = std::max(0, int((y2 * scale + paddings[1]) * 0.25)); + cv::Mat mask_roi = m1(cv::Range(my1, my2), cv::Range(mx1, mx2)); + + cv::Mat rm, det_mask; + cv::resize(mask_roi, rm, cv::Size(x2 - x1, y2 - y1)); + for (int r = 0; r < rm.rows; r++) + { + for (int c = 0; c < rm.cols; c++) + { + float pv = rm.at(r, c); + if (pv > 0.5) + { + rm.at(r, c) = 1.0; + } + else + { + rm.at(r, c) = 0.0; + } + } + } + + rm = rm * rng.uniform(0, 255); + rm.convertTo(det_mask, CV_8UC1); + if ((y1 + det_mask.rows) >= img_.rows) + { + y2 = img_.rows - 1; + } + if ((x1 + det_mask.cols) >= img_.cols) + { + x2 = img_.cols - 1; + } + + cv::Mat mask = cv::Mat::zeros(cv::Size(img_.cols, img_.rows), CV_8UC1); + det_mask(cv::Range(0, y2 - y1), cv::Range(0, x2 - x1)).copyTo(mask(cv::Range(y1, y2), cv::Range(x1, x2))); + add(rgb_mask, cv::Scalar(rng.uniform(0, 255), rng.uniform(0, 255), rng.uniform(0, 255)), rgb_mask, mask); + + boxes_x_.push_back(box.x); + boxes_y_.push_back(box.y); + boxes_w_.push_back(box.width); + boxes_h_.push_back(box.height); + + boxes_label_.push_back((int)class_id); + boxes_score_.push_back(class_scores[index]); + + cv::Mat mask_j = mask.clone(); + boxes_seg_.push_back(mask_j); + } + +#endif + } + + void CommonObjectDetectorIntelImpl::postprocess_img(std::vector &boxes_x_, + std::vector &boxes_y_, + std::vector &boxes_w_, + std::vector &boxes_h_, + std::vector &boxes_label_, + std::vector &boxes_score_, + double &thrs_conf, + double &thrs_nms) + { +#ifdef WITH_INTEL + const ov::Tensor &output_tensor = infer_request.get_output_tensor(); + ov::Shape output_shape = output_tensor.get_shape(); + float *detections = output_tensor.data(); + + std::vector boxes; + vector class_ids; + vector confidences; + for (int i = 0; i < output_shape[1]; i++) + { + float *detection = &detections[i * output_shape[2]]; + + float confidence = detection[4]; + if (confidence >= thrs_conf) + { + float *classes_scores = &detection[5]; + cv::Mat scores(1, output_shape[2] - 5, CV_32FC1, classes_scores); + cv::Point class_id; + double max_class_score; + cv::minMaxLoc(scores, 0, &max_class_score, 0, &class_id); + if (max_class_score > thrs_conf) + { + confidences.push_back(confidence); + class_ids.push_back(class_id.x); + float x = detection[0]; + float y = detection[1]; + float w = detection[2]; + float h = detection[3]; + float xmin = x - (w / 2); + float ymin = y - (h / 2); + + boxes.push_back(cv::Rect(xmin, ymin, w, h)); + } + } + } + + std::vector nms_result; + cv::dnn::NMSBoxes(boxes, confidences, thrs_conf, thrs_nms, nms_result); + + std::vector output; + for (int i = 0; i < nms_result.size(); i++) + { + Detection result; + int idx = nms_result[i]; + result.class_id = class_ids[idx]; + result.confidence = confidences[idx]; + result.box = boxes[idx]; + output.push_back(result); + } + + for (int i = 0; i < output.size(); i++) + { + auto detection = output[i]; + auto box = detection.box; + auto classId = detection.class_id; + auto confidence = detection.confidence; + + float xmax = box.x + box.width; + float ymax = box.y + box.height; + + boxes_x_.push_back(this->rx * box.x); + boxes_y_.push_back(this->ry * box.y); + boxes_w_.push_back(this->rx * box.width); + boxes_h_.push_back(this->ry * box.height); + + boxes_label_.push_back((int)detection.class_id); + boxes_score_.push_back(detection.confidence); + } +#endif + } + +} \ No newline at end of file diff --git a/algorithm/common_det/intel/common_det_intel_impl.h b/algorithm/common_det/intel/common_det_intel_impl.h new file mode 100644 index 0000000..ad4ac38 --- /dev/null +++ b/algorithm/common_det/intel/common_det_intel_impl.h @@ -0,0 +1,84 @@ +#ifndef __SV_COMMON_DET_INTEL__ +#define __SV_COMMON_DET_INTEL__ + +#include "sv_core.h" +#include +#include +#include +#include +#include +#include +#include + +#ifdef WITH_INTEL +#include +#endif + +struct Resize +{ + cv::Mat resized_image; + int dw; + int dh; +}; + +struct Detection +{ + int class_id; + float confidence; + cv::Rect box; +}; + +namespace sv +{ + + class CommonObjectDetectorIntelImpl + { + public: + CommonObjectDetectorIntelImpl(); + ~CommonObjectDetectorIntelImpl(); + + bool intelSetup(CommonObjectDetectorBase *base_, bool input_4k_); + void intelDetect( + CommonObjectDetectorBase *base_, + cv::Mat img_, + std::vector &boxes_x_, + std::vector &boxes_y_, + std::vector &boxes_w_, + std::vector &boxes_h_, + std::vector &boxes_label_, + std::vector &boxes_score_, + std::vector &boxes_seg_, + bool input_4k_); + void preprocess_img(cv::Mat &img_); + void preprocess_img_seg(cv::Mat &img_, std::vector &paddings); + void postprocess_img_seg(cv::Mat &img_, std::vector &paddings, std::vector &boxes_x_, std::vector &boxes_y_, std::vector &boxes_w_, std::vector &boxes_h_, + std::vector &boxes_label_, + std::vector &boxes_score_, + std::vector &boxes_seg_, + double &thrs_conf, + double &thrs_nms); + + void postprocess_img(std::vector &boxes_x_, std::vector &boxes_y_, std::vector &boxes_w_, + std::vector &boxes_h_, + std::vector &boxes_label_, + std::vector &boxes_score_, + double &thrs_conf, + double &thrs_nms); + +#ifdef WITH_INTEL + + int inpWidth; + int inpHeight; + bool with_segmentation; + float rx; // the width ratio of original image and resized image + float ry; // the height ratio of original image and resized image + Resize resize; + ov::Tensor input_tensor; + ov::InferRequest infer_request; + ov::CompiledModel compiled_model; + +#endif + }; + +} +#endif diff --git a/algorithm/common_det/sv_common_det.cpp b/algorithm/common_det/sv_common_det.cpp index 328a1bf..b64420e 100644 --- a/algorithm/common_det/sv_common_det.cpp +++ b/algorithm/common_det/sv_common_det.cpp @@ -8,6 +8,10 @@ #include "common_det_cuda_impl.h" #endif +#ifdef WITH_INTEL +#include +#include "common_det_intel_impl.h" +#endif namespace sv { @@ -18,6 +22,10 @@ CommonObjectDetector::CommonObjectDetector(bool input_4k) #ifdef WITH_CUDA this->_cuda_impl = new CommonObjectDetectorCUDAImpl; #endif + +#ifdef WITH_INTEL + this->_intel_impl = new CommonObjectDetectorIntelImpl; +#endif } CommonObjectDetector::~CommonObjectDetector() { @@ -28,6 +36,10 @@ bool CommonObjectDetector::setupImpl() #ifdef WITH_CUDA return this->_cuda_impl->cudaSetup(this, this->_input_4k); #endif + +#ifdef WITH_INTEL + return this->_intel_impl->intelSetup(this, this->_input_4k); +#endif return false; } @@ -56,11 +68,21 @@ void CommonObjectDetector::detectImpl( this->_input_4k ); #endif + +#ifdef WITH_INTEL + this->_intel_impl->intelDetect( + this, + img_, + boxes_x_, + boxes_y_, + boxes_w_, + boxes_h_, + boxes_label_, + boxes_score_, + boxes_seg_, + this->_input_4k); +#endif } - - - - } diff --git a/algorithm/landing_det/intel/landing_det_intel_impl.cpp b/algorithm/landing_det/intel/landing_det_intel_impl.cpp new file mode 100644 index 0000000..a989ebb --- /dev/null +++ b/algorithm/landing_det/intel/landing_det_intel_impl.cpp @@ -0,0 +1,95 @@ +#include "landing_det_intel_impl.h" +#include +#include + +#define SV_MODEL_DIR "/SpireCV/models/" +#define SV_ROOT_DIR "/SpireCV/" + +namespace sv +{ + +#ifdef WITH_INTEL + using namespace cv; + using namespace std; + using namespace dnn; +#endif + + LandingMarkerDetectorIntelImpl::LandingMarkerDetectorIntelImpl() + { + } + + LandingMarkerDetectorIntelImpl::~LandingMarkerDetectorIntelImpl() + { + } + + bool LandingMarkerDetectorIntelImpl::intelSetup() + { +#ifdef WITH_INTEL + std::string onnx_model_fn = get_home() + SV_MODEL_DIR + "LandingMarker.onnx"; + if (!is_file_exist(onnx_model_fn)) + { + throw std::runtime_error("SpireCV (104) Error loading the LandingMarker ONNX model (File Not Exist)"); + } + + // OpenVINO + ov::Core core; + std::shared_ptr model = core.read_model(onnx_model_fn); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); + ppp.input().tensor().set_element_type(ov::element::u8).set_layout("NHWC").set_color_format(ov::preprocess::ColorFormat::RGB); + ppp.input().preprocess().convert_element_type(ov::element::f32).convert_color(ov::preprocess::ColorFormat::RGB).scale({255, 255, 255}); // .scale({ 112, 112, 112 }); + ppp.input().model().set_layout("NCHW"); + ppp.output().tensor().set_element_type(ov::element::f32); + model = ppp.build(); + this->compiled_model = core.compile_model(model, "GPU"); + this->infer_request = compiled_model.create_infer_request(); + + return true; +#endif + return false; + } + + void LandingMarkerDetectorIntelImpl::intelRoiCNN( + std::vector &input_rois_, + std::vector &output_labels_) + { +#ifdef WITH_INTEL + output_labels_.clear(); + + for (int i = 0; i < input_rois_.size(); i++) + { + cv::Mat e_roi = input_rois_[i]; + + // Get input port for model with one input + auto input_port = compiled_model.input(); + // Create tensor from external memory + ov::Tensor input_tensor(input_port.get_element_type(), input_port.get_shape(), e_roi.ptr(0)); + // Set input tensor for model with one input + infer_request.set_input_tensor(input_tensor); + //preprocess_img(e_roi); + + // infer_request.infer(); + infer_request.start_async(); + infer_request.wait(); + + const ov::Tensor &output_tensor = infer_request.get_output_tensor(); + ov::Shape output_shape = output_tensor.get_shape(); + this->_p_prob = output_tensor.data(); + + // Find max index + double max = 0; + int label = 0; + for (int i = 0; i < 11; ++i) + { + if (max < this->_p_prob[i]) + { + max = this->_p_prob[i]; + label = i; + } + } + output_labels_.push_back(label); + } + +#endif + } + +} diff --git a/algorithm/landing_det/intel/landing_det_intel_impl.h b/algorithm/landing_det/intel/landing_det_intel_impl.h new file mode 100644 index 0000000..57aec46 --- /dev/null +++ b/algorithm/landing_det/intel/landing_det_intel_impl.h @@ -0,0 +1,37 @@ +#ifndef __SV_LANDING_DET_INTEL__ +#define __SV_LANDING_DET_INTEL__ + +#include "sv_core.h" +#include +#include +#include +#include +#include + +#ifdef WITH_INTEL +#include +#endif + +namespace sv +{ + class LandingMarkerDetectorIntelImpl + { + public: + LandingMarkerDetectorIntelImpl(); + ~LandingMarkerDetectorIntelImpl(); + + bool intelSetup(); + void intelRoiCNN( + std::vector &input_rois_, + std::vector &output_labels_); + +#ifdef WITH_INTEL + float *_p_prob; + + ov::Tensor input_tensor; + ov::InferRequest infer_request; + ov::CompiledModel compiled_model; +#endif + }; +} +#endif diff --git a/algorithm/landing_det/sv_landing_det.cpp b/algorithm/landing_det/sv_landing_det.cpp index 58264fb..480bc51 100644 --- a/algorithm/landing_det/sv_landing_det.cpp +++ b/algorithm/landing_det/sv_landing_det.cpp @@ -7,6 +7,10 @@ #include "landing_det_cuda_impl.h" #endif +#ifdef WITH_INTEL +#include +#include "landing_det_intel_impl.h" +#endif namespace sv { @@ -16,6 +20,10 @@ LandingMarkerDetector::LandingMarkerDetector() #ifdef WITH_CUDA this->_cuda_impl = new LandingMarkerDetectorCUDAImpl; #endif + +#ifdef WITH_INTEL + this->_intel_impl = new LandingMarkerDetectorIntelImpl; +#endif } LandingMarkerDetector::~LandingMarkerDetector() { @@ -26,6 +34,10 @@ bool LandingMarkerDetector::setupImpl() #ifdef WITH_CUDA return this->_cuda_impl->cudaSetup(); #endif + +#ifdef WITH_INTEL + return this->_intel_impl->intelSetup(); +#endif return false; } @@ -40,11 +52,13 @@ void LandingMarkerDetector::roiCNN( output_labels_ ); #endif -} - - - - - + +#ifdef WITH_INTEL + this->_intel_impl->intelRoiCNN( + input_rois_, + output_labels_ + ); +#endif +} } diff --git a/algorithm/sv_algorithm_base.cpp b/algorithm/sv_algorithm_base.cpp index 1745559..a543579 100644 --- a/algorithm/sv_algorithm_base.cpp +++ b/algorithm/sv_algorithm_base.cpp @@ -1002,6 +1002,11 @@ void CommonObjectDetectorBase::detect(cv::Mat img_, TargetsInFrame& tgts_, Box* if (this->_with_segmentation) { cv::Mat mask_j = boxes_seg[j].clone(); + #ifdef WITH_INTEL + tgt.setMask(mask_j); + #endif + + #ifdef WITH_CUDA int maskh = mask_j.rows, maskw = mask_j.cols; assert(maskh == maskw); @@ -1031,6 +1036,7 @@ void CommonObjectDetectorBase::detect(cv::Mat img_, TargetsInFrame& tgts_, Box* { tgt.setMask(mask_j); } + #endif } tgts_.targets.push_back(tgt); diff --git a/algorithm/veri/intel/veri_det_intel_impl.cpp b/algorithm/veri/intel/veri_det_intel_impl.cpp new file mode 100644 index 0000000..446af9c --- /dev/null +++ b/algorithm/veri/intel/veri_det_intel_impl.cpp @@ -0,0 +1,112 @@ +#include "veri_det_intel_impl.h" +#include +#include + +#define SV_MODEL_DIR "/SpireCV/models/" +#define SV_ROOT_DIR "/SpireCV/" + +#include +#include +int BAT = 1; +float cosineSimilarity(float *vec1, float *vec2, int size) +{ + // 计算向量的点积 + float dotProduct = 0.0f; + for (int i = 0; i < size; ++i) + { + dotProduct += vec1[i] * vec2[i]; + } + + // 计算向量的模长 + float magnitudeVec1 = 0.0f; + float magnitudeVec2 = 0.0f; + for (int i = 0; i < size; ++i) + { + magnitudeVec1 += vec1[i] * vec1[i]; + magnitudeVec2 += vec2[i] * vec2[i]; + } + magnitudeVec1 = std::sqrt(magnitudeVec1); + magnitudeVec2 = std::sqrt(magnitudeVec2); + + // 计算余弦相似性 + float similarity = dotProduct / (magnitudeVec1 * magnitudeVec2); + + return similarity; +} + +namespace sv +{ +#ifdef WITH_INTEL + using namespace cv; + using namespace std; + using namespace dnn; +#endif + + VeriDetectorIntelImpl::VeriDetectorIntelImpl() + { + } + + VeriDetectorIntelImpl::~VeriDetectorIntelImpl() + { + } + + bool VeriDetectorIntelImpl::intelSetup() + { +#ifdef WITH_INTEL + std::string onnx_model_fn = get_home() + SV_MODEL_DIR + "veri.onnx"; + if (!is_file_exist(onnx_model_fn)) + { + throw std::runtime_error("SpireCV (104) Error loading the VeriDetector openVINO model (File Not Exist)"); + } + + // OpenVINO + ov::Core core; + this->compiled_model = core.compile_model(onnx_model_fn, "GPU"); + this->infer_request = compiled_model.create_infer_request(); + + return true; +#endif + return false; + } + + void VeriDetectorIntelImpl::intelRoiCNN( + std::vector &input_rois_, + std::vector &output_labels_) + { +#ifdef WITH_INTEL + + Mat blobs; + blobFromImages(input_rois_, blobs, 1 / 255.0, Size(224, 224), Scalar(0, 0, 0), true, true); + + auto input_port = compiled_model.input(); + ov::Tensor input_tensor(input_port.get_element_type(), input_port.get_shape(), blobs.ptr(0)); + + infer_request.infer(); + + const ov::Tensor &label_pre = infer_request.get_output_tensor(0); + this->_p_prob1 = label_pre.data(); + + const ov::Tensor &proto_pre = infer_request.get_output_tensor(1); + this->_p_prob2 = proto_pre.data(); + + // Find max index + double max = 0; + int label = 0; + for (int i = 0; i < 576; ++i) + { + if (max < this->_p_prob1[i]) + { + max = this->_p_prob1[i]; + label = i; + } + } + + float similarity = cosineSimilarity(this->_p_prob2, this->_p_prob2 + 1280, 1280); + + output_labels_.push_back(label); + output_labels_.push_back(similarity); +#endif + } + + +} diff --git a/algorithm/veri/intel/veri_det_intel_impl.h b/algorithm/veri/intel/veri_det_intel_impl.h new file mode 100644 index 0000000..89123f6 --- /dev/null +++ b/algorithm/veri/intel/veri_det_intel_impl.h @@ -0,0 +1,41 @@ +#ifndef __SV_VERI_DET_INTEL__ +#define __SV_VERI_DET_INTEL__ + +#include "sv_core.h" +#include +#include +#include +#include +#include + +#ifdef WITH_INTEL +#include +#endif + +namespace sv +{ + class VeriDetectorIntelImpl + { + public: + VeriDetectorIntelImpl(); + ~VeriDetectorIntelImpl(); + + bool intelSetup(); + void intelRoiCNN( + std::vector &input_rois_, + std::vector &output_labels_); + +#ifdef WITH_INTEL + + float *_p_data; + float *_p_prob1; + float *_p_prob2; + + ov::Tensor input_tensor; + ov::InferRequest infer_request; + ov::CompiledModel compiled_model; +#endif + }; + +} +#endif diff --git a/algorithm/veri/sv_veri_det.cpp b/algorithm/veri/sv_veri_det.cpp index 17cba0d..591ef98 100644 --- a/algorithm/veri/sv_veri_det.cpp +++ b/algorithm/veri/sv_veri_det.cpp @@ -10,6 +10,11 @@ #include "veri_det_cuda_impl.h" #endif +#ifdef WITH_INTEL +#include +#include "veri_det_intel_impl.h" +#endif + #define SV_ROOT_DIR "/SpireCV/" namespace sv @@ -20,6 +25,10 @@ namespace sv #ifdef WITH_CUDA this->_cuda_impl = new VeriDetectorCUDAImpl; #endif + +#ifdef WITH_INTEL + this->_intel_impl = new VeriDetectorIntelImpl; +#endif } VeriDetector::~VeriDetector() { @@ -50,6 +59,9 @@ namespace sv return this->_cuda_impl->cudaSetup(); #endif +#ifdef WITH_INTEL + return this->_intel_impl->intelSetup(); +#endif return false; } @@ -63,6 +75,12 @@ namespace sv input_rois_, output_labels_); #endif + +#ifdef WITH_INTEL + this->_intel_impl->intelRoiCNN( + input_rois_, + output_labels_); +#endif } void VeriDetector::detect(cv::Mat img_, const cv::Rect &bounding_box_, sv::Target &tgt) @@ -94,23 +112,20 @@ namespace sv cv::Mat img_ground = cv::imread(img_ground_dir); cv::resize(img_ground, img_ground, cv::Size(224, 224)); std::vector input_rois_ = {crop, img_ground}; - -#ifdef WITH_CUDA + std::vector output_labels; +#ifdef WITH_CUDA roiCNN(input_rois_, output_labels); +#endif - // auto t1 = std::chrono::system_clock::now(); - // tgts_.setFPS(1000.0 / std::chrono::duration_cast(t1 - this->_t0).count()); - // this->_t0 = std::chrono::system_clock::now(); - // tgts_.setTimeNow(); - +#ifdef WITH_INTEL + roiCNN(input_rois_, output_labels); +#endif + if (output_labels.size() > 0) { - // tgt.category_id = output_labels[0]; tgt.sim_score = output_labels[1]; - // tgts_.targets.push_back(tgt); } -#endif } void VeriDetector::getSubwindow(cv::Mat &dstCrop, cv::Mat &srcImg, int originalSz, int resizeSz) diff --git a/include/sv_common_det.h b/include/sv_common_det.h index 30d39ab..f8511e9 100644 --- a/include/sv_common_det.h +++ b/include/sv_common_det.h @@ -8,33 +8,44 @@ #include #include - -namespace sv { - -class CommonObjectDetectorCUDAImpl; - -class CommonObjectDetector : public CommonObjectDetectorBase +namespace sv { -public: - CommonObjectDetector(bool input_4k=false); - ~CommonObjectDetector(); -protected: - bool setupImpl(); - void detectImpl( - cv::Mat img_, - std::vector& boxes_x_, - std::vector& boxes_y_, - std::vector& boxes_w_, - std::vector& boxes_h_, - std::vector& boxes_label_, - std::vector& boxes_score_, - std::vector& boxes_seg_ - ); +#ifdef WITH_CUDA + class CommonObjectDetectorCUDAImpl; +#endif - CommonObjectDetectorCUDAImpl* _cuda_impl; - bool _input_4k; -}; +#ifdef WITH_INTEL + class CommonObjectDetectorIntelImpl; +#endif + class CommonObjectDetector : public CommonObjectDetectorBase + { + public: + CommonObjectDetector(bool input_4k = false); + ~CommonObjectDetector(); + + protected: + bool setupImpl(); + void detectImpl( + cv::Mat img_, + std::vector &boxes_x_, + std::vector &boxes_y_, + std::vector &boxes_w_, + std::vector &boxes_h_, + std::vector &boxes_label_, + std::vector &boxes_score_, + std::vector &boxes_seg_); + +#ifdef WITH_CUDA + CommonObjectDetectorCUDAImpl *_cuda_impl; +#endif + +#ifdef WITH_INTEL + CommonObjectDetectorIntelImpl *_intel_impl; +#endif + + bool _input_4k; + }; } #endif diff --git a/include/sv_landing_det.h b/include/sv_landing_det.h index 493e021..76cd385 100644 --- a/include/sv_landing_det.h +++ b/include/sv_landing_det.h @@ -10,8 +10,13 @@ namespace sv { +#ifdef WITH_CUDA + class LandingMarkerDetectorCUDAImpl; +#endif -class LandingMarkerDetectorCUDAImpl; +#ifdef WITH_INTEL + class LandingMarkerDetectorIntelImpl; +#endif class LandingMarkerDetector : public LandingMarkerDetectorBase { @@ -25,7 +30,13 @@ protected: std::vector& output_labels_ ); - LandingMarkerDetectorCUDAImpl* _cuda_impl; +#ifdef WITH_CUDA + LandingMarkerDetectorCUDAImpl *_cuda_impl; +#endif + +#ifdef WITH_INTEL + LandingMarkerDetectorIntelImpl *_intel_impl; +#endif }; diff --git a/include/sv_veri_det.h b/include/sv_veri_det.h index 08e3a00..13a05fa 100644 --- a/include/sv_veri_det.h +++ b/include/sv_veri_det.h @@ -10,8 +10,13 @@ namespace sv { - +#ifdef WITH_CUDA class VeriDetectorCUDAImpl; +#endif + +#ifdef WITH_INTEL + class VeriDetectorIntelImpl; +#endif class VeriDetector : public LandingMarkerDetectorBase { @@ -35,8 +40,13 @@ namespace sv std::vector targetSz = {0, 0}; // H and W of bounding box std::vector targetPos = {0, 0}; // center point of bounding box (x, y) - +#ifdef WITH_CUDA VeriDetectorCUDAImpl *_cuda_impl; +#endif + +#ifdef WITH_INTEL + VeriDetectorIntelImpl *_intel_impl; +#endif }; }