!19 VERI test Edition

Merge pull request !19 from Daniel/lx

CMakeLists.txt

@@ -76,6 +76,7 @@ include_directories(
   ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/sot/ocv470
   ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/color_line
   ${CMAKE_CURRENT_SOURCE_DIR}/video_io
+    ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/veri/cuda
   ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/ellipse_det
   ${CMAKE_CURRENT_SOURCE_DIR}/utils
 )
@@ -116,6 +117,7 @@ set(
   include/sv_sot.h
   include/sv_mot.h
   include/sv_color_line.h
+  include/sv_veri_det.h
   include/sv_video_input.h
   include/sv_video_output.h
   include/sv_world.h
@@ -155,6 +157,7 @@ set(spirecv_SRCS
   algorithm/ellipse_det/ellipse_detector.cpp
   algorithm/common_det/sv_common_det.cpp
   algorithm/landing_det/sv_landing_det.cpp
+  algorithm/veri/sv_veri_det.cpp
   algorithm/sot/sv_sot.cpp
   algorithm/mot/sv_mot.cpp
   algorithm/color_line/sv_color_line.cpp
@@ -177,6 +180,8 @@ if(USE_CUDA)
   list(APPEND spirecv_SRCS ${ALG_SRC_FILES})
   file(GLOB ALG_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/landing_det/cuda/*.cpp)
   list(APPEND spirecv_SRCS ${ALG_SRC_FILES})
+  file(GLOB ALG_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/veri/cuda/*.cpp)
+  list(APPEND spirecv_SRCS ${ALG_SRC_FILES})
 endif()
 
 if(USE_FFMPEG)
@@ -263,6 +268,8 @@ add_executable(UdpDetectionInfoSender samples/demo/udp_detection_info_sender.cpp
 target_link_libraries(UdpDetectionInfoSender sv_world)
 add_executable(VideoSaving samples/demo/video_saving.cpp)
 target_link_libraries(VideoSaving sv_world)
+add_executable(VERI samples/demo/veri.cpp)
+target_link_libraries(VERI sv_world)
 add_executable(VideoStreaming samples/demo/video_streaming.cpp)
 target_link_libraries(VideoStreaming sv_world)
 add_executable(GimbalClickedTracking samples/demo/gimbal_detection_with_clicked_tracking.cpp)

algorithm/veri/cuda/veri_det_cuda_impl.cpp

@@ -0,0 +1,189 @@
+#include "veri_det_cuda_impl.h"
+#include <cmath>
+#include <fstream>
+
+#define SV_MODEL_DIR "/SpireCV/models/"
+#define SV_ROOT_DIR "/SpireCV/"
+
+#ifdef WITH_CUDA
+#include "yolov7/logging.h"
+#define TRTCHECK(status)                                 \
+  do                                                     \
+  {                                                      \
+    auto ret = (status);                                 \
+    if (ret != 0)                                        \
+    {                                                    \
+      std::cerr << "Cuda failure: " << ret << std::endl; \
+      abort();                                           \
+    }                                                    \
+  } while (0)
+
+#define DEVICE 0 // GPU id
+#define BATCH_SIZE 1
+
+#define MAX_IMAGE_INPUT_SIZE_THRESH 3000 * 3000 // ensure it exceed the maximum size in the input images !
+#endif
+
+#include <iostream>
+#include <cmath>
+int BAT = 1;
+float cosineSimilarity(float *vec1, float *vec2, int size)
+{
+  // 计算向量的点积
+  float dotProduct = 0.0f;
+  for (int i = 0; i < size; ++i)
+  {
+    dotProduct += vec1[i] * vec2[i];
+  }
+
+  // 计算向量的模长
+  float magnitudeVec1 = 0.0f;
+  float magnitudeVec2 = 0.0f;
+  for (int i = 0; i < size; ++i)
+  {
+    magnitudeVec1 += vec1[i] * vec1[i];
+    magnitudeVec2 += vec2[i] * vec2[i];
+  }
+  magnitudeVec1 = std::sqrt(magnitudeVec1);
+  magnitudeVec2 = std::sqrt(magnitudeVec2);
+
+  // 计算余弦相似性
+  float similarity = dotProduct / (magnitudeVec1 * magnitudeVec2);
+
+  return similarity;
+}
+
+namespace sv
+{
+
+  using namespace cv;
+
+#ifdef WITH_CUDA
+  using namespace nvinfer1;
+  static Logger g_nvlogger;
+#endif
+
+  VeriDetectorCUDAImpl::VeriDetectorCUDAImpl()
+  {
+  }
+
+  VeriDetectorCUDAImpl::~VeriDetectorCUDAImpl()
+  {
+  }
+
+  bool VeriDetectorCUDAImpl::cudaSetup()
+  {
+#ifdef WITH_CUDA
+    std::string trt_model_fn = get_home() + SV_MODEL_DIR + "model.engine";
+    if (!is_file_exist(trt_model_fn))
+    {
+      throw std::runtime_error("SpireCV (104) Error loading the VERI TensorRT model (File Not Exist)");
+    }
+    char *trt_model_stream{nullptr};
+    size_t trt_model_size{0};
+    try
+    {
+      std::ifstream file(trt_model_fn, std::ios::binary);
+      file.seekg(0, file.end);
+      trt_model_size = file.tellg();
+      file.seekg(0, file.beg);
+      trt_model_stream = new char[trt_model_size];
+      assert(trt_model_stream);
+      file.read(trt_model_stream, trt_model_size);
+      file.close();
+    }
+    catch (const std::runtime_error &e)
+    {
+      throw std::runtime_error("SpireCV (104) Error loading the TensorRT model!");
+    }
+
+    // TensorRT
+    IRuntime *runtime = nvinfer1::createInferRuntime(g_nvlogger);
+    assert(runtime != nullptr);
+    ICudaEngine *p_cu_engine = runtime->deserializeCudaEngine(trt_model_stream, trt_model_size);
+    assert(p_cu_engine != nullptr);
+    this->_trt_context = p_cu_engine->createExecutionContext();
+    assert(this->_trt_context != nullptr);
+
+    delete[] trt_model_stream;
+    const ICudaEngine &cu_engine = this->_trt_context->getEngine();
+    assert(cu_engine.getNbBindings() == 2);
+
+    this->_input_index = cu_engine.getBindingIndex("input");
+    this->_output_index1 = cu_engine.getBindingIndex("output");
+    this->_output_index2 = cu_engine.getBindingIndex("/head/layers.0/act/Mul_output_0");
+    TRTCHECK(cudaMalloc(&_p_buffers[this->_input_index], 2 * 3 * 224 * 224 * sizeof(float)));
+    TRTCHECK(cudaMalloc(&_p_buffers[this->_output_index1], 2 * 576 * sizeof(float)));
+    TRTCHECK(cudaMalloc(&_p_buffers[this->_output_index2], 2 * 1280 * sizeof(float)));
+    TRTCHECK(cudaStreamCreate(&_cu_stream));
+
+    auto input_dims = nvinfer1::Dims4{2, 3, 224, 224};
+    this->_trt_context->setBindingDimensions(this->_input_index, input_dims);
+
+    this->_p_data = new float[2 * 3 * 224 * 224];
+    this->_p_prob1 = new float[2 * 576];
+    this->_p_prob2 = new float[2 * 1280];
+    this->_p_prob3 = new float[2 * 1280];
+    // Input
+    TRTCHECK(cudaMemcpyAsync(_p_buffers[this->_input_index], this->_p_data, 2 * 3 * 224 * 224 * sizeof(float), cudaMemcpyHostToDevice, this->_cu_stream));
+    // this->_trt_context->enqueue(1, _p_buffers, this->_cu_stream, nullptr);
+    this->_trt_context->enqueueV2(_p_buffers, this->_cu_stream, nullptr);
+    // Output
+    TRTCHECK(cudaMemcpyAsync(this->_p_prob1, _p_buffers[this->_output_index1], 2 * 576 * sizeof(float), cudaMemcpyDeviceToHost, this->_cu_stream));
+    TRTCHECK(cudaMemcpyAsync(this->_p_prob2, _p_buffers[this->_output_index2], 2 * 1280 * sizeof(float), cudaMemcpyDeviceToHost, this->_cu_stream));
+    cudaStreamSynchronize(this->_cu_stream);
+    return true;
+#endif
+    return false;
+  }
+
+  void VeriDetectorCUDAImpl::cudaRoiCNN(
+      std::vector<cv::Mat> &input_rois_,
+      std::vector<int> &output_labels_)
+  {
+#ifdef WITH_CUDA
+
+    for (int i = 0; i < 2; ++i)
+    {
+      for (int row = 0; row < 224; ++row)
+      {
+        uchar *uc_pixel = input_rois_[i].data + row * input_rois_[i].step; // compute row id
+        for (int col = 0; col < 224; ++col)
+        {
+          // mean=[136.20, 141.50, 145.41], std=[44.77, 44.20, 44.30]
+          this->_p_data[224 * 224 * 3 * i + col + row * 224] = ((float)uc_pixel[0] - 136.20f) / 44.77f;
+          this->_p_data[224 * 224 * 3 * i + col + row * 224 + 224 * 224] = ((float)uc_pixel[1] - 141.50f) / 44.20f;
+          this->_p_data[224 * 224 * 3 * i + col + row * 224 + 224 * 224 * 2] = ((float)uc_pixel[2] - 145.41f) / 44.30f;
+          uc_pixel += 3;
+        }
+      }
+    }
+
+    // Input
+    TRTCHECK(cudaMemcpyAsync(_p_buffers[this->_input_index], this->_p_data, 2 * 3 * 224 * 224 * sizeof(float), cudaMemcpyHostToDevice, this->_cu_stream));
+    // this->_trt_context->enqueue(1, _p_buffers, this->_cu_stream, nullptr);
+    this->_trt_context->enqueueV2(_p_buffers, this->_cu_stream, nullptr);
+    // Output
+    TRTCHECK(cudaMemcpyAsync(this->_p_prob1, _p_buffers[this->_output_index1], 2 * 576 * sizeof(float), cudaMemcpyDeviceToHost, this->_cu_stream));
+    TRTCHECK(cudaMemcpyAsync(this->_p_prob2, _p_buffers[this->_output_index2], 2 * 1280 * sizeof(float), cudaMemcpyDeviceToHost, this->_cu_stream));
+    cudaStreamSynchronize(this->_cu_stream);
+
+    // Find max index
+    double max = 0;
+    int label = 0;
+    for (int i = 0; i < 576; ++i)
+    {
+      if (max < this->_p_prob1[i])
+      {
+        max = this->_p_prob1[i];
+        label = i;
+      }
+    }
+
+    // 计算两个数组的余弦相似性
+    float similarity = cosineSimilarity(_p_prob2, _p_prob2 + 1280, 1280);
+    std::cout << "余弦相似性: " << similarity << std::endl;
+    std::cout << "VERI LABEL: " << label << std::endl;
+  }
+#endif
+}

algorithm/veri/cuda/veri_det_cuda_impl.h

@@ -0,0 +1,51 @@
+#ifndef __SV_VERI_DET_CUDA__
+#define __SV_VERI_DET_CUDA__
+
+#include "sv_core.h"
+#include <opencv2/opencv.hpp>
+#include <opencv2/aruco.hpp>
+#include <opencv2/tracking.hpp>
+#include <string>
+#include <chrono>
+
+
+
+#ifdef WITH_CUDA
+#include <NvInfer.h>
+#include <cuda_runtime_api.h>
+#endif
+
+
+
+namespace sv {
+
+
+class VeriDetectorCUDAImpl
+{
+public:
+  VeriDetectorCUDAImpl();
+  ~VeriDetectorCUDAImpl();
+
+  bool cudaSetup();
+  void cudaRoiCNN(
+    std::vector<cv::Mat>& input_rois_,
+    std::vector<int>& output_labels_
+  );
+
+#ifdef WITH_CUDA
+  float *_p_data;
+  float *_p_prob1;
+  float *_p_prob2;
+  float *_p_prob3;
+  nvinfer1::IExecutionContext *_trt_context;
+  int _input_index;
+  int _output_index1;
+  int _output_index2;
+  void *_p_buffers[3];
+  cudaStream_t _cu_stream;
+#endif
+};
+
+
+}
+#endif

algorithm/veri/sv_veri_det.cpp

@@ -0,0 +1,61 @@
+#include "sv_veri_det.h"
+#include <cmath>
+#include <fstream>
+#ifdef WITH_CUDA
+#include <NvInfer.h>
+#include <cuda_runtime_api.h>
+#include "veri_det_cuda_impl.h"
+#endif
+
+
+namespace sv {
+
+
+VeriDetector::VeriDetector()
+{
+  this->_cuda_impl = new VeriDetectorCUDAImpl;
+}
+VeriDetector::~VeriDetector()
+{
+}
+
+bool VeriDetector::setupImpl()
+{
+#ifdef WITH_CUDA
+  return this->_cuda_impl->cudaSetup();
+#endif
+  return false;
+}
+
+void VeriDetector::roiCNN(
+  std::vector<cv::Mat>& input_rois_,
+  std::vector<int>& output_labels_
+)
+{
+#ifdef WITH_CUDA
+  this->_cuda_impl->cudaRoiCNN(
+    input_rois_,
+    output_labels_
+  );
+#endif
+}
+
+
+
+void VeriDetector::detect(cv::Mat img1_, cv::Mat img2_, TargetsInFrame& tgts_)
+{
+  if (!_params_loaded)
+  {
+    this->_load();
+    this->_loadLabels();
+    _params_loaded = true;
+  }
+
+  std::vector<cv::Mat> e_roi = {img1_, img2_};
+
+  std::vector<int> output_labels;
+  roiCNN(e_roi, output_labels);
+}
+
+}
+

include/sv_veri_det.h

@@ -0,0 +1,36 @@
+#ifndef __SV_VERI_DET__
+#define __SV_VERI_DET__
+
+#include "sv_core.h"
+#include <opencv2/opencv.hpp>
+#include <opencv2/aruco.hpp>
+#include <opencv2/tracking.hpp>
+#include <string>
+#include <chrono>
+
+
+namespace sv {
+
+class VeriDetectorCUDAImpl;
+
+class VeriDetector : public LandingMarkerDetectorBase
+{
+public:
+  VeriDetector();
+  ~VeriDetector();
+
+  void detect(cv::Mat img1_, cv::Mat img2_, TargetsInFrame &tgts_);
+
+protected:
+  bool setupImpl();
+  void roiCNN(
+    std::vector<cv::Mat>& input_rois_,
+    std::vector<int>& output_labels_
+  );
+
+  VeriDetectorCUDAImpl* _cuda_impl;
+};
+
+
+}
+#endif

include/sv_world.h

@@ -7,6 +7,7 @@
 #include "sv_sot.h"
 #include "sv_mot.h"
 #include "sv_color_line.h"
+#include "sv_veri_det.h"
 #include "sv_video_input.h"
 #include "sv_video_output.h"
 

samples/demo/camera_reading.cpp

@@ -25,3 +25,4 @@ int main(int argc, char *argv[]) {
 
   return 0;
 }
+

samples/demo/veri.cpp

@@ -0,0 +1,45 @@
+#include <iostream>
+#include <string>
+// 包含SpireCV SDK头文件
+#include <sv_world.h>
+
+using namespace std;
+
+int main(int argc, char *argv[]) {  
+  // 打开摄像头
+  sv::VeriDetector veri;
+  veri.loadCameraParams(sv::get_home() + "/SpireCV/calib_webcam_640x480.yaml");
+
+  cv::VideoCapture cap1("/home/amov/Videos/com/FlyVideo_2023-09-02_11-36-00.avi");
+  cv::VideoCapture cap2("/home/amov/Videos/com/FlyVideo_2023-09-02_11-41-55.avi");
+  // cap.setWH(640, 480);
+  // cap.setFps(30);
+  //cap.open(sv::CameraType::WEBCAM, 0);  // CameraID 0
+  // 实例化OpenCV的Mat类，用于内存单帧图像
+
+
+  cv::Mat img1,img2;
+  int frame_id = 0;
+  while (1)
+  {
+    // 实例化SpireCV的 单帧检测结果 接口类 TargetsInFrame
+    sv::TargetsInFrame tgts(frame_id++);
+
+    // 读取一帧图像到img
+    cap1.read(img1);
+    cap2.read(img2);
+
+    cv::resize(img1, img1, cv::Size(224, 224));
+    cv::resize(img2, img2, cv::Size(224, 224));
+
+    veri.detect(img1, img2, tgts);
+
+    
+
+    // 显示img
+    // cv::imshow("img", img);
+    // cv::waitKey(10);
+  }
+
+  return 0;
+}