diff --git a/CMakeLists.txt b/CMakeLists.txt index 44707a4..cb8a226 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,6 +76,7 @@ include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/sot/ocv470 ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/color_line ${CMAKE_CURRENT_SOURCE_DIR}/video_io + ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/veri/cuda ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/ellipse_det ${CMAKE_CURRENT_SOURCE_DIR}/utils ) @@ -116,6 +117,7 @@ set( include/sv_sot.h include/sv_mot.h include/sv_color_line.h + include/sv_veri_det.h include/sv_video_input.h include/sv_video_output.h include/sv_world.h @@ -155,6 +157,7 @@ set(spirecv_SRCS algorithm/ellipse_det/ellipse_detector.cpp algorithm/common_det/sv_common_det.cpp algorithm/landing_det/sv_landing_det.cpp + algorithm/veri/sv_veri_det.cpp algorithm/sot/sv_sot.cpp algorithm/mot/sv_mot.cpp algorithm/color_line/sv_color_line.cpp @@ -177,6 +180,8 @@ if(USE_CUDA) list(APPEND spirecv_SRCS ${ALG_SRC_FILES}) file(GLOB ALG_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/landing_det/cuda/*.cpp) list(APPEND spirecv_SRCS ${ALG_SRC_FILES}) + file(GLOB ALG_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/veri/cuda/*.cpp) + list(APPEND spirecv_SRCS ${ALG_SRC_FILES}) endif() if(USE_FFMPEG) @@ -263,6 +268,8 @@ add_executable(UdpDetectionInfoSender samples/demo/udp_detection_info_sender.cpp target_link_libraries(UdpDetectionInfoSender sv_world) add_executable(VideoSaving samples/demo/video_saving.cpp) target_link_libraries(VideoSaving sv_world) +add_executable(VERI samples/demo/veri.cpp) +target_link_libraries(VERI sv_world) add_executable(VideoStreaming samples/demo/video_streaming.cpp) target_link_libraries(VideoStreaming sv_world) add_executable(GimbalClickedTracking samples/demo/gimbal_detection_with_clicked_tracking.cpp) diff --git a/algorithm/veri/cuda/veri_det_cuda_impl.cpp b/algorithm/veri/cuda/veri_det_cuda_impl.cpp new file mode 100644 index 0000000..d5225ec --- /dev/null +++ b/algorithm/veri/cuda/veri_det_cuda_impl.cpp @@ -0,0 +1,189 @@ +#include "veri_det_cuda_impl.h" +#include +#include + +#define SV_MODEL_DIR "/SpireCV/models/" +#define SV_ROOT_DIR "/SpireCV/" + +#ifdef WITH_CUDA +#include "yolov7/logging.h" +#define TRTCHECK(status) \ + do \ + { \ + auto ret = (status); \ + if (ret != 0) \ + { \ + std::cerr << "Cuda failure: " << ret << std::endl; \ + abort(); \ + } \ + } while (0) + +#define DEVICE 0 // GPU id +#define BATCH_SIZE 1 + +#define MAX_IMAGE_INPUT_SIZE_THRESH 3000 * 3000 // ensure it exceed the maximum size in the input images ! +#endif + +#include +#include +int BAT = 1; +float cosineSimilarity(float *vec1, float *vec2, int size) +{ + // 计算向量的点积 + float dotProduct = 0.0f; + for (int i = 0; i < size; ++i) + { + dotProduct += vec1[i] * vec2[i]; + } + + // 计算向量的模长 + float magnitudeVec1 = 0.0f; + float magnitudeVec2 = 0.0f; + for (int i = 0; i < size; ++i) + { + magnitudeVec1 += vec1[i] * vec1[i]; + magnitudeVec2 += vec2[i] * vec2[i]; + } + magnitudeVec1 = std::sqrt(magnitudeVec1); + magnitudeVec2 = std::sqrt(magnitudeVec2); + + // 计算余弦相似性 + float similarity = dotProduct / (magnitudeVec1 * magnitudeVec2); + + return similarity; +} + +namespace sv +{ + + using namespace cv; + +#ifdef WITH_CUDA + using namespace nvinfer1; + static Logger g_nvlogger; +#endif + + VeriDetectorCUDAImpl::VeriDetectorCUDAImpl() + { + } + + VeriDetectorCUDAImpl::~VeriDetectorCUDAImpl() + { + } + + bool VeriDetectorCUDAImpl::cudaSetup() + { +#ifdef WITH_CUDA + std::string trt_model_fn = get_home() + SV_MODEL_DIR + "model.engine"; + if (!is_file_exist(trt_model_fn)) + { + throw std::runtime_error("SpireCV (104) Error loading the VERI TensorRT model (File Not Exist)"); + } + char *trt_model_stream{nullptr}; + size_t trt_model_size{0}; + try + { + std::ifstream file(trt_model_fn, std::ios::binary); + file.seekg(0, file.end); + trt_model_size = file.tellg(); + file.seekg(0, file.beg); + trt_model_stream = new char[trt_model_size]; + assert(trt_model_stream); + file.read(trt_model_stream, trt_model_size); + file.close(); + } + catch (const std::runtime_error &e) + { + throw std::runtime_error("SpireCV (104) Error loading the TensorRT model!"); + } + + // TensorRT + IRuntime *runtime = nvinfer1::createInferRuntime(g_nvlogger); + assert(runtime != nullptr); + ICudaEngine *p_cu_engine = runtime->deserializeCudaEngine(trt_model_stream, trt_model_size); + assert(p_cu_engine != nullptr); + this->_trt_context = p_cu_engine->createExecutionContext(); + assert(this->_trt_context != nullptr); + + delete[] trt_model_stream; + const ICudaEngine &cu_engine = this->_trt_context->getEngine(); + assert(cu_engine.getNbBindings() == 2); + + this->_input_index = cu_engine.getBindingIndex("input"); + this->_output_index1 = cu_engine.getBindingIndex("output"); + this->_output_index2 = cu_engine.getBindingIndex("/head/layers.0/act/Mul_output_0"); + TRTCHECK(cudaMalloc(&_p_buffers[this->_input_index], 2 * 3 * 224 * 224 * sizeof(float))); + TRTCHECK(cudaMalloc(&_p_buffers[this->_output_index1], 2 * 576 * sizeof(float))); + TRTCHECK(cudaMalloc(&_p_buffers[this->_output_index2], 2 * 1280 * sizeof(float))); + TRTCHECK(cudaStreamCreate(&_cu_stream)); + + auto input_dims = nvinfer1::Dims4{2, 3, 224, 224}; + this->_trt_context->setBindingDimensions(this->_input_index, input_dims); + + this->_p_data = new float[2 * 3 * 224 * 224]; + this->_p_prob1 = new float[2 * 576]; + this->_p_prob2 = new float[2 * 1280]; + this->_p_prob3 = new float[2 * 1280]; + // Input + TRTCHECK(cudaMemcpyAsync(_p_buffers[this->_input_index], this->_p_data, 2 * 3 * 224 * 224 * sizeof(float), cudaMemcpyHostToDevice, this->_cu_stream)); + // this->_trt_context->enqueue(1, _p_buffers, this->_cu_stream, nullptr); + this->_trt_context->enqueueV2(_p_buffers, this->_cu_stream, nullptr); + // Output + TRTCHECK(cudaMemcpyAsync(this->_p_prob1, _p_buffers[this->_output_index1], 2 * 576 * sizeof(float), cudaMemcpyDeviceToHost, this->_cu_stream)); + TRTCHECK(cudaMemcpyAsync(this->_p_prob2, _p_buffers[this->_output_index2], 2 * 1280 * sizeof(float), cudaMemcpyDeviceToHost, this->_cu_stream)); + cudaStreamSynchronize(this->_cu_stream); + return true; +#endif + return false; + } + + void VeriDetectorCUDAImpl::cudaRoiCNN( + std::vector &input_rois_, + std::vector &output_labels_) + { +#ifdef WITH_CUDA + + for (int i = 0; i < 2; ++i) + { + for (int row = 0; row < 224; ++row) + { + uchar *uc_pixel = input_rois_[i].data + row * input_rois_[i].step; // compute row id + for (int col = 0; col < 224; ++col) + { + // mean=[136.20, 141.50, 145.41], std=[44.77, 44.20, 44.30] + this->_p_data[224 * 224 * 3 * i + col + row * 224] = ((float)uc_pixel[0] - 136.20f) / 44.77f; + this->_p_data[224 * 224 * 3 * i + col + row * 224 + 224 * 224] = ((float)uc_pixel[1] - 141.50f) / 44.20f; + this->_p_data[224 * 224 * 3 * i + col + row * 224 + 224 * 224 * 2] = ((float)uc_pixel[2] - 145.41f) / 44.30f; + uc_pixel += 3; + } + } + } + + // Input + TRTCHECK(cudaMemcpyAsync(_p_buffers[this->_input_index], this->_p_data, 2 * 3 * 224 * 224 * sizeof(float), cudaMemcpyHostToDevice, this->_cu_stream)); + // this->_trt_context->enqueue(1, _p_buffers, this->_cu_stream, nullptr); + this->_trt_context->enqueueV2(_p_buffers, this->_cu_stream, nullptr); + // Output + TRTCHECK(cudaMemcpyAsync(this->_p_prob1, _p_buffers[this->_output_index1], 2 * 576 * sizeof(float), cudaMemcpyDeviceToHost, this->_cu_stream)); + TRTCHECK(cudaMemcpyAsync(this->_p_prob2, _p_buffers[this->_output_index2], 2 * 1280 * sizeof(float), cudaMemcpyDeviceToHost, this->_cu_stream)); + cudaStreamSynchronize(this->_cu_stream); + + // Find max index + double max = 0; + int label = 0; + for (int i = 0; i < 576; ++i) + { + if (max < this->_p_prob1[i]) + { + max = this->_p_prob1[i]; + label = i; + } + } + + // 计算两个数组的余弦相似性 + float similarity = cosineSimilarity(_p_prob2, _p_prob2 + 1280, 1280); + std::cout << "余弦相似性: " << similarity << std::endl; + std::cout << "VERI LABEL: " << label << std::endl; + } +#endif +} diff --git a/algorithm/veri/cuda/veri_det_cuda_impl.h b/algorithm/veri/cuda/veri_det_cuda_impl.h new file mode 100644 index 0000000..559a67f --- /dev/null +++ b/algorithm/veri/cuda/veri_det_cuda_impl.h @@ -0,0 +1,51 @@ +#ifndef __SV_VERI_DET_CUDA__ +#define __SV_VERI_DET_CUDA__ + +#include "sv_core.h" +#include +#include +#include +#include +#include + + + +#ifdef WITH_CUDA +#include +#include +#endif + + + +namespace sv { + + +class VeriDetectorCUDAImpl +{ +public: + VeriDetectorCUDAImpl(); + ~VeriDetectorCUDAImpl(); + + bool cudaSetup(); + void cudaRoiCNN( + std::vector& input_rois_, + std::vector& output_labels_ + ); + +#ifdef WITH_CUDA + float *_p_data; + float *_p_prob1; + float *_p_prob2; + float *_p_prob3; + nvinfer1::IExecutionContext *_trt_context; + int _input_index; + int _output_index1; + int _output_index2; + void *_p_buffers[3]; + cudaStream_t _cu_stream; +#endif +}; + + +} +#endif diff --git a/algorithm/veri/sv_veri_det.cpp b/algorithm/veri/sv_veri_det.cpp new file mode 100644 index 0000000..a7df6d7 --- /dev/null +++ b/algorithm/veri/sv_veri_det.cpp @@ -0,0 +1,61 @@ +#include "sv_veri_det.h" +#include +#include +#ifdef WITH_CUDA +#include +#include +#include "veri_det_cuda_impl.h" +#endif + + +namespace sv { + + +VeriDetector::VeriDetector() +{ + this->_cuda_impl = new VeriDetectorCUDAImpl; +} +VeriDetector::~VeriDetector() +{ +} + +bool VeriDetector::setupImpl() +{ +#ifdef WITH_CUDA + return this->_cuda_impl->cudaSetup(); +#endif + return false; +} + +void VeriDetector::roiCNN( + std::vector& input_rois_, + std::vector& output_labels_ +) +{ +#ifdef WITH_CUDA + this->_cuda_impl->cudaRoiCNN( + input_rois_, + output_labels_ + ); +#endif +} + + + +void VeriDetector::detect(cv::Mat img1_, cv::Mat img2_, TargetsInFrame& tgts_) +{ + if (!_params_loaded) + { + this->_load(); + this->_loadLabels(); + _params_loaded = true; + } + + std::vector e_roi = {img1_, img2_}; + + std::vector output_labels; + roiCNN(e_roi, output_labels); +} + +} + diff --git a/include/sv_veri_det.h b/include/sv_veri_det.h new file mode 100644 index 0000000..d372919 --- /dev/null +++ b/include/sv_veri_det.h @@ -0,0 +1,36 @@ +#ifndef __SV_VERI_DET__ +#define __SV_VERI_DET__ + +#include "sv_core.h" +#include +#include +#include +#include +#include + + +namespace sv { + +class VeriDetectorCUDAImpl; + +class VeriDetector : public LandingMarkerDetectorBase +{ +public: + VeriDetector(); + ~VeriDetector(); + + void detect(cv::Mat img1_, cv::Mat img2_, TargetsInFrame &tgts_); + +protected: + bool setupImpl(); + void roiCNN( + std::vector& input_rois_, + std::vector& output_labels_ + ); + + VeriDetectorCUDAImpl* _cuda_impl; +}; + + +} +#endif diff --git a/include/sv_world.h b/include/sv_world.h index 29ccde5..4122f9d 100644 --- a/include/sv_world.h +++ b/include/sv_world.h @@ -7,6 +7,7 @@ #include "sv_sot.h" #include "sv_mot.h" #include "sv_color_line.h" +#include "sv_veri_det.h" #include "sv_video_input.h" #include "sv_video_output.h" diff --git a/samples/demo/camera_reading.cpp b/samples/demo/camera_reading.cpp index 16d085d..e3ac844 100644 --- a/samples/demo/camera_reading.cpp +++ b/samples/demo/camera_reading.cpp @@ -25,3 +25,4 @@ int main(int argc, char *argv[]) { return 0; } + diff --git a/samples/demo/veri.cpp b/samples/demo/veri.cpp new file mode 100644 index 0000000..cbe71b6 --- /dev/null +++ b/samples/demo/veri.cpp @@ -0,0 +1,45 @@ +#include +#include +// 包含SpireCV SDK头文件 +#include + +using namespace std; + +int main(int argc, char *argv[]) { + // 打开摄像头 + sv::VeriDetector veri; + veri.loadCameraParams(sv::get_home() + "/SpireCV/calib_webcam_640x480.yaml"); + + cv::VideoCapture cap1("/home/amov/Videos/com/FlyVideo_2023-09-02_11-36-00.avi"); + cv::VideoCapture cap2("/home/amov/Videos/com/FlyVideo_2023-09-02_11-41-55.avi"); + // cap.setWH(640, 480); + // cap.setFps(30); + //cap.open(sv::CameraType::WEBCAM, 0); // CameraID 0 + // 实例化OpenCV的Mat类，用于内存单帧图像 + + + cv::Mat img1,img2; + int frame_id = 0; + while (1) + { + // 实例化SpireCV的 单帧检测结果 接口类 TargetsInFrame + sv::TargetsInFrame tgts(frame_id++); + + // 读取一帧图像到img + cap1.read(img1); + cap2.read(img2); + + cv::resize(img1, img1, cv::Size(224, 224)); + cv::resize(img2, img2, cv::Size(224, 224)); + + veri.detect(img1, img2, tgts); + + + + // 显示img + // cv::imshow("img", img); + // cv::waitKey(10); + } + + return 0; +} diff --git a/sv_algorithm_params.json b/sv_algorithm_params.json new file mode 100644 index 0000000..35fb108 --- /dev/null +++ b/sv_algorithm_params.json @@ -0,0 +1,187 @@ +{ + "CommonObjectDetector": { + "dataset": "COCO", + "inputSize": 1280, + "nmsThrs": 0.6, + "scoreThrs": 0.3, + "useWidthOrHeight": 1, + "withSegmentation": false, + "datasetPersonVehicle": { + "person": [0.5, 1.8], + "car": [4.1, 1.5], + "bus": [10, 3], + "truck": [-1, -1], + "bike": [-1, -1], + "train": [-1, -1], + "boat": [-1, -1], + "aeroplane": [-1, -1] + }, + "datasetDrone": { + "drone": [0.4, 0.2] + }, + "datasetCOCO": { + "person": [-1, -1], + "bicycle": [-1, -1], + "car": [-1, -1], + "motorcycle": [-1, -1], + "airplane": [-1, -1], + "bus": [-1, -1], + "train": [-1, -1], + "truck": [-1, -1], + "boat": [-1, -1], + "traffic light": [-1, -1], + "fire hydrant": [-1, -1], + "stop sign": [-1, -1], + "parking meter": [-1, -1], + "bench": [-1, -1], + "bird": [-1, -1], + "cat": [-1, -1], + "dog": [-1, -1], + "horse": [-1, -1], + "sheep": [-1, -1], + "cow": [-1, -1], + "elephant": [-1, -1], + "bear": [-1, -1], + "zebra": [-1, -1], + "giraffe": [-1, -1], + "backpack": [-1, -1], + "umbrella": [-1, -1], + "handbag": [-1, -1], + "tie": [-1, -1], + "suitcase": [-1, -1], + "frisbee": [-1, -1], + "skis": [-1, -1], + "snowboard": [-1, -1], + "sports ball": [-1, -1], + "kite": [-1, -1], + "baseball bat": [-1, -1], + "baseball glove": [-1, -1], + "skateboard": [-1, -1], + "surfboard": [-1, -1], + "tennis racket": [-1, -1], + "bottle": [-1, -1], + "wine glass": [-1, -1], + "cup": [-1, -1], + "fork": [-1, -1], + "knife": [-1, -1], + "spoon": [-1, -1], + "bowl": [-1, -1], + "banana": [-1, -1], + "apple": [-1, -1], + "sandwich": [-1, -1], + "orange": [-1, -1], + "broccoli": [-1, -1], + "carrot": [-1, -1], + "hot dog": [-1, -1], + "pizza": [-1, -1], + "donut": [-1, -1], + "cake": [-1, -1], + "chair": [-1, -1], + "couch": [-1, -1], + "potted plant": [-1, -1], + "bed": [-1, -1], + "dining table": [-1, -1], + "toilet": [-1, -1], + "tv": [-1, -1], + "laptop": [-1, -1], + "mouse": [-1, -1], + "remote": [-1, -1], + "keyboard": [-1, -1], + "cell phone": [-1, -1], + "microwave": [-1, -1], + "oven": [-1, -1], + "toaster": [-1, -1], + "sink": [-1, -1], + "refrigerator": [-1, -1], + "book": [-1, -1], + "clock": [-1, -1], + "vase": [-1, -1], + "scissors": [-1, -1], + "teddy bear": [-1, -1], + "hair drier": [-1, -1], + "toothbrush": [-1, -1] + } + }, + "AutoFocusObjectDetector": { + "lock_thres": 5, + "unlock_thres": 5, + "lock_scale_init": 12.0, + "lock_scale": 8.0, + "categories_filter": [], + "keep_unlocked": false, + "use_square_region": false + }, + "SingleObjectTracker": { + "algorithm": "nano", + "backend": 0, + "target": 0, + "useWidthOrHeight": 0, + "sigleobjectW":2, + "sigleobjectH":3 + }, + "LandingMarkerDetector": { + "labels": ["h","x","1","2","8"], + "maxCandidates": 3 + }, + "EllipseDetector": { + "radiusInMeter": 0.1535, + "preProcessingGaussKernel": 5, + "preProcessingGaussSigma": 1.306, + "thPosition": 1.0, + "maxCenterDistance": 0.05, + "minEdgeLength": 9, + "minOrientedRectSide": 2.984, + "distanceToEllipseContour": 0.111, + "minScore": 0.7, + "minReliability": 0.470, + "ns": 22, + "percentNe": 0.99, + "T_CNC": 0.121, + "T_TCN_L": 0.468, + "T_TCN_P": 0.560, + "thRadius": 0.202 + }, + "ArucoDetector": { + "dictionaryId": 10, + "markerIds": [-1], + "markerLengths": [0.2], + "adaptiveThreshConstant": 35, + "adaptiveThreshWinSizeMax": 23, + "adaptiveThreshWinSizeMin": 3, + "adaptiveThreshWinSizeStep": 10, + "aprilTagCriticalRad": 0.17453292519, + "aprilTagDeglitch": 0, + "aprilTagMaxLineFitMse": 10.0, + "aprilTagMaxNmaxima": 10, + "aprilTagMinClusterPixels": 5, + "aprilTagMinWhiteBlackDiff": 5, + "aprilTagQuadDecimate": 0.0, + "aprilTagQuadSigma": 0.0, + "cornerRefinementMaxIterations": 30, + "cornerRefinementMethod": 0, + "cornerRefinementMinAccuracy": 0.1, + "cornerRefinementWinSize": 5, + "detectInvertedMarker": false, + "errorCorrectionRate": 0.6, + "markerBorderBits": 1, + "maxErroneousBitsInBorderRate": 0.35, + "maxMarkerPerimeterRate": 4.0, + "minCornerDistanceRate": 0.05, + "minDistanceToBorder": 3, + "minMarkerDistanceRate": 0.05, + "minMarkerLengthRatioOriginalImg": 0, + "minMarkerPerimeterRate": 0.03, + "minOtsuStdDev": 5.0, + "minSideLengthCanonicalImg": 32, + "perspectiveRemoveIgnoredMarginPerCell": 0.13, + "perspectiveRemovePixelPerCell": 4, + "polygonalApproxAccuracyRate": 0.03, + "useAruco3Detection": false + }, + "ColorLineDetector": { + "line_color": "black", + "line_location": 0.5, + "line_location_a1": 0.3, + "line_location_a2": 0.7 + } +}