VERI test Edition

This commit is contained in:
lxm 2023-09-12 10:33:16 +08:00
parent 0c489ad9cb
commit 6d7d666e2c
9 changed files with 578 additions and 0 deletions

View File

@ -76,6 +76,7 @@ include_directories(
${CMAKE_CURRENT_SOURCE_DIR}/algorithm/sot/ocv470
${CMAKE_CURRENT_SOURCE_DIR}/algorithm/color_line
${CMAKE_CURRENT_SOURCE_DIR}/video_io
${CMAKE_CURRENT_SOURCE_DIR}/algorithm/veri/cuda
${CMAKE_CURRENT_SOURCE_DIR}/algorithm/ellipse_det
${CMAKE_CURRENT_SOURCE_DIR}/utils
)
@ -116,6 +117,7 @@ set(
include/sv_sot.h
include/sv_mot.h
include/sv_color_line.h
include/sv_veri_det.h
include/sv_video_input.h
include/sv_video_output.h
include/sv_world.h
@ -155,6 +157,7 @@ set(spirecv_SRCS
algorithm/ellipse_det/ellipse_detector.cpp
algorithm/common_det/sv_common_det.cpp
algorithm/landing_det/sv_landing_det.cpp
algorithm/veri/sv_veri_det.cpp
algorithm/sot/sv_sot.cpp
algorithm/mot/sv_mot.cpp
algorithm/color_line/sv_color_line.cpp
@ -177,6 +180,8 @@ if(USE_CUDA)
list(APPEND spirecv_SRCS ${ALG_SRC_FILES})
file(GLOB ALG_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/landing_det/cuda/*.cpp)
list(APPEND spirecv_SRCS ${ALG_SRC_FILES})
file(GLOB ALG_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/algorithm/veri/cuda/*.cpp)
list(APPEND spirecv_SRCS ${ALG_SRC_FILES})
endif()
if(USE_FFMPEG)
@ -263,6 +268,8 @@ add_executable(UdpDetectionInfoSender samples/demo/udp_detection_info_sender.cpp
target_link_libraries(UdpDetectionInfoSender sv_world)
add_executable(VideoSaving samples/demo/video_saving.cpp)
target_link_libraries(VideoSaving sv_world)
add_executable(VERI samples/demo/veri.cpp)
target_link_libraries(VERI sv_world)
add_executable(VideoStreaming samples/demo/video_streaming.cpp)
target_link_libraries(VideoStreaming sv_world)
add_executable(GimbalClickedTracking samples/demo/gimbal_detection_with_clicked_tracking.cpp)

View File

@ -0,0 +1,189 @@
#include "veri_det_cuda_impl.h"
#include <cmath>
#include <fstream>
#define SV_MODEL_DIR "/SpireCV/models/"
#define SV_ROOT_DIR "/SpireCV/"
#ifdef WITH_CUDA
#include "yolov7/logging.h"
#define TRTCHECK(status) \
do \
{ \
auto ret = (status); \
if (ret != 0) \
{ \
std::cerr << "Cuda failure: " << ret << std::endl; \
abort(); \
} \
} while (0)
#define DEVICE 0 // GPU id
#define BATCH_SIZE 1
#define MAX_IMAGE_INPUT_SIZE_THRESH 3000 * 3000 // ensure it exceed the maximum size in the input images !
#endif
#include <iostream>
#include <cmath>
int BAT = 1;
float cosineSimilarity(float *vec1, float *vec2, int size)
{
// 计算向量的点积
float dotProduct = 0.0f;
for (int i = 0; i < size; ++i)
{
dotProduct += vec1[i] * vec2[i];
}
// 计算向量的模长
float magnitudeVec1 = 0.0f;
float magnitudeVec2 = 0.0f;
for (int i = 0; i < size; ++i)
{
magnitudeVec1 += vec1[i] * vec1[i];
magnitudeVec2 += vec2[i] * vec2[i];
}
magnitudeVec1 = std::sqrt(magnitudeVec1);
magnitudeVec2 = std::sqrt(magnitudeVec2);
// 计算余弦相似性
float similarity = dotProduct / (magnitudeVec1 * magnitudeVec2);
return similarity;
}
namespace sv
{
using namespace cv;
#ifdef WITH_CUDA
using namespace nvinfer1;
static Logger g_nvlogger;
#endif
VeriDetectorCUDAImpl::VeriDetectorCUDAImpl()
{
}
VeriDetectorCUDAImpl::~VeriDetectorCUDAImpl()
{
}
bool VeriDetectorCUDAImpl::cudaSetup()
{
#ifdef WITH_CUDA
std::string trt_model_fn = get_home() + SV_MODEL_DIR + "model.engine";
if (!is_file_exist(trt_model_fn))
{
throw std::runtime_error("SpireCV (104) Error loading the VERI TensorRT model (File Not Exist)");
}
char *trt_model_stream{nullptr};
size_t trt_model_size{0};
try
{
std::ifstream file(trt_model_fn, std::ios::binary);
file.seekg(0, file.end);
trt_model_size = file.tellg();
file.seekg(0, file.beg);
trt_model_stream = new char[trt_model_size];
assert(trt_model_stream);
file.read(trt_model_stream, trt_model_size);
file.close();
}
catch (const std::runtime_error &e)
{
throw std::runtime_error("SpireCV (104) Error loading the TensorRT model!");
}
// TensorRT
IRuntime *runtime = nvinfer1::createInferRuntime(g_nvlogger);
assert(runtime != nullptr);
ICudaEngine *p_cu_engine = runtime->deserializeCudaEngine(trt_model_stream, trt_model_size);
assert(p_cu_engine != nullptr);
this->_trt_context = p_cu_engine->createExecutionContext();
assert(this->_trt_context != nullptr);
delete[] trt_model_stream;
const ICudaEngine &cu_engine = this->_trt_context->getEngine();
assert(cu_engine.getNbBindings() == 2);
this->_input_index = cu_engine.getBindingIndex("input");
this->_output_index1 = cu_engine.getBindingIndex("output");
this->_output_index2 = cu_engine.getBindingIndex("/head/layers.0/act/Mul_output_0");
TRTCHECK(cudaMalloc(&_p_buffers[this->_input_index], 2 * 3 * 224 * 224 * sizeof(float)));
TRTCHECK(cudaMalloc(&_p_buffers[this->_output_index1], 2 * 576 * sizeof(float)));
TRTCHECK(cudaMalloc(&_p_buffers[this->_output_index2], 2 * 1280 * sizeof(float)));
TRTCHECK(cudaStreamCreate(&_cu_stream));
auto input_dims = nvinfer1::Dims4{2, 3, 224, 224};
this->_trt_context->setBindingDimensions(this->_input_index, input_dims);
this->_p_data = new float[2 * 3 * 224 * 224];
this->_p_prob1 = new float[2 * 576];
this->_p_prob2 = new float[2 * 1280];
this->_p_prob3 = new float[2 * 1280];
// Input
TRTCHECK(cudaMemcpyAsync(_p_buffers[this->_input_index], this->_p_data, 2 * 3 * 224 * 224 * sizeof(float), cudaMemcpyHostToDevice, this->_cu_stream));
// this->_trt_context->enqueue(1, _p_buffers, this->_cu_stream, nullptr);
this->_trt_context->enqueueV2(_p_buffers, this->_cu_stream, nullptr);
// Output
TRTCHECK(cudaMemcpyAsync(this->_p_prob1, _p_buffers[this->_output_index1], 2 * 576 * sizeof(float), cudaMemcpyDeviceToHost, this->_cu_stream));
TRTCHECK(cudaMemcpyAsync(this->_p_prob2, _p_buffers[this->_output_index2], 2 * 1280 * sizeof(float), cudaMemcpyDeviceToHost, this->_cu_stream));
cudaStreamSynchronize(this->_cu_stream);
return true;
#endif
return false;
}
void VeriDetectorCUDAImpl::cudaRoiCNN(
std::vector<cv::Mat> &input_rois_,
std::vector<int> &output_labels_)
{
#ifdef WITH_CUDA
for (int i = 0; i < 2; ++i)
{
for (int row = 0; row < 224; ++row)
{
uchar *uc_pixel = input_rois_[i].data + row * input_rois_[i].step; // compute row id
for (int col = 0; col < 224; ++col)
{
// mean=[136.20, 141.50, 145.41], std=[44.77, 44.20, 44.30]
this->_p_data[224 * 224 * 3 * i + col + row * 224] = ((float)uc_pixel[0] - 136.20f) / 44.77f;
this->_p_data[224 * 224 * 3 * i + col + row * 224 + 224 * 224] = ((float)uc_pixel[1] - 141.50f) / 44.20f;
this->_p_data[224 * 224 * 3 * i + col + row * 224 + 224 * 224 * 2] = ((float)uc_pixel[2] - 145.41f) / 44.30f;
uc_pixel += 3;
}
}
}
// Input
TRTCHECK(cudaMemcpyAsync(_p_buffers[this->_input_index], this->_p_data, 2 * 3 * 224 * 224 * sizeof(float), cudaMemcpyHostToDevice, this->_cu_stream));
// this->_trt_context->enqueue(1, _p_buffers, this->_cu_stream, nullptr);
this->_trt_context->enqueueV2(_p_buffers, this->_cu_stream, nullptr);
// Output
TRTCHECK(cudaMemcpyAsync(this->_p_prob1, _p_buffers[this->_output_index1], 2 * 576 * sizeof(float), cudaMemcpyDeviceToHost, this->_cu_stream));
TRTCHECK(cudaMemcpyAsync(this->_p_prob2, _p_buffers[this->_output_index2], 2 * 1280 * sizeof(float), cudaMemcpyDeviceToHost, this->_cu_stream));
cudaStreamSynchronize(this->_cu_stream);
// Find max index
double max = 0;
int label = 0;
for (int i = 0; i < 576; ++i)
{
if (max < this->_p_prob1[i])
{
max = this->_p_prob1[i];
label = i;
}
}
// 计算两个数组的余弦相似性
float similarity = cosineSimilarity(_p_prob2, _p_prob2 + 1280, 1280);
std::cout << "余弦相似性: " << similarity << std::endl;
std::cout << "VERI LABEL: " << label << std::endl;
}
#endif
}

View File

@ -0,0 +1,51 @@
#ifndef __SV_VERI_DET_CUDA__
#define __SV_VERI_DET_CUDA__
#include "sv_core.h"
#include <opencv2/opencv.hpp>
#include <opencv2/aruco.hpp>
#include <opencv2/tracking.hpp>
#include <string>
#include <chrono>
#ifdef WITH_CUDA
#include <NvInfer.h>
#include <cuda_runtime_api.h>
#endif
namespace sv {
class VeriDetectorCUDAImpl
{
public:
VeriDetectorCUDAImpl();
~VeriDetectorCUDAImpl();
bool cudaSetup();
void cudaRoiCNN(
std::vector<cv::Mat>& input_rois_,
std::vector<int>& output_labels_
);
#ifdef WITH_CUDA
float *_p_data;
float *_p_prob1;
float *_p_prob2;
float *_p_prob3;
nvinfer1::IExecutionContext *_trt_context;
int _input_index;
int _output_index1;
int _output_index2;
void *_p_buffers[3];
cudaStream_t _cu_stream;
#endif
};
}
#endif

View File

@ -0,0 +1,61 @@
#include "sv_veri_det.h"
#include <cmath>
#include <fstream>
#ifdef WITH_CUDA
#include <NvInfer.h>
#include <cuda_runtime_api.h>
#include "veri_det_cuda_impl.h"
#endif
namespace sv {
VeriDetector::VeriDetector()
{
this->_cuda_impl = new VeriDetectorCUDAImpl;
}
VeriDetector::~VeriDetector()
{
}
bool VeriDetector::setupImpl()
{
#ifdef WITH_CUDA
return this->_cuda_impl->cudaSetup();
#endif
return false;
}
void VeriDetector::roiCNN(
std::vector<cv::Mat>& input_rois_,
std::vector<int>& output_labels_
)
{
#ifdef WITH_CUDA
this->_cuda_impl->cudaRoiCNN(
input_rois_,
output_labels_
);
#endif
}
void VeriDetector::detect(cv::Mat img1_, cv::Mat img2_, TargetsInFrame& tgts_)
{
if (!_params_loaded)
{
this->_load();
this->_loadLabels();
_params_loaded = true;
}
std::vector<cv::Mat> e_roi = {img1_, img2_};
std::vector<int> output_labels;
roiCNN(e_roi, output_labels);
}
}

36
include/sv_veri_det.h Normal file
View File

@ -0,0 +1,36 @@
#ifndef __SV_VERI_DET__
#define __SV_VERI_DET__
#include "sv_core.h"
#include <opencv2/opencv.hpp>
#include <opencv2/aruco.hpp>
#include <opencv2/tracking.hpp>
#include <string>
#include <chrono>
namespace sv {
class VeriDetectorCUDAImpl;
class VeriDetector : public LandingMarkerDetectorBase
{
public:
VeriDetector();
~VeriDetector();
void detect(cv::Mat img1_, cv::Mat img2_, TargetsInFrame &tgts_);
protected:
bool setupImpl();
void roiCNN(
std::vector<cv::Mat>& input_rois_,
std::vector<int>& output_labels_
);
VeriDetectorCUDAImpl* _cuda_impl;
};
}
#endif

View File

@ -7,6 +7,7 @@
#include "sv_sot.h"
#include "sv_mot.h"
#include "sv_color_line.h"
#include "sv_veri_det.h"
#include "sv_video_input.h"
#include "sv_video_output.h"

View File

@ -25,3 +25,4 @@ int main(int argc, char *argv[]) {
return 0;
}

45
samples/demo/veri.cpp Normal file
View File

@ -0,0 +1,45 @@
#include <iostream>
#include <string>
// 包含SpireCV SDK头文件
#include <sv_world.h>
using namespace std;
int main(int argc, char *argv[]) {
// 打开摄像头
sv::VeriDetector veri;
veri.loadCameraParams(sv::get_home() + "/SpireCV/calib_webcam_640x480.yaml");
cv::VideoCapture cap1("/home/amov/Videos/com/FlyVideo_2023-09-02_11-36-00.avi");
cv::VideoCapture cap2("/home/amov/Videos/com/FlyVideo_2023-09-02_11-41-55.avi");
// cap.setWH(640, 480);
// cap.setFps(30);
//cap.open(sv::CameraType::WEBCAM, 0); // CameraID 0
// 实例化OpenCV的Mat类用于内存单帧图像
cv::Mat img1,img2;
int frame_id = 0;
while (1)
{
// 实例化SpireCV的 单帧检测结果 接口类 TargetsInFrame
sv::TargetsInFrame tgts(frame_id++);
// 读取一帧图像到img
cap1.read(img1);
cap2.read(img2);
cv::resize(img1, img1, cv::Size(224, 224));
cv::resize(img2, img2, cv::Size(224, 224));
veri.detect(img1, img2, tgts);
// 显示img
// cv::imshow("img", img);
// cv::waitKey(10);
}
return 0;
}

187
sv_algorithm_params.json Normal file
View File

@ -0,0 +1,187 @@
{
"CommonObjectDetector": {
"dataset": "COCO",
"inputSize": 1280,
"nmsThrs": 0.6,
"scoreThrs": 0.3,
"useWidthOrHeight": 1,
"withSegmentation": false,
"datasetPersonVehicle": {
"person": [0.5, 1.8],
"car": [4.1, 1.5],
"bus": [10, 3],
"truck": [-1, -1],
"bike": [-1, -1],
"train": [-1, -1],
"boat": [-1, -1],
"aeroplane": [-1, -1]
},
"datasetDrone": {
"drone": [0.4, 0.2]
},
"datasetCOCO": {
"person": [-1, -1],
"bicycle": [-1, -1],
"car": [-1, -1],
"motorcycle": [-1, -1],
"airplane": [-1, -1],
"bus": [-1, -1],
"train": [-1, -1],
"truck": [-1, -1],
"boat": [-1, -1],
"traffic light": [-1, -1],
"fire hydrant": [-1, -1],
"stop sign": [-1, -1],
"parking meter": [-1, -1],
"bench": [-1, -1],
"bird": [-1, -1],
"cat": [-1, -1],
"dog": [-1, -1],
"horse": [-1, -1],
"sheep": [-1, -1],
"cow": [-1, -1],
"elephant": [-1, -1],
"bear": [-1, -1],
"zebra": [-1, -1],
"giraffe": [-1, -1],
"backpack": [-1, -1],
"umbrella": [-1, -1],
"handbag": [-1, -1],
"tie": [-1, -1],
"suitcase": [-1, -1],
"frisbee": [-1, -1],
"skis": [-1, -1],
"snowboard": [-1, -1],
"sports ball": [-1, -1],
"kite": [-1, -1],
"baseball bat": [-1, -1],
"baseball glove": [-1, -1],
"skateboard": [-1, -1],
"surfboard": [-1, -1],
"tennis racket": [-1, -1],
"bottle": [-1, -1],
"wine glass": [-1, -1],
"cup": [-1, -1],
"fork": [-1, -1],
"knife": [-1, -1],
"spoon": [-1, -1],
"bowl": [-1, -1],
"banana": [-1, -1],
"apple": [-1, -1],
"sandwich": [-1, -1],
"orange": [-1, -1],
"broccoli": [-1, -1],
"carrot": [-1, -1],
"hot dog": [-1, -1],
"pizza": [-1, -1],
"donut": [-1, -1],
"cake": [-1, -1],
"chair": [-1, -1],
"couch": [-1, -1],
"potted plant": [-1, -1],
"bed": [-1, -1],
"dining table": [-1, -1],
"toilet": [-1, -1],
"tv": [-1, -1],
"laptop": [-1, -1],
"mouse": [-1, -1],
"remote": [-1, -1],
"keyboard": [-1, -1],
"cell phone": [-1, -1],
"microwave": [-1, -1],
"oven": [-1, -1],
"toaster": [-1, -1],
"sink": [-1, -1],
"refrigerator": [-1, -1],
"book": [-1, -1],
"clock": [-1, -1],
"vase": [-1, -1],
"scissors": [-1, -1],
"teddy bear": [-1, -1],
"hair drier": [-1, -1],
"toothbrush": [-1, -1]
}
},
"AutoFocusObjectDetector": {
"lock_thres": 5,
"unlock_thres": 5,
"lock_scale_init": 12.0,
"lock_scale": 8.0,
"categories_filter": [],
"keep_unlocked": false,
"use_square_region": false
},
"SingleObjectTracker": {
"algorithm": "nano",
"backend": 0,
"target": 0,
"useWidthOrHeight": 0,
"sigleobjectW":2,
"sigleobjectH":3
},
"LandingMarkerDetector": {
"labels": ["h","x","1","2","8"],
"maxCandidates": 3
},
"EllipseDetector": {
"radiusInMeter": 0.1535,
"preProcessingGaussKernel": 5,
"preProcessingGaussSigma": 1.306,
"thPosition": 1.0,
"maxCenterDistance": 0.05,
"minEdgeLength": 9,
"minOrientedRectSide": 2.984,
"distanceToEllipseContour": 0.111,
"minScore": 0.7,
"minReliability": 0.470,
"ns": 22,
"percentNe": 0.99,
"T_CNC": 0.121,
"T_TCN_L": 0.468,
"T_TCN_P": 0.560,
"thRadius": 0.202
},
"ArucoDetector": {
"dictionaryId": 10,
"markerIds": [-1],
"markerLengths": [0.2],
"adaptiveThreshConstant": 35,
"adaptiveThreshWinSizeMax": 23,
"adaptiveThreshWinSizeMin": 3,
"adaptiveThreshWinSizeStep": 10,
"aprilTagCriticalRad": 0.17453292519,
"aprilTagDeglitch": 0,
"aprilTagMaxLineFitMse": 10.0,
"aprilTagMaxNmaxima": 10,
"aprilTagMinClusterPixels": 5,
"aprilTagMinWhiteBlackDiff": 5,
"aprilTagQuadDecimate": 0.0,
"aprilTagQuadSigma": 0.0,
"cornerRefinementMaxIterations": 30,
"cornerRefinementMethod": 0,
"cornerRefinementMinAccuracy": 0.1,
"cornerRefinementWinSize": 5,
"detectInvertedMarker": false,
"errorCorrectionRate": 0.6,
"markerBorderBits": 1,
"maxErroneousBitsInBorderRate": 0.35,
"maxMarkerPerimeterRate": 4.0,
"minCornerDistanceRate": 0.05,
"minDistanceToBorder": 3,
"minMarkerDistanceRate": 0.05,
"minMarkerLengthRatioOriginalImg": 0,
"minMarkerPerimeterRate": 0.03,
"minOtsuStdDev": 5.0,
"minSideLengthCanonicalImg": 32,
"perspectiveRemoveIgnoredMarginPerCell": 0.13,
"perspectiveRemovePixelPerCell": 4,
"polygonalApproxAccuracyRate": 0.03,
"useAruco3Detection": false
},
"ColorLineDetector": {
"line_color": "black",
"line_location": 0.5,
"line_location_a1": 0.3,
"line_location_a2": 0.7
}
}