diff --git a/CMakeLists.txt b/CMakeLists.txt index c2986df..9081303 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -264,6 +264,8 @@ add_executable(ColorLineDetection samples/demo/color_line_detect.cpp) target_link_libraries(ColorLineDetection sv_world) add_executable(UdpDetectionInfoReceiver samples/demo/udp_detection_info_receiver.cpp) target_link_libraries(UdpDetectionInfoReceiver sv_world) +add_executable(UdpDetectionInfoReceiverByAirborne samples/demo/udp_detection_info_receiver_by_airborne.cpp) +target_link_libraries(UdpDetectionInfoReceiverByAirborne sv_world) add_executable(UdpDetectionInfoSender samples/demo/udp_detection_info_sender.cpp) target_link_libraries(UdpDetectionInfoSender sv_world) add_executable(VideoSaving samples/demo/video_saving.cpp) diff --git a/algorithm/veri/cuda/veri_det_cuda_impl.cpp b/algorithm/veri/cuda/veri_det_cuda_impl.cpp index a35d795..c307ff1 100644 --- a/algorithm/veri/cuda/veri_det_cuda_impl.cpp +++ b/algorithm/veri/cuda/veri_det_cuda_impl.cpp @@ -181,7 +181,7 @@ namespace sv } } - float similarity = cosineSimilarity(_p_prob2, _p_prob2 + 1280, 1280); + float similarity = cosineSimilarity(this->_p_prob2, this->_p_prob2 + 1280, 1280); output_labels_.push_back(label); output_labels_.push_back(similarity); } diff --git a/algorithm/veri/sv_veri_det.cpp b/algorithm/veri/sv_veri_det.cpp index 279afe0..bf12630 100644 --- a/algorithm/veri/sv_veri_det.cpp +++ b/algorithm/veri/sv_veri_det.cpp @@ -1,12 +1,17 @@ #include "sv_veri_det.h" #include #include +#include "gason.h" +#include "sv_util.h" + #ifdef WITH_CUDA #include #include #include "veri_det_cuda_impl.h" #endif +#define SV_ROOT_DIR "/SpireCV/" + namespace sv { @@ -18,14 +23,35 @@ namespace sv { } + void VeriDetector::_load() + { + JsonValue all_value; + JsonAllocator allocator; + _load_all_json(this->alg_params_fn, all_value, allocator); + + JsonValue veriliner_params_value; + _parser_algorithm_params("VeriDetector", all_value, veriliner_params_value); + + for (auto i : veriliner_params_value) + { + if ("vehicle_ID" == std::string(i->key)) + { + this->vehicle_id = i->value.toString(); + std::cout << "vehicle_ID Load Sucess!" << std::endl; + } + } + } + bool VeriDetector::setupImpl() { #ifdef WITH_CUDA return this->_cuda_impl->cudaSetup(); #endif + return false; } + void VeriDetector::roiCNN( std::vector &input_rois_, std::vector &output_labels_) @@ -37,7 +63,7 @@ namespace sv #endif } - void VeriDetector::detect(cv::Mat img1_, cv::Mat img2_, TargetsInFrame &tgts_) + void VeriDetector::detect(cv::Mat img_, const cv::Rect &bounding_box_, sv::Target &tgt) { if (!_params_loaded) { @@ -46,28 +72,79 @@ namespace sv _params_loaded = true; } - std::vector input_rois_ = {img1_, img2_}; + // convert Rect2d from left-up to center. + targetPos[0] = float(bounding_box_.x) + float(bounding_box_.width) * 0.5f; + targetPos[1] = float(bounding_box_.y) + float(bounding_box_.height) * 0.5f; + + targetSz[0] = float(bounding_box_.width); + targetSz[1] = float(bounding_box_.height); + + // Extent the bounding box. + float sumSz = targetSz[0] + targetSz[1]; + float wExtent = targetSz[0] + 0.5 * (sumSz); + float hExtent = targetSz[1] + 0.5 * (sumSz); + int sz = int(cv::sqrt(wExtent * hExtent)); + + cv::Mat crop; + getSubwindow(crop, img_, sz, 224); + + std::string img_ground_dir = get_home() + SV_ROOT_DIR + this->vehicle_id; + cv::Mat img_ground = cv::imread(img_ground_dir); + cv::resize(img_ground, img_ground, cv::Size(224, 224)); + std::vector input_rois_ = {crop, img_ground}; #ifdef WITH_CUDA std::vector output_labels; roiCNN(input_rois_, output_labels); #endif + // auto t1 = std::chrono::system_clock::now(); + // tgts_.setFPS(1000.0 / std::chrono::duration_cast(t1 - this->_t0).count()); + // this->_t0 = std::chrono::system_clock::now(); + // tgts_.setTimeNow(); - tgts_.setSize(img1_.cols, img1_.rows); - tgts_.setFOV(this->fov_x, this->fov_y); - auto t1 = std::chrono::system_clock::now(); - tgts_.setFPS(1000.0 / std::chrono::duration_cast(t1 - this->_t0).count()); - this->_t0 = std::chrono::system_clock::now(); - tgts_.setTimeNow(); - - if (output_labels.size() > 0) + if (output_labels.size() > 0) { - Target tgt; - - tgt.category_id = output_labels[0]; - tgt.score = output_labels[1]; - tgts_.targets.push_back(tgt); + // tgt.category_id = output_labels[0]; + tgt.sim_score = output_labels[1]; + // tgts_.targets.push_back(tgt); } } + void VeriDetector::getSubwindow(cv::Mat &dstCrop, cv::Mat &srcImg, int originalSz, int resizeSz) + { + cv::Scalar avgChans = mean(srcImg); + cv::Size imgSz = srcImg.size(); + int c = (originalSz + 1) / 2; + + int context_xmin = (int)(targetPos[0]) - c; + int context_xmax = context_xmin + originalSz - 1; + int context_ymin = (int)(targetPos[1]) - c; + int context_ymax = context_ymin + originalSz - 1; + + int left_pad = std::max(0, -context_xmin); + int top_pad = std::max(0, -context_ymin); + int right_pad = std::max(0, context_xmax - imgSz.width + 1); + int bottom_pad = std::max(0, context_ymax - imgSz.height + 1); + + context_xmin += left_pad; + context_xmax += left_pad; + context_ymin += top_pad; + context_ymax += top_pad; + + cv::Mat cropImg; + if (left_pad == 0 && top_pad == 0 && right_pad == 0 && bottom_pad == 0) + { + // Crop image without padding. + cropImg = srcImg(cv::Rect(context_xmin, context_ymin, + context_xmax - context_xmin + 1, context_ymax - context_ymin + 1)); + } + else // Crop image with padding, and the padding value is avgChans + { + cv::Mat tmpMat; + cv::copyMakeBorder(srcImg, tmpMat, top_pad, bottom_pad, left_pad, right_pad, cv::BORDER_CONSTANT, avgChans); + cropImg = tmpMat(cv::Rect(context_xmin, context_ymin, context_xmax - context_xmin + 1, context_ymax - context_ymin + 1)); + } + resize(cropImg, dstCrop, cv::Size(resizeSz, resizeSz)); + } + } diff --git a/include/sv_veri_det.h b/include/sv_veri_det.h index a788694..08e3a00 100644 --- a/include/sv_veri_det.h +++ b/include/sv_veri_det.h @@ -8,29 +8,36 @@ #include #include - -namespace sv { - -class VeriDetectorCUDAImpl; - -class VeriDetector : public LandingMarkerDetectorBase +namespace sv { -public: - VeriDetector(); - ~VeriDetector(); - void detect(cv::Mat img1_, cv::Mat img2_, TargetsInFrame &tgts_); + class VeriDetectorCUDAImpl; -protected: - bool setupImpl(); - void roiCNN( - std::vector& input_rois_, - std::vector& output_labels_ - ); + class VeriDetector : public LandingMarkerDetectorBase + { + public: + VeriDetector(); + ~VeriDetector(); - VeriDetectorCUDAImpl* _cuda_impl; -}; + void detect(cv::Mat img_, const cv::Rect &bounding_box_, sv::Target &tgt); + protected: + void _load(); + bool setupImpl(); + void roiCNN( + std::vector &input_rois_, + std::vector &output_labels_); + void getSubwindow(cv::Mat &dstCrop, cv::Mat &srcImg, int originalSz, int resizeSz); + + std::string vehicle_id; + + // Save the target bounding box for each frame. + std::vector targetSz = {0, 0}; // H and W of bounding box + std::vector targetPos = {0, 0}; // center point of bounding box (x, y) + + + VeriDetectorCUDAImpl *_cuda_impl; + }; } #endif diff --git a/include/sv_video_base.h b/include/sv_video_base.h index f026ea3..39e1817 100644 --- a/include/sv_video_base.h +++ b/include/sv_video_base.h @@ -91,6 +91,9 @@ public: //! The angle of the target in the image coordinate system, (unit: degree) [-180, 180]. double yaw_a; + //! Similarity, Confidence, (0, 1] + double sim_score; + //! Whether the height&width of the target can be obtained. bool has_hw; //! Whether the category of the target can be obtained. diff --git a/params/a-params/sv_algorithm_params.json b/params/a-params/sv_algorithm_params.json index 6d2bd9d..49025d5 100644 --- a/params/a-params/sv_algorithm_params.json +++ b/params/a-params/sv_algorithm_params.json @@ -190,5 +190,8 @@ "line_location": 0.5, "line_location_a1": 0.3, "line_location_a2": 0.7 + }, + "VeriDetector": { + "vehicle_ID": "car_id.jpeg" } } diff --git a/samples/demo/udp_detection_info_receiver_by_airborne.cpp b/samples/demo/udp_detection_info_receiver_by_airborne.cpp new file mode 100644 index 0000000..9eb2ef1 --- /dev/null +++ b/samples/demo/udp_detection_info_receiver_by_airborne.cpp @@ -0,0 +1,140 @@ +#include +#include +#include +#include +#include +#include + +#define PORT 12346 // UDP端口号 + +// 枚举定义消息类型 +enum MessageType { + TIMESTAMP = 0x00, + CLICK_COORDINATES = 0x01, + TARGET_BOX_COORDINATES = 0x02, + CLICK_COORDINATES_FRAME_ID = 0x03, + TARGET_BOX_COORDINATES_FRAME_ID = 0x04 +}; + +// 定义消息结构体 +#pragma pack(1) // 使用1字节对齐方式 +struct Message { + unsigned char header[2]; // 帧头 + unsigned char type; // 消息类型 + unsigned char reserved; // 保留字段 + unsigned int requestID; // 请求ID + unsigned int dataLength; // 数据长度 + unsigned char* data; // 数据段 +}; +#pragma pack() // 恢复默认对齐方式 + +// 解析时间戳消息 +void parseTimestampMessage(unsigned char* data) { + unsigned short year = ntohs(*reinterpret_cast(data)); + unsigned char month = *(data + 2); + unsigned char day = *(data + 3); + unsigned char hour = *(data + 4); + unsigned char minute = *(data + 5); + unsigned char second = *(data + 6); + unsigned short millisecond = ntohs(*reinterpret_cast(data + 7)); + + std::cout << "Timestamp: " << static_cast(year) << "-" << static_cast(month) + << "-" << static_cast(day) << " " << static_cast(hour) << ":" + << static_cast(minute) << ":" << static_cast(second) << "." + << static_cast(millisecond) << std::endl; +} + +// 解析点击坐标消息 +void parseClickCoordinatesMessage(unsigned char* data) { + float x = *reinterpret_cast(data); + float y = *reinterpret_cast(data + 4); + + std::cout << "Click coordinates: (" << x << ", " << y << ")" << std::endl; +} + +// 解析目标框坐标消息 +void parseTargetBoxCoordinatesMessage(unsigned char* data) { + float x1 = *reinterpret_cast(data); + float y1 = *reinterpret_cast(data + 4); + float x2 = *reinterpret_cast(data + 8); + float y2 = *reinterpret_cast(data + 12); + + std::cout << "Target box coordinates: (" << x1 << ", " << y1 << ") - (" + << x2 << ", " << y2 << ")" << std::endl; +} + +// 解析UDP数据包 +void parseUDPData(unsigned char* buffer, int length) { + if (length < 13) { + std::cout << "Invalid UDP data format" << std::endl; + return; + } + + Message message; + memcpy(&message, buffer, sizeof(Message)); + + // 解析消息类型 + MessageType messageType = static_cast(message.type); + + // 根据消息类型处理数据 + switch (messageType) { + case TIMESTAMP: + if (length != 9) { + std::cout << "Invalid timestamp message length" << std::endl; + return; + } + parseTimestampMessage(message.data); + break; + case CLICK_COORDINATES: + if (length != 13) { + std::cout << "Invalid click coordinates message length" << std::endl; + return; + } + parseClickCoordinatesMessage(message.data); + break; + case TARGET_BOX_COORDINATES: + if (length != 21) { + std::cout << "Invalid target box coordinates message length" << std::endl; + return; + } + parseTargetBoxCoordinatesMessage(message.data); + break; + default: + std::cout << "Unsupported message type" << std::endl; + break; + } +} + +int main() { + int sockfd; + struct sockaddr_in servaddr, cliaddr; + unsigned char buffer[4096]; + + // 创建UDP套接字 + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + + memset(&servaddr, 0, sizeof(servaddr)); + memset(&cliaddr, 0, sizeof(cliaddr)); + + servaddr.sin_family = AF_INET; + servaddr.sin_addr.s_addr = htonl(INADDR_ANY); + servaddr.sin_port = htons(PORT); + + // 绑定UDP套接字到端口 + bind(sockfd, (const struct sockaddr*)&servaddr, sizeof(servaddr)); + + while (true) { + socklen_t len = sizeof(cliaddr); + + // 接收UDP数据包 + ssize_t n = recvfrom(sockfd, buffer, sizeof(buffer), MSG_WAITALL, + (struct sockaddr*)&cliaddr, &len); + + // 解析UDP数据包 + parseUDPData(buffer, n); + } + + close(sockfd); + + return 0; +} diff --git a/samples/demo/veri.cpp b/samples/demo/veri.cpp index e9cff74..dcb7155 100644 --- a/samples/demo/veri.cpp +++ b/samples/demo/veri.cpp @@ -5,67 +5,82 @@ using namespace std; +struct node +{ + double x, y; +}; + +node p1; + int main(int argc, char *argv[]) { - // 实例化 圆形降落标志 检测器类 + // 实例化 框选目标跟踪类 sv::VeriDetector veri; // 手动导入相机参数，如果使用Amov的G1等吊舱或相机，则可以忽略该步骤，将自动下载相机参数文件 veri.loadCameraParams(sv::get_home() + "/SpireCV/calib_webcam_640x480.yaml"); + sv::CommonObjectDetector cod; + cod.loadCameraParams(sv::get_home() + "/SpireCV/calib_webcam_640x480.yaml"); + // 打开摄像头 - sv::Camera cap1,cap2; - cap1.setWH(640, 480); - cap1.setFps(30); - cap1.open(sv::CameraType::WEBCAM, 4); // CameraID 0 - - cap2.setWH(640, 480); - cap2.setFps(30); - cap2.open(sv::CameraType::WEBCAM, 10); // CameraID 0 - - - + sv::Camera cap; + // cap.setWH(640, 480); + // cap.setFps(30); + cap.open(sv::CameraType::WEBCAM, 4); // CameraID 0 // 实例化OpenCV的Mat类，用于内存单帧图像 - cv::Mat img1, img2; + cv::Mat img; int frame_id = 0; - int w = 224; - while (1) { + // 实例化SpireCV的 单帧检测结果 接口类 TargetsInFrame sv::TargetsInFrame tgts(frame_id++); // 读取一帧图像到img - cap1.read(img1); - cap2.read(img2); + cap.read(img); + cv::resize(img, img, cv::Size(cod.image_width, cod.image_height)); - cv::resize(img1, img1, cv::Size(w, w)); - cv::resize(img2, img2, cv::Size(w, w)); + // 执行通用目标检测 + cod.detect(img, tgts); - // 执行 降落标志 检测 - veri.detect(img1, img2, tgts); // 可视化检测结果，叠加到img上 - // sv::drawTargetsInFrame(img, tgts); + sv::drawTargetsInFrame(img, tgts); - // // 控制台打印 降落标志 检测结果 + // 控制台打印通用目标检测结果 printf("Frame-[%d]\n", frame_id); // 打印当前检测的FPS printf(" FPS = %.2f\n", tgts.fps); // 打印当前相机的视场角（degree） printf(" FOV (fx, fy) = (%.2f, %.2f)\n", tgts.fov_x, tgts.fov_y); - // 打印当前输入图像的像素宽度和高度 - printf(" Frame Size (width, height) = (%d, %d)\n", tgts.width, tgts.height); for (int i = 0; i < tgts.targets.size(); i++) { + printf("Frame-[%d], Object-[%d]\n", frame_id, i); + // 打印每个目标的中心位置，cx，cy的值域为[0, 1] + printf(" Object Center (cx, cy) = (%.3f, %.3f)\n", tgts.targets[i].cx, tgts.targets[i].cy); + // 打印每个目标的外接矩形框的宽度、高度，w，h的值域为(0, 1] + printf(" Object Size (w, h) = (%.3f, %.3f)\n", tgts.targets[i].w, tgts.targets[i].h); + // 打印每个目标的置信度 + printf(" Object Score = %.3f\n", tgts.targets[i].score); + // 打印每个目标的类别，字符串类型 + printf(" Object Category = %s, Category ID = [%d]\n", tgts.targets[i].category.c_str(), tgts.targets[i].category_id); + // 打印每个目标的视线角，跟相机视场相关 + printf(" Object Line-of-sight (ax, ay) = (%.3f, %.3f)\n", tgts.targets[i].los_ax, tgts.targets[i].los_ay); + // 打印每个目标的3D位置（在相机坐标系下），跟目标实际长宽、相机参数相关 + printf(" Object Position = (x, y, z) = (%.3f, %.3f, %.3f)\n", tgts.targets[i].px, tgts.targets[i].py, tgts.targets[i].pz); - // 打印每个 降落标志 的置信度 - printf(" Similarity Score = %.3f\n", tgts.targets[i].score); - printf(" Category ID = %d\n", tgts.targets[i].category_id); + if (tgts.targets[i].category_id == 2) + { + p1.x = tgts.targets[i].cx * tgts.width - tgts.targets[i].w * tgts.width / 2; + p1.y = tgts.targets[i].cy * tgts.height - tgts.targets[i].h * tgts.height / 2; + rect_sel = cv::Rect(p1.x, p1.y, tgts.targets[i].w * tgts.width, tgts.targets[i].h * tgts.height); + veri.detect(img, rect_sel, tgts.targets[i]); + // 打印每个目标的CosineSimilarity + printf(" CosineSimilarity Score = %.3f\n", tgts.targets[i].sim_score); + } } // 显示检测结果img - cv::imshow("img1", img1); - cv::imshow("img2", img2); + cv::imshow("img", img); cv::waitKey(10); } - return 0; }