diff --git a/CMakeLists.txt b/CMakeLists.txt index e640190..5e3c8a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,6 +52,7 @@ if(USE_FFMPEG) endif() +find_package(Eigen3 REQUIRED) add_definitions(-DWITH_OCV470) find_package(OpenCV 4.7 REQUIRED) message(STATUS "OpenCV library status:") @@ -60,6 +61,7 @@ message(STATUS " libraries: ${OpenCV_LIBS}") message(STATUS " include path: ${OpenCV_INCLUDE_DIRS}") +include_directories(${EIGEN3_INCLUDE_DIRS}) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/gimbal_ctrl/IOs/serial/include diff --git a/algorithm/mot/sv_mot.cpp b/algorithm/mot/sv_mot.cpp index a954b62..a326a0d 100644 --- a/algorithm/mot/sv_mot.cpp +++ b/algorithm/mot/sv_mot.cpp @@ -4,6 +4,8 @@ #include "gason.h" #include "sv_util.h" +using namespace std; +using namespace Eigen; namespace sv { @@ -11,9 +13,12 @@ namespace sv { MultipleObjectTracker::MultipleObjectTracker() { this->_params_loaded = false; + this->_sort_impl = NULL; } MultipleObjectTracker::~MultipleObjectTracker() { + if (this->_sort_impl) + delete this->_sort_impl; } void MultipleObjectTracker::track(cv::Mat img_, TargetsInFrame& tgts_) @@ -23,16 +28,25 @@ void MultipleObjectTracker::track(cv::Mat img_, TargetsInFrame& tgts_) this->_load(); this->_params_loaded = true; } - + if ("sort" == this->_algorithm && this->_sort_impl) + { + this->_detector->detect(img_, tgts_); + this->_sort_impl->update(tgts_); + } } -void MultipleObjectTracker::init() +void MultipleObjectTracker::init(CommonObjectDetector* detector_) { if (!this->_params_loaded) { this->_load(); this->_params_loaded = true; } + if ("sort" == this->_algorithm) + { + this->_sort_impl = new SORT(this->_iou_thres, this->_max_age, this->_min_hits); + } + this->_detector = detector_; } void MultipleObjectTracker::_load() @@ -94,5 +108,306 @@ void MultipleObjectTracker::_load() } } + + +KalmanFilter::KalmanFilter() +{ + this->_chi2inv95 << 3.8415, 5.9915, 7.8147, 9.4877, 11.070, 12.592, 14.067, 15.507, 16.919; + this->_F = MatrixXd::Identity(8, 8); + for (int i=0; i<4; i++) + { + this->_F(i,i+4) = 1; + } + this->_H = MatrixXd::Identity(4, 8); + this->_std_weight_position = 1. / 20; + this->_std_weight_vel = 1. / 160; +} + +KalmanFilter::~KalmanFilter() +{ +} + +pair, Matrix > KalmanFilter::initiate(Vector4d &bbox) +{ + Matrix mean; + mean << bbox(0), bbox(1), bbox(2)/bbox(3), bbox(3), 0, 0, 0, 0; + VectorXd stds(8); + stds << 2 * this->_std_weight_position * mean(3), 2 * this->_std_weight_position * mean(3), 0.01, 2 * this->_std_weight_position * mean(3), \ + 10 * this->_std_weight_vel * mean(3), 10 * this->_std_weight_vel * mean(3), 1e-5, 10 * this->_std_weight_vel * mean(3); + MatrixXd squared = stds.array().square(); + Matrix covariances; + covariances = squared.asDiagonal(); + return make_pair(mean, covariances); +} + +pair, Matrix > KalmanFilter::update(Matrix mean, Matrix covariances, sv::Box &box) +{ + MatrixXd R; + Vector4d stds; + + stds << this->_std_weight_position * mean(3), this->_std_weight_position * mean(3), 0.1, this->_std_weight_position * mean(3); + MatrixXd squared = stds.array().square(); + R = squared.asDiagonal(); + MatrixXd S = this->_H * covariances * this->_H.transpose() + R; + + MatrixXd Kalman_gain = covariances * this->_H.transpose() * S.inverse(); + VectorXd measurement(4); + measurement << box.x1, box.y1, (box.x2-box.x1)/(box.y2-box.y1), box.y2-box.y1; + Matrix new_mean = mean + Kalman_gain * (measurement - this->_H * mean); + Matrix new_covariances = (MatrixXd::Identity(8, 8) - Kalman_gain * this->_H) * covariances; + return make_pair(new_mean, new_covariances); +} + +pair, Matrix > KalmanFilter::predict(Matrix mean, Matrix covariances) +{ + VectorXd stds(8); + stds << this->_std_weight_position * mean(3), this->_std_weight_position * mean(3), 0.01, this->_std_weight_position * mean(3), \ + this->_std_weight_vel * mean(3), this->_std_weight_vel * mean(3), 1e-5, this->_std_weight_vel * mean(3); + MatrixXd squared = stds.array().square(); + MatrixXd Q = squared.asDiagonal(); + Matrix pre_mean = this->_F * mean; + Matrix pre_cov = this->_F * covariances * this->_F.transpose() + Q; + return make_pair(pre_mean, pre_cov); +} + + +SORT::~SORT() +{ +} + +void SORT::update(TargetsInFrame& tgts) +{ + sv::KalmanFilter kf; + if (! this->_tracklets.size()) + { + Vector4d bbox; + for (int i=0; i_next_tracklet_id; + // cout << tracklet.id << endl; + tgts.targets[i].tracked_id = this->_next_tracklet_id; + + tracklet.bbox << box.x1,box.y1,box.x2-box.x1,box.y2-box.y1; // x,y,w,h + tracklet.age = 0; + tracklet.hits = 1; + tracklet.misses = 0; + // initate the motion + pair, Matrix > motion = kf.initiate(tracklet.bbox); + tracklet.mean=motion.first; + tracklet.covariance = motion.second; + + this->_tracklets.push_back(tracklet); + } + } + else + { + for (int i=0; i match_det; + match_det.fill(-1); + // predict the next state of each tracklet + for (auto& tracklet : this->_tracklets) + { + tracklet.age++; + pair, Matrix > motion = kf.predict(tracklet.mean, tracklet.covariance); + tracklet.bbox << motion.first(0), motion.first(1), motion.first(2) * motion.first(3), motion.first(3); + tracklet.mean = motion.first; + tracklet.covariance = motion.second; + } + + // Match the detections to the existing tracklets + // cout << "the num of targets: " << tgts.targets.size() << endl; + // cout << "the num of tracklets: " << this->_tracklets.size() << endl; + vector > iouMatrix(this->_tracklets.size(), vector (tgts.targets.size(), 0)); + for (int i=0; i_tracklets.size(); i++) + { + for (int j=0; j_iou(this->_tracklets[i], box); + } + } + vector > matches = this->_hungarian(iouMatrix); + for (auto& match : matches) + { + int trackletIndex = match.first; + int detectionIndex = match.second; + if (trackletIndex >= 0 && detectionIndex >= 0) + { + if(iouMatrix[match.first][match.second] >= 0) + { + sv::Box box; + tgts.targets[detectionIndex].getBox(box); + this->_tracklets[trackletIndex].age = 0; + this->_tracklets[trackletIndex].hits++; + this->_tracklets[trackletIndex].bbox << box.x1, box.y1, box.x2-box.x1, box.y2-box.y1; + + auto[mean, covariance] = kf.update(this->_tracklets[trackletIndex].mean, this->_tracklets[trackletIndex].covariance, box); + this->_tracklets[trackletIndex].mean = mean; + this->_tracklets[trackletIndex].covariance = covariance; + + tgts.targets[detectionIndex].tracked_id = this->_tracklets[trackletIndex].id; + match_det[detectionIndex] = detectionIndex; + } + } + } + // create new tracklets for unmatched detections + for (int i = 0; i < tgts.targets.size(); i++) + { + if (match_det[i]==-1) + { + sv::Box box; + tgts.targets[i].getBox(box); + Tracklet tracklet; + tracklet.id = ++ this->_next_tracklet_id; + tracklet.bbox << box.x1, box.y1, box.x2-box.x1, box.y2-box.y1; + + tracklet.age = 0; + tracklet.hits = 1; + tracklet.misses = 0; + + auto[new_mean, new_covariance] = kf.initiate(tracklet.bbox); + tracklet.mean = new_mean; + tracklet.covariance = new_covariance; + + tgts.targets[i].tracked_id = this->_next_tracklet_id; + this->_tracklets.push_back(tracklet); + } + } + } +} + +vector SORT::getTracklets() const +{ + return this->_tracklets; +} + +double SORT::_iou(Tracklet& tracklet, sv::Box& box) +{ + double trackletX1 = tracklet.bbox(0); + double trackletY1 = tracklet.bbox(1); + double trackletX2 = tracklet.bbox(0) + tracklet.bbox(2); + double trackletY2 = tracklet.bbox(1) + tracklet.bbox(3); + + double detectionX1 = box.x1; + double detectionY1 = box.y1; + double detectionX2 = box.x2; + double detectionY2 = box.y2; + double intersectionX1 = max(trackletX1, detectionX1); + double intersectionY1 = max(trackletY1, detectionY1); + double intersectionX2 = min(trackletX2, detectionX2); + double intersectionY2 = min(trackletY2, detectionY2); + + double w = (intersectionX2-intersectionX1 > 0.0) ? (intersectionX2-intersectionX1) : 0.0; + double h = (intersectionY2-intersectionY1 > 0.0) ? (intersectionY2-intersectionY1) : 0.0; + double intersectionArea = w * h; + + double trackletArea = tracklet.bbox(2) * tracklet.bbox(3); + + double detectionArea = (box.x2-box.x1) * (box.y2-box.y1); + double unionArea = trackletArea + detectionArea - intersectionArea; + double iou = (1 - intersectionArea / unionArea * 1.0); + + return iou; +} + +vector > SORT::_hungarian(vector > costMatrix) +{ + int numRows = costMatrix.size(); + int numCols = costMatrix[0].size(); + + const bool transposed = numCols > numRows; + // transpose the matrix if necessary + if (transposed) + { + vector > transposedMatrix(numCols, vector(numRows)); + for (int i=0; irowMin(numRows, numeric_limits::infinity()); + vectorcolMin(numCols, numeric_limits::infinity()); + vectorrowMatch(numRows, -1); + vectorcolMatch(numCols, -1); + vector > matches; + // step1: Subtract the row minimums from each row + for (int i=0; i visited(numCols, false); + _augment(costMatrix, i, rowMatch, colMatch, visited); + } + // step4: calculate the matches + matches.clear(); + for (int j=0; j >& costMatrix, int row, vector& rowMatch, vector& colMatch, vector& visited) +{ + int numCols = costMatrix[0].size(); + for (int j=0; j #include #include #include #include - +#include namespace sv { +class SORT; + + class MultipleObjectTracker : public CameraAlgorithm { public: - MultipleObjectTracker(); - ~MultipleObjectTracker(); + MultipleObjectTracker(); + ~MultipleObjectTracker(); - void init(); - void track(cv::Mat img_, TargetsInFrame& tgts_); + void init(CommonObjectDetector* detector_); + void track(cv::Mat img_, TargetsInFrame& tgts_); private: - void _load(); - bool _params_loaded; + void _load(); + bool _params_loaded; - std::string _algorithm; - bool _with_reid; - int _reid_input_h; - int _reid_input_w; - int _reid_num_samples; - double _reid_match_thres; - double _iou_thres; - int _max_age; - int _min_hits; + std::string _algorithm; + bool _with_reid; + int _reid_input_h; + int _reid_input_w; + int _reid_num_samples; + double _reid_match_thres; + double _iou_thres; + int _max_age; + int _min_hits; + + SORT* _sort_impl; + CommonObjectDetector* _detector; +}; + + +// define the tracklet struct to store the tracked objects. +struct Tracklet +{ +/* data */ +public: + Eigen::Vector4d bbox; // double x, y, w, h; + int id=0; + int age; + int hits; + int misses; + std::vector features; + Eigen::Matrix mean; + Eigen::Matrix covariance; +}; + + +class KalmanFilter { +public: + KalmanFilter(); + ~KalmanFilter(); + std::pair, Eigen::Matrix > initiate(Eigen::Vector4d &bbox); + std::pair, Eigen::Matrix > update(Eigen::Matrix mean, Eigen::Matrix covariances, Box &box); + std::pair, Eigen::Matrix > predict(Eigen::Matrix mean, Eigen::Matrix covariances); +private: + Eigen::Matrix _F; + Eigen::Matrix _H; + Eigen::Matrix _chi2inv95; + double _std_weight_position; + double _std_weight_vel; +}; + + +class SORT { +public: + SORT(double iou_threshold, int max_age, int min_hits): _iou_threshold(iou_threshold), _max_age(max_age), _min_hits(min_hits), _next_tracklet_id(0) {}; + ~SORT(); + void update(TargetsInFrame &tgts); + std::vector getTracklets() const; +private: + double _iou(Tracklet &tracklet, Box &box); + std::vector> _hungarian(std::vector> costMatrix); + bool _augment(const std::vector>& costMatrix, int row, std::vector& rowMatch, std::vector& colMatch, std::vector& visited); + + double _iou_threshold; + int _max_age; + int _min_hits; + int _next_tracklet_id; + std::vector _tracklets; }; diff --git a/samples/demo/multiple_object_tracking.cpp b/samples/demo/multiple_object_tracking.cpp index 613012a..a61e5f0 100644 --- a/samples/demo/multiple_object_tracking.cpp +++ b/samples/demo/multiple_object_tracking.cpp @@ -7,9 +7,13 @@ using namespace std; int main(int argc, char *argv[]) { // 实例化 + sv::CommonObjectDetector cod; + // 手动导入相机参数,如果使用Amov的G1等吊舱或相机,则可以忽略该步骤,将自动下载相机参数文件 + cod.loadCameraParams(sv::get_home() + "/SpireCV/calib_webcam_640x480.yaml"); sv::MultipleObjectTracker mot; // 手动导入相机参数,如果使用Amov的G1等吊舱或相机,则可以忽略该步骤,将自动下载相机参数文件 mot.loadCameraParams(sv::get_home() + "/SpireCV/calib_webcam_640x480.yaml"); + mot.init(&cod); // 打开摄像头 sv::Camera cap;