更换文档检测模型

paddle_detection/deploy/pptracking/cpp/include/config_parser.h

@@ -0,0 +1,137 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <iostream>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "yaml-cpp/yaml.h"
+
+#ifdef _WIN32
+#define OS_PATH_SEP "\\"
+#else
+#define OS_PATH_SEP "/"
+#endif
+
+namespace PaddleDetection {
+
+// Inference model configuration parser
+class ConfigPaser {
+ public:
+  ConfigPaser() {}
+
+  ~ConfigPaser() {}
+
+  bool load_config(const std::string& model_dir,
+                   const std::string& cfg = "infer_cfg.yml") {
+    // Load as a YAML::Node
+    YAML::Node config;
+    config = YAML::LoadFile(model_dir + OS_PATH_SEP + cfg);
+
+    // Get runtime mode : paddle, trt_fp16, trt_fp32
+    if (config["mode"].IsDefined()) {
+      mode_ = config["mode"].as<std::string>();
+    } else {
+      std::cerr << "Please set mode, "
+                << "support value : paddle/trt_fp16/trt_fp32." << std::endl;
+      return false;
+    }
+
+    // Get model arch: FairMot or YOLO/Picodet/LCNet for DeepSort
+    if (config["arch"].IsDefined()) {
+      arch_ = config["arch"].as<std::string>();
+    } else {
+      std::cerr << "Please set model arch,"
+                << "support value : FairMot, YOLO, PicoDet, LCNet etc"
+                << std::endl;
+      return false;
+    }
+
+    // Get min_subgraph_size for tensorrt
+    if (config["min_subgraph_size"].IsDefined()) {
+      min_subgraph_size_ = config["min_subgraph_size"].as<int>();
+    } else {
+      std::cerr << "Please set min_subgraph_size." << std::endl;
+      return false;
+    }
+    // Get draw_threshold for visualization
+    if (config["draw_threshold"].IsDefined()) {
+      draw_threshold_ = config["draw_threshold"].as<float>();
+    } else {
+      std::cerr << "Please set draw_threshold." << std::endl;
+      return false;
+    }
+    // Get Preprocess for preprocessing
+    if (config["Preprocess"].IsDefined()) {
+      preprocess_info_ = config["Preprocess"];
+    } else {
+      std::cerr << "Please set Preprocess." << std::endl;
+      return false;
+    }
+    // Get label_list for visualization
+    if (config["label_list"].IsDefined()) {
+      label_list_ = config["label_list"].as<std::vector<std::string>>();
+    } else {
+      std::cerr << "Please set label_list." << std::endl;
+      return false;
+    }
+
+    // Get use_dynamic_shape for TensorRT
+    if (config["use_dynamic_shape"].IsDefined()) {
+      use_dynamic_shape_ = config["use_dynamic_shape"].as<bool>();
+    } else {
+      std::cerr << "Please set use_dynamic_shape." << std::endl;
+      return false;
+    }
+
+    // Get conf_thresh for tracker
+    if (config["tracker"].IsDefined()) {
+      if (config["tracker"]["conf_thres"].IsDefined()) {
+        conf_thresh_ = config["tracker"]["conf_thres"].as<float>();
+      } else {
+        std::cerr << "Please set conf_thres in tracker." << std::endl;
+        return false;
+      }
+    }
+
+    // Get NMS for postprocess
+    if (config["NMS"].IsDefined()) {
+      nms_info_ = config["NMS"];
+    }
+    // Get fpn_stride in PicoDet
+    if (config["fpn_stride"].IsDefined()) {
+      fpn_stride_.clear();
+      for (auto item : config["fpn_stride"]) {
+        fpn_stride_.emplace_back(item.as<int>());
+      }
+    }
+
+    return true;
+  }
+  std::string mode_;
+  float draw_threshold_;
+  std::string arch_;
+  int min_subgraph_size_;
+  YAML::Node preprocess_info_;
+  YAML::Node nms_info_;
+  std::vector<std::string> label_list_;
+  std::vector<int> fpn_stride_;
+  bool use_dynamic_shape_;
+  float conf_thresh_;
+};
+
+}  // namespace PaddleDetection

paddle_detection/deploy/pptracking/cpp/include/jde_predictor.h

@@ -0,0 +1,98 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ctime>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "paddle_inference_api.h"  // NOLINT
+
+#include "include/config_parser.h"
+#include "include/preprocess_op.h"
+#include "include/utils.h"
+
+using namespace paddle_infer;  // NOLINT
+
+namespace PaddleDetection {
+
+class JDEPredictor {
+ public:
+  explicit JDEPredictor(const std::string& device = "CPU",
+                        const std::string& model_dir = "",
+                        const double threshold = -1.,
+                        const std::string& run_mode = "paddle",
+                        const int gpu_id = 0,
+                        const bool use_mkldnn = false,
+                        const int cpu_threads = 1,
+                        bool trt_calib_mode = false,
+                        const int min_box_area = 200) {
+    this->device_ = device;
+    this->gpu_id_ = gpu_id;
+    this->use_mkldnn_ = use_mkldnn;
+    this->cpu_math_library_num_threads_ = cpu_threads;
+    this->trt_calib_mode_ = trt_calib_mode;
+    this->min_box_area_ = min_box_area;
+
+    config_.load_config(model_dir);
+    this->min_subgraph_size_ = config_.min_subgraph_size_;
+    preprocessor_.Init(config_.preprocess_info_);
+    LoadModel(model_dir, run_mode);
+    this->conf_thresh_ = config_.conf_thresh_;
+    this->threshold_ = config_.conf_thresh_;
+  }
+
+  // Load Paddle inference model
+  void LoadModel(const std::string& model_dir,
+                 const std::string& run_mode = "paddle");
+
+  // Run predictor
+  void Predict(const std::vector<cv::Mat> imgs,
+               const double threshold = 0.5,
+               MOTResult* result = nullptr,
+               std::vector<double>* times = nullptr);
+
+ private:
+  std::string device_ = "CPU";
+  float threhold = 0.5;
+  int gpu_id_ = 0;
+  bool use_mkldnn_ = false;
+  int cpu_math_library_num_threads_ = 1;
+  int min_subgraph_size_ = 3;
+  bool trt_calib_mode_ = false;
+
+  // Preprocess image and copy data to input buffer
+  void Preprocess(const cv::Mat& image_mat);
+  // Postprocess result
+  void Postprocess(const cv::Mat dets, const cv::Mat emb, MOTResult* result);
+
+  std::shared_ptr<Predictor> predictor_;
+  Preprocessor preprocessor_;
+  ImageBlob inputs_;
+  std::vector<float> bbox_data_;
+  std::vector<float> emb_data_;
+  double threshold_;
+  ConfigPaser config_;
+  float min_box_area_;
+  float conf_thresh_;
+};
+
+}  // namespace PaddleDetection

paddle_detection/deploy/pptracking/cpp/include/lapjv.h

@@ -0,0 +1,64 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The code is based on:
+// https://github.com/gatagat/lap/blob/master/lap/lapjv.h
+// Ths copyright of gatagat/lap is as follows:
+// MIT License
+
+#ifndef DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_
+#define DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_
+#define LARGE 1000000
+
+#if !defined TRUE
+#define TRUE 1
+#endif
+#if !defined FALSE
+#define FALSE 0
+#endif
+
+#define NEW(x, t, n)                                               \
+  if ((x = reinterpret_cast<t *>(malloc(sizeof(t) * (n)))) == 0) { \
+    return -1;                                                     \
+  }
+#define FREE(x) \
+  if (x != 0) { \
+    free(x);    \
+    x = 0;      \
+  }
+#define SWAP_INDICES(a, b) \
+  {                        \
+    int_t _temp_index = a; \
+    a = b;                 \
+    b = _temp_index;       \
+  }
+#include <opencv2/opencv.hpp>
+
+namespace PaddleDetection {
+
+typedef signed int int_t;
+typedef unsigned int uint_t;
+typedef double cost_t;
+typedef char boolean;
+typedef enum fp_t { FP_1 = 1, FP_2 = 2, FP_DYNAMIC = 3 } fp_t;
+
+int lapjv_internal(const cv::Mat &cost,
+                   const bool extend_cost,
+                   const float cost_limit,
+                   int *x,
+                   int *y);
+
+}  // namespace PaddleDetection
+
+#endif  // DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_

paddle_detection/deploy/pptracking/cpp/include/pipeline.h

@@ -0,0 +1,142 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef DEPLOY_PPTRACKING_CPP_INCLUDE_PIPELINE_H_
+#define DEPLOY_PPTRACKING_CPP_INCLUDE_PIPELINE_H_
+
+#include <glog/logging.h>
+
+#include <math.h>
+#include <sys/types.h>
+#include <algorithm>
+#include <iostream>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#ifdef _WIN32
+#include <direct.h>
+#include <io.h>
+#elif LINUX
+#include <stdarg.h>
+#include <sys/stat.h>
+#endif
+
+#include "include/jde_predictor.h"
+#include "include/sde_predictor.h"
+
+namespace PaddleDetection {
+
+class Pipeline {
+ public:
+  explicit Pipeline(const std::string& device,
+                    const double threshold,
+                    const std::string& output_dir,
+                    const std::string& run_mode = "paddle",
+                    const int gpu_id = 0,
+                    const bool use_mkldnn = false,
+                    const int cpu_threads = 1,
+                    const bool trt_calib_mode = false,
+                    const bool do_entrance_counting = false,
+                    const bool save_result = false,
+                    const std::string& scene = "pedestrian",
+                    const bool tiny_obj = false,
+                    const bool is_mtmct = false,
+                    const int secs_interval = 10,
+                    const std::string track_model_dir = "",
+                    const std::string det_model_dir = "",
+                    const std::string reid_model_dir = "") {
+    std::vector<std::string> input;
+    this->input_ = input;
+    this->device_ = device;
+    this->threshold_ = threshold;
+    this->output_dir_ = output_dir;
+    this->run_mode_ = run_mode;
+    this->gpu_id_ = gpu_id;
+    this->use_mkldnn_ = use_mkldnn;
+    this->cpu_threads_ = cpu_threads;
+    this->trt_calib_mode_ = trt_calib_mode;
+    this->do_entrance_counting_ = do_entrance_counting;
+    this->secs_interval_ = secs_interval_;
+    this->save_result_ = save_result;
+    SelectModel(scene,
+                tiny_obj,
+                is_mtmct,
+                track_model_dir,
+                det_model_dir,
+                reid_model_dir);
+    InitPredictor();
+  }
+
+  // Set input, it must execute before Run()
+  void SetInput(const std::string& input_video);
+  void ClearInput();
+
+  // Run pipeline in video
+  void Run();
+  void PredictMOT(const std::string& video_path);
+  void PredictMTMCT(const std::vector<std::string> video_inputs);
+
+  // Run pipeline in stream
+  void RunMOTStream(const cv::Mat img,
+                    const int frame_id,
+                    const int video_fps,
+                    const Rect entrance,
+                    cv::Mat out_img,
+                    std::vector<std::string>* records,
+                    std::set<int>* count_set,
+                    std::set<int>* interval_count_set,
+                    std::vector<int>* in_count_list,
+                    std::vector<int>* out_count_list,
+                    std::map<int, std::vector<float>>* prev_center,
+                    std::vector<std::string>* flow_records);
+  void RunMTMCTStream(const std::vector<cv::Mat> imgs,
+                      std::vector<std::string>* records);
+
+  void PrintBenchmarkLog(const std::vector<double> det_time, const int img_num);
+
+ private:
+  // Select model according to scenes, it must execute before Run()
+  void SelectModel(const std::string& scene = "pedestrian",
+                   const bool tiny_obj = false,
+                   const bool is_mtmct = false,
+                   const std::string track_model_dir = "",
+                   const std::string det_model_dir = "",
+                   const std::string reid_model_dir = "");
+  void InitPredictor();
+
+  std::shared_ptr<PaddleDetection::JDEPredictor> jde_sct_;
+  std::shared_ptr<PaddleDetection::SDEPredictor> sde_sct_;
+
+  std::vector<std::string> input_;
+  std::vector<cv::Mat> stream_;
+  std::string device_;
+  double threshold_;
+  std::string output_dir_;
+  std::string track_model_dir_;
+  std::string det_model_dir_;
+  std::string reid_model_dir_;
+  std::string run_mode_ = "paddle";
+  int gpu_id_ = 0;
+  bool use_mkldnn_ = false;
+  int cpu_threads_ = 1;
+  bool trt_calib_mode_ = false;
+  bool do_entrance_counting_ = false;
+  bool save_result_ = false;
+  int secs_interval_ = 10;
+};
+
+}  // namespace PaddleDetection
+
+#endif  // DEPLOY_PPTRACKING_CPP_INCLUDE_PIPELINE_H_

paddle_detection/deploy/pptracking/cpp/include/postprocess.h

@@ -0,0 +1,62 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <glog/logging.h>
+
+#include <ctime>
+#include <memory>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "include/utils.h"
+
+namespace PaddleDetection {
+
+// Generate visualization color
+cv::Scalar GetColor(int idx);
+
+// Visualize Tracking Results
+cv::Mat VisualizeTrackResult(const cv::Mat& img,
+                             const MOTResult& results,
+                             const float fps,
+                             const int frame_id);
+
+// Pedestrian/Vehicle Counting
+void FlowStatistic(const MOTResult& results,
+                   const int frame_id,
+                   const int secs_interval,
+                   const bool do_entrance_counting,
+                   const int video_fps,
+                   const Rect entrance,
+                   std::set<int>* id_set,
+                   std::set<int>* interval_id_set,
+                   std::vector<int>* in_id_list,
+                   std::vector<int>* out_id_list,
+                   std::map<int, std::vector<float>>* prev_center,
+                   std::vector<std::string>* records);
+
+// Save Tracking Results
+void SaveMOTResult(const MOTResult& results,
+                   const int frame_id,
+                   std::vector<std::string>* records);
+
+}  // namespace PaddleDetection

paddle_detection/deploy/pptracking/cpp/include/predictor.h

@@ -0,0 +1,99 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ctime>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "paddle_inference_api.h"  // NOLINT
+
+#include "include/config_parser.h"
+#include "include/jde_predictor.h"
+#include "include/preprocess_op.h"
+#include "include/sde_predictor.h"
+
+using namespace paddle_infer;  // NOLINT
+
+namespace PaddleDetection {
+
+class Predictor {
+ public:
+  explicit Predictor(const std::string& device = "CPU",
+                     const std::string& track_model_dir = "",
+                     const std::string& det_model_dir = "",
+                     const std::string& reid_model_dir = "",
+                     const double threshold = -1.,
+                     const std::string& run_mode = "paddle",
+                     const int gpu_id = 0,
+                     const bool use_mkldnn = false,
+                     const int cpu_threads = 1,
+                     bool trt_calib_mode = false,
+                     const int min_box_area = 200) {
+    if (track_model_dir.empty() && det_model_dir.empty()) {
+      throw "Predictor must receive track_model or det_model!";
+    }
+
+    if (!track_model_dir.empty() && !det_model_dir.empty()) {
+      throw "Predictor only receive one of track_model or det_model!";
+    }
+
+    if (!track_model_dir.empty()) {
+      jde_sct_ =
+          std::make_shared<PaddleDetection::JDEPredictor>(device,
+                                                          track_model_dir,
+                                                          threshold,
+                                                          run_mode,
+                                                          gpu_id,
+                                                          use_mkldnn,
+                                                          cpu_threads,
+                                                          trt_calib_mode,
+                                                          min_box_area);
+      use_jde_ = true;
+    }
+    if (!det_model_dir.empty()) {
+      sde_sct_ = std::make_shared<PaddleDetection::SDEPredictor>(device,
+                                                                 det_model_dir,
+                                                                 reid_model_dir,
+                                                                 threshold,
+                                                                 run_mode,
+                                                                 gpu_id,
+                                                                 use_mkldnn,
+                                                                 cpu_threads,
+                                                                 trt_calib_mode,
+                                                                 min_box_area);
+      use_jde_ = false;
+    }
+  }
+
+  // Run predictor
+  void Predict(const std::vector<cv::Mat> imgs,
+               const double threshold = 0.5,
+               MOTResult* result = nullptr,
+               std::vector<double>* times = nullptr);
+
+ private:
+  std::shared_ptr<PaddleDetection::JDEPredictor> jde_sct_;
+  std::shared_ptr<PaddleDetection::SDEPredictor> sde_sct_;
+  bool use_jde_ = true;
+};
+
+}  // namespace PaddleDetection

paddle_detection/deploy/pptracking/cpp/include/preprocess_op.h

@@ -0,0 +1,171 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <glog/logging.h>
+#include <yaml-cpp/yaml.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+namespace PaddleDetection {
+
+// Object for storing all preprocessed data
+class ImageBlob {
+ public:
+  // image width and height
+  std::vector<float> im_shape_;
+  // Buffer for image data after preprocessing
+  std::vector<float> im_data_;
+  // in net data shape(after pad)
+  std::vector<float> in_net_shape_;
+  // Evaluation image width and height
+  // std::vector<float>  eval_im_size_f_;
+  // Scale factor for image size to origin image size
+  std::vector<float> scale_factor_;
+};
+
+// Abstraction of preprocessing opration class
+class PreprocessOp {
+ public:
+  virtual void Init(const YAML::Node& item) = 0;
+  virtual void Run(cv::Mat* im, ImageBlob* data) = 0;
+};
+
+class InitInfo : public PreprocessOp {
+ public:
+  virtual void Init(const YAML::Node& item) {}
+  virtual void Run(cv::Mat* im, ImageBlob* data);
+};
+
+class NormalizeImage : public PreprocessOp {
+ public:
+  virtual void Init(const YAML::Node& item) {
+    mean_ = item["mean"].as<std::vector<float>>();
+    scale_ = item["std"].as<std::vector<float>>();
+    is_scale_ = item["is_scale"].as<bool>();
+  }
+
+  virtual void Run(cv::Mat* im, ImageBlob* data);
+
+ private:
+  // CHW or HWC
+  std::vector<float> mean_;
+  std::vector<float> scale_;
+  bool is_scale_;
+};
+
+class Permute : public PreprocessOp {
+ public:
+  virtual void Init(const YAML::Node& item) {}
+  virtual void Run(cv::Mat* im, ImageBlob* data);
+};
+
+class Resize : public PreprocessOp {
+ public:
+  virtual void Init(const YAML::Node& item) {
+    interp_ = item["interp"].as<int>();
+    keep_ratio_ = item["keep_ratio"].as<bool>();
+    target_size_ = item["target_size"].as<std::vector<int>>();
+  }
+
+  // Compute best resize scale for x-dimension, y-dimension
+  std::pair<float, float> GenerateScale(const cv::Mat& im);
+
+  virtual void Run(cv::Mat* im, ImageBlob* data);
+
+ private:
+  int interp_;
+  bool keep_ratio_;
+  std::vector<int> target_size_;
+  std::vector<int> in_net_shape_;
+};
+
+class LetterBoxResize : public PreprocessOp {
+ public:
+  virtual void Init(const YAML::Node& item) {
+    target_size_ = item["target_size"].as<std::vector<int>>();
+  }
+
+  float GenerateScale(const cv::Mat& im);
+
+  virtual void Run(cv::Mat* im, ImageBlob* data);
+
+ private:
+  std::vector<int> target_size_;
+  std::vector<int> in_net_shape_;
+};
+// Models with FPN need input shape % stride == 0
+class PadStride : public PreprocessOp {
+ public:
+  virtual void Init(const YAML::Node& item) {
+    stride_ = item["stride"].as<int>();
+  }
+
+  virtual void Run(cv::Mat* im, ImageBlob* data);
+
+ private:
+  int stride_;
+};
+
+class Preprocessor {
+ public:
+  void Init(const YAML::Node& config_node) {
+    // initialize image info at first
+    ops_["InitInfo"] = std::make_shared<InitInfo>();
+    for (const auto& item : config_node) {
+      auto op_name = item["type"].as<std::string>();
+
+      ops_[op_name] = CreateOp(op_name);
+      ops_[op_name]->Init(item);
+    }
+  }
+
+  std::shared_ptr<PreprocessOp> CreateOp(const std::string& name) {
+    if (name == "Resize") {
+      return std::make_shared<Resize>();
+    } else if (name == "LetterBoxResize") {
+      return std::make_shared<LetterBoxResize>();
+    } else if (name == "Permute") {
+      return std::make_shared<Permute>();
+    } else if (name == "NormalizeImage") {
+      return std::make_shared<NormalizeImage>();
+    } else if (name == "PadStride") {
+      // use PadStride instead of PadBatch
+      return std::make_shared<PadStride>();
+    }
+    std::cerr << "can not find function of OP: " << name
+              << " and return: nullptr" << std::endl;
+    return nullptr;
+  }
+
+  void Run(cv::Mat* im, ImageBlob* data);
+
+ public:
+  static const std::vector<std::string> RUN_ORDER;
+
+ private:
+  std::unordered_map<std::string, std::shared_ptr<PreprocessOp>> ops_;
+};
+
+}  // namespace PaddleDetection

paddle_detection/deploy/pptracking/cpp/include/sde_predictor.h

@@ -0,0 +1,106 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ctime>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "paddle_inference_api.h"  // NOLINT
+
+#include "include/config_parser.h"
+#include "include/preprocess_op.h"
+#include "include/utils.h"
+
+using namespace paddle_infer;  // NOLINT
+
+namespace PaddleDetection {
+
+class SDEPredictor {
+ public:
+  explicit SDEPredictor(const std::string& device,
+                        const std::string& det_model_dir = "",
+                        const std::string& reid_model_dir = "",
+                        const double threshold = -1.,
+                        const std::string& run_mode = "paddle",
+                        const int gpu_id = 0,
+                        const bool use_mkldnn = false,
+                        const int cpu_threads = 1,
+                        bool trt_calib_mode = false,
+                        const int min_box_area = 200) {
+    this->device_ = device;
+    this->gpu_id_ = gpu_id;
+    this->use_mkldnn_ = use_mkldnn;
+    this->cpu_math_library_num_threads_ = cpu_threads;
+    this->trt_calib_mode_ = trt_calib_mode;
+    this->min_box_area_ = min_box_area;
+
+    det_config_.load_config(det_model_dir);
+    this->min_subgraph_size_ = det_config_.min_subgraph_size_;
+    det_preprocessor_.Init(det_config_.preprocess_info_);
+
+    reid_config_.load_config(reid_model_dir);
+    reid_preprocessor_.Init(reid_config_.preprocess_info_);
+
+    LoadModel(det_model_dir, reid_model_dir, run_mode);
+    this->conf_thresh_ = det_config_.conf_thresh_;
+  }
+
+  // Load Paddle inference model
+  void LoadModel(const std::string& det_model_dir,
+                 const std::string& reid_model_dir,
+                 const std::string& run_mode = "paddle");
+
+  // Run predictor
+  void Predict(const std::vector<cv::Mat> imgs,
+               const double threshold = 0.5,
+               MOTResult* result = nullptr,
+               std::vector<double>* times = nullptr);
+
+ private:
+  std::string device_ = "CPU";
+  float threhold = 0.5;
+  int gpu_id_ = 0;
+  bool use_mkldnn_ = false;
+  int cpu_math_library_num_threads_ = 1;
+  int min_subgraph_size_ = 3;
+  bool trt_calib_mode_ = false;
+
+  // Preprocess image and copy data to input buffer
+  void Preprocess(const cv::Mat& image_mat);
+  // Postprocess result
+  void Postprocess(const cv::Mat dets, const cv::Mat emb, MOTResult* result);
+
+  std::shared_ptr<Predictor> det_predictor_;
+  std::shared_ptr<Predictor> reid_predictor_;
+  Preprocessor det_preprocessor_;
+  Preprocessor reid_preprocessor_;
+  ImageBlob inputs_;
+  std::vector<float> bbox_data_;
+  std::vector<float> emb_data_;
+  double threshold_;
+  ConfigPaser det_config_;
+  ConfigPaser reid_config_;
+  float min_box_area_ = 200;
+  float conf_thresh_;
+};
+
+}  // namespace PaddleDetection

paddle_detection/deploy/pptracking/cpp/include/tracker.h

@@ -0,0 +1,72 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The code is based on:
+// https://github.com/CnybTseng/JDE/blob/master/platforms/common/jdetracker.h
+// Ths copyright of CnybTseng/JDE is as follows:
+// MIT License
+
+#pragma once
+
+#include <map>
+#include <vector>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+#include "include/trajectory.h"
+
+namespace PaddleDetection {
+
+typedef std::map<int, int> Match;
+typedef std::map<int, int>::iterator MatchIterator;
+
+struct Track {
+  int id;
+  float score;
+  cv::Vec4f ltrb;
+};
+
+class JDETracker {
+ public:
+  static JDETracker *instance(void);
+  virtual bool update(const cv::Mat &dets,
+                      const cv::Mat &emb,
+                      std::vector<Track> *tracks);
+
+ private:
+  JDETracker(void);
+  virtual ~JDETracker(void) {}
+  cv::Mat motion_distance(const TrajectoryPtrPool &a, const TrajectoryPool &b);
+  void linear_assignment(const cv::Mat &cost,
+                         float cost_limit,
+                         Match *matches,
+                         std::vector<int> *mismatch_row,
+                         std::vector<int> *mismatch_col);
+  void remove_duplicate_trajectory(TrajectoryPool *a,
+                                   TrajectoryPool *b,
+                                   float iou_thresh = 0.15f);
+
+ private:
+  static JDETracker *me;
+  int timestamp;
+  TrajectoryPool tracked_trajectories;
+  TrajectoryPool lost_trajectories;
+  TrajectoryPool removed_trajectories;
+  int max_lost_time;
+  float lambda;
+  float det_thresh;
+};
+
+}  // namespace PaddleDetection

paddle_detection/deploy/pptracking/cpp/include/trajectory.h

@@ -0,0 +1,230 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The code is based on:
+// https://github.com/CnybTseng/JDE/blob/master/platforms/common/trajectory.h
+// Ths copyright of CnybTseng/JDE is as follows:
+// MIT License
+
+#pragma once
+
+#include <vector>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+#include "opencv2/video/tracking.hpp"
+
+namespace PaddleDetection {
+
+typedef enum { New = 0, Tracked = 1, Lost = 2, Removed = 3 } TrajectoryState;
+
+class Trajectory;
+typedef std::vector<Trajectory> TrajectoryPool;
+typedef std::vector<Trajectory>::iterator TrajectoryPoolIterator;
+typedef std::vector<Trajectory *> TrajectoryPtrPool;
+typedef std::vector<Trajectory *>::iterator TrajectoryPtrPoolIterator;
+
+class TKalmanFilter : public cv::KalmanFilter {
+ public:
+  TKalmanFilter(void);
+  virtual ~TKalmanFilter(void) {}
+  virtual void init(const cv::Mat &measurement);
+  virtual const cv::Mat &predict();
+  virtual const cv::Mat &correct(const cv::Mat &measurement);
+  virtual void project(cv::Mat *mean, cv::Mat *covariance) const;
+
+ private:
+  float std_weight_position;
+  float std_weight_velocity;
+};
+
+inline TKalmanFilter::TKalmanFilter(void) : cv::KalmanFilter(8, 4) {
+  cv::KalmanFilter::transitionMatrix = cv::Mat::eye(8, 8, CV_32F);
+  for (int i = 0; i < 4; ++i)
+    cv::KalmanFilter::transitionMatrix.at<float>(i, i + 4) = 1;
+  cv::KalmanFilter::measurementMatrix = cv::Mat::eye(4, 8, CV_32F);
+  std_weight_position = 1 / 20.f;
+  std_weight_velocity = 1 / 160.f;
+}
+
+class Trajectory : public TKalmanFilter {
+ public:
+  Trajectory();
+  Trajectory(const cv::Vec4f &ltrb, float score, const cv::Mat &embedding);
+  Trajectory(const Trajectory &other);
+  Trajectory &operator=(const Trajectory &rhs);
+  virtual ~Trajectory(void) {}
+
+  static int next_id();
+  virtual const cv::Mat &predict(void);
+  virtual void update(Trajectory *traj,
+                      int timestamp,
+                      bool update_embedding = true);
+  virtual void activate(int timestamp);
+  virtual void reactivate(Trajectory *traj, int timestamp, bool newid = false);
+  virtual void mark_lost(void);
+  virtual void mark_removed(void);
+
+  friend TrajectoryPool operator+(const TrajectoryPool &a,
+                                  const TrajectoryPool &b);
+  friend TrajectoryPool operator+(const TrajectoryPool &a,
+                                  const TrajectoryPtrPool &b);
+  friend TrajectoryPool &operator+=(TrajectoryPool &a,  // NOLINT
+                                    const TrajectoryPtrPool &b);
+  friend TrajectoryPool operator-(const TrajectoryPool &a,
+                                  const TrajectoryPool &b);
+  friend TrajectoryPool &operator-=(TrajectoryPool &a,  // NOLINT
+                                    const TrajectoryPool &b);
+  friend TrajectoryPtrPool operator+(const TrajectoryPtrPool &a,
+                                     const TrajectoryPtrPool &b);
+  friend TrajectoryPtrPool operator+(const TrajectoryPtrPool &a,
+                                     TrajectoryPool *b);
+  friend TrajectoryPtrPool operator-(const TrajectoryPtrPool &a,
+                                     const TrajectoryPtrPool &b);
+
+  friend cv::Mat embedding_distance(const TrajectoryPool &a,
+                                    const TrajectoryPool &b);
+  friend cv::Mat embedding_distance(const TrajectoryPtrPool &a,
+                                    const TrajectoryPtrPool &b);
+  friend cv::Mat embedding_distance(const TrajectoryPtrPool &a,
+                                    const TrajectoryPool &b);
+
+  friend cv::Mat mahalanobis_distance(const TrajectoryPool &a,
+                                      const TrajectoryPool &b);
+  friend cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a,
+                                      const TrajectoryPtrPool &b);
+  friend cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a,
+                                      const TrajectoryPool &b);
+
+  friend cv::Mat iou_distance(const TrajectoryPool &a, const TrajectoryPool &b);
+  friend cv::Mat iou_distance(const TrajectoryPtrPool &a,
+                              const TrajectoryPtrPool &b);
+  friend cv::Mat iou_distance(const TrajectoryPtrPool &a,
+                              const TrajectoryPool &b);
+
+ private:
+  void update_embedding(const cv::Mat &embedding);
+
+ public:
+  TrajectoryState state;
+  cv::Vec4f ltrb;
+  cv::Mat smooth_embedding;
+  int id;
+  bool is_activated;
+  int timestamp;
+  int starttime;
+  float score;
+
+ private:
+  static int count;
+  cv::Vec4f xyah;
+  cv::Mat current_embedding;
+  float eta;
+  int length;
+};
+
+inline cv::Vec4f ltrb2xyah(const cv::Vec4f &ltrb) {
+  cv::Vec4f xyah;
+  xyah[0] = (ltrb[0] + ltrb[2]) * 0.5f;
+  xyah[1] = (ltrb[1] + ltrb[3]) * 0.5f;
+  xyah[3] = ltrb[3] - ltrb[1];
+  xyah[2] = (ltrb[2] - ltrb[0]) / xyah[3];
+  return xyah;
+}
+
+inline Trajectory::Trajectory()
+    : state(New),
+      ltrb(cv::Vec4f()),
+      smooth_embedding(cv::Mat()),
+      id(0),
+      is_activated(false),
+      timestamp(0),
+      starttime(0),
+      score(0),
+      eta(0.9),
+      length(0) {}
+
+inline Trajectory::Trajectory(const cv::Vec4f &ltrb_,
+                              float score_,
+                              const cv::Mat &embedding)
+    : state(New),
+      ltrb(ltrb_),
+      smooth_embedding(cv::Mat()),
+      id(0),
+      is_activated(false),
+      timestamp(0),
+      starttime(0),
+      score(score_),
+      eta(0.9),
+      length(0) {
+  xyah = ltrb2xyah(ltrb);
+  update_embedding(embedding);
+}
+
+inline Trajectory::Trajectory(const Trajectory &other)
+    : state(other.state),
+      ltrb(other.ltrb),
+      id(other.id),
+      is_activated(other.is_activated),
+      timestamp(other.timestamp),
+      starttime(other.starttime),
+      xyah(other.xyah),
+      score(other.score),
+      eta(other.eta),
+      length(other.length) {
+  other.smooth_embedding.copyTo(smooth_embedding);
+  other.current_embedding.copyTo(current_embedding);
+  // copy state in KalmanFilter
+
+  other.statePre.copyTo(cv::KalmanFilter::statePre);
+  other.statePost.copyTo(cv::KalmanFilter::statePost);
+  other.errorCovPre.copyTo(cv::KalmanFilter::errorCovPre);
+  other.errorCovPost.copyTo(cv::KalmanFilter::errorCovPost);
+}
+
+inline Trajectory &Trajectory::operator=(const Trajectory &rhs) {
+  this->state = rhs.state;
+  this->ltrb = rhs.ltrb;
+  rhs.smooth_embedding.copyTo(this->smooth_embedding);
+  this->id = rhs.id;
+  this->is_activated = rhs.is_activated;
+  this->timestamp = rhs.timestamp;
+  this->starttime = rhs.starttime;
+  this->xyah = rhs.xyah;
+  this->score = rhs.score;
+  rhs.current_embedding.copyTo(this->current_embedding);
+  this->eta = rhs.eta;
+  this->length = rhs.length;
+
+  // copy state in KalmanFilter
+
+  rhs.statePre.copyTo(cv::KalmanFilter::statePre);
+  rhs.statePost.copyTo(cv::KalmanFilter::statePost);
+  rhs.errorCovPre.copyTo(cv::KalmanFilter::errorCovPre);
+  rhs.errorCovPost.copyTo(cv::KalmanFilter::errorCovPost);
+
+  return *this;
+}
+
+inline int Trajectory::next_id() {
+  ++count;
+  return count;
+}
+
+inline void Trajectory::mark_lost(void) { state = Lost; }
+
+inline void Trajectory::mark_removed(void) { state = Removed; }
+
+}  // namespace PaddleDetection

paddle_detection/deploy/pptracking/cpp/include/utils.h

@@ -0,0 +1,44 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <algorithm>
+#include <ctime>
+#include <numeric>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "include/tracker.h"
+
+namespace PaddleDetection {
+
+struct Rect {
+  float left;
+  float top;
+  float right;
+  float bottom;
+};
+
+struct MOTTrack {
+  int ids;
+  float score;
+  Rect rects;
+  int class_id = -1;
+};
+
+typedef std::vector<MOTTrack> MOTResult;
+
+}  // namespace PaddleDetection