更换文档检测模型

paddle_detection/deploy/lite/include/config_parser.h

@@ -0,0 +1,104 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <fstream>
+#include <iostream>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "json/json.h"
+
+#ifdef _WIN32
+#define OS_PATH_SEP "\\"
+#else
+#define OS_PATH_SEP "/"
+#endif
+
+namespace PaddleDetection {
+
+void load_jsonf(std::string jsonfile, Json::Value& jsondata);
+
+// Inference model configuration parser
+class ConfigPaser {
+ public:
+  ConfigPaser() {}
+
+  ~ConfigPaser() {}
+
+  bool load_config(const std::string& model_dir,
+                   const std::string& cfg = "infer_cfg") {
+    Json::Value config;
+    load_jsonf(model_dir + OS_PATH_SEP + cfg + ".json", config);
+
+    // Get model arch : YOLO, SSD, RetinaNet, RCNN, Face, PicoDet, HRNet
+    if (config.isMember("arch")) {
+      arch_ = config["arch"].as<std::string>();
+    } else {
+      std::cerr
+          << "Please set model arch,"
+          << "support value : YOLO, SSD, RetinaNet, RCNN, Face, PicoDet, HRNet."
+          << std::endl;
+      return false;
+    }
+
+    // Get draw_threshold for visualization
+    if (config.isMember("draw_threshold")) {
+      draw_threshold_ = config["draw_threshold"].as<float>();
+    } else {
+      std::cerr << "Please set draw_threshold." << std::endl;
+      return false;
+    }
+    // Get Preprocess for preprocessing
+    if (config.isMember("Preprocess")) {
+      preprocess_info_ = config["Preprocess"];
+    } else {
+      std::cerr << "Please set Preprocess." << std::endl;
+      return false;
+    }
+    // Get label_list for visualization
+    if (config.isMember("label_list")) {
+      label_list_.clear();
+      for (auto item : config["label_list"]) {
+        label_list_.emplace_back(item.as<std::string>());
+      }
+    } else {
+      std::cerr << "Please set label_list." << std::endl;
+      return false;
+    }
+
+    // Get NMS for postprocess
+    if (config.isMember("NMS")) {
+      nms_info_ = config["NMS"];
+    }
+    // Get fpn_stride in PicoDet
+    if (config.isMember("fpn_stride")) {
+      fpn_stride_.clear();
+      for (auto item : config["fpn_stride"]) {
+        fpn_stride_.emplace_back(item.as<int>());
+      }
+    }
+
+    return true;
+  }
+  float draw_threshold_;
+  std::string arch_;
+  Json::Value preprocess_info_;
+  Json::Value nms_info_;
+  std::vector<std::string> label_list_;
+  std::vector<int> fpn_stride_;
+};
+
+}  // namespace PaddleDetection

paddle_detection/deploy/lite/include/keypoint_detector.h

@@ -0,0 +1,107 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ctime>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "paddle_api.h"  // NOLINT
+
+#include "include/config_parser.h"
+#include "include/keypoint_postprocess.h"
+#include "include/preprocess_op.h"
+
+using namespace paddle::lite_api;  // NOLINT
+
+namespace PaddleDetection {
+// Object KeyPoint Result
+struct KeyPointResult {
+  // Keypoints: shape(N x 3); N: number of Joints; 3: x,y,conf
+  std::vector<float> keypoints;
+  int num_joints = -1;
+};
+
+// Visualiztion KeyPoint Result
+cv::Mat VisualizeKptsResult(const cv::Mat& img,
+                            const std::vector<KeyPointResult>& results,
+                            const std::vector<int>& colormap,
+                            float threshold = 0.2);
+
+class KeyPointDetector {
+ public:
+  explicit KeyPointDetector(const std::string& model_dir,
+                            int cpu_threads = 1,
+                            const int batch_size = 1,
+                            bool use_dark = true) {
+    config_.load_config(model_dir);
+    threshold_ = config_.draw_threshold_;
+    use_dark_ = use_dark;
+    preprocessor_.Init(config_.preprocess_info_);
+    printf("before keypoint detector\n");
+    LoadModel(model_dir, cpu_threads);
+    printf("create keypoint detector\n");
+  }
+
+  // Load Paddle inference model
+  void LoadModel(std::string model_file, int num_theads);
+
+  // Run predictor
+  void Predict(const std::vector<cv::Mat> imgs,
+               std::vector<std::vector<float>>& center,
+               std::vector<std::vector<float>>& scale,
+               const int warmup = 0,
+               const int repeats = 1,
+               std::vector<KeyPointResult>* result = nullptr,
+               std::vector<double>* times = nullptr);
+
+  // Get Model Label list
+  const std::vector<std::string>& GetLabelList() const {
+    return config_.label_list_;
+  }
+
+  bool use_dark(){return this->use_dark_;}
+
+  inline float get_threshold() {return threshold_;};
+
+ private:
+  // Preprocess image and copy data to input buffer
+  void Preprocess(const cv::Mat& image_mat);
+  // Postprocess result
+  void Postprocess(std::vector<float>& output,
+                   std::vector<int64_t>& output_shape,
+                   std::vector<int64_t>& idxout,
+                   std::vector<int64_t>& idx_shape,
+                   std::vector<KeyPointResult>* result,
+                   std::vector<std::vector<float>>& center,
+                   std::vector<std::vector<float>>& scale);
+
+  std::shared_ptr<PaddlePredictor> predictor_;
+  Preprocessor preprocessor_;
+  ImageBlob inputs_;
+  std::vector<float> output_data_;
+  std::vector<int64_t> idx_data_;
+  float threshold_;
+  ConfigPaser config_;
+  bool use_dark_;
+};
+
+}  // namespace PaddleDetection

paddle_detection/deploy/lite/include/keypoint_postprocess.h

@@ -0,0 +1,57 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+#include <vector>
+
+std::vector<float> get_3rd_point(std::vector<float>& a, std::vector<float>& b);
+std::vector<float> get_dir(float src_point_x, float src_point_y, float rot_rad);
+void affine_tranform(
+    float pt_x, float pt_y, cv::Mat& trans, std::vector<float>& x, int p, int num);
+cv::Mat get_affine_transform(std::vector<float>& center,
+                             std::vector<float>& scale,
+                             float rot,
+                             std::vector<int>& output_size,
+                             int inv);
+void transform_preds(std::vector<float>& coords,
+                     std::vector<float>& center,
+                     std::vector<float>& scale,
+                     std::vector<int>& output_size,
+                     std::vector<int>& dim,
+                     std::vector<float>& target_coords,
+                     bool affine);
+void box_to_center_scale(std::vector<int>& box,
+                         int width,
+                         int height,
+                         std::vector<float>& center,
+                         std::vector<float>& scale);
+void get_max_preds(std::vector<float>& heatmap,
+                   std::vector<int64_t>& dim,
+                   std::vector<float>& preds,
+                   std::vector<float>& maxvals,
+                   int batchid,
+                   int joint_idx);
+void get_final_preds(std::vector<float>& heatmap,
+                     std::vector<int64_t>& dim,
+                     std::vector<int64_t>& idxout,
+                     std::vector<int64_t>& idxdim,
+                     std::vector<float>& center,
+                     std::vector<float> scale,
+                     std::vector<float>& preds,
+                     int batchid,
+                     bool DARK = true);

paddle_detection/deploy/lite/include/object_detector.h

@@ -0,0 +1,98 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <ctime>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "paddle_api.h"  // NOLINT
+
+#include "include/config_parser.h"
+#include "include/preprocess_op.h"
+#include "include/utils.h"
+#include "include/picodet_postprocess.h"
+
+using namespace paddle::lite_api;  // NOLINT
+
+namespace PaddleDetection {
+
+// Generate visualization colormap for each class
+std::vector<int> GenerateColorMap(int num_class);
+
+// Visualiztion Detection Result
+cv::Mat VisualizeResult(const cv::Mat& img,
+                        const std::vector<PaddleDetection::ObjectResult>& results,
+                        const std::vector<std::string>& lables,
+                        const std::vector<int>& colormap,
+                        const bool is_rbox);
+
+class ObjectDetector {
+ public:
+  explicit ObjectDetector(const std::string& model_dir,
+                          int cpu_threads = 1,
+                          const int batch_size = 1) {
+    config_.load_config(model_dir);
+    printf("config created\n");
+    threshold_ = config_.draw_threshold_;
+    preprocessor_.Init(config_.preprocess_info_);
+    printf("before object detector\n");
+    LoadModel(model_dir, cpu_threads);
+    printf("create object detector\n");
+  }
+
+  // Load Paddle inference model
+  void LoadModel(std::string model_file, int num_theads);
+
+  // Run predictor
+  void Predict(const std::vector<cv::Mat>& imgs,
+               const double threshold = 0.5,
+               const int warmup = 0,
+               const int repeats = 1,
+               std::vector<PaddleDetection::ObjectResult>* result = nullptr,
+               std::vector<int>* bbox_num = nullptr,
+               std::vector<double>* times = nullptr);
+
+  // Get Model Label list
+  const std::vector<std::string>& GetLabelList() const {
+    return config_.label_list_;
+  }
+
+ private:
+  // Preprocess image and copy data to input buffer
+  void Preprocess(const cv::Mat& image_mat);
+  // Postprocess result
+  void Postprocess(const std::vector<cv::Mat> mats,
+                   std::vector<PaddleDetection::ObjectResult>* result,
+                   std::vector<int> bbox_num,
+                   bool is_rbox);
+
+  std::shared_ptr<PaddlePredictor> predictor_;
+  Preprocessor preprocessor_;
+  ImageBlob inputs_;
+  std::vector<float> output_data_;
+  std::vector<int> out_bbox_num_data_;
+  float threshold_;
+  ConfigPaser config_;
+
+};
+
+}  // namespace PaddleDetection

paddle_detection/deploy/lite/include/picodet_postprocess.h

@@ -0,0 +1,39 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <utility>
+#include <ctime>
+#include <numeric>
+#include <math.h>
+
+#include "include/utils.h"
+
+namespace PaddleDetection {
+
+void PicoDetPostProcess(std::vector<PaddleDetection::ObjectResult>* results,
+                         std::vector<const float *> outs,
+                         std::vector<int> fpn_stride,
+                         std::vector<float> im_shape,
+                         std::vector<float> scale_factor,
+                         float score_threshold = 0.3,
+                         float nms_threshold = 0.5,
+                         int num_class = 80,
+                         int reg_max = 7);
+
+}  // namespace PaddleDetection

paddle_detection/deploy/lite/include/preprocess_op.h

@@ -0,0 +1,188 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+#include "json/json.h"
+
+namespace PaddleDetection {
+
+// Object for storing all preprocessed data
+class ImageBlob {
+ public:
+  // image width and height
+  std::vector<float> im_shape_;
+  // Buffer for image data after preprocessing
+  std::vector<float> im_data_;
+  // in net data shape(after pad)
+  std::vector<float> in_net_shape_;
+  // Evaluation image width and height
+  // std::vector<float>  eval_im_size_f_;
+  // Scale factor for image size to origin image size
+  std::vector<float> scale_factor_;
+};
+
+// Abstraction of preprocessing opration class
+class PreprocessOp {
+ public:
+  virtual void Init(const Json::Value& item) = 0;
+  virtual void Run(cv::Mat* im, ImageBlob* data) = 0;
+};
+
+class InitInfo : public PreprocessOp {
+ public:
+  virtual void Init(const Json::Value& item) {}
+  virtual void Run(cv::Mat* im, ImageBlob* data);
+};
+
+class NormalizeImage : public PreprocessOp {
+ public:
+  virtual void Init(const Json::Value& item) {
+    mean_.clear();
+    scale_.clear();
+    for (auto tmp : item["mean"]) {
+      mean_.emplace_back(tmp.as<float>());
+    }
+    for (auto tmp : item["std"]) {
+      scale_.emplace_back(tmp.as<float>());
+    }
+    is_scale_ = item["is_scale"].as<bool>();
+  }
+
+  virtual void Run(cv::Mat* im, ImageBlob* data);
+
+ private:
+  // CHW or HWC
+  std::vector<float> mean_;
+  std::vector<float> scale_;
+  bool is_scale_;
+};
+
+class Permute : public PreprocessOp {
+ public:
+  virtual void Init(const Json::Value& item) {}
+  virtual void Run(cv::Mat* im, ImageBlob* data);
+};
+
+class Resize : public PreprocessOp {
+ public:
+  virtual void Init(const Json::Value& item) {
+    interp_ = item["interp"].as<int>();
+    // max_size_ = item["target_size"].as<int>();
+    keep_ratio_ = item["keep_ratio"].as<bool>();
+    target_size_.clear();
+    for (auto tmp : item["target_size"]) {
+      target_size_.emplace_back(tmp.as<int>());
+    }
+  }
+
+  // Compute best resize scale for x-dimension, y-dimension
+  std::pair<float, float> GenerateScale(const cv::Mat& im);
+
+  virtual void Run(cv::Mat* im, ImageBlob* data);
+
+ private:
+  int interp_;
+  bool keep_ratio_;
+  std::vector<int> target_size_;
+  std::vector<int> in_net_shape_;
+};
+
+// Models with FPN need input shape % stride == 0
+class PadStride : public PreprocessOp {
+ public:
+  virtual void Init(const Json::Value& item) {
+    stride_ = item["stride"].as<int>();
+  }
+
+  virtual void Run(cv::Mat* im, ImageBlob* data);
+
+ private:
+  int stride_;
+};
+
+class TopDownEvalAffine : public PreprocessOp {
+ public:
+  virtual void Init(const Json::Value& item) {
+    trainsize_.clear();
+    for (auto tmp : item["trainsize"]) {
+      trainsize_.emplace_back(tmp.as<int>());
+    }
+  }
+
+  virtual void Run(cv::Mat* im, ImageBlob* data);
+
+ private:
+  int interp_ = 1;
+  std::vector<int> trainsize_;
+};
+
+void CropImg(cv::Mat& img,
+             cv::Mat& crop_img,
+             std::vector<int>& area,
+             std::vector<float>& center,
+             std::vector<float>& scale,
+             float expandratio = 0.15);
+
+class Preprocessor {
+ public:
+  void Init(const Json::Value& config_node) {
+    // initialize image info at first
+    ops_["InitInfo"] = std::make_shared<InitInfo>();
+    for (const auto& item : config_node) {
+      auto op_name = item["type"].as<std::string>();
+
+      ops_[op_name] = CreateOp(op_name);
+      ops_[op_name]->Init(item);
+    }
+  }
+
+  std::shared_ptr<PreprocessOp> CreateOp(const std::string& name) {
+    if (name == "Resize") {
+      return std::make_shared<Resize>();
+    } else if (name == "Permute") {
+      return std::make_shared<Permute>();
+    } else if (name == "NormalizeImage") {
+      return std::make_shared<NormalizeImage>();
+    } else if (name == "PadStride") {
+      // use PadStride instead of PadBatch
+      return std::make_shared<PadStride>();
+    } else if (name == "TopDownEvalAffine") {
+      return std::make_shared<TopDownEvalAffine>();
+    }
+    std::cerr << "can not find function of OP: " << name
+              << " and return: nullptr" << std::endl;
+    return nullptr;
+  }
+
+  void Run(cv::Mat* im, ImageBlob* data);
+
+ public:
+  static const std::vector<std::string> RUN_ORDER;
+
+ private:
+  std::unordered_map<std::string, std::shared_ptr<PreprocessOp>> ops_;
+};
+
+}  // namespace PaddleDetection

paddle_detection/deploy/lite/include/utils.h

@@ -0,0 +1,39 @@
+//   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include <memory>
+#include <utility>
+#include <ctime>
+#include <numeric>
+#include <algorithm>
+
+namespace PaddleDetection {
+
+// Object Detection Result
+struct ObjectResult {
+  // Rectangle coordinates of detected object: left, right, top, down
+  std::vector<int> rect;
+  // Class id of detected object
+  int class_id;
+  // Confidence of detected object
+  float confidence;
+};
+
+void nms(std::vector<ObjectResult> &input_boxes, float nms_threshold);
+
+}  // namespace PaddleDetection