new feature: object detection v0.1

opencv-motion-detect/evmlmotion.cpp

@@ -23,6 +23,7 @@ update: 2019/09/10
 #include "common.hpp"
 #include "avcvhelpers.hpp"
 #include "database.h"
+#include "yolo.hpp"
 
 using namespace std;
 using namespace zmqhelper;
@@ -81,6 +82,7 @@ private:
     long long packetTsDelta = 0;
     float pps = 0;
     int pktLag = 0;
+    unique_ptr<YoloDectect> pYolo = nullptr;
     //
 
     int handleCloudMsg(vector<vector<uint8_t> > v)
@@ -358,6 +360,9 @@ private:
             }
             //ping
             ret = ping();
+
+            //
+            pYolo = unique_ptr<YoloDectect>(new YoloDectect("../opencv-yolo"));
         }
         catch(exception &e) {
             spdlog::error("evmlmotion {} exception in EvPuller.init {:s} retrying", selfId, e.what());
@@ -559,10 +564,26 @@ private:
     void detectMotion(AVPixelFormat format,AVFrame *pFrame, bool detect = true)
     {
         static bool first = true;
+        static unsigned long detCnt = 0;
         static cv::Mat avg;
         static vector<vector<cv::Point> > cnts;
         cv::Mat origin, gray, thresh;
         avcvhelpers::frame2mat(format, pFrame, origin);
+        if(detCnt % 6 == 0) {
+            auto objDetRes = pYolo->process(origin, nullptr);
+            if(objDetRes.size() > 0) {
+                string s = fmt::format("{} {} object detected:\n", selfId, objDetRes.size());
+                int i = 0;
+                for(auto &[n, c, _]: objDetRes) {
+                    i++;
+                    s += fmt::format("\t\tobj {}:{}, prob: {}, x:{}, y:{}, w: {}, h:{}\n", i, n, c, _.x, _.y, _.width, _.height);
+                }
+
+                spdlog::info(s);
+            }
+        }
+        detCnt++;
+        
         cv::resize(origin, gray, cv::Size(FRAME_SIZE,FRAME_SIZE));
         cv::cvtColor(gray, thresh, cv::COLOR_BGR2GRAY);
         float fent = avcvhelpers::getEntropy(thresh);

opencv-yolo/main.cpp

@@ -35,6 +35,9 @@ private:
     bool bOutputIsImg = false;
     string outFileBase;
     bool cmdStop = false;
+    unsigned int wrapNum = 0;
+    unsigned long numFrameProcessed = 0;
+    unsigned int numLogSkip = 0;
 
     // Get the names of the output layers
     vector<String> getOutputsNames(const Net& net)
@@ -77,7 +80,7 @@ private:
     }
 
     // post process
-    vector<tuple<string, double, Rect>> postprocess(Mat& frame, const vector<Mat>& outs)
+    vector<tuple<string, double, Rect>> postprocess(Mat& frame, const vector<Mat>& outs, bool bModify = true)
     {
         vector<int> classIds;
         vector<float> confidences;
@@ -117,6 +120,7 @@ private:
             int idx = indices[i];
             Rect box = boxes[idx];
             ret.push_back(tuple<string, double, Rect>(classes[classIds[idx]], confidences[idx], box));
+            if(bModify)
             drawPred(classIds[idx], confidences[idx], box.x, box.y, box.x + box.width, box.y + box.height, frame);
         }
 
@@ -129,12 +133,15 @@ protected:
     //
 public:
     typedef int (*callback)(vector<tuple<string, double, Rect>>&, Mat);
-    YoloDectect(string path = "")
+    YoloDectect(string path = "", unsigned int _wrapNum = 10, unsigned int _numLogSkip = 380)
     {
         if(path.empty()) {
             path = ".";
         }
 
+        wrapNum = _wrapNum;
+        numLogSkip = _numLogSkip;
+        
         // Load names of classes
         string classesFile = path + "/coco.names";
         // Give the configuration and weight files for the model
@@ -159,7 +166,7 @@ public:
         spdlog::info("{} inited", selfId);
     }
 
-    vector<tuple<string, double, Rect>> process(Mat &inFrame, Mat &outFrame)
+    vector<tuple<string, double, Rect>> process(Mat &inFrame, Mat* pOutFrame)
     {
         if(inFrame.empty()) {
             return vector<tuple<string, double, Rect>>();
@@ -176,15 +183,20 @@ public:
         net.forward(outs, getOutputsNames(net));
 
         // Remove the bounding boxes with low confidence
-        auto ret = postprocess(inFrame, outs);
+        auto ret = postprocess(inFrame, outs, true);
 
         // The function getPerfProfile returns the overall time for inference(t) and the timings for each of the layers(in layersTimes)
         vector<double> layersTimes;
+        if(numLogSkip == 0 || numFrameProcessed % numLogSkip == 0) {
             double freq = getTickFrequency() / 1000;
             double t = net.getPerfProfile(layersTimes) / freq;
             spdlog::info("{} infer time: {} ms", selfId, t);
+        }
+        if(pOutFrame != nullptr){
+            inFrame.convertTo(*pOutFrame, CV_8U);
+        }
         
-        inFrame.convertTo(outFrame, CV_8U);
+        numFrameProcessed++;
         return ret;
     }
 
@@ -217,8 +229,8 @@ public:
 
         spdlog::info("{} try to process video {} to {}", selfId, inVideoUri, outFile);
 
-        long frameCnt = 0;
-        long detCnt = 0, skipCnt = 0;
+        unsigned long frameCnt = 0;
+        unsigned long detCnt = 0, skipCnt = 0;
         Mat frame, outFrame;
         while (waitKey(1) < 0) {
             // get frame from the video
@@ -238,11 +250,11 @@ public:
                 continue;
             }
 
-            vector<tuple<string, double, Rect>> ret = process(frame, outFrame);
+            vector<tuple<string, double, Rect>> ret = process(frame, &outFrame);
             if(cb == nullptr) {
                 if(ret.size() == 0 && bOutputIsImg) {
                     // no detection
-                    if(skipCnt % 100 == 0) {
+                    if(numLogSkip == 0|| skipCnt % numLogSkip == 0) {
                         spdlog::info("{} no valid object detected skipped frame count {}", selfId, skipCnt);
                     }
                     skipCnt++;    
@@ -250,6 +262,10 @@ public:
                 }
 
                 if (bOutputIsImg) {
+                    if(wrapNum > 0) {
+                        detCnt = detCnt % wrapNum;
+                    }
+
                     string ofname = outFileBase + to_string(detCnt) + ".jpg";
                     imwrite(ofname, outFrame);
                     detCnt++;
@@ -272,7 +288,7 @@ public:
 int main(int argc, char** argv)
 {
     YoloDectect det;
-    det.process("rtsp://admin:ZQEAAI@192.168.0.101:554/h264/ch1/main/av_stream", "a.avi");
+    det.process("rtsp://admin:ZQEAAI@192.168.0.101:554/h264/ch1/main/av_stream", "a.jpg");
 
     return 0;
 }