evdameon, evcloudsvc: subsystem monitor thread and defered uploading

opencv-motion-detect/evdaemon.cpp

@@ -83,6 +83,12 @@ private:
     json mapSubSystems;
     json jsonIPs;
 
+    /// monitoring thread
+    condition_variable cvSubMonitor;
+    mutex mutSubMonitor;
+    thread thSubMonitor;
+
+
     int ping(void *s)
     {
         int ret = 0;
@@ -562,13 +568,15 @@ private:
                         peerData["pids"].erase(selfId);
                     }
 
-                    if(this->bBootstrap) {
-                        spdlog::warn("evdaemon {} peer {} disconnected. restarting it.", devSn, selfId);
-                        startSubSystems({selfId});
-                    }
-                    else {
-                        spdlog::warn("evdaemon {} peer {} disconnected. won't restart it since BOOTSTRAP=false", devSn, selfId);
-                    }
+                    spdlog::warn("evdaemon {} peer {} disconnected. will restarting it.", devSn, selfId);
+                    /// let thSubSystem do the following job
+                    // if(this->bBootstrap) {
+                    //     spdlog::warn("evdaemon {} peer {} disconnected. restarting it.", devSn, selfId);
+                    //     startSubSystems({selfId});
+                    // }
+                    // else {
+                    //     spdlog::warn("evdaemon {} peer {} disconnected. won't restart it since BOOTSTRAP=false", devSn, selfId);
+                    // }
                 }
             }
 
@@ -1064,6 +1072,82 @@ public:
         });
         thHeartBeat.detach();
 
+        thSubMonitor = thread([this](){
+            while(true){
+                {
+                    unique_lock<mutex> lk(this->mutSubMonitor);
+                    this->cvSubMonitor.wait_for(lk, 5s, [this] {return !this->bColdStart;});
+
+                    vector<string> tmp;
+                    json unkown;
+                    vector<string> terms;
+                    string info;
+                    json ret;
+                    int cnt = 0;
+                    for(auto &[k,v]: this->peerData["config"].items()) {
+                        if(this->peerData["enabled"].count(k) != 0 && this->peerData["enabled"][k] != 0) {
+                            if((this->peerData["status"].count(k) == 0 || this->peerData["status"][k] == 0)) {
+                                tmp.push_back(k);
+                                info += (cnt == 0? "" : string(", ")) + k;
+                            }
+                            else if(this->peerData["status"][k] == -1) {
+                                unkown[k] = -1;
+                            }
+                        }
+                        else {
+                            terms.push_back(k);
+                        }
+                        cnt++;
+                    }
+
+                    spdlog::info("evdaemon {} will start following subsystems: {}", devSn, info);
+                    //
+                    for(string &e : tmp) {
+                        pid_t pid = 0;
+                        this->peerData["status"][e] = 0;
+                        ret = zmqhelper::forkSubsystem(devSn, e, portRouter, pid);
+                        if(0 == ret) {
+                            this->peerData["pids"][e] = pid;
+                            spdlog::info("evdaemon {} created subsystem {}", devSn, e);
+                        }
+                        else {
+                            spdlog::info("evdaemon {} failed to create subsystem {}", devSn, e);
+                        }
+                    }
+
+                    for(string &e: terms) {
+                        if(this->peerData["pids"].count(e) != 0) {
+                            kill(this->peerData["pids"][e], SIGTERM);
+                        }
+                    }
+
+                    while(unkown.size() != 0 && cnt < 3) {
+                        this_thread::sleep_for(chrono::seconds(3));
+                        for(auto &[k,v]: unkown.items()) {
+                            if(this->peerData["status"][k] != -1 && this->peerData["status"][k] != 0) {
+                                // no need to start
+                                unkown.erase(k);
+                            }
+                        }
+                        cnt++;
+                    }
+
+                    for(auto &[k,v]: unkown.items()) {
+                        pid_t pid = 0;
+                        this->peerData["status"][k] = 0;
+                        ret = zmqhelper::forkSubsystem(devSn, k, portRouter, pid);
+                        if(0 == ret) {
+                            this->peerData["pids"][k] = pid;
+                            spdlog::info("evdaemon {} created subsystem {}", devSn, k);
+                        }
+                        else {
+                            spdlog::info("evdaemon {} failed to create subsystem {}", devSn, k);
+                        }
+                    }
+                }
+            }
+        });
+
         this->thIdMain = this_thread::get_id();
     };
     ~EvDaemon() {};

opencv-motion-detect/inc/zmqhelper.cpp

@@ -207,6 +207,7 @@ int forkSubsystem(string devSn, string peerId, int drPort, pid_t &pid){
     auto v = strutils::split(peerId, ':');
     string modName = v[1];
     string sn = v[0];
+    system(string("killall " + modName).c_str());
     if( (pid = fork()) == -1 ) {
         spdlog::error("evdamon {} failed to fork subsytem: {}", devSn, peerId);
         return -1;