Merge pull request #50 from supermeng/master

Added notify for stop engine

Author: supermeng

engine/engine.go

@@ -697,6 +697,8 @@ func (engine *OrcEngine) onClusterNodeLost(nodeName string, downCount int) {
 	log.Warnf("Cluster node is down, [%q], %d nodes down in all, will check if need stop the engine", nodeName, downCount)
 	if downCount >= maxDownNode {
 		log.Warnf("Too many cluster nodes stoped in a short period, need stop the engine")
+		ntfController.Send(NewNotifySpec("Cluster", "Deployd",
+			1, time.Now(), NotifyClusterAbnormal))
 		engine.Stop()
 	}
 }

engine/notify.go

@@ -24,6 +24,7 @@ var (
 	NotifyPodUnHealthy = "LAIN found pod Unhealthy, please check your service"
 
 	NotifyClusterUnHealthy = "LAIN found Cluster Manager Unhealthy, please check your cluster"
+	NotifyClusterAbnormal = "LAIN found too many cluster nodes stoped in a short period, need stop the engine, please check your cluster"
 
 	NotifyUpgradeFailedTmplt = "LAIN found Last version:%d upgrade is terrrible, please check your code carefully!!"
 )

utils/elector/elect.go

@@ -84,6 +84,7 @@ func (e *Elector) watch(leaderCh chan string, stop chan struct{}) {
 		}
 		retry = 0
 
+		// libkv locker refresh ttl every ttl / 3, so will receive data each refresh time.
 		for kv := range ch {
 			value := string(kv.Value)
 			log.Debugf("Get watch event, leader value changed to %s", value)