add test case

.gitignore

@@ -12,3 +12,5 @@
 
 # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
 .glide/
+
+.vscode/

election.go

@@ -147,6 +147,8 @@ func (e *elector) ElectionLoop(ctx context.Context, group uint64, current string
 					time.Sleep(time.Second * time.Duration(e.options.leaseSec))
 				}
 			}
+
+			time.Sleep(loopInterval)
 		}
 	}
 }

election_test.go

@@ -90,8 +90,10 @@ func (t *electorTester) notifyStarted() {
 }
 
 func TestElectionLoop(t *testing.T) {
-	stopC, port, err := startTestSingleEtcd()
-	assert.Nil(t, err, "start embed etcd failed")
+	stopC, port, err := startTestSingleEtcd(t)
+	if err != nil {
+		assert.FailNowf(t, "start embed etcd failed", "error: %+v", err)
+	}
 	defer close(stopC)
 
 	client, err := clientv3.New(clientv3.Config{
@@ -103,17 +105,24 @@ func TestElectionLoop(t *testing.T) {
 
 	elector, err := NewElector(client, WithLeaderLeaseSeconds(1))
 	assert.Nil(t, err, "create elector failed")
+	defer elector.Stop(0)
 
 	value1 := newElectorTester(0, "1", elector)
-	value1.start()
-	assert.True(t, value1.isLeader(), "value1 must be leader")
-
 	value2 := newElectorTester(0, "2", elector)
+
+	value1.start()
 	value2.start()
+
+	defer value1.stop(0)
+	defer value2.stop(0)
+
+	assert.True(t, value1.isLeader(), "value1 must be leader")
 	assert.False(t, value2.isLeader(), "value2 must be follower")
 
 	value3 := newElectorTester(0, "", elector)
 	value3.start()
+	defer value3.stop(0)
+
 	assert.False(t, value3.isLeader(), "value3 must be follower")
 
 	value1.stop(0)
@@ -127,8 +136,10 @@ func TestElectionLoop(t *testing.T) {
 }
 
 func TestElectionLoopWithDistributedLock(t *testing.T) {
-	stopC, port, err := startTestSingleEtcd()
-	assert.Nil(t, err, "start embed etcd failed")
+	stopC, port, err := startTestSingleEtcd(t)
+	if err != nil {
+		assert.FailNowf(t, "start embed etcd failed", "error: %+v", err)
+	}
 	defer close(stopC)
 
 	client, err := clientv3.New(clientv3.Config{
@@ -142,26 +153,33 @@ func TestElectionLoopWithDistributedLock(t *testing.T) {
 		WithLeaderLeaseSeconds(1),
 		WithLockIfBecomeLeader(true))
 	assert.Nil(t, err, "create elector failed")
+	defer elector.Stop(0)
 
 	value1 := newElectorTester(0, "1", elector)
-	value1.start()
-	assert.True(t, value1.isLeader(), "value1 must be leader")
-
 	value2 := newElectorTester(0, "2", elector)
+
+	value1.start()
 	value2.start()
+
+	defer value1.stop(0)
+	defer value2.stop(0)
+
+	assert.True(t, value1.isLeader(), "value1 must be leader")
 	assert.False(t, value2.isLeader(), "value2 must be follower")
 
 	value1.stop(time.Second * 2)
-	time.Sleep(time.Second)
+	time.Sleep(time.Second + time.Millisecond*200)
 	assert.False(t, value2.isLeader(), "value2 must be follower before distributed lock released")
 
-	time.Sleep(time.Second + time.Millisecond*100)
+	time.Sleep(time.Second + time.Millisecond*200)
 	assert.True(t, value2.isLeader(), "value2 must be leader after distributed lock released")
 }
 
 func TestChangeLeaderTo(t *testing.T) {
-	stopC, port, err := startTestSingleEtcd()
-	assert.Nil(t, err, "start embed etcd failed")
+	stopC, port, err := startTestSingleEtcd(t)
+	if err != nil {
+		assert.FailNowf(t, "start embed etcd failed", "error: %+v", err)
+	}
 	defer close(stopC)
 
 	client, err := clientv3.New(clientv3.Config{
@@ -174,27 +192,33 @@ func TestChangeLeaderTo(t *testing.T) {
 	elector, err := NewElector(client,
 		WithLeaderLeaseSeconds(1))
 	assert.Nil(t, err, "create elector failed")
+	defer elector.Stop(0)
 
 	value1 := newElectorTester(0, "1", elector)
 	value2 := newElectorTester(0, "2", elector)
 
 	value1.start()
 	value2.start()
 
+	defer value1.stop(0)
+	defer value2.stop(0)
+
 	err = elector.ChangeLeaderTo(0, "2", "3")
 	assert.NotNil(t, err, "only leader node can transfer leader")
 
 	err = elector.ChangeLeaderTo(0, "1", "2")
 	assert.Nil(t, err, "change leader failed")
 
-	time.Sleep(time.Second)
+	time.Sleep(time.Second + time.Millisecond*200)
 	assert.False(t, value1.isLeader(), "value1 must be follower")
 	assert.True(t, value2.isLeader(), "value2 must be leader")
 }
 
 func TestCurrentLeader(t *testing.T) {
-	stopC, port, err := startTestSingleEtcd()
-	assert.Nil(t, err, "start embed etcd failed")
+	stopC, port, err := startTestSingleEtcd(t)
+	if err != nil {
+		assert.FailNowf(t, "start embed etcd failed", "error: %+v", err)
+	}
 	defer close(stopC)
 
 	client, err := clientv3.New(clientv3.Config{
@@ -207,29 +231,35 @@ func TestCurrentLeader(t *testing.T) {
 	elector, err := NewElector(client,
 		WithLeaderLeaseSeconds(1))
 	assert.Nil(t, err, "create elector failed")
+	defer elector.Stop(0)
 
 	value1 := newElectorTester(0, "1", elector)
 	value2 := newElectorTester(0, "2", elector)
 
 	value1.start()
 	value2.start()
 
+	defer value1.stop(0)
+	defer value2.stop(0)
+
 	data, err := elector.CurrentLeader(0)
 	assert.Nil(t, err, "get current leader failed")
 	assert.Equal(t, "1", string(data), "current leader failed")
 
 	elector.ChangeLeaderTo(0, "1", "2")
 	assert.Nil(t, err, "get current leader failed")
 
-	time.Sleep(time.Second*1 + time.Millisecond*200)
+	time.Sleep(time.Second + time.Millisecond*200)
 	data, err = elector.CurrentLeader(0)
 	assert.Nil(t, err, "get current leader failed")
 	assert.Equalf(t, "2", string(data), "current leader failed, %+v", value2.isLeader())
 }
 
 func TestStop(t *testing.T) {
-	stopC, port, err := startTestSingleEtcd()
-	assert.Nil(t, err, "start embed etcd failed")
+	stopC, port, err := startTestSingleEtcd(t)
+	if err != nil {
+		assert.FailNowf(t, "start embed etcd failed", "error: %+v", err)
+	}
 	defer close(stopC)
 
 	client, err := clientv3.New(clientv3.Config{
@@ -239,25 +269,31 @@ func TestStop(t *testing.T) {
 	assert.Nil(t, err, "create etcd client failed")
 	defer client.Close()
 
-	elector, err := NewElector(client,
+	e, err := NewElector(client,
 		WithLeaderLeaseSeconds(1))
 	assert.Nil(t, err, "create elector failed")
+	defer e.Stop(0)
 
-	value1 := newElectorTester(0, "1", elector)
-	value2 := newElectorTester(0, "2", elector)
+	value1 := newElectorTester(0, "1", e)
+	value2 := newElectorTester(0, "2", e)
 
 	value1.start()
 	value2.start()
 
-	elector.Stop(0)
+	defer value1.stop(0)
+	defer value2.stop(0)
 
-	time.Sleep(time.Second + time.Microsecond*200)
-	assert.False(t, value1.isLeader(), "value1 must be follower")
+	e.Stop(0)
+
+	assert.Equal(t, 0, len(e.(*elector).watchers), "watchers must be clear")
+	assert.Equal(t, 0, len(e.(*elector).leasors), "leasors must be clear")
 }
 
 func TestDoIfLeader(t *testing.T) {
-	stopC, port, err := startTestSingleEtcd()
-	assert.Nil(t, err, "start embed etcd failed")
+	stopC, port, err := startTestSingleEtcd(t)
+	if err != nil {
+		assert.FailNowf(t, "start embed etcd failed", "error: %+v", err)
+	}
 	defer close(stopC)
 
 	client, err := clientv3.New(clientv3.Config{
@@ -277,6 +313,9 @@ func TestDoIfLeader(t *testing.T) {
 	value1.start()
 	value2.start()
 
+	defer value1.stop(0)
+	defer value2.stop(0)
+
 	ok, err := elector.DoIfLeader(0, "1", nil, clientv3.OpPut("/key1", "value1"))
 	assert.Nil(t, err, "check do if leader failed")
 	assert.True(t, ok, "check do if leader failed")

go.mod

@@ -21,8 +21,6 @@ require (
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.9.6 // indirect
 	github.com/jonboulle/clockwork v0.1.0 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/prometheus/client_golang v1.1.0 // indirect
 	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
 	github.com/soheilhy/cmux v0.1.4 // indirect
@@ -36,7 +34,6 @@ require (
 	golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 // indirect
 	golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
-	google.golang.org/appengine v1.4.0 // indirect
 	google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 // indirect
 	google.golang.org/grpc v1.23.0 // indirect
 	sigs.k8s.io/yaml v1.1.0 // indirect

go.sum

@@ -166,6 +166,7 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

local_test.go

@@ -113,8 +113,10 @@ func TestRange(t *testing.T) {
 }
 
 func TestBootstrapCluster(t *testing.T) {
-	stopC, port, err := startTestSingleEtcd()
-	assert.Nil(t, err, "start embed etcd failed")
+	stopC, port, err := startTestSingleEtcd(t)
+	if err != nil {
+		assert.FailNowf(t, "start embed etcd failed", "error: %+v", err)
+	}
 	defer close(stopC)
 
 	client, err := clientv3.New(clientv3.Config{
@@ -128,7 +130,9 @@ func TestBootstrapCluster(t *testing.T) {
 	defer ctrl.Finish()
 
 	elector, _ := NewElector(client)
-	newElectorTester(math.MaxUint64, "c1", elector).start()
+	et := newElectorTester(math.MaxUint64, "c1", elector)
+	et.start()
+	defer et.stop(0)
 
 	rpc := newTestRPC(ctrl, 0)
 	pd := newTestProphet(ctrl, newEtcdStore(client, newTestAdapter(ctrl), "c1", elector), rpc, client, func() {})
@@ -171,8 +175,10 @@ func TestBootstrapCluster(t *testing.T) {
 }
 
 func TestMustPutResource(t *testing.T) {
-	stopC, port, err := startTestSingleEtcd()
-	assert.Nil(t, err, "start embed etcd failed")
+	stopC, port, err := startTestSingleEtcd(t)
+	if err != nil {
+		assert.FailNowf(t, "start embed etcd failed", "error: %+v", err)
+	}
 	defer close(stopC)
 
 	client, err := clientv3.New(clientv3.Config{
@@ -186,7 +192,9 @@ func TestMustPutResource(t *testing.T) {
 	defer ctrl.Finish()
 
 	elector, _ := NewElector(client)
-	newElectorTester(math.MaxUint64, "c1", elector).start()
+	et := newElectorTester(math.MaxUint64, "c1", elector)
+	et.start()
+	defer et.stop(0)
 
 	rpc := newTestRPC(ctrl, 0)
 	pd := newTestProphet(ctrl, newEtcdStore(client, newTestAdapter(ctrl), "c1", elector), rpc, client, func() {})
@@ -214,8 +222,10 @@ func TestMustPutResource(t *testing.T) {
 }
 
 func TestMustRemoveResource(t *testing.T) {
-	stopC, port, err := startTestSingleEtcd()
-	assert.Nil(t, err, "start embed etcd failed")
+	stopC, port, err := startTestSingleEtcd(t)
+	if err != nil {
+		assert.FailNowf(t, "start embed etcd failed", "error: %+v", err)
+	}
 	defer close(stopC)
 
 	client, err := clientv3.New(clientv3.Config{
@@ -229,7 +239,9 @@ func TestMustRemoveResource(t *testing.T) {
 	defer ctrl.Finish()
 
 	elector, _ := NewElector(client)
-	newElectorTester(math.MaxUint64, "c1", elector).start()
+	et := newElectorTester(math.MaxUint64, "c1", elector)
+	et.start()
+	defer et.stop(0)
 
 	rpc := newTestRPC(ctrl, 0)
 	pd := newTestProphet(ctrl, newEtcdStore(client, newTestAdapter(ctrl), "c1", elector), rpc, client, func() {})
@@ -253,9 +265,11 @@ func TestMustRemoveResource(t *testing.T) {
 	assert.Equal(t, 0, store.MustCountResources(), "remove resource failed")
 }
 
-func TestMustAllocIDe(t *testing.T) {
-	stopC, port, err := startTestSingleEtcd()
-	assert.Nil(t, err, "start embed etcd failed")
+func TestMustAllocID(t *testing.T) {
+	stopC, port, err := startTestSingleEtcd(t)
+	if err != nil {
+		assert.FailNowf(t, "start embed etcd failed", "error: %+v", err)
+	}
 	defer close(stopC)
 
 	client, err := clientv3.New(clientv3.Config{
@@ -269,7 +283,9 @@ func TestMustAllocIDe(t *testing.T) {
 	defer ctrl.Finish()
 
 	elector, _ := NewElector(client)
-	newElectorTester(math.MaxUint64, "c1", elector).start()
+	et := newElectorTester(math.MaxUint64, "c1", elector)
+	et.start()
+	defer et.stop(0)
 
 	rpc := newTestRPC(ctrl, 0)
 	pd := newTestProphet(ctrl, newEtcdStore(client, newTestAdapter(ctrl), "c1", elector), rpc, client, func() {})