Merge branch '3810-cordon-control'

Author: bboreham

examples/k8s/cluster-role.yaml

@@ -15,7 +15,6 @@ rules:
   - pods/log
   - replicationcontrollers
   - services
-  - nodes
   - namespaces
   - persistentvolumes
   - persistentvolumeclaims
@@ -95,3 +94,13 @@ rules:
   verbs:
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch

go.sum

@@ -1,9 +1,11 @@
 camlistore.org v0.0.0-20171230002226-a5a65f0d8b22 h1:VP9VuyosMHmS9zdzd5Co9TJKWPbMTfmtKc/XWctszyQ=
 camlistore.org v0.0.0-20171230002226-a5a65f0d8b22/go.mod h1:mzAP6ICVzPdfO0f3N9hAVWhO7qplHF7mbFhGsGdErTI=
+cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/DataDog/datadog-go v2.2.0+incompatible h1:V5BKkxACZLjzHjSgBbr2gvLA2Ae49yhc6CSY7MLy5k4=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
@@ -39,7 +41,9 @@ github.com/certifi/gocertifi v0.0.0-20150906030631-84c0a38a18fc h1:zSPFItDTOJZPd
 github.com/certifi/gocertifi v0.0.0-20150906030631-84c0a38a18fc/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible h1:C29Ae4G5GtYyYMm1aztcyj/J5ckgJm2zwdDajFbx1NY=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
+github.com/circonus-labs/circonusllhist v0.1.3 h1:TJH+oke8D16535+jHExHj4nQvzlZrj7ug5D7I/orNUA=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
@@ -97,6 +101,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4 h1:6o8aP0LGMKzo3NzwhhX6EJsiJ3ejmj+9yA/3p8Fjjlw=
 github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
+github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -134,6 +139,7 @@ github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxB
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs=
 github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4=
+github.com/hashicorp/go-retryablehttp v0.5.3 h1:QlWt0KvWT0lq8MFppF9tsJGF+ynG7ztc2KIPhzRGk7s=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
@@ -274,6 +280,7 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
 github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/tylerb/graceful v1.2.13 h1:yKdTh6eHcWdD8Jm3wxgJ6pNf8Lb3wwbV4Ip8fHbeMLE=
 github.com/tylerb/graceful v1.2.13/go.mod h1:LPYTbOYmUTdabwRt0TGhLllQ0MUNbs0Y5q1WXJOI9II=

probe/kubernetes/client.go

@@ -65,6 +65,10 @@ type Client interface {
 	DeleteVolumeSnapshot(namespaceID, volumeSnapshotID string) error
 	ScaleUp(namespaceID, id string) error
 	ScaleDown(namespaceID, id string) error
+	// Cordon or Uncordon a node based on whether `desired` is true or false respectively.
+	CordonNode(name string, desired bool) error
+	// Returns a list of kubernetes nodes.
+	GetNodes() ([]apiv1.Node, error)
 }
 
 // ResourceMap is the mapping of resource and their GroupKind
@@ -624,3 +628,33 @@ func (c *client) modifyScale(namespaceID, id string, f func(*autoscalingv1.Scale
 func (c *client) Stop() {
 	close(c.quit)
 }
+
+func (c *client) CordonNode(name string, desired bool) error {
+	node, err := c.client.CoreV1().Nodes().Get(name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	helper := newCordonHelper(node)
+	if updateRequired := helper.updateIfRequired(desired); !updateRequired {
+		return nil
+	}
+
+	err, patchErr := helper.patchOrReplace(c.client, false)
+	if patchErr != nil {
+		return patchErr
+	}
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *client) GetNodes() ([]apiv1.Node, error) {
+	l, err := c.client.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return l.Items, nil
+}

probe/kubernetes/controls.go

@@ -21,6 +21,8 @@ const (
 	DeleteVolumeSnapshot = report.KubernetesDeleteVolumeSnapshot
 	ScaleUp              = report.KubernetesScaleUp
 	ScaleDown            = report.KubernetesScaleDown
+	CordonNode           = report.KubernetesCordonNode
+	UncordonNode         = report.KubernetesUncordonNode
 )
 
 // GroupName and version used by CRDs
@@ -474,6 +476,17 @@ func (r *Reporter) CaptureJob(f func(xfer.Request, string, string) xfer.Response
 	}
 }
 
+// CaptureNode is exported for testing
+func (r *Reporter) CaptureNode(f func(xfer.Request, string) xfer.Response) func(xfer.Request) xfer.Response {
+	return func(req xfer.Request) xfer.Response {
+		nodeID, ok := report.ParseHostNodeID(req.NodeID)
+		if !ok {
+			return xfer.ResponseErrorf("Invalid ID: %s", req.NodeID)
+		}
+		return f(req, nodeID)
+	}
+}
+
 // ScaleUp is the control to scale up a deployment
 func (r *Reporter) ScaleUp(req xfer.Request, namespace, id string) xfer.Response {
 	return xfer.ResponseError(r.client.ScaleUp(namespace, id))
@@ -484,6 +497,16 @@ func (r *Reporter) ScaleDown(req xfer.Request, namespace, id string) xfer.Respon
 	return xfer.ResponseError(r.client.ScaleDown(namespace, id))
 }
 
+// CordonNode is the control to cordon a node.
+func (r *Reporter) CordonNode(req xfer.Request, name string) xfer.Response {
+	return xfer.ResponseError(r.client.CordonNode(name, true))
+}
+
+// UncordonNode is the control to un-cordon a node.
+func (r *Reporter) UncordonNode(req xfer.Request, name string) xfer.Response {
+	return xfer.ResponseError(r.client.CordonNode(name, false))
+}
+
 func (r *Reporter) registerControls() {
 	controls := map[string]xfer.ControlHandlerFunc{
 		CloneVolumeSnapshot:  r.CaptureVolumeSnapshot(r.cloneVolumeSnapshot),
@@ -494,6 +517,8 @@ func (r *Reporter) registerControls() {
 		DeleteVolumeSnapshot: r.CaptureVolumeSnapshot(r.deleteVolumeSnapshot),
 		ScaleUp:              r.CaptureDeployment(r.ScaleUp),
 		ScaleDown:            r.CaptureDeployment(r.ScaleDown),
+		CordonNode:           r.CaptureNode(r.CordonNode),
+		UncordonNode:         r.CaptureNode(r.UncordonNode),
 	}
 	r.handlerRegistry.Batch(nil, controls)
 }
@@ -508,6 +533,8 @@ func (r *Reporter) deregisterControls() {
 		DeleteVolumeSnapshot,
 		ScaleUp,
 		ScaleDown,
+		CordonNode,
+		UncordonNode,
 	}
 	r.handlerRegistry.Batch(controls, nil)
 }

probe/kubernetes/cordon.go

@@ -0,0 +1,87 @@
+/*
+Copied from
+
+	https://github.com/kubernetes/kubectl/blob/master/pkg/drain/cordon.go
+
+at commit f9460c53339c4bb60b20031e3c6125e8bac679e2 to add node cordon feature.
+*/
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubernetes
+
+import (
+	"encoding/json"
+
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+	"k8s.io/client-go/kubernetes"
+)
+
+// CordonHelper wraps functionality to cordon/uncordon nodes
+type cordonHelper struct {
+	node    *apiv1.Node
+	desired bool
+}
+
+// NewCordonHelper returns a new CordonHelper
+func newCordonHelper(node *apiv1.Node) *cordonHelper {
+	return &cordonHelper{
+		node: node,
+	}
+}
+
+// UpdateIfRequired returns true if c.node.Spec.Unschedulable isn't already set,
+// or false when no change is needed
+func (c *cordonHelper) updateIfRequired(desired bool) bool {
+	c.desired = desired
+
+	return c.node.Spec.Unschedulable != c.desired
+}
+
+// PatchOrReplace uses given clientset to update the node status, either by patching or
+// updating the given node object; it may return error if the object cannot be encoded as
+// JSON, or if either patch or update calls fail; it will also return a second error
+// whenever creating a patch has failed
+func (c *cordonHelper) patchOrReplace(clientset kubernetes.Interface, serverDryRun bool) (error, error) {
+	client := clientset.CoreV1().Nodes()
+
+	oldData, err := json.Marshal(c.node)
+	if err != nil {
+		return err, nil
+	}
+
+	c.node.Spec.Unschedulable = c.desired
+
+	newData, err := json.Marshal(c.node)
+	if err != nil {
+		return err, nil
+	}
+
+	patchBytes, patchErr := strategicpatch.CreateTwoWayMergePatch(oldData, newData, c.node)
+	if patchErr == nil {
+		_, err = client.Patch(c.node.Name, types.StrategicMergePatchType, patchBytes)
+	} else {
+		updateOptions := metav1.UpdateOptions{}
+		if serverDryRun {
+			updateOptions.DryRun = []string{metav1.DryRunAll}
+		}
+		_, err = client.Update(c.node)
+	}
+	return err, patchErr
+}

probe/kubernetes/reporter.go

@@ -181,6 +181,21 @@ var (
 		Icon:  "fa fa-file-text",
 		Rank:  2,
 	}
+
+	CordonControl = []report.Control{
+		{
+			ID:    CordonNode,
+			Human: "Cordon",
+			Icon:  "fa fa-toggle-off",
+			Rank:  1,
+		},
+		{
+			ID:    UncordonNode,
+			Human: "Uncordon",
+			Icon:  "fa fa-toggle-on",
+			Rank:  0,
+		},
+	}
 )
 
 // Reporter generate Reports containing Container and ContainerImage topologies
@@ -345,6 +360,10 @@ func (r *Reporter) Report() (report.Report, error) {
 	if err != nil {
 		return result, err
 	}
+	hostTopology, err := r.hostTopology()
+	if err != nil {
+		return result, err
+	}
 
 	result.Pod = result.Pod.Merge(podTopology)
 	result.Service = result.Service.Merge(serviceTopology)
@@ -359,6 +378,8 @@ func (r *Reporter) Report() (report.Report, error) {
 	result.VolumeSnapshot = result.VolumeSnapshot.Merge(volumeSnapshotTopology)
 	result.VolumeSnapshotData = result.VolumeSnapshotData.Merge(volumeSnapshotDataTopology)
 	result.Job = result.Job.Merge(jobTopology)
+	result.Host = result.Host.Merge(hostTopology)
+
 	return result, nil
 }
 
@@ -678,3 +699,32 @@ func (r *Reporter) namespaceTopology() (report.Topology, error) {
 	})
 	return result, err
 }
+
+func (r *Reporter) hostTopology() (report.Topology, error) {
+	result := report.MakeTopology()
+	// Add buttons for Host view, with the ID of the Kubernetes probe
+	for _, control := range CordonControl {
+		control.ProbeID = r.probeID
+		result.Controls.AddControl(control)
+	}
+
+	nodes, err := r.client.GetNodes()
+	if err != nil {
+		return result, err
+	}
+
+	for _, n := range nodes {
+		var activeControl string
+		if n.Spec.Unschedulable {
+			activeControl = UncordonNode
+		} else {
+			activeControl = CordonNode
+		}
+		result.AddNode(
+			report.MakeNode(report.MakeHostNodeID(n.Name)).
+				WithTopology(report.Host).
+				WithLatestActiveControls(activeControl),
+		)
+	}
+	return result, nil
+}

probe/kubernetes/reporter_test.go

@@ -205,6 +205,14 @@ func (c *mockClient) Describe(namespaceID, resourceID string, groupKind schema.G
 	return nil, nil
 }
 
+func (c *mockClient) CordonNode(name string, desired bool) error {
+	return nil
+}
+
+func (c *mockClient) GetNodes() ([]apiv1.Node, error) {
+	return nil, nil
+}
+
 type mockPipeClient map[string]xfer.Pipe
 
 func (c mockPipeClient) PipeConnection(appID, id string, pipe xfer.Pipe) error {

render/detailed/node.go

@@ -113,6 +113,9 @@ func controlsFor(topology report.Topology, nodeID string) []ControlInstance {
 	}
 	for _, controlID := range node.ActiveControls() {
 		if control, ok := topology.Controls[controlID]; ok {
+			if control.ProbeID != "" { // does this Control have an override for the node probe?
+				probeID = control.ProbeID
+			}
 			result = append(result, ControlInstance{
 				ProbeID: probeID,
 				NodeID:  nodeID,

report/controls.go

@@ -10,6 +10,7 @@ type Control struct {
 	Icon         string `json:"icon"` // from https://fortawesome.github.io/Font-Awesome/cheatsheet/ please
 	Confirmation string `json:"confirmation,omitempty"`
 	Rank         int    `json:"rank"`
+	ProbeID      string `json:"probeId,omitempty"`
 }
 
 // Merge merges other with cs, returning a fresh Controls.

report/map_keys.go

@@ -95,6 +95,8 @@ const (
 	KubernetesCloneVolumeSnapshot  = "kubernetes_clone_volume_snapshot"
 	KubernetesDeleteVolumeSnapshot = "kubernetes_delete_volume_snapshot"
 	KubernetesDescribe             = "kubernetes_describe"
+	KubernetesCordonNode           = "kubernetes_cordon_node"
+	KubernetesUncordonNode         = "kubernetes_uncordon_node"
 	// probe/awsecs
 	ECSCluster             = "ecs_cluster"
 	ECSCreatedAt           = "ecs_created_at"