Merge pull request #1952 from hajiler/mount-error-metrics-branch

Introduce metrics for pdcsi node

Author: k8s-ci-robot

cmd/gce-pd-csi-driver/main.go

@@ -151,13 +151,22 @@ func handle() {
 	}
 
 	var metricsManager *metrics.MetricsManager = nil
-	if *runControllerService && *httpEndpoint != "" {
+	runServiceWithMetrics := *runControllerService || *runNodeService
+	if runServiceWithMetrics && *httpEndpoint != "" {
 		mm := metrics.NewMetricsManager()
 		mm.InitializeHttpHandler(*httpEndpoint, *metricsPath)
-		mm.RegisterPDCSIMetric()
 
-		if metrics.IsGKEComponentVersionAvailable() {
-			mm.EmitGKEComponentVersion()
+		switch {
+		case *runControllerService:
+			mm.RegisterPDCSIMetric()
+			if metrics.IsGKEComponentVersionAvailable() {
+				mm.EmitGKEComponentVersion()
+			}
+		case *runNodeService:
+			if err := mm.EmmitProcessStartTime(); err != nil {
+				klog.Errorf("Failed to emit process start time: %v", err.Error())
+			}
+			mm.RegisterMountMetric()
 		}
 		metricsManager = &mm
 	}
@@ -266,6 +275,7 @@ func handle() {
 			EnableDataCache:          *enableDataCacheFlag,
 			DataCacheEnabledNodePool: isDataCacheEnabledNodePool,
 			SysfsPath:                "/sys",
+			MetricsManager:           metricsManager,
 		}
 		nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter, nsArgs)
 

pkg/gce-pd-csi-driver/gce-pd-driver.go

@@ -158,6 +158,7 @@ func NewNodeServer(gceDriver *GCEDriver, mounter *mount.SafeFormatAndMount, devi
 		EnableDataCache:          args.EnableDataCache,
 		DataCacheEnabledNodePool: args.DataCacheEnabledNodePool,
 		SysfsPath:                args.SysfsPath,
+		metricsManager:           args.MetricsManager,
 	}
 }
 

pkg/gce-pd-csi-driver/node.go

@@ -40,6 +40,7 @@ import (
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils"
 	metadataservice "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/metadata"
+	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/metrics"
 	mountmanager "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/mount-manager"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/resizefs"
 )
@@ -77,6 +78,8 @@ type GCENodeServer struct {
 	// Embed UnimplementedNodeServer to ensure the driver returns Unimplemented for any
 	// new RPC methods that might be introduced in future versions of the spec.
 	csi.UnimplementedNodeServer
+
+	metricsManager *metrics.MetricsManager
 }
 
 type NodeServerArgs struct {
@@ -92,6 +95,8 @@ type NodeServerArgs struct {
 
 	// SysfsPath defaults to "/sys", except if it's a unit test.
 	SysfsPath string
+
+	MetricsManager *metrics.MetricsManager
 }
 
 var _ csi.NodeServer = &GCENodeServer{}
@@ -442,6 +447,7 @@ func (ns *GCENodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStage
 				return &csi.NodeStageVolumeResponse{}, nil
 			}
 		}
+
 		return nil, status.Error(codes.Internal,
 			fmt.Sprintf("Failed to format and mount device from (%q) to (%q) with fstype (%q) and options (%q): %v",
 				devicePath, stagingTargetPath, fstype, options, err.Error()))

pkg/gce-pd-csi-driver/utils_linux.go

@@ -75,7 +75,12 @@ func (ns *GCENodeServer) formatAndMount(source, target, fstype string, options [
 			}
 		}()
 	}
-	return m.FormatAndMount(source, target, fstype, options)
+
+	err := m.FormatAndMount(source, target, fstype, options)
+	if ns.metricsManager != nil {
+		ns.metricsManager.RecordMountErrorMetric(fstype, err)
+	}
+	return err
 }
 
 func preparePublishPath(path string, m *mount.SafeFormatAndMount) error {

pkg/metrics/metrics.go

@@ -25,6 +25,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"k8s.io/component-base/metrics"
 	"k8s.io/klog/v2"
+	"k8s.io/mount-utils"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
 )
 
@@ -53,6 +54,15 @@ var (
 			StabilityLevel: metrics.ALPHA,
 		},
 		[]string{"driver_name", "method_name", "grpc_status_code", "disk_type", "enable_confidential_storage", "enable_storage_pools"})
+
+	mountErrorMetric = metrics.NewCounterVec(&metrics.CounterOpts{
+		Subsystem:      "node",
+		Name:           "mount_errors",
+		Help:           "Node server file system mounting errors",
+		StabilityLevel: metrics.ALPHA,
+	},
+		[]string{"driver_name", "file_system_format", "error_type"},
+	)
 )
 
 type MetricsManager struct {
@@ -78,6 +88,10 @@ func (mm *MetricsManager) RegisterPDCSIMetric() {
 	mm.registry.MustRegister(pdcsiOperationErrorsMetric)
 }
 
+func (mm *MetricsManager) RegisterMountMetric() {
+	mm.registry.MustRegister(mountErrorMetric)
+}
+
 func (mm *MetricsManager) recordComponentVersionMetric() error {
 	v := getEnvVar(envGKEPDCSIVersion)
 	if v == "" {
@@ -101,6 +115,16 @@ func (mm *MetricsManager) RecordOperationErrorMetrics(
 	klog.Infof("Recorded PDCSI operation error code: %q", errCode)
 }
 
+func (mm *MetricsManager) RecordMountErrorMetric(fs_format string, err error) {
+	errType := mountErrorType(err)
+	mountErrorMetric.WithLabelValues(pdcsiDriverName, fs_format, errType).Inc()
+	klog.Infof("Recorded mount error type: %q", errType)
+}
+
+func (mm *MetricsManager) EmmitProcessStartTime() error {
+	return metrics.RegisterProcessStartTime(mm.registry.Register)
+}
+
 func (mm *MetricsManager) EmitGKEComponentVersion() error {
 	mm.registerComponentVersionMetric()
 	if err := mm.recordComponentVersionMetric(); err != nil {
@@ -169,3 +193,16 @@ func errorCodeLabelValue(operationErr error) string {
 	}
 	return err
 }
+
+func mountErrorType(err error) string {
+	if err == nil {
+		return "OK"
+	}
+
+	mntErr := &mount.MountError{}
+	if !errors.As(err, mntErr) {
+		return "UnknownError"
+	}
+
+	return string(mntErr.Type)
+}

pkg/metrics/metrics_test.go

@@ -29,6 +29,7 @@ import (
 	"google.golang.org/api/googleapi"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"k8s.io/mount-utils"
 
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
 	gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute"
@@ -186,3 +187,35 @@ func TestErrorCodeLabelValue(t *testing.T) {
 		}
 	}
 }
+
+func TestMountOperationError(t *testing.T) {
+	testCases := []struct {
+		name string
+		err  error
+		want string
+	}{
+		{
+			name: "no error",
+			want: "OK",
+		},
+		{
+			name: "unknown error",
+			err:  fmt.Errorf("fake error"),
+			want: "UnknownError",
+		},
+		{
+			name: "mount error",
+			err:  mount.NewMountError(mount.FormatFailed, "file system format failed"),
+			want: string(mount.FormatFailed),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := mountErrorType(tc.err)
+			if diff := cmp.Diff(tc.want, got); diff != "" {
+				t.Errorf("%s: -want err, +got err\n%s", tc.name, diff)
+			}
+		})
+	}
+}