Add quantiles to request metrics, correct metric postfix

...the new summary collector will expose the metrics with three names:
http_server_requests_seconds(_sum|_count)?, making the former counter
metric redundant and making the metric names more consistent with
conventions. Leaving in a "legacy" metric collector to avoid breaking existing monitoring
and alerting setups that depend on the metric with postfix "_sum_sum".

Author: Zeidan, Omar

web/middleware/requestmetrics/requestmetrics.go

@@ -5,37 +5,40 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/prometheus/client_golang/prometheus"
+
 	"net/http"
 	"strings"
 	"time"
 )
 
 var (
-	RequestCounterName  = "http_server_requests_seconds_count"
-	RequestDurationName = "http_server_requests_seconds_sum"
+	requestSummaryMetricName       = "http_server_requests_seconds"
+	requestSummaryLegacyMetricName = "http_server_requests_seconds_sum"
 
-	reqs    *prometheus.CounterVec
-	latency *prometheus.SummaryVec
+	reqs                       *prometheus.CounterVec
+	requestSummaryMetric       *prometheus.SummaryVec
+	requestSummaryLegacyMetric *prometheus.SummaryVec
 )
 
 func Setup() {
-	reqs = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: RequestCounterName,
-			Help: "Number of incoming HTTP requests processed, partitioned by status code, method and HTTP path (grouped by patterns).",
+	requestSummaryMetric = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       requestSummaryMetricName,
+			Help:       "Request counts, durations, accumulated and in quantiles, partitioned by method, \"outcome\", status code, and HTTP path (grouped by patterns).",
+			Objectives: map[float64]float64{0.5: 0.05, 0.95: 0.01, 0.99: 0.001},
 		},
 		[]string{"method", "outcome", "status", "uri"},
 	)
-	prometheus.MustRegister(reqs)
 
-	latency = prometheus.NewSummaryVec(
+	requestSummaryLegacyMetric = prometheus.NewSummaryVec(
 		prometheus.SummaryOpts{
-			Name: RequestDurationName,
-			Help: "How long it took to process requests, partitioned by status code, method and HTTP path (grouped by patterns).",
+			Name: requestSummaryLegacyMetricName,
+			Help: "(Legacy, replaced by \"http_server_requests_seconds(_sum|_count)?\") Accumulated request durations and counts partitioned by method, \"outcome\", status code, and HTTP path (grouped by patterns).",
 		},
 		[]string{"method", "outcome", "status", "uri"},
 	)
-	prometheus.MustRegister(latency)
+
+	prometheus.MustRegister(requestSummaryMetric)
 }
 
 func RecordRequestMetrics(next http.Handler) http.Handler {
@@ -49,7 +52,8 @@ func RecordRequestMetrics(next http.Handler) http.Handler {
 		routePattern = strings.Replace(routePattern, "/*/", "/", -1)
 
 		reqs.WithLabelValues(r.Method, outcome(ww.Status()), fmt.Sprintf("%d", ww.Status()), routePattern).Inc()
-		latency.WithLabelValues(r.Method, outcome(ww.Status()), fmt.Sprintf("%d", ww.Status()), routePattern).Observe(float64(time.Since(start).Microseconds()) / 1000000)
+		requestSummaryMetric.WithLabelValues(r.Method, outcome(ww.Status()), fmt.Sprintf("%d", ww.Status()), routePattern).Observe(float64(time.Since(start).Microseconds()) / 1000000)
+		requestSummaryLegacyMetric.WithLabelValues(r.Method, outcome(ww.Status()), fmt.Sprintf("%d", ww.Status()), routePattern).Observe(float64(time.Since(start).Microseconds()) / 1000000)
 	}
 	return http.HandlerFunc(fn)
 }