Upstream of updates and misc improvements

Readme.md

@@ -20,7 +20,7 @@ Based on the blog post and code from [Footballradar](https://engineering.footbal
 Include it in your project by adding the following to your build.sbt:
 
 ```scala
-libraryDependencies += "com.github.kovszilard" %% "twitter-server-prometheus" % "19.10.0"
+libraryDependencies += "com.github.kovszilard" %% "twitter-server-prometheus" % "20.10.0"
 ```
 
 Once you have the SBT dependency, you can mix in the `PrometheusExporter` trait to your App.
@@ -30,3 +30,13 @@ object MyApp extends TwitterServer with PrometheusExporter {
   // ...
 }
 ```
+
+## Example
+
+See Example.scala
+
+and run it with:
+
+```
+sbt example/runMain com.github.kovszilard.twitter.server.prometheus.Example
+```

build.sbt

@@ -1,6 +1,6 @@
 import Dependencies._
 
-ThisBuild / scalaVersion     := "2.12.10"
+ThisBuild / scalaVersion     := "2.13.10"
 ThisBuild / version          := twitterVersion
 ThisBuild / organization     := "com.github.kovszilard"
 ThisBuild / organizationName := "kovszilard"
@@ -18,7 +18,7 @@ lazy val root = (project in file("."))
 lazy val twitterServerPrometheus = (project in file("twitter-server-prometheus"))
     .settings(
         name := "twitter-server-prometheus",
-        crossScalaVersions := Seq("2.12.10", "2.11.12"),
+        crossScalaVersions := Seq("2.13.10", "2.12.10", "2.11.12"),
         libraryDependencies ++= Seq(
             twitterServer,
             finagleStats,
@@ -30,7 +30,7 @@ lazy val twitterServerPrometheus = (project in file("twitter-server-prometheus")
 lazy val example = (project in file("example"))
     .settings(
         name := "example",
-        crossScalaVersions := Seq("2.12.10", "2.11.12"),
+        crossScalaVersions := Seq("2.13.10", "2.12.10", "2.11.12"),
         libraryDependencies ++= Seq(
             twitterServerLogback,
             logback

end-to-end-test.sh

@@ -9,7 +9,7 @@ SCALA_VERSION=$1
 # If SCALA_VERSION not provided
 if [ -z "$SCALA_VERSION" ]
 then
-  SCALA_VERSION=2.12.10
+  SCALA_VERSION=2.13.10
 fi
 
 sbt "project example" "++$SCALA_VERSION run" &

example/src/main/scala/Example.scala

@@ -9,7 +9,28 @@ import com.twitter.util.{Await, Future}
 object Example extends TwitterServer with PrometheusExporter {
 
   val receiver = LoadedStatsReceiver.scope("prometheus_demo")
-  val requests = receiver.counter("http_requests")
+  override lazy val metricsCodec: PrometheusMetricsCodec = new PrometheusMetricsCodec {
+    override def fromMetricName(metricName: String): (String, List[(String, String)]) = {
+      val name :: params = metricName.split('?').toList
+      val labels = params.map(_.split('&').flatMap(_.split("=").toList).toList).flatMap {
+        case Nil =>
+          None
+        case name :: Nil =>
+          Some(name -> "")
+        case name :: value :: _ =>
+          Some(name -> value)
+      }
+      (name, labels)
+    }
+    override def toMetricName(name: String, metadata: List[(String, String)]): String = {
+      val params: List[String] = metadata.map { case (name, value) => s"$name=$value" }
+      name + params.mkString("?", "&", "")
+    }
+  }
+
+
+  val requests = receiver.counter(metricsCodec.toMetricName("http_requests", List("id" -> "4")))
+
 
   val helloWorldService = new Service[Request, Response] {
     def apply(request: Request): Future[Response] = {

project/Dependencies.scala

@@ -1,15 +1,15 @@
 import sbt._
 
 object Dependencies {
-  val twitterVersion = "19.10.0"
+  val twitterVersion = "22.12.0"
 
   lazy val twitterServer = "com.twitter" %% "twitter-server" % twitterVersion
   lazy val finagleStats = "com.twitter" %% "finagle-stats" % twitterVersion
-  lazy val prometheusSimpleClient = "io.prometheus" % "simpleclient" % "0.8.0"
-  lazy val prometheusSimpleClientCommon = "io.prometheus" % "simpleclient_common" % "0.8.0"
+  lazy val prometheusSimpleClient = "io.prometheus" % "simpleclient" % "0.16.0"
+  lazy val prometheusSimpleClientCommon = "io.prometheus" % "simpleclient_common" % "0.16.0"
 
   lazy val twitterServerLogback = "com.twitter" %% "twitter-server-logback-classic" % twitterVersion
-  lazy val logback = "ch.qos.logback" % "logback-classic" % "1.2.3"
+  lazy val logback = "ch.qos.logback" % "logback-classic" % "1.4.5"
 
-  lazy val scalaTest = "org.scalatest" %% "scalatest" % "3.0.5"
+  lazy val scalaTest = "org.scalatest" %% "scalatest" % "3.2.14"
 }

project/build.properties

@@ -1 +1 @@
-sbt.version = 1.3.3
+sbt.version = 1.8.0

twitter-server-prometheus/src/main/scala/PrometheusExporter.scala

@@ -1,14 +1,17 @@
 package com.github.kovszilard.twitter.server.prometheus
 
 import com.twitter.app.App
-import com.twitter.finagle.stats.PrometheusMetricsCollector
+import com.twitter.finagle.stats.{MetricsStatsReceiver, PrometheusMetricsCollector}
 import com.twitter.server.Admin.Grouping
 import com.twitter.server.AdminHttpServer.Route
 import com.twitter.server.{AdminHttpServer, Stats}
 
 trait PrometheusExporter { self: App with AdminHttpServer with Stats =>
 
-  PrometheusMetricsCollector().register()
+  lazy val metricsCodec: PrometheusMetricsCodec = new PrometheusMetricsCodec {}
+  lazy val metricsRegistry = MetricsStatsReceiver.defaultRegistry
+
+  PrometheusMetricsCollector(metricsCodec, metricsRegistry).register()
 
   val metricsRoute: Route = Route.isolate(Route(
       path = "/metrics",

twitter-server-prometheus/src/main/scala/PrometheusMetricsCodec.scala

@@ -0,0 +1,6 @@
+package com.github.kovszilard.twitter.server.prometheus
+
+trait PrometheusMetricsCodec {
+  def fromMetricName(metricName: String): (String, List[(String, String)]) = (metricName, List.empty)
+  def toMetricName(name: String, metadata: List[(String, String)]): String = name
+}

twitter-server-prometheus/src/main/scala/PrometheusMetricsCollector.scala

@@ -1,58 +1,82 @@
 package com.twitter.finagle.stats
 
+import com.github.kovszilard.twitter.server.prometheus.PrometheusMetricsCodec
 import io.prometheus.client.Collector
 import io.prometheus.client.Collector.MetricFamilySamples.Sample
 import io.prometheus.client.Collector._
 
-import scala.collection.JavaConversions._
+import scala.jdk.CollectionConverters._
+import scala.util.Try
 
 
-class PrometheusMetricsCollector(registry: MetricsView = MetricsStatsReceiver.defaultRegistry) extends Collector {
+case class PrometheusMetricsCollector(codec: PrometheusMetricsCodec, registry: Metrics) extends Collector {
+
+  implicit def listConverter[A](list: List[A]): java.util.List[A] = list.asJava
+  implicit def mapConverter[K, V](map: java.util.Map[K, V]): scala.collection.mutable.Map[K, V] = map.asScala
 
   override def collect(): java.util.List[MetricFamilySamples] = {
-    val gauges = registry.gauges.map{ case (name: String, value: Number) => fromGauge(name, value)}
-    val counters = registry.counters.map{ case (name: String, value: Number) => fromCounter(name, value)}
-    val histograms = registry.histograms.map{ case (name: String, value: Snapshot) => fromHistogram(name, value)}
+    val gauges = registry.gauges.map { gaugeSnapshot =>
+      fromGauge(gaugeSnapshot.hierarchicalName, gaugeSnapshot.value)
+    }
+    val counters = registry.counters.map { counterSnapshot =>
+      fromCounter(counterSnapshot.hierarchicalName, counterSnapshot.value)
+    }
+    val histograms = registry.histograms.map{ histogramSnapshot =>
+      fromHistogram(histogramSnapshot.hierarchicalName, histogramSnapshot.value)
+    }
+
     (gauges ++ counters ++ histograms).toList
   }
 
+  def extractAndSanitizeMetricAndLabels(rawName: String): (String, List[(String, String)]) = {
+    val (name, labels) = Try {
+      codec.fromMetricName(rawName)
+    }.getOrElse{
+      // TODO Fix potential high cardinality if dynamic non-envodable values end up as the rawName
+      rawName -> List("finagleName" -> rawName)
+    }
+    (sanitizeMetricName(name), labels)
+  }
+
   def fromGauge(name: String, value: Number): MetricFamilySamples = {
-    val prometheusName = sanitizeMetricName(name)
+    val (metric, labels) = extractAndSanitizeMetricAndLabels(name)
 
     new MetricFamilySamples(
-      prometheusName,
+      metric,
       Type.GAUGE,
       genHelpMessage(name, "gauge"),
-      List(new Sample(prometheusName, List("finagleName"), List(name), value.doubleValue()))
+      List(new Sample(metric, labels.map(_._1), labels.map(_._2), value.doubleValue()))
     )
   }
 
   def fromCounter(name: String, value: Number): MetricFamilySamples = {
-    val prometheusName = sanitizeMetricName(name)
+    val (metric, labels) = extractAndSanitizeMetricAndLabels(name)
 
     new MetricFamilySamples(
-      prometheusName,
+      metric,
       Type.COUNTER,
       genHelpMessage(name, "counter"),
-      List(new Sample(prometheusName, List("finagleName"), List(name), value.doubleValue()))
+      List(new Sample(metric, labels.map(_._1), labels.map(_._2), value.doubleValue()))
     )
   }
 
   def fromHistogram(name: String, snapshot: Snapshot): MetricFamilySamples = {
-    val prometheusName = sanitizeMetricName(name)
-
-    val count = new Sample(s"${prometheusName}_count", Nil, Nil, snapshot.count.toDouble)
-    val sum = new Sample(s"${prometheusName}_sum", Nil, Nil, snapshot.sum.toDouble)
-    val max = new Sample(s"${prometheusName}_max", Nil, Nil, snapshot.max.toDouble)
-    val min = new Sample(s"${prometheusName}_min", Nil, Nil, snapshot.min.toDouble)
-    val avg = new Sample(s"${prometheusName}_avg", Nil, Nil, snapshot.average)
-
-    val percentiles = snapshot.percentiles.map{percentile =>
-      new Sample(prometheusName, List("quantile"), List(percentile.quantile.toString), percentile.value.toDouble)
+    val (metric, labels) = extractAndSanitizeMetricAndLabels(name)
+    val labelNames = labels.map(_._1)
+    val labelValues = labels.map(_._2)
+
+    val count = new Sample(s"${metric}_count", labelNames, labelValues, snapshot.count.toDouble)
+    val sum = new Sample(s"${metric}_sum", labelNames, labelValues, snapshot.sum.toDouble)
+    val max = new Sample(s"${metric}_max", labelNames, labelValues, snapshot.max.toDouble)
+    val min = new Sample(s"${metric}_min", labelNames, labelValues, snapshot.min.toDouble)
+    val avg = new Sample(s"${metric}_avg", labelNames, labelValues, snapshot.average)
+
+    val percentiles = snapshot.percentiles.map { percentile =>
+      new Sample(metric, labelNames :+ "quantile", labelValues :+ percentile.quantile.toString, percentile.value.toDouble)
     }
 
     new MetricFamilySamples(
-      prometheusName,
+      metric,
       Type.SUMMARY,
       genHelpMessage(name, "histogram"),
       List(count, sum, max, min, avg) ++ percentiles
@@ -64,8 +88,3 @@ class PrometheusMetricsCollector(registry: MetricsView = MetricsStatsReceiver.de
   }
 
 }
-
-object PrometheusMetricsCollector {
-  def apply() = new PrometheusMetricsCollector()
-  def apply(registry: Metrics) = new PrometheusMetricsCollector(registry)
-}