diff --git a/Readme.md b/Readme.md index 4e29fb0..e3e4922 100644 --- a/Readme.md +++ b/Readme.md @@ -20,7 +20,7 @@ Based on the blog post and code from [Footballradar](https://engineering.footbal Include it in your project by adding the following to your build.sbt: ```scala -libraryDependencies += "com.github.kovszilard" %% "twitter-server-prometheus" % "19.10.0" +libraryDependencies += "com.github.kovszilard" %% "twitter-server-prometheus" % "20.10.0" ``` Once you have the SBT dependency, you can mix in the `PrometheusExporter` trait to your App. @@ -30,3 +30,13 @@ object MyApp extends TwitterServer with PrometheusExporter { // ... } ``` + +## Example + +See Example.scala + +and run it with: + +``` +sbt example/runMain com.github.kovszilard.twitter.server.prometheus.Example +``` diff --git a/build.sbt b/build.sbt index bc293f6..df58d81 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ import Dependencies._ -ThisBuild / scalaVersion := "2.12.10" +ThisBuild / scalaVersion := "2.13.10" ThisBuild / version := twitterVersion ThisBuild / organization := "com.github.kovszilard" ThisBuild / organizationName := "kovszilard" @@ -18,7 +18,7 @@ lazy val root = (project in file(".")) lazy val twitterServerPrometheus = (project in file("twitter-server-prometheus")) .settings( name := "twitter-server-prometheus", - crossScalaVersions := Seq("2.12.10", "2.11.12"), + crossScalaVersions := Seq("2.13.10", "2.12.10", "2.11.12"), libraryDependencies ++= Seq( twitterServer, finagleStats, @@ -30,7 +30,7 @@ lazy val twitterServerPrometheus = (project in file("twitter-server-prometheus") lazy val example = (project in file("example")) .settings( name := "example", - crossScalaVersions := Seq("2.12.10", "2.11.12"), + crossScalaVersions := Seq("2.13.10", "2.12.10", "2.11.12"), libraryDependencies ++= Seq( twitterServerLogback, logback diff --git a/end-to-end-test.sh b/end-to-end-test.sh index 2426ba5..218c1d8 100755 --- a/end-to-end-test.sh +++ b/end-to-end-test.sh @@ -9,7 +9,7 @@ SCALA_VERSION=$1 # If SCALA_VERSION not provided if [ -z "$SCALA_VERSION" ] then - SCALA_VERSION=2.12.10 + SCALA_VERSION=2.13.10 fi sbt "project example" "++$SCALA_VERSION run" & diff --git a/example/src/main/scala/Example.scala b/example/src/main/scala/Example.scala index 623b8df..a1ece79 100644 --- a/example/src/main/scala/Example.scala +++ b/example/src/main/scala/Example.scala @@ -9,7 +9,28 @@ import com.twitter.util.{Await, Future} object Example extends TwitterServer with PrometheusExporter { val receiver = LoadedStatsReceiver.scope("prometheus_demo") - val requests = receiver.counter("http_requests") + override lazy val metricsCodec: PrometheusMetricsCodec = new PrometheusMetricsCodec { + override def fromMetricName(metricName: String): (String, List[(String, String)]) = { + val name :: params = metricName.split('?').toList + val labels = params.map(_.split('&').flatMap(_.split("=").toList).toList).flatMap { + case Nil => + None + case name :: Nil => + Some(name -> "") + case name :: value :: _ => + Some(name -> value) + } + (name, labels) + } + override def toMetricName(name: String, metadata: List[(String, String)]): String = { + val params: List[String] = metadata.map { case (name, value) => s"$name=$value" } + name + params.mkString("?", "&", "") + } + } + + + val requests = receiver.counter(metricsCodec.toMetricName("http_requests", List("id" -> "4"))) + val helloWorldService = new Service[Request, Response] { def apply(request: Request): Future[Response] = { diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 0267665..4187da5 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,15 +1,15 @@ import sbt._ object Dependencies { - val twitterVersion = "19.10.0" + val twitterVersion = "22.12.0" lazy val twitterServer = "com.twitter" %% "twitter-server" % twitterVersion lazy val finagleStats = "com.twitter" %% "finagle-stats" % twitterVersion - lazy val prometheusSimpleClient = "io.prometheus" % "simpleclient" % "0.8.0" - lazy val prometheusSimpleClientCommon = "io.prometheus" % "simpleclient_common" % "0.8.0" + lazy val prometheusSimpleClient = "io.prometheus" % "simpleclient" % "0.16.0" + lazy val prometheusSimpleClientCommon = "io.prometheus" % "simpleclient_common" % "0.16.0" lazy val twitterServerLogback = "com.twitter" %% "twitter-server-logback-classic" % twitterVersion - lazy val logback = "ch.qos.logback" % "logback-classic" % "1.2.3" + lazy val logback = "ch.qos.logback" % "logback-classic" % "1.4.5" - lazy val scalaTest = "org.scalatest" %% "scalatest" % "3.0.5" + lazy val scalaTest = "org.scalatest" %% "scalatest" % "3.2.14" } diff --git a/project/build.properties b/project/build.properties index 010613d..9a19778 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 1.3.3 \ No newline at end of file +sbt.version = 1.8.0 diff --git a/twitter-server-prometheus/src/main/scala/PrometheusExporter.scala b/twitter-server-prometheus/src/main/scala/PrometheusExporter.scala index 0773335..13aca86 100644 --- a/twitter-server-prometheus/src/main/scala/PrometheusExporter.scala +++ b/twitter-server-prometheus/src/main/scala/PrometheusExporter.scala @@ -1,14 +1,17 @@ package com.github.kovszilard.twitter.server.prometheus import com.twitter.app.App -import com.twitter.finagle.stats.PrometheusMetricsCollector +import com.twitter.finagle.stats.{MetricsStatsReceiver, PrometheusMetricsCollector} import com.twitter.server.Admin.Grouping import com.twitter.server.AdminHttpServer.Route import com.twitter.server.{AdminHttpServer, Stats} trait PrometheusExporter { self: App with AdminHttpServer with Stats => - PrometheusMetricsCollector().register() + lazy val metricsCodec: PrometheusMetricsCodec = new PrometheusMetricsCodec {} + lazy val metricsRegistry = MetricsStatsReceiver.defaultRegistry + + PrometheusMetricsCollector(metricsCodec, metricsRegistry).register() val metricsRoute: Route = Route.isolate(Route( path = "/metrics", diff --git a/twitter-server-prometheus/src/main/scala/PrometheusMetricsCodec.scala b/twitter-server-prometheus/src/main/scala/PrometheusMetricsCodec.scala new file mode 100644 index 0000000..942207e --- /dev/null +++ b/twitter-server-prometheus/src/main/scala/PrometheusMetricsCodec.scala @@ -0,0 +1,6 @@ +package com.github.kovszilard.twitter.server.prometheus + +trait PrometheusMetricsCodec { + def fromMetricName(metricName: String): (String, List[(String, String)]) = (metricName, List.empty) + def toMetricName(name: String, metadata: List[(String, String)]): String = name +} diff --git a/twitter-server-prometheus/src/main/scala/PrometheusMetricsCollector.scala b/twitter-server-prometheus/src/main/scala/PrometheusMetricsCollector.scala index 4192838..461bc15 100644 --- a/twitter-server-prometheus/src/main/scala/PrometheusMetricsCollector.scala +++ b/twitter-server-prometheus/src/main/scala/PrometheusMetricsCollector.scala @@ -1,58 +1,82 @@ package com.twitter.finagle.stats +import com.github.kovszilard.twitter.server.prometheus.PrometheusMetricsCodec import io.prometheus.client.Collector import io.prometheus.client.Collector.MetricFamilySamples.Sample import io.prometheus.client.Collector._ -import scala.collection.JavaConversions._ +import scala.jdk.CollectionConverters._ +import scala.util.Try -class PrometheusMetricsCollector(registry: MetricsView = MetricsStatsReceiver.defaultRegistry) extends Collector { +case class PrometheusMetricsCollector(codec: PrometheusMetricsCodec, registry: Metrics) extends Collector { + + implicit def listConverter[A](list: List[A]): java.util.List[A] = list.asJava + implicit def mapConverter[K, V](map: java.util.Map[K, V]): scala.collection.mutable.Map[K, V] = map.asScala override def collect(): java.util.List[MetricFamilySamples] = { - val gauges = registry.gauges.map{ case (name: String, value: Number) => fromGauge(name, value)} - val counters = registry.counters.map{ case (name: String, value: Number) => fromCounter(name, value)} - val histograms = registry.histograms.map{ case (name: String, value: Snapshot) => fromHistogram(name, value)} + val gauges = registry.gauges.map { gaugeSnapshot => + fromGauge(gaugeSnapshot.hierarchicalName, gaugeSnapshot.value) + } + val counters = registry.counters.map { counterSnapshot => + fromCounter(counterSnapshot.hierarchicalName, counterSnapshot.value) + } + val histograms = registry.histograms.map{ histogramSnapshot => + fromHistogram(histogramSnapshot.hierarchicalName, histogramSnapshot.value) + } + (gauges ++ counters ++ histograms).toList } + def extractAndSanitizeMetricAndLabels(rawName: String): (String, List[(String, String)]) = { + val (name, labels) = Try { + codec.fromMetricName(rawName) + }.getOrElse{ + // TODO Fix potential high cardinality if dynamic non-envodable values end up as the rawName + rawName -> List("finagleName" -> rawName) + } + (sanitizeMetricName(name), labels) + } + def fromGauge(name: String, value: Number): MetricFamilySamples = { - val prometheusName = sanitizeMetricName(name) + val (metric, labels) = extractAndSanitizeMetricAndLabels(name) new MetricFamilySamples( - prometheusName, + metric, Type.GAUGE, genHelpMessage(name, "gauge"), - List(new Sample(prometheusName, List("finagleName"), List(name), value.doubleValue())) + List(new Sample(metric, labels.map(_._1), labels.map(_._2), value.doubleValue())) ) } def fromCounter(name: String, value: Number): MetricFamilySamples = { - val prometheusName = sanitizeMetricName(name) + val (metric, labels) = extractAndSanitizeMetricAndLabels(name) new MetricFamilySamples( - prometheusName, + metric, Type.COUNTER, genHelpMessage(name, "counter"), - List(new Sample(prometheusName, List("finagleName"), List(name), value.doubleValue())) + List(new Sample(metric, labels.map(_._1), labels.map(_._2), value.doubleValue())) ) } def fromHistogram(name: String, snapshot: Snapshot): MetricFamilySamples = { - val prometheusName = sanitizeMetricName(name) - - val count = new Sample(s"${prometheusName}_count", Nil, Nil, snapshot.count.toDouble) - val sum = new Sample(s"${prometheusName}_sum", Nil, Nil, snapshot.sum.toDouble) - val max = new Sample(s"${prometheusName}_max", Nil, Nil, snapshot.max.toDouble) - val min = new Sample(s"${prometheusName}_min", Nil, Nil, snapshot.min.toDouble) - val avg = new Sample(s"${prometheusName}_avg", Nil, Nil, snapshot.average) - - val percentiles = snapshot.percentiles.map{percentile => - new Sample(prometheusName, List("quantile"), List(percentile.quantile.toString), percentile.value.toDouble) + val (metric, labels) = extractAndSanitizeMetricAndLabels(name) + val labelNames = labels.map(_._1) + val labelValues = labels.map(_._2) + + val count = new Sample(s"${metric}_count", labelNames, labelValues, snapshot.count.toDouble) + val sum = new Sample(s"${metric}_sum", labelNames, labelValues, snapshot.sum.toDouble) + val max = new Sample(s"${metric}_max", labelNames, labelValues, snapshot.max.toDouble) + val min = new Sample(s"${metric}_min", labelNames, labelValues, snapshot.min.toDouble) + val avg = new Sample(s"${metric}_avg", labelNames, labelValues, snapshot.average) + + val percentiles = snapshot.percentiles.map { percentile => + new Sample(metric, labelNames :+ "quantile", labelValues :+ percentile.quantile.toString, percentile.value.toDouble) } new MetricFamilySamples( - prometheusName, + metric, Type.SUMMARY, genHelpMessage(name, "histogram"), List(count, sum, max, min, avg) ++ percentiles @@ -64,8 +88,3 @@ class PrometheusMetricsCollector(registry: MetricsView = MetricsStatsReceiver.de } } - -object PrometheusMetricsCollector { - def apply() = new PrometheusMetricsCollector() - def apply(registry: Metrics) = new PrometheusMetricsCollector(registry) -} \ No newline at end of file