Merge pull request #2540 from htmldoug/retry-after

Retry on Retry-After response header

Author: exoego

modules/core/src/main/scala/org/scalasteward/core/application/Context.scala

@@ -19,6 +19,7 @@ package org.scalasteward.core.application
 import cats.effect._
 import cats.effect.implicits._
 import cats.syntax.all._
+import eu.timepit.refined.auto._
 import org.http4s.Uri
 import org.http4s.client.Client
 import org.http4s.headers.`User-Agent`
@@ -82,14 +83,16 @@ object Context {
       _ <- Resource.eval(printBanner(logger))
       _ <- Resource.eval(F.delay(System.setProperty("http.agent", userAgentString)))
       userAgent <- Resource.eval(F.fromEither(`User-Agent`.parse(userAgentString)))
-      userAgentMiddleware = ClientConfiguration.setUserAgent[F](userAgent)
+      middleware = ClientConfiguration
+        .setUserAgent[F](userAgent)
+        .andThen(ClientConfiguration.retryAfter[F](maxAttempts = 5))
       defaultClient <- ClientConfiguration.build(
         ClientConfiguration.BuilderMiddleware.default,
-        userAgentMiddleware
+        middleware
       )
       urlCheckerClient <- ClientConfiguration.build(
         ClientConfiguration.disableFollowRedirect[F],
-        userAgentMiddleware
+        middleware
       )
       fileAlg = FileAlg.create(logger, F)
       processAlg = ProcessAlg.create(config.processCfg)(logger, F)

modules/core/src/main/scala/org/scalasteward/core/client/ClientConfiguration.scala

@@ -16,10 +16,16 @@
 
 package org.scalasteward.core.client
 
+import cats.effect._
+import eu.timepit.refined.auto._
+import eu.timepit.refined.types.numeric.PosInt
+import org.http4s.Response
 import org.http4s.client._
 import org.http4s.headers.`User-Agent`
-import cats.effect._
 import org.http4s.okhttp.client.OkHttpBuilder
+import org.typelevel.ci._
+
+import scala.concurrent.duration._
 
 object ClientConfiguration {
   type BuilderMiddleware[F[_]] = OkHttpBuilder[F] => OkHttpBuilder[F]
@@ -37,6 +43,32 @@ object ClientConfiguration {
     Client[F](req => client.run(req.putHeaders(userAgent)))
   }
 
+  private val RetryAfterStatuses = Set(403, 429, 503)
+
+  /**
+    * @param maxAttempts max number times the HTTP request should be sent
+    *                    useful to avoid unexpected cloud provider costs
+    */
+  def retryAfter[F[_]: Temporal](maxAttempts: PosInt = 5): Middleware[F] = { client =>
+    Client[F] { req =>
+      def run(attempt: Int = 1): Resource[F, Response[F]] = client
+        .run(req.putHeaders("X-Attempt" -> attempt.toString))
+        .flatMap { response =>
+          val maybeRetried = for {
+            header <- response.headers.get(ci"Retry-After")
+            seconds <- header.head.value.toIntOption
+            if seconds > 0
+            duration = seconds.seconds
+            if RetryAfterStatuses.contains(response.status.code)
+            if attempt < maxAttempts.value
+          } yield Resource.eval(Temporal[F].sleep(duration)).flatMap(_ => run(attempt + 1))
+          maybeRetried.getOrElse(Resource.pure(response))
+        }
+
+      run()
+    }
+  }
+
   def disableFollowRedirect[F[_]](builder: OkHttpBuilder[F]): OkHttpBuilder[F] =
     builder.withOkHttpClient(
       builder.okHttpClient.newBuilder().followRedirects(false).build()

modules/core/src/test/scala/org/scalasteward/core/client/ClientConfigurationTest.scala

@@ -1,16 +1,18 @@
 package org.scalasteward.core.client
 
-import org.http4s.client._
-import org.http4s.headers.`User-Agent`
 import cats.effect._
 import cats.implicits._
-import org.http4s.implicits._
-import org.typelevel.ci._
 import munit.CatsEffectSuite
 import org.http4s.HttpRoutes
-import org.http4s.headers.Location
+import org.http4s.client._
+import org.http4s.headers.{`Retry-After`, `User-Agent`, Location}
+import org.http4s.implicits._
+import org.typelevel.ci._
+import eu.timepit.refined.auto._
+import eu.timepit.refined.types.numeric.PosInt
 
 class ClientConfigurationTest extends CatsEffectSuite {
+
   private val userAgentValue = "my-user-agent"
   private val dummyUserAgent =
     `User-Agent`.parse(userAgentValue).getOrElse(fail("unable to create user agent"))
@@ -35,12 +37,21 @@ class ClientConfigurationTest extends CatsEffectSuite {
 
       case GET -> Root / "redirected" =>
         BadRequest("Got redirected")
+
+      case req @ GET -> Root / "retry-after" =>
+        val maybeAttempt = req.headers.get(ci"X-Attempt").flatMap(_.head.value.toIntOption)
+        maybeAttempt match {
+          case Some(attempt) if attempt >= 2 =>
+            Ok()
+          case _ =>
+            Forbidden().map(_.putHeaders(`Retry-After`.fromLong(1)))
+        }
     }
   }
 
   test("setUserAgent add a specific user agent to requests") {
-    import org.http4s.client.dsl.io._
     import org.http4s.Method._
+    import org.http4s.client.dsl.io._
 
     val initialClient = Client.fromHttpApp[IO](routes.orNotFound)
     val setUserAgent = ClientConfiguration.setUserAgent[IO](dummyUserAgent)
@@ -59,9 +70,9 @@ class ClientConfigurationTest extends CatsEffectSuite {
   }
 
   test("disableFollowRedirect does not follow redirect") {
+    import org.http4s.Method._
     import org.http4s.blaze.server._
     import org.http4s.client.dsl.io._
-    import org.http4s.Method._
 
     val regularClient = ClientConfiguration.build[IO](
       ClientConfiguration.BuilderMiddleware.default,
@@ -84,4 +95,27 @@ class ClientConfigurationTest extends CatsEffectSuite {
     }
     test.assertEquals((400, 302))
   }
+
+  test("retries on retry-after response header") {
+    import org.http4s.Method._
+    import org.http4s.client.dsl.io._
+
+    def clientWithMaxAttempts(maxAttempts: PosInt): Client[IO] = {
+      val initialClient = Client.fromHttpApp[IO](routes.orNotFound)
+      val retryAfter = ClientConfiguration.retryAfter[IO](maxAttempts)
+      retryAfter(initialClient)
+    }
+
+    val notEnoughRetries = clientWithMaxAttempts(1)
+      .run(GET(uri"/retry-after"))
+      .use(r => r.status.code.pure[IO])
+      .assertEquals(403)
+
+    val exactlyEnoughRetries = clientWithMaxAttempts(2)
+      .run(GET(uri"/retry-after"))
+      .use(r => r.status.code.pure[IO])
+      .assertEquals(200)
+
+    notEnoughRetries.flatMap(_ => exactlyEnoughRetries)
+  }
 }