STAC Client pagination support (#327)

* StacClient supports pagination and has fs2.Streams in the interface

* Cleanup PaginationToken companion object

* Split StacClient and StreamingStacClient

* Make a StreamingStacClient G independent from F; finish search pagination

* Rename type alias

CHANGELOG.md

@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [Unreleased]
 ### Added
 - Add Scala 2.13 cross compilation [#310](https://github.com/azavea/stac4s/pull/310)
+- STAC Client pagination support [#327](https://github.com/azavea/stac4s/pull/327)
 
 ### Changed
 - Make StacClient type alias a trait [#325](https://github.com/azavea/stac4s/pull/325)

build.sbt

@@ -1,5 +1,4 @@
 import xerial.sbt.Sonatype._
-import Dependencies._
 
 lazy val commonSettings = Seq(
   // We are overriding the default behavior of sbt-git which, by default, only
@@ -104,22 +103,20 @@ lazy val credentialSettings = Seq(
 
 val jvmGeometryDependencies = Def.setting {
   Seq(
-    "org.locationtech.jts" % "jts-core" % Versions.Jts,
-    geotrellis("vector").value
+    "org.locationtech.jts"         % "jts-core"          % Versions.Jts,
+    "org.locationtech.geotrellis" %% "geotrellis-vector" % Versions.GeoTrellis.value
   )
 }
 
 val coreDependenciesJVM = Def.setting {
-  Seq(
-    "org.threeten" % "threeten-extra" % Versions.ThreeTenExtra
-  ) ++ jvmGeometryDependencies.value
+  Seq("org.threeten" % "threeten-extra" % Versions.ThreeTenExtra) ++ jvmGeometryDependencies.value
 }
 
 val testingDependenciesJVM = Def.setting {
   Seq(
-    geotrellis("vector").value,
-    "org.locationtech.jts" % "jts-core"       % Versions.Jts,
-    "org.threeten"         % "threeten-extra" % Versions.ThreeTenExtra
+    "org.locationtech.geotrellis" %% "geotrellis-vector" % Versions.GeoTrellis.value,
+    "org.locationtech.jts"         % "jts-core"          % Versions.Jts,
+    "org.threeten"                 % "threeten-extra"    % Versions.ThreeTenExtra
   )
 }
 
@@ -131,7 +128,7 @@ val testRunnerDependenciesJVM = Seq(
 
 lazy val root = project
   .in(file("."))
-  .settings(moduleName := "root")
+  .settings(name := "stac4s")
   .settings(commonSettings)
   .settings(publishSettings)
   .settings(noPublishSettings)
@@ -182,11 +179,7 @@ lazy val testing = crossProject(JSPlatform, JVMPlatform)
     )
   )
   .jvmSettings(libraryDependencies ++= testingDependenciesJVM.value)
-  .jsSettings(
-    libraryDependencies ++= Seq(
-      "io.github.cquiroz" %%% "scala-java-time" % "2.3.0" % Test
-    )
-  )
+  .jsSettings(libraryDependencies += "io.github.cquiroz" %%% "scala-java-time" % "2.3.0" % Test)
 
 lazy val testingJVM = testing.jvm
 lazy val testingJS  = testing.js
@@ -203,11 +196,7 @@ lazy val coreTest = crossProject(JSPlatform, JVMPlatform)
       "org.scalatestplus" %%% "scalacheck-1-14" % Versions.ScalatestPlusScalacheck % Test
     )
   )
-  .jsSettings(
-    libraryDependencies ++= Seq(
-      "io.github.cquiroz" %%% "scala-java-time" % "2.3.0" % Test
-    )
-  )
+  .jsSettings(libraryDependencies += "io.github.cquiroz" %%% "scala-java-time" % "2.3.0" % Test)
 
 lazy val coreTestJVM = coreTest.jvm
 lazy val coreTestJS  = coreTest.js
@@ -220,6 +209,8 @@ lazy val client = crossProject(JSPlatform, JVMPlatform)
   .settings(publishSettings)
   .settings(
     libraryDependencies ++= Seq(
+      "com.github.julien-truffaut"    %%% "monocle-core"  % Versions.Monocle,
+      "com.github.julien-truffaut"    %%% "monocle-macro" % Versions.Monocle,
       "io.circe"                      %%% "circe-core"    % Versions.Circe,
       "io.circe"                      %%% "circe-generic" % Versions.Circe,
       "io.circe"                      %%% "circe-refined" % Versions.Circe,
@@ -232,6 +223,7 @@ lazy val client = crossProject(JSPlatform, JVMPlatform)
       "com.softwaremill.sttp.client3" %%% "json-common"   % Versions.Sttp,
       "com.softwaremill.sttp.model"   %%% "core"          % Versions.SttpModel,
       "com.softwaremill.sttp.shared"  %%% "core"          % Versions.SttpShared,
+      "co.fs2"                        %%% "fs2-core"      % Versions.Fs2,
       "org.scalatest"                 %%% "scalatest"     % Versions.Scalatest % Test
     )
   )

modules/client/js/src/main/scala/com/azavea/stac4s/api/client/SearchFilters.scala

@@ -8,6 +8,8 @@ import com.azavea.stac4s.jsTypes.TemporalExtent
 import eu.timepit.refined.types.numeric.NonNegInt
 import io.circe._
 import io.circe.refined._
+import monocle.Lens
+import monocle.macros.GenLens
 
 case class SearchFilters(
     bbox: Option[Bbox] = None,
@@ -21,6 +23,7 @@ case class SearchFilters(
 )
 
 object SearchFilters extends ClientCodecs {
+  implicit val paginationTokenLens: Lens[SearchFilters, Option[PaginationToken]] = GenLens[SearchFilters](_.next)
 
   implicit val searchFiltersDecoder: Decoder[SearchFilters] = { c =>
     for {

modules/client/js/src/main/scala/com/azavea/stac4s/api/client/SttpStacClient.scala

@@ -1,11 +1,11 @@
 package com.azavea.stac4s.api.client
 
-import cats.MonadError
+import cats.MonadThrow
 import sttp.client3.SttpBackend
 import sttp.model.Uri
 
 object SttpStacClient {
 
-  def apply[F[_]: MonadError[*[_], Throwable]](client: SttpBackend[F, Any], baseUri: Uri): SttpStacClient[F] =
+  def apply[F[_]: MonadThrow](client: SttpBackend[F, Any], baseUri: Uri): SttpStacClient[F] =
     SttpStacClientF[F, SearchFilters](client, baseUri)
 }

modules/client/js/src/main/scala/com/azavea/stac4s/api/client/package.scala

@@ -1,5 +1,8 @@
 package com.azavea.stac4s.api
 
 package object client {
-  type SttpStacClient[F[_]] = SttpStacClientF[F, SearchFilters]
+  type SttpStacClient[F[_]]            = SttpStacClientF[F, SearchFilters]
+  type StacClient[F[_]]                = StacClientF[F, SearchFilters]
+  type StreamingStacClientFS2[F[_]]    = StreamingStacClientF[F, fs2.Stream[F, *], SearchFilters]
+  type StreamingStacClient[F[_], G[_]] = StreamingStacClientF[F, G, SearchFilters]
 }

modules/client/jvm/src/main/scala/com/azavea/stac4s/api/client/SearchFilters.scala

@@ -8,6 +8,8 @@ import eu.timepit.refined.types.numeric.NonNegInt
 import geotrellis.vector.{io => _, _}
 import io.circe._
 import io.circe.refined._
+import monocle.Lens
+import monocle.macros.GenLens
 
 case class SearchFilters(
     bbox: Option[Bbox] = None,
@@ -21,6 +23,7 @@ case class SearchFilters(
 )
 
 object SearchFilters extends ClientCodecs {
+  implicit val paginationTokenLens: Lens[SearchFilters, Option[PaginationToken]] = GenLens[SearchFilters](_.next)
 
   implicit val searchFiltersDecoder: Decoder[SearchFilters] = { c =>
     for {

modules/client/jvm/src/main/scala/com/azavea/stac4s/api/client/SttpStacClient.scala

@@ -1,11 +1,11 @@
 package com.azavea.stac4s.api.client
 
-import cats.MonadError
+import cats.MonadThrow
 import sttp.client3.SttpBackend
 import sttp.model.Uri
 
 object SttpStacClient {
 
-  def apply[F[_]: MonadError[*[_], Throwable]](client: SttpBackend[F, Any], baseUri: Uri): SttpStacClient[F] =
+  def apply[F[_]: MonadThrow](client: SttpBackend[F, Any], baseUri: Uri): SttpStacClient[F] =
     SttpStacClientF[F, SearchFilters](client, baseUri)
 }

modules/client/jvm/src/main/scala/com/azavea/stac4s/api/client/package.scala

@@ -0,0 +1,8 @@
+package com.azavea.stac4s.api
+
+package object client {
+  type SttpStacClient[F[_]]            = SttpStacClientF[F, SearchFilters]
+  type StacClient[F[_]]                = StacClientF[F, SearchFilters]
+  type StreamingStacClientFS2[F[_]]    = StreamingStacClientF[F, fs2.Stream[F, *], SearchFilters]
+  type StreamingStacClient[F[_], G[_]] = StreamingStacClientF[F, G, SearchFilters]
+}

modules/client/shared/src/main/scala/com/azavea/stac4s/api/client/PaginationToken.scala

@@ -2,12 +2,11 @@ package com.azavea.stac4s.api.client
 
 import cats.syntax.either._
 import eu.timepit.refined.types.numeric.PosInt
-import io.circe
 import io.circe.generic.semiauto._
 import io.circe.parser.parse
 import io.circe.refined._
 import io.circe.syntax._
-import io.circe.{Decoder, Encoder}
+import io.circe.{Decoder, Encoder, Error}
 
 import java.time.Instant
 import java.util.Base64
@@ -18,26 +17,26 @@ final case class PaginationToken(timestampAtLeast: Instant, serialIdGreaterThan:
   * https://github.com/azavea/franklin/blob/f5be8ddf48661c5bc43cbd22cb7277e961641803/application/src/main/scala/com/azavea/franklin/api/schemas/package.scala#L84-L85
   */
 object PaginationToken {
-  val b64Encoder = Base64.getEncoder
-  val b64Decoder = Base64.getDecoder
-
   val defaultDecoder: Decoder[PaginationToken] = deriveDecoder
   val defaultEncoder: Encoder[PaginationToken] = deriveEncoder
 
+  val b64Encoder: Base64.Encoder = Base64.getEncoder
+  val b64Decoder: Base64.Decoder = Base64.getDecoder
+
   def encPaginationToken(token: PaginationToken): String = b64Encoder.encodeToString(
     token.asJson(defaultEncoder).noSpaces.getBytes
   )
 
-  def decPaginationToken(encoded: String): Either[circe.Error, PaginationToken] = {
+  def decPaginationTokenEither(encoded: String): Either[Error, PaginationToken] = {
     val jsonString = new String(b64Decoder.decode(encoded))
     for {
       js      <- parse(jsonString)
       decoded <- js.as[PaginationToken](defaultDecoder)
     } yield decoded
   }
 
-  implicit val dec: Decoder[PaginationToken] =
-    Decoder.decodeString.emap(str => decPaginationToken(str).leftMap(_.getMessage))
+  implicit val paginationTokenDecoder: Decoder[PaginationToken] =
+    Decoder.decodeString.emap(str => decPaginationTokenEither(str).leftMap(_.getMessage))
 
-  implicit val enc: Encoder[PaginationToken] = { encPaginationToken(_).asJson }
+  implicit val paginationTokenEncoder: Encoder[PaginationToken] = { encPaginationToken(_).asJson }
 }

modules/client/shared/src/main/scala/com/azavea/stac4s/api/client/StacClientF.scala

@@ -5,12 +5,15 @@ import com.azavea.stac4s._
 import eu.timepit.refined.types.string.NonEmptyString
 
 trait StacClientF[F[_], S] {
-  def search: F[List[StacItem]]
-  def search(filter: S): F[List[StacItem]]
-  def collections: F[List[StacCollection]]
   def collection(collectionId: NonEmptyString): F[StacCollection]
-  def items(collectionId: NonEmptyString): F[List[StacItem]]
   def item(collectionId: NonEmptyString, itemId: NonEmptyString): F[StacItem]
   def itemCreate(collectionId: NonEmptyString, item: StacItem): F[StacItem]
   def collectionCreate(collection: StacCollection): F[StacCollection]
 }
+
+trait StreamingStacClientF[F[_], G[_], S] extends StacClientF[F, S] {
+  def search: G[StacItem]
+  def search(filter: S): G[StacItem]
+  def collections: G[StacCollection]
+  def items(collectionId: NonEmptyString): G[StacItem]
+}