Add ignore flag to exclude markers from value calculations

database/src/main/resources/db/migration/V1_113__Add_ignore_in_value_calculations_to_marker_cache.sql

@@ -0,0 +1,8 @@
+SELECT 'Add ignore_in_value_calculations column to marker_cache' AS comment;
+
+ALTER TABLE marker_cache
+    ADD COLUMN IF NOT EXISTS ignore_in_value_calculations BOOLEAN NOT NULL DEFAULT FALSE;
+
+CREATE INDEX IF NOT EXISTS marker_cache_ignore_in_value_calculations_idx 
+    ON marker_cache(ignore_in_value_calculations) 
+    WHERE ignore_in_value_calculations = TRUE;

service/src/main/kotlin/io/provenance/explorer/domain/entities/Markers.kt

@@ -42,6 +42,7 @@ object MarkerCacheTable : IntIdTable(name = "marker_cache") {
     val supply = decimal("supply", 100, 10)
     val lastTx = datetime("last_tx_timestamp").nullable()
     val data = jsonb<MarkerCacheTable, MarkerAccount>("data", OBJECT_MAPPER).nullable()
+    val ignoreInValueCalculations = bool("ignore_in_value_calculations").default(false)
 }
 
 enum class BaseDenomType { DENOM, IBC_DENOM }
@@ -109,6 +110,19 @@ class MarkerCacheRecord(id: EntityID<Int>) : IntEntity(id) {
         fun findCountByIbc() = transaction {
             MarkerCacheRecord.find { MarkerCacheTable.markerType eq BaseDenomType.IBC_DENOM.name }.count()
         }
+
+        /**
+         * Returns a map of marker addresses to denoms for markers that should be ignored in value calculations
+         */
+        fun getIgnoredMarkers(): Map<String, String> = transaction {
+            MarkerCacheRecord.find { MarkerCacheTable.ignoreInValueCalculations eq true }
+                .mapNotNull { record ->
+                    record.markerAddress?.let { address ->
+                        address to record.denom
+                    }
+                }
+                .toMap()
+        }
     }
 
     fun toCoinStrWithPrice(price: BigDecimal?) =
@@ -121,6 +135,7 @@ class MarkerCacheRecord(id: EntityID<Int>) : IntEntity(id) {
     var supply by MarkerCacheTable.supply
     var lastTx by MarkerCacheTable.lastTx
     var data by MarkerCacheTable.data
+    var ignoreInValueCalculations by MarkerCacheTable.ignoreInValueCalculations
 }
 
 object TokenDistributionAmountsTable : IntIdTable(name = "token_distribution_amounts") {

service/src/main/kotlin/io/provenance/explorer/domain/entities/NavEvents.kt

@@ -198,7 +198,7 @@ class NavEventsRecord(id: EntityID<Int>) : IntEntity(id) {
             val fromDateQuery = toDate?.let { "AND block_time <= ?" } ?: ""
 
             val query = """
-                select sum(price_amount)
+                select scope_id, price_amount
                 from (select scope_id, price_amount, row_number() over (partition by scope_id order by block_height desc) as r
                       from nav_events where source = 'metadata' and price_amount > 0 $fromDateQuery ) s
                 where r = 1
@@ -208,13 +208,13 @@ class NavEventsRecord(id: EntityID<Int>) : IntEntity(id) {
                 query.execAndMap(
                     listOf(Pair(JavaLocalDateTimeColumnType(), toDate))
                 ) {
-                    BigDecimal(it.getString(1))
+                    Pair(it.getString("scope_id"), BigDecimal(it.getString("price_amount")))
                 }
             } else {
                 query.execAndMap {
-                    BigDecimal(it.getString(1))
+                    Pair(it.getString("scope_id"), BigDecimal(it.getString("price_amount")))
                 }
-            }.firstOrNull() ?: BigDecimal.ZERO
+            }
         }
 
         fun latestScopeNavsByEntity(

service/src/main/kotlin/io/provenance/explorer/domain/entities/Nfts.kt

@@ -2,6 +2,7 @@ package io.provenance.explorer.domain.entities
 
 import io.provenance.explorer.OBJECT_MAPPER
 import io.provenance.explorer.domain.core.sql.jsonb
+import io.provenance.explorer.domain.core.sql.toDbQueryList
 import io.provenance.explorer.domain.extensions.execAndMap
 import io.provenance.explorer.domain.models.explorer.NftVOTransferObj
 import io.provenance.explorer.domain.models.explorer.toNftData
@@ -89,6 +90,20 @@ class NftScopeRecord(id: EntityID<Int>) : IntEntity(id) {
             NftScopeRecord.find { NftScopeTable.scope.isNull() }.toList()
         }
 
+        /**
+         * Returns a set of scope addresses that have value_owner_address in the given set of addresses
+         */
+        fun findScopeAddressesByValueOwners(valueOwnerAddresses: Set<String>): Set<String> = transaction {
+            if (valueOwnerAddresses.isEmpty()) return@transaction emptySet()
+
+            val query = """
+                SELECT address FROM nft_scope
+                WHERE scope ->> 'value_owner_address' IN (${valueOwnerAddresses.toDbQueryList()})
+            """.trimIndent()
+
+            query.execAndMap { it.getString("address") }.toSet()
+        }
+
         fun insertOrUpdate(uuid: String, address: String, scope: Scope) =
             transaction {
                 findByUuid(uuid)?.apply {

service/src/main/kotlin/io/provenance/explorer/service/AssetService.kt

@@ -35,6 +35,7 @@ import io.provenance.explorer.model.base.PagedResults
 import io.provenance.explorer.model.base.USD_LOWER
 import io.provenance.explorer.model.base.USD_UPPER
 import io.provenance.marker.v1.MarkerStatus
+import jakarta.annotation.PostConstruct
 import kotlinx.coroutines.async
 import kotlinx.coroutines.runBlocking
 import org.jetbrains.exposed.dao.id.EntityID
@@ -58,6 +59,16 @@ class AssetService(
 
     private var assetPricinglastRun: OffsetDateTime? = null
 
+    @PostConstruct
+    fun initializeAssetPricingLastRun() {
+        assetPricinglastRun = transaction {
+            AssetPricingRecord.getLastUpdatedTime()?.let {
+                OffsetDateTime.of(it, ZoneOffset.UTC)
+            }
+        }
+        logger.info("Initialized assetPricinglastRun from database: $assetPricinglastRun")
+    }
+
     fun validateDenom(denom: String) =
         requireNotNullToMessage(MarkerCacheRecord.findByDenom(denom)) { "Denom $denom does not exist." }
 
@@ -236,14 +247,21 @@ class AssetService(
 
         logger.info("Updating asset pricing, last run at: $assetPricinglastRun")
 
+        val ignoredDenoms = transaction {
+            MarkerCacheRecord.getIgnoredMarkers().values
+        }
+
         val latestPrices = NavEventsRecord.getLatestNavEvents(
             priceDenoms = usdPriceDenoms,
             includeMarkers = true,
             includeScopes = false,
             fromDate = assetPricinglastRun?.toDateTime()
         )
 
-        latestPrices.filter { it.denom !in usdPriceDenoms }.forEach { price ->
+        latestPrices.filter {
+            it.denom !in usdPriceDenoms &&
+            it.denom !in ignoredDenoms
+        }.forEach { price ->
             if (price.denom != UTILITY_TOKEN) {
                 val marker = getAssetRaw(price.denom!!)
 

service/src/main/kotlin/io/provenance/explorer/service/NftService.kt

@@ -2,6 +2,7 @@ package io.provenance.explorer.service
 
 import com.google.protobuf.util.JsonFormat
 import io.provenance.explorer.domain.core.logger
+import io.provenance.explorer.domain.entities.MarkerCacheRecord
 import io.provenance.explorer.domain.entities.NavEventsRecord
 import io.provenance.explorer.domain.entities.NftContractSpecRecord
 import io.provenance.explorer.domain.entities.NftScopeRecord
@@ -258,15 +259,21 @@ class NftService(
         }
     }
 
-    fun getScopeTotalForNavEvents() =
-        // TODO could query for marker owned scopes and filter them out here after scope migration
+    fun getScopeTotalForNavEvents() = transaction {
+        val ignoredMarkerAddresses = MarkerCacheRecord.getIgnoredMarkers().keys
+        val ignoredScopeAddresses = NftScopeRecord.findScopeAddressesByValueOwners(ignoredMarkerAddresses)
+
         NavEventsRecord.getLatestNavEvents(
             priceDenoms = listOf(USD_LOWER),
             includeMarkers = false,
             includeScopes = true,
-        ).sumOf {
+        ).filter { navEvent ->
+            // Filter out if scope's value_owner_address points to an ignored marker
+            navEvent.scopeId?.let { it !in ignoredScopeAddresses } ?: true
+        }.sumOf {
             it.calculateUsdPricePerUnit()
         }
+    }
 
     fun populateScopes() = runBlocking {
         NftScopeRecord.findWithMissingScope().forEach {

service/src/main/kotlin/io/provenance/explorer/service/PricingService.kt

@@ -23,7 +23,8 @@ class PricingService(
         val baseMap = transaction {
             MarkerCacheRecord.find {
                 (MarkerCacheTable.status eq MarkerStatus.MARKER_STATUS_ACTIVE.name) and
-                    (MarkerCacheTable.supply greater BigDecimal.ZERO)
+                    (MarkerCacheTable.supply greater BigDecimal.ZERO) and
+                    (MarkerCacheTable.ignoreInValueCalculations eq false)
             }.filterNot {
                 // excluding portfolio manager pools in favor of scope navs
                 // we may be able to remove this after scope data migration

service/src/main/kotlin/io/provenance/explorer/service/PulseMetricService.kt

@@ -20,8 +20,10 @@ import io.provenance.explorer.domain.entities.BlockCacheRecord
 import io.provenance.explorer.domain.entities.EntityNavEvent
 import io.provenance.explorer.domain.entities.LedgerEntityRecord
 import io.provenance.explorer.domain.entities.LedgerEntitySpecRecord
+import io.provenance.explorer.domain.entities.MarkerCacheRecord
 import io.provenance.explorer.domain.entities.NavEvent
 import io.provenance.explorer.domain.entities.NavEventsRecord
+import io.provenance.explorer.domain.entities.NftScopeRecord
 import io.provenance.explorer.domain.entities.PulseCacheRecord
 import io.provenance.explorer.domain.entities.TxCacheRecord
 import io.provenance.explorer.domain.extensions.pageCountOfResults
@@ -107,6 +109,24 @@ class PulseMetricService(
             maximumSize(100)
         }.build()
 
+    /**
+     * Returns a map of marker addresses to denoms for markers that should be ignored in value calculations.
+     */
+    private val ignoredMarkersCache: Cache<String, Map<String, String>> =
+        Caffeine.newBuilder().apply {
+            expireAfterWrite(1, TimeUnit.HOURS)
+            maximumSize(1)
+        }.build()
+
+    /**
+     * Returns a set of scope addresses that should be ignored in value calculations
+     */
+    private val ignoredScopeAddressesCache: Cache<String, Set<String>> =
+        Caffeine.newBuilder().apply {
+            expireAfterWrite(1, TimeUnit.HOURS)
+            maximumSize(1)
+        }.build()
+
     /* so it turns out that the `usd` in metadata nav events
        use 3 decimal places - :|
      */
@@ -594,12 +614,19 @@ class PulseMetricService(
             range = range,
             atDateTime = atDateTime
         ) {
-            NavEventsRecord.totalMetadataNavs(atDateTime).let {
-                PulseMetric.build(
-                    base = USD_UPPER,
-                    amount = it.times(scopeNAVDecimal)
-                )
-            }
+            val ignoredScopeAddresses = getIgnoredScopeAddresses()
+
+            NavEventsRecord.totalMetadataNavs(atDateTime)
+                .filter { (scopeId, _) ->
+                    scopeId?.let { it !in ignoredScopeAddresses } ?: true
+                }
+                .sumOf { (_, priceAmount) -> priceAmount }
+                .let {
+                    PulseMetric.build(
+                        base = USD_UPPER,
+                        amount = it.times(scopeNAVDecimal)
+                    )
+                }
         }
 
     /**
@@ -778,15 +805,17 @@ class PulseMetricService(
             committedAssetTotals(atDateTime).committedAssetsToValue()
         }
 
-    private fun Map<String, BigDecimal>.committedAssetsToValue() = this.map {
-        calcExchangeTotalValueForAsset(it.key, it.value)
-    }.sumOf { it }
-        .let {
-            PulseMetric.build(
-                base = USD_UPPER,
-                amount = it
-            )
-        }
+    private fun Map<String, BigDecimal>.committedAssetsToValue() = this.let { committedAssets ->
+        val ignoredDenoms = getIgnoredMarkers().values
+        committedAssets.filterKeys { it !in ignoredDenoms }.map {
+            calcExchangeTotalValueForAsset(it.key, it.value)
+        }.sumOf { it }
+    }.let {
+        PulseMetric.build(
+            base = USD_UPPER,
+            amount = it
+        )
+    }
 
     private fun Map<String, BigDecimal>.committedAssetsToVolume() = this.map {
         convertDenomToDisplayUnits(it.key, it.value)
@@ -1536,6 +1565,24 @@ class PulseMetricService(
     private fun inversePowerOfTen(exp: Int) =
         10.0.pow(exp.toDouble() * -1).toBigDecimal()
 
+    /**
+     * Returns a map of marker addresses to denoms for ignored markers
+     */
+    private fun getIgnoredMarkers(): Map<String, String> =
+        ignoredMarkersCache.get("ignored_markers") {
+            transaction {
+                MarkerCacheRecord.getIgnoredMarkers()
+            }
+        }!!
+
+    private fun getIgnoredScopeAddresses(): Set<String> =
+        ignoredScopeAddressesCache.get("ignored_scope_addresses") {
+            transaction {
+                val ignoredMarkerAddresses = getIgnoredMarkers().keys
+                NftScopeRecord.findScopeAddressesByValueOwners(ignoredMarkerAddresses)
+            }
+        }!!
+
     /**
      * Build cache of exchange-traded asset summaries using nav events from
      * exchange module for USD-based assets
@@ -2041,13 +2088,16 @@ class PulseMetricService(
      * TODO - this is problematic because it assumes all assets are USD quoted
      */
     fun pulseAssetSummaries(atDateTime: LocalDateTime? = null): List<PulseAssetSummary> {
+        val ignoredDenoms = getIgnoredMarkers().values
         val committedTotals = committedAssetTotals(atDateTime)
-        return committedTotals.keys.distinct().map { denom ->
-            buildAssetSummaryForDenom(denom, atDateTime)
-        }.toMutableList().also {
-            // add FIGR_HELOC supply/price/volume to pulse assets
-            it.add(buildFigureHelocAssetSummary(atDateTime))
-        }
+        return committedTotals.keys.distinct()
+            .filter { it !in ignoredDenoms }
+            .map { denom ->
+                buildAssetSummaryForDenom(denom, atDateTime)
+            }.toMutableList().also {
+                // add FIGR_HELOC supply/price/volume to pulse assets
+                it.add(buildFigureHelocAssetSummary(atDateTime))
+            }
             .sortedWith(
                 compareBy(
                     { it.symbol.isEmpty() },