From 86311fc5e1d32b9422cc84c003d655f8fc4e3029 Mon Sep 17 00:00:00 2001 From: Eric Thompson Date: Wed, 4 Mar 2026 09:43:17 -0700 Subject: [PATCH 1/3] Add ignore flag to exclude markers from value calculations Add ignore_in_value_calculations column to marker_cache to prevent invalid NAVs from affecting chain value metrics. Filter ignored denoms and scopes in all value calculation services. --- ..._in_value_calculations_to_marker_cache.sql | 8 ++ .../explorer/domain/entities/Markers.kt | 15 +++ .../explorer/domain/entities/NavEvents.kt | 8 +- .../explorer/domain/entities/Nfts.kt | 15 +++ .../explorer/service/AssetService.kt | 20 +++- .../provenance/explorer/service/NftService.kt | 13 ++- .../explorer/service/PricingService.kt | 3 +- .../explorer/service/PulseMetricService.kt | 92 ++++++++++++++----- 8 files changed, 144 insertions(+), 30 deletions(-) create mode 100644 database/src/main/resources/db/migration/V1_112__Add_ignore_in_value_calculations_to_marker_cache.sql diff --git a/database/src/main/resources/db/migration/V1_112__Add_ignore_in_value_calculations_to_marker_cache.sql b/database/src/main/resources/db/migration/V1_112__Add_ignore_in_value_calculations_to_marker_cache.sql new file mode 100644 index 00000000..f038ee70 --- /dev/null +++ b/database/src/main/resources/db/migration/V1_112__Add_ignore_in_value_calculations_to_marker_cache.sql @@ -0,0 +1,8 @@ +SELECT 'Add ignore_in_value_calculations column to marker_cache' AS comment; + +ALTER TABLE marker_cache + ADD COLUMN IF NOT EXISTS ignore_in_value_calculations BOOLEAN NOT NULL DEFAULT FALSE; + +CREATE INDEX IF NOT EXISTS marker_cache_ignore_in_value_calculations_idx + ON marker_cache(ignore_in_value_calculations) + WHERE ignore_in_value_calculations = TRUE; diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Markers.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Markers.kt index 964f9263..bfd169d5 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Markers.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Markers.kt @@ -42,6 +42,7 @@ object MarkerCacheTable : IntIdTable(name = "marker_cache") { val supply = decimal("supply", 100, 10) val lastTx = datetime("last_tx_timestamp").nullable() val data = jsonb("data", OBJECT_MAPPER).nullable() + val ignoreInValueCalculations = bool("ignore_in_value_calculations").default(false) } enum class BaseDenomType { DENOM, IBC_DENOM } @@ -109,6 +110,19 @@ class MarkerCacheRecord(id: EntityID) : IntEntity(id) { fun findCountByIbc() = transaction { MarkerCacheRecord.find { MarkerCacheTable.markerType eq BaseDenomType.IBC_DENOM.name }.count() } + + /** + * Returns a map of marker addresses to denoms for markers that should be ignored in value calculations + */ + fun getIgnoredMarkers(): Map = transaction { + MarkerCacheRecord.find { MarkerCacheTable.ignoreInValueCalculations eq true } + .mapNotNull { record -> + record.markerAddress?.let { address -> + address to record.denom + } + } + .toMap() + } } fun toCoinStrWithPrice(price: BigDecimal?) = @@ -121,6 +135,7 @@ class MarkerCacheRecord(id: EntityID) : IntEntity(id) { var supply by MarkerCacheTable.supply var lastTx by MarkerCacheTable.lastTx var data by MarkerCacheTable.data + var ignoreInValueCalculations by MarkerCacheTable.ignoreInValueCalculations } object TokenDistributionAmountsTable : IntIdTable(name = "token_distribution_amounts") { diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/NavEvents.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/NavEvents.kt index e9087be3..43501edf 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/NavEvents.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/NavEvents.kt @@ -198,7 +198,7 @@ class NavEventsRecord(id: EntityID) : IntEntity(id) { val fromDateQuery = toDate?.let { "AND block_time <= ?" } ?: "" val query = """ - select sum(price_amount) + select scope_id, price_amount from (select scope_id, price_amount, row_number() over (partition by scope_id order by block_height desc) as r from nav_events where source = 'metadata' and price_amount > 0 $fromDateQuery ) s where r = 1 @@ -208,13 +208,13 @@ class NavEventsRecord(id: EntityID) : IntEntity(id) { query.execAndMap( listOf(Pair(JavaLocalDateTimeColumnType(), toDate)) ) { - BigDecimal(it.getString(1)) + Pair(it.getString("scope_id"), BigDecimal(it.getString("price_amount"))) } } else { query.execAndMap { - BigDecimal(it.getString(1)) + Pair(it.getString("scope_id"), BigDecimal(it.getString("price_amount"))) } - }.firstOrNull() ?: BigDecimal.ZERO + } } fun latestScopeNavsByEntity( diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Nfts.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Nfts.kt index d9f1c983..ac1899d6 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/Nfts.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/Nfts.kt @@ -2,6 +2,7 @@ package io.provenance.explorer.domain.entities import io.provenance.explorer.OBJECT_MAPPER import io.provenance.explorer.domain.core.sql.jsonb +import io.provenance.explorer.domain.core.sql.toDbQueryList import io.provenance.explorer.domain.extensions.execAndMap import io.provenance.explorer.domain.models.explorer.NftVOTransferObj import io.provenance.explorer.domain.models.explorer.toNftData @@ -89,6 +90,20 @@ class NftScopeRecord(id: EntityID) : IntEntity(id) { NftScopeRecord.find { NftScopeTable.scope.isNull() }.toList() } + /** + * Returns a set of scope addresses that have value_owner_address in the given set of addresses + */ + fun findScopeAddressesByValueOwners(valueOwnerAddresses: Set): Set = transaction { + if (valueOwnerAddresses.isEmpty()) return@transaction emptySet() + + val query = """ + SELECT address FROM nft_scope + WHERE scope ->> 'value_owner_address' IN (${valueOwnerAddresses.toDbQueryList()}) + """.trimIndent() + + query.execAndMap { it.getString("address") }.toSet() + } + fun insertOrUpdate(uuid: String, address: String, scope: Scope) = transaction { findByUuid(uuid)?.apply { diff --git a/service/src/main/kotlin/io/provenance/explorer/service/AssetService.kt b/service/src/main/kotlin/io/provenance/explorer/service/AssetService.kt index 1e36b936..19836e98 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/AssetService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/AssetService.kt @@ -35,6 +35,7 @@ import io.provenance.explorer.model.base.PagedResults import io.provenance.explorer.model.base.USD_LOWER import io.provenance.explorer.model.base.USD_UPPER import io.provenance.marker.v1.MarkerStatus +import jakarta.annotation.PostConstruct import kotlinx.coroutines.async import kotlinx.coroutines.runBlocking import org.jetbrains.exposed.dao.id.EntityID @@ -58,6 +59,16 @@ class AssetService( private var assetPricinglastRun: OffsetDateTime? = null + @PostConstruct + fun initializeAssetPricingLastRun() { + assetPricinglastRun = transaction { + AssetPricingRecord.getLastUpdatedTime()?.let { + OffsetDateTime.of(it, ZoneOffset.UTC) + } + } + logger.info("Initialized assetPricinglastRun from database: $assetPricinglastRun") + } + fun validateDenom(denom: String) = requireNotNullToMessage(MarkerCacheRecord.findByDenom(denom)) { "Denom $denom does not exist." } @@ -236,6 +247,10 @@ class AssetService( logger.info("Updating asset pricing, last run at: $assetPricinglastRun") + val ignoredDenoms = transaction { + MarkerCacheRecord.getIgnoredMarkers().values + } + val latestPrices = NavEventsRecord.getLatestNavEvents( priceDenoms = usdPriceDenoms, includeMarkers = true, @@ -243,7 +258,10 @@ class AssetService( fromDate = assetPricinglastRun?.toDateTime() ) - latestPrices.filter { it.denom !in usdPriceDenoms }.forEach { price -> + latestPrices.filter { + it.denom !in usdPriceDenoms && + it.denom !in ignoredDenoms + }.forEach { price -> if (price.denom != UTILITY_TOKEN) { val marker = getAssetRaw(price.denom!!) diff --git a/service/src/main/kotlin/io/provenance/explorer/service/NftService.kt b/service/src/main/kotlin/io/provenance/explorer/service/NftService.kt index 6e07b120..5c3e254a 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/NftService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/NftService.kt @@ -2,6 +2,7 @@ package io.provenance.explorer.service import com.google.protobuf.util.JsonFormat import io.provenance.explorer.domain.core.logger +import io.provenance.explorer.domain.entities.MarkerCacheRecord import io.provenance.explorer.domain.entities.NavEventsRecord import io.provenance.explorer.domain.entities.NftContractSpecRecord import io.provenance.explorer.domain.entities.NftScopeRecord @@ -258,15 +259,21 @@ class NftService( } } - fun getScopeTotalForNavEvents() = - // TODO could query for marker owned scopes and filter them out here after scope migration + fun getScopeTotalForNavEvents() = transaction { + val ignoredMarkerAddresses = MarkerCacheRecord.getIgnoredMarkers().keys + val ignoredScopeAddresses = NftScopeRecord.findScopeAddressesByValueOwners(ignoredMarkerAddresses) + NavEventsRecord.getLatestNavEvents( priceDenoms = listOf(USD_LOWER), includeMarkers = false, includeScopes = true, - ).sumOf { + ).filter { navEvent -> + // Filter out if scope's value_owner_address points to an ignored marker + navEvent.scopeId?.let { it !in ignoredScopeAddresses } ?: true + }.sumOf { it.calculateUsdPricePerUnit() } + } fun populateScopes() = runBlocking { NftScopeRecord.findWithMissingScope().forEach { diff --git a/service/src/main/kotlin/io/provenance/explorer/service/PricingService.kt b/service/src/main/kotlin/io/provenance/explorer/service/PricingService.kt index 3edae0c0..8e466d92 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/PricingService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/PricingService.kt @@ -23,7 +23,8 @@ class PricingService( val baseMap = transaction { MarkerCacheRecord.find { (MarkerCacheTable.status eq MarkerStatus.MARKER_STATUS_ACTIVE.name) and - (MarkerCacheTable.supply greater BigDecimal.ZERO) + (MarkerCacheTable.supply greater BigDecimal.ZERO) and + (MarkerCacheTable.ignoreInValueCalculations eq false) }.filterNot { // excluding portfolio manager pools in favor of scope navs // we may be able to remove this after scope data migration diff --git a/service/src/main/kotlin/io/provenance/explorer/service/PulseMetricService.kt b/service/src/main/kotlin/io/provenance/explorer/service/PulseMetricService.kt index 6633125f..f9033118 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/PulseMetricService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/PulseMetricService.kt @@ -20,8 +20,10 @@ import io.provenance.explorer.domain.entities.BlockCacheRecord import io.provenance.explorer.domain.entities.EntityNavEvent import io.provenance.explorer.domain.entities.LedgerEntityRecord import io.provenance.explorer.domain.entities.LedgerEntitySpecRecord +import io.provenance.explorer.domain.entities.MarkerCacheRecord import io.provenance.explorer.domain.entities.NavEvent import io.provenance.explorer.domain.entities.NavEventsRecord +import io.provenance.explorer.domain.entities.NftScopeRecord import io.provenance.explorer.domain.entities.PulseCacheRecord import io.provenance.explorer.domain.entities.TxCacheRecord import io.provenance.explorer.domain.extensions.pageCountOfResults @@ -107,6 +109,24 @@ class PulseMetricService( maximumSize(100) }.build() + /** + * Returns a map of marker addresses to denoms for markers that should be ignored in value calculations. + */ + private val ignoredMarkersCache: Cache> = + Caffeine.newBuilder().apply { + expireAfterWrite(1, TimeUnit.HOURS) + maximumSize(1) + }.build() + + /** + * Returns a set of scope addresses that should be ignored in value calculations + */ + private val ignoredScopeAddressesCache: Cache> = + Caffeine.newBuilder().apply { + expireAfterWrite(1, TimeUnit.HOURS) + maximumSize(1) + }.build() + /* so it turns out that the `usd` in metadata nav events use 3 decimal places - :| */ @@ -593,12 +613,19 @@ class PulseMetricService( range = range, atDateTime = atDateTime ) { - NavEventsRecord.totalMetadataNavs(atDateTime).let { - PulseMetric.build( - base = USD_UPPER, - amount = it.times(scopeNAVDecimal) - ) - } + val ignoredScopeAddresses = getIgnoredScopeAddresses() + + NavEventsRecord.totalMetadataNavs(atDateTime) + .filter { (scopeId, _) -> + scopeId?.let { it !in ignoredScopeAddresses } ?: true + } + .sumOf { (_, priceAmount) -> priceAmount } + .let { + PulseMetric.build( + base = USD_UPPER, + amount = it.times(scopeNAVDecimal) + ) + } } /** @@ -777,15 +804,17 @@ class PulseMetricService( committedAssetTotals(atDateTime).committedAssetsToValue() } - private fun Map.committedAssetsToValue() = this.map { - calcExchangeTotalValueForAsset(it.key, it.value) - }.sumOf { it } - .let { - PulseMetric.build( - base = USD_UPPER, - amount = it - ) - } + private fun Map.committedAssetsToValue() = this.let { committedAssets -> + val ignoredDenoms = getIgnoredMarkers().values + committedAssets.filterKeys { it !in ignoredDenoms }.map { + calcExchangeTotalValueForAsset(it.key, it.value) + }.sumOf { it } + }.let { + PulseMetric.build( + base = USD_UPPER, + amount = it + ) + } private fun Map.committedAssetsToVolume() = this.map { convertDenomToDisplayUnits(it.key, it.value) @@ -1526,6 +1555,24 @@ class PulseMetricService( private fun inversePowerOfTen(exp: Int) = 10.0.pow(exp.toDouble() * -1).toBigDecimal() + /** + * Returns a map of marker addresses to denoms for ignored markers + */ + private fun getIgnoredMarkers(): Map = + ignoredMarkersCache.get("ignored_markers") { + transaction { + MarkerCacheRecord.getIgnoredMarkers() + } + }!! + + private fun getIgnoredScopeAddresses(): Set = + ignoredScopeAddressesCache.get("ignored_scope_addresses") { + transaction { + val ignoredMarkerAddresses = getIgnoredMarkers().keys + NftScopeRecord.findScopeAddressesByValueOwners(ignoredMarkerAddresses) + } + }!! + /** * Build cache of exchange-traded asset summaries using nav events from * exchange module for USD-based assets @@ -2031,13 +2078,16 @@ class PulseMetricService( * TODO - this is problematic because it assumes all assets are USD quoted */ fun pulseAssetSummaries(atDateTime: LocalDateTime? = null): List { + val ignoredDenoms = getIgnoredMarkers().values val committedTotals = committedAssetTotals(atDateTime) - return committedTotals.keys.distinct().map { denom -> - buildAssetSummaryForDenom(denom, atDateTime) - }.toMutableList().also { - // add FIGR_HELOC supply/price/volume to pulse assets - it.add(buildFigureHelocAssetSummary(atDateTime)) - } + return committedTotals.keys.distinct() + .filter { it !in ignoredDenoms } + .map { denom -> + buildAssetSummaryForDenom(denom, atDateTime) + }.toMutableList().also { + // add FIGR_HELOC supply/price/volume to pulse assets + it.add(buildFigureHelocAssetSummary(atDateTime)) + } .sortedWith( compareBy( { it.symbol.isEmpty() }, From 4a9ee1061ade89a86036da6241933beb8fb7f43c Mon Sep 17 00:00:00 2001 From: Eric Thompson Date: Tue, 10 Mar 2026 10:46:54 -0600 Subject: [PATCH 2/3] bump migration version --- ...__Add_ignore_in_value_calculations_to_marker_cache.sql | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 database/src/main/resources/db/migration/V1_113__Add_ignore_in_value_calculations_to_marker_cache.sql diff --git a/database/src/main/resources/db/migration/V1_113__Add_ignore_in_value_calculations_to_marker_cache.sql b/database/src/main/resources/db/migration/V1_113__Add_ignore_in_value_calculations_to_marker_cache.sql new file mode 100644 index 00000000..f038ee70 --- /dev/null +++ b/database/src/main/resources/db/migration/V1_113__Add_ignore_in_value_calculations_to_marker_cache.sql @@ -0,0 +1,8 @@ +SELECT 'Add ignore_in_value_calculations column to marker_cache' AS comment; + +ALTER TABLE marker_cache + ADD COLUMN IF NOT EXISTS ignore_in_value_calculations BOOLEAN NOT NULL DEFAULT FALSE; + +CREATE INDEX IF NOT EXISTS marker_cache_ignore_in_value_calculations_idx + ON marker_cache(ignore_in_value_calculations) + WHERE ignore_in_value_calculations = TRUE; From f066eeb955741a3c4e54795b21cf4b10f599ea04 Mon Sep 17 00:00:00 2001 From: Eric Thompson Date: Tue, 10 Mar 2026 10:48:25 -0600 Subject: [PATCH 3/3] remove dup migration --- ...__Add_ignore_in_value_calculations_to_marker_cache.sql | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 database/src/main/resources/db/migration/V1_112__Add_ignore_in_value_calculations_to_marker_cache.sql diff --git a/database/src/main/resources/db/migration/V1_112__Add_ignore_in_value_calculations_to_marker_cache.sql b/database/src/main/resources/db/migration/V1_112__Add_ignore_in_value_calculations_to_marker_cache.sql deleted file mode 100644 index f038ee70..00000000 --- a/database/src/main/resources/db/migration/V1_112__Add_ignore_in_value_calculations_to_marker_cache.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT 'Add ignore_in_value_calculations column to marker_cache' AS comment; - -ALTER TABLE marker_cache - ADD COLUMN IF NOT EXISTS ignore_in_value_calculations BOOLEAN NOT NULL DEFAULT FALSE; - -CREATE INDEX IF NOT EXISTS marker_cache_ignore_in_value_calculations_idx - ON marker_cache(ignore_in_value_calculations) - WHERE ignore_in_value_calculations = TRUE;