Master fix node country decoding (#145)

* check if node country/city includes '\x00'

Signed-off-by: mario <[email protected]>

* add --start-block flag to substrate-ingest

Signed-off-by: mario <[email protected]>

* add setBlockRange to processor to enable  modifying start/end blocks

Signed-off-by: mario <[email protected]>

* bump version to 2.11.5

Signed-off-by: mario <[email protected]>

* validate all strings before saving to db

Signed-off-by: mario <[email protected]>

---------

Signed-off-by: mario <[email protected]>

Author: mariobassem

indexer/chart/Chart.yaml

@@ -2,4 +2,4 @@ name: tfchainindexer
 description: Helm Chart for the tfchain hydra indexer
 version: 2.7.7
 apiVersion: v2
-appVersion: '2.11.3'
+appVersion: '2.11.5'

indexer/docker-compose.yml

@@ -27,7 +27,9 @@ services:
         "--out",
         "postgres://root@db:26257/defaultdb",
         "--types-bundle",
-        "/configs/typesBundle.json"
+        "/configs/typesBundle.json",
+        "--start-block",
+        "${START_HEIGHT}"
       ]
 
   gateway:

package-lock.json

@@ -1,7 +1,7 @@
 {
   "name": "graphql_tfgrid",
   "private": "true",
-  "version": "2.11.3",
+  "version": "2.11.5",
   "description": "GraphQL server and Substrate indexer. Generated with ♥ by Hydra-CLI",
   "author": "",
   "license": "ISC",

package.json

@@ -2,4 +2,4 @@ apiVersion: v2
 name: tfchain-processor
 description: A chart for the tfchain graphql processor and query node
 version: 1.0.6
-appVersion: '2.11.3'
+appVersion: '2.11.5'

processor-chart/Chart.yaml

@@ -15,6 +15,7 @@ import {
   SmartContractModuleNruConsumptionReportReceivedEvent, SmartContractModuleRentContractCanceledEvent,
   SmartContractModuleContractGracePeriodStartedEvent, SmartContractModuleContractGracePeriodEndedEvent
 } from "../types/events";
+import { validateString } from "./nodes"
 
 export async function contractCreated(
   ctx: Ctx,
@@ -47,7 +48,7 @@ export async function contractCreated(
     let newNameContract = new NameContract()
     newNameContract.id = item.event.id
     contract = contractEvent.contractType.value
-    newNameContract.name = contract.name.toString()
+    newNameContract.name = validateString(ctx, contract.name.toString())
     newNameContract.contractID = contractEvent.contractId
     newNameContract.gridVersion = contractEvent.version
     newNameContract.twinID = contractEvent.twinId
@@ -71,16 +72,9 @@ export async function contractCreated(
     newNodeContract.nodeID = contract.nodeId
     newNodeContract.numberOfPublicIPs = contract.publicIps
 
-    if (contract.deploymentData.toString().indexOf('\x00') >= 0) {
-      newNodeContract.deploymentData = ""
-    } else {
-      newNodeContract.deploymentData = contract.deploymentData.toString()
-    }
-    if (contract.deploymentHash.toString().indexOf('\x00') >= 0) {
-      newNodeContract.deploymentHash = ""
-    } else {
-      newNodeContract.deploymentHash = contract.deploymentHash.toString()
-    }
+
+    newNodeContract.deploymentData = validateString(ctx, contract.deploymentData.toString())
+    newNodeContract.deploymentHash = validateString(ctx, contract.deploymentHash.toString())
 
     // newNodeContract.deploymentHash = contract.deploymentHash.toString()
     newNodeContract.state = state
@@ -164,16 +158,16 @@ export async function contractUpdated(
 
   const SavedNodeContract = await ctx.store.get(NodeContract, { where: { contractID: contractEvent.contractId } })
   if (SavedNodeContract) {
-    await updateNodeContract(contractEvent, SavedNodeContract, ctx.store)
+    await updateNodeContract(ctx, contractEvent, SavedNodeContract, ctx.store)
   }
 
   const SavedNameContract = await ctx.store.get(NameContract, { where: { contractID: contractEvent.contractId } })
   if (SavedNameContract) {
-    await updateNameContract(contractEvent, SavedNameContract, ctx.store)
+    await updateNameContract(ctx, contractEvent, SavedNameContract, ctx.store)
   }
 }
 
-async function updateNodeContract(ctr: any, contract: NodeContract, store: Store) {
+async function updateNodeContract(ctx: Ctx, ctr: any, contract: NodeContract, store: Store) {
   if (ctr.contractType.__kind !== "NodeContract") return
 
   const parsedNodeContract = ctr.contractType.value
@@ -184,16 +178,9 @@ async function updateNodeContract(ctr: any, contract: NodeContract, store: Store
   contract.nodeID = parsedNodeContract.nodeId
   contract.numberOfPublicIPs = parsedNodeContract.publicIps
 
-  if (contract.deploymentData.toString().indexOf('\x00') >= 0) {
-    contract.deploymentData = ""
-  } else {
-    contract.deploymentData = contract.deploymentData.toString()
-  }
-  if (contract.deploymentHash.toString().indexOf('\x00') >= 0) {
-    contract.deploymentHash = ""
-  } else {
-    contract.deploymentHash = contract.deploymentHash.toString()
-  }
+
+  contract.deploymentData = validateString(ctx, contract.deploymentData.toString())
+  contract.deploymentHash = validateString(ctx, contract.deploymentHash.toString())
 
   let state = ContractState.OutOfFunds
   switch (ctr.state.__kind) {
@@ -208,15 +195,15 @@ async function updateNodeContract(ctr: any, contract: NodeContract, store: Store
   await store.save<NodeContract>(contract)
 }
 
-async function updateNameContract(ctr: any, contract: NameContract, store: Store) {
+async function updateNameContract(ctx: Ctx, ctr: any, contract: NameContract, store: Store) {
   if (ctr.contractType.__kind !== "NameContract") return
 
   const parsedNameContract = ctr.contractType.value
 
   contract.contractID = ctr.contractId
   contract.gridVersion = ctr.version
   contract.twinID = ctr.twinId
-  contract.name = parsedNameContract.name.toString()
+  contract.name = validateString(ctx, parsedNameContract.name.toString())
 
   let state = ContractState.OutOfFunds
   switch (ctr.state.__kind) {

src/mappings/contracts.ts

@@ -4,6 +4,7 @@ import { EventItem } from '@subsquid/substrate-processor/lib/interfaces/dataSele
 
 import { Ctx } from '../processor'
 import * as v63 from '../types/v63'
+import { validateString } from "./nodes"
 
 export class FarmWithIPs {
   constructor(farmID: number, ips: PublicIp[]) {
@@ -41,7 +42,7 @@ export async function farmStored(
   newFarm.id = item.event.id
   newFarm.gridVersion = farmStoredEventParsed.version
   newFarm.farmID = farmStoredEventParsed.id
-  newFarm.name = farmStoredEventParsed.name.toString()
+  newFarm.name = validateString(ctx, farmStoredEventParsed.name.toString())
   newFarm.twinID = farmStoredEventParsed.twinId
   newFarm.pricingPolicyID = farmStoredEventParsed.pricingPolicyId
   newFarm.dedicatedFarm = false
@@ -56,15 +57,8 @@ export async function farmStored(
 
     newIP.id = item.event.id
 
-    if (ip.ip.toString().indexOf('\x00') >= 0) {
-      return
-    }
-    newIP.ip = ip.ip.toString()
-
-    if (ip.gateway.toString().indexOf('\x00') >= 0) {
-      return
-    }
-    newIP.gateway = ip.gateway.toString()
+    newIP.ip = validateString(ctx, ip.ip.toString())
+    newIP.gateway = validateString(ctx, ip.gateway.toString())
 
     newIP.contractId = ip.contractId
     newIP.farm = newFarm
@@ -114,7 +108,7 @@ export async function farmUpdated(
   if (!savedFarm) return
 
   savedFarm.gridVersion = farmUpdatedEventParsed.version
-  savedFarm.name = farmUpdatedEventParsed.name.toString()
+  savedFarm.name = validateString(ctx, farmUpdatedEventParsed.name.toString())
   savedFarm.twinID = farmUpdatedEventParsed.twinId
   savedFarm.pricingPolicyID = farmUpdatedEventParsed.pricingPolicyId
   savedFarm.certification = certification
@@ -128,14 +122,14 @@ export async function farmUpdated(
     const savedIP = await ctx.store.get(PublicIp, { where: { ip: ip.ip.toString() }, relations: { farm: true } })
     // ip is already there in storage, don't save it again
     if (savedIP) {
-      savedIP.ip = ip.ip.toString()
-      savedIP.gateway = ip.gateway.toString()
+      savedIP.ip = validateString(ctx, ip.ip.toString()) // not effective, but for since we already check for \x00
+      savedIP.gateway = validateString(ctx, ip.gateway.toString())
       await ctx.store.save<PublicIp>(savedIP)
     } else {
       const newIP = new PublicIp()
       newIP.id = item.event.id
-      newIP.ip = ip.ip.toString()
-      newIP.gateway = ip.gateway.toString()
+      newIP.ip = validateString(ctx, ip.ip.toString())
+      newIP.gateway = validateString(ctx, ip.gateway.toString())
       newIP.contractId = ip.contractId
       newIP.farm = savedFarm
 
@@ -151,7 +145,7 @@ export async function farmUpdated(
 
   const publicIPsOfFarm = await ctx.store.find<PublicIp>(PublicIp, { where: { farm: { id: savedFarm.id } }, relations: { farm: true } })
   publicIPsOfFarm.forEach(async ip => {
-    if (eventPublicIPs.filter(eventIp => eventIp.ip.toString() === ip.ip).length === 0) {
+    if (eventPublicIPs.filter(eventIp => validateString(ctx, eventIp.ip.toString()) === ip.ip).length === 0) {
       // IP got removed from farm
       await ctx.store.remove<PublicIp>(ip)
     }
@@ -188,7 +182,7 @@ export async function farmPayoutV2AddressRegistered(
   if (savedFarm) {
     let address = ''
     if (!stellarAddress.includes(0)) {
-      address = stellarAddress.toString()
+      address = validateString(ctx, stellarAddress.toString())
     }
 
     savedFarm.stellarAddress = address
@@ -252,7 +246,7 @@ export async function farmCreateOrUpdateOrDelete(ctx: Ctx): Promise<[Farm[], Far
         newFarm.id = eventID
         newFarm.gridVersion = farmStoredEventParsed.version
         newFarm.farmID = farmStoredEventParsed.id
-        newFarm.name = farmStoredEventParsed.name.toString()
+        newFarm.name = validateString(ctx, farmStoredEventParsed.name.toString())
         newFarm.twinID = farmStoredEventParsed.twinId
         newFarm.pricingPolicyID = farmStoredEventParsed.pricingPolicyId
         newFarm.dedicatedFarm = false
@@ -267,15 +261,8 @@ export async function farmCreateOrUpdateOrDelete(ctx: Ctx): Promise<[Farm[], Far
 
           newIP.id = eventID
 
-          if (ip.ip.toString().indexOf('\x00') >= 0) {
-            return
-          }
-          newIP.ip = ip.ip.toString()
-
-          if (ip.gateway.toString().indexOf('\x00') >= 0) {
-            return
-          }
-          newIP.gateway = ip.gateway.toString()
+          newIP.ip = validateString(ctx, ip.ip.toString())
+          newIP.gateway = validateString(ctx, ip.gateway.toString())
 
           newIP.contractId = ip.contractId
           newIP.farm = newFarm
@@ -313,7 +300,7 @@ export async function farmCreateOrUpdateOrDelete(ctx: Ctx): Promise<[Farm[], Far
         if (foundInNewListIndex != -1) {
           const savedFarm: Farm = newFarms[foundInNewListIndex]
           savedFarm.gridVersion = farmUpdatedEventParsed.version
-          savedFarm.name = farmUpdatedEventParsed.name.toString()
+          savedFarm.name = validateString(ctx, farmUpdatedEventParsed.name.toString())
           savedFarm.twinID = farmUpdatedEventParsed.twinId
           savedFarm.pricingPolicyID = farmUpdatedEventParsed.pricingPolicyId
           // const pubIps = updatePublicIPs(ctx, farmUpdatedEventParsed.publicIps, eventID, savedFarm)
@@ -328,7 +315,7 @@ export async function farmCreateOrUpdateOrDelete(ctx: Ctx): Promise<[Farm[], Far
         if (foundInUpdatedListIndex != -1) {
           let savedFarm: Farm = updatedFarms[foundInUpdatedListIndex]
           savedFarm.gridVersion = farmUpdatedEventParsed.version
-          savedFarm.name = farmUpdatedEventParsed.name.toString()
+          savedFarm.name = validateString(ctx, farmUpdatedEventParsed.name.toString())
           savedFarm.twinID = farmUpdatedEventParsed.twinId
           savedFarm.pricingPolicyID = farmUpdatedEventParsed.pricingPolicyId
           updatedFarms[foundInUpdatedListIndex] = savedFarm
@@ -342,7 +329,7 @@ export async function farmCreateOrUpdateOrDelete(ctx: Ctx): Promise<[Farm[], Far
         if (!savedFarm) continue
 
         savedFarm.gridVersion = farmUpdatedEventParsed.version
-        savedFarm.name = farmUpdatedEventParsed.name.toString()
+        savedFarm.name = validateString(ctx, farmUpdatedEventParsed.name.toString())
         savedFarm.twinID = farmUpdatedEventParsed.twinId
         savedFarm.pricingPolicyID = farmUpdatedEventParsed.pricingPolicyId
 
@@ -379,13 +366,13 @@ function getPublicIPs(ctx: Ctx, farm: Farm, savedFarmIps: FarmWithIPs[], newIps:
     // console.log(`found index: ${foundIdx}`)
     if (foundIdx !== -1) {
       toModify[0].publicIPs[foundIdx].contractId = ip.contractId
-      toModify[0].publicIPs[foundIdx].ip = ip.ip.toString()
-      toModify[0].publicIPs[foundIdx].gateway = ip.gateway.toString()
+      toModify[0].publicIPs[foundIdx].ip = validateString(ctx, ip.ip.toString())
+      toModify[0].publicIPs[foundIdx].gateway = validateString(ctx, ip.gateway.toString())
     } else {
       const newIP = new PublicIp()
       newIP.id = eventID
       newIP.ip = ip.ip.toString()
-      newIP.gateway = ip.gateway.toString()
+      newIP.gateway = validateString(ctx, ip.gateway.toString())
       newIP.contractId = ip.contractId
       newIP.farm = farm
       // ctx.log.info(`saving new ip: ${newIP.ip}`)