feat(sdk,web-client): add auth token support for subgraph API

sdk/.env.example

@@ -1,7 +1,10 @@
-CONTRACT_ADDRESS=0x0810ea246310b737A8a3aE048Ad7ef1Ac27E9932
-GENESIS_BLOCK_NUMBER=12725104
+CONTRACT_ADDRESS=0xF34d1BF74B98f0a67621F14633E4c807457E1400
+GENESIS_BLOCK_NUMBER=69237935
 MIN_REWARD=0.1
-SUBGRAPH_ID=QmTATJr6FZiQKDS8j9tfuXeH4tvzDWZWqzJewSA1L5Dtip
+SUBGRAPH_ID=8ZnqbDNeqA83VE2SiZnHmqMHWUEe4At3hE8nVbDN71bn
+SUBGRAPH_AUTH_TOKEN=your-auth-token-here
 INTERVAL=10
 PRIVATE_KEY=5e...b
-RPC_URL=https://polygon-mumbai.infura.io/v3/...
+RPC_URL=https://polygon-rpc.com
+# PAGE_SIZE: Number of blocks to scan at once (default: 1000, reduce for restrictive RPCs like polygon-rpc.com)
+# PAGE_SIZE=100

sdk/scripts/run.ts

@@ -35,6 +35,8 @@ async function main() {
     const [tree, startingBlock, insertedQueueIds] = await coldStart(
         env.SUBGRAPH_ID,
         env.GENESIS_BLOCK_NUMBER,
+        log,
+        env.SUBGRAPH_AUTH_TOKEN,
     );
 
     const db = new MemCache(insertedQueueIds);
@@ -43,6 +45,8 @@ async function main() {
         env.CONTRACT_ADDRESS,
         startingBlock,
         db,
+        log,
+        env.PAGE_SIZE,
     );
 
     const batchProcessing = new BatchProcessing(tree, scanner, db);

sdk/src/env.ts

@@ -5,12 +5,14 @@ import {providers, utils, Wallet} from 'ethers';
 
 import {EnvVariables} from './types';
 
-export const requiredVars: Array<keyof EnvVariables> = [
+type RequiredEnvVars = Exclude<keyof EnvVariables, 'PAGE_SIZE'>;
+export const requiredVars: Array<RequiredEnvVars> = [
     'INTERVAL',
     'PRIVATE_KEY',
     'RPC_URL',
     'CONTRACT_ADDRESS',
     'SUBGRAPH_ID',
+    'SUBGRAPH_AUTH_TOKEN',
     'GENESIS_BLOCK_NUMBER',
     'MIN_REWARD',
 ];
@@ -39,6 +41,9 @@ export function parseEnvVariables(env: NodeJS.ProcessEnv): EnvVariables {
         }
     }
 
+    // Optional PAGE_SIZE with default of 1000
+    parsed.PAGE_SIZE = env.PAGE_SIZE ? parseInt(env.PAGE_SIZE) : 1000;
+
     return parsed as EnvVariables;
 }
 

sdk/src/event-scanner.ts

@@ -10,21 +10,23 @@ import {LogFn, log as defaultLog} from './logging';
 import {MemCache} from './mem-cache';
 import {BusBatchOnboardedEventRecord, UtxoBusQueuedEventRecord} from './types';
 
-const PAGE_SIZE = 1_000; // Amount of blocks to scan at once
+const DEFAULT_PAGE_SIZE = 1000;
 
 export class EventScanner {
     private contract: ForestTree;
     private db: MemCache;
     private filters: EventFilter[];
     private startingBlock: number;
     private log: LogFn;
+    private pageSize: number;
 
     constructor(
         rpcEndpoint: string,
         address: string,
         startingBlock: number,
         db: MemCache,
         log: LogFn = defaultLog,
+        pageSize: number = DEFAULT_PAGE_SIZE,
     ) {
         this.contract = initializeReadOnlyBusContract(rpcEndpoint, address);
         this.filters = [
@@ -34,6 +36,7 @@ export class EventScanner {
         this.db = db;
         this.startingBlock = startingBlock;
         this.log = log;
+        this.pageSize = pageSize;
     }
 
     public async scan(): Promise<void> {
@@ -45,8 +48,8 @@ export class EventScanner {
             const totalBlocks = currentBlock - this.startingBlock;
             let scannedBlocks = 0;
 
-            for (let i = this.startingBlock; i < currentBlock; i += PAGE_SIZE) {
-                const endBlock = Math.min(i + PAGE_SIZE, currentBlock);
+            for (let i = this.startingBlock; i < currentBlock; i += this.pageSize) {
+                const endBlock = Math.min(i + this.pageSize, currentBlock);
                 const progress = Math.floor((scannedBlocks / totalBlocks) * 100);
                 this.log(`Scanning block range ${i} - ${endBlock} [${progress}%]`);
                 await this.scanBlockRangeAndSave(i, endBlock);

sdk/src/runner.ts

@@ -78,11 +78,15 @@ export async function coldStart(
     subgraphId: string,
     genesisBlockNumber: number,
     log: LogFn = defaultLog,
+    subgraphAuthToken?: string,
 ): Promise<[MinerTree, number, number[]]> {
     log('Starting cold start');
-    const [tree, filledBatches] = await initializeMinerTree(subgraphId);
+    const [tree, filledBatches] = await initializeMinerTree(
+        subgraphId,
+        subgraphAuthToken,
+    );
     const insertedQueueIds = filledBatches.map(batch => Number(batch.queueId));
-    const startingBlock = await getOldestBlockNumber(subgraphId);
+    const startingBlock = await getOldestBlockNumber(subgraphId, subgraphAuthToken);
 
     const blockNumber = Math.max(
         genesisBlockNumber,
@@ -100,9 +104,10 @@ export async function coldStart(
 // Initializes MinerTree and returns sorted onboarded batches
 async function initializeMinerTree(
     subgraphId: string,
+    subgraphAuthToken?: string,
 ): Promise<[MinerTree, BusBatchOnboardedEvent[]]> {
     const tree = new MinerTree();
-    const subgraph = new Subgraph(subgraphId);
+    const subgraph = new Subgraph(subgraphId, subgraphAuthToken);
     const filledBranches = await subgraph.getFilledBranches();
     filledBranches.sort((a, b) => a.branchIndex - b.branchIndex);
     filledBranches.forEach(branch => {
@@ -125,8 +130,9 @@ async function initializeMinerTree(
 // Gets oldest block number excluding inserted queueIds
 async function getOldestBlockNumber(
     subgraphId: string,
+    subgraphAuthToken?: string,
 ): Promise<number | null> {
-    const subgraph = new Subgraph(subgraphId);
+    const subgraph = new Subgraph(subgraphId, subgraphAuthToken);
     return subgraph.getOldestBlockNumber();
 }
 

sdk/src/subgraph.ts

@@ -8,8 +8,20 @@ import {BusBatchOnboardedEvent, BranchFilledEvent} from './types';
 const PAGINATION_WINDOW_SIZE = 1000;
 
 // Handles all Subgraph API requests
-async function requestSubgraph(url: string, query: string): Promise<any> {
-    const response = await axios.post(url, {query});
+async function requestSubgraph(
+    url: string,
+    query: string,
+    authToken?: string,
+): Promise<any> {
+    const headers: Record<string, string> = {
+        'Content-Type': 'application/json',
+    };
+
+    if (authToken) {
+        headers['Authorization'] = `Bearer ${authToken}`;
+    }
+
+    const response = await axios.post(url, {query}, {headers});
 
     if (response.data.errors?.[0]?.message || response.status !== 200) {
         console.error(response.data.errors?.[0]?.message);
@@ -40,13 +52,19 @@ class QueryBuilder {
 
 export class Subgraph {
     private readonly url: string;
+    private readonly authToken?: string;
 
-    constructor(id: string) {
-        this.url = `https://api.thegraph.com/subgraphs/id/${id}`;
+    constructor(id: string, authToken?: string) {
+        this.url = `https://gateway.thegraph.com/api/subgraphs/id/${id}`;
+        this.authToken = authToken;
     }
 
     private async fetchFromSubgraph(queryBuilder: QueryBuilder): Promise<any> {
-        return await requestSubgraph(this.url, queryBuilder.build());
+        return await requestSubgraph(
+            this.url,
+            queryBuilder.build(),
+            this.authToken,
+        );
     }
 
     public async getFilledBranches(): Promise<BranchFilledEvent[]> {

sdk/src/types.ts

@@ -65,8 +65,10 @@ export type EnvVariables = {
     CONTRACT_ADDRESS: string;
     INTERVAL: number;
     SUBGRAPH_ID: string;
+    SUBGRAPH_AUTH_TOKEN: string;
     GENESIS_BLOCK_NUMBER: number;
     MIN_REWARD: string;
+    PAGE_SIZE: number;
 };
 
 export type ProofInputs = {

web-client/.env.example

@@ -1,7 +1,8 @@
-CONTRACT_ADDRESS=0x0810ea246310b737A8a3aE048Ad7ef1Ac27E9932
-GENESIS_BLOCK_NUMBER=12725104
+CONTRACT_ADDRESS=0xF34d1BF74B98f0a67621F14633E4c807457E1400
+GENESIS_BLOCK_NUMBER=69237935
 MIN_REWARD=0.001
 SUBGRAPH_ID=QmTATJr6FZiQKDS8j9tfuXeH4tvzDWZWqzJewSA1L5Dtip
+SUBGRAPH_AUTH_TOKEN=your-auth-token-here
 ZKP_TOKEN_ADDRESS=0x9C56E89D8Aa0d4A1fB769DfbEa80D6C29e5A2893
 INTERVAL=5
 RPC_URL=https://polygon-mumbai.gateway.tenderly.co

web-client/src/components/MinerClientParams/index.tsx

@@ -26,6 +26,7 @@ const MinerClientParamsForm = () => {
         rpcUrl: string;
         address: string;
         subgraphId: string;
+        subgraphAuthToken: string;
         genesisBlockNumber: string;
         minReward: string;
     }>({
@@ -34,6 +35,7 @@ const MinerClientParamsForm = () => {
         rpcUrl: env.RPC_URL || '',
         address: env.CONTRACT_ADDRESS || '',
         subgraphId: env.SUBGRAPH_ID || '',
+        subgraphAuthToken: env.SUBGRAPH_AUTH_TOKEN || '',
         genesisBlockNumber: env.GENESIS_BLOCK_NUMBER || '',
         minReward: env.MIN_REWARD || '0.001',
     });
@@ -87,6 +89,10 @@ const MinerClientParamsForm = () => {
                 condition: !state.subgraphId,
                 message: 'Subgraph ID field is required',
             },
+            {
+                condition: !state.subgraphAuthToken,
+                message: 'Subgraph Auth Token field is required',
+            },
         ];
 
         for (const validation of validations) {
@@ -128,6 +134,17 @@ const MinerClientParamsForm = () => {
                         />
                     </div>
                 </div>
+                <div className="flex items-end justify-between space-x-5 mb-4">
+                    <div className="w-full">
+                        <Input
+                            label="Subgraph Auth Token"
+                            value={state.subgraphAuthToken}
+                            name="subgraphAuthToken"
+                            onChange={updateStateHandler}
+                            type="password"
+                        />
+                    </div>
+                </div>
 
                 <div className="flex items-end justify-between space-x-5 mb-4">
                     <div className="w-full">
@@ -209,6 +226,7 @@ const MinerClientParamsForm = () => {
                             rpcUrl: state.rpcUrl,
                             address: state.address,
                             subgraphId: state.subgraphId,
+                            subgraphAuthToken: state.subgraphAuthToken,
                             genesisBlockNumber,
                             minReward: state.minReward,
                         };

web-client/src/redux/slices/miner/minerParams.ts

@@ -7,6 +7,7 @@ const initialState: MinerClientParams = {
     rpcUrl: '',
     address: '',
     subgraphId: '',
+    subgraphAuthToken: '',
     genesisBlockNumber: 0,
     minReward: '0.001',
 };

web-client/src/services/env.ts

@@ -3,6 +3,7 @@ export const env = Object.freeze({
     RPC_URL: process.env.RPC_URL!,
     CONTRACT_ADDRESS: process.env.CONTRACT_ADDRESS!,
     SUBGRAPH_ID: process.env.SUBGRAPH_ID!,
+    SUBGRAPH_AUTH_TOKEN: process.env.SUBGRAPH_AUTH_TOKEN!,
     ZKP_TOKEN_ADDRESS: process.env.ZKP_TOKEN_ADDRESS!,
     GENESIS_BLOCK_NUMBER: process.env.GENESIS_BLOCK_NUMBER!,
     MIN_REWARD: process.env.MIN_REWARD!,
@@ -11,6 +12,7 @@ export const env = Object.freeze({
 export const requiredEnvVars: Array<keyof typeof env> = [
     'CONTRACT_ADDRESS',
     'SUBGRAPH_ID',
+    'SUBGRAPH_AUTH_TOKEN',
     'GENESIS_BLOCK_NUMBER',
     'INTERVAL',
     'RPC_URL',

web-client/src/services/miner.worker.ts

@@ -41,6 +41,7 @@ async function handleMining(eventData: MinerClientParams) {
         interval,
         address,
         subgraphId,
+        subgraphAuthToken,
         genesisBlockNumber,
         minReward,
     } = eventData;
@@ -50,6 +51,7 @@ async function handleMining(eventData: MinerClientParams) {
         subgraphId,
         Number(env.GENESIS_BLOCK_NUMBER),
         notify,
+        subgraphAuthToken,
     );
     const db = new MemCache(insertedQueueIds, notify);
     const scanner = new EventScanner(

web-client/src/types/worker.ts

@@ -12,6 +12,7 @@ export type MinerClientParams = {
     rpcUrl: string;
     address: string;
     subgraphId: string;
+    subgraphAuthToken: string;
     genesisBlockNumber: number;
     minReward: string;
 };