DON'T MERGE - Use db to persist cache storage

packages/api/package.json

@@ -23,7 +23,8 @@
     "lint:fix": "eslint src --fix && pnpm format",
     "format": "prettier --write src",
     "migration:generate": "tsx ./node_modules/typeorm/cli.js migration:generate ./src/migrations/migration -d ./src/db.ts",
-    "migration:run": "tsx ./node_modules/typeorm/cli.js migration:run -d ./src/db.ts"
+    "migration:run": "tsx ./node_modules/typeorm/cli.js migration:run -d ./src/db.ts",
+    "migration:revert": "tsx ./node_modules/typeorm/cli.js migration:revert -d ./src/db.ts"
   },
   "keywords": [],
   "dependencies": {

packages/api/src/db.ts

@@ -9,6 +9,7 @@ import { MarketPrice } from './models/MarketPrice';
 import { RenderJob } from './models/RenderJob';
 import { CollateralTransfer } from './models/CollateralTransfer';
 import { Resource } from './models/Resource';
+import { PerformanceCache } from './models/PerformanceCache';
 import dotenv from 'dotenv';
 import { fileURLToPath } from 'url';
 import path, { dirname } from 'path';
@@ -39,6 +40,7 @@ const devDataSource: DataSource = new DataSource({
     RenderJob,
     CollateralTransfer,
     Resource,
+    PerformanceCache,
   ],
 });
 
@@ -60,6 +62,7 @@ const postgresDataSource: DataSource = new DataSource({
     RenderJob,
     CollateralTransfer,
     Resource,
+    PerformanceCache,
   ],
 });
 
@@ -90,5 +93,7 @@ export const marketPriceRepository = dataSource.getRepository(MarketPrice);
 export const renderJobRepository = dataSource.getRepository(RenderJob);
 export const collateralTransferRepository =
   dataSource.getRepository(CollateralTransfer);
+export const performanceCacheRepository =
+  dataSource.getRepository(PerformanceCache);
 
 export default dataSource;

packages/api/src/migrations/1741790918671-migration.ts

@@ -0,0 +1,15 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class Migration1741790918671 implements MigrationInterface {
+  name = 'Migration1741790918671';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "performance_cache" ("id" SERIAL NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "resourceSlug" character varying NOT NULL, "interval" character varying NOT NULL, "jsonSection" character varying NOT NULL, "storageVersion" character varying, "latestTimestamp" character varying NOT NULL, "storage" jsonb NOT NULL, CONSTRAINT "PK_93aab8268ebc22e5129c77cdc2d" PRIMARY KEY ("id"))`
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP TABLE "performance_cache"`);
+  }
+}

packages/api/src/models/PerformanceCache.ts

@@ -0,0 +1,33 @@
+import {
+  Entity,
+  PrimaryGeneratedColumn,
+  Column,
+  CreateDateColumn,
+} from 'typeorm';
+
+@Entity()
+export class PerformanceCache {
+  @PrimaryGeneratedColumn()
+  id: number;
+
+  @CreateDateColumn()
+  createdAt: Date;
+
+  @Column({ type: 'varchar' })
+  resourceSlug: string;
+
+  @Column({ type: 'varchar' })
+  interval: number;
+
+  @Column({ type: 'varchar' })
+  jsonSection: string;
+
+  @Column({ type: 'varchar', nullable: true })
+  storageVersion: string;
+
+  @Column({ type: 'varchar' })
+  latestTimestamp: number;
+
+  @Column({ type: 'jsonb' })
+  storage: string;
+}

packages/api/src/performance/helper.ts

@@ -1,60 +1,47 @@
 import { IntervalStore } from './types';
-import * as fs from 'fs';
-import * as path from 'path';
+import { performanceCacheRepository } from '../db';
 
-const FILE_VERSION = 1;
+const STORAGE_VERSION = '1';
 
-export async function saveStorageToFile(
+export async function persistStorage(
   storage: IntervalStore,
   latestResourceTimestamp: number,
   latestMarketTimestamp: number,
   resourceSlug: string,
   resourceName: string,
-  sectionName: string
+  interval: number,
+  jsonSection: string
 ): Promise<undefined> {
   if (process.env.SAVE_STORAGE !== 'true') {
     return;
   }
 
   console.time(
-    `  ResourcePerformance - processResourceData.${resourceName}.${sectionName}.saveStorage`
+    `  ResourcePerformance - processResourceData.${resourceName}.${interval}.${jsonSection}.saveStorage`
   );
-  const storageDir = process.env.STORAGE_PATH;
-  if (!storageDir) {
-    throw new Error('STORAGE_PATH is not set');
-  }
 
-  if (!fs.existsSync(storageDir)) {
-    fs.mkdirSync(storageDir, { recursive: true });
-  }
-
-  const filename = path.join(
-    storageDir,
-    `${resourceSlug}-${sectionName}-storage.json`
-  );
-  await fs.promises.writeFile(
-    filename,
-    JSON.stringify(
-      {
-        fileVersion: FILE_VERSION,
-        latestResourceTimestamp,
-        latestMarketTimestamp,
-        store: storage,
-      },
-      (key, value) => (typeof value === 'bigint' ? value.toString() : value),
-      2
-    )
-  );
+  // Create or update the cache entry
+  await performanceCacheRepository.save({
+    resourceSlug,
+    interval,
+    jsonSection,
+    storageVersion: STORAGE_VERSION,
+    latestTimestamp,
+    storage: JSON.stringify(storage, (key, value) =>
+      typeof value === 'bigint' ? value.toString() : value
+    ),
+  });
 
   console.timeEnd(
-    `  ResourcePerformance - processResourceData.${resourceName}.${sectionName}.saveStorage`
+    `  ResourcePerformance - processResourceData.${resourceName}.${interval}.${jsonSection}.saveStorage`
   );
-  console.log(`  ResourcePerformance --> Saved storage to ${filename}`);
+  console.log(`  ResourcePerformance --> Saved storage to database`);
 }
 
-export async function loadStorageFromFile(
+export async function restorePersistedStorage(
   resourceSlug: string,
   resourceName: string,
+  interval: number,
   sectionName: string
 ): Promise<
   | {
@@ -69,32 +56,35 @@ export async function loadStorageFromFile(
   }
 
   console.time(
-    `  ResourcePerformance - processResourceData.${resourceName}.${sectionName}.loadStorage`
+    `  ResourcePerformance - processResourceData.${resourceName}.${interval}.${jsonSection}.loadStorage`
   );
-  const storageDir = process.env.STORAGE_PATH;
-  if (!storageDir) {
-    throw new Error('STORAGE_PATH is not set');
-  }
 
-  const filename = path.join(
-    storageDir,
-    `${resourceSlug}-${sectionName}-storage.json`
-  );
-  if (!fs.existsSync(filename)) {
-    console.log(`!! Storage file ${filename} does not exist`);
+  const cacheEntry = await performanceCacheRepository.findOne({
+    where: {
+      resourceSlug,
+      interval,
+      jsonSection,
+      storageVersion: STORAGE_VERSION,
+    },
+    order: {
+      createdAt: 'DESC',
+    },
+  });
+
+  if (!cacheEntry) {
+    console.log(
+      `!! Storage entry for ${resourceSlug}-${interval}-${jsonSection} does not exist`
+    );
     return undefined;
   }
 
-  try {
-    const fileContent = await fs.promises.readFile(filename, 'utf-8');
-    const storage = JSON.parse(fileContent, (key, value) => {
-      // Convert string numbers that might be bigints back to bigint
-      if (typeof value === 'string' && /^\d+$/.test(value)) {
-        try {
-          return BigInt(value);
-        } catch {
-          return value;
-        }
+  const storage = JSON.parse(cacheEntry.storage, (key, value) => {
+    // Convert string numbers that might be bigints back to bigint
+    if (typeof value === 'string' && /^\d+$/.test(value)) {
+      try {
+        return BigInt(value);
+      } catch {
+        return value;
       }
       return value;
     }) as {
@@ -113,6 +103,18 @@ export async function loadStorageFromFile(
       );
       return undefined;
     }
+    return value;
+  });
+
+  console.timeEnd(
+    `  ResourcePerformance - processResourceData.${resourceName}.${interval}.${jsonSection}.loadStorage`
+  );
+  console.log(`  ResourcePerformance - -> Loaded storage from database`);
+
+  return {
+    latestTimestamp: cacheEntry.latestTimestamp,
+    store: storage,
+  };
     return {
       latestResourceTimestamp: storage.latestResourceTimestamp,
       latestMarketTimestamp: storage.latestMarketTimestamp,
@@ -127,29 +129,13 @@ export async function loadStorageFromFile(
   }
 }
 
-export async function clearStorageFiles(): Promise<void> {
-  const storageDir = process.env.STORAGE_PATH;
-  if (!storageDir) {
-    throw new Error('STORAGE_PATH is not set');
-  }
-
-  if (!fs.existsSync(storageDir)) {
-    return; // Nothing to clear
-  }
+export async function clearPersistedStore(): Promise<void> {
+  console.time('  ResourcePerformance - clearStorage');
 
-  console.time('  ResourcePerformance - clearStorageFiles');
+  await performanceCacheRepository.delete({});
 
-  const files = await fs.promises.readdir(storageDir);
-  for (const file of files) {
-    if (file.endsWith('-storage.json')) {
-      await fs.promises.unlink(path.join(storageDir, file));
-    }
-  }
-
-  console.timeEnd('  ResourcePerformance - clearStorageFiles');
-  console.log(
-    `  ResourcePerformance --> Cleared ${files.length} storage files`
-  );
+  console.timeEnd('  ResourcePerformance - clearStorage');
+  console.log('  ResourcePerformance --> Cleared performance cache storage');
 }
 
 export function maxBigInt(a: bigint, b: bigint) {

packages/api/src/performance/resourcePerformance.ts

@@ -27,10 +27,10 @@ import {
 } from './constants';
 
 import {
-  loadStorageFromFile,
+  restorePersistedStorage,
   maxBigInt,
   minBigInt,
-  saveStorageToFile,
+  persistStorage,
 } from './helper';
 
 export class ResourcePerformance {
@@ -447,13 +447,14 @@ export class ResourcePerformance {
 
     for (const interval of this.intervals) {
       // Interval resource store
-      await saveStorageToFile(
+      await persistStorage(
         storage[interval],
         lastResourceTimestampProcessed,
         lastMarketTimestampProcessed,
         resourceSlug,
         resourceName,
-        interval.toString()
+        interval,
+        'intervalStore'
       );
     }
   }
@@ -472,10 +473,11 @@ export class ResourcePerformance {
     let latestResourceTimestamp = 0;
     let latestMarketTimestamp = 0;
     for (const interval of this.intervals) {
-      const storageInterval = await loadStorageFromFile(
+      const storageInterval = await restorePersistedStorage(
         resourceSlug,
         resourceName,
-        interval.toString()
+        interval,
+        'intervalStore'
       );
       if (!storageInterval) {
         return undefined;

packages/api/src/performance/resourcePerformanceManager.ts

@@ -1,6 +1,6 @@
 import { ResourcePerformance } from './resourcePerformance';
 import { Resource } from 'src/models/Resource';
-import { clearStorageFiles } from './helper';
+import { clearPersistedStore } from './helper';
 export class ResourcePerformanceManager {
   private static _instance: ResourcePerformanceManager;
   private static _initialized: boolean = false;
@@ -143,7 +143,7 @@ export class ResourcePerformanceManager {
 
     // Remove files from disk (hard init will recreate them)
     if (hardInitialize) {
-      await clearStorageFiles();
+      await clearPersistedStore();
     }
 
     // Get rid of existing resource performances and start fresh