Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use database to persist cache store #360

Merged
merged 6 commits into from
Mar 18, 2025
Merged

Use database to persist cache store #360

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2754570
Use database to persist cache store
llamalm Mar 11, 2025
a5eebf5
Merge branch 'main' into use-db-to-persist-cache-storage
llamalm Mar 12, 2025
0b8f2a6
update migrations
llamalm Mar 12, 2025
8d95fa3
Add revert migration as packages command
llamalm Mar 12, 2025
16af2e9
Merge branch 'main' into use-db-to-persist-cache-storage
llamalm Mar 18, 2025
c69875d
Merge branch 'main' into use-db-to-persist-cache-storage
noahlitvin Mar 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion packages/api/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@
"lint:fix": "eslint src --fix && pnpm format",
"format": "prettier --write src",
"migration:generate": "tsx ./node_modules/typeorm/cli.js migration:generate ./src/migrations/migration -d ./src/db.ts",
"migration:run": "tsx ./node_modules/typeorm/cli.js migration:run -d ./src/db.ts"
"migration:run": "tsx ./node_modules/typeorm/cli.js migration:run -d ./src/db.ts",
"migration:revert": "tsx ./node_modules/typeorm/cli.js migration:revert -d ./src/db.ts"
},
"keywords": [],
"dependencies": {
Expand Down
5 changes: 5 additions & 0 deletions packages/api/src/db.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { MarketPrice } from './models/MarketPrice';
import { RenderJob } from './models/RenderJob';
import { CollateralTransfer } from './models/CollateralTransfer';
import { Resource } from './models/Resource';
import { PerformanceCache } from './models/PerformanceCache';
import dotenv from 'dotenv';
import { fileURLToPath } from 'url';
import path, { dirname } from 'path';
Expand Down Expand Up @@ -39,6 +40,7 @@ const devDataSource: DataSource = new DataSource({
RenderJob,
CollateralTransfer,
Resource,
PerformanceCache,
],
});

Expand All @@ -60,6 +62,7 @@ const postgresDataSource: DataSource = new DataSource({
RenderJob,
CollateralTransfer,
Resource,
PerformanceCache,
],
});

Expand Down Expand Up @@ -90,5 +93,7 @@ export const marketPriceRepository = dataSource.getRepository(MarketPrice);
export const renderJobRepository = dataSource.getRepository(RenderJob);
export const collateralTransferRepository =
dataSource.getRepository(CollateralTransfer);
export const performanceCacheRepository =
dataSource.getRepository(PerformanceCache);

export default dataSource;
117 changes: 84 additions & 33 deletions packages/api/src/migrations/1741722842647-migration.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,38 +1,89 @@
import { MigrationInterface, QueryRunner } from "typeorm";
import { MigrationInterface, QueryRunner } from 'typeorm';

export class Migration1741722842647 implements MigrationInterface {
name = 'Migration1741722842647'
name = 'Migration1741722842647';

public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`CREATE INDEX "IDX_02755ce1b56a981eef76c0b59b" ON "epoch" ("marketId") `);
await queryRunner.query(`CREATE INDEX "IDX_f89ec06faf22da268399ae6a9b" ON "epoch" ("epochId") `);
await queryRunner.query(`CREATE INDEX "IDX_187fa56af532560ce204719ea3" ON "resource_price" ("resourceId") `);
await queryRunner.query(`CREATE INDEX "IDX_5bbe200849d138539d19b7caa6" ON "resource_price" ("blockNumber") `);
await queryRunner.query(`CREATE INDEX "IDX_a369700ab879af9ef6061c6dbe" ON "resource_price" ("timestamp") `);
await queryRunner.query(`CREATE INDEX "IDX_82453de75cd894e19c42844e70" ON "resource" ("slug") `);
await queryRunner.query(`CREATE INDEX "IDX_58232d6050e212b4a0f7eb02da" ON "market" ("address") `);
await queryRunner.query(`CREATE INDEX "IDX_33f985ce349688238dfeb8560e" ON "market" ("chainId") `);
await queryRunner.query(`CREATE INDEX "IDX_5430e2d7fe1df2bcada2c12deb" ON "event" ("blockNumber") `);
await queryRunner.query(`CREATE INDEX "IDX_2c15918ff289396205521c5f3c" ON "event" ("timestamp") `);
await queryRunner.query(`CREATE INDEX "IDX_a9346cdd1ea1e53a6b87e409ad" ON "market_price" ("timestamp") `);
await queryRunner.query(`CREATE INDEX "IDX_1ebf6f07652ca11d9f4618b64a" ON "collateral_transfer" ("transactionHash") `);
await queryRunner.query(`CREATE INDEX "IDX_927edd2b828777f0052366195e" ON "position" ("positionId") `);
}

public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP INDEX "public"."IDX_927edd2b828777f0052366195e"`);
await queryRunner.query(`DROP INDEX "public"."IDX_1ebf6f07652ca11d9f4618b64a"`);
await queryRunner.query(`DROP INDEX "public"."IDX_a9346cdd1ea1e53a6b87e409ad"`);
await queryRunner.query(`DROP INDEX "public"."IDX_2c15918ff289396205521c5f3c"`);
await queryRunner.query(`DROP INDEX "public"."IDX_5430e2d7fe1df2bcada2c12deb"`);
await queryRunner.query(`DROP INDEX "public"."IDX_33f985ce349688238dfeb8560e"`);
await queryRunner.query(`DROP INDEX "public"."IDX_58232d6050e212b4a0f7eb02da"`);
await queryRunner.query(`DROP INDEX "public"."IDX_82453de75cd894e19c42844e70"`);
await queryRunner.query(`DROP INDEX "public"."IDX_a369700ab879af9ef6061c6dbe"`);
await queryRunner.query(`DROP INDEX "public"."IDX_5bbe200849d138539d19b7caa6"`);
await queryRunner.query(`DROP INDEX "public"."IDX_187fa56af532560ce204719ea3"`);
await queryRunner.query(`DROP INDEX "public"."IDX_f89ec06faf22da268399ae6a9b"`);
await queryRunner.query(`DROP INDEX "public"."IDX_02755ce1b56a981eef76c0b59b"`);
}
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE INDEX "IDX_02755ce1b56a981eef76c0b59b" ON "epoch" ("marketId") `
);
await queryRunner.query(
`CREATE INDEX "IDX_f89ec06faf22da268399ae6a9b" ON "epoch" ("epochId") `
);
await queryRunner.query(
`CREATE INDEX "IDX_187fa56af532560ce204719ea3" ON "resource_price" ("resourceId") `
);
await queryRunner.query(
`CREATE INDEX "IDX_5bbe200849d138539d19b7caa6" ON "resource_price" ("blockNumber") `
);
await queryRunner.query(
`CREATE INDEX "IDX_a369700ab879af9ef6061c6dbe" ON "resource_price" ("timestamp") `
);
await queryRunner.query(
`CREATE INDEX "IDX_82453de75cd894e19c42844e70" ON "resource" ("slug") `
);
await queryRunner.query(
`CREATE INDEX "IDX_58232d6050e212b4a0f7eb02da" ON "market" ("address") `
);
await queryRunner.query(
`CREATE INDEX "IDX_33f985ce349688238dfeb8560e" ON "market" ("chainId") `
);
await queryRunner.query(
`CREATE INDEX "IDX_5430e2d7fe1df2bcada2c12deb" ON "event" ("blockNumber") `
);
await queryRunner.query(
`CREATE INDEX "IDX_2c15918ff289396205521c5f3c" ON "event" ("timestamp") `
);
await queryRunner.query(
`CREATE INDEX "IDX_a9346cdd1ea1e53a6b87e409ad" ON "market_price" ("timestamp") `
);
await queryRunner.query(
`CREATE INDEX "IDX_1ebf6f07652ca11d9f4618b64a" ON "collateral_transfer" ("transactionHash") `
);
await queryRunner.query(
`CREATE INDEX "IDX_927edd2b828777f0052366195e" ON "position" ("positionId") `
);
}

public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`DROP INDEX "public"."IDX_927edd2b828777f0052366195e"`
);
await queryRunner.query(
`DROP INDEX "public"."IDX_1ebf6f07652ca11d9f4618b64a"`
);
await queryRunner.query(
`DROP INDEX "public"."IDX_a9346cdd1ea1e53a6b87e409ad"`
);
await queryRunner.query(
`DROP INDEX "public"."IDX_2c15918ff289396205521c5f3c"`
);
await queryRunner.query(
`DROP INDEX "public"."IDX_5430e2d7fe1df2bcada2c12deb"`
);
await queryRunner.query(
`DROP INDEX "public"."IDX_33f985ce349688238dfeb8560e"`
);
await queryRunner.query(
`DROP INDEX "public"."IDX_58232d6050e212b4a0f7eb02da"`
);
await queryRunner.query(
`DROP INDEX "public"."IDX_82453de75cd894e19c42844e70"`
);
await queryRunner.query(
`DROP INDEX "public"."IDX_a369700ab879af9ef6061c6dbe"`
);
await queryRunner.query(
`DROP INDEX "public"."IDX_5bbe200849d138539d19b7caa6"`
);
await queryRunner.query(
`DROP INDEX "public"."IDX_187fa56af532560ce204719ea3"`
);
await queryRunner.query(
`DROP INDEX "public"."IDX_f89ec06faf22da268399ae6a9b"`
);
await queryRunner.query(
`DROP INDEX "public"."IDX_02755ce1b56a981eef76c0b59b"`
);
}
}
15 changes: 15 additions & 0 deletions packages/api/src/migrations/1741790918671-migration.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import { MigrationInterface, QueryRunner } from 'typeorm';

export class Migration1741790918671 implements MigrationInterface {
name = 'Migration1741790918671';

public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TABLE "performance_cache" ("id" SERIAL NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "resourceSlug" character varying NOT NULL, "interval" character varying NOT NULL, "jsonSection" character varying NOT NULL, "storageVersion" character varying, "latestTimestamp" character varying NOT NULL, "storage" jsonb NOT NULL, CONSTRAINT "PK_93aab8268ebc22e5129c77cdc2d" PRIMARY KEY ("id"))`
);
}

public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP TABLE "performance_cache"`);
}
}
33 changes: 33 additions & 0 deletions packages/api/src/models/PerformanceCache.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import {
Entity,
PrimaryGeneratedColumn,
Column,
CreateDateColumn,
} from 'typeorm';

@Entity()
export class PerformanceCache {
@PrimaryGeneratedColumn()
id: number;

@CreateDateColumn()
createdAt: Date;

@Column({ type: 'varchar' })
resourceSlug: string;

@Column({ type: 'varchar' })
interval: number;

@Column({ type: 'varchar' })
jsonSection: string;

@Column({ type: 'varchar', nullable: true })
storageVersion: string;

@Column({ type: 'varchar' })
latestTimestamp: number;

@Column({ type: 'jsonb' })
storage: string;
}
132 changes: 51 additions & 81 deletions packages/api/src/performance/helper.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,86 +1,74 @@
import { IntervalStore } from './types';
import * as fs from 'fs';
import * as path from 'path';
import { performanceCacheRepository } from '../db';

export async function saveStorageToFile(
export async function persistStorage(
storage: IntervalStore,
latestTimestamp: number,
resourceSlug: string,
resourceName: string,
sectionName: string
interval: number,
jsonSection: string
): Promise<undefined> {
if (process.env.SAVE_STORAGE !== 'true') {
return;
}

console.time(
` ResourcePerformance - processResourceData.${resourceName}.${sectionName}.saveStorage`
` ResourcePerformance - processResourceData.${resourceName}.${interval}.${jsonSection}.saveStorage`
);
const storageDir = process.env.STORAGE_PATH;
if (!storageDir) {
throw new Error('STORAGE_PATH is not set');
}

if (!fs.existsSync(storageDir)) {
fs.mkdirSync(storageDir, { recursive: true });
}

const filename = path.join(
storageDir,
`${resourceSlug}-${sectionName}-storage.json`
);
await fs.promises.writeFile(
filename,
JSON.stringify(
{
latestTimestamp,
store: storage,
},
(key, value) => (typeof value === 'bigint' ? value.toString() : value),
2
)
);
// Create or update the cache entry
await performanceCacheRepository.save({
resourceSlug,
interval,
jsonSection,
storageVersion: '1', // You may want to manage versions
latestTimestamp,
storage: JSON.stringify(storage, (key, value) =>
typeof value === 'bigint' ? value.toString() : value
),
});

console.timeEnd(
` ResourcePerformance - processResourceData.${resourceName}.${sectionName}.saveStorage`
` ResourcePerformance - processResourceData.${resourceName}.${interval}.${jsonSection}.saveStorage`
);
console.log(` ResourcePerformance --> Saved storage to ${filename}`);
console.log(` ResourcePerformance --> Saved storage to database`);
}

export async function loadStorageFromFile(
export async function restorePersistedStorage(
resourceSlug: string,
resourceName: string,
sectionName: string
): Promise<
| {
latestTimestamp: number;
store: IntervalStore;
}
| undefined
> {
interval: number,
jsonSection: string
): Promise<{ latestTimestamp: number; store: IntervalStore } | undefined> {
if (process.env.SAVE_STORAGE !== 'true') {
return undefined;
}

console.time(
` ResourcePerformance - processResourceData.${resourceName}.${sectionName}.loadStorage`
` ResourcePerformance - processResourceData.${resourceName}.${interval}.${jsonSection}.loadStorage`
);
const storageDir = process.env.STORAGE_PATH;
if (!storageDir) {
throw new Error('STORAGE_PATH is not set');
}

const filename = path.join(
storageDir,
`${resourceSlug}-${sectionName}-storage.json`
);
if (!fs.existsSync(filename)) {
console.log(`!! Storage file ${filename} does not exist`);
const cacheEntry = await performanceCacheRepository.findOne({
where: {
resourceSlug,
interval,
jsonSection,
storageVersion: '1',
},
order: {
createdAt: 'DESC',
},
});

if (!cacheEntry) {
console.log(
`!! Storage entry for ${resourceSlug}-${interval}-${jsonSection} does not exist`
);
return undefined;
}

const fileContent = await fs.promises.readFile(filename, 'utf-8');
const storage = JSON.parse(fileContent, (key, value) => {
const storage = JSON.parse(cacheEntry.storage, (key, value) => {
// Convert string numbers that might be bigints back to bigint
if (typeof value === 'string' && /^\d+$/.test(value)) {
try {
Expand All @@ -90,44 +78,26 @@ export async function loadStorageFromFile(
}
}
return value;
}) as {
latestTimestamp: number;
store: IntervalStore;
};
});

console.timeEnd(
` ResourcePerformance - processResourceData.${resourceName}.${sectionName}.loadStorage`
` ResourcePerformance - processResourceData.${resourceName}.${interval}.${jsonSection}.loadStorage`
);
console.log(` ResourcePerformance - -> Loaded storage from ${filename}`);
console.log(` ResourcePerformance - -> Loaded storage from database`);

return {
latestTimestamp: storage.latestTimestamp,
store: storage.store,
latestTimestamp: cacheEntry.latestTimestamp,
store: storage,
};
}

export async function clearStorageFiles(): Promise<void> {
const storageDir = process.env.STORAGE_PATH;
if (!storageDir) {
throw new Error('STORAGE_PATH is not set');
}

if (!fs.existsSync(storageDir)) {
return; // Nothing to clear
}

console.time(' ResourcePerformance - clearStorageFiles');
export async function clearPersistedStore(): Promise<void> {
console.time(' ResourcePerformance - clearStorage');

const files = await fs.promises.readdir(storageDir);
for (const file of files) {
if (file.endsWith('-storage.json')) {
await fs.promises.unlink(path.join(storageDir, file));
}
}
await performanceCacheRepository.delete({});

console.timeEnd(' ResourcePerformance - clearStorageFiles');
console.log(
` ResourcePerformance --> Cleared ${files.length} storage files`
);
console.timeEnd(' ResourcePerformance - clearStorage');
console.log(' ResourcePerformance --> Cleared performance cache storage');
}

export function maxBigInt(a: bigint, b: bigint) {
Expand Down
Loading