Skip to content

Commit 1292e50

Browse files
llamalmnoahlitvin
llamalm
and
noahlitvin
authored
Use database to persist cache store
* Use database to persist cache store * update migrations * Add revert migration as packages command --------- Co-authored-by: Noah Litvin <[email protected]>
1 parent bc654cd commit 1292e50

8 files changed

+200
-123
lines changed

‎packages/api/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@
2323
"lint:fix": "eslint src --fix && pnpm format",
2424
"format": "prettier --write src",
2525
"migration:generate": "tsx ./node_modules/typeorm/cli.js migration:generate ./src/migrations/migration -d ./src/db.ts",
26-
"migration:run": "tsx ./node_modules/typeorm/cli.js migration:run -d ./src/db.ts"
26+
"migration:run": "tsx ./node_modules/typeorm/cli.js migration:run -d ./src/db.ts",
27+
"migration:revert": "tsx ./node_modules/typeorm/cli.js migration:revert -d ./src/db.ts"
2728
},
2829
"keywords": [],
2930
"dependencies": {

‎packages/api/src/db.ts

+5
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import { MarketPrice } from './models/MarketPrice';
99
import { RenderJob } from './models/RenderJob';
1010
import { CollateralTransfer } from './models/CollateralTransfer';
1111
import { Resource } from './models/Resource';
12+
import { PerformanceCache } from './models/PerformanceCache';
1213
import dotenv from 'dotenv';
1314
import { fileURLToPath } from 'url';
1415
import path, { dirname } from 'path';
@@ -39,6 +40,7 @@ const devDataSource: DataSource = new DataSource({
3940
RenderJob,
4041
CollateralTransfer,
4142
Resource,
43+
PerformanceCache,
4244
],
4345
});
4446

@@ -60,6 +62,7 @@ const postgresDataSource: DataSource = new DataSource({
6062
RenderJob,
6163
CollateralTransfer,
6264
Resource,
65+
PerformanceCache,
6366
],
6467
});
6568

@@ -90,5 +93,7 @@ export const marketPriceRepository = dataSource.getRepository(MarketPrice);
9093
export const renderJobRepository = dataSource.getRepository(RenderJob);
9194
export const collateralTransferRepository =
9295
dataSource.getRepository(CollateralTransfer);
96+
export const performanceCacheRepository =
97+
dataSource.getRepository(PerformanceCache);
9398

9499
export default dataSource;

‎packages/api/src/migrations/1741722842647-migration.ts

+84-33
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,89 @@
1-
import { MigrationInterface, QueryRunner } from "typeorm";
1+
import { MigrationInterface, QueryRunner } from 'typeorm';
22

33
export class Migration1741722842647 implements MigrationInterface {
4-
name = 'Migration1741722842647'
4+
name = 'Migration1741722842647';
55

6-
public async up(queryRunner: QueryRunner): Promise<void> {
7-
await queryRunner.query(`CREATE INDEX "IDX_02755ce1b56a981eef76c0b59b" ON "epoch" ("marketId") `);
8-
await queryRunner.query(`CREATE INDEX "IDX_f89ec06faf22da268399ae6a9b" ON "epoch" ("epochId") `);
9-
await queryRunner.query(`CREATE INDEX "IDX_187fa56af532560ce204719ea3" ON "resource_price" ("resourceId") `);
10-
await queryRunner.query(`CREATE INDEX "IDX_5bbe200849d138539d19b7caa6" ON "resource_price" ("blockNumber") `);
11-
await queryRunner.query(`CREATE INDEX "IDX_a369700ab879af9ef6061c6dbe" ON "resource_price" ("timestamp") `);
12-
await queryRunner.query(`CREATE INDEX "IDX_82453de75cd894e19c42844e70" ON "resource" ("slug") `);
13-
await queryRunner.query(`CREATE INDEX "IDX_58232d6050e212b4a0f7eb02da" ON "market" ("address") `);
14-
await queryRunner.query(`CREATE INDEX "IDX_33f985ce349688238dfeb8560e" ON "market" ("chainId") `);
15-
await queryRunner.query(`CREATE INDEX "IDX_5430e2d7fe1df2bcada2c12deb" ON "event" ("blockNumber") `);
16-
await queryRunner.query(`CREATE INDEX "IDX_2c15918ff289396205521c5f3c" ON "event" ("timestamp") `);
17-
await queryRunner.query(`CREATE INDEX "IDX_a9346cdd1ea1e53a6b87e409ad" ON "market_price" ("timestamp") `);
18-
await queryRunner.query(`CREATE INDEX "IDX_1ebf6f07652ca11d9f4618b64a" ON "collateral_transfer" ("transactionHash") `);
19-
await queryRunner.query(`CREATE INDEX "IDX_927edd2b828777f0052366195e" ON "position" ("positionId") `);
20-
}
21-
22-
public async down(queryRunner: QueryRunner): Promise<void> {
23-
await queryRunner.query(`DROP INDEX "public"."IDX_927edd2b828777f0052366195e"`);
24-
await queryRunner.query(`DROP INDEX "public"."IDX_1ebf6f07652ca11d9f4618b64a"`);
25-
await queryRunner.query(`DROP INDEX "public"."IDX_a9346cdd1ea1e53a6b87e409ad"`);
26-
await queryRunner.query(`DROP INDEX "public"."IDX_2c15918ff289396205521c5f3c"`);
27-
await queryRunner.query(`DROP INDEX "public"."IDX_5430e2d7fe1df2bcada2c12deb"`);
28-
await queryRunner.query(`DROP INDEX "public"."IDX_33f985ce349688238dfeb8560e"`);
29-
await queryRunner.query(`DROP INDEX "public"."IDX_58232d6050e212b4a0f7eb02da"`);
30-
await queryRunner.query(`DROP INDEX "public"."IDX_82453de75cd894e19c42844e70"`);
31-
await queryRunner.query(`DROP INDEX "public"."IDX_a369700ab879af9ef6061c6dbe"`);
32-
await queryRunner.query(`DROP INDEX "public"."IDX_5bbe200849d138539d19b7caa6"`);
33-
await queryRunner.query(`DROP INDEX "public"."IDX_187fa56af532560ce204719ea3"`);
34-
await queryRunner.query(`DROP INDEX "public"."IDX_f89ec06faf22da268399ae6a9b"`);
35-
await queryRunner.query(`DROP INDEX "public"."IDX_02755ce1b56a981eef76c0b59b"`);
36-
}
6+
public async up(queryRunner: QueryRunner): Promise<void> {
7+
await queryRunner.query(
8+
`CREATE INDEX "IDX_02755ce1b56a981eef76c0b59b" ON "epoch" ("marketId") `
9+
);
10+
await queryRunner.query(
11+
`CREATE INDEX "IDX_f89ec06faf22da268399ae6a9b" ON "epoch" ("epochId") `
12+
);
13+
await queryRunner.query(
14+
`CREATE INDEX "IDX_187fa56af532560ce204719ea3" ON "resource_price" ("resourceId") `
15+
);
16+
await queryRunner.query(
17+
`CREATE INDEX "IDX_5bbe200849d138539d19b7caa6" ON "resource_price" ("blockNumber") `
18+
);
19+
await queryRunner.query(
20+
`CREATE INDEX "IDX_a369700ab879af9ef6061c6dbe" ON "resource_price" ("timestamp") `
21+
);
22+
await queryRunner.query(
23+
`CREATE INDEX "IDX_82453de75cd894e19c42844e70" ON "resource" ("slug") `
24+
);
25+
await queryRunner.query(
26+
`CREATE INDEX "IDX_58232d6050e212b4a0f7eb02da" ON "market" ("address") `
27+
);
28+
await queryRunner.query(
29+
`CREATE INDEX "IDX_33f985ce349688238dfeb8560e" ON "market" ("chainId") `
30+
);
31+
await queryRunner.query(
32+
`CREATE INDEX "IDX_5430e2d7fe1df2bcada2c12deb" ON "event" ("blockNumber") `
33+
);
34+
await queryRunner.query(
35+
`CREATE INDEX "IDX_2c15918ff289396205521c5f3c" ON "event" ("timestamp") `
36+
);
37+
await queryRunner.query(
38+
`CREATE INDEX "IDX_a9346cdd1ea1e53a6b87e409ad" ON "market_price" ("timestamp") `
39+
);
40+
await queryRunner.query(
41+
`CREATE INDEX "IDX_1ebf6f07652ca11d9f4618b64a" ON "collateral_transfer" ("transactionHash") `
42+
);
43+
await queryRunner.query(
44+
`CREATE INDEX "IDX_927edd2b828777f0052366195e" ON "position" ("positionId") `
45+
);
46+
}
3747

48+
public async down(queryRunner: QueryRunner): Promise<void> {
49+
await queryRunner.query(
50+
`DROP INDEX "public"."IDX_927edd2b828777f0052366195e"`
51+
);
52+
await queryRunner.query(
53+
`DROP INDEX "public"."IDX_1ebf6f07652ca11d9f4618b64a"`
54+
);
55+
await queryRunner.query(
56+
`DROP INDEX "public"."IDX_a9346cdd1ea1e53a6b87e409ad"`
57+
);
58+
await queryRunner.query(
59+
`DROP INDEX "public"."IDX_2c15918ff289396205521c5f3c"`
60+
);
61+
await queryRunner.query(
62+
`DROP INDEX "public"."IDX_5430e2d7fe1df2bcada2c12deb"`
63+
);
64+
await queryRunner.query(
65+
`DROP INDEX "public"."IDX_33f985ce349688238dfeb8560e"`
66+
);
67+
await queryRunner.query(
68+
`DROP INDEX "public"."IDX_58232d6050e212b4a0f7eb02da"`
69+
);
70+
await queryRunner.query(
71+
`DROP INDEX "public"."IDX_82453de75cd894e19c42844e70"`
72+
);
73+
await queryRunner.query(
74+
`DROP INDEX "public"."IDX_a369700ab879af9ef6061c6dbe"`
75+
);
76+
await queryRunner.query(
77+
`DROP INDEX "public"."IDX_5bbe200849d138539d19b7caa6"`
78+
);
79+
await queryRunner.query(
80+
`DROP INDEX "public"."IDX_187fa56af532560ce204719ea3"`
81+
);
82+
await queryRunner.query(
83+
`DROP INDEX "public"."IDX_f89ec06faf22da268399ae6a9b"`
84+
);
85+
await queryRunner.query(
86+
`DROP INDEX "public"."IDX_02755ce1b56a981eef76c0b59b"`
87+
);
88+
}
3889
}

‎packages/api/src/migrations/1741790918671-migration.ts

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import { MigrationInterface, QueryRunner } from 'typeorm';
2+
3+
export class Migration1741790918671 implements MigrationInterface {
4+
name = 'Migration1741790918671';
5+
6+
public async up(queryRunner: QueryRunner): Promise<void> {
7+
await queryRunner.query(
8+
`CREATE TABLE "performance_cache" ("id" SERIAL NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "resourceSlug" character varying NOT NULL, "interval" character varying NOT NULL, "jsonSection" character varying NOT NULL, "storageVersion" character varying, "latestTimestamp" character varying NOT NULL, "storage" jsonb NOT NULL, CONSTRAINT "PK_93aab8268ebc22e5129c77cdc2d" PRIMARY KEY ("id"))`
9+
);
10+
}
11+
12+
public async down(queryRunner: QueryRunner): Promise<void> {
13+
await queryRunner.query(`DROP TABLE "performance_cache"`);
14+
}
15+
}

‎packages/api/src/models/PerformanceCache.ts

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import {
2+
Entity,
3+
PrimaryGeneratedColumn,
4+
Column,
5+
CreateDateColumn,
6+
} from 'typeorm';
7+
8+
@Entity()
9+
export class PerformanceCache {
10+
@PrimaryGeneratedColumn()
11+
id: number;
12+
13+
@CreateDateColumn()
14+
createdAt: Date;
15+
16+
@Column({ type: 'varchar' })
17+
resourceSlug: string;
18+
19+
@Column({ type: 'varchar' })
20+
interval: number;
21+
22+
@Column({ type: 'varchar' })
23+
jsonSection: string;
24+
25+
@Column({ type: 'varchar', nullable: true })
26+
storageVersion: string;
27+
28+
@Column({ type: 'varchar' })
29+
latestTimestamp: number;
30+
31+
@Column({ type: 'jsonb' })
32+
storage: string;
33+
}

‎packages/api/src/performance/helper.ts

+51-81
Original file line numberDiff line numberDiff line change
@@ -1,86 +1,74 @@
11
import { IntervalStore } from './types';
2-
import * as fs from 'fs';
3-
import * as path from 'path';
2+
import { performanceCacheRepository } from '../db';
43

5-
export async function saveStorageToFile(
4+
export async function persistStorage(
65
storage: IntervalStore,
76
latestTimestamp: number,
87
resourceSlug: string,
98
resourceName: string,
10-
sectionName: string
9+
interval: number,
10+
jsonSection: string
1111
): Promise<undefined> {
1212
if (process.env.SAVE_STORAGE !== 'true') {
1313
return;
1414
}
1515

1616
console.time(
17-
` ResourcePerformance - processResourceData.${resourceName}.${sectionName}.saveStorage`
17+
` ResourcePerformance - processResourceData.${resourceName}.${interval}.${jsonSection}.saveStorage`
1818
);
19-
const storageDir = process.env.STORAGE_PATH;
20-
if (!storageDir) {
21-
throw new Error('STORAGE_PATH is not set');
22-
}
23-
24-
if (!fs.existsSync(storageDir)) {
25-
fs.mkdirSync(storageDir, { recursive: true });
26-
}
2719

28-
const filename = path.join(
29-
storageDir,
30-
`${resourceSlug}-${sectionName}-storage.json`
31-
);
32-
await fs.promises.writeFile(
33-
filename,
34-
JSON.stringify(
35-
{
36-
latestTimestamp,
37-
store: storage,
38-
},
39-
(key, value) => (typeof value === 'bigint' ? value.toString() : value),
40-
2
41-
)
42-
);
20+
// Create or update the cache entry
21+
await performanceCacheRepository.save({
22+
resourceSlug,
23+
interval,
24+
jsonSection,
25+
storageVersion: '1', // You may want to manage versions
26+
latestTimestamp,
27+
storage: JSON.stringify(storage, (key, value) =>
28+
typeof value === 'bigint' ? value.toString() : value
29+
),
30+
});
4331

4432
console.timeEnd(
45-
` ResourcePerformance - processResourceData.${resourceName}.${sectionName}.saveStorage`
33+
` ResourcePerformance - processResourceData.${resourceName}.${interval}.${jsonSection}.saveStorage`
4634
);
47-
console.log(` ResourcePerformance --> Saved storage to ${filename}`);
35+
console.log(` ResourcePerformance --> Saved storage to database`);
4836
}
4937

50-
export async function loadStorageFromFile(
38+
export async function restorePersistedStorage(
5139
resourceSlug: string,
5240
resourceName: string,
53-
sectionName: string
54-
): Promise<
55-
| {
56-
latestTimestamp: number;
57-
store: IntervalStore;
58-
}
59-
| undefined
60-
> {
41+
interval: number,
42+
jsonSection: string
43+
): Promise<{ latestTimestamp: number; store: IntervalStore } | undefined> {
6144
if (process.env.SAVE_STORAGE !== 'true') {
6245
return undefined;
6346
}
6447

6548
console.time(
66-
` ResourcePerformance - processResourceData.${resourceName}.${sectionName}.loadStorage`
49+
` ResourcePerformance - processResourceData.${resourceName}.${interval}.${jsonSection}.loadStorage`
6750
);
68-
const storageDir = process.env.STORAGE_PATH;
69-
if (!storageDir) {
70-
throw new Error('STORAGE_PATH is not set');
71-
}
7251

73-
const filename = path.join(
74-
storageDir,
75-
`${resourceSlug}-${sectionName}-storage.json`
76-
);
77-
if (!fs.existsSync(filename)) {
78-
console.log(`!! Storage file ${filename} does not exist`);
52+
const cacheEntry = await performanceCacheRepository.findOne({
53+
where: {
54+
resourceSlug,
55+
interval,
56+
jsonSection,
57+
storageVersion: '1',
58+
},
59+
order: {
60+
createdAt: 'DESC',
61+
},
62+
});
63+
64+
if (!cacheEntry) {
65+
console.log(
66+
`!! Storage entry for ${resourceSlug}-${interval}-${jsonSection} does not exist`
67+
);
7968
return undefined;
8069
}
8170

82-
const fileContent = await fs.promises.readFile(filename, 'utf-8');
83-
const storage = JSON.parse(fileContent, (key, value) => {
71+
const storage = JSON.parse(cacheEntry.storage, (key, value) => {
8472
// Convert string numbers that might be bigints back to bigint
8573
if (typeof value === 'string' && /^\d+$/.test(value)) {
8674
try {
@@ -90,44 +78,26 @@ export async function loadStorageFromFile(
9078
}
9179
}
9280
return value;
93-
}) as {
94-
latestTimestamp: number;
95-
store: IntervalStore;
96-
};
81+
});
9782

9883
console.timeEnd(
99-
` ResourcePerformance - processResourceData.${resourceName}.${sectionName}.loadStorage`
84+
` ResourcePerformance - processResourceData.${resourceName}.${interval}.${jsonSection}.loadStorage`
10085
);
101-
console.log(` ResourcePerformance - -> Loaded storage from ${filename}`);
86+
console.log(` ResourcePerformance - -> Loaded storage from database`);
87+
10288
return {
103-
latestTimestamp: storage.latestTimestamp,
104-
store: storage.store,
89+
latestTimestamp: cacheEntry.latestTimestamp,
90+
store: storage,
10591
};
10692
}
10793

108-
export async function clearStorageFiles(): Promise<void> {
109-
const storageDir = process.env.STORAGE_PATH;
110-
if (!storageDir) {
111-
throw new Error('STORAGE_PATH is not set');
112-
}
113-
114-
if (!fs.existsSync(storageDir)) {
115-
return; // Nothing to clear
116-
}
117-
118-
console.time(' ResourcePerformance - clearStorageFiles');
94+
export async function clearPersistedStore(): Promise<void> {
95+
console.time(' ResourcePerformance - clearStorage');
11996

120-
const files = await fs.promises.readdir(storageDir);
121-
for (const file of files) {
122-
if (file.endsWith('-storage.json')) {
123-
await fs.promises.unlink(path.join(storageDir, file));
124-
}
125-
}
97+
await performanceCacheRepository.delete({});
12698

127-
console.timeEnd(' ResourcePerformance - clearStorageFiles');
128-
console.log(
129-
` ResourcePerformance --> Cleared ${files.length} storage files`
130-
);
99+
console.timeEnd(' ResourcePerformance - clearStorage');
100+
console.log(' ResourcePerformance --> Cleared performance cache storage');
131101
}
132102

133103
export function maxBigInt(a: bigint, b: bigint) {

0 commit comments

Comments
 (0)