Skip to content

Commit 73082cb

Browse files
7aydllamalm
7ayd
and
llamalm
authored
adding 7-day trailing average
* adding 7-day trailing average * variable fix * checkpoint: propagate changes * make it start * protobuf * add protobuf dependency * Revert "add protobuf dependency" This reverts commit e385614. * Revert "protobuf" This reverts commit 8140b5b. * use 28 days trailing average * remove duplicate code * use MessagePack * reduce storage size --------- Co-authored-by: llamalm <[email protected]>
1 parent f7b1beb commit 73082cb

File tree

10 files changed

+388
-311
lines changed

10 files changed

+388
-311
lines changed

‎packages/api/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
"dependencies": {
3030
"@apollo/server": "^4.11.3",
3131
"@apollo/server-plugin-response-cache": "^4.1.4",
32+
"@msgpack/msgpack": "^3.1.1",
3233
"@sentry/cli": "^2.39.0",
3334
"@sentry/node": "^8.40.0",
3435
"axios": "^1.7.7",

‎packages/api/schema.graphql

+1-1
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ type Query {
6868
resource(slug: String!): ResourceType
6969
resourceCandles(from: Int!, interval: Int!, slug: String!, to: Int!): [CandleType!]!
7070
resourcePrices: [ResourcePriceType!]!
71-
resourceTrailingAverageCandles(from: Int!, interval: Int!, slug: String!, to: Int!): [CandleType!]!
71+
resourceTrailingAverageCandles(from: Int!, interval: Int!, slug: String!, to: Int!, trailingAvgTime: Int!): [CandleType!]!
7272
resources: [ResourceType!]!
7373
transactions(positionId: Int): [TransactionType!]!
7474
}

‎packages/api/src/fixtures.ts

+13
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,19 @@ import celestiaIndexer from './resourcePriceFunctions/celestiaIndexer';
55
import btcIndexer from './resourcePriceFunctions/btcIndexer';
66
import { Deployment, MarketInfo } from './interfaces';
77

8+
// TAT = Trailing Average Time
9+
export const TIME_INTERVALS = {
10+
intervals: {
11+
INTERVAL_1_MINUTE: 60,
12+
INTERVAL_5_MINUTES: 5 * 60,
13+
INTERVAL_15_MINUTES: 15 * 60,
14+
INTERVAL_30_MINUTES: 30 * 60,
15+
INTERVAL_4_HOURS: 4 * 60 * 60,
16+
INTERVAL_1_DAY: 24 * 60 * 60,
17+
INTERVAL_7_DAYS: 7 * 24 * 60 * 60,
18+
INTERVAL_28_DAYS: 28 * 24 * 60 * 60,
19+
},
20+
};
821
const safeRequire = async (path: string): Promise<Deployment | null> => {
922
try {
1023
const module = await import(path);

‎packages/api/src/graphql/resolvers/CandleResolver.ts

+4-2
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,8 @@ export class CandleResolver {
334334
@Arg('slug', () => String) slug: string,
335335
@Arg('from', () => Int) from: number,
336336
@Arg('to', () => Int) to: number,
337-
@Arg('interval', () => Int) interval: number
337+
@Arg('interval', () => Int) interval: number,
338+
@Arg('trailingAvgTime', () => Int) trailingAvgTime: number
338339
): Promise<CandleType[]> {
339340
const resourcePerformanceManager = ResourcePerformanceManager.getInstance();
340341
const resourcePerformance =
@@ -347,7 +348,8 @@ export class CandleResolver {
347348
const prices = await resourcePerformance.getTrailingAvgPrices(
348349
from,
349350
to,
350-
interval
351+
interval,
352+
trailingAvgTime
351353
);
352354

353355
return prices;

‎packages/api/src/performance/constants.ts

+10-7
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
export const INTERVAL_1_MINUTE = 60;
2-
export const INTERVAL_5_MINUTES = 5 * 60;
3-
export const INTERVAL_15_MINUTES = 15 * 60;
4-
export const INTERVAL_30_MINUTES = 30 * 60;
5-
export const INTERVAL_4_HOURS = 4 * 60 * 60;
6-
export const INTERVAL_1_DAY = 24 * 60 * 60;
7-
export const INTERVAL_28_DAYS = 28 * 24 * 60 * 60;
1+
export const INTERVAL_TIMES = {
2+
INTERVAL_1_MINUTE: 60,
3+
INTERVAL_5_MINUTES: 5 * 60,
4+
INTERVAL_15_MINUTES: 15 * 60,
5+
INTERVAL_30_MINUTES: 30 * 60,
6+
INTERVAL_4_HOURS: 4 * 60 * 60,
7+
INTERVAL_1_DAY: 24 * 60 * 60,
8+
INTERVAL_7_DAYS: 7 * 24 * 60 * 60,
9+
INTERVAL_28_DAYS: 28 * 24 * 60 * 60,
10+
};

‎packages/api/src/performance/helper.ts

+32-43
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import { IntervalStore } from './types';
22
import * as fs from 'fs';
33
import * as path from 'path';
4+
import { encode, decode } from '@msgpack/msgpack';
45

5-
const FILE_VERSION = 1;
6+
const FILE_VERSION = 3;
67

78
export async function saveStorageToFile(
89
storage: IntervalStore,
@@ -30,26 +31,26 @@ export async function saveStorageToFile(
3031

3132
const filename = path.join(
3233
storageDir,
33-
`${resourceSlug}-${sectionName}-storage.json`
34-
);
35-
await fs.promises.writeFile(
36-
filename,
37-
JSON.stringify(
38-
{
39-
fileVersion: FILE_VERSION,
40-
latestResourceTimestamp,
41-
latestMarketTimestamp,
42-
store: storage,
43-
},
44-
(key, value) => (typeof value === 'bigint' ? value.toString() : value),
45-
2
46-
)
34+
`${resourceSlug}-${sectionName}-storage.msgpack`
4735
);
4836

37+
const data = {
38+
fileVersion: FILE_VERSION,
39+
latestResourceTimestamp,
40+
latestMarketTimestamp,
41+
store: storage,
42+
};
43+
44+
// Encode and save
45+
const buffer = encode(data);
46+
await fs.promises.writeFile(filename, buffer);
47+
4948
console.timeEnd(
5049
` ResourcePerformance - processResourceData.${resourceName}.${sectionName}.saveStorage`
5150
);
52-
console.log(` ResourcePerformance --> Saved storage to ${filename}`);
51+
console.log(
52+
` ResourcePerformance --> Saved storage to ${filename} (${buffer.length} bytes)`
53+
);
5354
}
5455

5556
export async function loadStorageFromFile(
@@ -78,48 +79,36 @@ export async function loadStorageFromFile(
7879

7980
const filename = path.join(
8081
storageDir,
81-
`${resourceSlug}-${sectionName}-storage.json`
82+
`${resourceSlug}-${sectionName}-storage.msgpack`
8283
);
83-
if (!fs.existsSync(filename)) {
84-
console.log(`!! Storage file ${filename} does not exist`);
85-
return undefined;
86-
}
8784

8885
try {
89-
const fileContent = await fs.promises.readFile(filename, 'utf-8');
90-
const storage = JSON.parse(fileContent, (key, value) => {
91-
// Convert string numbers that might be bigints back to bigint
92-
if (typeof value === 'string' && /^\d+$/.test(value)) {
93-
try {
94-
return BigInt(value);
95-
} catch {
96-
return value;
97-
}
98-
}
99-
return value;
100-
}) as {
86+
const buffer = await fs.promises.readFile(filename);
87+
const data = decode(buffer) as {
10188
fileVersion: number;
10289
latestResourceTimestamp: number;
10390
latestMarketTimestamp: number;
10491
store: IntervalStore;
10592
};
106-
console.timeEnd(
107-
` ResourcePerformance - processResourceData.${resourceName}.${sectionName}.loadStorage`
108-
);
109-
console.log(` ResourcePerformance - -> Loaded storage from ${filename}`);
110-
if (storage.fileVersion !== FILE_VERSION) {
93+
94+
if (data.fileVersion !== FILE_VERSION) {
11195
console.log(
112-
`!! Storage file ${filename} has an unsupported version -${storage.fileVersion}-. Expected -${FILE_VERSION}-`
96+
`!! Storage file ${filename} has an unsupported version -${data.fileVersion}-. Expected -${FILE_VERSION}-`
11397
);
11498
return undefined;
11599
}
100+
101+
console.timeEnd(
102+
` ResourcePerformance - processResourceData.${resourceName}.${sectionName}.loadStorage`
103+
);
104+
console.log(` ResourcePerformance - -> Loaded storage from ${filename}`);
116105
return {
117-
latestResourceTimestamp: storage.latestResourceTimestamp,
118-
latestMarketTimestamp: storage.latestMarketTimestamp,
119-
store: storage.store,
106+
latestResourceTimestamp: data.latestResourceTimestamp,
107+
latestMarketTimestamp: data.latestMarketTimestamp,
108+
store: data.store,
120109
};
121110
} catch (error) {
122-
console.error(`!! Error loading storage from ${filename}: ${error}`);
111+
console.log(` ResourcePerformance - load storage failed: ${error}`);
123112
console.timeEnd(
124113
` ResourcePerformance - processResourceData.${resourceName}.${sectionName}.loadStorage`
125114
);

0 commit comments

Comments
 (0)