Skip to content

Commit 28da30d

Browse files
llamalmnoahlitvin
llamalm
and
noahlitvin
authored
Add endpoint to refresh selected resources
* Add endpoint to refresh selected resources * Improve hard refresh + better logs * lint * ui update --------- Co-authored-by: Noah Litvin <[email protected]>
1 parent 9b71b98 commit 28da30d

File tree

5 files changed

+225
-40
lines changed

5 files changed

+225
-40
lines changed

‎packages/api/src/performance/helper.ts

+25
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,31 @@ export async function loadStorageFromFile(
105105
};
106106
}
107107

108+
export async function clearStorageFiles(): Promise<void> {
109+
const storageDir = process.env.STORAGE_PATH;
110+
if (!storageDir) {
111+
throw new Error('STORAGE_PATH is not set');
112+
}
113+
114+
if (!fs.existsSync(storageDir)) {
115+
return; // Nothing to clear
116+
}
117+
118+
console.time(' ResourcePerformance - clearStorageFiles');
119+
120+
const files = await fs.promises.readdir(storageDir);
121+
for (const file of files) {
122+
if (file.endsWith('-storage.json')) {
123+
await fs.promises.unlink(path.join(storageDir, file));
124+
}
125+
}
126+
127+
console.timeEnd(' ResourcePerformance - clearStorageFiles');
128+
console.log(
129+
` ResourcePerformance --> Cleared ${files.length} storage files`
130+
);
131+
}
132+
108133
export function maxBigInt(a: bigint, b: bigint) {
109134
return a > b ? a : b;
110135
}

‎packages/api/src/performance/resourcePerformance.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ export class ResourcePerformance {
156156
}
157157

158158
console.time(
159-
` ResourcePerformance.processResourceData.${this.resource.slug} (${initialTimestamp})`
159+
` ResourcePerformance.processResourceData.${this.resource.slug}.total (${initialTimestamp})`
160160
);
161161

162162
this.runtime.processingResourceItems = true;
@@ -211,7 +211,7 @@ export class ResourcePerformance {
211211
);
212212

213213
console.timeEnd(
214-
` ResourcePerformance.processResourceData.${this.resource.slug} (${initialTimestamp})`
214+
` ResourcePerformance.processResourceData.${this.resource.slug}.total (${initialTimestamp})`
215215
);
216216
this.runtime.processingResourceItems = false;
217217
}

‎packages/api/src/performance/resourcePerformanceManager.ts

+102-31
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
import { ResourcePerformance } from './resourcePerformance';
22
import { Resource } from 'src/models/Resource';
3-
3+
import { clearStorageFiles } from './helper';
44
export class ResourcePerformanceManager {
55
private static _instance: ResourcePerformanceManager;
66
private static _initialized: boolean = false;
77
private static _initializing: boolean = false;
88

9+
private actionIdx: number = 0;
10+
911
private resources: Resource[] = [];
1012
private resourcePerformances: {
1113
[resourceSlug: string]: ResourcePerformance;
@@ -27,49 +29,69 @@ export class ResourcePerformanceManager {
2729
if (ResourcePerformanceManager._initializing) {
2830
return;
2931
}
30-
console.time('ResourcePerformanceManager.initialize');
32+
console.time(
33+
`ResourcePerformanceManager.initialize - op# ${this.actionIdx}`
34+
);
3135
ResourcePerformanceManager._initializing = true;
3236
await this.initializeResources(resources, false);
3337
ResourcePerformanceManager._initialized = true;
3438
ResourcePerformanceManager._initializing = false;
35-
console.timeEnd('ResourcePerformanceManager.initialize');
39+
console.timeEnd(
40+
`ResourcePerformanceManager.initialize - op# ${this.actionIdx}`
41+
);
42+
this.actionIdx++;
3643
}
3744

38-
private async initializeResources(
39-
resources: Resource[],
40-
hardInitialize: boolean
41-
) {
42-
this.resources = resources;
43-
for (const resource of this.resources) {
44-
// if (resource.slug != 'ethereum-gas') {
45-
// continue;
46-
// }
47-
this.resourcePerformances[resource.slug] = new ResourcePerformance(
48-
resource
49-
);
50-
if (hardInitialize) {
51-
console.log(
52-
`ResourcePerformanceManager Hard initializing resource ${resource.slug}`
53-
);
54-
await this.resourcePerformances[resource.slug].hardInitialize();
55-
} else {
56-
console.log(
57-
`ResourcePerformanceManager Soft initializing resource ${resource.slug}`
58-
);
59-
await this.resourcePerformances[resource.slug].softInitialize();
60-
}
61-
console.log(`ResourcePerformanceManager Resource ${resource.slug} done`);
45+
public async hardRefreshResource(resourceSlug: string) {
46+
console.log(
47+
`ResourcePerformanceManager Hard Refresh Resource ${resourceSlug} - op# ${this.actionIdx}`
48+
);
49+
const resource = this.resources.find((r) => r.slug === resourceSlug);
50+
if (!resource) {
51+
throw new Error(`Resource ${resourceSlug} not found`);
52+
}
53+
await this.updateResourceCache(resource, true, 'refresh');
54+
console.log(
55+
`ResourcePerformanceManager Hard Refresh Resource ${resourceSlug} done - op# ${this.actionIdx}`
56+
);
57+
this.actionIdx++;
58+
}
59+
60+
public async softRefreshResource(resourceSlug: string) {
61+
console.log(
62+
`ResourcePerformanceManager Soft Refresh Resource ${resourceSlug} - op# ${this.actionIdx}`
63+
);
64+
const resource = this.resources.find((r) => r.slug === resourceSlug);
65+
if (!resource) {
66+
throw new Error(`Resource ${resourceSlug} not found`);
6267
}
68+
await this.updateResourceCache(resource, false, 'refresh');
69+
console.log(
70+
`ResourcePerformanceManager Soft Refresh Resource ${resourceSlug} done - op# ${this.actionIdx}`
71+
);
72+
this.actionIdx++;
6373
}
6474

65-
public async hardRefresh(resources: Resource[]) {
66-
console.log('ResourcePerformanceManager Hard Refresh');
75+
public async hardRefreshAllResources(resources: Resource[]) {
76+
console.log(
77+
`ResourcePerformanceManager Hard Refresh All Resources - op# ${this.actionIdx}`
78+
);
6779
await this.initializeResources(resources, true);
80+
console.log(
81+
`ResourcePerformanceManager Hard Refresh All Resources done - op# ${this.actionIdx}`
82+
);
83+
this.actionIdx++;
6884
}
6985

70-
public async softRefresh(resources: Resource[]) {
71-
console.log('ResourcePerformanceManager Soft Refresh');
86+
public async softRefreshAllResources(resources: Resource[]) {
87+
console.log(
88+
`ResourcePerformanceManager Soft Refresh All Resources - op# ${this.actionIdx}`
89+
);
7290
await this.initializeResources(resources, false);
91+
console.log(
92+
`ResourcePerformanceManager Soft Refresh All Resources done - op# ${this.actionIdx}`
93+
);
94+
this.actionIdx++;
7395
}
7496

7597
public getResourcePerformance(resourceSlug: string) {
@@ -107,4 +129,53 @@ export class ResourcePerformanceManager {
107129
}
108130
return this.resourcePerformances;
109131
}
132+
133+
private async initializeResources(
134+
resources: Resource[],
135+
hardInitialize: boolean
136+
) {
137+
// Clean up existing resource performances
138+
// await Promise.all(
139+
// Object.values(this.resourcePerformances).map(async (rp) => {
140+
// await rp.cleanup(); // TODO implement this method in ResourcePerformance class. It should stop (using AbortController) any running process (db or fs)
141+
// })
142+
// );
143+
144+
// Remove files from disk (hard init will recreate them)
145+
if (hardInitialize) {
146+
await clearStorageFiles();
147+
}
148+
149+
// Get rid of existing resource performances and start fresh
150+
this.resourcePerformances = {};
151+
152+
this.resources = resources;
153+
for (const resource of this.resources) {
154+
this.resourcePerformances[resource.slug] = new ResourcePerformance(
155+
resource
156+
);
157+
await this.updateResourceCache(resource, hardInitialize, 'initialize');
158+
console.log(
159+
`ResourcePerformanceManager Initialize Resource ${resource.slug} done - op# ${this.actionIdx}`
160+
);
161+
}
162+
}
163+
164+
private async updateResourceCache(
165+
resource: Resource,
166+
hardInitialize: boolean,
167+
logMode: 'initialize' | 'refresh'
168+
) {
169+
if (hardInitialize) {
170+
console.log(
171+
`ResourcePerformanceManager Hard ${logMode} resource ${resource.slug}`
172+
);
173+
await this.resourcePerformances[resource.slug].hardInitialize();
174+
} else {
175+
console.log(
176+
`ResourcePerformanceManager Soft ${logMode} resource ${resource.slug}`
177+
);
178+
await this.resourcePerformances[resource.slug].softInitialize();
179+
}
180+
}
110181
}

‎packages/api/src/routes/refreshCache.ts

+49-3
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,15 @@ router.get(
3131
}
3232
}
3333

34-
// For local development
3534
try {
3635
console.log('Starting Cache Refresh');
3736
const resources = await resourceRepository.find();
3837
const resourcePerformanceManager =
3938
ResourcePerformanceManager.getInstance();
4039
if (hardInitialize && hardInitialize.toLowerCase() === 'true') {
41-
await resourcePerformanceManager.hardRefresh(resources);
40+
await resourcePerformanceManager.hardRefreshAllResources(resources);
4241
} else {
43-
await resourcePerformanceManager.softRefresh(resources);
42+
await resourcePerformanceManager.softRefreshAllResources(resources);
4443
}
4544
console.log('Cache Refresh Completed');
4645
res.json({ success: true });
@@ -54,4 +53,51 @@ router.get(
5453
})
5554
);
5655

56+
router.get(
57+
'/refresh/:resourceSlug',
58+
handleAsyncErrors(async (req: Request, res: Response) => {
59+
const { signature, signatureTimestamp, hardInitialize } = req.query as {
60+
signature: string;
61+
signatureTimestamp: string;
62+
hardInitialize: string;
63+
};
64+
const resourceSlug = req.params.resourceSlug.toLowerCase();
65+
const isProduction =
66+
process.env.NODE_ENV === 'production' ||
67+
process.env.NODE_ENV === 'staging';
68+
69+
// For production/staging environments
70+
if (isProduction) {
71+
// Verify signature
72+
const isAuthenticated = await isValidWalletSignature(
73+
signature as `0x${string}`,
74+
Number(signatureTimestamp)
75+
);
76+
if (!isAuthenticated) {
77+
res.status(401).json({ error: 'Unauthorized' });
78+
return;
79+
}
80+
}
81+
82+
try {
83+
console.log(`Starting Cache Refresh for ${resourceSlug}`);
84+
const resourcePerformanceManager =
85+
ResourcePerformanceManager.getInstance();
86+
if (hardInitialize && hardInitialize.toLowerCase() === 'true') {
87+
await resourcePerformanceManager.hardRefreshResource(resourceSlug);
88+
} else {
89+
await resourcePerformanceManager.softRefreshResource(resourceSlug);
90+
}
91+
console.log(`Cache Refresh Completed for ${resourceSlug}`);
92+
res.json({ success: true });
93+
} catch (error: unknown) {
94+
if (error instanceof Error) {
95+
res.status(500).json({ error: error.message });
96+
} else {
97+
res.status(500).json({ error: 'An unknown error occurred' });
98+
}
99+
}
100+
})
101+
);
102+
57103
export { router };

‎packages/app/src/app/admin/page.tsx

+47-4
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ const Admin = () => {
5353
const [indexResourceOpen, setIndexResourceOpen] = useState(false);
5454
const [refreshCacheOpen, setRefreshCacheOpen] = useState(false);
5555
const [selectedResource, setSelectedResource] = useState('');
56+
const [refreshResourceSlug, setRefreshResourceSlug] = useState('all');
5657
const [startTimestamp, setStartTimestamp] = useState('');
5758
const [endTimestamp, setEndTimestamp] = useState('');
5859
const { signMessageAsync } = useSignMessage();
@@ -143,17 +144,25 @@ const Admin = () => {
143144
message: ADMIN_AUTHENTICATE_MSG,
144145
});
145146

146-
const response = await foilApi.get(
147-
`/cache/refresh?hardInitialize=true&signature=${signature}&signatureTimestamp=${timestamp}`
148-
);
147+
// Build the endpoint URL based on whether a specific resource is selected
148+
const endpoint =
149+
refreshResourceSlug && refreshResourceSlug !== 'all'
150+
? `/cache/refresh/${refreshResourceSlug}?hardInitialize=true&signature=${signature}&signatureTimestamp=${timestamp}`
151+
: `/cache/refresh?hardInitialize=true&signature=${signature}&signatureTimestamp=${timestamp}`;
152+
153+
const response = await foilApi.get(endpoint);
149154

150155
if (response.success) {
151156
toast({
152157
title: 'Cache refreshed',
153-
description: 'Cache has been successfully refreshed',
158+
description:
159+
refreshResourceSlug && refreshResourceSlug !== 'all'
160+
? `Cache for ${refreshResourceSlug} has been successfully refreshed`
161+
: 'Cache has been successfully refreshed for all resources',
154162
variant: 'default',
155163
});
156164
setRefreshCacheOpen(false);
165+
setRefreshResourceSlug('all'); // Reset to "all" instead of empty string
157166
} else {
158167
toast({
159168
title: 'Cache refresh failed',
@@ -345,6 +354,40 @@ const Admin = () => {
345354
operation requires authentication.
346355
</p>
347356

357+
<div className="space-y-2">
358+
<label className="block">
359+
<span className="text-sm font-medium">Resource (Optional)</span>
360+
<Select
361+
value={refreshResourceSlug}
362+
onValueChange={setRefreshResourceSlug}
363+
>
364+
<SelectTrigger>
365+
<SelectValue
366+
placeholder={
367+
resourcesLoading
368+
? 'Loading...'
369+
: 'All resources (default)'
370+
}
371+
/>
372+
</SelectTrigger>
373+
<SelectContent>
374+
<SelectItem value="all">All resources</SelectItem>
375+
{resourcesData?.map(
376+
(resource: { slug: string; name: string }) => (
377+
<SelectItem key={resource.slug} value={resource.slug}>
378+
{resource.name}
379+
</SelectItem>
380+
)
381+
)}
382+
</SelectContent>
383+
</Select>
384+
</label>
385+
<p className="text-xs text-muted-foreground">
386+
Select a specific resource to refresh, or leave empty to refresh
387+
all resources.
388+
</p>
389+
</div>
390+
348391
<Button
349392
onClick={handleRefreshCache}
350393
disabled={loadingAction.refreshCache}

0 commit comments

Comments
 (0)