Skip to content

Commit fa04441

Browse files
chenaskidenis-urban
chenaski
and
denis-urban
authored
Add RequestQueue for fetching queue, retries and errors (#48)
* Add `RequestQueue` for fetching queue, retries and errors * bump up package version --------- Co-authored-by: Denis Urban <[email protected]>
1 parent 8b4069b commit fa04441

File tree

4 files changed

+223
-34
lines changed

4 files changed

+223
-34
lines changed

‎CLAUDE.md‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# CLAUDE.md
2+
3+
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
4+
5+
## Commands
6+
7+
### Development
8+
- `yarn dev` - Watch and compile code continuously
9+
- `yarn storybook` - Run Storybook on port 6001
10+
- `yarn tdd` - Run Jest tests in watch mode
11+
- `yarn start` - Run dev, storybook, and tdd in parallel
12+
- `yarn test` - Run all Jest tests
13+
- `yarn test:smoke` - Run smoke tests with open handles detection
14+
- `yarn update-schema` - Update GraphQL introspection schema
15+
- `yarn prepare` - Build the package using package-prepare
16+
- `npm publish` - Transpile and publish to NPM
17+
18+
### Testing
19+
Tests are located in the `tests/` directory. To run a specific test file:
20+
```bash
21+
yarn test tests/[filename].test.js
22+
```
23+
24+
## Architecture
25+
26+
This package is a GraphQL content layer for fetching and processing conference content from GraphCMS. It:
27+
28+
1. **Fetches data** from GraphCMS using GraphQL queries through multiple fetch modules (`fetch-*.js`)
29+
2. **Processes content** through a post-processing layer that merges talks, Q&A sessions, and populates speaker activities
30+
3. **Exposes content** via the `getContent` async function for consumption
31+
4. **Generates Storybook** for visualizing both CMS and content layers
32+
33+
### Key Components
34+
35+
- **Entry point**: `src/index.js` - Creates GraphQL client and orchestrates all content fetching
36+
- **Content fetchers**: `src/fetch-*.js` files - Each handles a specific content type (speakers, talks, sponsors, etc.)
37+
- **Post-processing**: `src/postprocess.js` - Merges and enriches content relationships
38+
- **Configuration**: Requires `CMS_ENDPOINT` and `CMS_TOKEN` environment variables for GraphCMS connection
39+
- **Conference settings**: Must be passed to `getContent()` with conference-specific data including `conferenceTitle`, `eventYear`, `tagColors`, and `speakerAvatar` dimensions
40+
41+
### Content Flow
42+
1. Conference settings are passed to `getContent(conferenceSettings)`
43+
2. All fetch modules run in parallel via Promise.all
44+
3. Content pieces are merged with conflict resolution for duplicate keys
45+
4. Post-processing enriches the content (populates speaker talks, merges Q&A sessions)
46+
5. Schedule items are sorted chronologically
47+
6. Final processed content is returned
48+
49+
### GraphQL Schema
50+
The GraphQL schema is stored in `schema.graphql` and can be updated using `yarn update-schema`. The schema endpoint is configured in `.graphqlconfig`.

‎package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@focus-reactive/graphql-content-layer",
3-
"version": "3.2.6",
3+
"version": "3.2.7",
44
"private": false,
55
"main": "dist/index.js",
66
"scripts": {
@@ -94,4 +94,4 @@
9494
"gitnation",
9595
"conference"
9696
]
97-
}
97+
}

‎src/index.js‎

Lines changed: 59 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
const { GraphQLClient } = require('graphql-request');
22

33
const { credentials } = require('./config');
4+
const RequestQueue = require('./request-queue');
45
const textContent = require('./fetch-texts');
56
const pageContent = require('./fetch-pages');
67
const brandContent = require('./fetch-brand');
@@ -42,40 +43,65 @@ const getQueriesData = (content, conferenceSettings) => {
4243
};
4344

4445
const getContent = async conferenceSettings => {
45-
const fetchAll = [
46-
textContent,
47-
pageContent,
48-
brandContent,
49-
speakerContent,
50-
advisersContent,
51-
performanceContent,
52-
sponsorContent,
53-
talksContent,
54-
workshopContent,
55-
mcContent,
56-
faqContent,
57-
extContent,
58-
jobsContent,
59-
committeeContent,
60-
diversityContent,
61-
latestLinksContent,
62-
].map(async content => {
63-
try {
64-
getQueriesData(content, conferenceSettings);
65-
const getVarsFromSettings = content.selectSettings || (() => undefined);
66-
const { conferenceTitle, eventYear } = conferenceSettings;
67-
return await content.fetchData(client, {
68-
conferenceTitle,
69-
eventYear,
70-
...getVarsFromSettings(conferenceSettings),
71-
});
72-
} catch (err) {
73-
console.error(err);
74-
process.exit(1);
75-
}
46+
const queue = new RequestQueue({
47+
concurrency: 5,
48+
retryAttempts: 3,
49+
retryDelay: 2000,
50+
maxRetryDelay: 30000,
51+
timeout: 60000,
52+
});
53+
54+
const contentModules = [
55+
{ module: textContent, name: 'texts' },
56+
{ module: pageContent, name: 'pages' },
57+
{ module: brandContent, name: 'brand' },
58+
{ module: speakerContent, name: 'speakers' },
59+
{ module: advisersContent, name: 'advisers' },
60+
{ module: performanceContent, name: 'performance' },
61+
{ module: sponsorContent, name: 'sponsors' },
62+
{ module: talksContent, name: 'talks' },
63+
{ module: workshopContent, name: 'workshops' },
64+
{ module: mcContent, name: 'mc' },
65+
{ module: faqContent, name: 'faq' },
66+
{ module: extContent, name: 'extended' },
67+
{ module: jobsContent, name: 'jobs' },
68+
{ module: committeeContent, name: 'committee' },
69+
{ module: diversityContent, name: 'diversity' },
70+
{ module: latestLinksContent, name: 'landings' },
71+
];
72+
73+
const fetchPromises = contentModules.map(({ module: content, name }) => {
74+
return queue.add(async () => {
75+
try {
76+
getQueriesData(content, conferenceSettings);
77+
const getVarsFromSettings = content.selectSettings || (() => undefined);
78+
const { conferenceTitle, eventYear } = conferenceSettings;
79+
80+
// eslint-disable-next-line no-console
81+
console.log(`Fetching ${name} for ${conferenceTitle} ${eventYear}`);
82+
83+
const result = await content.fetchData(client, {
84+
conferenceTitle,
85+
eventYear,
86+
...getVarsFromSettings(conferenceSettings),
87+
});
88+
89+
// eslint-disable-next-line no-console
90+
console.log(
91+
`Successfully fetched ${name} for ${conferenceTitle} ${eventYear}`,
92+
);
93+
return result;
94+
} catch (err) {
95+
console.error(
96+
`Failed to fetch ${name} for ${conferenceSettings.conferenceTitle}:`,
97+
err,
98+
);
99+
throw err;
100+
}
101+
}, name);
76102
});
77103

78-
const contentArray = await Promise.all(fetchAll);
104+
const contentArray = await Promise.all(fetchPromises);
79105
const contentMap = contentArray.reduce((content, piece) => {
80106
try {
81107
const newKeys = Object.keys(piece);
@@ -86,6 +112,7 @@ const getContent = async conferenceSettings => {
86112
piece[k] = { ...content[k], ...piece[k] };
87113
});
88114
} catch (err) {
115+
// eslint-disable-next-line no-console
89116
console.log('content, piece', piece);
90117
console.error(err);
91118
}

‎src/request-queue.js‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
2+
3+
class RequestQueue {
4+
constructor(options = {}) {
5+
this.concurrency = options.concurrency || 5;
6+
this.retryAttempts = options.retryAttempts || 3;
7+
this.retryDelay = options.retryDelay || 1000;
8+
this.maxRetryDelay = options.maxRetryDelay || 30000;
9+
this.timeout = options.timeout || 60000;
10+
11+
this.queue = [];
12+
this.running = 0;
13+
this.results = new Map();
14+
}
15+
16+
async add(fn, id) {
17+
return new Promise((resolve, reject) => {
18+
this.queue.push({ fn, id, resolve, reject, attempts: 0 });
19+
this.process();
20+
});
21+
}
22+
23+
async process() {
24+
while (this.running < this.concurrency && this.queue.length > 0) {
25+
const task = this.queue.shift();
26+
this.running++;
27+
this.executeTask(task);
28+
}
29+
}
30+
31+
async executeTask(task) {
32+
const { fn, id, resolve, reject, attempts } = task;
33+
34+
try {
35+
const timeoutPromise = new Promise((_, reject) =>
36+
setTimeout(
37+
() => reject(new Error(`Request timeout after ${this.timeout}ms`)),
38+
this.timeout,
39+
),
40+
);
41+
42+
const result = await Promise.race([fn(), timeoutPromise]);
43+
44+
resolve(result);
45+
if (id) {
46+
this.results.set(id, { success: true, data: result });
47+
}
48+
} catch (error) {
49+
const isRetriableError = this.isRetriable(error);
50+
const nextAttempt = attempts + 1;
51+
52+
if (isRetriableError && nextAttempt < this.retryAttempts) {
53+
const delay = Math.min(
54+
this.retryDelay * Math.pow(2, attempts),
55+
this.maxRetryDelay,
56+
);
57+
58+
console.warn(
59+
`Request failed (attempt ${nextAttempt}/${this.retryAttempts}), retrying in ${delay}ms...`,
60+
{
61+
error: error.message || error,
62+
id,
63+
},
64+
);
65+
66+
await sleep(delay);
67+
68+
task.attempts = nextAttempt;
69+
this.queue.unshift(task);
70+
} else {
71+
console.error(`Request failed after ${nextAttempt} attempts`, {
72+
error: error.message || error,
73+
id,
74+
});
75+
76+
if (id) {
77+
this.results.set(id, { success: false, error });
78+
}
79+
reject(error);
80+
}
81+
} finally {
82+
this.running--;
83+
this.process();
84+
}
85+
}
86+
87+
isRetriable(error) {
88+
const errorMessage = error.message || '';
89+
const errorCode = error.code || '';
90+
const statusCode = error.response && error.response.status;
91+
92+
if (statusCode === 429) return true;
93+
94+
if (errorCode === 'ETIMEDOUT' || errorCode === 'ECONNRESET') return true;
95+
96+
if (
97+
errorMessage.includes('timeout') ||
98+
errorMessage.includes('ETIMEDOUT') ||
99+
errorMessage.includes('429') ||
100+
errorMessage.includes('Too Many Requests') ||
101+
errorMessage.includes('rate limit')
102+
) {
103+
return true;
104+
}
105+
106+
if (statusCode >= 500 && statusCode < 600) return true;
107+
108+
return false;
109+
}
110+
}
111+
112+
module.exports = RequestQueue;

0 commit comments

Comments
 (0)