Skip to content

Commit 636dc4e

Browse files
RomanBobrovskiyRomanBobrovskiy
authored
Merge pull request #52 from DBB-Software/feat/PLATFORM-1745-dynamodb
feat: added dynamo db
2 parents 0a75634 + 3f98ca8 commit 636dc4e

17 files changed

+2079
-275
lines changed

‎package-lock.json‎

Lines changed: 1712 additions & 208 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
"@semantic-release/git": "10.0.1",
4040
"@semantic-release/npm": "12.0.1",
4141
"@types/aws-lambda": "8.10.138",
42+
"@types/express": "5.0.0",
4243
"@types/jest": "29.5.14",
4344
"@types/lodash": "4.17.13",
4445
"@types/node": "20.12.11",
@@ -61,6 +62,7 @@
6162
"dependencies": {
6263
"@aws-sdk/client-cloudformation": "3.590.0",
6364
"@aws-sdk/client-cloudfront": "3.590.0",
65+
"@aws-sdk/client-dynamodb": "3.709.0",
6466
"@aws-sdk/client-elastic-beanstalk": "3.590.0",
6567
"@aws-sdk/client-s3": "3.591.0",
6668
"@aws-sdk/client-sqs": "3.682.0",
@@ -70,9 +72,11 @@
7072
"@dbbs/next-cache-handler-core": "1.3.0",
7173
"aws-cdk-lib": "2.144.0",
7274
"aws-sdk": "2.1635.0",
75+
"body-parser": "^1.20.3",
7376
"cdk-assets": "2.144.0",
7477
"constructs": "10.3.0",
7578
"esbuild": "0.21.4",
79+
"express": "4.21.2",
7680
"lodash": "4.17.21",
7781
"yargs": "17.7.2"
7882
},

‎src/build/cache/revalidateServer.ts‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
import express from 'express'
2+
import { json } from 'body-parser'
3+
import { S3 } from '@aws-sdk/client-s3'
4+
import { DynamoDB, type AttributeValue } from '@aws-sdk/client-dynamodb'
5+
import http from 'http'
6+
import { chunkArray } from '../../common/array'
7+
8+
const port = parseInt(process.env.PORT || '', 10) || 3000
9+
const nextServerPort = 3001
10+
const nextServerHostname = process.env.HOSTNAME || '0.0.0.0'
11+
12+
const PAGE_CACHE_EXTENSIONS = ['json', 'html', 'rsc']
13+
const CHUNK_LIMIT = 1000
14+
const DYNAMODB_BATCH_LIMIT = 25
15+
16+
interface RevalidateBody {
17+
paths: string[]
18+
cacheSegment?: string
19+
}
20+
21+
const s3 = new S3({ region: process.env.AWS_REGION })
22+
const dynamoDB = new DynamoDB({ region: process.env.AWS_REGION })
23+
24+
async function deleteS3Objects(bucketName: string, keys: string[]) {
25+
if (!keys.length) return
26+
27+
// Delete objects in chunks to stay within AWS limits
28+
await Promise.allSettled(
29+
chunkArray(keys, CHUNK_LIMIT).map((chunk) => {
30+
return s3.deleteObjects({
31+
Bucket: bucketName,
32+
Delete: { Objects: chunk.map((Key) => ({ Key })) }
33+
})
34+
})
35+
)
36+
}
37+
38+
async function batchDeleteFromDynamoDB(tableName: string, items: Record<string, AttributeValue>[]) {
39+
if (!items.length) return
40+
41+
// Split items into chunks of 25 (DynamoDB batch limit)
42+
const chunks = chunkArray(items, DYNAMODB_BATCH_LIMIT)
43+
44+
await Promise.all(
45+
chunks.map(async (chunk) => {
46+
const deleteRequests = chunk.map((item) => ({
47+
DeleteRequest: {
48+
Key: item
49+
}
50+
}))
51+
52+
try {
53+
await dynamoDB.batchWriteItem({
54+
RequestItems: {
55+
[tableName]: deleteRequests
56+
}
57+
})
58+
} catch (error) {
59+
console.error('Error in batch delete:', error)
60+
// Handle unprocessed items if needed
61+
throw error
62+
}
63+
})
64+
)
65+
}
66+
67+
const app = express()
68+
69+
app.use(json())
70+
71+
app.post('/api/revalidate-pages', async (req, res) => {
72+
try {
73+
const { paths, cacheSegment } = req.body as RevalidateBody
74+
75+
if (!paths.length) {
76+
res.status(400).json({ Message: 'paths is required.' }).end()
77+
} else {
78+
const attributeValues: Record<string, AttributeValue> = {}
79+
const keyConditionExpression =
80+
paths.length === 1 ? 'pageKey = :path0' : 'pageKey IN (' + paths.map((_, i) => `:path${i}`).join(',') + ')'
81+
82+
paths.forEach((path, index) => {
83+
attributeValues[`:path${index}`] = { S: path.substring(1) }
84+
})
85+
86+
if (cacheSegment) {
87+
attributeValues[':segment'] = { S: cacheSegment }
88+
}
89+
90+
const result = await dynamoDB.query({
91+
TableName: process.env.DYNAMODB_CACHE_TABLE!,
92+
IndexName: 'cacheKey-index',
93+
KeyConditionExpression: keyConditionExpression,
94+
FilterExpression: cacheSegment ? 'cacheKey = :segment' : undefined,
95+
ExpressionAttributeValues: attributeValues
96+
})
97+
98+
if (result?.Items?.length) {
99+
const s3KeysToDelete = result.Items.flatMap((item) => {
100+
return PAGE_CACHE_EXTENSIONS.map((ext) => `${item.s3Key.S}.${ext}`)
101+
})
102+
await deleteS3Objects(process.env.STATIC_BUCKET_NAME!, s3KeysToDelete)
103+
await batchDeleteFromDynamoDB(process.env.DYNAMODB_CACHE_TABLE!, result.Items)
104+
}
105+
106+
await Promise.all(
107+
paths.map((path) =>
108+
http.get({
109+
hostname: nextServerHostname,
110+
port: nextServerPort,
111+
path
112+
})
113+
)
114+
)
115+
}
116+
117+
res.status(200).json({ Message: 'Revalidated.' })
118+
} catch (err) {
119+
res.status(400).json({ Message: err })
120+
}
121+
})
122+
123+
app.use((_req, res) => {
124+
res.status(404).json({ error: 'Not found' })
125+
})
126+
127+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
128+
app.use((err: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
129+
console.error('Server error:', err)
130+
res.status(500).json({ error: 'Internal server error' })
131+
})
132+
133+
app.listen(port, () => {
134+
console.log(`> Revalidation server ready on port ${port}`)
135+
})

‎src/build/next.ts‎

Lines changed: 6 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,9 @@ import fs from 'fs/promises'
33
import path from 'node:path'
44
import type { PrerenderManifest, RoutesManifest } from 'next/dist/build'
55
import { type ProjectPackager, type ProjectSettings } from '../common/project'
6-
import appRouterRevalidateTemplate from './cache/handlers/appRouterRevalidate'
76

87
interface BuildOptions {
98
packager: ProjectPackager
10-
nextConfigPath: string
11-
isAppDir: boolean
12-
projectPath: string
139
}
1410

1511
interface BuildAppOptions {
@@ -23,27 +19,11 @@ const setNextEnvs = () => {
2319
process.env.NEXT_SERVERLESS_DEPLOYING_PHASE = 'true'
2420
}
2521

26-
const appendRevalidateApi = async (projectPath: string, isAppDir: boolean): Promise<string> => {
27-
const routeFolderPath = path.join(projectPath, isAppDir ? 'src/app' : 'src', 'api', 'revalidate')
28-
const routePath = path.join(routeFolderPath, 'route.ts')
29-
30-
await fs.mkdir(routeFolderPath, { recursive: true })
31-
await fs.writeFile(routePath, appRouterRevalidateTemplate, 'utf-8')
32-
33-
return routePath
34-
}
35-
36-
export const buildNext = async (options: BuildOptions): Promise<() => Promise<void>> => {
37-
const { packager, projectPath, isAppDir } = options
22+
export const buildNext = async (options: BuildOptions) => {
23+
const { packager } = options
3824

3925
setNextEnvs()
40-
const revalidateRoutePath = await appendRevalidateApi(projectPath, isAppDir)
4126
childProcess.execSync(packager.buildCommand, { stdio: 'inherit' })
42-
43-
// Reverts changes to the next project
44-
return async () => {
45-
await fs.rm(revalidateRoutePath)
46-
}
4727
}
4828

4929
const copyAssets = async (outputPath: string, appPath: string, appRelativePath: string) => {
@@ -85,19 +65,16 @@ export const getNextCachedRoutesMatchers = async (outputPath: string, appRelativ
8565
export const buildApp = async (options: BuildAppOptions) => {
8666
const { projectSettings, outputPath } = options
8767

88-
const { packager, nextConfigPath, projectPath, isAppDir, root, isMonorepo } = projectSettings
68+
const { packager, projectPath, root, isMonorepo } = projectSettings
8969

90-
const cleanNextApp = await buildNext({
91-
packager,
92-
nextConfigPath,
93-
isAppDir,
94-
projectPath
70+
await buildNext({
71+
packager
9572
})
9673

9774
const appRelativePath = isMonorepo ? path.relative(root, projectPath) : ''
9875

9976
await copyAssets(outputPath, projectPath, appRelativePath)
10077
const nextCachedRoutesMatchers = await getNextCachedRoutesMatchers(outputPath, appRelativePath)
10178

102-
return { cleanNextApp, nextCachedRoutesMatchers }
79+
return { nextCachedRoutesMatchers }
10380
}

‎src/cacheHandler/strategy/s3.spec.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ describe('S3Cache', () => {
165165

166166
expect(await s3Cache.get(cacheKey, cacheKey)).toEqual(mockCacheEntryWithTags.value.pageData)
167167

168-
await s3Cache.revalidateTag(cacheKey, [])
168+
await s3Cache.revalidateTag(cacheKey)
169169

170170
expect(await s3Cache.get(cacheKey, cacheKey)).toBeNull()
171171
})

‎src/cacheHandler/strategy/s3.ts‎

Lines changed: 50 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { NEXT_CACHE_TAGS_HEADER } from 'next/dist/lib/constants'
2-
import { ListObjectsV2CommandOutput, S3 } from '@aws-sdk/client-s3'
3-
import { PutObjectCommandInput } from '@aws-sdk/client-s3/dist-types/commands/PutObjectCommand'
2+
import { type ListObjectsV2CommandOutput, type PutObjectCommandInput, S3 } from '@aws-sdk/client-s3'
3+
import { DynamoDB } from '@aws-sdk/client-dynamodb'
44
import { chunkArray } from '../../common/array'
55
import type { CacheEntry, CacheStrategy, CacheContext } from '@dbbs/next-cache-handler-core'
66

@@ -13,16 +13,17 @@ enum CacheExtension {
1313
}
1414
const PAGE_CACHE_EXTENSIONS = Object.values(CacheExtension)
1515
const CHUNK_LIMIT = 1000
16-
const EXT_REGEX = new RegExp(`.(${PAGE_CACHE_EXTENSIONS.join('|')})$`)
1716

1817
export class S3Cache implements CacheStrategy {
1918
public readonly client: S3
2019
public readonly bucketName: string
20+
#dynamoDBClient: DynamoDB
2121

2222
constructor(bucketName: string) {
2323
const region = process.env.AWS_REGION
2424
this.client = new S3({ region })
2525
this.bucketName = bucketName
26+
this.#dynamoDBClient = new DynamoDB({ region })
2627
}
2728

2829
buildTagKeys(tags?: string | string[]) {
@@ -69,12 +70,25 @@ export class S3Cache implements CacheStrategy {
6970
Metadata: {
7071
'Cache-Fragment-Key': cacheKey
7172
},
72-
...(data.revalidate ? { CacheControl: `max-age=${data.revalidate}` } : undefined)
73+
...(data.revalidate ? { CacheControl: `smax-age=${data.revalidate}, stale-while-revalidate` } : undefined)
7374
}
7475

7576
if (data.value?.kind === 'PAGE' || data.value?.kind === 'ROUTE') {
7677
const headersTags = this.buildTagKeys(data.value.headers?.[NEXT_CACHE_TAGS_HEADER]?.toString())
77-
const input: PutObjectCommandInput = { ...baseInput, ...(headersTags ? { Tagging: headersTags } : {}) }
78+
const input: PutObjectCommandInput = { ...baseInput }
79+
80+
promises.push(
81+
this.#dynamoDBClient.putItem({
82+
TableName: process.env.DYNAMODB_CACHE_TABLE!,
83+
Item: {
84+
pageKey: { S: pageKey },
85+
cacheKey: { S: cacheKey },
86+
s3Key: { S: baseInput.Key! },
87+
tags: { S: [headersTags, this.buildTagKeys(data.tags)].filter(Boolean).join('&') },
88+
createdAt: { S: new Date().toISOString() }
89+
}
90+
})
91+
)
7892

7993
if (data.value?.kind === 'PAGE') {
8094
promises.push(
@@ -119,18 +133,32 @@ export class S3Cache implements CacheStrategy {
119133
...baseInput,
120134
Key: `${baseInput.Key}.${CacheExtension.JSON}`,
121135
Body: JSON.stringify(data),
122-
ContentType: 'application/json',
123-
...(data.tags?.length ? { Tagging: `${this.buildTagKeys(data.tags)}` } : {})
136+
ContentType: 'application/json'
137+
// ...(data.tags?.length ? { Tagging: `${this.buildTagKeys(data.tags)}` } : {})
124138
})
125139
)
126140
}
127141

128142
await Promise.all(promises)
129143
}
130144

131-
async revalidateTag(tag: string, allowCacheKeys: string[]): Promise<void> {
145+
async revalidateTag(tag: string): Promise<void> {
132146
const keysToDelete: string[] = []
133147
let nextContinuationToken: string | undefined = undefined
148+
149+
const result = await this.#dynamoDBClient.query({
150+
TableName: process.env.DYNAMODB_CACHE_TABLE!,
151+
KeyConditionExpression: '#field = :value',
152+
ExpressionAttributeNames: {
153+
'#field': 'tags'
154+
},
155+
ExpressionAttributeValues: {
156+
':value': { S: tag }
157+
}
158+
})
159+
160+
console.log('HERE_IS_RESULT', result)
161+
console.log('HERE_IS_RESULT_ITEMS', result.Items)
134162
do {
135163
const { Contents: contents = [], NextContinuationToken: token }: ListObjectsV2CommandOutput =
136164
await this.client.listObjectsV2({
@@ -141,11 +169,9 @@ export class S3Cache implements CacheStrategy {
141169

142170
keysToDelete.push(
143171
...(await contents.reduce<Promise<string[]>>(async (acc, { Key: key }) => {
144-
if (
145-
!key ||
146-
(allowCacheKeys.length && !allowCacheKeys.some((allowKey) => key.replace(EXT_REGEX, '').endsWith(allowKey)))
147-
)
172+
if (!key) {
148173
return acc
174+
}
149175

150176
const { TagSet = [] } = await this.client.getObjectTagging({ Bucket: this.bucketName, Key: key })
151177
const tags = TagSet.filter(({ Key: key }) => key?.startsWith(TAG_PREFIX)).map(({ Value: tags }) => tags || '')
@@ -163,6 +189,7 @@ export class S3Cache implements CacheStrategy {
163189
}
164190

165191
async delete(pageKey: string, cacheKey: string): Promise<void> {
192+
console.log('HERE_IS_CALL_DELETE')
166193
await this.client.deleteObjects({
167194
Bucket: this.bucketName,
168195
Delete: { Objects: PAGE_CACHE_EXTENSIONS.map((ext) => ({ Key: `${pageKey}/${cacheKey}.${ext}` })) }
@@ -172,6 +199,17 @@ export class S3Cache implements CacheStrategy {
172199
async deleteAllByKeyMatch(pageKey: string, cacheKey: string): Promise<void> {
173200
if (cacheKey) {
174201
await this.deleteObjects(PAGE_CACHE_EXTENSIONS.map((ext) => `${pageKey}/${cacheKey}.${ext}`))
202+
await this.#dynamoDBClient.deleteItem({
203+
TableName: process.env.DYNAMODB_CACHE_TABLE!,
204+
Key: {
205+
pageKey: {
206+
S: pageKey
207+
},
208+
cacheKey: {
209+
S: cacheKey
210+
}
211+
}
212+
})
175213
return
176214
}
177215
const keysToDelete: string[] = []

0 commit comments

Comments
 (0)