feat/platform 1627 - revalidate server

package.json

@@ -66,6 +66,7 @@
     "@aws-sdk/client-dynamodb": "3.709.0",
     "@aws-sdk/client-elastic-beanstalk": "3.590.0",
     "@aws-sdk/client-s3": "3.591.0",
+    "@aws-sdk/client-secrets-manager": "3.758.0",
     "@aws-sdk/client-sqs": "3.682.0",
     "@aws-sdk/client-sts": "3.590.0",
     "@aws-sdk/credential-providers": "3.590.0",
@@ -80,6 +81,7 @@
     "express": "4.21.2",
     "lodash": "4.17.21",
     "mime-types": "2.1.35",
+    "minimatch": "10.0.1",
     "yargs": "17.7.2"
   },
   "peerDependencies": {

src/build/cache/revalidateServer.ts

@@ -1,7 +1,11 @@
 import express from 'express'
 import { json } from 'body-parser'
 import http from 'http'
-
+import { S3Client, DeleteObjectsCommand, ListObjectsV2Command } from '@aws-sdk/client-s3'
+import { DynamoDBClient, BatchWriteItemCommand, ScanCommand, AttributeValue } from '@aws-sdk/client-dynamodb'
+import { CloudFrontClient, CreateInvalidationCommand } from '@aws-sdk/client-cloudfront'
+import { minimatch } from 'minimatch'
+import chunk from 'lodash/chunk'
 const port = parseInt(process.env.PORT || '', 10) || 3000
 const nextServerPort = 3001
 const nextServerHostname = process.env.HOSTNAME || '0.0.0.0'
@@ -14,6 +18,122 @@ const app = express()
 
 app.use(json())
 
+const s3Client = new S3Client({ region: process.env.AWS_REGION })
+const dynamoClient = new DynamoDBClient({ region: process.env.AWS_REGION })
+const cloudfrontClient = new CloudFrontClient({ region: process.env.AWS_REGION })
+
+function transformPathPattern(pattern: string) {
+  const cleanedPattern = pattern.startsWith('/') ? pattern.slice(1) : pattern
+  const hasWildcard = cleanedPattern.includes('*')
+  const prefix = cleanedPattern.split('*')[0]
+  const minimatchString = hasWildcard
+    ? cleanedPattern.replace(/\/\*/g, '/**')
+    : cleanedPattern.endsWith('/')
+      ? cleanedPattern + '*'
+      : cleanedPattern + '/*'
+  return { prefix, hasWildcard, minimatchString }
+}
+
+async function listS3Objects(pattern: string) {
+  const { prefix, minimatchString } = transformPathPattern(pattern)
+
+  const { Contents = [] } = await s3Client.send(
+    new ListObjectsV2Command({
+      Bucket: process.env.STATIC_BUCKET_NAME,
+      Prefix: prefix
+    })
+  )
+
+  return Contents.filter((obj) => obj.Key && minimatch(obj.Key, minimatchString)).map((obj) => ({
+    Key: obj.Key!
+  }))
+}
+
+async function listDynamoItems(pattern: string): Promise<{ key: Record<string, AttributeValue> }[]> {
+  const { prefix, minimatchString } = transformPathPattern(pattern)
+
+  const { Items = [] } = await dynamoClient.send(
+    new ScanCommand({
+      TableName: process.env.DYNAMODB_CACHE_TABLE,
+      FilterExpression: 'contains(pageKey, :pattern)',
+      ExpressionAttributeValues: {
+        ':pattern': { S: prefix }
+      }
+    })
+  )
+
+  return Items.filter((item) => item.pageKey?.S && minimatch(item.pageKey.S, minimatchString)).map((item) => ({
+    key: { pageKey: item.pageKey }
+  }))
+}
+
+async function deleteS3Objects(objects: { Key: string }[]) {
+  if (!objects.length) return
+
+  // S3 can delete max 1000 objects in one call
+  const chunks = chunk(objects, 1000)
+  await Promise.all(
+    chunks.map((batch) =>
+      s3Client.send(
+        new DeleteObjectsCommand({
+          Bucket: process.env.STATIC_BUCKET_NAME,
+          Delete: { Objects: batch }
+        })
+      )
+    )
+  )
+}
+
+async function deleteDynamoItems(items: { key: Record<string, AttributeValue> }[]) {
+  if (!items.length) return
+
+  const chunks = chunk(items, 10)
+  await Promise.all(
+    chunks.map((batch) =>
+      dynamoClient.send(
+        new BatchWriteItemCommand({
+          RequestItems: {
+            [process.env.DYNAMODB_CACHE_TABLE!]: batch.map((item) => ({
+              DeleteRequest: { Key: item.key }
+            }))
+          }
+        })
+      )
+    )
+  )
+}
+
+function categorizePaths(paths: string[]) {
+  return paths.reduce(
+    (acc, path) => {
+      if (path.includes('*')) {
+        acc.wildcardPaths.push(path)
+      } else {
+        acc.exactPaths.push(path)
+      }
+      return acc
+    },
+    { wildcardPaths: [] as string[], exactPaths: [] as string[] }
+  )
+}
+
+async function revalidateNextPages(paths: string[]) {
+  await Promise.all(
+    paths.map((path) =>
+      http.get({
+        hostname: nextServerHostname,
+        port: nextServerPort,
+        path
+      })
+    )
+  )
+}
+
+async function handleWildcardPath(wildcardPath: string) {
+  const [s3Objects, dynamoItems] = await Promise.all([listS3Objects(wildcardPath), listDynamoItems(wildcardPath)])
+  return Promise.all([deleteS3Objects(s3Objects), deleteDynamoItems(dynamoItems)])
+}
+
 app.post('/api/revalidate-pages', async (req, res) => {
   try {
     const { paths } = req.body as RevalidateBody
@@ -23,14 +143,18 @@ app.post('/api/revalidate-pages', async (req, res) => {
       return
     }
 
-    await Promise.all(
-      paths.map((path) =>
-        http.get({
-          hostname: nextServerHostname,
-          port: nextServerPort,
-          path
-        })
-      )
+    const { exactPaths, wildcardPaths } = categorizePaths(paths)
+
+    await Promise.all([revalidateNextPages(exactPaths), ...wildcardPaths.map(handleWildcardPath)])
+
+    await cloudfrontClient.send(
+      new CreateInvalidationCommand({
+        DistributionId: process.env.CLOUDFRONT_DISTRIBUTION_ID,
+        InvalidationBatch: {
+          Paths: { Quantity: paths.length, Items: paths },
+          CallerReference: Date.now().toString()
+        }
+      })
     )
 
     res.status(200).json({ Message: 'Revalidated.' })

src/cdk/constructs/CloudFrontDistribution.ts

@@ -2,6 +2,7 @@ import { Construct } from 'constructs'
 import { Duration } from 'aws-cdk-lib'
 import * as cloudfront from 'aws-cdk-lib/aws-cloudfront'
 import * as s3 from 'aws-cdk-lib/aws-s3'
+import * as lambda from 'aws-cdk-lib/aws-lambda'
 import * as origins from 'aws-cdk-lib/aws-cloudfront-origins'
 import { addOutput } from '../../common/cdk'
 import { DeployConfig } from '../../types'
@@ -13,6 +14,7 @@ interface CloudFrontPropsDistribution {
   requestEdgeFunction: cloudfront.experimental.EdgeFunction
   viewerResponseEdgeFunction: cloudfront.experimental.EdgeFunction
   viewerRequestLambdaEdge: cloudfront.experimental.EdgeFunction
+  revalidateLambdaUrl: lambda.FunctionUrl
   deployConfig: DeployConfig
   imageTTL?: number
 }
@@ -35,6 +37,7 @@ export class CloudFrontDistribution extends Construct {
       requestEdgeFunction,
       viewerResponseEdgeFunction,
       viewerRequestLambdaEdge,
+      revalidateLambdaUrl,
       deployConfig,
       renderServerDomain,
       imageTTL
@@ -102,6 +105,8 @@ export class CloudFrontDistribution extends Construct {
       httpPort: 80
     })
 
+    const revalidateLambdaOrigin = new origins.FunctionUrlOrigin(revalidateLambdaUrl)
+
     this.cf = new cloudfront.Distribution(this, id, {
       defaultBehavior: {
         origin: s3Origin,
@@ -135,6 +140,13 @@ export class CloudFrontDistribution extends Construct {
           cachePolicy: splitCachePolicy,
           compress: true
         },
+        '/_next/revalidate': {
+          origin: revalidateLambdaOrigin,
+          cachePolicy: cloudfront.CachePolicy.CACHING_DISABLED,
+          originRequestPolicy: cloudfront.OriginRequestPolicy.ALL_VIEWER_EXCEPT_HOST_HEADER,
+          viewerProtocolPolicy: cloudfront.ViewerProtocolPolicy.ALLOW_ALL,
+          allowedMethods: cloudfront.AllowedMethods.ALLOW_ALL
+        },
         '/_next/image*': {
           origin: nextServerOrigin,
           cachePolicy: imageCachePolicy

src/cdk/constructs/RevalidateLambda.ts

@@ -0,0 +1,69 @@
+import { Construct } from 'constructs'
+import * as cdk from 'aws-cdk-lib'
+import * as lambda from 'aws-cdk-lib/aws-lambda'
+import * as logs from 'aws-cdk-lib/aws-logs'
+import * as iam from 'aws-cdk-lib/aws-iam'
+import { buildLambda } from '../../common/esbuild'
+import path from 'node:path'
+import { addOutput } from '../../common/cdk'
+
+interface RevalidateLambdaProps extends cdk.StackProps {
+  buildOutputPath: string
+  nodejs?: string
+  sqsRegion: string
+  sqsQueueUrl: string
+}
+
+const NodeJSEnvironmentMapping: Record<string, lambda.Runtime> = {
+  '18': lambda.Runtime.NODEJS_18_X,
+  '20': lambda.Runtime.NODEJS_20_X
+}
+
+export class RevalidateLambda extends Construct {
+  public readonly lambda: lambda.Function
+  public readonly lambdaHttpUrl: lambda.FunctionUrl
+
+  constructor(scope: Construct, id: string, props: RevalidateLambdaProps) {
+    const { nodejs, buildOutputPath, sqsRegion, sqsQueueUrl } = props
+    super(scope, id)
+
+    const nodeJSEnvironment = NodeJSEnvironmentMapping[nodejs ?? ''] ?? NodeJSEnvironmentMapping['20']
+    const name = 'revalidate'
+
+    buildLambda(name, buildOutputPath)
+
+    const logGroup = new logs.LogGroup(this, 'RevalidateLambdaLogGroup', {
+      logGroupName: `/aws/lambda/${id}-${name}`,
+      removalPolicy: cdk.RemovalPolicy.DESTROY,
+      retention: logs.RetentionDays.FIVE_DAYS
+    })
+
+    this.lambda = new lambda.Function(this, 'RevalidateLambda', {
+      runtime: nodeJSEnvironment,
+      code: lambda.Code.fromAsset(path.join(buildOutputPath, 'server-functions', name)),
+      handler: 'index.handler',
+      logGroup,
+      environment: {
+        SQS_AWS_REGION: sqsRegion,
+        SECRET_ID: 'x-api-key',
+        SQS_QUEUE_URL: sqsQueueUrl
+      }
+    })
+
+    logGroup.grantWrite(this.lambda)
+
+    const policyStatement = new iam.PolicyStatement({
+      actions: ['logs:CreateLogStream', 'logs:PutLogEvents'],
+      resources: [`${logGroup.logGroupArn}:*`]
+    })
+
+    this.lambda.addToRolePolicy(policyStatement)
+
+    this.lambdaHttpUrl = this.lambda.addFunctionUrl({
+      authType: lambda.FunctionUrlAuthType.NONE,
+      invokeMode: lambda.InvokeMode.RESPONSE_STREAM
+    })
+
+    addOutput(this, `${id}-RevalidateLambdaUrl`, this.lambdaHttpUrl.url)
+  }
+}

src/cdk/constructs/SecretManagerDistribution.ts

@@ -0,0 +1,14 @@
+import { Construct } from 'constructs'
+import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'
+
+export class SecretManagerDistribution extends Construct {
+  public readonly xApiKey: secretsmanager.Secret
+
+  constructor(scope: Construct, id: string) {
+    super(scope, id)
+
+    this.xApiKey = new secretsmanager.Secret(this, 'XApiKey', {
+      secretName: 'x-api-key'
+    })
+  }
+}

src/cdk/stacks/NextCloudfrontStack.ts

@@ -5,7 +5,10 @@ import { OriginRequestLambdaEdge } from '../constructs/OriginRequestLambdaEdge'
 import { CloudFrontDistribution } from '../constructs/CloudFrontDistribution'
 import { ViewerResponseLambdaEdge } from '../constructs/ViewerResponseLambdaEdge'
 import { ViewerRequestLambdaEdge } from '../constructs/ViewerRequestLambdaEdge'
+import { RevalidateLambda } from '../constructs/RevalidateLambda'
+import { SecretManagerDistribution } from '../constructs/SecretManagerDistribution'
 import { DeployConfig, NextRedirects, NextI18nConfig, NextRewrites } from '../../types'
+import * as iam from 'aws-cdk-lib/aws-iam'
 
 export interface NextCloudfrontStackProps extends StackProps {
   nodejs?: string
@@ -20,14 +23,16 @@ export interface NextCloudfrontStackProps extends StackProps {
   cachedRoutesMatchers: string[]
   rewritesConfig: NextRewrites
   isTrailingSlashEnabled: boolean
+  sqsQueueUrl: string
+  sqsQueueArn: string
 }
 
 export class NextCloudfrontStack extends Stack {
   public readonly originRequestLambdaEdge: OriginRequestLambdaEdge
   public readonly viewerResponseLambdaEdge: ViewerResponseLambdaEdge
   public readonly viewerRequestLambdaEdge: ViewerRequestLambdaEdge
   public readonly cloudfront: CloudFrontDistribution
-
+  public readonly revalidateLambda: RevalidateLambda
   constructor(scope: Construct, id: string, props: NextCloudfrontStackProps) {
     super(scope, id, props)
     const {
@@ -42,7 +47,9 @@ export class NextCloudfrontStack extends Stack {
       cachedRoutesMatchers,
       nextI18nConfig,
       rewritesConfig,
-      isTrailingSlashEnabled
+      isTrailingSlashEnabled,
+      sqsQueueUrl,
+      sqsQueueArn
     } = props
 
     this.originRequestLambdaEdge = new OriginRequestLambdaEdge(this, `${id}-OriginRequestLambdaEdge`, {
@@ -69,17 +76,36 @@ export class NextCloudfrontStack extends Stack {
       buildOutputPath
     })
 
+    this.revalidateLambda = new RevalidateLambda(this, `${id}-RevalidateLambda`, {
+      nodejs,
+      buildOutputPath,
+      sqsRegion: region,
+      sqsQueueUrl
+    })
+
     const staticBucket = s3.Bucket.fromBucketAttributes(this, `${id}-StaticAssetsBucket`, {
       bucketName: staticBucketName,
       region
     })
 
+    const secretManager = new SecretManagerDistribution(this, `${id}-SecretManagerDistribution`)
+
+    secretManager.xApiKey.grantRead(this.revalidateLambda.lambda)
+
+    this.revalidateLambda.lambda.addToRolePolicy(
+      new iam.PolicyStatement({
+        actions: ['sqs:SendMessage'],
+        resources: [sqsQueueArn]
+      })
+    )
+
     this.cloudfront = new CloudFrontDistribution(this, `${id}-NextCloudFront`, {
       staticBucket,
       renderServerDomain,
       requestEdgeFunction: this.originRequestLambdaEdge.lambdaEdge,
       viewerResponseEdgeFunction: this.viewerResponseLambdaEdge.lambdaEdge,
       viewerRequestLambdaEdge: this.viewerRequestLambdaEdge.lambdaEdge,
+      revalidateLambdaUrl: this.revalidateLambda.lambdaHttpUrl,
       deployConfig: deployConfig,
       imageTTL
     })