Skip to content
This repository was archived by the owner on Aug 6, 2025. It is now read-only.

Commit f2b5539

Browse files
authored
[DOP-3640]: Define the Autobuilder Core Construct (#821)
* [DOP-3639]: Create base API construct * [DOP-3639]: Create ts files to export specific lambda handler * [DOP-3639]: Create ts files to export specific lambda handler * [DOP-3639]: Create methods for triggering builds * [DOP-3639]: Remove the jobQueue as an attribute for the webhooks construct * [DOP-3639]: Rename construct to better reflect what it represents * [DOP-3639]: Use correct path * [DOP-3639]: Retrieve SSM params * [DOP-3639]: Add DB name * [DOP-3639]: Add dochub env vars * [DOP-3639]: Add correct dochub path * [DOP-3639]: use correct method to retrieve secure strings * [DOP-3639]: Delete zip script in favor of using NodejsFunction construct * [DOP-3639]: Update esbuild config for nodejsfunctions * [DOP-3639]: Remove bundle script and handlers * [DOP-3639]: Get tests working * [DOP-3639]: Delete handlers and remove zip script * [DOP-3639]: refactor so that queues are a separate construct from the api * [DOP-3639]: add correct identifiers for constructs * [DOP-3639]: Update comments for bundling * [DOP-3639]: Update tests * [DOP-3639]: Clean up some typing * [DOP-3639]: Add script to bundle lambda config with zip * [DOP-3639]: Add permission to execute script * [DOP-3639]: rename file * [DOP-3639]: Use the regular function construct again and update bundling script * [DOP-3639]: Add updated script and add snapshot test * [DOP-3639]: Add updated script * [DOP-3639]: Prune dev dependencies when deploying lambda * [DOP-3639]: Add slack auth token * [DOP-3639]: Add queue urls as env vars * [DOP-3639]: Grab DB info from ssm * [DOP-3639]: Update env vars * [DOP-3639]: Remove cdk.context.json * [DOP-3639]: Update gitignore and use valueFromLookup * [DOP-3639]: Refactor code to successfully deploy and use config variables * [DOP-3639]: Remove unnecessary build command * [DOP-3639]: Remove unnecessary build script * [DOP-3639]: Refactor env vars into separate construct * [DOP-3639]: Remove build.zip from gitignore * [DOP-3639]: Remove bundle command * [DOP-3680]: Add v2 handlers * [DOP-3680]: Use v2 handlers * [DOP-3680]: Revert slack v1 * [DOP-3639]: Remove additional slash * [DOP-3639]: Remove unused params * [DOP-3639]: Remove unused import * [DOP-3639]: Use esModuleInterop * [DOP-3640]: Rename to simplify stuff * [DOP-3640]: Add util to get bucket names * [DOP-3640]: Add task definition for fargate and create cluster * [DOP-3640]: Refactor Stack and constructs * [DOP-3640]: Add CDK infra to docker ignore * [DOP-3640]: Rename files to be less generic and add task definition for lambdas * [DOP-3640]: Work on creating bucket constructs * [DOP-3640]: destroy buckets when tearing down * [DOP-3640]: Grant ecs task permission to read and write from buckets * [DOP-3640]: Add VPC endpoint for ECR * [DOP-3640]: Add custom dockerfile for testing enhanced app * [DOP-3640]: Refactor path names to take into account the environment * [DOP-3640]: Add a bunch of env vars * [DOP-3640]: Add a bunch of env vars for worker * [DOP-3640]: Remove unused env var * [DOP-3640]: Refactor object destructure for consistency * [DOP-3640]: Add first routing rule * [DOP-3640]: Add routing rules for root bucket * [DOP-3640]: Refactor container props to be a little nicer * [DOP-3640]: Add feature branch deploy logic * [DOP-3640]: Add env to worker * [DOP-3640]: Move stuff around and work on getting secure strings using aws sdk * [DOP-3640]: Add SSM client SDK to query secure strings * [DOP-3640]: Add remaining secure strings and refactor stack to use async ssm client * [DOP-3640]: Grant permissions to the ECS cluster to read and write to queues * [DOP-3640]: Add Tag for stack to help debug costs * [DOP-3640]: Refactor layout to be a bit simpler * [DOP-3640]: Refactor enhanced job to check for correct job type * [DOP-3640]: Remove unnecesssary code * [DOP-3640]: Update readme * [DOP-3640]: Update prettier to ignore cdk-infra * [DOP-3640]: Use clustername for task definition family (that's the value that is actually used) * [DOP-3640]: Update README * [DOP-3640]: Update readme and rename context variables * [DOP-3640]: Refactor how environment variables are retrieved to not depend on current scope * [DOP-3640]: Add routing rule for 1.1.4 of atlas cli * [DOP-3640]: Call initContextVars
1 parent d2c57ad commit f2b5539

26 files changed

+1960
-142
lines changed

.dockerignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
tests/
22
node_modules/
3+
cdk-infra/

Dockerfile.enhanced

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
# Build the Typescript app
2+
FROM node:18.16.0-alpine as ts-compiler
3+
WORKDIR /home/docsworker-xlarge
4+
COPY config config/
5+
COPY package*.json ./
6+
COPY tsconfig*.json ./
7+
RUN npm install
8+
COPY . ./
9+
RUN npm run build
10+
11+
# install persistence module
12+
RUN cd ./modules/persistence \
13+
&& npm install \
14+
&& npm run build
15+
16+
# Build modules
17+
# OAS Page Builder
18+
RUN cd ./modules/oas-page-builder \
19+
&& npm install \
20+
&& npm run build
21+
22+
# where repo work will happen
23+
FROM ubuntu:20.04
24+
ARG WORK_DIRECTORY=/home/docsworker-xlarge
25+
ARG SNOOTY_PARSER_VERSION=0.14.0
26+
ARG SNOOTY_FRONTEND_VERSION=0.14.6
27+
ARG MUT_VERSION=0.10.3
28+
ARG REDOC_CLI_VERSION=1.2.0
29+
ARG NPM_BASE_64_AUTH
30+
ARG NPM_EMAIL
31+
ENV DEBIAN_FRONTEND=noninteractive
32+
33+
# install legacy build environment for docs
34+
RUN apt-get -o Acquire::Check-Valid-Until=false update
35+
RUN apt-get -y install libpython2.7-dev python2.7 git rsync unzip curl
36+
RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
37+
RUN python2.7 get-pip.py
38+
RUN pip install requests virtualenv virtualenvwrapper py-dateutil
39+
RUN python2.7 -m pip install python-dateutil
40+
RUN virtualenv /venv
41+
RUN /venv/bin/pip install --upgrade --force setuptools
42+
RUN /venv/bin/pip install -r https://raw.githubusercontent.com/mongodb/docs-tools/master/giza/requirements.txt
43+
44+
# helper libraries for docs builds
45+
RUN apt-get update && apt-get install -y vim git
46+
47+
48+
ENV PATH="${PATH}:/opt/snooty:/opt/mut:/home/docsworker-xlarge/.local/bin:/usr/local/lib/python2.7/dist-packages/virtualenv/bin"
49+
50+
# get node 18
51+
# https://gist.github.com/RinatMullayanov/89687a102e696b1d4cab
52+
RUN apt-get install --yes curl
53+
RUN curl --location https://deb.nodesource.com/setup_18.x | bash -
54+
RUN apt-get install --yes nodejs
55+
RUN apt-get install --yes build-essential
56+
57+
# use npm 8.*
58+
RUN npm install -g npm@8
59+
60+
# install snooty parser
61+
RUN curl -L -o snooty-parser.zip https://github.com/mongodb/snooty-parser/releases/download/v${SNOOTY_PARSER_VERSION}/snooty-v${SNOOTY_PARSER_VERSION}-linux_x86_64.zip \
62+
&& unzip -d /opt/ snooty-parser.zip
63+
64+
# install mut
65+
RUN curl -L -o mut.zip https://github.com/mongodb/mut/releases/download/v${MUT_VERSION}/mut-v${MUT_VERSION}-linux_x86_64.zip \
66+
&& unzip -d /opt/ mut.zip
67+
68+
# setup user and root directory
69+
RUN useradd -ms /bin/bash docsworker-xlarge
70+
RUN chmod 755 -R ${WORK_DIRECTORY}
71+
RUN chown -Rv docsworker-xlarge ${WORK_DIRECTORY}
72+
USER docsworker-xlarge
73+
74+
WORKDIR ${WORK_DIRECTORY}
75+
76+
# get shared.mk
77+
RUN curl https://raw.githubusercontent.com/mongodb/docs-worker-pool/meta/makefiles/shared.mk -o shared.mk
78+
79+
# install snooty frontend and docs-tools
80+
RUN git clone -b v${SNOOTY_FRONTEND_VERSION} --depth 1 https://github.com/mongodb/snooty.git \
81+
&& cd snooty \
82+
&& npm ci --legacy-peer-deps --omit=dev \
83+
&& git clone --depth 1 https://github.com/mongodb/docs-tools.git \
84+
&& mkdir -p ./static/images \
85+
&& mv ./docs-tools/themes/mongodb/static ./static/docs-tools \
86+
&& mv ./docs-tools/themes/guides/static/images/bg-accent.svg ./static/docs-tools/images/bg-accent.svg
87+
88+
# install redoc fork
89+
RUN git clone -b @dop/redoc-cli@${REDOC_CLI_VERSION} --depth 1 https://github.com/mongodb-forks/redoc.git redoc \
90+
# Install dependencies for Redoc CLI
91+
&& cd redoc/ \
92+
&& npm ci --prefix cli/ --omit=dev
93+
94+
COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/package*.json ./
95+
COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/config config/
96+
COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/build ./
97+
RUN npm install
98+
99+
# Persistence module copy
100+
# Create directory and add permissions to allow node module installation
101+
RUN mkdir -p modules/persistence && chmod 755 modules/persistence
102+
COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/persistence/package*.json ./modules/persistence/
103+
COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/persistence/dist ./modules/persistence/
104+
ENV PERSISTENCE_MODULE_PATH=${WORK_DIRECTORY}/modules/persistence/index.js
105+
RUN cd ./modules/persistence/ && ls && npm install
106+
107+
# OAS Page Builder module copy
108+
# Create directory and add permissions to allow node module installation
109+
RUN mkdir -p modules/oas-page-builder && chmod 755 modules/oas-page-builder
110+
COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/oas-page-builder/package*.json ./modules/oas-page-builder/
111+
COPY --from=ts-compiler --chown=docsworker-xlarge /home/docsworker-xlarge/modules/oas-page-builder/dist ./modules/oas-page-builder/
112+
RUN cd ./modules/oas-page-builder/ && npm install
113+
114+
# Needed for OAS Page Builder module in shared.mk
115+
ENV REDOC_PATH=${WORK_DIRECTORY}/redoc/cli/index.js
116+
ENV OAS_MODULE_PATH=${WORK_DIRECTORY}/modules/oas-page-builder/index.js
117+
118+
RUN mkdir repos && chmod 755 repos
119+
EXPOSE 3000
120+
CMD ["node", "enhanced/enhancedApp.js"]

cdk-infra/README.md

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,20 @@
1-
# Welcome to your CDK TypeScript project
1+
# Docs Platform
2+
3+
## Tips
4+
5+
To verify the AWS CDK code adequately replicates the current AWS infrastructure, it is helpful to reference the CloudFormation templates that exist within the `infrastructure` directory, as well as understanding the `serverless.yml` files' contents.
6+
7+
## Testing
8+
9+
To verify the CloudFormation is being generated successfully, you can use the `cdk synth` command to generate the CloudFormation. In the `/cdk-infra` directory, execute the following command:
10+
11+
```zsh
12+
npm run cdk synth -- -c enhanced=true -c customFeatureName=enhancedApp > cdk.out/template.yaml
13+
```
14+
15+
Make sure to update your `~/.aws/credentials` file. The `enhanced` context variable, if set to true, will use the `Dockerfile.enhanced` dockerfile instead of the standard one. The `featureName` context variable is used to provide a different name for a custom stack other than the branch name. In the future for feature branches, the context variable `isFeature` will be used to use the Git branch name and append that to the stack name.
16+
17+
# Welcome to your CDK TypeScript project (Auto-generated readme below)
218

319
This is a blank project for CDK development with TypeScript.
420

cdk-infra/bin/cdk-infra.ts

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,43 @@
22
import 'source-map-support/register';
33
import * as cdk from 'aws-cdk-lib';
44
import { AutoBuilderStack } from '../lib/auto-builder-stack';
5+
import { getSsmPathPrefix, getWebhookSecureStrings, getWorkerSecureStrings } from '../utils/ssm';
6+
import { getFeatureName, initContextVars } from '../utils/env';
57

6-
const app = new cdk.App();
7-
new AutoBuilderStack(app, 'AutoBuilderStack', {
8-
/* If you don't specify 'env', this stack will be environment-agnostic.
9-
* Account/Region-dependent features and context lookups will not work,
10-
* but a single synthesized template can be deployed anywhere. */
11-
/* Uncomment the next line to specialize this stack for the AWS Account
12-
* and Region that are implied by the current CLI configuration. */
13-
// env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION },
14-
/* Uncomment the next line if you know exactly what Account and Region you
15-
* want to deploy the stack to. */
16-
env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION },
17-
/* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */
18-
});
8+
async function main() {
9+
const app = new cdk.App();
10+
11+
// calling this here so that we can call functions like getEnv and getSsmPathPrefix
12+
// without having to pass in current construct. More intuitive for developers to look at.
13+
// This will grab context variables that we pass in from the CLI and add it to a map.
14+
initContextVars(app);
15+
16+
const ssmPrefix = getSsmPathPrefix();
17+
18+
// Constructors can't be async, so since I am doing this workaround for the secure strings,
19+
// they need to be retrieved before we create the stack.
20+
const workerSecureStrings = await getWorkerSecureStrings(ssmPrefix);
21+
const webhookSecureStrings = await getWebhookSecureStrings(ssmPrefix);
22+
23+
const stackName = `auto-builder-stack-${getFeatureName()}`;
24+
25+
new AutoBuilderStack(app, stackName, {
26+
/* If you don't specify 'env', this stack will be environment-agnostic.
27+
* Account/Region-dependent features and context lookups will not work,
28+
* but a single synthesized template can be deployed anywhere. */
29+
/* Uncomment the next line to specialize this stack for the AWS Account
30+
* and Region that are implied by the current CLI configuration. */
31+
// env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION },
32+
/* Uncomment the next line if you know exactly what Account and Region you
33+
* want to deploy the stack to. */
34+
env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION },
35+
workerSecureStrings,
36+
webhookSecureStrings,
37+
tags: {
38+
stackName,
39+
},
40+
/* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */
41+
});
42+
}
43+
44+
main();
Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,50 @@
1-
import * as cdk from 'aws-cdk-lib';
1+
import { Stack, StackProps } from 'aws-cdk-lib';
22
import { Construct } from 'constructs';
3-
import { AutoBuilderApiConstruct } from './constructs/auto-builder-api-construct';
4-
import { AutoBuilderQueuesConstruct } from './constructs/auto-builder-queues-construct';
5-
import { AutoBuilderEnvConstruct } from './constructs/auto-builder-env-construct';
63

7-
export class AutoBuilderStack extends cdk.Stack {
8-
constructor(scope: Construct, id: string, props?: cdk.StackProps) {
4+
import { WebhookApiConstruct } from './constructs/api/webhook-api-construct';
5+
import { WebhookEnvConstruct } from './constructs/api/webhook-env-construct';
6+
import { AutoBuilderQueuesConstruct } from './constructs/queue/queues-construct';
7+
import { WorkerBucketsConstruct } from './constructs/worker/buckets-construct';
8+
import { WorkerConstruct } from './constructs/worker/worker-construct';
9+
import { WorkerEnvConstruct } from './constructs/worker/worker-env-construct';
10+
11+
interface AutoBuilderStackProps extends StackProps {
12+
workerSecureStrings: Record<string, string>;
13+
webhookSecureStrings: Record<string, string>;
14+
}
15+
export class AutoBuilderStack extends Stack {
16+
constructor(
17+
scope: Construct,
18+
id: string,
19+
{ workerSecureStrings, webhookSecureStrings, ...props }: AutoBuilderStackProps
20+
) {
921
super(scope, id, props);
1022

1123
const queues = new AutoBuilderQueuesConstruct(this, 'queues');
12-
const { environment } = new AutoBuilderEnvConstruct(this, 'ssmVars', queues);
1324

14-
new AutoBuilderApiConstruct(this, 'api', { ...queues, environment });
25+
const { environment: webhookEnvironment } = new WebhookEnvConstruct(this, 'ssmVars', {
26+
...queues,
27+
secureStrings: webhookSecureStrings,
28+
});
29+
const { environment: workerEnvironment } = new WorkerEnvConstruct(this, 'workerSsmVars', {
30+
...queues,
31+
secureStrings: workerSecureStrings,
32+
});
33+
34+
const { clusterName, ecsTaskRole } = new WorkerConstruct(this, 'worker', {
35+
environment: workerEnvironment,
36+
...queues,
37+
});
38+
39+
const { buckets } = new WorkerBucketsConstruct(this, 'workerBuckets');
40+
41+
new WebhookApiConstruct(this, 'api', {
42+
...queues,
43+
environment: { ...webhookEnvironment, TASK_DEFINITION_FAMILY: clusterName },
44+
});
45+
46+
buckets.forEach((bucket) => {
47+
bucket.grantReadWrite(ecsTaskRole);
48+
});
1549
}
1650
}

cdk-infra/lib/constructs/auto-builder-api-construct.ts renamed to cdk-infra/lib/constructs/api/webhook-api-construct.ts

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
import { Cors, CorsOptions, LambdaIntegration, LambdaRestApi } from 'aws-cdk-lib/aws-apigateway';
22
import { Code, Function, Runtime } from 'aws-cdk-lib/aws-lambda';
3+
import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources';
34
import { BundlingOptions, NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs';
45
import { IQueue } from 'aws-cdk-lib/aws-sqs';
56
import { Construct } from 'constructs';
67
import path from 'path';
78

8-
const HANDLERS_PATH = path.join(__dirname, '/../../../api/controllers/v2');
9+
const HANDLERS_PATH = path.join(__dirname, '/../../../../api/controllers/v2');
910

1011
const bundling: BundlingOptions = {
1112
sourceMap: true,
@@ -24,18 +25,16 @@ const bundling: BundlingOptions = {
2425

2526
const runtime = Runtime.NODEJS_18_X;
2627

27-
interface AutoBuilderApiConstructProps {
28+
interface WebhookApiConstructProps {
2829
jobsQueue: IQueue;
2930
jobUpdatesQueue: IQueue;
3031
environment: Record<string, string>;
3132
}
3233

33-
export class AutoBuilderApiConstruct extends Construct {
34-
constructor(scope: Construct, id: string, props: AutoBuilderApiConstructProps) {
34+
export class WebhookApiConstruct extends Construct {
35+
constructor(scope: Construct, id: string, { jobsQueue, jobUpdatesQueue, environment }: WebhookApiConstructProps) {
3536
super(scope, id);
3637

37-
const { jobsQueue, jobUpdatesQueue, environment } = props;
38-
3938
const slackTriggerLambda = new NodejsFunction(this, 'slackTriggerLambda', {
4039
entry: `${HANDLERS_PATH}/slack.ts`,
4140
runtime,
@@ -75,6 +74,14 @@ export class AutoBuilderApiConstruct extends Construct {
7574
bundling,
7675
});
7776

77+
const handleJobsLambda = new NodejsFunction(this, 'handleJobsLambda', {
78+
entry: `${HANDLERS_PATH}/jobs.ts`,
79+
runtime,
80+
handler: 'HandleJobs',
81+
environment,
82+
bundling,
83+
});
84+
7885
// generic handler for the root endpoint
7986
const rootEndpointLambda = new Function(this, 'RootEndpointLambda', {
8087
code: Code.fromInline('exports.default = (event) => { console.log("hello, world!!"); }'),
@@ -130,6 +137,15 @@ export class AutoBuilderApiConstruct extends Construct {
130137
jobsQueue.grantSendMessages(triggerLocalBuildLambda);
131138

132139
// grant permission for lambdas to enqueue messages to the job updates queue
140+
jobUpdatesQueue.grantSendMessages(slackTriggerLambda);
141+
jobUpdatesQueue.grantSendMessages(githubTriggerLambda);
133142
jobUpdatesQueue.grantSendMessages(triggerLocalBuildLambda);
143+
144+
// grant permission to read from jobUpdatesQueue
145+
jobUpdatesQueue.grantConsumeMessages(handleJobsLambda);
146+
147+
const handleJobsQueueSource = new SqsEventSource(jobUpdatesQueue);
148+
149+
handleJobsLambda.addEventSource(handleJobsQueueSource);
134150
}
135151
}
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import { IQueue } from 'aws-cdk-lib/aws-sqs';
2+
import { StringParameter } from 'aws-cdk-lib/aws-ssm';
3+
import { Construct } from 'constructs';
4+
import { getSsmPathPrefix } from '../../../utils/ssm';
5+
6+
interface WebhookEnvConstructProps {
7+
jobsQueue: IQueue;
8+
jobUpdatesQueue: IQueue;
9+
secureStrings: Record<string, string>;
10+
}
11+
export class WebhookEnvConstruct extends Construct {
12+
readonly environment: Record<string, string>;
13+
14+
constructor(scope: Construct, id: string, { jobsQueue, jobUpdatesQueue, secureStrings }: WebhookEnvConstructProps) {
15+
super(scope, id);
16+
17+
const ssmPrefix = getSsmPathPrefix();
18+
19+
const dbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/dbname`);
20+
const dbUsername = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/username`);
21+
const dbHost = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/host`);
22+
23+
const dbPassword = secureStrings[`${ssmPrefix}/atlas/password`];
24+
this.environment = {
25+
...secureStrings,
26+
MONGO_ATLAS_USERNAME: dbUsername,
27+
MONGO_ATLAS_PASSWORD: dbPassword,
28+
MONGO_ATLAS_URL: `mongodb+srv://${dbUsername}:${dbPassword}@${dbHost}/admin?retryWrites=true`,
29+
DB_NAME: dbName,
30+
31+
NODE_CONFIG_DIR: './config',
32+
JOBS_QUEUE_URL: jobsQueue.queueUrl,
33+
JOB_UPDATES_QUEUE_URL: jobUpdatesQueue.queueUrl,
34+
};
35+
}
36+
}

0 commit comments

Comments
 (0)