From b616a1893d6572b6a82368737a0ea6a5a17ec4f6 Mon Sep 17 00:00:00 2001 From: Eugene Cheah Date: Fri, 10 Jan 2020 16:18:31 +0000 Subject: [PATCH] Optimized docker build caching (sort of) --- .gitignore | 1 + api/app.js | 20 ++++- api/config/cacheControl.js | 11 +++ api/src/api/mailGetHtml.js | 8 +- api/src/api/{mailGetKey.js => mailGetInfo.js} | 30 ++++--- api/src/api/mailGetUrl.js | 8 +- api/src/api/mailList.js | 6 +- docker-entrypoint.sh | 46 ++++++++--- dockerfile | 79 ++++++++++++------- 9 files changed, 148 insertions(+), 61 deletions(-) create mode 100644 api/config/cacheControl.js rename api/src/api/{mailGetKey.js => mailGetInfo.js} (51%) diff --git a/.gitignore b/.gitignore index ad61f98..7cc9d61 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ ui/config/apiconfig.js # Include sample config files !api/config/*.sample.js +!api/config/cacheControl.js # Whitelist anchor files & .gitignore !.gitignore diff --git a/api/app.js b/api/app.js index b46fa95..6db1bc5 100644 --- a/api/app.js +++ b/api/app.js @@ -1,16 +1,30 @@ +// Cache control settings +const cacheControl = require("./config/cacheControl"); + // app package loading let app = require("./src/app-setup"); // Setup the routes app.get("/api/v1/mail/list", require("./src/api/mailList")); -app.get("/api/v1/mail/getKey", require("./src/api/mailGetKey")); +app.get("/api/v1/mail/getInfo", require("./src/api/mailGetInfo")); app.get("/api/v1/mail/getHtml", require("./src/api/mailGetHtml")); -// Static folder hosting -app.use( app.express.static("public") ) +// Legacy fallback behaviour - +// Note this is to be deprecated (after updating UI) +app.get("/api/v1/mail/getKey", require("./src/api/mailGetInfo")); + +// Static folder hosting with cache control +// See express static options: https://expressjs.com/en/4x/api.html#express.static +app.use( app.express.static("public", { + etag: true, + setHeaders: function (res, path, stat) { + res.set('cache-control', cacheControl.static) + } +}) ) // Custom 404 handling - use index.html app.use(function(req, res) { + res.set('cache-control', cacheControl.static) res.sendFile(__dirname + '/public/index.html'); }); diff --git a/api/config/cacheControl.js b/api/config/cacheControl.js new file mode 100644 index 0000000..d74ef79 --- /dev/null +++ b/api/config/cacheControl.js @@ -0,0 +1,11 @@ +/** + * Configure the various level of cache controls + */ +module.exports = { + // Frequent changing dynamic content + "dynamic": "public, max-age=1, max-stale=1, stale-while-revalidate=10, stale-if-error=86400", + + // Rarely changing static content + // with very aggressive caching + "static": "public, max-age=10, max-stale=10, stale-while-revalidate=3600, stale-if-error=86400" +} \ No newline at end of file diff --git a/api/src/api/mailGetHtml.js b/api/src/api/mailGetHtml.js index 12d7d7f..4de1446 100644 --- a/api/src/api/mailGetHtml.js +++ b/api/src/api/mailGetHtml.js @@ -1,11 +1,12 @@ // Loading mailgun reader and config const mailgunReader = require("../mailgunReader"); const mailgunConfig = require("../../config/mailgunConfig"); +const cacheControl = require("../../config/cacheControl"); const reader = new mailgunReader(mailgunConfig); /** - * Get and return the URL content from the mailgun API + * Get and return the etatic email HTML content from the mailgun API, given the mailKey * * @param {*} req * @param {*} res @@ -31,9 +32,10 @@ module.exports = function(req, res){ // eslint-disable-next-line '<\/script>' - res.status(200).send(body) + res.set('cache-control', cacheControl.static) + res.status(200).send(body) }) .catch(e => { - res.status(500).send("{error: '"+e+"'}") + res.status(500).send("{error: '"+e+"'}") }); } diff --git a/api/src/api/mailGetKey.js b/api/src/api/mailGetInfo.js similarity index 51% rename from api/src/api/mailGetKey.js rename to api/src/api/mailGetInfo.js index 612a92d..38b69d6 100644 --- a/api/src/api/mailGetKey.js +++ b/api/src/api/mailGetInfo.js @@ -1,11 +1,12 @@ // Loading mailgun reader and config const mailgunReader = require("../mailgunReader"); const mailgunConfig = require("../../config/mailgunConfig"); +const cacheControl = require("../../config/cacheControl"); const reader = new mailgunReader(mailgunConfig); /** - * Get and return the URL content from the mailgun API + * Get and return the static email header details from the mailgun API given the mailKey * * @param {*} req * @param {*} res @@ -18,22 +19,25 @@ module.exports = function(req, res){ } reader.getKey(mailKey).then(response => { - let emailDetails = {} + let emailDetails = {} - // Format and extract the name of the user - let [name, ...rest] = formatName(response.from) - emailDetails.name = name + // Format and extract the name of the user + let [name, ...rest] = formatName(response.from) + emailDetails.name = name - // Extract the rest of the email domain after splitting - if (rest[0].length > 0) { - emailDetails.emailAddress = ' <' + rest - } + // Extract the rest of the email domain after splitting + if (rest[0].length > 0) { + emailDetails.emailAddress = ' <' + rest + } - // Extract the subject of the response - emailDetails.subject = response.subject + // Extract the subject of the response + emailDetails.subject = response.subject - // Extract the recipients - emailDetails.recipients = response.recipients + // Extract the recipients + emailDetails.recipients = response.recipients + + // Return with cache control + res.set('cache-control', cacheControl.static) res.status(200).send(emailDetails) }) .catch(e => { diff --git a/api/src/api/mailGetUrl.js b/api/src/api/mailGetUrl.js index a1585d7..3c091e3 100644 --- a/api/src/api/mailGetUrl.js +++ b/api/src/api/mailGetUrl.js @@ -1,11 +1,14 @@ // Loading mailgun reader and config const mailgunReader = require("../mailgunReader"); const mailgunConfig = require("../../config/mailgunConfig"); +const cacheControl = require("../../config/cacheControl"); const reader = new mailgunReader(mailgunConfig); /** - * Get and return the URL content from the mailgun API + * Get and return the URL link from the mailgun API - for the mail gcontent + * + * NOTE - this is to be deprecated * * @param {*} req * @param {*} res @@ -18,7 +21,8 @@ module.exports = function(req, res){ } reader.getUrl(url).then(response => { - res.status(200).send(response) + res.set('cache-control', cacheControl.static) + res.status(200).send(response) }) .catch(e => { res.status(500).send("{error: '"+e+"'}") diff --git a/api/src/api/mailList.js b/api/src/api/mailList.js index 81bdee0..4769696 100644 --- a/api/src/api/mailList.js +++ b/api/src/api/mailList.js @@ -1,11 +1,12 @@ // Loading mailgun reader and config const mailgunReader = require("../mailgunReader"); const mailgunConfig = require("../../config/mailgunConfig"); +const cacheControl = require("../../config/cacheControl"); const reader = new mailgunReader(mailgunConfig); /** - * Mail listing api, returns the item list + * Mail listing api, returns the list of emails * * @param {*} req * @param {*} res @@ -24,7 +25,8 @@ module.exports = function(req, res){ } reader.recipientEventList(recipient+"@"+mailgunConfig.emailDomain).then(response => { - res.status(200).send(response.items) + res.set('cache-control', cacheControl.dynamic) + res.status(200).send(response.items) }) .catch(e => { res.status(500).send("{error: '"+e+"'}") diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 520da94..540461a 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,5 +1,12 @@ #!/bin/sh +# +# Entrypoint start +# +echo ">>---------------------------------------------------------------------" +echo ">> Starting inboxkitten container : Get Mail Nyow!" +echo ">>---------------------------------------------------------------------" + # # Getting the various configuration settings from command line / environment variable # @@ -11,9 +18,8 @@ else fi if [ -z "$MAILGUN_API_KEY" ]; then - echo ">> Please type in your MAILGUN_API_KEY"; - read -sp '>> MAILGUN_API_KEY : ' MAILGUN_API_KEY; - echo ""; + echo "[FATAL ERROR] Missing MAILGUN_API_KEY"; + exit 1; else echo ">> Detected MAILGUN_API_KEY env variable : [intentionally redacted]"; fi @@ -25,25 +31,47 @@ else echo ">> Detected WEBSITE_DOMAIN env variable : $WEBSITE_DOMAIN"; fi +# +# End of env variable checks +# Moving to config setups +# +echo ">>---------------------------------------------------------------------" + +# Debug check +# ls /application/ + # # Setup the UI # echo ">> Setting up UI" # Clone the files -rm -rf /application/api/public/* -cp -r /application/ui-dist/ /application/api/public/ +rm -rf /application/api/public/ +mkdir /application/api/public/ +cp -r /application/ui-dist/* /application/api/public/ + +# Debug check +# ls /application/api/public/ + +# Search token (so that it does not get character substituted) +TOKEN_MAILGUN_EMAIL_DOMAIN='${MAILGUN_EMAIL_DOMAIN}' +TOKEN_WEBSITE_DOMAIN='${WEBSITE_DOMAIN}' # Find and replace -find /application/api/public/ -type f -exec sed -i "s/\$\{MAILGUN_EMAIL_DOMAIN\}/$MAILGUN_EMAIL_DOMAIN/g" {} + -find /application/api/public/ -type f -exec sed -i "s/\$\{WEBSITE_DOMAIN\}/$WEBSITE_DOMAIN/g" {} + +find /application/api/public/ -type f -exec sed -i "s/$TOKEN_MAILGUN_EMAIL_DOMAIN/$MAILGUN_EMAIL_DOMAIN/g" {} + +find /application/api/public/ -type f -exec sed -i "s/$TOKEN_WEBSITE_DOMAIN/$WEBSITE_DOMAIN/g" {} + # # Setup the API # echo ">> Setting up API config" -cat "application/api/config/mailgunConfig.sample.js" | envsubst > "application/api/config/mailgunConfig.js" +cat "/application/api/config/mailgunConfig.sample.js" | envsubst > "/application/api/config/mailgunConfig.js" -# Start the API +# +# Start the server +# +echo ">>---------------------------------------------------------------------" +echo ">> Starting the server" +echo ">>---------------------------------------------------------------------" cd /application/api/ npm start diff --git a/dockerfile b/dockerfile index d3b4d03..26694e3 100755 --- a/dockerfile +++ b/dockerfile @@ -1,61 +1,79 @@ +#------------------------------------------------------- # +# Base alpine images with all the runtime os dependencies # -# Base alpine image with all the various dependencies -# -# Note that the node sass is broken with +# Note that the node-sass is broken with node 12 # https://github.com/nodejs/docker-node/issues/1028 # -# -FROM node:10-alpine AS baseimage +#------------------------------------------------------- -# Install dependencies +# Does basic node, and runtime dependencies +FROM node:10-alpine AS baseimage RUN apk add --no-cache gettext - -# Setup the /application/ directory RUN mkdir -p /application/ # WORKDIR /application/ +#------------------------------------------------------- # +# code builders (used by dockerbuilders) # -# Initial docker builder (resets node_modules) -# -# -FROM baseimage AS builder +#------------------------------------------------------- -# node-gyp installation +# Install dependencies for some NPM modules +FROM baseimage AS codebuilder RUN apk add --no-cache make gcc g++ python -# Copy over the requried files -COPY api /application/api/ -COPY ui /application/ui/ -COPY docker-entrypoint.sh /application/docker-entrypoint.sh +#------------------------------------------------------- +# +# Docker builders (also resets node_modules) +# +# Note each major dependency is compiled seperately +# so as to isolate the impact of each code change +# +#------------------------------------------------------- -# Scrub out node_modules and built files +# Build the API +# with reseted node_modules +FROM codebuilder AS apibuilder +# copy and reset the code +COPY api /application/api/ RUN rm -rf /application/api/node_modules +RUN cd /application/api && ls && npm install + +# Build the UI +# with reseted node_modules +FROM codebuilder AS uibuilder +# copy and reset the code +COPY ui /application/ui/ RUN rm -rf /application/ui/node_modules RUN rm -rf /application/ui/dist - -# Lets do the initial npm install RUN cd /application/ui && ls && npm install -RUN cd /application/api && ls && npm install - # Lets do the UI build RUN cp /application/ui/config/apiconfig.sample.js /application/ui/config/apiconfig.js RUN cd /application/ui && npm run build +# Entry script +# & Permission reset +FROM codebuilder AS entrypointbuilder +COPY docker-entrypoint.sh /application/docker-entrypoint.sh +RUN chmod +x /application/docker-entrypoint.sh + +#------------------------------------------------------- # +# Full Docker application # -# Docker application -# -# -FROM node:12-alpine as application +#------------------------------------------------------- +FROM baseimage as inboxkitten # Copy over the built files -COPY --from=builder /application/api /application/ -COPY --from=builder /application/ui/dist /application/ui-dist +COPY --from=apibuilder /application/api /application/api +COPY --from=uibuilder /application/ui/dist /application/ui-dist +COPY --from=entrypointbuilder /application/docker-entrypoint.sh /application/docker-entrypoint.sh # Debugging logging -RUN ls /application +# RUN ls /application/./ +# RUN ls /application/ui-dist +# RUN ls /application/api # Expose the server port EXPOSE 8000 @@ -73,6 +91,9 @@ ENV WEBSITE_DOMAIN="" # RUN cd /application/ui && ls && npm install # RUN cd /application/api && ls && npm install +# Setup the workdir +WORKDIR "/application/" + # Setup the entrypoint ENTRYPOINT [ "/application/docker-entrypoint.sh" ] CMD []