fix!: remove esm package, use native Node ES modules (#2064)

BREAKING CHANGE: Node v12.20+, v14.14+, or v16.0+ is required

* fix!: remove esm package, use native Node ES modules

* fix: fix some CJS imports

.circleci/config.yml

@@ -1,5 +1,7 @@
 version: 2.1
 
+orbs:
+  browser-tools: circleci/[email protected]
 workflows:
   version: 2
   build_and_test:
@@ -15,13 +17,11 @@ executors:
   node:
     working_directory: ~/pinafore
     docker:
-      # we want Node v12, not v14
-      # see https://discuss.circleci.com/t/build-failed-the-engine-node-is-incompatible-with-this-module-expected-version-12-x-got-14-15-0/37921/7
-      - image: circleci/ruby@sha256:b018ec2a8f0bbf06880735d2801402bad316c465edb60663be83ac8f1086b805
+      - image: cimg/ruby:2.7.2-browsers
   node_and_ruby:
     working_directory: ~/pinafore
     docker:
-      - image: circleci/ruby@sha256:b018ec2a8f0bbf06880735d2801402bad316c465edb60663be83ac8f1086b805
+      - image: cimg/ruby:2.7.2-browsers
       - image: circleci/postgres:12.2
         environment:
           POSTGRES_USER: pinafore
@@ -30,6 +30,45 @@ executors:
           BROWSER: chrome:headless
       - image: circleci/redis:5-alpine
 commands:
+  install_mastodon_system_dependencies:
+    description: Install system dependencies that Mastodon requires
+    steps:
+      - run:
+          name: Install system dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y \
+              ffmpeg \
+              fonts-noto-color-emoji \
+              imagemagick \
+              libicu-dev \
+              libidn11-dev \
+              libprotobuf-dev \
+              postgresql-contrib \
+              protobuf-compiler
+  install_browsers:
+    description: Install browsers and tools
+    steps:
+      - browser-tools/install-chrome:
+          chrome-version: 91.0.4472.114
+      - browser-tools/install-chromedriver
+      - run:
+          name: "Check browser version"
+          command: |
+            google-chrome --version
+  install_node:
+    description: Install Node.js
+    steps:
+      - run:
+          name: "Install Node.js"
+          # via https://circleci.com/docs/2.0/circleci-images/#notes-on-pinning-images
+          command: |
+            curl -sSL "https://nodejs.org/dist/v12.22.2/node-v12.22.2-linux-x64.tar.xz" \
+            | sudo tar --strip-components=2 -xJ -C /usr/local/bin/ node-v12.22.2-linux-x64/bin/node
+      - run:
+          name: Check current version of node
+          command: node -v
+
   save_workspace:
     description: Persist workspace
     steps:
@@ -47,51 +86,46 @@ commands:
     steps:
       - restore_cache:
           name: Restore yarn cache
-          key: yarn-v3-{{ checksum "yarn.lock" }}
+          key: yarn-v4-{{ checksum "yarn.lock" }}
   save_yarn_cache:
     description: Save yarn cache
     steps:
       - save_cache:
           name: Save yarn cache
-          key: yarn-v3-{{ checksum "yarn.lock" }}
+          key: yarn-v4-{{ checksum "yarn.lock" }}
           paths:
             - ~/.cache/yarn
   restore_yarn_cache_mastodon:
     description: Restore yarn cache for Mastodon
     steps:
       - restore_cache:
           name: Restore yarn cache for Mastodon
-          key: yarn-v3-{{ checksum "mastodon/yarn.lock" }}
+          key: yarn-v4-{{ checksum "mastodon/yarn.lock" }}
   save_yarn_cache_mastodon:
     description: Save yarn cache for Mastodon
     steps:
       - save_cache:
           name: Save yarn cache for Mastodon
-          key: yarn-v3-{{ checksum "mastodon/yarn.lock" }}
+          key: yarn-v4-{{ checksum "mastodon/yarn.lock" }}
           paths:
             - ~/.cache/yarn
   restore_bundler_cache:
     description: Restore bundler cache
     steps:
       - restore_cache:
           name: Restore bundler cache
-          key: bundler-v2-{{ checksum "mastodon/Gemfile.lock" }}
+          key: bundler-v4-{{ checksum "mastodon/Gemfile.lock" }}
   save_bundler_cache:
     description: Save bundler cache
     steps:
       - save_cache:
           name: Save bundler cache
-          key: bundler-v2-{{ checksum "mastodon/Gemfile.lock" }}
+          key: bundler-v4-{{ checksum "mastodon/Gemfile.lock" }}
           paths:
             - mastodon/vendor/bundle
   install_mastodon:
     description: Install Mastodon and set up Postgres/Redis
     steps:
-      - run:
-          name: Install system dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y ffmpeg fonts-noto-color-emoji libicu-dev libidn11-dev libprotobuf-dev postgresql-contrib protobuf-compiler
       - run:
           name: Clone mastodon
           command: yarn clone-mastodon
@@ -127,6 +161,7 @@ jobs:
     executor: node
     steps:
       - checkout
+      - install_node
       - restore_yarn_cache
       - run:
           name: Yarn install
@@ -156,6 +191,9 @@ jobs:
   integration_test_readonly:
     executor: node_and_ruby
     steps:
+      - install_mastodon_system_dependencies
+      - install_browsers
+      - install_node
       - load_workspace
       - install_mastodon
       - run:
@@ -164,6 +202,9 @@ jobs:
   integration_test_readwrite:
     executor: node_and_ruby
     steps:
+      - install_mastodon_system_dependencies
+      - install_browsers
+      - install_node
       - load_workspace
       - install_mastodon
       - run:

.gitignore

@@ -14,3 +14,4 @@ yarn-error.log
 
 .now
 .vercel
+/webpack/*.cjs

bin/build-assets.js

@@ -1,9 +1,10 @@
 import path from 'path'
 import fs from 'fs'
 import { promisify } from 'util'
-import { LOCALE } from '../src/routes/_static/intl'
-import { getIntl, trimWhitespace } from './getIntl'
+import { LOCALE } from '../src/routes/_static/intl.js'
+import { getIntl, trimWhitespace } from './getIntl.js'
 
+const __dirname = path.dirname(new URL(import.meta.url).pathname)
 const readFile = promisify(fs.readFile)
 const writeFile = promisify(fs.writeFile)
 

bin/build-inline-script.js

@@ -5,13 +5,12 @@ import path from 'path'
 import { rollup } from 'rollup'
 import { terser } from 'rollup-plugin-terser'
 import replace from '@rollup/plugin-replace'
-import fromPairs from 'lodash-es/fromPairs'
-import { themes } from '../src/routes/_static/themes'
-import terserOptions from './terserOptions'
+import { themes } from '../src/routes/_static/themes.js'
+import terserOptions from './terserOptions.js'
 
+const __dirname = path.dirname(new URL(import.meta.url).pathname)
 const writeFile = promisify(fs.writeFile)
-
-const themeColors = fromPairs(themes.map(_ => ([_.name, _.color])))
+const themeColors = Object.fromEntries(themes.map(_ => ([_.name, _.color])))
 
 export async function buildInlineScript () {
   const inlineScriptPath = path.join(__dirname, '../src/inline-script/inline-script.js')
@@ -42,7 +41,7 @@ export async function buildInlineScript () {
   const checksum = crypto.createHash('sha256').update(fullCode, 'utf8').digest('base64')
 
   await writeFile(path.resolve(__dirname, '../src/inline-script/checksum.js'),
-    `module.exports = ${JSON.stringify(checksum)}`, 'utf8')
+    `export default ${JSON.stringify(checksum)}`, 'utf8')
   await writeFile(path.resolve(__dirname, '../static/inline-script.js.map'),
     map.toString(), 'utf8')
 

bin/build-sass.js

@@ -3,10 +3,12 @@ import path from 'path'
 import fs from 'fs'
 import { promisify } from 'util'
 import cssDedoupe from 'css-dedoupe'
-import { TextDecoder } from 'text-encoding'
+import textEncodingPackage from 'text-encoding'
+const { TextDecoder } = textEncodingPackage
 
 const writeFile = promisify(fs.writeFile)
 const readdir = promisify(fs.readdir)
+const __dirname = path.dirname(new URL(import.meta.url).pathname)
 
 const globalScss = path.join(__dirname, '../src/scss/global.scss')
 const defaultThemeScss = path.join(__dirname, '../src/scss/themes/_default.scss')

bin/build-svg.js

@@ -1,10 +1,13 @@
-import svgs from './svgs'
+import svgs from './svgs.js'
 import path from 'path'
 import fs from 'fs'
 import { promisify } from 'util'
 import { optimize } from 'svgo'
-import $ from 'cheerio'
+import cheerioPackage from 'cheerio'
 
+const { default: $ } = cheerioPackage
+
+const __dirname = path.dirname(new URL(import.meta.url).pathname)
 const readFile = promisify(fs.readFile)
 const writeFile = promisify(fs.writeFile)
 

bin/build-template-html.js

@@ -2,15 +2,18 @@ import chokidar from 'chokidar'
 import fs from 'fs'
 import path from 'path'
 import { promisify } from 'util'
-import { buildSass } from './build-sass'
-import { buildInlineScript } from './build-inline-script'
-import { buildSvg } from './build-svg'
+import { buildSass } from './build-sass.js'
+import { buildInlineScript } from './build-inline-script.js'
+import { buildSvg } from './build-svg.js'
 import { performance } from 'perf_hooks'
-import debounce from 'lodash-es/debounce'
-import applyIntl from '../webpack/svelte-intl-loader'
-import { LOCALE } from '../src/routes/_static/intl'
-import { getLangDir } from 'rtl-detect'
+import { debounce } from '../src/routes/_thirdparty/lodash/timers.js'
+import applyIntl from '../webpack/svelte-intl-loader.js'
+import { LOCALE } from '../src/routes/_static/intl.js'
+import rtlDetectPackage from 'rtl-detect'
 
+const { getLangDir } = rtlDetectPackage
+
+const __dirname = path.dirname(new URL(import.meta.url).pathname)
 const writeFile = promisify(fs.writeFile)
 const LOCALE_DIRECTION = getLangDir(LOCALE)
 const DEBOUNCE = 500

bin/build-vercel-json.js

@@ -5,11 +5,12 @@
 import path from 'path'
 import fs from 'fs'
 import { promisify } from 'util'
-import { routes } from '../__sapper__/service-worker'
-import cloneDeep from 'lodash-es/cloneDeep'
-import inlineScriptChecksum from '../src/inline-script/checksum'
-import { sapperInlineScriptChecksums } from '../src/server/sapperInlineScriptChecksums'
+import { routes } from '../__sapper__/service-worker.js'
+import { cloneDeep } from '../src/routes/_utils/lodash-lite.js'
+import inlineScriptChecksum from '../src/inline-script/checksum.js'
+import { sapperInlineScriptChecksums } from '../src/server/sapperInlineScriptChecksums.js'
 
+const __dirname = path.dirname(new URL(import.meta.url).pathname)
 const writeFile = promisify(fs.writeFile)
 
 const JSON_TEMPLATE = {

bin/clone-mastodon.js

@@ -2,11 +2,13 @@ import { promisify } from 'util'
 import childProcessPromise from 'child-process-promise'
 import path from 'path'
 import fs from 'fs'
-import { envFile, RUBY_VERSION } from './mastodon-config'
+import { envFile, RUBY_VERSION } from './mastodon-config.js'
+import esMain from 'es-main'
 
 const exec = childProcessPromise.exec
 const stat = promisify(fs.stat)
 const writeFile = promisify(fs.writeFile)
+const __dirname = path.dirname(new URL(import.meta.url).pathname)
 const dir = __dirname
 
 const GIT_URL = 'https://github.com/tootsuite/mastodon.git'
@@ -25,7 +27,7 @@ export default async function cloneMastodon () {
   }
 }
 
-if (require.main === module) {
+if (esMain(import.meta)) {
   cloneMastodon().catch(err => {
     console.error(err)
     process.exit(1)

bin/getIntl.js

@@ -1,9 +1,19 @@
-import { get } from 'lodash-es'
-import { DEFAULT_LOCALE, LOCALE } from '../src/routes/_static/intl'
-import path from 'path'
+import { get } from '../src/routes/_utils/lodash-lite.js'
+import { DEFAULT_LOCALE, LOCALE } from '../src/routes/_static/intl.js'
 
-const intl = require(path.join(__dirname, '../src/intl', LOCALE + '.js')).default
-const defaultIntl = require(path.join(__dirname, '../src/intl', DEFAULT_LOCALE + '.js')).default
+import enUS from '../src/intl/en-US.js'
+import fr from '../src/intl/fr.js'
+import de from '../src/intl/de.js'
+
+// TODO: make it so we don't have to explicitly list these out
+const locales = {
+  'en-US': enUS,
+  fr,
+  de
+}
+
+const intl = locales[LOCALE]
+const defaultIntl = locales[DEFAULT_LOCALE]
 
 export function warningOrError (message) { // avoid crashing the whole server on `yarn dev`
   if (process.env.NODE_ENV === 'production') {
@@ -14,6 +24,7 @@ export function warningOrError (message) { // avoid crashing the whole server on
 }
 
 export function getIntl (path) {
+  path = path.split('.')
   const res = get(intl, path, get(defaultIntl, path))
   if (typeof res !== 'string') {
     return warningOrError('Unknown intl string: ' + JSON.stringify(path))

bin/install-mastodon.js

@@ -2,12 +2,14 @@ import { promisify } from 'util'
 import childProcessPromise from 'child-process-promise'
 import path from 'path'
 import fs from 'fs'
-import { DB_NAME, DB_PASS, DB_USER, mastodonDir, env } from './mastodon-config'
+import { DB_NAME, DB_PASS, DB_USER, mastodonDir, env } from './mastodon-config.js'
 import mkdirp from 'mkdirp'
+import esMain from 'es-main'
 
 const exec = childProcessPromise.exec
 const stat = promisify(fs.stat)
 const writeFile = promisify(fs.writeFile)
+const __dirname = path.dirname(new URL(import.meta.url).pathname)
 const dir = __dirname
 
 async function setupMastodonDatabase () {
@@ -69,7 +71,7 @@ export default async function installMastodon () {
   await installMastodonDependencies()
 }
 
-if (require.main === module) {
+if (esMain(import.meta)) {
   installMastodon().catch(err => {
     console.error(err)
     process.exit(1)

bin/mastodon-config.js

@@ -17,10 +17,9 @@ DB_NAME=${DB_NAME}
 DB_PASS=${DB_PASS}
 `
 
-// Need a Ruby version that CircleCI bundles with Node v12, not Node v14 which doesn't
-// work for streaming
-export const RUBY_VERSION = '2.6.6'
+export const RUBY_VERSION = '2.7.2'
 
+const __dirname = path.dirname(new URL(import.meta.url).pathname)
 export const mastodonDir = path.join(__dirname, '../mastodon')
 
 export const env = Object.assign({}, process.env, {

bin/mastodon-data.js

@@ -1,4 +1,4 @@
-import times from 'lodash-es/times'
+import { times } from '../src/routes/_utils/lodash-lite.js'
 
 function unrollThread (user, prefix, privacy, thread) {
   const res = []

bin/restore-mastodon-data.js

@@ -1,13 +1,13 @@
-import { actions } from './mastodon-data'
-import { users } from '../tests/users'
-import { postStatus } from '../src/routes/_api/statuses'
-import { followAccount } from '../src/routes/_api/follow'
-import { favoriteStatus } from '../src/routes/_api/favorite'
-import { reblogStatus } from '../src/routes/_api/reblog'
+import { actions } from './mastodon-data.js'
+import { users } from '../tests/users.js'
+import { postStatus } from '../src/routes/_api/statuses.js'
+import { followAccount } from '../src/routes/_api/follow.js'
+import { favoriteStatus } from '../src/routes/_api/favorite.js'
+import { reblogStatus } from '../src/routes/_api/reblog.js'
 import fetch from 'node-fetch'
 import FileApi from 'file-api'
-import { pinStatus } from '../src/routes/_api/pin'
-import { submitMedia } from '../tests/submitMedia'
+import { pinStatus } from '../src/routes/_api/pin.js'
+import { submitMedia } from '../tests/submitMedia.js'
 
 global.File = FileApi.File
 global.FormData = FileApi.FormData