From 83d165e7edf0c97aa6a959f835b7993e67b3d4e0 Mon Sep 17 00:00:00 2001 From: Brad Turner Date: Sat, 24 May 2025 11:38:50 +1000 Subject: [PATCH 1/5] Switch config file require to dynamic import --- lib/index.js | 133 ++++++++++++++++++-------------------------- lib/parseOptions.js | 23 ++++---- 2 files changed, 68 insertions(+), 88 deletions(-) diff --git a/lib/index.js b/lib/index.js index 7905a906..e36e4c6b 100644 --- a/lib/index.js +++ b/lib/index.js @@ -8,15 +8,10 @@ import UpdateRenderer from 'listr-update-renderer' import VerboseRenderer from 'listr-verbose-renderer' import startCase from 'lodash.startcase' import mkdirp from 'mkdirp' -import { differenceInSeconds } from 'date-fns/differenceInSeconds' -import { formatDistance } from 'date-fns/formatDistance' +import differenceInSeconds from 'date-fns/differenceInSeconds' +import formatDistance from 'date-fns/formatDistance' -import { - setupLogging, - displayErrorLog, - wrapTask, - writeErrorLogFile -} from 'contentful-batch-libs' +import { setupLogging, displayErrorLog, wrapTask, writeErrorLogFile } from 'contentful-batch-libs' import downloadAssets from './tasks/download-assets' import getSpaceData from './tasks/get-space-data' @@ -31,7 +26,7 @@ const tableOptions = { style: { head: [], border: [] } } -function createListrOptions (options) { +function createListrOptions(options) { if (options.useVerboseRenderer) { return { renderer: VerboseRenderer @@ -43,9 +38,9 @@ function createListrOptions (options) { } } -export default function runContentfulExport (params) { +export default async function runContentfulExport(params) { const log = [] - const options = parseOptions(params) + const options = await parseOptions(params) const listrOptions = createListrOptions(options) @@ -97,9 +92,7 @@ export default function runContentfulExport (params) { { title: 'Download assets', task: wrapTask(downloadAssets(options)), - skip: (ctx) => - !options.downloadAssets || - !Object.prototype.hasOwnProperty.call(ctx.data, 'assets') + skip: (ctx) => !options.downloadAssets || !Object.prototype.hasOwnProperty.call(ctx.data, 'assets') }, { title: 'Write export log file', @@ -141,78 +134,62 @@ export default function runContentfulExport (params) { listrOptions ) - return tasks - .run({ - data: {} - }) - .then((ctx) => { - const resultTypes = Object.keys(ctx.data) - if (resultTypes.length) { - const resultTable = new Table(tableOptions) + try { + const ctx = await tasks.run({ data: {} }) + const resultTypes = Object.keys(ctx.data) + if (resultTypes.length) { + const resultTable = new Table(tableOptions) - resultTable.push([{ colSpan: 2, content: 'Exported entities' }]) + resultTable.push([{ colSpan: 2, content: 'Exported entities' }]) - resultTypes.forEach((type) => { - resultTable.push([startCase(type), ctx.data[type].length]) - }) + resultTypes.forEach((type) => { + resultTable.push([startCase(type), ctx.data[type].length]) + }) - console.log(resultTable.toString()) - } else { - console.log('No data was exported') - } + console.log(resultTable.toString()) + } else { + console.log('No data was exported') + } - if ('assetDownloads' in ctx) { - const downloadsTable = new Table(tableOptions) - downloadsTable.push([ - { colSpan: 2, content: 'Asset file download results' } - ]) - downloadsTable.push(['Successful', ctx.assetDownloads.successCount]) - downloadsTable.push(['Warnings ', ctx.assetDownloads.warningCount]) - downloadsTable.push(['Errors ', ctx.assetDownloads.errorCount]) - console.log(downloadsTable.toString()) - } + if ('assetDownloads' in ctx) { + const downloadsTable = new Table(tableOptions) + downloadsTable.push([{ colSpan: 2, content: 'Asset file download results' }]) + downloadsTable.push(['Successful', ctx.assetDownloads.successCount]) + downloadsTable.push(['Warnings ', ctx.assetDownloads.warningCount]) + downloadsTable.push(['Errors ', ctx.assetDownloads.errorCount]) + console.log(downloadsTable.toString()) + } - const endTime = new Date() - const durationHuman = formatDistance(endTime, options.startTime) - const durationSeconds = differenceInSeconds(endTime, options.startTime) + const endTime = new Date() + const durationHuman = formatDistance(endTime, options.startTime) + const durationSeconds = differenceInSeconds(endTime, options.startTime) - console.log(`The export took ${durationHuman} (${durationSeconds}s)`) - if (options.saveFile) { - console.log( - `\nStored space data to json file at: ${options.logFilePath}` - ) - } - return ctx.data - }) - .catch((err) => { - log.push({ - ts: new Date().toJSON(), - level: 'error', - error: err + console.log(`The export took ${durationHuman} (${durationSeconds}s)`) + if (options.saveFile) { + console.log(`\nStored space data to json file at: ${options.logFilePath}`) + } + + const errorLog = log.filter((logMessage) => logMessage.level !== 'info' && logMessage.level !== 'warning') + const displayLog = log.filter((logMessage) => logMessage.level !== 'info') + displayErrorLog(displayLog) + + if (errorLog.length) { + return writeErrorLogFile(options.errorLogFile, errorLog).then(() => { + const multiError = new Error('Errors occured') + multiError.name = 'ContentfulMultiError' + Object.assign(multiError, { errors: errorLog }) + throw multiError }) - }) - .then((data) => { - // @todo this should live in batch libs - const errorLog = log.filter( - (logMessage) => - logMessage.level !== 'info' && logMessage.level !== 'warning' - ) - const displayLog = log.filter( - (logMessage) => logMessage.level !== 'info' - ) - displayErrorLog(displayLog) - - if (errorLog.length) { - return writeErrorLogFile(options.errorLogFile, errorLog).then(() => { - const multiError = new Error('Errors occured') - multiError.name = 'ContentfulMultiError' - Object.assign(multiError, { errors: errorLog }) - throw multiError - }) - } + } - console.log('The export was successful.') + console.log('The export was successful.') - return data + return ctx.data + } catch (err) { + log.push({ + ts: new Date().toJSON(), + level: 'error', + error: err }) + } } diff --git a/lib/parseOptions.js b/lib/parseOptions.js index 24516ad0..bef975d6 100644 --- a/lib/parseOptions.js +++ b/lib/parseOptions.js @@ -1,12 +1,11 @@ import { addSequenceHeader, agentFromProxy, proxyStringToObject } from 'contentful-batch-libs' -import { format } from 'date-fns/format' +import format from 'date-fns/format' import { resolve } from 'path' import qs from 'querystring' - import { version } from '../package.json' import { getHeadersConfig } from './utils/headers' -export default function parseOptions (params) { +export default async function parseOptions(params) { const defaultOptions = { environmentId: 'master', exportDir: process.cwd(), @@ -25,10 +24,7 @@ export default function parseOptions (params) { rawProxy: false } - const configFile = params.config - // eslint-disable-next-line @typescript-eslint/no-require-imports - ? require(resolve(process.cwd(), params.config)) - : {} + const configFile = await (params.config ? import(resolve(process.cwd(), params.config)) : Promise.resolve({})) const options = { ...defaultOptions, @@ -47,12 +43,17 @@ export default function parseOptions (params) { } options.startTime = new Date() - options.contentFile = options.contentFile || `contentful-export-${options.spaceId}-${options.environmentId}-${format(options.startTime, "yyyy-MM-dd'T'HH-mm-ss")}.json` + options.contentFile = + options.contentFile || + `contentful-export-${options.spaceId}-${options.environmentId}-${format(options.startTime, "yyyy-MM-dd'T'HH-mm-ss")}.json` options.logFilePath = resolve(options.exportDir, options.contentFile) if (!options.errorLogFile) { - options.errorLogFile = resolve(options.exportDir, `contentful-export-error-log-${options.spaceId}-${options.environmentId}-${format(options.startTime, "yyyy-MM-dd'T'HH-mm-ss")}.json`) + options.errorLogFile = resolve( + options.exportDir, + `contentful-export-error-log-${options.spaceId}-${options.environmentId}-${format(options.startTime, "yyyy-MM-dd'T'HH-mm-ss")}.json` + ) } else { options.errorLogFile = resolve(process.cwd(), options.errorLogFile) } @@ -65,7 +66,9 @@ export default function parseOptions (params) { const proxySimpleExp = /.+:\d+/ const proxyAuthExp = /.+:.+@.+:\d+/ if (!(proxySimpleExp.test(options.proxy) || proxyAuthExp.test(options.proxy))) { - throw new Error('Please provide the proxy config in the following format:\nhost:port or user:password@host:port') + throw new Error( + 'Please provide the proxy config in the following format:\nhost:port or user:password@host:port' + ) } options.proxy = proxyStringToObject(options.proxy) } From 9cba224a4e38a9b0a8d5b7e80de08e671ae7fab5 Mon Sep 17 00:00:00 2001 From: Brad Turner Date: Sat, 24 May 2025 11:50:52 +1000 Subject: [PATCH 2/5] Use default export from external config file --- lib/index.js | 8 ++------ lib/parseOptions.js | 8 ++++++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/index.js b/lib/index.js index e36e4c6b..85b93e1b 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,5 +1,4 @@ import { access } from 'fs' - import bfj from 'bfj' import Promise from 'bluebird' import Table from 'cli-table3' @@ -8,15 +7,12 @@ import UpdateRenderer from 'listr-update-renderer' import VerboseRenderer from 'listr-verbose-renderer' import startCase from 'lodash.startcase' import mkdirp from 'mkdirp' -import differenceInSeconds from 'date-fns/differenceInSeconds' -import formatDistance from 'date-fns/formatDistance' - +import { differenceInSeconds } from 'date-fns/differenceInSeconds' +import { formatDistance } from 'date-fns/formatDistance' import { setupLogging, displayErrorLog, wrapTask, writeErrorLogFile } from 'contentful-batch-libs' - import downloadAssets from './tasks/download-assets' import getSpaceData from './tasks/get-space-data' import initClient from './tasks/init-client' - import parseOptions from './parseOptions' const accessP = Promise.promisify(access) diff --git a/lib/parseOptions.js b/lib/parseOptions.js index bef975d6..e6ac3a7b 100644 --- a/lib/parseOptions.js +++ b/lib/parseOptions.js @@ -1,5 +1,5 @@ import { addSequenceHeader, agentFromProxy, proxyStringToObject } from 'contentful-batch-libs' -import format from 'date-fns/format' +import { format } from 'date-fns/format' import { resolve } from 'path' import qs from 'querystring' import { version } from '../package.json' @@ -24,7 +24,11 @@ export default async function parseOptions(params) { rawProxy: false } - const configFile = await (params.config ? import(resolve(process.cwd(), params.config)) : Promise.resolve({})) + let configFile = {}; + if (params.config) { + const externalConfigFile = await import(resolve(process.cwd(), params.config)) + configFile = externalConfigFile.default; + } const options = { ...defaultOptions, From 35f9f942b99c335031072053b652d11c856101a6 Mon Sep 17 00:00:00 2001 From: Brad Turner Date: Sat, 24 May 2025 12:11:35 +1000 Subject: [PATCH 3/5] Fix dumb lint rules --- lib/index.js | 4 ++-- lib/parseOptions.js | 6 +++--- lib/tasks/download-assets.js | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/index.js b/lib/index.js index 85b93e1b..528901fa 100644 --- a/lib/index.js +++ b/lib/index.js @@ -22,7 +22,7 @@ const tableOptions = { style: { head: [], border: [] } } -function createListrOptions(options) { +function createListrOptions (options) { if (options.useVerboseRenderer) { return { renderer: VerboseRenderer @@ -34,7 +34,7 @@ function createListrOptions(options) { } } -export default async function runContentfulExport(params) { +export default async function runContentfulExport (params) { const log = [] const options = await parseOptions(params) diff --git a/lib/parseOptions.js b/lib/parseOptions.js index e6ac3a7b..84094931 100644 --- a/lib/parseOptions.js +++ b/lib/parseOptions.js @@ -5,7 +5,7 @@ import qs from 'querystring' import { version } from '../package.json' import { getHeadersConfig } from './utils/headers' -export default async function parseOptions(params) { +export default async function parseOptions (params) { const defaultOptions = { environmentId: 'master', exportDir: process.cwd(), @@ -24,10 +24,10 @@ export default async function parseOptions(params) { rawProxy: false } - let configFile = {}; + let configFile = {} if (params.config) { const externalConfigFile = await import(resolve(process.cwd(), params.config)) - configFile = externalConfigFile.default; + configFile = externalConfigFile.default } const options = { diff --git a/lib/tasks/download-assets.js b/lib/tasks/download-assets.js index b5348f70..40366d1e 100644 --- a/lib/tasks/download-assets.js +++ b/lib/tasks/download-assets.js @@ -33,8 +33,8 @@ async function downloadAsset ({ url, directory, httpClient }) { try { // download asset const assetRequest = await httpClient.get(url, { - responseType: "stream", - transformResponse: [(data) => data], + responseType: 'stream', + transformResponse: [(data) => data] }) // Wait for stream to be consumed before returning local file @@ -92,7 +92,7 @@ export default function downloadAssets (options) { return startingPromise .then(downloadAsset) - .then((_downLoadedFile) => { + .then(() => { task.output = `${figures.tick} downloaded ${entityName} (${url})` successCount++ }) From 7f04a3132c4e04376a8243f58bc0013a68c75b44 Mon Sep 17 00:00:00 2001 From: Brad Turner Date: Sat, 24 May 2025 12:34:41 +1000 Subject: [PATCH 4/5] Adjust tests and fix index.js file --- lib/index.js | 37 ++++++++++---------- test/unit/parseOptions.test.js | 62 +++++++++++++++++----------------- 2 files changed, 51 insertions(+), 48 deletions(-) diff --git a/lib/index.js b/lib/index.js index 528901fa..abf8cff9 100644 --- a/lib/index.js +++ b/lib/index.js @@ -130,8 +130,9 @@ export default async function runContentfulExport (params) { listrOptions ) + let ctx; try { - const ctx = await tasks.run({ data: {} }) + ctx = await tasks.run({ data: {} }) const resultTypes = Object.keys(ctx.data) if (resultTypes.length) { const resultTable = new Table(tableOptions) @@ -165,22 +166,7 @@ export default async function runContentfulExport (params) { console.log(`\nStored space data to json file at: ${options.logFilePath}`) } - const errorLog = log.filter((logMessage) => logMessage.level !== 'info' && logMessage.level !== 'warning') - const displayLog = log.filter((logMessage) => logMessage.level !== 'info') - displayErrorLog(displayLog) - - if (errorLog.length) { - return writeErrorLogFile(options.errorLogFile, errorLog).then(() => { - const multiError = new Error('Errors occured') - multiError.name = 'ContentfulMultiError' - Object.assign(multiError, { errors: errorLog }) - throw multiError - }) - } - - console.log('The export was successful.') - - return ctx.data + } catch (err) { log.push({ ts: new Date().toJSON(), @@ -188,4 +174,21 @@ export default async function runContentfulExport (params) { error: err }) } + + const errorLog = log.filter((logMessage) => logMessage.level !== 'info' && logMessage.level !== 'warning') + const displayLog = log.filter((logMessage) => logMessage.level !== 'info') + displayErrorLog(displayLog) + + if (errorLog.length) { + return writeErrorLogFile(options.errorLogFile, errorLog).then(() => { + const multiError = new Error('Errors occured') + multiError.name = 'ContentfulMultiError' + Object.assign(multiError, { errors: errorLog }) + throw multiError + }) + } + + console.log('The export was successful.') + + return ctx.data } diff --git a/test/unit/parseOptions.test.js b/test/unit/parseOptions.test.js index b60c5ef3..00f84c4d 100644 --- a/test/unit/parseOptions.test.js +++ b/test/unit/parseOptions.test.js @@ -12,25 +12,25 @@ const toBeAbsolutePathWithPattern = (received, pattern) => { return (!isAbsolute(received) || !RegExp(`/${escapedPattern}$/`).test(received)) } -test('parseOptions sets requires spaceId', () => { - expect( +test('parseOptions sets requires spaceId', async () => { + await expect( () => parseOptions({}) - ).toThrow('The `spaceId` option is required.') + ).rejects.toThrow('The `spaceId` option is required.') }) -test('parseOptions sets requires managementToken', () => { - expect( +test('parseOptions sets requires managementToken', async () => { + await expect( () => parseOptions({ spaceId: 'someSpaceId' }) - ).toThrow('The `managementToken` option is required.') + ).rejects.toThrow('The `managementToken` option is required.') }) test('parseOptions sets correct default options', async () => { const { default: packageJson } = await import(resolve(basePath, 'package.json')) const version = packageJson.version - const options = parseOptions({ spaceId, managementToken }) + const options = await parseOptions({ spaceId, managementToken }) const contentFileNamePattern = `contentful-export-${spaceId}-master-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}\\.json` const errorFileNamePattern = `contentful-export-error-log-${spaceId}-master-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}\\.json` @@ -63,15 +63,15 @@ test('parseOption accepts config file', async () => { const configFileName = 'example-config.test.json' const { default: config } = await import(resolve(basePath, configFileName)) - const options = parseOptions({ config: configFileName }) + const options = await parseOptions({ config: configFileName }) Object.keys(config).forEach((key) => { expect(options[key]).toBe(config[key]) }) }) -test('parseOption overwrites errorLogFile', () => { +test('parseOption overwrites errorLogFile', async () => { const errorLogFile = 'error.log' - const options = parseOptions({ + const options = await parseOptions({ spaceId, managementToken, errorLogFile @@ -79,16 +79,16 @@ test('parseOption overwrites errorLogFile', () => { expect(options.errorLogFile).toBe(resolve(basePath, errorLogFile)) }) -test('parseOption throws with invalid proxy', () => { - expect(() => parseOptions({ +test('parseOption throws with invalid proxy', async () => { + await expect(() => parseOptions({ spaceId, managementToken, proxy: 'invalid' - })).toThrow('Please provide the proxy config in the following format:\nhost:port or user:password@host:port') + })).rejects.toThrow('Please provide the proxy config in the following format:\nhost:port or user:password@host:port') }) -test('parseOption accepts proxy config as string', () => { - const options = parseOptions({ +test('parseOption accepts proxy config as string', async () => { + const options = await parseOptions({ spaceId, managementToken, proxy: 'localhost:1234' @@ -97,8 +97,8 @@ test('parseOption accepts proxy config as string', () => { expect(options.httpsAgent).toBeInstanceOf(HttpsProxyAgent) }) -test('parseOption accepts proxy config as object', () => { - const options = parseOptions({ +test('parseOption accepts proxy config as object', async () => { + const options = await parseOptions({ spaceId, managementToken, proxy: { @@ -112,8 +112,8 @@ test('parseOption accepts proxy config as object', () => { expect(options.httpsAgent).toBeInstanceOf(HttpsProxyAgent) }) -test('parseOptions parses queryEntries option', () => { - const options = parseOptions({ +test('parseOptions parses queryEntries option', async () => { + const options = await parseOptions({ spaceId, managementToken, queryEntries: [ @@ -127,8 +127,8 @@ test('parseOptions parses queryEntries option', () => { }) }) -test('parseOptions parses queryAssets option', () => { - const options = parseOptions({ +test('parseOptions parses queryAssets option', async () => { + const options = await parseOptions({ spaceId, managementToken, queryAssets: [ @@ -142,8 +142,8 @@ test('parseOptions parses queryAssets option', () => { }) }) -test('parseOptions sets correct options given contentOnly', () => { - const options = parseOptions({ +test('parseOptions sets correct options given contentOnly', async () => { + const options = await parseOptions({ spaceId, managementToken, contentOnly: true @@ -153,11 +153,11 @@ test('parseOptions sets correct options given contentOnly', () => { expect(options.skipWebhooks).toBe(true) }) -test('parseOptions accepts custom application & feature', () => { +test('parseOptions accepts custom application & feature', async () => { const managementApplication = 'managementApplicationMock' const managementFeature = 'managementFeatureMock' - const options = parseOptions({ + const options = await parseOptions({ spaceId, managementToken, managementApplication, @@ -168,8 +168,8 @@ test('parseOptions accepts custom application & feature', () => { expect(options.feature).toBe(managementFeature) }) -test('parseOption parses deliveryToken option', () => { - const options = parseOptions({ +test('parseOption parses deliveryToken option', async () => { + const options = await parseOptions({ spaceId, managementToken, deliveryToken: 'testDeliveryToken' @@ -179,8 +179,8 @@ test('parseOption parses deliveryToken option', () => { expect(options.deliveryToken).toBe('testDeliveryToken') }) -test('parseOption parses headers option', () => { - const options = parseOptions({ +test('parseOption parses headers option', async () => { + const options = await parseOptions({ spaceId, managementToken, headers: { @@ -195,8 +195,8 @@ test('parseOption parses headers option', () => { }) }) -test('parses params.header if provided', function () { - const config = parseOptions({ +test('parses params.header if provided', async () => { + const config = await parseOptions({ spaceId, managementToken, header: ['Accept : application/json ', ' X-Header: 1'] From 0218c159aa731a09dd944e55c9785dbd148f69f4 Mon Sep 17 00:00:00 2001 From: Brad Turner Date: Sat, 24 May 2025 12:36:31 +1000 Subject: [PATCH 5/5] More lint fixes :( --- lib/index.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/index.js b/lib/index.js index abf8cff9..ba30aa4f 100644 --- a/lib/index.js +++ b/lib/index.js @@ -130,7 +130,7 @@ export default async function runContentfulExport (params) { listrOptions ) - let ctx; + let ctx try { ctx = await tasks.run({ data: {} }) const resultTypes = Object.keys(ctx.data) @@ -165,8 +165,6 @@ export default async function runContentfulExport (params) { if (options.saveFile) { console.log(`\nStored space data to json file at: ${options.logFilePath}`) } - - } catch (err) { log.push({ ts: new Date().toJSON(),