diff --git a/integration-tests/worker/CHANGELOG.md b/integration-tests/worker/CHANGELOG.md index 7f6aece75..0d67f01f3 100644 --- a/integration-tests/worker/CHANGELOG.md +++ b/integration-tests/worker/CHANGELOG.md @@ -4,6 +4,12 @@ ### Patch Changes +- Updated dependencies [2fde0ad] +- Updated dependencies [2fde0ad] + - @openfn/logger@1.0.1 + - @openfn/engine-multi@1.0.1 + - @openfn/lightning-mock@2.0.1 + - @openfn/ws-worker@1.0.1 - Updated dependencies [4f5f1dd] - Updated dependencies [58e0d11] - Updated dependencies [58e0d11] diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 922085f3d..92603c99c 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,14 @@ # @openfn/cli +## 1.1.1 + +### Patch Changes + +- Updated dependencies [2fde0ad] + - @openfn/logger@1.0.1 + - @openfn/compiler@0.0.41 + - @openfn/deploy@0.4.3 + - @openfn/runtime@1.0.1 ## 1.1.0 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index b7de5b514..17ad7ae8d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/cli", - "version": "1.1.0", + "version": "1.1.1", "description": "CLI devtools for the openfn toolchain.", "engines": { "node": ">=18", diff --git a/packages/compiler/CHANGELOG.md b/packages/compiler/CHANGELOG.md index c50e21489..a88109535 100644 --- a/packages/compiler/CHANGELOG.md +++ b/packages/compiler/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/compiler +## 0.0.41 + +### Patch Changes + +- Updated dependencies [2fde0ad] + - @openfn/logger@1.0.1 + ## 0.0.40 ### Patch Changes diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 45d5718a0..4a1782c9f 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/compiler", - "version": "0.0.40", + "version": "0.0.41", "description": "Compiler and language tooling for openfn jobs.", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/deploy/CHANGELOG.md b/packages/deploy/CHANGELOG.md index 66e70c31d..e7e627e2a 100644 --- a/packages/deploy/CHANGELOG.md +++ b/packages/deploy/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/deploy +## 0.4.3 + +### Patch Changes + +- Updated dependencies [2fde0ad] + - @openfn/logger@1.0.1 + ## 0.4.2 ### Patch Changes diff --git a/packages/deploy/package.json b/packages/deploy/package.json index aef2ded1c..fcf37d1fe 100644 --- a/packages/deploy/package.json +++ b/packages/deploy/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/deploy", - "version": "0.4.2", + "version": "0.4.3", "description": "Deploy projects to Lightning instances", "type": "module", "exports": { diff --git a/packages/engine-multi/CHANGELOG.md b/packages/engine-multi/CHANGELOG.md index c1ba43d97..7a2c7ef77 100644 --- a/packages/engine-multi/CHANGELOG.md +++ b/packages/engine-multi/CHANGELOG.md @@ -1,5 +1,15 @@ # engine-multi +## 1.1.1 + +### Patch Changes + +- 2fde0ad: Slightly better error reporting for exceptions +- Updated dependencies [2fde0ad] + - @openfn/logger@1.0.1 + - @openfn/compiler@0.0.41 + - @openfn/lexicon@1.0.0 + - @openfn/runtime@1.0.1 ## 1.1.0 ### Minor Changes diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index 251523fbb..0492232f3 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/engine-multi", - "version": "1.1.0", + "version": "1.1.1", "description": "Multi-process runtime engine", "main": "dist/index.js", "type": "module", diff --git a/packages/engine-multi/src/errors.ts b/packages/engine-multi/src/errors.ts index 5468aa390..cae5e1a39 100644 --- a/packages/engine-multi/src/errors.ts +++ b/packages/engine-multi/src/errors.ts @@ -33,11 +33,13 @@ export class ExecutionError extends EngineError { message; original: any; // this is the original error + constructor(original: any) { super(); this.original = original; this.message = original.message; + this.stack = original.stack; } } diff --git a/packages/engine-multi/src/worker/thread/helpers.ts b/packages/engine-multi/src/worker/thread/helpers.ts index fb3e4d9ee..06c924a44 100644 --- a/packages/engine-multi/src/worker/thread/helpers.ts +++ b/packages/engine-multi/src/worker/thread/helpers.ts @@ -74,6 +74,7 @@ export const execute = async ( workflowId, // Map the error out of the thread in a serializable format error: serializeError(err), + stack: err?.stack // TODO job id maybe }); }; @@ -89,6 +90,18 @@ export const execute = async ( // Note that if this occurs after the execute promise resolved, // it'll be ignored (ie the workerEmit call will fail) process.on('uncaughtException', async (err: any) => { + // Log this error to local stdout. This won't be sent out of the worker thread. + console.debug(`Uncaught exception in worker thread (workflow ${workflowId} )`) + console.debug(err) + + // Also try and log to the workflow's logger + try { + console.error(`Uncaught exception in worker thread (workflow ${workflowId} )`) + console.error(err) + } catch(e) { + console.error(`Uncaught exception in worker thread`) + } + // For now, we'll write this off as a crash-level generic execution error // TODO did this come from job or adaptor code? const e = new ExecutionError(err); diff --git a/packages/lightning-mock/CHANGELOG.md b/packages/lightning-mock/CHANGELOG.md index 1ccb71880..e404eaae5 100644 --- a/packages/lightning-mock/CHANGELOG.md +++ b/packages/lightning-mock/CHANGELOG.md @@ -4,6 +4,12 @@ ### Patch Changes +- Updated dependencies [2fde0ad] +- Updated dependencies [2fde0ad] + - @openfn/logger@1.0.1 + - @openfn/engine-multi@1.0.1 + - @openfn/lexicon@1.0.0 + - @openfn/runtime@1.0.1 - Updated dependencies [4f5f1dd] - Updated dependencies [58e0d11] - @openfn/engine-multi@1.1.0 diff --git a/packages/logger/CHANGELOG.md b/packages/logger/CHANGELOG.md index e6ffd9de3..3b7147a98 100644 --- a/packages/logger/CHANGELOG.md +++ b/packages/logger/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/logger +## 1.0.1 + +### Patch Changes + +- 2fde0ad: Don't blow up if an object with a null prototype is sent through + ## 1.0.0 ### Major Changes diff --git a/packages/logger/package.json b/packages/logger/package.json index 1b202bf12..19f8171c0 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/logger", - "version": "1.0.0", + "version": "1.0.1", "description": "Cross-package logging utility", "module": "dist/index.js", "author": "Open Function Group ", diff --git a/packages/logger/src/sanitize.ts b/packages/logger/src/sanitize.ts index 7597a8d95..d537b518d 100644 --- a/packages/logger/src/sanitize.ts +++ b/packages/logger/src/sanitize.ts @@ -49,6 +49,12 @@ const sanitize = (item: any, options: SanitizeOptions = {}) => { const maybeStringify = (o: any) => options.stringify === false ? o : stringify(o, undefined, 2); + // Ignore primitive values + if (/^(number|string|boolean)$/.exec(typeof item)) { + return item; + } + + // Serialize errors if (isError(item)) { if (options.serializeErrors) { return { @@ -62,10 +68,9 @@ const sanitize = (item: any, options: SanitizeOptions = {}) => { if (options.policy?.match(/^(remove|obfuscate|summarize)$/)) { return scrubbers[options.policy](item); - } else if ( - Array.isArray(item) || - (isNaN(item) && item && typeof item !== 'string') - ) { + } else if (Array.isArray(item)) { + return maybeStringify(item); + } else if (item) { const obj = item as Record; if (obj && obj.configuration) { // This looks sensitive, so let's sanitize it @@ -81,6 +86,7 @@ const sanitize = (item: any, options: SanitizeOptions = {}) => { } return maybeStringify(obj); } + return item; }; diff --git a/packages/logger/test/logger.test.ts b/packages/logger/test/logger.test.ts index 3acaf7522..2ccf2f415 100644 --- a/packages/logger/test/logger.test.ts +++ b/packages/logger/test/logger.test.ts @@ -107,6 +107,16 @@ test('do log null after a string', (t) => { t.is(logger._history.length, 1); }); +test("log objects with null prototype", (t) => { + const logger = createLogger(undefined, { level: 'debug' }); + + const obj = Object.create(null) + logger.log(obj); + + t.is(logger._history.length, 1); +}); + + test('sanitize: remove object', (t) => { const logger = createLogger(undefined, { level: 'debug', diff --git a/packages/logger/test/mock.test.ts b/packages/logger/test/mock.test.ts index 4444ffbc6..454f51bfd 100644 --- a/packages/logger/test/mock.test.ts +++ b/packages/logger/test/mock.test.ts @@ -99,6 +99,7 @@ test('_parse raw message', (t) => { logger.success('x', 1, true); const { messageRaw } = logger._parse(logger._last); + t.is(messageRaw[0], 'x'); t.is(messageRaw[1], 1); t.true(messageRaw[2]); diff --git a/packages/logger/test/sanitize.test.ts b/packages/logger/test/sanitize.test.ts index 834571657..5ed0f78b0 100644 --- a/packages/logger/test/sanitize.test.ts +++ b/packages/logger/test/sanitize.test.ts @@ -1,6 +1,7 @@ import test from 'ava'; import sanitize, { SECRET } from '../src/sanitize'; + test('simply return a string', (t) => { const result = sanitize('x'); t.is(result, 'x'); @@ -16,6 +17,17 @@ test('simply return a number', (t) => { t.true(result === 0); }); + +test('simply return true', (t) => { + const result = sanitize(true); + t.true(result); +}); + +test('simply return false', (t) => { + const result = sanitize(false); + t.false(result); +}); + test('simply return undefined', (t) => { const result = sanitize(undefined); t.deepEqual(result, undefined); @@ -105,6 +117,13 @@ test('preserve top level stuff after sanitizing', (t) => { t.deepEqual(json, expectedState); }); +test("don't blow up on null prototypes", (t) => { + const obj = Object.create(null) + const result = sanitize(obj); + + t.deepEqual(result, '{}'); +}); + test('ignore a string with obfuscation', (t) => { const result = sanitize('x', { policy: 'obfuscate' }); t.is(result, 'x'); diff --git a/packages/runtime/CHANGELOG.md b/packages/runtime/CHANGELOG.md index 756d2abc3..2af1d4e1d 100644 --- a/packages/runtime/CHANGELOG.md +++ b/packages/runtime/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/runtime +## 1.1.1 + +### Patch Changes + +- Updated dependencies [2fde0ad] + - @openfn/logger@1.0.1 ## 1.1.0 ### Minor Changes diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 79aed3a63..506668af1 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/runtime", - "version": "1.1.0", + "version": "1.1.1", "description": "Job processing runtime.", "type": "module", "exports": { diff --git a/packages/ws-worker/CHANGELOG.md b/packages/ws-worker/CHANGELOG.md index e959ac5d9..416386159 100644 --- a/packages/ws-worker/CHANGELOG.md +++ b/packages/ws-worker/CHANGELOG.md @@ -1,16 +1,15 @@ # ws-worker -## 1.1.0 - -Allow runs to use multiple versions of the same adaptor +## 1.1.1 ### Patch Changes -- 58e0d11: Move version log to workflow start -- Updated dependencies [4f5f1dd] -- Updated dependencies [58e0d11] - - @openfn/engine-multi@1.1.0 - - @openfn/runtime@1.1.0 +- Updated dependencies [2fde0ad] +- Updated dependencies [2fde0ad] + - @openfn/logger@1.0.1 + - @openfn/engine-multi@1.0.1 + - @openfn/lexicon@1.0.0 + - @openfn/runtime@1.0.1 ## 1.0.0 diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index af5a2f972..2f178435f 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/ws-worker", - "version": "1.1.0", + "version": "1.1.1", "description": "A Websocket Worker to connect Lightning to a Runtime Engine", "main": "dist/index.js", "type": "module", diff --git a/packages/ws-worker/test/worker.test.ts b/packages/ws-worker/test/worker.test.ts new file mode 100644 index 000000000..19f4fc5c6 --- /dev/null +++ b/packages/ws-worker/test/worker.test.ts @@ -0,0 +1,100 @@ +// general tests against the worker + +import test from 'ava'; +import createRTE from '@openfn/engine-multi'; +import { createMockLogger } from '@openfn/logger'; + +import type { ExitReason } from '@openfn/lexicon/lightning'; + +import { createPlan } from './util'; +import { execute as doExecute } from '../src/api/execute'; +import { mockChannel } from '../src/mock/sockets'; +import { + STEP_START, + STEP_COMPLETE, + RUN_LOG, + RUN_START, + RUN_COMPLETE, + GET_CREDENTIAL, +} from '../src/events'; +import { ExecutionPlan } from '@openfn/lexicon'; + +let engine: any; +let logger: any; + +test.before(async () => { + logger = createMockLogger(); + // logger = createLogger(null, { level: 'debug' }); + + // Note: this is the REAL engine, not a mock + engine = await createRTE({ + maxWorkers: 1, + logger, + }); +}); + +test.after(async () => engine.destroy()); + +// Run code on the worker with a fake channel, no lightning +const execute = async (plan: ExecutionPlan, input = {}, options = {}) => + new Promise<{ reason: ExitReason; state: any }>((done) => { + // TODO allow handlers to be passed + const channel = mockChannel({ + [RUN_START]: async () => true, + [STEP_START]: async () => true, + [RUN_LOG]: async (_evt) => { + //console.log(evt.source, evt.message) + return true + }, + [STEP_COMPLETE]: async () => true, + [RUN_COMPLETE]: async () => true, + [GET_CREDENTIAL]: async () => { + throw new Error('err'); + }, + }); + + const onFinish = (result: any) => { + done(result); + }; + + doExecute(channel, engine, logger, plan, input, options, onFinish); + }); + + +// Repro for https://github.com/OpenFn/kit/issues/616 +// This will not run in CI unless the env is set +if (process.env.OPENFN_TEST_SF_TOKEN && process.env.OPENFN_TEST_SF_PASSWORD) { + // hard skipping the test because the insert actually fails (permissions) + test.skip('salesforce issue', async (t) => { + const plan = createPlan({ + id: 'x', + expression: `bulk( + 'Contact', + 'insert', + { + failOnError: true, + allowNoOp: true, + }, + state => ([{ + "name": "testy mctestface", + "email": "test@test.com" + }]) + )`, + adaptor: '@openfn/language-salesforce@4.5.0', + configuration: { + username: 'demo@openfn.org', + securityToken: process.env.OPENFN_TEST_SF_TOKEN, + password: process.env.OPENFN_TEST_SF_PASSWORD, + loginUrl: 'https://login.salesforce.com', + } + }); + + const input = { data: { result: 42 } }; + + const result= await execute(plan, input); + t.log(result) + + // Actually this fails right but it's a permissions thing on the sandbox + t.is(result.reason.reason, 'success'); + }) +} \ No newline at end of file