diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index efc4b75c6..45d64e799 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -6,6 +6,7 @@ import deployCommand from './deploy/command'; import docgenCommand from './docgen/command'; import docsCommand from './docs/command'; import executeCommand from './execute/command'; +import generateCommand from './generate/command'; import metadataCommand from './metadata/command'; import pullCommand from './pull/command'; import { Opts } from './options'; @@ -26,6 +27,7 @@ export const cmd = y .command(metadataCommand as any) .command(docgenCommand as any) .command(pullCommand as any) + .command(generateCommand as any) .command({ command: 'version', describe: diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index ecb7ed6f6..eeda9f863 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -5,6 +5,7 @@ import test from './test/handler'; import deploy from './deploy/handler'; import docgen from './docgen/handler'; import docs from './docs/handler'; +import generateAdaptor from './generate/adaptor'; import metadata from './metadata/handler'; import pull from './pull/handler'; import { clean, install, pwd, list } from './repo/handler'; @@ -21,6 +22,7 @@ export type CommandList = | 'docgen' | 'docs' | 'execute' + | 'generate-adaptor' | 'metadata' | 'pull' | 'repo-clean' @@ -37,6 +39,7 @@ const handlers = { deploy, docgen, docs, + ['generate-adaptor']: generateAdaptor, metadata, pull, ['repo-clean']: clean, @@ -81,7 +84,7 @@ const parse = async (options: Opts, log?: Logger) => { // TODO it would be nice to do this in the repoDir option, but // the logger isn't available yet if ( - !/^(pull|deploy|test|version)$/.test(options.command!) && + !/^(pull|deploy|test|version|generate-adaptor)$/.test(options.command!) && !options.repoDir ) { logger.warn( diff --git a/packages/cli/src/generate/adaptor.ts b/packages/cli/src/generate/adaptor.ts new file mode 100644 index 000000000..690627447 --- /dev/null +++ b/packages/cli/src/generate/adaptor.ts @@ -0,0 +1,141 @@ +/** + * Handler to generate adaptor code + */ +import path from 'node:path'; +import fs from 'node:fs/promises'; + +import { Opts } from '../options'; +import { Logger } from '../util'; +import loadGenSpec from './adaptor/load-gen-spec'; + +// TODO: really I just want one domain here +const endpoints = { + signature: 'http://localhost:8001/generate_signature', + code: 'http://localhost:8002/generate_code/', +}; + +export type AdaptorGenOptions = Pick< + Opts, + | 'command' + | 'path' // path to spec - we proably want to override the description + | 'log' // same log rules + | 'logJson' + | 'monorepoPath' // maybe use the monorepo (or use env var) + | 'outputPath' // where to output to. Defaults to monorepo or as sibling of the spec +> & { + adaptor?: string; + spec?: string; + + // TODO spec overrides +}; + +// spec.spec is silly, so what is this object? +export type Spec = { + adaptor?: string; // adaptor name. TOOD rename to name? + + spec: any; // OpenAPI spec. TODO rename to api? + + instruction: string; // for now... but we'll use endpoints later + + endpoints?: string[]; // TODO not supported yet + + model?: string; // TODO not supported yet +}; + +const generateAdaptor = async (opts: AdaptorGenOptions, logger: Logger) => { + // Load the input spec from the cli options + const spec = await loadGenSpec(opts, logger); + + // TODO Validate that the spec looks correct + + // if we're using the monorepo, and no adaptor with this name exists + // prompt to generate it + // humm is that worth it? it'll create a git diff anyway + + const sig = await generateSignature(spec, logger); + const code = await generateCode(spec, sig, logger); + + await simpleOutput(opts, logger, sig, code); + + return { sig, code }; +}; + +export default generateAdaptor; + +// throw if the spec is missing anything +const validateSpec = () => {}; + +// simple output means we write adaptor.js and adaptor.d.ts to disk +// next to the input path +// This is what we run in non-monorepo mode +const simpleOutput = async ( + opts: AdaptorGenOptions, + logger: Logger, + sig: string, + code: string +) => { + const outputPath = path.resolve(path.dirname(opts.path ?? '.')); + + const sigPath = `${outputPath}/adaptor.d.ts`; + logger.debug(`Writing sig to ${sigPath}`); + await fs.writeFile(sigPath, sig); + + const codePath = `${outputPath}/adaptor.js`; + logger.debug(`Writing code to ${sigPath}`); + await fs.writeFile(codePath, code); + + logger.success(`Output adaptor.js and adaptor.d.ts to ${outputPath}`); +}; + +const monorepoOutput = async ( + opts: AdaptorGenOptions, + logger: Logger, + sig: string, + code: string +) => { + // Check if this adaptor exists in the monorepo + // If not, call a monorepo helper to generate a stub + // Now write adaptor.d.ts and adaptor.js into the monorepo structure + // (Right now, we don't care if this overwrites existing code) +}; + +const convertSpec = (spec: Spec) => + JSON.stringify({ + open_api_spec: spec.spec, + instruction: spec.instruction, + + // For now we force this model + model: 'gpt3_turbo', + }); + +const generateSignature = async (spec: Spec, logger: Logger) => { + // generate signature + const result = await fetch(endpoints.signature, { + method: 'POST', + body: convertSpec(spec), + headers: { + ['Content-Type']: 'application/json', + }, + }); + const json = await result.json(); + logger.success('Generated signature:\n', json.signature); + + return json.signature; +}; + +const generateCode = async (spec: Spec, signature: string, logger: Logger) => { + const result = await fetch(endpoints.code, { + method: 'POST', + body: JSON.stringify({ + // TODO why doesn't code gen use the spec?? + signature, + model: 'gpt3_turbo', + }), + headers: { + ['Content-Type']: 'application/json', + }, + }); + const json = await result.json(); + logger.success('Generated code:\n', json.implementation); + return json.implementation; +}; diff --git a/packages/cli/src/generate/adaptor/generate-project.ts b/packages/cli/src/generate/adaptor/generate-project.ts new file mode 100644 index 000000000..4230b7d1e --- /dev/null +++ b/packages/cli/src/generate/adaptor/generate-project.ts @@ -0,0 +1,79 @@ +/** + * This fella will generate an adaptor project stub into an empty directory + * + */ + +type Files { + [path: string]: string +} + +const generatePkg = (name: string) => JSON.stringify({ + // TODO how much monorepo boilerplate do we really need? + // Should not the mono repo provide this, and we call it to generate the stub? + // what if you're not generating in the monorepo? + + // OK so this will be a very minimal package.json, just for now + "name": `@openfn/language-${name}`, + "version": "1.0.0", + "description": "", + "type": "module", + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.cjs" + }, + "./package.json": "./package.json" + }, + "author": "Open Function Group", + "license": "LGPLv3", + "files": [ + "dist/", + "types/", + "ast.json", + "configuration-schema.json" + ], + "dependencies": { + "@openfn/language-common": "^1.12.0" + }, + "repository": { + "type": "git", + "url": "https://github.com/openfn/adaptors.git" + }, +}, null, 2); + +const generate = (name:string, path: string, files: Files = {}) => { + + // TODO throw or warn if path/name exists + + // create root door "name" + + // ensure that some key paths are set + if (!files['src/index.js']) { + files['src/index.js'] = `import * as Adaptor from './Adaptor'; + export default Adaptor; + + export * from './Adaptor';` + } + + if (!files['src/Adaptor.js']) { + files['src/Adaptor.js'] = `import { fn } from "@openfn/language-common"; + + export { fn };` + } + + if (!files['package.json']) { + files['package.json'] = generatePkg(name) + } + + // TODO there's loads we need todo here tbh + // license, readme, maybe eslintrc + // I really think this functionality should mostly move to adaptors, + // where we'll generate a stub + + // If you ask for codegen without the monorepo, we'll jsut emit stub files + + // generate each path + +} + +export default generate \ No newline at end of file diff --git a/packages/cli/src/generate/adaptor/load-gen-spec.ts b/packages/cli/src/generate/adaptor/load-gen-spec.ts new file mode 100644 index 000000000..1b62235dc --- /dev/null +++ b/packages/cli/src/generate/adaptor/load-gen-spec.ts @@ -0,0 +1,64 @@ +import path from 'node:path'; +import fs from 'node:fs/promises'; + +import type { AdaptorGenOptions, Spec } from '../adaptor'; +import { Logger, abort } from '../../util'; + +// single standalone function to parse all the options and return a spec object +// I can unit test this comprehensively you see +const loadGenSpec = async (opts: AdaptorGenOptions, logger: Logger) => { + let spec: Partial = {}; + + if (opts.path) { + const inputPath = path.resolve(opts.path); + logger.debug(`Loading input spec from ${inputPath}`); + + try { + const text = await fs.readFile(inputPath, 'utf8'); + spec = JSON.parse(text); + } catch (e) { + return abort( + logger, + 'spec load error', + undefined, + `Failed to load a codegen specc from ${inputPath}` + ); + } + } + + if (opts.spec) { + spec.spec = opts.spec; + } + if (opts.adaptor) { + spec.adaptor = opts.adaptor; + } + + if (typeof spec.spec === 'string') { + // TODO if opts.path isn't set I think this will blow up + const specPath = path.resolve(path.dirname(opts.path ?? '.'), spec.spec); + logger.debug(`Loading OpenAPI spec from ${specPath}`); + try { + const text = await fs.readFile(specPath, 'utf8'); + spec.spec = JSON.parse(text); + } catch (e) { + return abort( + logger, + 'OpenAPI error', + undefined, + `Failed to load openAPI spec from ${specPath}` + ); + } + } + + // if no name provided, see if we can pull one from the spec + if (!spec.adaptor) { + // TOOD use a lib for this? + spec.adaptor = spec.spec.info?.title?.toLowerCase().replace(/\W/g, '-'); + } + + logger.debug(`Final spec: ${JSON.stringify(spec, null, 2)}`); + + return spec as Required; +}; + +export default loadGenSpec; diff --git a/packages/cli/src/generate/command.ts b/packages/cli/src/generate/command.ts new file mode 100644 index 000000000..a40b4c7bf --- /dev/null +++ b/packages/cli/src/generate/command.ts @@ -0,0 +1,48 @@ +import yargs from 'yargs'; +import * as o from '../options'; +import type { Opts } from '../options'; +import { build, ensure } from '../util/command-builders'; + +export type TestOptions = Pick; + +const options = [o.log, o.logJson, o.useAdaptorsMonorepo, o.outputPath]; + +options.push({ + name: 'adaptor', + yargs: { + description: 'The name of the adaptor to generate', + }, +} as o.CLIOption); + +// Adaptor generation subcommand +const adaptor = { + command: 'adaptor [path]', + desc: 'Generate adaptor code', + handler: ensure('generate-adaptor', options), + builder: (yargs) => + build(options, yargs) + .example( + 'generate adaptor ./spec.json', + 'Generate adaptor code based on spec.json' + ) + .positional('path', { + describe: 'The path spec.json', + }), +} as yargs.CommandModule<{}>; + +export default { + command: 'generate [subcommand]', + desc: 'Generate code (only adaptors supported now)', + handler: () => { + // TODO: better error handling + console.error('ERROR: invalid command'); + console.error('Try:\n\n openfn generate adaptor\n'); + }, + builder: (yargs: yargs.Argv) => + yargs + .command(adaptor) + .example( + 'generate adaptor ./spec.json', + 'Generate adaptor code based on spec.json' + ), +} as yargs.CommandModule<{}>; diff --git a/packages/cli/src/util/index.ts b/packages/cli/src/util/index.ts index 640967359..af8b942c4 100644 --- a/packages/cli/src/util/index.ts +++ b/packages/cli/src/util/index.ts @@ -1,6 +1,7 @@ import expandAdaptors from './expand-adaptors'; import ensureLogOpts from './ensure-log-opts'; +import abort from './abort'; export * from './logger'; -export { expandAdaptors, ensureLogOpts }; +export { expandAdaptors, ensureLogOpts, abort }; diff --git a/packages/cli/test/generate/adaptor/load-gen-spec.test.ts b/packages/cli/test/generate/adaptor/load-gen-spec.test.ts new file mode 100644 index 000000000..854a028d9 --- /dev/null +++ b/packages/cli/test/generate/adaptor/load-gen-spec.test.ts @@ -0,0 +1,185 @@ +import test from 'ava'; +import { mockFs, resetMockFs } from '../../util'; +import loadGenSpec from '../../../src/generate/adaptor/load-gen-spec'; +import { createMockLogger } from '@openfn/logger'; + +const logger = createMockLogger(); + +test.after(resetMockFs); + +test.serial('should load a spec from a json file', async (t) => { + const spec = { + adaptor: 'a', + spec: {}, + instruction: 'abc', + }; + + // Note that this takes like 4 seconds to initialise?! + mockFs({ + 'spec.json': JSON.stringify(spec), + }); + + const options = { + path: 'spec.json', + }; + + const result = await loadGenSpec(options, logger); + + t.deepEqual(result, spec); +}); + +test.serial( + 'should load a spec from a json file with overriden name', + async (t) => { + const spec = { + adaptor: 'a', + spec: {}, + instruction: 'abc', + }; + + mockFs({ + 'spec.json': JSON.stringify(spec), + }); + + const options = { + path: 'spec.json', + adaptor: 'b', + }; + + const result = await loadGenSpec(options, logger); + + t.deepEqual(result, { + ...spec, + adaptor: 'b', + }); + } +); + +test.serial( + 'should load a spec from a json file with overridden api spec', + async (t) => { + const api = { x: 1 }; + + const spec = { + adaptor: 'a', + spec: {}, + instruction: 'abc', + }; + + mockFs({ + 'spec.json': JSON.stringify(spec), + }); + + const options = { + path: 'spec.json', + spec: api, + }; + + // @ts-ignore options typing + const result = await loadGenSpec(options, logger); + + t.deepEqual(result, { + ...spec, + spec: api, + }); + } +); + +test.serial('should load an api spec from a path', async (t) => { + const api = { x: 1 }; + + const spec = { + adaptor: 'a', + spec: 'api.json', + instruction: 'abc', + }; + + mockFs({ + 'spec.json': JSON.stringify(spec), + 'api.json': JSON.stringify(api), + }); + + const options = { + path: 'spec.json', + }; + + const result = await loadGenSpec(options, logger); + + t.deepEqual(result, { + ...spec, + spec: api, + }); +}); + +test.serial('should load an api spec from a path override', async (t) => { + const api = { x: 1 }; + + const spec = { + adaptor: 'a', + spec: {}, + instruction: 'abc', + }; + + mockFs({ + 'spec.json': JSON.stringify(spec), + 'api.json': JSON.stringify(api), + }); + + const options = { + path: 'spec.json', + spec: 'api.json', + }; + + const result = await loadGenSpec(options, logger); + + t.deepEqual(result, { + ...spec, + spec: api, + }); +}); + +test.serial('should generate a default name from the API spec', async (t) => { + const spec = { + spec: { + info: { + title: 'my Friendly API', + }, + }, + instruction: 'abc', + }; + + mockFs({ + 'spec.json': JSON.stringify(spec), + }); + + const options = { + path: 'spec.json', + }; + + const result = await loadGenSpec(options, logger); + + t.deepEqual(result, { + ...spec, + adaptor: 'my-friendly-api', + }); +}); + +test.serial('should load an API spec CLI path and name', async (t) => { + const api = { x: 1 }; + + mockFs({ + 'api.json': JSON.stringify(api), + }); + + const options = { + spec: 'api.json', + adaptor: 'a', + }; + + const result = await loadGenSpec(options, logger); + + t.deepEqual(result, { + adaptor: 'a', + spec: api, + }); +});