CLI: Adaptor codegen

packages/cli/src/cli.ts

@@ -6,6 +6,7 @@ import deployCommand from './deploy/command';
 import docgenCommand from './docgen/command';
 import docsCommand from './docs/command';
 import executeCommand from './execute/command';
+import generateCommand from './generate/command';
 import metadataCommand from './metadata/command';
 import pullCommand from './pull/command';
 import { Opts } from './options';
@@ -26,6 +27,7 @@ export const cmd = y
   .command(metadataCommand as any)
   .command(docgenCommand as any)
   .command(pullCommand as any)
+  .command(generateCommand as any)
   .command({
     command: 'version',
     describe:

packages/cli/src/commands.ts

@@ -5,6 +5,7 @@ import test from './test/handler';
 import deploy from './deploy/handler';
 import docgen from './docgen/handler';
 import docs from './docs/handler';
+import generateAdaptor from './generate/adaptor';
 import metadata from './metadata/handler';
 import pull from './pull/handler';
 import { clean, install, pwd, list } from './repo/handler';
@@ -21,6 +22,7 @@ export type CommandList =
   | 'docgen'
   | 'docs'
   | 'execute'
+  | 'generate-adaptor'
   | 'metadata'
   | 'pull'
   | 'repo-clean'
@@ -37,6 +39,7 @@ const handlers = {
   deploy,
   docgen,
   docs,
+  ['generate-adaptor']: generateAdaptor,
   metadata,
   pull,
   ['repo-clean']: clean,
@@ -81,7 +84,7 @@ const parse = async (options: Opts, log?: Logger) => {
   // TODO it would be nice to do this in the repoDir option, but
   // the logger isn't available yet
   if (
-    !/^(pull|deploy|test|version)$/.test(options.command!) &&
+    !/^(pull|deploy|test|version|generate-adaptor)$/.test(options.command!) &&
     !options.repoDir
   ) {
     logger.warn(

packages/cli/src/generate/adaptor.ts

@@ -0,0 +1,141 @@
+/**
+ * Handler to generate adaptor code
+ */
+import path from 'node:path';
+import fs from 'node:fs/promises';
+
+import { Opts } from '../options';
+import { Logger } from '../util';
+import loadGenSpec from './adaptor/load-gen-spec';
+
+// TODO: really I just want one domain here
+const endpoints = {
+  signature: 'http://localhost:8001/generate_signature',
+  code: 'http://localhost:8002/generate_code/',
+};
+
+export type AdaptorGenOptions = Pick<
+  Opts,
+  | 'command'
+  | 'path' // path to spec - we proably want to override the description
+  | 'log' // same log rules
+  | 'logJson'
+  | 'monorepoPath' // maybe use the monorepo (or use env var)
+  | 'outputPath' // where to output to. Defaults to monorepo or as sibling of the spec
+> & {
+  adaptor?: string;
+  spec?: string;
+
+  // TODO spec overrides
+};
+
+// spec.spec is silly, so what is this object?
+export type Spec = {
+  adaptor?: string; // adaptor name. TOOD rename to name?
+
+  spec: any; // OpenAPI spec. TODO rename to api?
+
+  instruction: string; // for now... but we'll use endpoints later
+
+  endpoints?: string[]; // TODO not supported yet
+
+  model?: string; // TODO not supported yet
+};
+
+const generateAdaptor = async (opts: AdaptorGenOptions, logger: Logger) => {
+  // Load the input spec from the cli options
+  const spec = await loadGenSpec(opts, logger);
+
+  // TODO Validate that the spec looks correct
+
+  // if we're using the monorepo, and no adaptor with this name exists
+  // prompt to generate it
+  // humm is that worth it? it'll create a git diff anyway
+
+  const sig = await generateSignature(spec, logger);
+  const code = await generateCode(spec, sig, logger);
+
+  await simpleOutput(opts, logger, sig, code);
+
+  return { sig, code };
+};
+
+export default generateAdaptor;
+
+// throw if the spec is missing anything
+const validateSpec = () => {};
+
+// simple output means we write adaptor.js and adaptor.d.ts to disk
+// next to the input path
+// This is what we run in non-monorepo mode
+const simpleOutput = async (
+  opts: AdaptorGenOptions,
+  logger: Logger,
+  sig: string,
+  code: string
+) => {
+  const outputPath = path.resolve(path.dirname(opts.path ?? '.'));
+
+  const sigPath = `${outputPath}/adaptor.d.ts`;
+  logger.debug(`Writing sig to ${sigPath}`);
+  await fs.writeFile(sigPath, sig);
+
+  const codePath = `${outputPath}/adaptor.js`;
+  logger.debug(`Writing code to ${sigPath}`);
+  await fs.writeFile(codePath, code);
+
+  logger.success(`Output adaptor.js and adaptor.d.ts to ${outputPath}`);
+};
+
+const monorepoOutput = async (
+  opts: AdaptorGenOptions,
+  logger: Logger,
+  sig: string,
+  code: string
+) => {
+  // Check if this adaptor exists in the monorepo
+  // If not, call a monorepo helper to generate a stub
+  // Now write adaptor.d.ts and adaptor.js into the monorepo structure
+  // (Right now, we don't care if this overwrites existing code)
+};
+
+const convertSpec = (spec: Spec) =>
+  JSON.stringify({
+    open_api_spec: spec.spec,
+    instruction: spec.instruction,
+
+    // For now we force this model
+    model: 'gpt3_turbo',
+  });
+
+const generateSignature = async (spec: Spec, logger: Logger) => {
+  // generate signature
+  const result = await fetch(endpoints.signature, {
+    method: 'POST',
+    body: convertSpec(spec),
+    headers: {
+      ['Content-Type']: 'application/json',
+    },
+  });
+  const json = await result.json();
+  logger.success('Generated signature:\n', json.signature);
+
+  return json.signature;
+};
+
+const generateCode = async (spec: Spec, signature: string, logger: Logger) => {
+  const result = await fetch(endpoints.code, {
+    method: 'POST',
+    body: JSON.stringify({
+      // TODO why doesn't code gen use the spec??
+      signature,
+      model: 'gpt3_turbo',
+    }),
+    headers: {
+      ['Content-Type']: 'application/json',
+    },
+  });
+  const json = await result.json();
+  logger.success('Generated code:\n', json.implementation);
+  return json.implementation;
+};

packages/cli/src/generate/adaptor/generate-project.ts

@@ -0,0 +1,79 @@
+/**
+ * This fella will generate an adaptor project stub into an empty directory
+ * 
+ */
+
+type Files {
+  [path: string]: string
+}
+
+const generatePkg = (name: string) => JSON.stringify({
+  // TODO how much monorepo boilerplate do we really need?
+  // Should not the mono repo provide this, and we call it to generate the stub?
+  // what if you're not generating in the monorepo?
+
+  // OK so this will be a very minimal package.json, just for now
+  "name": `@openfn/language-${name}`,
+  "version": "1.0.0",
+  "description": "",
+  "type": "module",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "author": "Open Function Group",
+  "license": "LGPLv3",
+  "files": [
+    "dist/",
+    "types/",
+    "ast.json",
+    "configuration-schema.json"
+  ],
+  "dependencies": {
+    "@openfn/language-common": "^1.12.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/openfn/adaptors.git"
+  },
+}, null, 2);
+
+const generate = (name:string,  path: string, files: Files = {}) => {
+
+  // TODO throw or warn if path/name exists
+
+  // create root door "name"
+
+  // ensure that some key paths are set
+  if (!files['src/index.js']) {
+    files['src/index.js'] = `import * as Adaptor from './Adaptor';
+    export default Adaptor;
+
+    export * from './Adaptor';`
+  }
+
+  if (!files['src/Adaptor.js']) {
+    files['src/Adaptor.js'] = `import { fn } from "@openfn/language-common";
+
+    export { fn };`
+  }
+
+  if (!files['package.json']) {
+    files['package.json'] = generatePkg(name)
+  }
+
+  // TODO there's loads we need todo here tbh
+  // license, readme, maybe eslintrc
+  // I really think this functionality should mostly move to adaptors,
+  // where we'll generate a stub
+
+  // If you ask for codegen without the monorepo, we'll jsut emit stub files
+
+  // generate each path
+
+}
+
+export default generate

packages/cli/src/generate/adaptor/load-gen-spec.ts

@@ -0,0 +1,64 @@
+import path from 'node:path';
+import fs from 'node:fs/promises';
+
+import type { AdaptorGenOptions, Spec } from '../adaptor';
+import { Logger, abort } from '../../util';
+
+// single standalone function to parse all the options and return a spec object
+// I can unit test this comprehensively you see
+const loadGenSpec = async (opts: AdaptorGenOptions, logger: Logger) => {
+  let spec: Partial<Spec> = {};
+
+  if (opts.path) {
+    const inputPath = path.resolve(opts.path);
+    logger.debug(`Loading input spec from ${inputPath}`);
+
+    try {
+      const text = await fs.readFile(inputPath, 'utf8');
+      spec = JSON.parse(text);
+    } catch (e) {
+      return abort(
+        logger,
+        'spec load error',
+        undefined,
+        `Failed to load a codegen specc from ${inputPath}`
+      );
+    }
+  }
+
+  if (opts.spec) {
+    spec.spec = opts.spec;
+  }
+  if (opts.adaptor) {
+    spec.adaptor = opts.adaptor;
+  }
+
+  if (typeof spec.spec === 'string') {
+    // TODO if opts.path isn't set I think this will blow up
+    const specPath = path.resolve(path.dirname(opts.path ?? '.'), spec.spec);
+    logger.debug(`Loading OpenAPI spec from ${specPath}`);
+    try {
+      const text = await fs.readFile(specPath, 'utf8');
+      spec.spec = JSON.parse(text);
+    } catch (e) {
+      return abort(
+        logger,
+        'OpenAPI error',
+        undefined,
+        `Failed to load openAPI spec from ${specPath}`
+      );
+    }
+  }
+
+  // if no name provided, see if we can pull one from the spec
+  if (!spec.adaptor) {
+    // TOOD use a lib for this?
+    spec.adaptor = spec.spec.info?.title?.toLowerCase().replace(/\W/g, '-');
+  }
+
+  logger.debug(`Final spec: ${JSON.stringify(spec, null, 2)}`);
+
+  return spec as Required<Spec>;
+};
+
+export default loadGenSpec;

packages/cli/src/generate/command.ts

@@ -0,0 +1,48 @@
+import yargs from 'yargs';
+import * as o from '../options';
+import type { Opts } from '../options';
+import { build, ensure } from '../util/command-builders';
+
+export type TestOptions = Pick<Opts, 'stateStdin' | 'log' | 'logJson'>;
+
+const options = [o.log, o.logJson, o.useAdaptorsMonorepo, o.outputPath];
+
+options.push({
+  name: 'adaptor',
+  yargs: {
+    description: 'The name of the adaptor to generate',
+  },
+} as o.CLIOption);
+
+// Adaptor generation subcommand
+const adaptor = {
+  command: 'adaptor [path]',
+  desc: 'Generate adaptor code',
+  handler: ensure('generate-adaptor', options),
+  builder: (yargs) =>
+    build(options, yargs)
+      .example(
+        'generate adaptor ./spec.json',
+        'Generate adaptor code based on spec.json'
+      )
+      .positional('path', {
+        describe: 'The path spec.json',
+      }),
+} as yargs.CommandModule<{}>;
+
+export default {
+  command: 'generate [subcommand]',
+  desc: 'Generate code (only adaptors supported now)',
+  handler: () => {
+    // TODO: better error handling
+    console.error('ERROR: invalid command');
+    console.error('Try:\n\n  openfn generate adaptor\n');
+  },
+  builder: (yargs: yargs.Argv) =>
+    yargs
+      .command(adaptor)
+      .example(
+        'generate adaptor ./spec.json',
+        'Generate adaptor code based on spec.json'
+      ),
+} as yargs.CommandModule<{}>;

packages/cli/src/util/index.ts

@@ -1,6 +1,7 @@
 import expandAdaptors from './expand-adaptors';
 import ensureLogOpts from './ensure-log-opts';
+import abort from './abort';
 
 export * from './logger';
 
-export { expandAdaptors, ensureLogOpts };
+export { expandAdaptors, ensureLogOpts, abort };