Vscode extension (dataform-co#803)

* Simple extension starter

* Simple sidebar command

* fix a couple of issues

* Add message passing between extension and webview

* Language server hello world

* Refactor

* Adds compile error logging to vscode notification

* WIP on dry runs command

* Fix definition searching and compile on save now

* Add manual compile

* Fix boot up of extension

* Add a quick readme and variable

* Bunch of tightening up

* Fix linter

Co-authored-by: Lewis Hemens <[email protected]>

package.json

@@ -47,6 +47,7 @@
     "@types/stack-trace": "^0.0.29",
     "@types/swagger-schema-official": "^2.0.20",
     "@types/uuid": "^7.0.2",
+    "@types/vscode": "^1.45.1",
     "@types/webpack-dev-server": "^3.1.7",
     "@types/yargs": "^13.0.2",
     "@zeit/next-css": "1.0.1",
@@ -134,6 +135,9 @@
     "untildify": "^4.0.0",
     "uuid": "^7.0.2",
     "vm2": "^3.9.2",
+    "vscode-languageclient": "^6.1.3",
+    "vscode-languageserver": "^6.1.1",
+    "vscode-languageserver-textdocument": "^1.0.1",
     "webpack": "4.20.2",
     "webpack-cli": "^3.3.10",
     "webpack-dev-server": "^3.8.2",

vscode/BUILD

@@ -0,0 +1,34 @@
+load("@npm_bazel_typescript//:index.bzl", "ts_library")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "vscode",
+    srcs = glob(["**/*.ts"]),
+    deps = [
+        "//api",
+        "//protos:ts",
+        "@npm//@types/node",
+        "@npm//@types/vscode",
+        "@npm//vscode-languageclient",
+        "@npm//vscode-languageserver",
+        "@npm//vscode-languageserver-textdocument",
+    ],
+)
+
+filegroup(
+    name = "vscode-sources",
+    srcs = [":vscode"],
+    output_group = "es5_sources",
+)
+
+sh_binary(
+    name = "bin",
+    srcs = ["run.sh"],
+    data = [
+        ":language-configuration.json",
+        ":package.json",
+        ":sqlx.grammar.json",
+        ":vscode-sources",
+    ],
+)

vscode/README.md

@@ -0,0 +1,5 @@
+** Under construction **
+
+To run you need vscode and bazel installed. Then run: `bazel run vscode:bin -- /path/to/dataform-repo`
+
+This will open a vscode instance in developer mode. Go to a `.sqlx` file to use the extension.

vscode/extension.ts

@@ -0,0 +1,56 @@
+import * as vscode from "vscode";
+import { workspace } from "vscode";
+import {
+  LanguageClient,
+  LanguageClientOptions,
+  ServerOptions,
+  TransportKind
+} from "vscode-languageclient";
+
+let client: LanguageClient;
+
+export async function activate(context: vscode.ExtensionContext) {
+  const serverModule = context.asAbsolutePath("server.js");
+  const debugOptions = { execArgv: ["--nolazy", "--inspect=6009"] };
+
+  const serverOptions: ServerOptions = {
+    run: { module: serverModule, transport: TransportKind.ipc },
+    debug: {
+      module: serverModule,
+      transport: TransportKind.ipc,
+      options: debugOptions
+    }
+  };
+
+  const clientOptions: LanguageClientOptions = {
+    // register server for sqlx files
+    documentSelector: [{ scheme: "file", language: "sqlx" }],
+    synchronize: {
+      fileEvents: workspace.createFileSystemWatcher("**/.clientrc")
+    }
+  };
+
+  client = new LanguageClient(
+    "dataformLanguageServer",
+    "Dataform Language Server",
+    serverOptions,
+    clientOptions
+  );
+
+  const compile = vscode.commands.registerCommand("dataform.compile", () => {
+    const _ = client.sendRequest("compile");
+  });
+
+  context.subscriptions.push(compile);
+
+  client.start();
+
+  // wait for client to be ready before setting up notification handlers
+  await client.onReady();
+  client.onNotification("error", errorMessage => {
+    vscode.window.showErrorMessage(errorMessage);
+  });
+  client.onNotification("success", message => {
+    vscode.window.showInformationMessage(message);
+  });
+}

vscode/language-configuration.json

@@ -0,0 +1,64 @@
+{
+  "comments": {
+    "lineComment": "--",
+    "blockComment": ["/*", "*/"]
+  },
+  "brackets": [
+    ["{", "}"],
+    ["[", "]"],
+    ["(", ")"]
+  ],
+  "autoClosingPairs": [
+    {
+      "open": "{",
+      "close": "}"
+    },
+    {
+      "open": "[",
+      "close": "]"
+    },
+    {
+      "open": "(",
+      "close": ")"
+    },
+    {
+      "open": "\"",
+      "close": "\""
+    },
+    {
+      "open": "'",
+      "close": "'"
+    },
+    {
+      "open": "`",
+      "close": "`"
+    }
+  ],
+  "surroundingPairs": [
+    {
+      "open": "{",
+      "close": "}"
+    },
+    {
+      "open": "[",
+      "close": "]"
+    },
+    {
+      "open": "(",
+      "close": ")"
+    },
+    {
+      "open": "\"",
+      "close": "\""
+    },
+    {
+      "open": "'",
+      "close": "'"
+    },
+    {
+      "open": "`",
+      "close": "`",
+      "notIn": ["string", "comment"]
+    }
+  ]
+}

vscode/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "dataform",
+  "version": "0.0.1",
+  "engines": {
+    "vscode": "^1.45.0"
+  },
+  "activationEvents": [
+    "workspaceContains:**/*.sqlx"
+  ],
+  "main": "extension.js",
+  "contributes": {
+    "commands": [
+      {
+        "command": "dataform.compile",
+        "title": "Recompile dataform",
+        "category": "cli"
+      }
+    ],
+    "languages": [
+      {
+        "id": "sqlx",
+        "extensions": [
+          ".sqlx"
+        ],
+        "configuration": "language-configuration.json"
+      }
+    ],
+    "grammars": [
+      {
+        "language": "sqlx",
+        "scopeName": "source.sqlx",
+        "path": "sqlx.grammar.json"
+      }
+    ]
+  }
+}

vscode/run.sh

@@ -0,0 +1 @@
+code --extensionDevelopmentPath=$PWD/vscode $@

vscode/server.ts

@@ -0,0 +1,108 @@
+import { ChildProcess, exec } from "child_process";
+import { dataform } from "df/protos/ts";
+import {
+  createConnection,
+  HandlerResult,
+  Location,
+  ProposedFeatures,
+  TextDocuments,
+  TextDocumentSyncKind
+} from "vscode-languageserver";
+import { TextDocument } from "vscode-languageserver-textdocument";
+
+const connection = createConnection(ProposedFeatures.all);
+const documents: TextDocuments<TextDocument> = new TextDocuments(TextDocument);
+let CACHED_COMPILE_GRAPH: dataform.ICompiledGraph = null;
+let WORKSPACE_ROOT_FOLDER: string = null;
+
+connection.onInitialize(() => {
+  return {
+    capabilities: {
+      textDocumentSync: TextDocumentSyncKind.Incremental,
+      // Tell the client that the server supports code completion and definitions
+      completionProvider: {
+        resolveProvider: true
+      },
+      definitionProvider: true
+    }
+  };
+});
+
+connection.onInitialized(async () => {
+  const _ = compileAndValidate();
+  const workSpaceFolders = await connection.workspace.getWorkspaceFolders();
+  // the first item is the root folder
+  WORKSPACE_ROOT_FOLDER = workSpaceFolders[0].uri;
+});
+
+connection.onRequest("compile", async () => {
+  const _ = compileAndValidate();
+});
+
+documents.onDidSave(change => {
+  const _ = compileAndValidate();
+});
+
+async function compileAndValidate() {
+  const compileResult = await getProcessResult(exec("dataform compile --json"));
+  const parsedResult: dataform.ICompiledGraph = JSON.parse(compileResult.stdout);
+  if (parsedResult?.graphErrors?.compilationErrors) {
+    parsedResult.graphErrors.compilationErrors.forEach(compilationError => {
+      connection.sendNotification("error", compilationError.message);
+    });
+  } else {
+    connection.sendNotification("success", "Project compiled successfully");
+  }
+  CACHED_COMPILE_GRAPH = parsedResult;
+}
+
+async function getProcessResult(childProcess: ChildProcess) {
+  let stdout = "";
+  childProcess.stderr.pipe(process.stderr);
+  childProcess.stdout.pipe(process.stdout);
+  childProcess.stdout.on("data", chunk => (stdout += String(chunk)));
+  const exitCode: number = await new Promise(resolve => {
+    childProcess.on("close", resolve);
+  });
+  return { exitCode, stdout };
+}
+
+function gatherAllActions(
+  graph = CACHED_COMPILE_GRAPH
+): Array<dataform.Table | dataform.Declaration | dataform.Operation | dataform.Assertion> {
+  return [].concat(graph.tables, graph.operations, graph.assertions, graph.declarations);
+}
+
+function retrieveLinkedFileName(ref: string) {
+  const allActions = gatherAllActions();
+  const foundCompileAction = allActions.find(action => action.name.split(".").slice(-1)[0] === ref);
+  return foundCompileAction.fileName;
+}
+
+connection.onDefinition(
+  (params): HandlerResult<Location, void> => {
+    const currentFile = documents.get(params.textDocument.uri);
+    const lineWithRef = currentFile.getText({
+      start: { line: params.position.line, character: 0 },
+      end: { line: params.position.line + 1, character: 0 }
+    });
+
+    const refRegex = new RegExp(/(?<=ref\(\"|'\s*).*?(?=\s*\"|'\))/g); // tslint:disable-line
+    const refContents = lineWithRef.match(refRegex);
+    if (!refContents || refContents.length === 0) {
+      return null;
+    }
+    const linkedFileName = retrieveLinkedFileName(refContents.join(""));
+    const fileString = `${WORKSPACE_ROOT_FOLDER}/${linkedFileName}`;
+    return {
+      uri: fileString,
+      range: {
+        start: { line: 0, character: 0 },
+        end: { line: 1, character: 0 }
+      }
+    } as Location;
+  }
+);
+
+documents.listen(connection);
+connection.listen();

vscode/sqlx.grammar.json

@@ -0,0 +1,64 @@
+{
+  "$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json",
+  "name": "sqlx",
+  "patterns": [
+    {
+      "include": "source.sql"
+    },
+    {
+      "name": "meta.section.configblock",
+      "contentName": "markup.other.section.example",
+      "begin": "(config)\\s+({)",
+      "beginCaptures": {
+        "1": { "name": "keyword.function" },
+        "2": { "name": "punctuation.definition.function.example" }
+      },
+      "end": "}",
+      "endCaptures": {
+        "0": { "name": "punctuation.definition.end.example" }
+      },
+      "patterns": [
+        {
+          "include": "source.js#object-member"
+        }
+      ]
+    },
+    {
+      "name": "meta.section.jsblock",
+      "contentName": "markup.other.section.example",
+      "begin": "(js)\\s+({)",
+      "beginCaptures": {
+        "1": { "name": "keyword.function" },
+        "2": { "name": "punctuation.definition.function.example" }
+      },
+      "end": "}",
+      "endCaptures": {
+        "0": { "name": "punctuation.definition.end.example" }
+      },
+      "patterns": [
+        {
+          "include": "source.js"
+        }
+      ]
+    },
+    {
+      "name": "meta.section.inlinejs",
+      "contentName": "markup.other.section.example",
+      "begin": "\\${",
+      "beginCaptures": {
+        "0": { "name": "punctuation.definition.function.example" }
+      },
+      "end": "}",
+      "endCaptures": {
+        "0": { "name": "punctuation.definition.end.example" }
+      },
+      "patterns": [
+        {
+          "include": "source.js"
+        }
+      ]
+    }
+  ],
+
+  "scopeName": "source.sqlx"
+}