Integrate Python Language Server to Monaco Editor (Closes #1908, #571)

.deployment/README.md

@@ -5,9 +5,11 @@ The script `deploy-deepforge` is used for standard deployment of deepforge using
 
 Additionally, this contains a file with customizations to the standard docker-compose.yml file which allows us to modify the entrypoint and install a version of tensorflow [compatible with the CPU of the deployment machine](https://github.com/deepforge-dev/deepforge/issues/1561).
 
-The deployment is updated by first creating the custom docker compose file using [yaml-merge](https://github.com/alexlafroscia/yaml-merge):
+Moreover, we also host a proxy server that spins up different language servers that we provide for intelligent syntax highlighting in DeepForge's browser based text editor. For more information checkout the language server's docker [file](../docker/Dockerfile.langservers). We use [jq](https://stedolan.github.io/jq/manual/) to update [components.json](../config/components.json) to include available language servers' configuration.
+
+The deployment is updated by first creating the custom docker compose file using [yq](https://github.com/mikefarah/yq):
 ```
-yaml-merge docker-compose.yml .deployment/docker-compose-overrides.yml > custom-docker-compose.yml
+yq m -a docker/docker-compose.yml "$DEEPFORGE_DEPLOYMENT_DIR"/docker-compose-overrides.yml > custom-docker-compose.yml
 ```
 Next, the generated file can be used with docker-compose:
 ```

.deployment/deploy-deepforge

@@ -13,8 +13,9 @@ fi;
 < config/components.json  jq '.Compute.backends=(.Compute.backends | map(select(. != "local")))' \
 > config/components2.json
 < config/components2.json jq '.Storage.backends=(.Storage.backends | map(select(. != "gme")))' \
-> config/components.json
-rm config/components2.json
+> config/components3.json
+jq '.LanguageServers = input' config/components3.json "$DEEPFORGE_DEPLOYMENT_DIR"/languageServers.json > config/components.json
+rm config/components3.json config/components2.json
 
 # Merging the custom override yml file
 yq m -a docker/docker-compose.yml "$DEEPFORGE_DEPLOYMENT_DIR"/docker-compose-overrides.yml > custom-docker-compose.yml
@@ -25,4 +26,12 @@ docker-compose --file custom-docker-compose.yml -p deepforge stop $SERVER_NAME
 docker-compose --file custom-docker-compose.yml -p deepforge rm -f $SERVER_NAME
 docker-compose --file custom-docker-compose.yml -p deepforge up -d $SERVER_NAME
 
+if [[ $SERVER_NAME = "server" ]]; then
+  docker-compose --file custom-docker-compose.yml -p deepforge pull language_servers
+  docker-compose --file custom-docker-compose.yml -p deepforge stop language_servers
+  docker-compose --file custom-docker-compose.yml -p deepforge rm -f language_servers
+  docker-compose --file custom-docker-compose.yml -p deepforge up -d language_servers
+fi;
+
+
 docker image prune -f

.deployment/docker-compose-overrides.yml

@@ -4,6 +4,8 @@ services:
     environment:
       - "DEEPFORGE_HOST=https://dev.deepforge.org"
       - "DEEPFORGE_INTERACTIVE_COMPUTE_HOST=wss://dev-compute.deepforge.org"
+      - "DEEPFORGE_LANGUAGE_SERVER_HOST=https://langservers.deepforge.org"
+
     volumes:
       - "${DEEPFORGE_DEPLOYMENT_DIR}:/.deployment"
       - "${DEEPFORGE_DEPLOYMENT_DIR}/../config/components.json:/deepforge/config/components.json"
@@ -16,6 +18,7 @@ services:
       - "DEEPFORGE_INTERACTIVE_COMPUTE_HOST=wss://compute.deepforge.org"
       - "DEEPFORGE_PUBLIC_KEY=/token_keys/public_key"
       - "DEEPFORGE_PRIVATE_KEY=/token_keys/private_key"
+      - "DEEPFORGE_LANGUAGE_SERVER_HOST=https://langservers.deepforge.org"
     image: deepforge/kitchen-sink:stable
     ports:
       - "9000:8888"
@@ -26,3 +29,15 @@ services:
       - "${DEEPFORGE_DEPLOYMENT_DIR}/../config/components.json:/deepforge/config/components.json"
     depends_on:
       - mongo
+
+  language_servers:
+    image: deepforge/language-servers:latest
+    ports:
+      - "5000:5000"
+    volumes:
+      - "${DEEPFORGE_DEPLOYMENT_DIR}:/.deployment"
+      - "/tmp/python-models:/tmp/python-models"
+    entrypoint: /.deployment/langserver-entrypoint.sh
+    depends_on:
+      - server
+      - server_stable

.deployment/langserver-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Remove pypi tensorflow in favor of conda installation
+source activate base
+pip uninstall tensorflow -y
+conda install tensorflow==1.14 -y
+
+jsonrpc-ws-proxy --port $PORT --languageServers /languageServers.yml
+

.deployment/languageServers.json

@@ -0,0 +1,18 @@
+{
+  "hostName": "https://langservers.deepforge.org",
+  "servers": {
+    "python": {
+      "command": "Microsoft.Python.LanguageServer",
+      "init": {
+        "interpreter": {
+          "properties": {
+            "InterpreterPath": "/opt/conda/bin/python3"
+          },
+          "code": "import func",
+          "item": "functools"
+        }
+      },
+      "workspace": "/tmp/python-models/"
+    }
+  }
+}

.dockerignore

@@ -1,2 +1,2 @@
 /node_modules
-/.deployment
+!/.deployment/languageServers.json

.github/workflows/docker-build.yml

@@ -4,6 +4,11 @@ on:
   push:
     branches:
       - master
+
+  pull_request:
+    branches:
+      - master
+
   release:
     types:
       - published
@@ -27,15 +32,15 @@ jobs:
         run: |
           echo "TAG=latest" >> $GITHUB_ENV
           echo "RELEASE_TAG=$(echo ${GITHUB_REF:11})" >> $GITHUB_ENV
-        if: github.event_name == 'push'
+
 
       - name: Publish to docker hub (server)
         uses: elgohr/Publish-Docker-Github-Action@master
         with:
           name: deepforge/server
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-          tags: "${{ env.TAG }},${{ env.RELEASE_TAG }}"
+          tags: "1908-pyls-integration"
           dockerfile: docker/Dockerfile
 
       - name: Publish to docker hub (kitchen-sink)
@@ -45,9 +50,20 @@ jobs:
           name: deepforge/kitchen-sink
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-          tags: "${{ env.TAG }},${{ env.RELEASE_TAG }}"
+          tags: "1908-pyls-integration"
           dockerfile: docker/Dockerfile.kitchensink
-          buildargs: "TAG=${{ env.TAG }}"
+          buildargs: "TAG=1908-pyls-integration"
+
+      - name: Publish to docker hub (language-servers)
+        uses: elgohr/Publish-Docker-Github-Action@master
+
+        with:
+          name: deepforge/language-servers
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          tags: latest
+          dockerfile: docker/Dockerfile.langservers
+
 
       - name: Deploy (dev.deepforge.org)
         uses: appleboy/ssh-action@master
@@ -58,9 +74,8 @@ jobs:
           port: ${{ secrets.PORT }}
           passphrase: ${{ secrets.PASSPHRASE }}
           script: |
-            rm -rf deepforge && git clone [email protected]:deepforge-dev/deepforge && cd deepforge
+            rm -rf deepforge && git clone [email protected]:deepforge-dev/deepforge && cd deepforge && git checkout -b 1908-pyls-integration origin/1908-pyls-integration
             chmod +x ./.deployment/deploy-deepforge && ./.deployment/deploy-deepforge
-        if: github.event_name == 'push'
 
       - name: Deploy (editor.deepforge.org)
         uses: appleboy/ssh-action@master

docker/Dockerfile.langservers

@@ -0,0 +1,29 @@
+FROM continuumio/anaconda3:2019.10
+
+LABEL maintainer1.name="Brian Broll"\
+      maintainer1.email="[email protected]"
+
+LABEL maintainer2.name="Umesh Timalsina"\
+      maintainer2.email="[email protected]"
+
+EXPOSE 5000
+
+ENV PORT 5000
+
+RUN apt-get update && curl -sL https://deb.nodesource.com/setup_12.x | bash - &&\
+ apt-get install -y nodejs && apt-get install unzip
+
+RUN npm install -g npm && npm -g config set user root && npm install -g jsonrpc-ws-proxy js-yaml
+
+COPY utils/install-microsoft-python-langserver.sh .
+
+COPY docker/create-servers-yml.js .
+
+COPY .deployment/languageServers.json .deployment/
+
+RUN ./install-microsoft-python-langserver.sh && npm link js-yaml &&\
+ node ./create-servers-yml.js && rm create-servers-yml.js
+
+RUN python -m pip install tensorflow==1.14 keras==2.2.5 matplotlib==3.2.2 simplejson==3.17.2
+
+ENTRYPOINT jsonrpc-ws-proxy --port $PORT --languageServers languageServers.yml

docker/create-servers-yml.js

@@ -0,0 +1,22 @@
+const yaml = require('js-yaml');
+const fs = require('fs');
+const path = require('path');
+const langServers = require(path.resolve(__dirname, '..', '.deployment', 'languageServers'));
+
+function dumpLangServerYaml(fileName) {
+    const serversYml = {
+        langservers: {}
+    };
+
+    Object.keys(langServers.servers).forEach(server => {
+        const command = langServers.servers[server].command;
+        serversYml.langservers[server] = Array.isArray(command) ? command : [command];
+    });
+
+    fs.writeFileSync(fileName, yaml.dump(serversYml));
+}
+
+
+if (require.main === module) {
+    dumpLangServerYaml('languageServers.yml');
+}

docker/docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - "MONGO_URI=mongodb://mongo:27017/deepforge"
       - "DEEPFORGE_PUBLIC_KEY=/token_keys/public_key"
       - "DEEPFORGE_PRIVATE_KEY=/token_keys/private_key"
-    image: deepforge/kitchen-sink:latest
+    image: deepforge/kitchen-sink:1908-pyls-integration
     ports:
       - "8888:8888"
       - "8889:8889"