From 622f9485737f2061ac34698c20dfbad8e4ef795a Mon Sep 17 00:00:00 2001 From: Ajai Date: Mon, 11 Jan 2021 16:57:32 -0800 Subject: [PATCH] [CDAP-17575] Integrate gitpod with CDAP repo to help spin up CDAP UI directly from pull requests --- .gitpod.Dockerfile | 4 ++ .gitpod.yml | 37 ++++++++++ cdap-ui/gitpod/bamboo_plan_info.json | 3 + cdap-ui/gitpod/package.json | 15 ++++ cdap-ui/gitpod/setup.js | 101 +++++++++++++++++++++++++++ cdap-ui/gitpod/start.js | 42 +++++++++++ cdap-ui/gitpod/unzip.js | 68 ++++++++++++++++++ cdap-ui/gitpod/yarn.lock | 56 +++++++++++++++ cdap-ui/pom.xml | 1 + pom.xml | 2 + 10 files changed, 329 insertions(+) create mode 100644 .gitpod.Dockerfile create mode 100644 .gitpod.yml create mode 100644 cdap-ui/gitpod/bamboo_plan_info.json create mode 100644 cdap-ui/gitpod/package.json create mode 100644 cdap-ui/gitpod/setup.js create mode 100644 cdap-ui/gitpod/start.js create mode 100644 cdap-ui/gitpod/unzip.js create mode 100644 cdap-ui/gitpod/yarn.lock diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile new file mode 100644 index 000000000000..6ae92263a214 --- /dev/null +++ b/.gitpod.Dockerfile @@ -0,0 +1,4 @@ +FROM gitpod/workspace-full + +RUN bash -c ". /home/gitpod/.sdkman/bin/sdkman-init.sh \ + && sdk install java 8.0.265-open" diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 000000000000..8e8cc377ef36 --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,37 @@ +image: + file: .gitpod.Dockerfile +github: + prebuilds: + develop: true + # enable for all branches in this repo (defaults to false) + branches: true + # enable for pull requests coming from this repo (defaults to true) + pullRequests: true + # enable for pull requests coming from forks (defaults to false) + pullRequestsFromForks: true + # add a check to pull requests (defaults to true) + addCheck: true + # add a "Review in Gitpod" button as a comment to pull requests (defaults to false) + addComment: true + # add a "Review in Gitpod" button to the pull request's description (defaults to false) + addBadge: false + # add a label once the prebuild is ready to pull requests (defaults to false) + addLabel: false +tasks: + - init: >- + cd /workspace/cdap/cdap-ui/gitpod && + yarn && + yarn setup + command: >- + cd /workspace/cdap/cdap-ui && + yarn && + ./node_modules/bower/bin/bower install && + yarn cdap-full-build && + cat <<< $(jq '.["dashboard.bind.port"]="11012"' server/config/development/cdap.json) > server/config/development/cdap.json && + yarn start + name: Start UI server @ 11012 +ports: + - port: 11011 + - port: 11012 + onOpen: open-preview + - port: 11015 diff --git a/cdap-ui/gitpod/bamboo_plan_info.json b/cdap-ui/gitpod/bamboo_plan_info.json new file mode 100644 index 000000000000..f7ccadce7f69 --- /dev/null +++ b/cdap-ui/gitpod/bamboo_plan_info.json @@ -0,0 +1,3 @@ +{ + "planName": "CDAP-BUT" +} diff --git a/cdap-ui/gitpod/package.json b/cdap-ui/gitpod/package.json new file mode 100644 index 000000000000..159c42cf0d55 --- /dev/null +++ b/cdap-ui/gitpod/package.json @@ -0,0 +1,15 @@ +{ + "name": "gitpod", + "version": "1.0.0", + "main": "index.js", + "license": "Apache-2.0", + "scripts": { + "setup": "node setup.js" + }, + "dependencies": { + "mkdirp": "^1.0.4", + "node-fetch": "^2.6.1", + "xml2js": "^0.4.23", + "yauzl": "^2.10.0" + } +} diff --git a/cdap-ui/gitpod/setup.js b/cdap-ui/gitpod/setup.js new file mode 100644 index 000000000000..2208cd22e0b5 --- /dev/null +++ b/cdap-ui/gitpod/setup.js @@ -0,0 +1,101 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. +*/ + +const branchInfo = require('./bamboo_plan_info.json'); +const path = require('path'); +var xml2js = require('xml2js'); +const fs = require('fs'); +const fetch = require('node-fetch'); +const unzip = require('./unzip'); +const start = require('./start'); + + +// Get CDAP version from the pom file. +async function getCDAPVersion() { + return new Promise((resolve, reject) => { + var parser = new xml2js.Parser(); + fs.readFile(path.join(__dirname , '..', '..', 'pom.xml'), function(err, data) { + if (err) { + reject(err); + } + parser.parseString(data, function (err, result) { + if (err) { + reject(err); + } + const version = result.project.version[0]; + resolve(version); + }); + }); + }); +} + +async function downloadSDK(res, pathToZipFile) { + const fileStream = fs.createWriteStream(pathToZipFile); + return new Promise((resolve, reject) => { + res.body.pipe(fileStream); + res.body.on("error", reject); + fileStream.on("finish", () => { + console.log('Done downloading!'); + resolve(); + }); + }); +} + +async function main() { + // Get the last successful build from appropriate plan branch. This plan branch corresponds to bamboo plan name for respective branches + const branchResultsRaw = await fetch(`https://builds.cask.co/rest/api/latest/result/${branchInfo.planName}.json?buildstate=successful&max-result=1`); + const branchResults = await branchResultsRaw.json(); + if (!branchResults.results.result.length) { + throw `No successful build found in ${branchInfo.planName}`; + } + const latestSuccessfulBuildName = branchResults.results.result[0].key; + let cdapversion; + try { + cdapversion = await getCDAPVersion(); + console.log('CDAP version: ', cdapversion); + } catch(e) { + throw 'Unable to fetch CDAP version from pom.xml file'; + } + + // Based on the version of CDAP construct the path to the SDK zip file from bamboo build plan. + const SDKZipPath = `https://builds.cask.co/browse/${latestSuccessfulBuildName}/artifact/shared/SDK/cdap/cdap-standalone/target/cdap-sandbox-${cdapversion}.zip`; + console.log('SDK zip path: ', SDKZipPath); + let res; + try { + res = await fetch(SDKZipPath); + } catch(e) { + throw `Invalid url to SDK zip file: ${SDKZipPath} - ${e}` + } + + // Download and unzip SDK + const pathToZipFile = path.join('/workspace', `cdap-sandbox-${cdapversion}.zip`); + try { + await downloadSDK(res, pathToZipFile); + await unzip(pathToZipFile); + console.log(`SDK is unzipped and ready to start!`); + } catch(e) { + console.error('Unable to download or unzip the SDK'); + throw e.message; + } + + try { + await start(path.join('/workspace', `cdap-sandbox-${cdapversion}`, 'bin', 'cdap')); + } catch(e) { + throw e; + } +} + +main(); diff --git a/cdap-ui/gitpod/start.js b/cdap-ui/gitpod/start.js new file mode 100644 index 000000000000..1294fa7b8035 --- /dev/null +++ b/cdap-ui/gitpod/start.js @@ -0,0 +1,42 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. +*/ + +const { spawn } = require('child_process'); +const path = require('path'); +const { StringDecoder } = require('string_decoder'); +const decoder = new StringDecoder('utf8'); + +function stringifyop(data) { + var textChunk = decoder.write(data); + if (textChunk) { + console.log(textChunk); + } +} +function startCDAP(pathToCDAP) { + return new Promise((resolve, reject) => { + const tool = spawn(path.join(pathToCDAP), ['sandbox', 'start']); + tool.stdout.on('data', stringifyop); + tool.stderr.on('data', stringifyop); + tool.on('exit', function(code) { + if (code > 0) { + reject(`Unable to start CDAP. Please check logs for more information.`); + } + resolve(code); + }); + }); +} + +module.exports = startCDAP; diff --git a/cdap-ui/gitpod/unzip.js b/cdap-ui/gitpod/unzip.js new file mode 100644 index 000000000000..bdba0cbb7690 --- /dev/null +++ b/cdap-ui/gitpod/unzip.js @@ -0,0 +1,68 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. +*/ + +const fs = require('fs'); +const path = require('path'); +const yauzl = require("yauzl"); +const mkdirp = require("mkdirp"); + +async function unzipSDK(sdkzippath) { + return new Promise((resolve, reject) => { + try { + yauzl.open(sdkzippath, {lazyEntries: true, autoClose: true}, function(err, zipfile) { + if (err) { + reject(err); + return; + } + zipfile.readEntry(); + zipfile.on("entry", async function(entry) { + if (/\/$/.test(entry.fileName)) { + // directory file names end with '/' + try { + await mkdirp(path.join('/workspace/', entry.fileName)) + zipfile.readEntry(); + } catch(err) { + reject(err); + return; + } + } else { + // file entry + zipfile.openReadStream(entry, async function(err, readStream) { + if (err) {reject(err); return; } + // ensure parent directory exists + try { + await mkdirp(path.dirname(path.join('/workspace/', entry.fileName))); + readStream.pipe(fs.createWriteStream(path.join('/workspace/', entry.fileName), {flags: 'w+', mode: 0o755})); + readStream.on("end", function() { + zipfile.readEntry(); + }); + } catch(err) { + reject(err); + return; + } + }); + } + }); + zipfile.once('close', function() { + resolve(); + }); + }); + } catch(e) { + reject(e); + } + }); +} +module.exports = unzipSDK; diff --git a/cdap-ui/gitpod/yarn.lock b/cdap-ui/gitpod/yarn.lock new file mode 100644 index 000000000000..ea7bb242dd51 --- /dev/null +++ b/cdap-ui/gitpod/yarn.lock @@ -0,0 +1,56 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +buffer-crc32@~0.2.3: + version "0.2.13" + resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" + integrity sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI= + +fd-slicer@~1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e" + integrity sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4= + dependencies: + pend "~1.2.0" + +mkdirp@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== + +node-fetch@^2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" + integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== + +pend@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" + integrity sha1-elfrVQpng/kRUzH89GY9XI4AelA= + +sax@>=0.6.0: + version "1.2.4" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" + integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== + +xml2js@^0.4.23: + version "0.4.23" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" + integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xmlbuilder@~11.0.0: + version "11.0.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" + integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== + +yauzl@^2.10.0: + version "2.10.0" + resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" + integrity sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk= + dependencies: + buffer-crc32 "~0.2.3" + fd-slicer "~1.1.0" diff --git a/cdap-ui/pom.xml b/cdap-ui/pom.xml index 76d0ba36767e..6c29e8623d34 100644 --- a/cdap-ui/pom.xml +++ b/cdap-ui/pom.xml @@ -182,6 +182,7 @@ **/*.snap **/*.rst **/cypress/fixtures/** + **/gitpod/** diff --git a/pom.xml b/pom.xml index c556fe6e536f..ae79948edf54 100644 --- a/pom.xml +++ b/pom.xml @@ -1648,6 +1648,8 @@ **/MANIFEST.MF **/.rubocop*.yml third-party-licenses/** + .gitpod.yml + .gitpod.Dockerfile