From 31fd049f5301c9488eb99fe7a15b5dbd73361776 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Fri, 16 Aug 2024 15:41:58 +0100 Subject: [PATCH 1/6] First pass at file api for k8s --- kubernetes.js | 63 +++++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 44 ++++++++++++++++++++++++++------- package.json | 1 + 3 files changed, 99 insertions(+), 9 deletions(-) diff --git a/kubernetes.js b/kubernetes.js index d435823..6c9de7f 100644 --- a/kubernetes.js +++ b/kubernetes.js @@ -1,4 +1,5 @@ const got = require('got') +const FormData = require('form-data') const k8s = require('@kubernetes/client-node') const _ = require('lodash') const awsEFS = require('./lib/aws-efs.js') @@ -498,6 +499,11 @@ const getEndpoints = async (project) => { } } +async function getStaticFileUrl (project, filePath) { + const prefix = project.safeName.match(/^[0-9]/) ? 'srv-' : '' + return `http://${prefix}${project.safeName}.${this._namespace}:2880/flowforge/files_/${filePath}` +} + module.exports = { /** * Initialises this driver @@ -1117,5 +1123,62 @@ module.exports = { } return properties + }, + + // Static Assets API + listFiles: async (instance, filePath) => { + const fileUrl = await getStaticFileUrl(instance, filePath) + try { + return got.get(fileUrl).json() + } catch (err) { + err.statusCode = err.response.statusCode + throw err + } + }, + + updateFile: async (instance, filePath, update) => { + const fileUrl = await getStaticFileUrl(instance, filePath) + try { + return got.put(fileUrl, { + json: update + }) + } catch (err) { + err.statusCode = err.response.statusCode + throw err + } + }, + + deleteFile: async (instance, filePath) => { + const fileUrl = await getStaticFileUrl(instance, filePath) + try { + return got.delete(fileUrl) + } catch (err) { + err.statusCode = err.response.statusCode + throw err + } + }, + createDirectory: async (instance, filePath, directoryName) => { + const fileUrl = await getStaticFileUrl(instance, filePath) + try { + return got.post(fileUrl, { + json: { path: directoryName } + }) + } catch (err) { + err.statusCode = err.response.statusCode + throw err + } + }, + uploadFile: async (instance, filePath, fileBuffer) => { + const form = new FormData() + form.append('file', fileBuffer, { filename: filePath }) + const fileUrl = await getStaticFileUrl(instance, filePath) + try { + return got.post(fileUrl, { + body: form + }) + } catch (err) { + err.statusCode = err.response.statusCode + throw err + } } } diff --git a/package-lock.json b/package-lock.json index f8bb377..2e05e06 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@aws-sdk/client-efs": "^3.600.0", "@kubernetes/client-node": "^0.18.1", + "form-data": "^4.0.0", "got": "^11.8.0", "lodash": "^4.17.21" }, @@ -1408,6 +1409,19 @@ "form-data": "^2.5.0" } }, + "node_modules/@types/request/node_modules/form-data": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" + } + }, "node_modules/@types/responselike": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz", @@ -2529,16 +2543,16 @@ } }, "node_modules/form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", "dependencies": { "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", + "combined-stream": "^1.0.8", "mime-types": "^2.1.12" }, "engines": { - "node": ">= 0.12" + "node": ">= 6" } }, "node_modules/fs-minipass": { @@ -5543,6 +5557,18 @@ "@types/node": "*", "@types/tough-cookie": "*", "form-data": "^2.5.0" + }, + "dependencies": { + "form-data": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + } } }, "@types/responselike": { @@ -6381,12 +6407,12 @@ "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==" }, "form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", "requires": { "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", + "combined-stream": "^1.0.8", "mime-types": "^2.1.12" } }, diff --git a/package.json b/package.json index f2d2fe2..410c74b 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "dependencies": { "@aws-sdk/client-efs": "^3.600.0", "@kubernetes/client-node": "^0.18.1", + "form-data": "^4.0.0", "got": "^11.8.0", "lodash": "^4.17.21" }, From a9289edf3ab6062774fd2e8b20ffb3c9dbbec975 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 20 Aug 2024 15:27:06 +0100 Subject: [PATCH 2/6] Fix namespace access --- kubernetes.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes.js b/kubernetes.js index 6c9de7f..969ce10 100644 --- a/kubernetes.js +++ b/kubernetes.js @@ -499,7 +499,7 @@ const getEndpoints = async (project) => { } } -async function getStaticFileUrl (project, filePath) { +const getStaticFileUrl = async (project, filePath) => { const prefix = project.safeName.match(/^[0-9]/) ? 'srv-' : '' return `http://${prefix}${project.safeName}.${this._namespace}:2880/flowforge/files_/${filePath}` } From cd0dd933c76cbfbd7f3846cb9129126885ac3da3 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 20 Aug 2024 15:43:44 +0100 Subject: [PATCH 3/6] debug --- kubernetes.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kubernetes.js b/kubernetes.js index 969ce10..4a223f5 100644 --- a/kubernetes.js +++ b/kubernetes.js @@ -1128,9 +1128,11 @@ module.exports = { // Static Assets API listFiles: async (instance, filePath) => { const fileUrl = await getStaticFileUrl(instance, filePath) + console.log(fileUrl) try { return got.get(fileUrl).json() } catch (err) { + console.log(err) err.statusCode = err.response.statusCode throw err } From 6da11b0ae9623a272dc139ab9073033180130fa5 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 20 Aug 2024 15:45:00 +0100 Subject: [PATCH 4/6] Fix path --- kubernetes.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes.js b/kubernetes.js index 4a223f5..ce4c814 100644 --- a/kubernetes.js +++ b/kubernetes.js @@ -501,7 +501,7 @@ const getEndpoints = async (project) => { const getStaticFileUrl = async (project, filePath) => { const prefix = project.safeName.match(/^[0-9]/) ? 'srv-' : '' - return `http://${prefix}${project.safeName}.${this._namespace}:2880/flowforge/files_/${filePath}` + return `http://${prefix}${project.safeName}.${this._namespace}:2880/flowforge/files/_/${filePath}` } module.exports = { From 523a6b231ca38ce45ae2939102a488beab169b71 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Thu, 22 Aug 2024 10:50:53 +0100 Subject: [PATCH 5/6] swap project for instance --- kubernetes.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes.js b/kubernetes.js index ce4c814..9ea96e1 100644 --- a/kubernetes.js +++ b/kubernetes.js @@ -499,9 +499,9 @@ const getEndpoints = async (project) => { } } -const getStaticFileUrl = async (project, filePath) => { - const prefix = project.safeName.match(/^[0-9]/) ? 'srv-' : '' - return `http://${prefix}${project.safeName}.${this._namespace}:2880/flowforge/files/_/${filePath}` +const getStaticFileUrl = async (instance, filePath) => { + const prefix = instance.safeName.match(/^[0-9]/) ? 'srv-' : '' + return `http://${prefix}${instance.safeName}.${this._namespace}:2880/flowforge/files/_/${filePath}` } module.exports = { From 9aec7074833a0bfdb2b098aa1eb9a5dbb4e11e72 Mon Sep 17 00:00:00 2001 From: Ben Hardill Date: Tue, 27 Aug 2024 19:24:00 +0100 Subject: [PATCH 6/6] URI encode paths --- kubernetes.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes.js b/kubernetes.js index 9ea96e1..9d3b32d 100644 --- a/kubernetes.js +++ b/kubernetes.js @@ -501,7 +501,7 @@ const getEndpoints = async (project) => { const getStaticFileUrl = async (instance, filePath) => { const prefix = instance.safeName.match(/^[0-9]/) ? 'srv-' : '' - return `http://${prefix}${instance.safeName}.${this._namespace}:2880/flowforge/files/_/${filePath}` + return `http://${prefix}${instance.safeName}.${this._namespace}:2880/flowforge/files/_/${encodeURIComponent(filePath)}` } module.exports = {