From a99f4c474f3d85e27d50510df7ab3e53ada58689 Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Mon, 4 Apr 2022 16:17:39 -0600 Subject: [PATCH 01/23] building deps snapshot and sending it to the frontier dashboard --- .gitignore | 3 ++- bin/compile | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ebe4b53..f198681 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea/ -node_modules \ No newline at end of file +node_modules +deps.json \ No newline at end of file diff --git a/bin/compile b/bin/compile index 6e7f89a..994dcb4 100644 --- a/bin/compile +++ b/bin/compile @@ -21,9 +21,12 @@ source ${BP_DIR}/lib/output.sh ### Upload with Node cd ${BP_DIR} npm install | output "$LOG_FILE" +npm ls --json > deps.json BUILD_DIR=${BUILD_DIR} \ CACHE_DIR=${CACHE_DIR} \ ENV_DIR=${ENV_DIR} \ SOURCE_VERSION=${SOURCE_VERSION} \ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" +echo "FRONTIER_DASHBOARD_TOKEN: ${FRONTIER_DASHBOARD_TOKEN}" +curl -X POST -H "x-api-key: ${FRONTIER_DASHBOARD_TOKEN}" -d "$(cat ./deps.json)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot \ No newline at end of file From dfb627ee13ebda88a4771d9085c1578e716d4cc0 Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Mon, 4 Apr 2022 16:30:59 -0600 Subject: [PATCH 02/23] bind dashboard token var --- bin/compile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 994dcb4..80edc12 100644 --- a/bin/compile +++ b/bin/compile @@ -28,5 +28,6 @@ CACHE_DIR=${CACHE_DIR} \ ENV_DIR=${ENV_DIR} \ SOURCE_VERSION=${SOURCE_VERSION} \ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" -echo "FRONTIER_DASHBOARD_TOKEN: ${FRONTIER_DASHBOARD_TOKEN}" -curl -X POST -H "x-api-key: ${FRONTIER_DASHBOARD_TOKEN}" -d "$(cat ./deps.json)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot \ No newline at end of file +FRONTIER_DASHBOARD_TOKEN=${FRONTIER_DASHBOARD_TOKEN} \ +curl -X POST -H "x-api-key: ${FRONTIER_DASHBOARD_TOKEN}" -d "$(cat ./deps.json)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot +# echo "FRONTIER_DASHBOARD_TOKEN: ${FRONTIER_DASHBOARD_TOKEN}" \ No newline at end of file From 4df59fe5760543c8cf6ecbc9cd55faf56f7aea2e Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Mon, 4 Apr 2022 17:04:42 -0600 Subject: [PATCH 03/23] trying env formats --- bin/compile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/compile b/bin/compile index 80edc12..8c7cd0b 100644 --- a/bin/compile +++ b/bin/compile @@ -21,13 +21,14 @@ source ${BP_DIR}/lib/output.sh ### Upload with Node cd ${BP_DIR} npm install | output "$LOG_FILE" -npm ls --json > deps.json +npm ls --json --depth 1 > deps.json BUILD_DIR=${BUILD_DIR} \ CACHE_DIR=${CACHE_DIR} \ ENV_DIR=${ENV_DIR} \ SOURCE_VERSION=${SOURCE_VERSION} \ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" -FRONTIER_DASHBOARD_TOKEN=${FRONTIER_DASHBOARD_TOKEN} \ +FRONTIER_DASHBOARD_TOKEN=${ENV["FRONTIER_DASHBOARD_TOKEN"]} \ curl -X POST -H "x-api-key: ${FRONTIER_DASHBOARD_TOKEN}" -d "$(cat ./deps.json)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot -# echo "FRONTIER_DASHBOARD_TOKEN: ${FRONTIER_DASHBOARD_TOKEN}" \ No newline at end of file +FRONTIER_DASHBOARD_TOKEN=${ENV["FRONTIER_DASHBOARD_TOKEN"]} \ +echo "FRONTIER_DASHBOARD_TOKEN: ${FRONTIER_DASHBOARD_TOKEN}" \ No newline at end of file From f3916964ef393f0979c38277c2d50ae4221499ca Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Tue, 5 Apr 2022 10:19:18 -0600 Subject: [PATCH 04/23] hardcode frontier dashboard token for now --- bin/compile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bin/compile b/bin/compile index 8c7cd0b..f9995ba 100644 --- a/bin/compile +++ b/bin/compile @@ -21,14 +21,13 @@ source ${BP_DIR}/lib/output.sh ### Upload with Node cd ${BP_DIR} npm install | output "$LOG_FILE" -npm ls --json --depth 1 > deps.json BUILD_DIR=${BUILD_DIR} \ CACHE_DIR=${CACHE_DIR} \ ENV_DIR=${ENV_DIR} \ SOURCE_VERSION=${SOURCE_VERSION} \ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" -FRONTIER_DASHBOARD_TOKEN=${ENV["FRONTIER_DASHBOARD_TOKEN"]} \ -curl -X POST -H "x-api-key: ${FRONTIER_DASHBOARD_TOKEN}" -d "$(cat ./deps.json)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot -FRONTIER_DASHBOARD_TOKEN=${ENV["FRONTIER_DASHBOARD_TOKEN"]} \ -echo "FRONTIER_DASHBOARD_TOKEN: ${FRONTIER_DASHBOARD_TOKEN}" \ No newline at end of file + +### Send deps to Frontier Dashboard +FRONTIER_DASHBOARD_TOKEN=KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7 \ +curl -X POST -H "x-api-key: ${FRONTIER_DASHBOARD_TOKEN}" -d "$(cat ./deps.json)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot \ No newline at end of file From 1c5670801697f291a9b460483faf0575f9fc8750 Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Tue, 5 Apr 2022 10:26:08 -0600 Subject: [PATCH 05/23] bind frontier dash var more statically --- bin/compile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index f9995ba..f85fb0f 100644 --- a/bin/compile +++ b/bin/compile @@ -29,5 +29,4 @@ SOURCE_VERSION=${SOURCE_VERSION} \ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" ### Send deps to Frontier Dashboard -FRONTIER_DASHBOARD_TOKEN=KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7 \ -curl -X POST -H "x-api-key: ${FRONTIER_DASHBOARD_TOKEN}" -d "$(cat ./deps.json)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot \ No newline at end of file +curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot \ No newline at end of file From 2e9cc639148207f5a824f9378fabdc892c8667c6 Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Tue, 5 Apr 2022 10:28:59 -0600 Subject: [PATCH 06/23] it helps to actually build the file you send --- bin/compile | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/compile b/bin/compile index f85fb0f..e8193b4 100644 --- a/bin/compile +++ b/bin/compile @@ -29,4 +29,5 @@ SOURCE_VERSION=${SOURCE_VERSION} \ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" ### Send deps to Frontier Dashboard +npm ls --json --depth 1 > deps.json curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot \ No newline at end of file From 1b8f26bf7677606c6f739112f03340bd5e8e65ff Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Fri, 8 Apr 2022 11:50:17 -0600 Subject: [PATCH 07/23] working node flattener and sender --- .gitignore | 2 +- bin/compile | 6 ++++-- bin/deps-test.sh | 14 ++++++++++++++ bin/flatten-deps.js | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 bin/deps-test.sh create mode 100644 bin/flatten-deps.js diff --git a/.gitignore b/.gitignore index f198681..9616d58 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ .idea/ node_modules -deps.json \ No newline at end of file +deps.json* \ No newline at end of file diff --git a/bin/compile b/bin/compile index e8193b4..8ef75b2 100644 --- a/bin/compile +++ b/bin/compile @@ -29,5 +29,7 @@ SOURCE_VERSION=${SOURCE_VERSION} \ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" ### Send deps to Frontier Dashboard -npm ls --json --depth 1 > deps.json -curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot \ No newline at end of file +echo "Sending deps to Frontier Dashboard" +npm ls --json > deps.json && echo "Done scanning deps..." +node bin/flatten-deps.js && echo "Done flattening deps..." +curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && echo "Done sending deps to Frontier Dashboard..." \ No newline at end of file diff --git a/bin/deps-test.sh b/bin/deps-test.sh new file mode 100644 index 0000000..8a217df --- /dev/null +++ b/bin/deps-test.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# bin/compile + +### Configure environment + +set -o errexit # always exit on error +set -o pipefail # don't ignore exit codes when piping output +set -o nounset # fail on unset variables +unset GIT_DIR # Avoid GIT_DIR leak from previous build steps + +### Send deps to Frontier Dashboard +npm ls --json > deps.json +node bin/flatten-deps.js +curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot \ No newline at end of file diff --git a/bin/flatten-deps.js b/bin/flatten-deps.js new file mode 100644 index 0000000..71360d9 --- /dev/null +++ b/bin/flatten-deps.js @@ -0,0 +1,36 @@ +const path = require('path'); +const fs = require('fs'); +const deps = require(path.join(__dirname, '../deps.json')); + +const output = { + name: deps.name, + version: deps.version +} + +const recursiveSearch = (obj, searchKey, results = []) => { + const r = results; + Object.keys(obj).forEach(key => { + const value = obj[key]; + console.log('key:', key) + // push package version + if (key !== searchKey && typeof value === 'object' && value.hasOwnProperty('version')) { + r.push({ name: key, version: value.version }); + } + // recurse into deps + if (key === searchKey && typeof value === 'object') { + recursiveSearch(value, searchKey, r); + } + // recurse into sub-deps + if (typeof value === 'object' && value.hasOwnProperty(searchKey)) { + recursiveSearch(value[searchKey], searchKey, r); + } + }); + return r; +}; + +const flatDeps = recursiveSearch(deps, 'dependencies'); +console.log(flatDeps.length); + +output.dependencies = flatDeps + +fs.writeFileSync(path.join(__dirname, '../deps.json.flat'), JSON.stringify(output, null, 2)); \ No newline at end of file From 330835e826c03c61a156677d2ef32b0ab42f0a9d Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Fri, 8 Apr 2022 11:59:30 -0600 Subject: [PATCH 08/23] reduce debugging logs --- bin/flatten-deps.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/flatten-deps.js b/bin/flatten-deps.js index 71360d9..cc36fbb 100644 --- a/bin/flatten-deps.js +++ b/bin/flatten-deps.js @@ -11,7 +11,6 @@ const recursiveSearch = (obj, searchKey, results = []) => { const r = results; Object.keys(obj).forEach(key => { const value = obj[key]; - console.log('key:', key) // push package version if (key !== searchKey && typeof value === 'object' && value.hasOwnProperty('version')) { r.push({ name: key, version: value.version }); @@ -29,7 +28,7 @@ const recursiveSearch = (obj, searchKey, results = []) => { }; const flatDeps = recursiveSearch(deps, 'dependencies'); -console.log(flatDeps.length); +console.log("dependency count:",flatDeps.length); output.dependencies = flatDeps From 3bba6f5d0691b0a73007ef9a00eb2f173bfd3a73 Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Fri, 8 Apr 2022 12:03:08 -0600 Subject: [PATCH 09/23] Formatting dep sender logs nicer --- bin/compile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/compile b/bin/compile index 8ef75b2..779bec0 100644 --- a/bin/compile +++ b/bin/compile @@ -29,7 +29,7 @@ SOURCE_VERSION=${SOURCE_VERSION} \ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" ### Send deps to Frontier Dashboard -echo "Sending deps to Frontier Dashboard" -npm ls --json > deps.json && echo "Done scanning deps..." -node bin/flatten-deps.js && echo "Done flattening deps..." -curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && echo "Done sending deps to Frontier Dashboard..." \ No newline at end of file +echo "\nSending deps to Frontier Dashboard" +npm ls --json > deps.json && echo "\nDone scanning deps..." +node bin/flatten-deps.js && echo "\nDone flattening deps..." +curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && echo "\nDone sending deps to Frontier Dashboard..." \ No newline at end of file From 7b697c92503b8180448d3c80bb70fd800b591242 Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Tue, 12 Apr 2022 15:48:49 -0600 Subject: [PATCH 10/23] flatten and separate sets of deps --- bin/compile | 9 +++++---- bin/deps-test.sh | 8 +++++--- bin/flatten-deps.js | 27 ++++++++++++++++++++++++--- package.json | 9 +++++++++ 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/bin/compile b/bin/compile index 779bec0..fac8985 100644 --- a/bin/compile +++ b/bin/compile @@ -29,7 +29,8 @@ SOURCE_VERSION=${SOURCE_VERSION} \ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" ### Send deps to Frontier Dashboard -echo "\nSending deps to Frontier Dashboard" -npm ls --json > deps.json && echo "\nDone scanning deps..." -node bin/flatten-deps.js && echo "\nDone flattening deps..." -curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && echo "\nDone sending deps to Frontier Dashboard..." \ No newline at end of file +printf "\nFrontier Dashboard steps:\n" +printf "Scanning deps... " && npm ls --json > deps.json && printf "Done.\n" +printf "Flattening deps... " && node bin/flatten-deps.js && printf "Done.\n" +printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" +printf "Done with Frontier Dashboard steps\n" \ No newline at end of file diff --git a/bin/deps-test.sh b/bin/deps-test.sh index 8a217df..0278ee7 100644 --- a/bin/deps-test.sh +++ b/bin/deps-test.sh @@ -9,6 +9,8 @@ set -o nounset # fail on unset variables unset GIT_DIR # Avoid GIT_DIR leak from previous build steps ### Send deps to Frontier Dashboard -npm ls --json > deps.json -node bin/flatten-deps.js -curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot \ No newline at end of file +printf "\nFrontier Dashboard steps:\n" +printf "Scanning deps... " && npm ls --json > deps.json && printf "Done.\n" +printf "Flattening deps... " && node bin/flatten-deps.js && printf "Done.\n" +printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" +printf "Done with Frontier Dashboard steps\n" \ No newline at end of file diff --git a/bin/flatten-deps.js b/bin/flatten-deps.js index cc36fbb..d75406d 100644 --- a/bin/flatten-deps.js +++ b/bin/flatten-deps.js @@ -1,6 +1,10 @@ const path = require('path'); const fs = require('fs'); const deps = require(path.join(__dirname, '../deps.json')); +// get host app's package.json +const package = require(path.join(__dirname, 'package.json')); +// testing: use the local repo's package.json +// const package = require(path.join(__dirname, '../package.json')); const output = { name: deps.name, @@ -27,9 +31,26 @@ const recursiveSearch = (obj, searchKey, results = []) => { return r; }; -const flatDeps = recursiveSearch(deps, 'dependencies'); -console.log("dependency count:",flatDeps.length); +function findDepVersion(deps, searchKey) { + let dep = {name: searchKey}; + for (const d of deps) { + if (d.name === searchKey) { + dep = d; + break; + } + }; + return dep +} + +const deepDeps = recursiveSearch(deps, 'dependencies'); +console.log("Deep dependency count:", deepDeps.length); -output.dependencies = flatDeps +// normal deps +output.deps = package.dependencies ? Object.keys(package.dependencies).map(d => findDepVersion(deepDeps, d)): []; +output.devDeps = package.devDependencies ? Object.keys(package.devDependencies).map(d => findDepVersion(deepDeps, d)) : []; +output.peerDeps = package.peerDependencies ? Object.keys(package.peerDependencies).map(d => findDepVersion(deepDeps, d)) : []; +// deps that aren't in deps, devDeps, or peerDeps +output.secondaryDeps = deepDeps.filter(d => !output.deps.includes(d) && !output.devDeps.includes(d) && !output.peerDeps.includes(d)); +// Write to file fs.writeFileSync(path.join(__dirname, '../deps.json.flat'), JSON.stringify(output, null, 2)); \ No newline at end of file diff --git a/package.json b/package.json index 147db69..117abe0 100644 --- a/package.json +++ b/package.json @@ -24,5 +24,14 @@ "lodash": "^4.15.0", "mime-types": "^2.1.11", "shelljs": "^0.8.4" + }, + "devDependencies": { + "async": "^3.2.0", + "aws-sdk": "^2.5.0" + }, + "peerDependencies": { + "lodash": "^4.15.0", + "mime-types": "^2.1.11", + "shelljs": "^0.8.4" } } From 565afa279e8a13bcd1f162f6e654465ccd35f7bc Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Tue, 12 Apr 2022 16:06:49 -0600 Subject: [PATCH 11/23] use heroku build dir during deploy --- bin/flatten-deps.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/flatten-deps.js b/bin/flatten-deps.js index d75406d..1d878c1 100644 --- a/bin/flatten-deps.js +++ b/bin/flatten-deps.js @@ -1,8 +1,11 @@ const path = require('path'); const fs = require('fs'); -const deps = require(path.join(__dirname, '../deps.json')); +// heroku buildpack dir or local test +const buildDir = process.env.BUILD_DIR || '../' +console.log('buildDir', buildDir); +const deps = require(path.join(buildDir, 'deps.json')); // get host app's package.json -const package = require(path.join(__dirname, 'package.json')); +const package = require(path.join(buildDir, 'package.json')); // testing: use the local repo's package.json // const package = require(path.join(__dirname, '../package.json')); From 3dee30aea01afe754bf6c44fb6f70bd5406ca681 Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Tue, 12 Apr 2022 16:08:00 -0600 Subject: [PATCH 12/23] cleanup --- bin/flatten-deps.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/bin/flatten-deps.js b/bin/flatten-deps.js index 1d878c1..8190463 100644 --- a/bin/flatten-deps.js +++ b/bin/flatten-deps.js @@ -2,12 +2,9 @@ const path = require('path'); const fs = require('fs'); // heroku buildpack dir or local test const buildDir = process.env.BUILD_DIR || '../' -console.log('buildDir', buildDir); const deps = require(path.join(buildDir, 'deps.json')); // get host app's package.json const package = require(path.join(buildDir, 'package.json')); -// testing: use the local repo's package.json -// const package = require(path.join(__dirname, '../package.json')); const output = { name: deps.name, From 8b6c3e20f60a44cbe0e47a14577cd0f304785918 Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Tue, 12 Apr 2022 16:14:06 -0600 Subject: [PATCH 13/23] debug and logging --- bin/compile | 1 + bin/flatten-deps.js | 1 + 2 files changed, 2 insertions(+) diff --git a/bin/compile b/bin/compile index fac8985..9fc9bec 100644 --- a/bin/compile +++ b/bin/compile @@ -30,6 +30,7 @@ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" ### Send deps to Frontier Dashboard printf "\nFrontier Dashboard steps:\n" +printf "$(ls -la)" printf "Scanning deps... " && npm ls --json > deps.json && printf "Done.\n" printf "Flattening deps... " && node bin/flatten-deps.js && printf "Done.\n" printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" diff --git a/bin/flatten-deps.js b/bin/flatten-deps.js index 8190463..fa5ea41 100644 --- a/bin/flatten-deps.js +++ b/bin/flatten-deps.js @@ -5,6 +5,7 @@ const buildDir = process.env.BUILD_DIR || '../' const deps = require(path.join(buildDir, 'deps.json')); // get host app's package.json const package = require(path.join(buildDir, 'package.json')); +console.log('Flattening dependencies for', package.name, '...'); const output = { name: deps.name, From 0f2000d70d8bf0c2a9a2a665abe869e3162a20ab Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Tue, 12 Apr 2022 16:40:21 -0600 Subject: [PATCH 14/23] finding app package --- bin/compile | 14 ++++++++------ bin/deps-test.sh | 1 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/bin/compile b/bin/compile index 9fc9bec..5340e12 100644 --- a/bin/compile +++ b/bin/compile @@ -29,9 +29,11 @@ SOURCE_VERSION=${SOURCE_VERSION} \ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" ### Send deps to Frontier Dashboard -printf "\nFrontier Dashboard steps:\n" -printf "$(ls -la)" -printf "Scanning deps... " && npm ls --json > deps.json && printf "Done.\n" -printf "Flattening deps... " && node bin/flatten-deps.js && printf "Done.\n" -printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" -printf "Done with Frontier Dashboard steps\n" \ No newline at end of file +if [ -f $1/package.json ]; then + printf "\nFrontier Dashboard steps:\n" + printf "$(ls -la BUILD_DIR)\n" + printf "Scanning deps... " && npm ls --json > deps.json && printf "Done.\n" + printf "Flattening deps... " && node bin/flatten-deps.js && printf "Done.\n" + printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" + printf "Done with Frontier Dashboard steps\n" +fi \ No newline at end of file diff --git a/bin/deps-test.sh b/bin/deps-test.sh index 0278ee7..6c3e085 100644 --- a/bin/deps-test.sh +++ b/bin/deps-test.sh @@ -10,6 +10,7 @@ unset GIT_DIR # Avoid GIT_DIR leak from previous build steps ### Send deps to Frontier Dashboard printf "\nFrontier Dashboard steps:\n" +printf "$(ls -la)" printf "Scanning deps... " && npm ls --json > deps.json && printf "Done.\n" printf "Flattening deps... " && node bin/flatten-deps.js && printf "Done.\n" printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" From a862e7277dc039054c02fbe2cc1818434ba298a8 Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Tue, 12 Apr 2022 16:45:55 -0600 Subject: [PATCH 15/23] finding app package --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 5340e12..5127b28 100644 --- a/bin/compile +++ b/bin/compile @@ -31,9 +31,9 @@ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" ### Send deps to Frontier Dashboard if [ -f $1/package.json ]; then printf "\nFrontier Dashboard steps:\n" - printf "$(ls -la BUILD_DIR)\n" + printf "$(ls -la $BUILD_DIR)\n" printf "Scanning deps... " && npm ls --json > deps.json && printf "Done.\n" - printf "Flattening deps... " && node bin/flatten-deps.js && printf "Done.\n" + printf "Flattening deps... " && BUILD_DIR=${BUILD_DIR} node bin/flatten-deps.js && printf "Done.\n" printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" printf "Done with Frontier Dashboard steps\n" fi \ No newline at end of file From 81c692f8336b8a4e88b6e7b50675b0254ea5487e Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Tue, 12 Apr 2022 16:57:30 -0600 Subject: [PATCH 16/23] getting closer to right bp and app paths --- bin/flatten-deps.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/flatten-deps.js b/bin/flatten-deps.js index fa5ea41..51a17ba 100644 --- a/bin/flatten-deps.js +++ b/bin/flatten-deps.js @@ -2,7 +2,8 @@ const path = require('path'); const fs = require('fs'); // heroku buildpack dir or local test const buildDir = process.env.BUILD_DIR || '../' -const deps = require(path.join(buildDir, 'deps.json')); +const bpDir = process.env.BP_DIR || './' +const deps = require(path.join(bpDir, 'deps.json')); // get host app's package.json const package = require(path.join(buildDir, 'package.json')); console.log('Flattening dependencies for', package.name, '...'); @@ -54,4 +55,4 @@ output.peerDeps = package.peerDependencies ? Object.keys(package.peerDependencie output.secondaryDeps = deepDeps.filter(d => !output.deps.includes(d) && !output.devDeps.includes(d) && !output.peerDeps.includes(d)); // Write to file -fs.writeFileSync(path.join(__dirname, '../deps.json.flat'), JSON.stringify(output, null, 2)); \ No newline at end of file +fs.writeFileSync(path.join(bpDir, 'deps.json.flat'), JSON.stringify(output, null, 2)); \ No newline at end of file From a21bc08a392cb058856dfcfbcd48759f90656d0a Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Tue, 12 Apr 2022 16:58:29 -0600 Subject: [PATCH 17/23] pass bp di --- bin/compile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/compile b/bin/compile index 5127b28..4c09979 100644 --- a/bin/compile +++ b/bin/compile @@ -33,7 +33,7 @@ if [ -f $1/package.json ]; then printf "\nFrontier Dashboard steps:\n" printf "$(ls -la $BUILD_DIR)\n" printf "Scanning deps... " && npm ls --json > deps.json && printf "Done.\n" - printf "Flattening deps... " && BUILD_DIR=${BUILD_DIR} node bin/flatten-deps.js && printf "Done.\n" + printf "Flattening deps... " && BUILD_DIR=${BUILD_DIR} BP_DIR=${BP_DIR} node bin/flatten-deps.js && printf "Done.\n" printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" printf "Done with Frontier Dashboard steps\n" fi \ No newline at end of file From 9a6c5a6f470d7c7436a46acf797f0ae15fd3905b Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Tue, 12 Apr 2022 17:23:04 -0600 Subject: [PATCH 18/23] npm ls in app dir --- bin/compile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/compile b/bin/compile index 4c09979..5f2f67c 100644 --- a/bin/compile +++ b/bin/compile @@ -17,7 +17,7 @@ LOG_FILE='/tmp/node-build-log.txt' ### Load dependencies source ${BP_DIR}/lib/output.sh - +pwd ### Upload with Node cd ${BP_DIR} npm install | output "$LOG_FILE" @@ -28,12 +28,13 @@ ENV_DIR=${ENV_DIR} \ SOURCE_VERSION=${SOURCE_VERSION} \ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" + ### Send deps to Frontier Dashboard if [ -f $1/package.json ]; then printf "\nFrontier Dashboard steps:\n" printf "$(ls -la $BUILD_DIR)\n" - printf "Scanning deps... " && npm ls --json > deps.json && printf "Done.\n" - printf "Flattening deps... " && BUILD_DIR=${BUILD_DIR} BP_DIR=${BP_DIR} node bin/flatten-deps.js && printf "Done.\n" + printf "Scanning deps... " && npm ls --json --prefix $1 > deps.json && printf "Done.\n" + printf "Flattening deps... " && BUILD_DIR=${BUILD_DIR} BP_DIR=${BP_DIR} node ${BP_DIR}/bin/flatten-deps.js && printf "Done.\n" printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" printf "Done with Frontier Dashboard steps\n" fi \ No newline at end of file From 8fb68c51caf689a2819b474672defc5ed42275ca Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Tue, 12 Apr 2022 17:23:59 -0600 Subject: [PATCH 19/23] pass prefix of build dir to npm ls --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 5f2f67c..3a45ec3 100644 --- a/bin/compile +++ b/bin/compile @@ -33,8 +33,8 @@ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" if [ -f $1/package.json ]; then printf "\nFrontier Dashboard steps:\n" printf "$(ls -la $BUILD_DIR)\n" - printf "Scanning deps... " && npm ls --json --prefix $1 > deps.json && printf "Done.\n" - printf "Flattening deps... " && BUILD_DIR=${BUILD_DIR} BP_DIR=${BP_DIR} node ${BP_DIR}/bin/flatten-deps.js && printf "Done.\n" + printf "Scanning deps... " && npm ls --json --prefix ${BUILD_DIR} > deps.json && printf "Done.\n" + printf "Flattening deps... " && BUILD_DIR=${BUILD_DIR} BP_DIR=${BP_DIR} node ${BP_DIR}/bin/flatten-deps.js && printf "Done.\n" printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" printf "Done with Frontier Dashboard steps\n" fi \ No newline at end of file From 702f0d1ab7983355761d56e89e29db723efeecca Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Wed, 27 Apr 2022 14:42:52 -0600 Subject: [PATCH 20/23] added deduper that keeps highest semver for each dep --- bin/compile | 3 +-- bin/flatten-deps.js | 27 ++++++++++++++++++++++----- package-lock.json | 21 +++++++++++++++++++++ package.json | 1 + 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/bin/compile b/bin/compile index 3a45ec3..68c05f0 100644 --- a/bin/compile +++ b/bin/compile @@ -17,7 +17,7 @@ LOG_FILE='/tmp/node-build-log.txt' ### Load dependencies source ${BP_DIR}/lib/output.sh -pwd + ### Upload with Node cd ${BP_DIR} npm install | output "$LOG_FILE" @@ -32,7 +32,6 @@ node ${BP_DIR}/lib/upload.js | output "$LOG_FILE" ### Send deps to Frontier Dashboard if [ -f $1/package.json ]; then printf "\nFrontier Dashboard steps:\n" - printf "$(ls -la $BUILD_DIR)\n" printf "Scanning deps... " && npm ls --json --prefix ${BUILD_DIR} > deps.json && printf "Done.\n" printf "Flattening deps... " && BUILD_DIR=${BUILD_DIR} BP_DIR=${BP_DIR} node ${BP_DIR}/bin/flatten-deps.js && printf "Done.\n" printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" diff --git a/bin/flatten-deps.js b/bin/flatten-deps.js index 51a17ba..0407bba 100644 --- a/bin/flatten-deps.js +++ b/bin/flatten-deps.js @@ -1,8 +1,9 @@ const path = require('path'); const fs = require('fs'); +const semverGt = require('semver/functions/gt') // heroku buildpack dir or local test const buildDir = process.env.BUILD_DIR || '../' -const bpDir = process.env.BP_DIR || './' +const bpDir = process.env.BP_DIR || '../' const deps = require(path.join(bpDir, 'deps.json')); // get host app's package.json const package = require(path.join(buildDir, 'package.json')); @@ -47,12 +48,28 @@ function findDepVersion(deps, searchKey) { const deepDeps = recursiveSearch(deps, 'dependencies'); console.log("Deep dependency count:", deepDeps.length); +// dedupe deps, keeping the highest semver version for each +const dedupedDeps = deepDeps.reduce((acc, dep) => { + const existing = acc.find(d => d.name === dep.name); + if (existing) { + console.log('exists, versions:', existing.version, dep.version); + if (semverGt(dep.version, existing.version)) { + acc.splice(acc.indexOf(existing), 1, dep); + } + } else { + acc.push(dep); + } + return acc; +}, []); +console.log("Deduped dependency count:", dedupedDeps.length); + + // normal deps -output.deps = package.dependencies ? Object.keys(package.dependencies).map(d => findDepVersion(deepDeps, d)): []; -output.devDeps = package.devDependencies ? Object.keys(package.devDependencies).map(d => findDepVersion(deepDeps, d)) : []; -output.peerDeps = package.peerDependencies ? Object.keys(package.peerDependencies).map(d => findDepVersion(deepDeps, d)) : []; +output.deps = package.dependencies ? Object.keys(package.dependencies).map(d => findDepVersion(dedupedDeps, d)): []; +output.devDeps = package.devDependencies ? Object.keys(package.devDependencies).map(d => findDepVersion(dedupedDeps, d)) : []; +output.peerDeps = package.peerDependencies ? Object.keys(package.peerDependencies).map(d => findDepVersion(dedupedDeps, d)) : []; // deps that aren't in deps, devDeps, or peerDeps -output.secondaryDeps = deepDeps.filter(d => !output.deps.includes(d) && !output.devDeps.includes(d) && !output.peerDeps.includes(d)); +output.secondaryDeps = dedupedDeps.filter(d => !output.deps.includes(d) && !output.devDeps.includes(d) && !output.peerDeps.includes(d)); // Write to file fs.writeFileSync(path.join(bpDir, 'deps.json.flat'), JSON.stringify(output, null, 2)); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index ebf3498..4de20b1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -308,6 +308,14 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, "merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -437,6 +445,14 @@ "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" }, + "semver": { + "version": "7.3.7", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz", + "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "requires": { + "lru-cache": "^6.0.0" + } + }, "shelljs": { "version": "0.8.4", "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz", @@ -492,6 +508,11 @@ "version": "9.0.7", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" } } } diff --git a/package.json b/package.json index 117abe0..18c9d44 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "glob": "^7.0.5", "lodash": "^4.15.0", "mime-types": "^2.1.11", + "semver": "^7.3.7", "shelljs": "^0.8.4" }, "devDependencies": { From 74abd1f7413726e5cf802ef7c83c74e75b693ea6 Mon Sep 17 00:00:00 2001 From: Jakob Anderson Date: Wed, 27 Apr 2022 16:21:49 -0600 Subject: [PATCH 21/23] added semver validation --- bin/flatten-deps.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/flatten-deps.js b/bin/flatten-deps.js index 0407bba..e86b29c 100644 --- a/bin/flatten-deps.js +++ b/bin/flatten-deps.js @@ -1,6 +1,7 @@ const path = require('path'); const fs = require('fs'); const semverGt = require('semver/functions/gt') +const semverValid = require('semver/functions/valid') // heroku buildpack dir or local test const buildDir = process.env.BUILD_DIR || '../' const bpDir = process.env.BP_DIR || '../' @@ -53,7 +54,7 @@ const dedupedDeps = deepDeps.reduce((acc, dep) => { const existing = acc.find(d => d.name === dep.name); if (existing) { console.log('exists, versions:', existing.version, dep.version); - if (semverGt(dep.version, existing.version)) { + if (semverValid(existing.version) && semverValid(dep.version) && semverGt(dep.version, existing.version)) { acc.splice(acc.indexOf(existing), 1, dep); } } else { From 174deeb6c54a02910b0afa91d9c6d2f1647e230d Mon Sep 17 00:00:00 2001 From: Heidi Sommer Date: Tue, 3 May 2022 11:20:44 -0600 Subject: [PATCH 22/23] Update compile --- bin/compile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/compile b/bin/compile index 68c05f0..962f660 100644 --- a/bin/compile +++ b/bin/compile @@ -34,6 +34,6 @@ if [ -f $1/package.json ]; then printf "\nFrontier Dashboard steps:\n" printf "Scanning deps... " && npm ls --json --prefix ${BUILD_DIR} > deps.json && printf "Done.\n" printf "Flattening deps... " && BUILD_DIR=${BUILD_DIR} BP_DIR=${BP_DIR} node ${BP_DIR}/bin/flatten-deps.js && printf "Done.\n" - printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" + printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: hmPlMe6As2aNih6Dg3sRj8mHhiM9mDWo1bldZoaz" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" printf "Done with Frontier Dashboard steps\n" -fi \ No newline at end of file +fi From 8a138814ab3fdf20fdfc350a9a835a1af28eea5b Mon Sep 17 00:00:00 2001 From: Heidi Sommer Date: Tue, 3 May 2022 11:22:17 -0600 Subject: [PATCH 23/23] Update deps-test.sh --- bin/deps-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/deps-test.sh b/bin/deps-test.sh index 6c3e085..7d6fc55 100644 --- a/bin/deps-test.sh +++ b/bin/deps-test.sh @@ -13,5 +13,5 @@ printf "\nFrontier Dashboard steps:\n" printf "$(ls -la)" printf "Scanning deps... " && npm ls --json > deps.json && printf "Done.\n" printf "Flattening deps... " && node bin/flatten-deps.js && printf "Done.\n" -printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: KrgZiZRMgxNKvH5gew3n6VBxkcradwu9lQrZe5C7" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" -printf "Done with Frontier Dashboard steps\n" \ No newline at end of file +printf "Sending deps to Frontier Dashboard... " curl -X POST -H "x-api-key: hmPlMe6As2aNih6Dg3sRj8mHhiM9mDWo1bldZoaz" -d "$(cat ./deps.json.flat)" https://tiagtww9kj.execute-api.us-east-1.amazonaws.com/dev/app/deploy/snapshot && printf "Done.\n" +printf "Done with Frontier Dashboard steps\n"