From e89810c26e167b3b9b5c72b86f82dc88301df01a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Mon, 23 Jan 2023 21:39:30 +0100 Subject: [PATCH 01/33] Update Neo4J schema to contain info about vulnerable components --- src/depvis-next/helpers/GraphHelper.ts | 17 ++++++++++++----- src/depvis-next/types/gqlTypes.ts | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/depvis-next/helpers/GraphHelper.ts b/src/depvis-next/helpers/GraphHelper.ts index f3de6d1..00296ae 100644 --- a/src/depvis-next/helpers/GraphHelper.ts +++ b/src/depvis-next/helpers/GraphHelper.ts @@ -2,10 +2,10 @@ import { gql } from '@apollo/client'; export const formatData = (data) => { const nodes = []; - const links = []; - if (!data || !data.projects || !data.projects[0].allComponents) return { nodes, links }; - const componentsCount = data.projects[0].allComponents.length / 20; - data.projects[0].allComponents.forEach((c) => { + let links = []; + if (!data || !data.projects || !data.projects[0].allVulnerableComponents) return { nodes, links }; + const componentsCount = data.projects[0].allVulnerableComponents.length / 20; + data.projects[0].allVulnerableComponents.forEach((c) => { nodes.push({ id: c.purl, name: c.name, @@ -19,6 +19,7 @@ export const formatData = (data) => { source: c.purl, target: d.purl, sourceDependsOnCount: c.dependsOnCount, + toVuln: false, }); }); } @@ -27,6 +28,7 @@ export const formatData = (data) => { links.push({ source: c.purl, target: v.id, + toVuln: true, }); nodes.push({ size: 1 + v.cvssScore, @@ -52,6 +54,11 @@ export const formatData = (data) => { }); } }); + //Filter out links that are not connected + links = links.filter((l) => { + console.log(l); + if (l.toVuln || nodes.find((n) => n.id === l.target)) return true; + }); console.log({ node: nodes, links: links }); return { nodes, links }; }; @@ -59,7 +66,7 @@ export const formatData = (data) => { export const getAllComponentsQuery = gql` query getProjectComponents($projectId: ID) { projects(where: { id: $projectId }) { - allComponents { + allVulnerableComponents { id name version diff --git a/src/depvis-next/types/gqlTypes.ts b/src/depvis-next/types/gqlTypes.ts index 76f8918..9818825 100644 --- a/src/depvis-next/types/gqlTypes.ts +++ b/src/depvis-next/types/gqlTypes.ts @@ -25,6 +25,15 @@ export const typeDefs = gql` dependsOn: [Component!]! @relationship(type: "DEPENDS_ON", direction: OUT) references: [Reference!]! @relationship(type: "HAS_REFERENCE", direction: OUT) vulnerabilities: [Vulnerability!]! @relationship(type: "HAS_VULNERABILITY", direction: OUT) + indirectVulnCount: Int + @cypher( + statement: """ + MATCH (v:Vulnerability) + CALL apoc.algo.dijkstra(this,v, "DEPENDS_ON>|HAS_VULNERABILITY>", "count",1) + YIELD weight + RETURN count(weight) + """ + ) } type Vulnerability { @@ -53,6 +62,17 @@ export const typeDefs = gql` RETURN distinct single """ ) + allVulnerableComponents: [Component!]! + @cypher( + statement: """ + MATCH a=(v:Vulnerability)<-[:HAS_VULNERABILITY]-(c1:Component)<-[:DEPENDS_ON*]-(c2)<-[:DEPENDS_ON]-(this) + WITH NODES(a) AS nodes + UNWIND nodes AS n + WITH n + WHERE 'Component' IN LABELS(n) + RETURN distinct n; + """ + ) component: [Component!]! @relationship(type: "DEPENDS_ON", direction: OUT) date: Date } From 4d48c85acf6b5aa159470d187beb658ba327cdb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Mon, 23 Jan 2023 21:39:30 +0100 Subject: [PATCH 02/33] Update Neo4J schema to contain info about vulnerable components --- src/depvis-next/helpers/GraphHelper.ts | 17 ++++++++++++----- src/depvis-next/types/gqlTypes.ts | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/depvis-next/helpers/GraphHelper.ts b/src/depvis-next/helpers/GraphHelper.ts index f3de6d1..00296ae 100644 --- a/src/depvis-next/helpers/GraphHelper.ts +++ b/src/depvis-next/helpers/GraphHelper.ts @@ -2,10 +2,10 @@ import { gql } from '@apollo/client'; export const formatData = (data) => { const nodes = []; - const links = []; - if (!data || !data.projects || !data.projects[0].allComponents) return { nodes, links }; - const componentsCount = data.projects[0].allComponents.length / 20; - data.projects[0].allComponents.forEach((c) => { + let links = []; + if (!data || !data.projects || !data.projects[0].allVulnerableComponents) return { nodes, links }; + const componentsCount = data.projects[0].allVulnerableComponents.length / 20; + data.projects[0].allVulnerableComponents.forEach((c) => { nodes.push({ id: c.purl, name: c.name, @@ -19,6 +19,7 @@ export const formatData = (data) => { source: c.purl, target: d.purl, sourceDependsOnCount: c.dependsOnCount, + toVuln: false, }); }); } @@ -27,6 +28,7 @@ export const formatData = (data) => { links.push({ source: c.purl, target: v.id, + toVuln: true, }); nodes.push({ size: 1 + v.cvssScore, @@ -52,6 +54,11 @@ export const formatData = (data) => { }); } }); + //Filter out links that are not connected + links = links.filter((l) => { + console.log(l); + if (l.toVuln || nodes.find((n) => n.id === l.target)) return true; + }); console.log({ node: nodes, links: links }); return { nodes, links }; }; @@ -59,7 +66,7 @@ export const formatData = (data) => { export const getAllComponentsQuery = gql` query getProjectComponents($projectId: ID) { projects(where: { id: $projectId }) { - allComponents { + allVulnerableComponents { id name version diff --git a/src/depvis-next/types/gqlTypes.ts b/src/depvis-next/types/gqlTypes.ts index 76f8918..9818825 100644 --- a/src/depvis-next/types/gqlTypes.ts +++ b/src/depvis-next/types/gqlTypes.ts @@ -25,6 +25,15 @@ export const typeDefs = gql` dependsOn: [Component!]! @relationship(type: "DEPENDS_ON", direction: OUT) references: [Reference!]! @relationship(type: "HAS_REFERENCE", direction: OUT) vulnerabilities: [Vulnerability!]! @relationship(type: "HAS_VULNERABILITY", direction: OUT) + indirectVulnCount: Int + @cypher( + statement: """ + MATCH (v:Vulnerability) + CALL apoc.algo.dijkstra(this,v, "DEPENDS_ON>|HAS_VULNERABILITY>", "count",1) + YIELD weight + RETURN count(weight) + """ + ) } type Vulnerability { @@ -53,6 +62,17 @@ export const typeDefs = gql` RETURN distinct single """ ) + allVulnerableComponents: [Component!]! + @cypher( + statement: """ + MATCH a=(v:Vulnerability)<-[:HAS_VULNERABILITY]-(c1:Component)<-[:DEPENDS_ON*]-(c2)<-[:DEPENDS_ON]-(this) + WITH NODES(a) AS nodes + UNWIND nodes AS n + WITH n + WHERE 'Component' IN LABELS(n) + RETURN distinct n; + """ + ) component: [Component!]! @relationship(type: "DEPENDS_ON", direction: OUT) date: Date } From f72fd1e8ccc84bdbb56eb0f8c263dcaa1c0ccd7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Tue, 31 Jan 2023 08:02:14 +0100 Subject: [PATCH 03/33] FIltering works --- .../components/Graph/GraphConfig.tsx | 17 ++++----- .../components/GraphControl/GraphControl.tsx | 17 ++++++++- .../components/Layout/Workspace.tsx | 15 +++++--- src/depvis-next/helpers/ApolloClientHelper.ts | 4 +-- src/depvis-next/helpers/GraphHelper.ts | 36 ++++++++++++++++--- src/depvis-next/pages/api/import/index.ts | 6 ++-- src/depvis-next/queues/ImportQueue.ts | 9 ++--- 7 files changed, 75 insertions(+), 29 deletions(-) diff --git a/src/depvis-next/components/Graph/GraphConfig.tsx b/src/depvis-next/components/Graph/GraphConfig.tsx index f27e2fa..50f69b7 100644 --- a/src/depvis-next/components/Graph/GraphConfig.tsx +++ b/src/depvis-next/components/Graph/GraphConfig.tsx @@ -1,9 +1,10 @@ export type GraphConfig = { - zoomLevel: number - color: string | Function, - label: string | Function, - linkDirectionalArrowLength: number, - linkDirectionalRelPos: number - nodeVal: number | Function - linkLength: number -} \ No newline at end of file + zoomLevel: number; + color: string | Function; + label: string | Function; + linkDirectionalArrowLength: number; + linkDirectionalRelPos: number; + nodeVal: number | Function; + linkLength: number; + showOnlyVulnerable: Boolean; +}; diff --git a/src/depvis-next/components/GraphControl/GraphControl.tsx b/src/depvis-next/components/GraphControl/GraphControl.tsx index bc5facd..0c0e842 100644 --- a/src/depvis-next/components/GraphControl/GraphControl.tsx +++ b/src/depvis-next/components/GraphControl/GraphControl.tsx @@ -10,17 +10,21 @@ const GraphControl = (props) => { useEffect(() => { props.onGraphConfigChange(graphConfig); + console.log(graphConfig); }, [graphConfig]); const handleNodeValToggle = (e) => { if (typeof graphConfig.nodeVal === 'function') { setGraphConfig({ ...graphConfig, nodeVal: 1 }); - console.log(graphConfig); } else { setGraphConfig({ ...graphConfig, nodeVal: getNodeValue }); } }; + const handleShowOnlyVulnerableToggle = (e) => { + setGraphConfig({ ...graphConfig, showOnlyVulnerable: !graphConfig.showOnlyVulnerable }); + }; + return (
Graph settings
@@ -35,6 +39,17 @@ const GraphControl = (props) => { checked={typeof graphConfig.nodeVal === 'function'} /> + + Show only affected components + { + handleShowOnlyVulnerableToggle(e); + }} + checked={graphConfig.showOnlyVulnerable as boolean} + /> + Length of links { diff --git a/src/depvis-next/components/Layout/Workspace.tsx b/src/depvis-next/components/Layout/Workspace.tsx index 4e51bba..56cbee1 100644 --- a/src/depvis-next/components/Layout/Workspace.tsx +++ b/src/depvis-next/components/Layout/Workspace.tsx @@ -28,6 +28,7 @@ const defaultGraphConfig: GraphConfig = { linkDirectionalRelPos: 0, linkLength: 10, nodeVal: getNodeValue, + showOnlyVulnerable: false, }; const Workspace = () => { @@ -48,7 +49,7 @@ const Workspace = () => { useEffect(() => { if (data) { - setGraphData(formatData(data)); + setGraphData(formatData(data.projects[0].allComponents)); } }, [data]); useEffect(() => { @@ -59,14 +60,20 @@ const Workspace = () => { getGraphData({ variables: { projectId: selectedProject } }); } }, [selectedProject]); + useEffect(() => { + handleShowOnlyVulnerableToggle(); + }, [graphConfig]); const handleNodeClick = (node) => { setNode(node); }; - const handleVuln = async () => { - const res = await fetch('/api/vuln'); - console.log(res); + const handleShowOnlyVulnerableToggle = () => { + if (graphConfig.showOnlyVulnerable) { + setGraphData(formatData(data.projects[0].allVulnerableComponents)); + } else { + setGraphData(formatData(data.projects[0].allComponents)); + } }; const handleSelectedSearchResult = (object) => { diff --git a/src/depvis-next/helpers/ApolloClientHelper.ts b/src/depvis-next/helpers/ApolloClientHelper.ts index 8078541..98aba03 100644 --- a/src/depvis-next/helpers/ApolloClientHelper.ts +++ b/src/depvis-next/helpers/ApolloClientHelper.ts @@ -1,14 +1,12 @@ import { ApolloClient, HttpLink, InMemoryCache } from '@apollo/client'; -import { gqlUrlPath } from '../pages/api/graphql'; /** * Function responsible for initialization of new Apollo Client used for GraphQL * @returns ApolloClient object */ export const createApolloClient = () => { - const uri = gqlUrlPath; const link = new HttpLink({ - uri: uri, + uri: process.env.NEXT_PUBLIC_GQL_URI, }); return new ApolloClient({ link, diff --git a/src/depvis-next/helpers/GraphHelper.ts b/src/depvis-next/helpers/GraphHelper.ts index 00296ae..1445f29 100644 --- a/src/depvis-next/helpers/GraphHelper.ts +++ b/src/depvis-next/helpers/GraphHelper.ts @@ -1,11 +1,16 @@ import { gql } from '@apollo/client'; -export const formatData = (data) => { +/** + * Function responsible for transforming the data to format that can be visualized + * @param components List of components + * @returns Object containing list of nodes and links + */ +export const formatData = (components) => { const nodes = []; let links = []; - if (!data || !data.projects || !data.projects[0].allVulnerableComponents) return { nodes, links }; - const componentsCount = data.projects[0].allVulnerableComponents.length / 20; - data.projects[0].allVulnerableComponents.forEach((c) => { + if (!components) return { nodes, links }; + const componentsCount = components.length / 20; + components.forEach((c) => { nodes.push({ id: c.purl, name: c.name, @@ -89,6 +94,29 @@ export const getAllComponentsQuery = gql` } } } + allComponents { + id + name + version + __typename + purl + dependsOnCount + dependsOn { + purl + } + vulnerabilities { + __typename + id + cve + name + description + cvssScore + references { + __typename + url + } + } + } } } `; diff --git a/src/depvis-next/pages/api/import/index.ts b/src/depvis-next/pages/api/import/index.ts index 7fc1113..216d9e3 100644 --- a/src/depvis-next/pages/api/import/index.ts +++ b/src/depvis-next/pages/api/import/index.ts @@ -75,7 +75,7 @@ async function parseXml(inputXml: string) { //Clear vuln queue emptyQueue(GetVulnQueue); - const job = await ImportQueue.add({ bom: xmlParsed.bom }); - return { jobId: job.id }; - //return await ImportSbom(xmlParsed.bom); + // const job = await ImportQueue.add({ bom: xmlParsed.bom }); + // return { jobId: job.id }; + return await ImportSbom(xmlParsed.bom); } diff --git a/src/depvis-next/queues/ImportQueue.ts b/src/depvis-next/queues/ImportQueue.ts index b798e1e..f7e4075 100644 --- a/src/depvis-next/queues/ImportQueue.ts +++ b/src/depvis-next/queues/ImportQueue.ts @@ -16,10 +16,7 @@ console.log(ImportQueueOptions); const ImportQueue = new Bull(ImportQueueName, ImportQueueOptions); ImportQueue.process(async (job) => { - try { - const res = ImportSbom(job.data.bom); - return res; - } catch (e) { - console.error(e); - } + console.log('Starting processing job %s', job.id); + const res = ImportSbom(job.data.bom); + return res; }); From 47a9e44181694662ab0e1c362aa622475180036c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Fri, 3 Feb 2023 09:42:22 +0100 Subject: [PATCH 04/33] New gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index cd62e76..5561297 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,7 @@ web_modules/ .env.test.local .env.production.local .env.local +.env* # parcel-bundler cache (https://parceljs.org/) .cache @@ -134,4 +135,5 @@ dist /runtime -*no_git* \ No newline at end of file +*no_git* + From d960c0f0aec524194b57e7f3997bb332d1d4b0b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Mon, 6 Feb 2023 21:30:26 +0100 Subject: [PATCH 05/33] WIP: Better import --- docker-compose.yml | 31 +- .../components/Import/ImportForm.tsx | 74 ++-- src/depvis-next/helpers/QueueHelper.ts | 24 +- src/depvis-next/package-lock.json | 402 ++++++++++-------- src/depvis-next/package.json | 2 +- src/depvis-next/pages/api/import/index.ts | 62 ++- src/depvis-next/queues/GetVulnQueue.ts | 32 +- src/depvis-next/queues/ImportQueue.ts | 32 +- 8 files changed, 397 insertions(+), 262 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8993394..6c1b37a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: - ./runtime/plugins:/var/lib/neo4j/plugins restart: unless-stopped environment: - - NEO4J_AUTH=neo4j/test + - NEO4J_AUTH=neo4j/${NEO4J_PASSWORD} - NEO4J_apoc_export_file_enabled=true - NEO4J_apoc_import_file_enabled=true - NEO4J_apoc_import_file_use__neo4j__config=true @@ -20,13 +20,28 @@ services: - NEO4J_dbms_security_procedures_unrestricted=apoc.*,algo.* redis-cache: - image: redis:alpine - restart: unless-stopped - ports: - - 6379:6379 - command: redis-server --save 20 1 --loglevel notice + image: redis:latest + command: redis-server --requirepass ${REDIS_PASSWORD} volumes: - cache:/data + ports: + - 6379:6379 + links: + - redis-commander + + redis-commander: + image: rediscommander/redis-commander:latest + restart: always + environment: + REDIS_HOSTS: redis-cache + REDIS_HOST: redis-cache + REDIS_PORT: redis-cache:6379 + REDIS_PASSWORD: ${REDIS_PASSWORD} + HTTP_USER: root + HTTP_PASSWORD: root + ports: + - 8081:8081 + depvis-next: depends_on: - neo4j @@ -36,6 +51,10 @@ services: ports: - 80:3000 restart: always + environment: + NEO4J_PASSWORD: ${NEO4J_PASSWORD} + REDIS_PASSWORD: ${REDIS_PASSWORD} + NEXT_PUBLIC_SONATYPE_OSS_AUTH: ${SONATYPE_OSS_AUTH} volumes: cache: driver: local diff --git a/src/depvis-next/components/Import/ImportForm.tsx b/src/depvis-next/components/Import/ImportForm.tsx index 3c3085e..a9fb779 100644 --- a/src/depvis-next/components/Import/ImportForm.tsx +++ b/src/depvis-next/components/Import/ImportForm.tsx @@ -1,15 +1,17 @@ -import { useState } from "react"; -import { Alert, Button, Container, Form } from "react-bootstrap"; -import { ImportResult } from "./ImportResult"; +import { useState } from 'react'; +import { Alert, Button, Container, Form } from 'react-bootstrap'; +import { ImportResult } from './ImportResult'; const allowedExtensionsRegex = /(\.json|\.xml)$/i; const ImportForm = () => { - const [file, setFile] = useState(""); - const [preview, setPreview] = useState(""); + const [file, setFile] = useState(''); + const [preview, setPreview] = useState(''); const [validated, setValidated] = useState(false); const [isSubmitted, setIsSubmitted] = useState(false); const [jobId, setJobId] = useState(null); + const [projectName, setProjectName] = useState(''); + const [projectVersion, setProjectVersion] = useState('1.0.0'); const handleFiles = (e: any) => { const files = e.target.files; @@ -17,8 +19,8 @@ const ImportForm = () => { const file = files[0]; console.log(file); if (!allowedExtensionsRegex.exec(file.name)) { - alert("This extension is not allowed!"); - setFile(""); + alert('This extension is not allowed!'); + setFile(''); return; } setFile(file); @@ -34,10 +36,14 @@ const ImportForm = () => { setValidated(true); console.log({ file: typeof file }); - const res = await fetch("/api/import", { - body: await file.text(), - headers: { "Content-Type": "application/xml" }, - method: "POST", + const formData = new FormData(); + formData.append('projectName', projectName); + formData.append('projectVersion', projectVersion); + formData.append('sbom', await file.text()); + const res = await fetch('/api/import', { + body: formData, + // headers: { 'Content-Type': 'application/xml' }, + method: 'POST', }); const json = await res.json(); setJobId(json.jobId); @@ -54,11 +60,35 @@ const ImportForm = () => { ) : ( - - All data currently stored in DB will be overwritten. - + All data currently stored in DB will be overwritten.
+ + Project name + { + setProjectName(e.target.value); + }} + value={projectName} + > + Please select any XML / JSON file with SBOM. + + + Project version + { + setProjectVersion(e.target.value); + }} + value={projectVersion} + > + Please select any XML / JSON file with SBOM. + SBOM File { handleFiles(e); }} > - - Please select any XML / JSON file with SBOM. - + Please select any XML / JSON file with SBOM. - -
diff --git a/src/depvis-next/helpers/QueueHelper.ts b/src/depvis-next/helpers/QueueHelper.ts index 2878d0c..b2457e2 100644 --- a/src/depvis-next/helpers/QueueHelper.ts +++ b/src/depvis-next/helpers/QueueHelper.ts @@ -1,6 +1,8 @@ +import { Queue } from 'bullmq'; + const getKeys = async (q) => { const multi = q.multi(); - multi.keys("*"); + multi.keys('*'); const keys = await multi.exec(); return keys[0][1]; }; @@ -16,9 +18,19 @@ const deleteKeys = async (q, keys) => { await multi.exec(); }; -export const emptyQueue = async (q) => { - const keys = await getKeys(q); - const queueKeys = filterQueueKeys(q, keys); - await deleteKeys(q, queueKeys); - console.log(`Queue ${q.name} is empty`); +export const emptyQueue = async (q: Queue) => { + q.drain(); + // const keys = await getKeys(q); + // const queueKeys = filterQueueKeys(q, keys); + // await deleteKeys(q, queueKeys); + // console.log(`Queue ${q.name} is empty`); +}; + +export const defaultBullConfig = { + connection: { + host: process.env.REDIS_HOST, + port: process.env.REDIS_PORT, + password: process.env.REDIS_PASSWORD, + enableOfflineQueue: false, + }, }; diff --git a/src/depvis-next/package-lock.json b/src/depvis-next/package-lock.json index 1fc6b57..b6322ad 100644 --- a/src/depvis-next/package-lock.json +++ b/src/depvis-next/package-lock.json @@ -12,7 +12,7 @@ "@neo4j/graphql": "^3.8.0", "apollo-server-micro": "^3.10.2", "bootstrap": "^5.2.1", - "bull": "^4.10.1", + "bullmq": "^3.6.2", "d3-force": "^3.0.0", "fast-xml-parser": "^4.0.10", "graphql": "^16.6.0", @@ -465,10 +465,70 @@ "resolved": "https://registry.npmjs.org/@josephg/resolvable/-/resolvable-1.0.1.tgz", "integrity": "sha512-CtzORUwWTTOTqfVtHaKRJ0I1kNQd1bpn3sUh8I3nJDVY+5/M/Oe1DnEWzPQvqq/xPIIkzzzIP7mfCoAjFRvDhg==" }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.0.tgz", + "integrity": "sha512-5qpnNHUyyEj9H3sm/4Um/bnx1lrQGhe8iqry/1d+cQYCRd/gzYA0YLeq0ezlk4hKx4vO+dsEsNyeowqRqslwQA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.0.tgz", + "integrity": "sha512-ZphTFFd6SFweNAMKD+QJCrWpgkjf4qBuHltiMkKkD6FFrB3NOTRVmetAGTkJ57pa+s6J0yCH06LujWB9rZe94g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.0.tgz", + "integrity": "sha512-ztKVV1dO/sSZyGse0PBCq3Pk1PkYjsA/dsEWE7lfrGoAK3i9HpS2o7XjGQ7V4va6nX+xPPOiuYpQwa4Bi6vlww==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.0.tgz", + "integrity": "sha512-NEX6hdSvP4BmVyegaIbrGxvHzHvTzzsPaxXCsUt0mbLbPpEftsvNwaEVKOowXnLoeuGeD4MaqSwL3BUK2elsUA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.0.tgz", + "integrity": "sha512-9uvdAkZMOPCY7SPRxZLW8XGqBOVNVEhqlgffenN8shA1XR9FWVsSM13nr/oHtNgXg6iVyML7RwWPyqUeThlwxg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.2.0.tgz", - "integrity": "sha512-XrC0JzsqQSvOyM3t04FMLO6z5gCuhPE6k4FXuLK5xf52ZbdvcFe1yBmo7meCew9B8G2f0T9iu9t3kfTYRYROgA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.0.tgz", + "integrity": "sha512-Wg0+9615kHKlr9iLVcG5I+/CHnf6w3x5UADRv8Ad16yA0Bu5l9eVOROjV7aHPG6uC8ZPFIVVaoSjDChD+Y0pzg==", "cpu": [ "x64" ], @@ -1468,8 +1528,7 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "node_modules/base64-js": { "version": "1.5.1", @@ -1561,31 +1620,56 @@ "ieee754": "^1.2.1" } }, - "node_modules/bull": { - "version": "4.10.2", - "resolved": "https://registry.npmjs.org/bull/-/bull-4.10.2.tgz", - "integrity": "sha512-xa65xtWjQsLqYU/eNaXxq9VRG8xd6qNsQEjR7yjYuae05xKrzbVMVj2QgrYsTMmSs/vsqJjHqHSRRiW1+IkGXQ==", + "node_modules/bullmq": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.6.2.tgz", + "integrity": "sha512-Nea7OWtdg6V47HP2vp0V2RUhG02KHCJE+VzbwpOFRONHS+1M6IehAVjEhfN3N036BNoPU776sMKCVxaegG0reA==", "dependencies": { - "cron-parser": "^4.2.1", - "debuglog": "^1.0.0", - "get-port": "^5.1.1", - "ioredis": "^5.0.0", + "cron-parser": "^4.6.0", + "glob": "^8.0.3", + "ioredis": "^5.3.0", "lodash": "^4.17.21", - "msgpackr": "^1.5.2", - "p-timeout": "^3.2.0", - "semver": "^7.3.2", - "uuid": "^8.3.0" + "msgpackr": "^1.6.2", + "semver": "^7.3.7", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, + "node_modules/bullmq/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/bullmq/node_modules/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" }, "engines": { "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/bull/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" + "node_modules/bullmq/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" } }, "node_modules/bytes": { @@ -1729,11 +1813,11 @@ } }, "node_modules/cron-parser": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.7.0.tgz", - "integrity": "sha512-BdAELR+MCT2ZWsIBhZKDuUqIUCBjHHulPJnm53OfdRLA4EWBjva3R+KM5NeidJuGsNXdEcZkjC7SCnkW5rAFSA==", + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.7.1.tgz", + "integrity": "sha512-WguFaoQ0hQ61SgsCZLHUcNbAvlK0lypKXu62ARguefYmjzaOXIVRNrAmyXzabTwUn4sQvQLkk6bjH+ipGfw8bA==", "dependencies": { - "luxon": "^3.1.0" + "luxon": "^3.2.1" }, "engines": { "node": ">=12.0.0" @@ -2000,14 +2084,6 @@ } } }, - "node_modules/debuglog": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", - "integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==", - "engines": { - "node": "*" - } - }, "node_modules/deep-equal": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.1.0.tgz", @@ -2847,8 +2923,7 @@ "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "node_modules/function-bind": { "version": "1.1.1", @@ -2894,17 +2969,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-port": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", - "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/get-symbol-description": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", @@ -3229,7 +3293,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -3271,14 +3334,14 @@ } }, "node_modules/ioredis": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.4.tgz", - "integrity": "sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.0.tgz", + "integrity": "sha512-Id9jKHhsILuIZpHc61QkagfVdUj2Rag5GzG1TGEvRNeM7dtTOjICgjC+tvqYxi//PuX2wjQ+Xjva2ONBuf92Pw==", "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", @@ -3855,32 +3918,32 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/msgpackr": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.8.1.tgz", - "integrity": "sha512-05fT4J8ZqjYlR4QcRDIhLCYKUOHXk7C/xa62GzMKj74l3up9k2QZ3LgFc6qWdsPHl91QA2WLWqWc8b8t7GLNNw==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.8.3.tgz", + "integrity": "sha512-m2JefwcKNzoHYXkH/5jzHRxAw7XLWsAdvu0FOJ+OLwwozwOV/J6UA62iLkfIMbg7G8+dIuRwgg6oz+QoQ4YkoA==", "optionalDependencies": { - "msgpackr-extract": "^2.2.0" + "msgpackr-extract": "^3.0.0" } }, "node_modules/msgpackr-extract": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-2.2.0.tgz", - "integrity": "sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.0.tgz", + "integrity": "sha512-oy6KCk1+X4Bn5m6Ycq5N1EWl9npqG/cLrE8ga8NX7ZqfqYUUBS08beCQaGq80fjbKBySur0E6x//yZjzNJDt3A==", "hasInstallScript": true, "optional": true, "dependencies": { - "node-gyp-build-optional-packages": "5.0.3" + "node-gyp-build-optional-packages": "5.0.7" }, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" }, "optionalDependencies": { - "@msgpackr-extract/msgpackr-extract-darwin-arm64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-darwin-x64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-linux-arm": "2.2.0", - "@msgpackr-extract/msgpackr-extract-linux-arm64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-linux-x64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-win32-x64": "2.2.0" + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.0" } }, "node_modules/multiparty": { @@ -4024,9 +4087,9 @@ } }, "node_modules/node-gyp-build-optional-packages": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz", - "integrity": "sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", "optional": true, "bin": { "node-gyp-build-optional-packages": "bin.js", @@ -4155,7 +4218,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "dependencies": { "wrappy": "1" } @@ -4186,14 +4248,6 @@ "node": ">= 0.8.0" } }, - "node_modules/p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", - "engines": { - "node": ">=4" - } - }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -4224,17 +4278,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "dependencies": { - "p-finally": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/packageurl-js": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/packageurl-js/-/packageurl-js-1.0.0.tgz", @@ -5338,8 +5381,7 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/xss": { "version": "1.0.14", @@ -5707,10 +5749,40 @@ "resolved": "https://registry.npmjs.org/@josephg/resolvable/-/resolvable-1.0.1.tgz", "integrity": "sha512-CtzORUwWTTOTqfVtHaKRJ0I1kNQd1bpn3sUh8I3nJDVY+5/M/Oe1DnEWzPQvqq/xPIIkzzzIP7mfCoAjFRvDhg==" }, + "@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.0.tgz", + "integrity": "sha512-5qpnNHUyyEj9H3sm/4Um/bnx1lrQGhe8iqry/1d+cQYCRd/gzYA0YLeq0ezlk4hKx4vO+dsEsNyeowqRqslwQA==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.0.tgz", + "integrity": "sha512-ZphTFFd6SFweNAMKD+QJCrWpgkjf4qBuHltiMkKkD6FFrB3NOTRVmetAGTkJ57pa+s6J0yCH06LujWB9rZe94g==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.0.tgz", + "integrity": "sha512-ztKVV1dO/sSZyGse0PBCq3Pk1PkYjsA/dsEWE7lfrGoAK3i9HpS2o7XjGQ7V4va6nX+xPPOiuYpQwa4Bi6vlww==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.0.tgz", + "integrity": "sha512-NEX6hdSvP4BmVyegaIbrGxvHzHvTzzsPaxXCsUt0mbLbPpEftsvNwaEVKOowXnLoeuGeD4MaqSwL3BUK2elsUA==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.0.tgz", + "integrity": "sha512-9uvdAkZMOPCY7SPRxZLW8XGqBOVNVEhqlgffenN8shA1XR9FWVsSM13nr/oHtNgXg6iVyML7RwWPyqUeThlwxg==", + "optional": true + }, "@msgpackr-extract/msgpackr-extract-win32-x64": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.2.0.tgz", - "integrity": "sha512-XrC0JzsqQSvOyM3t04FMLO6z5gCuhPE6k4FXuLK5xf52ZbdvcFe1yBmo7meCew9B8G2f0T9iu9t3kfTYRYROgA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.0.tgz", + "integrity": "sha512-Wg0+9615kHKlr9iLVcG5I+/CHnf6w3x5UADRv8Ad16yA0Bu5l9eVOROjV7aHPG6uC8ZPFIVVaoSjDChD+Y0pzg==", "optional": true }, "@neo4j/cypher-builder": { @@ -6399,8 +6471,7 @@ "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "base64-js": { "version": "1.5.1", @@ -6445,26 +6516,48 @@ "ieee754": "^1.2.1" } }, - "bull": { - "version": "4.10.2", - "resolved": "https://registry.npmjs.org/bull/-/bull-4.10.2.tgz", - "integrity": "sha512-xa65xtWjQsLqYU/eNaXxq9VRG8xd6qNsQEjR7yjYuae05xKrzbVMVj2QgrYsTMmSs/vsqJjHqHSRRiW1+IkGXQ==", + "bullmq": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.6.2.tgz", + "integrity": "sha512-Nea7OWtdg6V47HP2vp0V2RUhG02KHCJE+VzbwpOFRONHS+1M6IehAVjEhfN3N036BNoPU776sMKCVxaegG0reA==", "requires": { - "cron-parser": "^4.2.1", - "debuglog": "^1.0.0", - "get-port": "^5.1.1", - "ioredis": "^5.0.0", + "cron-parser": "^4.6.0", + "glob": "^8.0.3", + "ioredis": "^5.3.0", "lodash": "^4.17.21", - "msgpackr": "^1.5.2", - "p-timeout": "^3.2.0", - "semver": "^7.3.2", - "uuid": "^8.3.0" + "msgpackr": "^1.6.2", + "semver": "^7.3.7", + "tslib": "^2.0.0", + "uuid": "^9.0.0" }, "dependencies": { - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, + "glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + } + }, + "minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "requires": { + "brace-expansion": "^2.0.1" + } } } }, @@ -6564,11 +6657,11 @@ "dev": true }, "cron-parser": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.7.0.tgz", - "integrity": "sha512-BdAELR+MCT2ZWsIBhZKDuUqIUCBjHHulPJnm53OfdRLA4EWBjva3R+KM5NeidJuGsNXdEcZkjC7SCnkW5rAFSA==", + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.7.1.tgz", + "integrity": "sha512-WguFaoQ0hQ61SgsCZLHUcNbAvlK0lypKXu62ARguefYmjzaOXIVRNrAmyXzabTwUn4sQvQLkk6bjH+ipGfw8bA==", "requires": { - "luxon": "^3.1.0" + "luxon": "^3.2.1" } }, "cross-spawn": { @@ -6764,11 +6857,6 @@ "ms": "2.1.2" } }, - "debuglog": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", - "integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==" - }, "deep-equal": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.1.0.tgz", @@ -7415,8 +7503,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "function-bind": { "version": "1.1.1", @@ -7450,11 +7537,6 @@ "has-symbols": "^1.0.3" } }, - "get-port": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", - "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==" - }, "get-symbol-description": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", @@ -7671,7 +7753,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -7707,14 +7788,14 @@ } }, "ioredis": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.4.tgz", - "integrity": "sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.0.tgz", + "integrity": "sha512-Id9jKHhsILuIZpHc61QkagfVdUj2Rag5GzG1TGEvRNeM7dtTOjICgjC+tvqYxi//PuX2wjQ+Xjva2ONBuf92Pw==", "requires": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", @@ -8123,26 +8204,26 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "msgpackr": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.8.1.tgz", - "integrity": "sha512-05fT4J8ZqjYlR4QcRDIhLCYKUOHXk7C/xa62GzMKj74l3up9k2QZ3LgFc6qWdsPHl91QA2WLWqWc8b8t7GLNNw==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.8.3.tgz", + "integrity": "sha512-m2JefwcKNzoHYXkH/5jzHRxAw7XLWsAdvu0FOJ+OLwwozwOV/J6UA62iLkfIMbg7G8+dIuRwgg6oz+QoQ4YkoA==", "requires": { - "msgpackr-extract": "^2.2.0" + "msgpackr-extract": "^3.0.0" } }, "msgpackr-extract": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-2.2.0.tgz", - "integrity": "sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.0.tgz", + "integrity": "sha512-oy6KCk1+X4Bn5m6Ycq5N1EWl9npqG/cLrE8ga8NX7ZqfqYUUBS08beCQaGq80fjbKBySur0E6x//yZjzNJDt3A==", "optional": true, "requires": { - "@msgpackr-extract/msgpackr-extract-darwin-arm64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-darwin-x64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-linux-arm": "2.2.0", - "@msgpackr-extract/msgpackr-extract-linux-arm64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-linux-x64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-win32-x64": "2.2.0", - "node-gyp-build-optional-packages": "5.0.3" + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.0", + "node-gyp-build-optional-packages": "5.0.7" } }, "multiparty": { @@ -8240,9 +8321,9 @@ } }, "node-gyp-build-optional-packages": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz", - "integrity": "sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", "optional": true }, "object-assign": { @@ -8327,7 +8408,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "requires": { "wrappy": "1" } @@ -8355,11 +8435,6 @@ "word-wrap": "^1.2.3" } }, - "p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==" - }, "p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -8378,14 +8453,6 @@ "p-limit": "^3.0.2" } }, - "p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "requires": { - "p-finally": "^1.0.0" - } - }, "packageurl-js": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/packageurl-js/-/packageurl-js-1.0.0.tgz", @@ -9158,8 +9225,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "xss": { "version": "1.0.14", diff --git a/src/depvis-next/package.json b/src/depvis-next/package.json index 3e3126f..b37e8a1 100644 --- a/src/depvis-next/package.json +++ b/src/depvis-next/package.json @@ -13,7 +13,7 @@ "@neo4j/graphql": "^3.8.0", "apollo-server-micro": "^3.10.2", "bootstrap": "^5.2.1", - "bull": "^4.10.1", + "bullmq": "^3.6.2", "d3-force": "^3.0.0", "fast-xml-parser": "^4.0.10", "graphql": "^16.6.0", diff --git a/src/depvis-next/pages/api/import/index.ts b/src/depvis-next/pages/api/import/index.ts index 7fc1113..5800631 100644 --- a/src/depvis-next/pages/api/import/index.ts +++ b/src/depvis-next/pages/api/import/index.ts @@ -1,7 +1,7 @@ -import Bull from 'bull'; +import { Queue } from 'bullmq'; import { XMLParser } from 'fast-xml-parser'; import { ImportSbom } from '../../../helpers/ImportSbomHelper'; -import { emptyQueue } from '../../../helpers/QueueHelper'; +import { defaultBullConfig, emptyQueue } from '../../../helpers/QueueHelper'; import { GetVulnQueueName } from '../../../queues/GetVulnQueue'; import { ImportQueueName } from '../../../queues/ImportQueue'; @@ -20,23 +20,45 @@ const XMLParserOptions = { }; //Bull queue -const ImportQueue = new Bull(ImportQueueName); -const GetVulnQueue = new Bull(GetVulnQueueName); +const ImportQueue = new Queue(ImportQueueName, defaultBullConfig); +const GetVulnQueue = new Queue(GetVulnQueueName, defaultBullConfig); + type ImportResult = { isError: boolean; errorMessage?: string; jobId?: string; + sbom?: any; +}; + +type ImportRequestBody = { + projectName: string; + projectVersion: string; + sbom: string; }; export default async function handler(req, res) { try { - if (req.headers['content-type'] !== 'application/xml') { - res.status(500).json({ error: "Content-type must be 'application/xml'" }); - return; + // if (req.headers['content-type'] !== 'application/xml') { + // res.status(500).json({ error: "Content-type must be 'application/xml'" }); + // return; + // } + const body = req.body as ImportRequestBody; + console.log('Recieved body: %s', body.projectName); + const result = await parseXml(body.sbom); + if (result.isError) { + console.log('Import failed with %s', result.errorMessage); + return res.status(400).json(result); } - //TODO: check if XML is even valid - const result = await parseXml(req.body); - return res.status(200).json(result); + + //Clear vuln queue + emptyQueue(GetVulnQueue); + const job = await ImportQueue.add(body.projectName, { + bom: result.sbom, + projectName: body.projectName, + projectVersion: body.projectVersion, + }); + const response: ImportResult = { jobId: job.id, isError: false }; + return res.status(200).json(response); } catch (err) { console.error(err); return res.status(500).json({ error: 'failed to load data', content: err }); @@ -45,37 +67,31 @@ export default async function handler(req, res) { // Function validates that object contains required properties function validateSbomXml(parsedXml): ImportResult { - const bom = parsedXml.bom; - if (!bom) + const sbom = parsedXml.bom; + if (!sbom) return { isError: true, errorMessage: "Validation failed - Missing 'bom' parameter in the file.", }; - if (!bom.metadata) + if (!sbom.metadata) return { isError: true, errorMessage: "Validation failed - Missing 'metadata' parameter in the file.", }; - if (!bom.components) + if (!sbom.components) return { isError: true, errorMessage: "Validation failed - Missing 'components' parameter in the file.", }; - return { isError: false }; + return { isError: false, sbom: sbom }; } // Function takes XML in plain text and transforms it into object async function parseXml(inputXml: string) { const parser = new XMLParser(XMLParserOptions); + console.log('Input XML: %s', inputXml); const xmlParsed = parser.parse(inputXml); - const validateResult = validateSbomXml(xmlParsed); - if (validateResult.isError) return validateResult; - - //Clear vuln queue - emptyQueue(GetVulnQueue); - const job = await ImportQueue.add({ bom: xmlParsed.bom }); - return { jobId: job.id }; - //return await ImportSbom(xmlParsed.bom); + return validateSbomXml(xmlParsed); } diff --git a/src/depvis-next/queues/GetVulnQueue.ts b/src/depvis-next/queues/GetVulnQueue.ts index d85cbe2..e39ab81 100644 --- a/src/depvis-next/queues/GetVulnQueue.ts +++ b/src/depvis-next/queues/GetVulnQueue.ts @@ -1,21 +1,25 @@ -import Bull from 'bull'; +import { Job, Worker } from 'bullmq'; +import { CreateUpdateVulnerability } from '../helpers/DbDataHelper'; +import { defaultBullConfig } from '../helpers/QueueHelper'; import { VulnFetcherHandler } from '../vulnerability-mgmt/VulnFetcherHandler'; export const GetVulnQueueName = 'get-vuln-queue'; -export const GetVulnQueueOptions: Bull.QueueOptions = { - redis: { - port: parseInt(process.env.REDIS_PORT, 10), - host: process.env.REDIS_HOST, - password: process.env.REDIS_PASSWORD, +const worker = new Worker( + GetVulnQueueName, + async (job) => { + console.log(`Processing Vuln ${job.data.purl}`); + const vulnerabilityList = await VulnFetcherHandler(job.data.purl); + if (vulnerabilityList.length == 0) return; + return await CreateUpdateVulnerability(job.data.purl, vulnerabilityList); }, - limiter: { max: 3, duration: 1000 }, -}; + defaultBullConfig +); -const GetVulnQueue = new Bull(GetVulnQueueName, GetVulnQueueOptions); +worker.on('failed', (job, error) => { + console.log('Job %d failed with error %s', job.id, error.message); + console.error(error); +}); -GetVulnQueue.process(async (job) => { - console.log(`Processing Vuln ${job.data.purl}`); - const vulnerabilityList = await VulnFetcherHandler(job.data.purl); - // if (vulnerabilitiesList.length == 0) return; - //return await CreateUpdateVulnerability(job.data.purl, vulnerabilitiesList); +worker.on('completed', (job) => { + console.log('Job %d completed successfully!', job.id); }); diff --git a/src/depvis-next/queues/ImportQueue.ts b/src/depvis-next/queues/ImportQueue.ts index b798e1e..754f7dd 100644 --- a/src/depvis-next/queues/ImportQueue.ts +++ b/src/depvis-next/queues/ImportQueue.ts @@ -1,25 +1,23 @@ -import Bull from 'bull'; +import { Worker } from 'bullmq'; import { ImportSbom } from '../helpers/ImportSbomHelper'; +import { defaultBullConfig } from '../helpers/QueueHelper'; export const ImportQueueName = 'import-queue'; -export const ImportQueueOptions: Bull.QueueOptions = { - redis: { - port: parseInt(process.env.REDIS_PORT, 10), - host: process.env.REDIS_HOST, - password: process.env.REDIS_PASSWORD, +const worker = new Worker( + ImportQueueName, + async (job) => { + const res = ImportSbom(job.data.bom); + return res; }, - limiter: { max: 2, duration: 3600 }, -}; + defaultBullConfig +); -console.log(ImportQueueOptions); -const ImportQueue = new Bull(ImportQueueName, ImportQueueOptions); +worker.on('failed', (job, error) => { + console.log('Job %d failed with error %s', job.id, error.message); + console.error(error); +}); -ImportQueue.process(async (job) => { - try { - const res = ImportSbom(job.data.bom); - return res; - } catch (e) { - console.error(e); - } +worker.on('completed', (job) => { + console.log('Job %d completed successfully!', job.id); }); From 12c242ca0ea2bb93a0ed34a2aadb108f44f8c58e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Wed, 8 Feb 2023 18:15:27 +0100 Subject: [PATCH 06/33] remove file --- .gitignore | 2 +- src/depvis-next/.env.production | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) delete mode 100644 src/depvis-next/.env.production diff --git a/.gitignore b/.gitignore index cd62e76..e28a0e2 100644 --- a/.gitignore +++ b/.gitignore @@ -73,7 +73,7 @@ web_modules/ .yarn-integrity # dotenv environment variable files -.env +.env* .env.development.local .env.test.local .env.production.local diff --git a/src/depvis-next/.env.production b/src/depvis-next/.env.production deleted file mode 100644 index 094f4e4..0000000 --- a/src/depvis-next/.env.production +++ /dev/null @@ -1,13 +0,0 @@ -# Sample .env file that can be used for deployment via docker -# Do not forget to change passwords to correspond with docker-compose file -NEO4J_USER=neo4j -NEO4J_PASSWORD= -NEO4J_HOST=neo4j://neo4j:7687 - -NEXT_PUBLIC_SONATYPE_OSS_AUTH= - -GQL_ALLOW_DEV_TOOLS=true - -REDIS_HOST=redis-cache -REDIS_PORT=6379 -REDIS_PASSWORD= \ No newline at end of file From dd7a326010613d67ee5704ae5d9e521aec4923e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Wed, 8 Feb 2023 21:01:11 +0100 Subject: [PATCH 07/33] Add CORS support --- src/depvis-next/helpers/ApolloClientHelper.ts | 7 ++++++- src/depvis-next/pages/api/graphql.ts | 19 ++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/depvis-next/helpers/ApolloClientHelper.ts b/src/depvis-next/helpers/ApolloClientHelper.ts index 8078541..6b61be4 100644 --- a/src/depvis-next/helpers/ApolloClientHelper.ts +++ b/src/depvis-next/helpers/ApolloClientHelper.ts @@ -1,4 +1,5 @@ import { ApolloClient, HttpLink, InMemoryCache } from '@apollo/client'; +import urlJoin from 'url-join'; import { gqlUrlPath } from '../pages/api/graphql'; /** @@ -6,9 +7,13 @@ import { gqlUrlPath } from '../pages/api/graphql'; * @returns ApolloClient object */ export const createApolloClient = () => { - const uri = gqlUrlPath; + const uri = urlJoin(process.env.NEXT_PUBLIC_SERVER_URI || 'http://localhost:3000', gqlUrlPath); + console.log(`Creating GQL Client (connection to ${uri})`); const link = new HttpLink({ uri: uri, + fetchOptions: { + mode: 'cors', + }, }); return new ApolloClient({ link, diff --git a/src/depvis-next/pages/api/graphql.ts b/src/depvis-next/pages/api/graphql.ts index 79daa5c..ace55d9 100644 --- a/src/depvis-next/pages/api/graphql.ts +++ b/src/depvis-next/pages/api/graphql.ts @@ -25,7 +25,7 @@ const driver = neo4j.driver( neo4j.auth.basic(NEO4J_CONFIG.neo4jUsername, NEO4J_CONFIG.neo4jPassword) ); -export default async function handler(req, res) { +async function handler(req, res) { const neoSchema = new Neo4jGraphQL({ typeDefs, driver }); const apolloServer = new ApolloServer({ schema: await neoSchema.getSchema(), @@ -38,6 +38,23 @@ export default async function handler(req, res) { })(req, res); } +const allowCors = (fn) => async (req, res) => { + res.setHeader('Access-Control-Allow-Credentials', true); + const origin = process.env.CORS_ORIGIN || 'http://localhost:3000'; + res.setHeader('Access-Control-Allow-Origin', origin); + res.setHeader('Access-Control-Allow-Methods', 'GET,OPTIONS,PATCH,DELETE,POST,PUT'); + res.setHeader( + 'Access-Control-Allow-Headers', + 'X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, Content-Type, Date, X-Api-Version' + ); + if (req.method === 'OPTIONS') { + res.status(200).end(); + return; + } + return await fn(req, res); +}; + +export default allowCors(handler); export const config = { api: { bodyParser: false, From 3a4dcaa4a3c952107f5c3d37653a8ea473dbfb29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Wed, 8 Feb 2023 21:01:44 +0100 Subject: [PATCH 08/33] Iproved import flow --- .../components/Details/ComponentDetails.tsx | 1 + .../components/Import/ImportForm.tsx | 15 ++++---- src/depvis-next/helpers/QueueHelper.ts | 22 ------------ src/depvis-next/helpers/WorkspaceHelper.ts | 35 ++++++++++--------- src/depvis-next/pages/api/import/index.ts | 1 - src/depvis-next/pages/api/import/status.ts | 11 +++--- src/depvis-next/pages/api/vuln/[...purl].ts | 16 --------- 7 files changed, 34 insertions(+), 67 deletions(-) delete mode 100644 src/depvis-next/pages/api/vuln/[...purl].ts diff --git a/src/depvis-next/components/Details/ComponentDetails.tsx b/src/depvis-next/components/Details/ComponentDetails.tsx index 836870d..07df282 100644 --- a/src/depvis-next/components/Details/ComponentDetails.tsx +++ b/src/depvis-next/components/Details/ComponentDetails.tsx @@ -27,6 +27,7 @@ const ComponentDetails = (props) => { const renderLink = () => { const link = GetComponentRepositoryURL(data.components[0].purl); + if (!link) return; return ( {link} diff --git a/src/depvis-next/components/Import/ImportForm.tsx b/src/depvis-next/components/Import/ImportForm.tsx index a9fb779..76bdea7 100644 --- a/src/depvis-next/components/Import/ImportForm.tsx +++ b/src/depvis-next/components/Import/ImportForm.tsx @@ -35,14 +35,15 @@ const ImportForm = () => { } setValidated(true); - console.log({ file: typeof file }); - const formData = new FormData(); - formData.append('projectName', projectName); - formData.append('projectVersion', projectVersion); - formData.append('sbom', await file.text()); + const body = { + projectName: projectName, + projectVersion: projectVersion, + sbom: await file.text(), + }; + console.log(body); const res = await fetch('/api/import', { - body: formData, - // headers: { 'Content-Type': 'application/xml' }, + body: await JSON.stringify(body), + headers: { 'Content-Type': 'application/json' }, method: 'POST', }); const json = await res.json(); diff --git a/src/depvis-next/helpers/QueueHelper.ts b/src/depvis-next/helpers/QueueHelper.ts index b2457e2..fadaf3e 100644 --- a/src/depvis-next/helpers/QueueHelper.ts +++ b/src/depvis-next/helpers/QueueHelper.ts @@ -1,29 +1,7 @@ import { Queue } from 'bullmq'; -const getKeys = async (q) => { - const multi = q.multi(); - multi.keys('*'); - const keys = await multi.exec(); - return keys[0][1]; -}; - -const filterQueueKeys = (q, keys) => { - const prefix = `${q.keyPrefix}:${q.name}`; - return keys.filter((k) => k.includes(prefix)); -}; - -const deleteKeys = async (q, keys) => { - const multi = q.multi(); - keys.forEach((k) => multi.del(k)); - await multi.exec(); -}; - export const emptyQueue = async (q: Queue) => { q.drain(); - // const keys = await getKeys(q); - // const queueKeys = filterQueueKeys(q, keys); - // await deleteKeys(q, queueKeys); - // console.log(`Queue ${q.name} is empty`); }; export const defaultBullConfig = { diff --git a/src/depvis-next/helpers/WorkspaceHelper.ts b/src/depvis-next/helpers/WorkspaceHelper.ts index 78cc405..3455556 100644 --- a/src/depvis-next/helpers/WorkspaceHelper.ts +++ b/src/depvis-next/helpers/WorkspaceHelper.ts @@ -1,23 +1,26 @@ -import { PackageURL } from "packageurl-js"; +import { PackageURL } from 'packageurl-js'; import urlJoin from 'url-join'; - -const npmUrlBase = "https://www.npmjs.com/package/" -const nugetUrlBase = "https://www.nuget.org/packages/" -const pypiUrlBase = "https://pypi.org/project/" - +const npmUrlBase = 'https://www.npmjs.com/package/'; +const nugetUrlBase = 'https://www.nuget.org/packages/'; +const pypiUrlBase = 'https://pypi.org/project/'; export function GetComponentRepositoryURL(purl: string) { - const component = PackageURL.fromString(purl) + try { + const component = PackageURL.fromString(purl); switch (component.type) { - case "npm": - return urlJoin(npmUrlBase, component.name, component.version ? urlJoin("v", component.version) : "") - case "nuget": - return urlJoin(nugetUrlBase, component.name, component.version) - case "pypi": - return urlJoin(pypiUrlBase, component.name, component.version) - default: - return urlJoin("https://www.google.com/search?q=", purl) + case 'npm': + return urlJoin(npmUrlBase, component.name, component.version ? urlJoin('v', component.version) : ''); + case 'nuget': + return urlJoin(nugetUrlBase, component.name, component.version); + case 'pypi': + return urlJoin(pypiUrlBase, component.name, component.version); + default: + return urlJoin('https://www.google.com/search?q=', purl); } -} \ No newline at end of file + } catch { + console.log('Could not parse Package URL from %s', purl); + return; + } +} diff --git a/src/depvis-next/pages/api/import/index.ts b/src/depvis-next/pages/api/import/index.ts index 5800631..490af9d 100644 --- a/src/depvis-next/pages/api/import/index.ts +++ b/src/depvis-next/pages/api/import/index.ts @@ -90,7 +90,6 @@ function validateSbomXml(parsedXml): ImportResult { // Function takes XML in plain text and transforms it into object async function parseXml(inputXml: string) { const parser = new XMLParser(XMLParserOptions); - console.log('Input XML: %s', inputXml); const xmlParsed = parser.parse(inputXml); return validateSbomXml(xmlParsed); diff --git a/src/depvis-next/pages/api/import/status.ts b/src/depvis-next/pages/api/import/status.ts index 22633b1..70b5496 100644 --- a/src/depvis-next/pages/api/import/status.ts +++ b/src/depvis-next/pages/api/import/status.ts @@ -1,12 +1,13 @@ -import Bull from "bull"; -import { ImportQueueName } from "../../../queues/ImportQueue"; +import { Queue } from 'bullmq'; +import { defaultBullConfig } from '../../../helpers/QueueHelper'; +import { ImportQueueName } from '../../../queues/ImportQueue'; //Bull queue -const ImportQueue = new Bull(ImportQueueName); +const ImportQueue = new Queue(ImportQueueName, defaultBullConfig); export default async function handler(req, res) { - if (req.method !== "GET") return res.status(405); - if (!("id" in req.query && !isNaN(req.query.id))) + if (req.method !== 'GET') return res.status(405); + if (!('id' in req.query && !isNaN(req.query.id))) return res.status(400).json({ error: "'id' parameter is missing or its value is not a number", }); diff --git a/src/depvis-next/pages/api/vuln/[...purl].ts b/src/depvis-next/pages/api/vuln/[...purl].ts deleted file mode 100644 index 3ad4989..0000000 --- a/src/depvis-next/pages/api/vuln/[...purl].ts +++ /dev/null @@ -1,16 +0,0 @@ -import Bull from "bull"; -import { GetVulnQueueName } from "../../../queues/GetVulnQueue"; - -const VulnQueue = new Bull(GetVulnQueueName); - -export default async function handler(req, res) { - const { purl } = req.query; - const purl_j = purl.join("/"); - try { - const job = await VulnQueue.add({ purl: purl_j }); - return res.status(200).json({ jobId: job.id }); - } catch (error) { - console.log(error.message); - res.status(500).json({ error: error.message }); - } -} From de52799b4583416a1a2d739f4ff962d9816ff5de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Wed, 8 Feb 2023 21:23:01 +0100 Subject: [PATCH 09/33] ImportForm - define name and version --- src/depvis-next/helpers/ImportSbomHelper.ts | 7 ++++--- src/depvis-next/pages/api/import/index.ts | 6 +++--- src/depvis-next/queues/ImportQueue.ts | 10 +++++++++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/depvis-next/helpers/ImportSbomHelper.ts b/src/depvis-next/helpers/ImportSbomHelper.ts index 67a0d66..e86e11e 100644 --- a/src/depvis-next/helpers/ImportSbomHelper.ts +++ b/src/depvis-next/helpers/ImportSbomHelper.ts @@ -9,7 +9,7 @@ import { UpdateProjectDependencies, } from './DbDataHelper'; -export async function ImportSbom(bom: any) { +export async function ImportSbom(bom: any, projectName, projectVersion) { // Prepare main component if exists const mainComponentParsed = bom.metadata.component; const mainComponent: Component | undefined = mainComponentParsed @@ -25,10 +25,11 @@ export async function ImportSbom(bom: any) { // Prepare project let project: Project = { - name: bom.metadata.component.name, - version: bom.metadata.component.version || 'n/a', + name: projectName || bom.metadata.component.name, + version: projectVersion || bom.metadata.component.version || '1.0.0', date: bom.metadata.timestamp || '1970-01-01', }; + console.log(project); // Prepare dependencies let dependencies = GetDependencies(bom.dependencies.dependency); diff --git a/src/depvis-next/pages/api/import/index.ts b/src/depvis-next/pages/api/import/index.ts index 490af9d..c01b437 100644 --- a/src/depvis-next/pages/api/import/index.ts +++ b/src/depvis-next/pages/api/import/index.ts @@ -3,7 +3,7 @@ import { XMLParser } from 'fast-xml-parser'; import { ImportSbom } from '../../../helpers/ImportSbomHelper'; import { defaultBullConfig, emptyQueue } from '../../../helpers/QueueHelper'; import { GetVulnQueueName } from '../../../queues/GetVulnQueue'; -import { ImportQueueName } from '../../../queues/ImportQueue'; +import { ImportQueueName, ImportSbomJobData } from '../../../queues/ImportQueue'; export const config = { api: { @@ -53,10 +53,10 @@ export default async function handler(req, res) { //Clear vuln queue emptyQueue(GetVulnQueue); const job = await ImportQueue.add(body.projectName, { - bom: result.sbom, + sbom: result.sbom, projectName: body.projectName, projectVersion: body.projectVersion, - }); + } as ImportSbomJobData); const response: ImportResult = { jobId: job.id, isError: false }; return res.status(200).json(response); } catch (err) { diff --git a/src/depvis-next/queues/ImportQueue.ts b/src/depvis-next/queues/ImportQueue.ts index 754f7dd..e74785c 100644 --- a/src/depvis-next/queues/ImportQueue.ts +++ b/src/depvis-next/queues/ImportQueue.ts @@ -4,10 +4,18 @@ import { defaultBullConfig } from '../helpers/QueueHelper'; export const ImportQueueName = 'import-queue'; +export type ImportSbomJobData = { + projectName: string; + projectVersion: string; + sbom: string; +}; + const worker = new Worker( ImportQueueName, async (job) => { - const res = ImportSbom(job.data.bom); + const test = job.data; + console.log(test); + const res = ImportSbom(test.sbom, test.projectName, test.projectVersion); return res; }, defaultBullConfig From 4598b054634f8230cb8c73811552c8629555d751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Mon, 6 Feb 2023 21:30:26 +0100 Subject: [PATCH 10/33] WIP: Better import remove file Add CORS support Iproved import flow ImportForm - define name and version --- .gitignore | 2 +- docker-compose.yml | 31 +- src/depvis-next/.env.production | 13 - .../components/Details/ComponentDetails.tsx | 1 + .../components/Import/ImportForm.tsx | 77 ++-- src/depvis-next/helpers/ApolloClientHelper.ts | 9 +- src/depvis-next/helpers/ImportSbomHelper.ts | 7 +- src/depvis-next/helpers/QueueHelper.ts | 30 +- src/depvis-next/helpers/WorkspaceHelper.ts | 35 +- src/depvis-next/package-lock.json | 402 ++++++++++-------- src/depvis-next/package.json | 2 +- src/depvis-next/pages/api/graphql.ts | 19 +- src/depvis-next/pages/api/import/index.ts | 55 ++- src/depvis-next/pages/api/import/status.ts | 11 +- src/depvis-next/pages/api/vuln/[...purl].ts | 16 - src/depvis-next/queues/GetVulnQueue.ts | 32 +- src/depvis-next/queues/ImportQueue.ts | 37 +- 17 files changed, 456 insertions(+), 323 deletions(-) delete mode 100644 src/depvis-next/.env.production delete mode 100644 src/depvis-next/pages/api/vuln/[...purl].ts diff --git a/.gitignore b/.gitignore index 5561297..c6c16d4 100644 --- a/.gitignore +++ b/.gitignore @@ -73,7 +73,7 @@ web_modules/ .yarn-integrity # dotenv environment variable files -.env +.env* .env.development.local .env.test.local .env.production.local diff --git a/docker-compose.yml b/docker-compose.yml index 8993394..6c1b37a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: - ./runtime/plugins:/var/lib/neo4j/plugins restart: unless-stopped environment: - - NEO4J_AUTH=neo4j/test + - NEO4J_AUTH=neo4j/${NEO4J_PASSWORD} - NEO4J_apoc_export_file_enabled=true - NEO4J_apoc_import_file_enabled=true - NEO4J_apoc_import_file_use__neo4j__config=true @@ -20,13 +20,28 @@ services: - NEO4J_dbms_security_procedures_unrestricted=apoc.*,algo.* redis-cache: - image: redis:alpine - restart: unless-stopped - ports: - - 6379:6379 - command: redis-server --save 20 1 --loglevel notice + image: redis:latest + command: redis-server --requirepass ${REDIS_PASSWORD} volumes: - cache:/data + ports: + - 6379:6379 + links: + - redis-commander + + redis-commander: + image: rediscommander/redis-commander:latest + restart: always + environment: + REDIS_HOSTS: redis-cache + REDIS_HOST: redis-cache + REDIS_PORT: redis-cache:6379 + REDIS_PASSWORD: ${REDIS_PASSWORD} + HTTP_USER: root + HTTP_PASSWORD: root + ports: + - 8081:8081 + depvis-next: depends_on: - neo4j @@ -36,6 +51,10 @@ services: ports: - 80:3000 restart: always + environment: + NEO4J_PASSWORD: ${NEO4J_PASSWORD} + REDIS_PASSWORD: ${REDIS_PASSWORD} + NEXT_PUBLIC_SONATYPE_OSS_AUTH: ${SONATYPE_OSS_AUTH} volumes: cache: driver: local diff --git a/src/depvis-next/.env.production b/src/depvis-next/.env.production deleted file mode 100644 index 094f4e4..0000000 --- a/src/depvis-next/.env.production +++ /dev/null @@ -1,13 +0,0 @@ -# Sample .env file that can be used for deployment via docker -# Do not forget to change passwords to correspond with docker-compose file -NEO4J_USER=neo4j -NEO4J_PASSWORD= -NEO4J_HOST=neo4j://neo4j:7687 - -NEXT_PUBLIC_SONATYPE_OSS_AUTH= - -GQL_ALLOW_DEV_TOOLS=true - -REDIS_HOST=redis-cache -REDIS_PORT=6379 -REDIS_PASSWORD= \ No newline at end of file diff --git a/src/depvis-next/components/Details/ComponentDetails.tsx b/src/depvis-next/components/Details/ComponentDetails.tsx index 836870d..07df282 100644 --- a/src/depvis-next/components/Details/ComponentDetails.tsx +++ b/src/depvis-next/components/Details/ComponentDetails.tsx @@ -27,6 +27,7 @@ const ComponentDetails = (props) => { const renderLink = () => { const link = GetComponentRepositoryURL(data.components[0].purl); + if (!link) return; return ( {link} diff --git a/src/depvis-next/components/Import/ImportForm.tsx b/src/depvis-next/components/Import/ImportForm.tsx index 3c3085e..76bdea7 100644 --- a/src/depvis-next/components/Import/ImportForm.tsx +++ b/src/depvis-next/components/Import/ImportForm.tsx @@ -1,15 +1,17 @@ -import { useState } from "react"; -import { Alert, Button, Container, Form } from "react-bootstrap"; -import { ImportResult } from "./ImportResult"; +import { useState } from 'react'; +import { Alert, Button, Container, Form } from 'react-bootstrap'; +import { ImportResult } from './ImportResult'; const allowedExtensionsRegex = /(\.json|\.xml)$/i; const ImportForm = () => { - const [file, setFile] = useState(""); - const [preview, setPreview] = useState(""); + const [file, setFile] = useState(''); + const [preview, setPreview] = useState(''); const [validated, setValidated] = useState(false); const [isSubmitted, setIsSubmitted] = useState(false); const [jobId, setJobId] = useState(null); + const [projectName, setProjectName] = useState(''); + const [projectVersion, setProjectVersion] = useState('1.0.0'); const handleFiles = (e: any) => { const files = e.target.files; @@ -17,8 +19,8 @@ const ImportForm = () => { const file = files[0]; console.log(file); if (!allowedExtensionsRegex.exec(file.name)) { - alert("This extension is not allowed!"); - setFile(""); + alert('This extension is not allowed!'); + setFile(''); return; } setFile(file); @@ -33,11 +35,16 @@ const ImportForm = () => { } setValidated(true); - console.log({ file: typeof file }); - const res = await fetch("/api/import", { - body: await file.text(), - headers: { "Content-Type": "application/xml" }, - method: "POST", + const body = { + projectName: projectName, + projectVersion: projectVersion, + sbom: await file.text(), + }; + console.log(body); + const res = await fetch('/api/import', { + body: await JSON.stringify(body), + headers: { 'Content-Type': 'application/json' }, + method: 'POST', }); const json = await res.json(); setJobId(json.jobId); @@ -54,11 +61,35 @@ const ImportForm = () => { ) : ( - - All data currently stored in DB will be overwritten. - + All data currently stored in DB will be overwritten.
+ + Project name + { + setProjectName(e.target.value); + }} + value={projectName} + > + Please select any XML / JSON file with SBOM. + + + Project version + { + setProjectVersion(e.target.value); + }} + value={projectVersion} + > + Please select any XML / JSON file with SBOM. + SBOM File { handleFiles(e); }} > - - Please select any XML / JSON file with SBOM. - + Please select any XML / JSON file with SBOM. - -
diff --git a/src/depvis-next/helpers/ApolloClientHelper.ts b/src/depvis-next/helpers/ApolloClientHelper.ts index 98aba03..6b61be4 100644 --- a/src/depvis-next/helpers/ApolloClientHelper.ts +++ b/src/depvis-next/helpers/ApolloClientHelper.ts @@ -1,12 +1,19 @@ import { ApolloClient, HttpLink, InMemoryCache } from '@apollo/client'; +import urlJoin from 'url-join'; +import { gqlUrlPath } from '../pages/api/graphql'; /** * Function responsible for initialization of new Apollo Client used for GraphQL * @returns ApolloClient object */ export const createApolloClient = () => { + const uri = urlJoin(process.env.NEXT_PUBLIC_SERVER_URI || 'http://localhost:3000', gqlUrlPath); + console.log(`Creating GQL Client (connection to ${uri})`); const link = new HttpLink({ - uri: process.env.NEXT_PUBLIC_GQL_URI, + uri: uri, + fetchOptions: { + mode: 'cors', + }, }); return new ApolloClient({ link, diff --git a/src/depvis-next/helpers/ImportSbomHelper.ts b/src/depvis-next/helpers/ImportSbomHelper.ts index 67a0d66..e86e11e 100644 --- a/src/depvis-next/helpers/ImportSbomHelper.ts +++ b/src/depvis-next/helpers/ImportSbomHelper.ts @@ -9,7 +9,7 @@ import { UpdateProjectDependencies, } from './DbDataHelper'; -export async function ImportSbom(bom: any) { +export async function ImportSbom(bom: any, projectName, projectVersion) { // Prepare main component if exists const mainComponentParsed = bom.metadata.component; const mainComponent: Component | undefined = mainComponentParsed @@ -25,10 +25,11 @@ export async function ImportSbom(bom: any) { // Prepare project let project: Project = { - name: bom.metadata.component.name, - version: bom.metadata.component.version || 'n/a', + name: projectName || bom.metadata.component.name, + version: projectVersion || bom.metadata.component.version || '1.0.0', date: bom.metadata.timestamp || '1970-01-01', }; + console.log(project); // Prepare dependencies let dependencies = GetDependencies(bom.dependencies.dependency); diff --git a/src/depvis-next/helpers/QueueHelper.ts b/src/depvis-next/helpers/QueueHelper.ts index 2878d0c..fadaf3e 100644 --- a/src/depvis-next/helpers/QueueHelper.ts +++ b/src/depvis-next/helpers/QueueHelper.ts @@ -1,24 +1,14 @@ -const getKeys = async (q) => { - const multi = q.multi(); - multi.keys("*"); - const keys = await multi.exec(); - return keys[0][1]; -}; - -const filterQueueKeys = (q, keys) => { - const prefix = `${q.keyPrefix}:${q.name}`; - return keys.filter((k) => k.includes(prefix)); -}; +import { Queue } from 'bullmq'; -const deleteKeys = async (q, keys) => { - const multi = q.multi(); - keys.forEach((k) => multi.del(k)); - await multi.exec(); +export const emptyQueue = async (q: Queue) => { + q.drain(); }; -export const emptyQueue = async (q) => { - const keys = await getKeys(q); - const queueKeys = filterQueueKeys(q, keys); - await deleteKeys(q, queueKeys); - console.log(`Queue ${q.name} is empty`); +export const defaultBullConfig = { + connection: { + host: process.env.REDIS_HOST, + port: process.env.REDIS_PORT, + password: process.env.REDIS_PASSWORD, + enableOfflineQueue: false, + }, }; diff --git a/src/depvis-next/helpers/WorkspaceHelper.ts b/src/depvis-next/helpers/WorkspaceHelper.ts index 78cc405..3455556 100644 --- a/src/depvis-next/helpers/WorkspaceHelper.ts +++ b/src/depvis-next/helpers/WorkspaceHelper.ts @@ -1,23 +1,26 @@ -import { PackageURL } from "packageurl-js"; +import { PackageURL } from 'packageurl-js'; import urlJoin from 'url-join'; - -const npmUrlBase = "https://www.npmjs.com/package/" -const nugetUrlBase = "https://www.nuget.org/packages/" -const pypiUrlBase = "https://pypi.org/project/" - +const npmUrlBase = 'https://www.npmjs.com/package/'; +const nugetUrlBase = 'https://www.nuget.org/packages/'; +const pypiUrlBase = 'https://pypi.org/project/'; export function GetComponentRepositoryURL(purl: string) { - const component = PackageURL.fromString(purl) + try { + const component = PackageURL.fromString(purl); switch (component.type) { - case "npm": - return urlJoin(npmUrlBase, component.name, component.version ? urlJoin("v", component.version) : "") - case "nuget": - return urlJoin(nugetUrlBase, component.name, component.version) - case "pypi": - return urlJoin(pypiUrlBase, component.name, component.version) - default: - return urlJoin("https://www.google.com/search?q=", purl) + case 'npm': + return urlJoin(npmUrlBase, component.name, component.version ? urlJoin('v', component.version) : ''); + case 'nuget': + return urlJoin(nugetUrlBase, component.name, component.version); + case 'pypi': + return urlJoin(pypiUrlBase, component.name, component.version); + default: + return urlJoin('https://www.google.com/search?q=', purl); } -} \ No newline at end of file + } catch { + console.log('Could not parse Package URL from %s', purl); + return; + } +} diff --git a/src/depvis-next/package-lock.json b/src/depvis-next/package-lock.json index 1fc6b57..b6322ad 100644 --- a/src/depvis-next/package-lock.json +++ b/src/depvis-next/package-lock.json @@ -12,7 +12,7 @@ "@neo4j/graphql": "^3.8.0", "apollo-server-micro": "^3.10.2", "bootstrap": "^5.2.1", - "bull": "^4.10.1", + "bullmq": "^3.6.2", "d3-force": "^3.0.0", "fast-xml-parser": "^4.0.10", "graphql": "^16.6.0", @@ -465,10 +465,70 @@ "resolved": "https://registry.npmjs.org/@josephg/resolvable/-/resolvable-1.0.1.tgz", "integrity": "sha512-CtzORUwWTTOTqfVtHaKRJ0I1kNQd1bpn3sUh8I3nJDVY+5/M/Oe1DnEWzPQvqq/xPIIkzzzIP7mfCoAjFRvDhg==" }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.0.tgz", + "integrity": "sha512-5qpnNHUyyEj9H3sm/4Um/bnx1lrQGhe8iqry/1d+cQYCRd/gzYA0YLeq0ezlk4hKx4vO+dsEsNyeowqRqslwQA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.0.tgz", + "integrity": "sha512-ZphTFFd6SFweNAMKD+QJCrWpgkjf4qBuHltiMkKkD6FFrB3NOTRVmetAGTkJ57pa+s6J0yCH06LujWB9rZe94g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.0.tgz", + "integrity": "sha512-ztKVV1dO/sSZyGse0PBCq3Pk1PkYjsA/dsEWE7lfrGoAK3i9HpS2o7XjGQ7V4va6nX+xPPOiuYpQwa4Bi6vlww==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.0.tgz", + "integrity": "sha512-NEX6hdSvP4BmVyegaIbrGxvHzHvTzzsPaxXCsUt0mbLbPpEftsvNwaEVKOowXnLoeuGeD4MaqSwL3BUK2elsUA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.0.tgz", + "integrity": "sha512-9uvdAkZMOPCY7SPRxZLW8XGqBOVNVEhqlgffenN8shA1XR9FWVsSM13nr/oHtNgXg6iVyML7RwWPyqUeThlwxg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.2.0.tgz", - "integrity": "sha512-XrC0JzsqQSvOyM3t04FMLO6z5gCuhPE6k4FXuLK5xf52ZbdvcFe1yBmo7meCew9B8G2f0T9iu9t3kfTYRYROgA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.0.tgz", + "integrity": "sha512-Wg0+9615kHKlr9iLVcG5I+/CHnf6w3x5UADRv8Ad16yA0Bu5l9eVOROjV7aHPG6uC8ZPFIVVaoSjDChD+Y0pzg==", "cpu": [ "x64" ], @@ -1468,8 +1528,7 @@ "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "node_modules/base64-js": { "version": "1.5.1", @@ -1561,31 +1620,56 @@ "ieee754": "^1.2.1" } }, - "node_modules/bull": { - "version": "4.10.2", - "resolved": "https://registry.npmjs.org/bull/-/bull-4.10.2.tgz", - "integrity": "sha512-xa65xtWjQsLqYU/eNaXxq9VRG8xd6qNsQEjR7yjYuae05xKrzbVMVj2QgrYsTMmSs/vsqJjHqHSRRiW1+IkGXQ==", + "node_modules/bullmq": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.6.2.tgz", + "integrity": "sha512-Nea7OWtdg6V47HP2vp0V2RUhG02KHCJE+VzbwpOFRONHS+1M6IehAVjEhfN3N036BNoPU776sMKCVxaegG0reA==", "dependencies": { - "cron-parser": "^4.2.1", - "debuglog": "^1.0.0", - "get-port": "^5.1.1", - "ioredis": "^5.0.0", + "cron-parser": "^4.6.0", + "glob": "^8.0.3", + "ioredis": "^5.3.0", "lodash": "^4.17.21", - "msgpackr": "^1.5.2", - "p-timeout": "^3.2.0", - "semver": "^7.3.2", - "uuid": "^8.3.0" + "msgpackr": "^1.6.2", + "semver": "^7.3.7", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, + "node_modules/bullmq/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/bullmq/node_modules/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" }, "engines": { "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/bull/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" + "node_modules/bullmq/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" } }, "node_modules/bytes": { @@ -1729,11 +1813,11 @@ } }, "node_modules/cron-parser": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.7.0.tgz", - "integrity": "sha512-BdAELR+MCT2ZWsIBhZKDuUqIUCBjHHulPJnm53OfdRLA4EWBjva3R+KM5NeidJuGsNXdEcZkjC7SCnkW5rAFSA==", + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.7.1.tgz", + "integrity": "sha512-WguFaoQ0hQ61SgsCZLHUcNbAvlK0lypKXu62ARguefYmjzaOXIVRNrAmyXzabTwUn4sQvQLkk6bjH+ipGfw8bA==", "dependencies": { - "luxon": "^3.1.0" + "luxon": "^3.2.1" }, "engines": { "node": ">=12.0.0" @@ -2000,14 +2084,6 @@ } } }, - "node_modules/debuglog": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", - "integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==", - "engines": { - "node": "*" - } - }, "node_modules/deep-equal": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.1.0.tgz", @@ -2847,8 +2923,7 @@ "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "node_modules/function-bind": { "version": "1.1.1", @@ -2894,17 +2969,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-port": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", - "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/get-symbol-description": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", @@ -3229,7 +3293,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -3271,14 +3334,14 @@ } }, "node_modules/ioredis": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.4.tgz", - "integrity": "sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.0.tgz", + "integrity": "sha512-Id9jKHhsILuIZpHc61QkagfVdUj2Rag5GzG1TGEvRNeM7dtTOjICgjC+tvqYxi//PuX2wjQ+Xjva2ONBuf92Pw==", "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", @@ -3855,32 +3918,32 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/msgpackr": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.8.1.tgz", - "integrity": "sha512-05fT4J8ZqjYlR4QcRDIhLCYKUOHXk7C/xa62GzMKj74l3up9k2QZ3LgFc6qWdsPHl91QA2WLWqWc8b8t7GLNNw==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.8.3.tgz", + "integrity": "sha512-m2JefwcKNzoHYXkH/5jzHRxAw7XLWsAdvu0FOJ+OLwwozwOV/J6UA62iLkfIMbg7G8+dIuRwgg6oz+QoQ4YkoA==", "optionalDependencies": { - "msgpackr-extract": "^2.2.0" + "msgpackr-extract": "^3.0.0" } }, "node_modules/msgpackr-extract": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-2.2.0.tgz", - "integrity": "sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.0.tgz", + "integrity": "sha512-oy6KCk1+X4Bn5m6Ycq5N1EWl9npqG/cLrE8ga8NX7ZqfqYUUBS08beCQaGq80fjbKBySur0E6x//yZjzNJDt3A==", "hasInstallScript": true, "optional": true, "dependencies": { - "node-gyp-build-optional-packages": "5.0.3" + "node-gyp-build-optional-packages": "5.0.7" }, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" }, "optionalDependencies": { - "@msgpackr-extract/msgpackr-extract-darwin-arm64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-darwin-x64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-linux-arm": "2.2.0", - "@msgpackr-extract/msgpackr-extract-linux-arm64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-linux-x64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-win32-x64": "2.2.0" + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.0" } }, "node_modules/multiparty": { @@ -4024,9 +4087,9 @@ } }, "node_modules/node-gyp-build-optional-packages": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz", - "integrity": "sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", "optional": true, "bin": { "node-gyp-build-optional-packages": "bin.js", @@ -4155,7 +4218,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "dependencies": { "wrappy": "1" } @@ -4186,14 +4248,6 @@ "node": ">= 0.8.0" } }, - "node_modules/p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", - "engines": { - "node": ">=4" - } - }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -4224,17 +4278,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "dependencies": { - "p-finally": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/packageurl-js": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/packageurl-js/-/packageurl-js-1.0.0.tgz", @@ -5338,8 +5381,7 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/xss": { "version": "1.0.14", @@ -5707,10 +5749,40 @@ "resolved": "https://registry.npmjs.org/@josephg/resolvable/-/resolvable-1.0.1.tgz", "integrity": "sha512-CtzORUwWTTOTqfVtHaKRJ0I1kNQd1bpn3sUh8I3nJDVY+5/M/Oe1DnEWzPQvqq/xPIIkzzzIP7mfCoAjFRvDhg==" }, + "@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.0.tgz", + "integrity": "sha512-5qpnNHUyyEj9H3sm/4Um/bnx1lrQGhe8iqry/1d+cQYCRd/gzYA0YLeq0ezlk4hKx4vO+dsEsNyeowqRqslwQA==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.0.tgz", + "integrity": "sha512-ZphTFFd6SFweNAMKD+QJCrWpgkjf4qBuHltiMkKkD6FFrB3NOTRVmetAGTkJ57pa+s6J0yCH06LujWB9rZe94g==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.0.tgz", + "integrity": "sha512-ztKVV1dO/sSZyGse0PBCq3Pk1PkYjsA/dsEWE7lfrGoAK3i9HpS2o7XjGQ7V4va6nX+xPPOiuYpQwa4Bi6vlww==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.0.tgz", + "integrity": "sha512-NEX6hdSvP4BmVyegaIbrGxvHzHvTzzsPaxXCsUt0mbLbPpEftsvNwaEVKOowXnLoeuGeD4MaqSwL3BUK2elsUA==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.0.tgz", + "integrity": "sha512-9uvdAkZMOPCY7SPRxZLW8XGqBOVNVEhqlgffenN8shA1XR9FWVsSM13nr/oHtNgXg6iVyML7RwWPyqUeThlwxg==", + "optional": true + }, "@msgpackr-extract/msgpackr-extract-win32-x64": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.2.0.tgz", - "integrity": "sha512-XrC0JzsqQSvOyM3t04FMLO6z5gCuhPE6k4FXuLK5xf52ZbdvcFe1yBmo7meCew9B8G2f0T9iu9t3kfTYRYROgA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.0.tgz", + "integrity": "sha512-Wg0+9615kHKlr9iLVcG5I+/CHnf6w3x5UADRv8Ad16yA0Bu5l9eVOROjV7aHPG6uC8ZPFIVVaoSjDChD+Y0pzg==", "optional": true }, "@neo4j/cypher-builder": { @@ -6399,8 +6471,7 @@ "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "base64-js": { "version": "1.5.1", @@ -6445,26 +6516,48 @@ "ieee754": "^1.2.1" } }, - "bull": { - "version": "4.10.2", - "resolved": "https://registry.npmjs.org/bull/-/bull-4.10.2.tgz", - "integrity": "sha512-xa65xtWjQsLqYU/eNaXxq9VRG8xd6qNsQEjR7yjYuae05xKrzbVMVj2QgrYsTMmSs/vsqJjHqHSRRiW1+IkGXQ==", + "bullmq": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.6.2.tgz", + "integrity": "sha512-Nea7OWtdg6V47HP2vp0V2RUhG02KHCJE+VzbwpOFRONHS+1M6IehAVjEhfN3N036BNoPU776sMKCVxaegG0reA==", "requires": { - "cron-parser": "^4.2.1", - "debuglog": "^1.0.0", - "get-port": "^5.1.1", - "ioredis": "^5.0.0", + "cron-parser": "^4.6.0", + "glob": "^8.0.3", + "ioredis": "^5.3.0", "lodash": "^4.17.21", - "msgpackr": "^1.5.2", - "p-timeout": "^3.2.0", - "semver": "^7.3.2", - "uuid": "^8.3.0" + "msgpackr": "^1.6.2", + "semver": "^7.3.7", + "tslib": "^2.0.0", + "uuid": "^9.0.0" }, "dependencies": { - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, + "glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + } + }, + "minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "requires": { + "brace-expansion": "^2.0.1" + } } } }, @@ -6564,11 +6657,11 @@ "dev": true }, "cron-parser": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.7.0.tgz", - "integrity": "sha512-BdAELR+MCT2ZWsIBhZKDuUqIUCBjHHulPJnm53OfdRLA4EWBjva3R+KM5NeidJuGsNXdEcZkjC7SCnkW5rAFSA==", + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.7.1.tgz", + "integrity": "sha512-WguFaoQ0hQ61SgsCZLHUcNbAvlK0lypKXu62ARguefYmjzaOXIVRNrAmyXzabTwUn4sQvQLkk6bjH+ipGfw8bA==", "requires": { - "luxon": "^3.1.0" + "luxon": "^3.2.1" } }, "cross-spawn": { @@ -6764,11 +6857,6 @@ "ms": "2.1.2" } }, - "debuglog": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", - "integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==" - }, "deep-equal": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.1.0.tgz", @@ -7415,8 +7503,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "function-bind": { "version": "1.1.1", @@ -7450,11 +7537,6 @@ "has-symbols": "^1.0.3" } }, - "get-port": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", - "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==" - }, "get-symbol-description": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", @@ -7671,7 +7753,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -7707,14 +7788,14 @@ } }, "ioredis": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.4.tgz", - "integrity": "sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.0.tgz", + "integrity": "sha512-Id9jKHhsILuIZpHc61QkagfVdUj2Rag5GzG1TGEvRNeM7dtTOjICgjC+tvqYxi//PuX2wjQ+Xjva2ONBuf92Pw==", "requires": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", @@ -8123,26 +8204,26 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "msgpackr": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.8.1.tgz", - "integrity": "sha512-05fT4J8ZqjYlR4QcRDIhLCYKUOHXk7C/xa62GzMKj74l3up9k2QZ3LgFc6qWdsPHl91QA2WLWqWc8b8t7GLNNw==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.8.3.tgz", + "integrity": "sha512-m2JefwcKNzoHYXkH/5jzHRxAw7XLWsAdvu0FOJ+OLwwozwOV/J6UA62iLkfIMbg7G8+dIuRwgg6oz+QoQ4YkoA==", "requires": { - "msgpackr-extract": "^2.2.0" + "msgpackr-extract": "^3.0.0" } }, "msgpackr-extract": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-2.2.0.tgz", - "integrity": "sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.0.tgz", + "integrity": "sha512-oy6KCk1+X4Bn5m6Ycq5N1EWl9npqG/cLrE8ga8NX7ZqfqYUUBS08beCQaGq80fjbKBySur0E6x//yZjzNJDt3A==", "optional": true, "requires": { - "@msgpackr-extract/msgpackr-extract-darwin-arm64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-darwin-x64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-linux-arm": "2.2.0", - "@msgpackr-extract/msgpackr-extract-linux-arm64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-linux-x64": "2.2.0", - "@msgpackr-extract/msgpackr-extract-win32-x64": "2.2.0", - "node-gyp-build-optional-packages": "5.0.3" + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.0", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.0", + "node-gyp-build-optional-packages": "5.0.7" } }, "multiparty": { @@ -8240,9 +8321,9 @@ } }, "node-gyp-build-optional-packages": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz", - "integrity": "sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", "optional": true }, "object-assign": { @@ -8327,7 +8408,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "requires": { "wrappy": "1" } @@ -8355,11 +8435,6 @@ "word-wrap": "^1.2.3" } }, - "p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==" - }, "p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -8378,14 +8453,6 @@ "p-limit": "^3.0.2" } }, - "p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "requires": { - "p-finally": "^1.0.0" - } - }, "packageurl-js": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/packageurl-js/-/packageurl-js-1.0.0.tgz", @@ -9158,8 +9225,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "xss": { "version": "1.0.14", diff --git a/src/depvis-next/package.json b/src/depvis-next/package.json index 3e3126f..b37e8a1 100644 --- a/src/depvis-next/package.json +++ b/src/depvis-next/package.json @@ -13,7 +13,7 @@ "@neo4j/graphql": "^3.8.0", "apollo-server-micro": "^3.10.2", "bootstrap": "^5.2.1", - "bull": "^4.10.1", + "bullmq": "^3.6.2", "d3-force": "^3.0.0", "fast-xml-parser": "^4.0.10", "graphql": "^16.6.0", diff --git a/src/depvis-next/pages/api/graphql.ts b/src/depvis-next/pages/api/graphql.ts index 79daa5c..ace55d9 100644 --- a/src/depvis-next/pages/api/graphql.ts +++ b/src/depvis-next/pages/api/graphql.ts @@ -25,7 +25,7 @@ const driver = neo4j.driver( neo4j.auth.basic(NEO4J_CONFIG.neo4jUsername, NEO4J_CONFIG.neo4jPassword) ); -export default async function handler(req, res) { +async function handler(req, res) { const neoSchema = new Neo4jGraphQL({ typeDefs, driver }); const apolloServer = new ApolloServer({ schema: await neoSchema.getSchema(), @@ -38,6 +38,23 @@ export default async function handler(req, res) { })(req, res); } +const allowCors = (fn) => async (req, res) => { + res.setHeader('Access-Control-Allow-Credentials', true); + const origin = process.env.CORS_ORIGIN || 'http://localhost:3000'; + res.setHeader('Access-Control-Allow-Origin', origin); + res.setHeader('Access-Control-Allow-Methods', 'GET,OPTIONS,PATCH,DELETE,POST,PUT'); + res.setHeader( + 'Access-Control-Allow-Headers', + 'X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, Content-Type, Date, X-Api-Version' + ); + if (req.method === 'OPTIONS') { + res.status(200).end(); + return; + } + return await fn(req, res); +}; + +export default allowCors(handler); export const config = { api: { bodyParser: false, diff --git a/src/depvis-next/pages/api/import/index.ts b/src/depvis-next/pages/api/import/index.ts index 216d9e3..5eee3f7 100644 --- a/src/depvis-next/pages/api/import/index.ts +++ b/src/depvis-next/pages/api/import/index.ts @@ -1,9 +1,9 @@ -import Bull from 'bull'; +import { Queue } from 'bullmq'; import { XMLParser } from 'fast-xml-parser'; import { ImportSbom } from '../../../helpers/ImportSbomHelper'; -import { emptyQueue } from '../../../helpers/QueueHelper'; +import { defaultBullConfig, emptyQueue } from '../../../helpers/QueueHelper'; import { GetVulnQueueName } from '../../../queues/GetVulnQueue'; -import { ImportQueueName } from '../../../queues/ImportQueue'; +import { ImportQueueName, ImportSbomJobData } from '../../../queues/ImportQueue'; export const config = { api: { @@ -20,23 +20,45 @@ const XMLParserOptions = { }; //Bull queue -const ImportQueue = new Bull(ImportQueueName); -const GetVulnQueue = new Bull(GetVulnQueueName); +const ImportQueue = new Queue(ImportQueueName, defaultBullConfig); +const GetVulnQueue = new Queue(GetVulnQueueName, defaultBullConfig); + type ImportResult = { isError: boolean; errorMessage?: string; jobId?: string; + sbom?: any; +}; + +type ImportRequestBody = { + projectName: string; + projectVersion: string; + sbom: string; }; export default async function handler(req, res) { try { - if (req.headers['content-type'] !== 'application/xml') { - res.status(500).json({ error: "Content-type must be 'application/xml'" }); - return; + // if (req.headers['content-type'] !== 'application/xml') { + // res.status(500).json({ error: "Content-type must be 'application/xml'" }); + // return; + // } + const body = req.body as ImportRequestBody; + console.log('Recieved body: %s', body.projectName); + const result = await parseXml(body.sbom); + if (result.isError) { + console.log('Import failed with %s', result.errorMessage); + return res.status(400).json(result); } - //TODO: check if XML is even valid - const result = await parseXml(req.body); - return res.status(200).json(result); + + //Clear vuln queue + emptyQueue(GetVulnQueue); + const job = await ImportQueue.add(body.projectName, { + sbom: result.sbom, + projectName: body.projectName, + projectVersion: body.projectVersion, + } as ImportSbomJobData); + const response: ImportResult = { jobId: job.id, isError: false }; + return res.status(200).json(response); } catch (err) { console.error(err); return res.status(500).json({ error: 'failed to load data', content: err }); @@ -45,24 +67,24 @@ export default async function handler(req, res) { // Function validates that object contains required properties function validateSbomXml(parsedXml): ImportResult { - const bom = parsedXml.bom; - if (!bom) + const sbom = parsedXml.bom; + if (!sbom) return { isError: true, errorMessage: "Validation failed - Missing 'bom' parameter in the file.", }; - if (!bom.metadata) + if (!sbom.metadata) return { isError: true, errorMessage: "Validation failed - Missing 'metadata' parameter in the file.", }; - if (!bom.components) + if (!sbom.components) return { isError: true, errorMessage: "Validation failed - Missing 'components' parameter in the file.", }; - return { isError: false }; + return { isError: false, sbom: sbom }; } // Function takes XML in plain text and transforms it into object @@ -78,4 +100,5 @@ async function parseXml(inputXml: string) { // const job = await ImportQueue.add({ bom: xmlParsed.bom }); // return { jobId: job.id }; return await ImportSbom(xmlParsed.bom); + return validateSbomXml(xmlParsed); } diff --git a/src/depvis-next/pages/api/import/status.ts b/src/depvis-next/pages/api/import/status.ts index 22633b1..70b5496 100644 --- a/src/depvis-next/pages/api/import/status.ts +++ b/src/depvis-next/pages/api/import/status.ts @@ -1,12 +1,13 @@ -import Bull from "bull"; -import { ImportQueueName } from "../../../queues/ImportQueue"; +import { Queue } from 'bullmq'; +import { defaultBullConfig } from '../../../helpers/QueueHelper'; +import { ImportQueueName } from '../../../queues/ImportQueue'; //Bull queue -const ImportQueue = new Bull(ImportQueueName); +const ImportQueue = new Queue(ImportQueueName, defaultBullConfig); export default async function handler(req, res) { - if (req.method !== "GET") return res.status(405); - if (!("id" in req.query && !isNaN(req.query.id))) + if (req.method !== 'GET') return res.status(405); + if (!('id' in req.query && !isNaN(req.query.id))) return res.status(400).json({ error: "'id' parameter is missing or its value is not a number", }); diff --git a/src/depvis-next/pages/api/vuln/[...purl].ts b/src/depvis-next/pages/api/vuln/[...purl].ts deleted file mode 100644 index 3ad4989..0000000 --- a/src/depvis-next/pages/api/vuln/[...purl].ts +++ /dev/null @@ -1,16 +0,0 @@ -import Bull from "bull"; -import { GetVulnQueueName } from "../../../queues/GetVulnQueue"; - -const VulnQueue = new Bull(GetVulnQueueName); - -export default async function handler(req, res) { - const { purl } = req.query; - const purl_j = purl.join("/"); - try { - const job = await VulnQueue.add({ purl: purl_j }); - return res.status(200).json({ jobId: job.id }); - } catch (error) { - console.log(error.message); - res.status(500).json({ error: error.message }); - } -} diff --git a/src/depvis-next/queues/GetVulnQueue.ts b/src/depvis-next/queues/GetVulnQueue.ts index d85cbe2..e39ab81 100644 --- a/src/depvis-next/queues/GetVulnQueue.ts +++ b/src/depvis-next/queues/GetVulnQueue.ts @@ -1,21 +1,25 @@ -import Bull from 'bull'; +import { Job, Worker } from 'bullmq'; +import { CreateUpdateVulnerability } from '../helpers/DbDataHelper'; +import { defaultBullConfig } from '../helpers/QueueHelper'; import { VulnFetcherHandler } from '../vulnerability-mgmt/VulnFetcherHandler'; export const GetVulnQueueName = 'get-vuln-queue'; -export const GetVulnQueueOptions: Bull.QueueOptions = { - redis: { - port: parseInt(process.env.REDIS_PORT, 10), - host: process.env.REDIS_HOST, - password: process.env.REDIS_PASSWORD, +const worker = new Worker( + GetVulnQueueName, + async (job) => { + console.log(`Processing Vuln ${job.data.purl}`); + const vulnerabilityList = await VulnFetcherHandler(job.data.purl); + if (vulnerabilityList.length == 0) return; + return await CreateUpdateVulnerability(job.data.purl, vulnerabilityList); }, - limiter: { max: 3, duration: 1000 }, -}; + defaultBullConfig +); -const GetVulnQueue = new Bull(GetVulnQueueName, GetVulnQueueOptions); +worker.on('failed', (job, error) => { + console.log('Job %d failed with error %s', job.id, error.message); + console.error(error); +}); -GetVulnQueue.process(async (job) => { - console.log(`Processing Vuln ${job.data.purl}`); - const vulnerabilityList = await VulnFetcherHandler(job.data.purl); - // if (vulnerabilitiesList.length == 0) return; - //return await CreateUpdateVulnerability(job.data.purl, vulnerabilitiesList); +worker.on('completed', (job) => { + console.log('Job %d completed successfully!', job.id); }); diff --git a/src/depvis-next/queues/ImportQueue.ts b/src/depvis-next/queues/ImportQueue.ts index f7e4075..e74785c 100644 --- a/src/depvis-next/queues/ImportQueue.ts +++ b/src/depvis-next/queues/ImportQueue.ts @@ -1,22 +1,31 @@ -import Bull from 'bull'; +import { Worker } from 'bullmq'; import { ImportSbom } from '../helpers/ImportSbomHelper'; +import { defaultBullConfig } from '../helpers/QueueHelper'; export const ImportQueueName = 'import-queue'; -export const ImportQueueOptions: Bull.QueueOptions = { - redis: { - port: parseInt(process.env.REDIS_PORT, 10), - host: process.env.REDIS_HOST, - password: process.env.REDIS_PASSWORD, - }, - limiter: { max: 2, duration: 3600 }, +export type ImportSbomJobData = { + projectName: string; + projectVersion: string; + sbom: string; }; -console.log(ImportQueueOptions); -const ImportQueue = new Bull(ImportQueueName, ImportQueueOptions); +const worker = new Worker( + ImportQueueName, + async (job) => { + const test = job.data; + console.log(test); + const res = ImportSbom(test.sbom, test.projectName, test.projectVersion); + return res; + }, + defaultBullConfig +); + +worker.on('failed', (job, error) => { + console.log('Job %d failed with error %s', job.id, error.message); + console.error(error); +}); -ImportQueue.process(async (job) => { - console.log('Starting processing job %s', job.id); - const res = ImportSbom(job.data.bom); - return res; +worker.on('completed', (job) => { + console.log('Job %d completed successfully!', job.id); }); From 23464a132383f485b36682f80e0860d6dd3be3d4 Mon Sep 17 00:00:00 2001 From: Matej Groman Date: Thu, 9 Feb 2023 09:25:06 +0100 Subject: [PATCH 11/33] Fix import API endpoint --- src/depvis-next/pages/api/import/index.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/depvis-next/pages/api/import/index.ts b/src/depvis-next/pages/api/import/index.ts index 5eee3f7..36f6e76 100644 --- a/src/depvis-next/pages/api/import/index.ts +++ b/src/depvis-next/pages/api/import/index.ts @@ -92,13 +92,5 @@ async function parseXml(inputXml: string) { const parser = new XMLParser(XMLParserOptions); const xmlParsed = parser.parse(inputXml); - const validateResult = validateSbomXml(xmlParsed); - if (validateResult.isError) return validateResult; - - //Clear vuln queue - emptyQueue(GetVulnQueue); - // const job = await ImportQueue.add({ bom: xmlParsed.bom }); - // return { jobId: job.id }; - return await ImportSbom(xmlParsed.bom); return validateSbomXml(xmlParsed); -} +} \ No newline at end of file From 711c90ab886ab409fd151988b8139f94555a854a Mon Sep 17 00:00:00 2001 From: Matej Groman Date: Fri, 10 Feb 2023 10:14:16 +0100 Subject: [PATCH 12/33] WIP: Merge with filtering --- docker-compose.yml | 2 +- .../components/Layout/Workspace.tsx | 1 + src/depvis-next/helpers/ApolloClientHelper.ts | 6 +++++- src/depvis-next/helpers/ImportSbomHelper.ts | 4 ++-- src/depvis-next/helpers/QueueHelper.ts | 18 +++++++++++------- src/depvis-next/pages/api/import/index.ts | 4 ++-- 6 files changed, 22 insertions(+), 13 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6c1b37a..fb24bfa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,7 +49,7 @@ services: build: ./src/depvis-next image: depvis-next:latest ports: - - 80:3000 + - 8123:3000 restart: always environment: NEO4J_PASSWORD: ${NEO4J_PASSWORD} diff --git a/src/depvis-next/components/Layout/Workspace.tsx b/src/depvis-next/components/Layout/Workspace.tsx index 56cbee1..43c021f 100644 --- a/src/depvis-next/components/Layout/Workspace.tsx +++ b/src/depvis-next/components/Layout/Workspace.tsx @@ -69,6 +69,7 @@ const Workspace = () => { }; const handleShowOnlyVulnerableToggle = () => { + if (!data) return; if (graphConfig.showOnlyVulnerable) { setGraphData(formatData(data.projects[0].allVulnerableComponents)); } else { diff --git a/src/depvis-next/helpers/ApolloClientHelper.ts b/src/depvis-next/helpers/ApolloClientHelper.ts index 6b61be4..d13978a 100644 --- a/src/depvis-next/helpers/ApolloClientHelper.ts +++ b/src/depvis-next/helpers/ApolloClientHelper.ts @@ -7,7 +7,11 @@ import { gqlUrlPath } from '../pages/api/graphql'; * @returns ApolloClient object */ export const createApolloClient = () => { - const uri = urlJoin(process.env.NEXT_PUBLIC_SERVER_URI || 'http://localhost:3000', gqlUrlPath); + if (!process.env.NEXT_PUBLIC_SERVER_URI) { + console.error("No server URI was provided, using default connection") + } + + const uri = urlJoin(process.env.NEXT_PUBLIC_SERVER_URI || "http://localhost:3000", gqlUrlPath); console.log(`Creating GQL Client (connection to ${uri})`); const link = new HttpLink({ uri: uri, diff --git a/src/depvis-next/helpers/ImportSbomHelper.ts b/src/depvis-next/helpers/ImportSbomHelper.ts index e86e11e..d2d599d 100644 --- a/src/depvis-next/helpers/ImportSbomHelper.ts +++ b/src/depvis-next/helpers/ImportSbomHelper.ts @@ -16,7 +16,7 @@ export async function ImportSbom(bom: any, projectName, projectVersion) { ? { type: mainComponentParsed.type, name: mainComponentParsed.name, - purl: mainComponentParsed.purl, + purl: mainComponentParsed.purl || `${mainComponentParsed.name}@${mainComponentParsed.version}`, version: mainComponentParsed.version, author: mainComponentParsed.author, publisher: mainComponentParsed.publisher, @@ -63,7 +63,7 @@ function GetComponents(bom: any) { type: c.type, name: c.name, purl: c.purl, - version: c.version, + version: `${c.version}`, author: c.author, publisher: c.publisher, }; diff --git a/src/depvis-next/helpers/QueueHelper.ts b/src/depvis-next/helpers/QueueHelper.ts index fadaf3e..e804147 100644 --- a/src/depvis-next/helpers/QueueHelper.ts +++ b/src/depvis-next/helpers/QueueHelper.ts @@ -1,14 +1,18 @@ -import { Queue } from 'bullmq'; +import { Queue, RedisOptions } from 'bullmq'; +import IORedis from 'ioredis'; export const emptyQueue = async (q: Queue) => { q.drain(); }; +const REDIS_OPTIONS: RedisOptions = { + host: process.env.REDIS_HOST, + port: +process.env.REDIS_PORT!, + password: process.env.REDIS_PASSWORD, + enableOfflineQueue: false, + showFriendlyErrorStack: true, + enableReadyCheck: true, +} export const defaultBullConfig = { - connection: { - host: process.env.REDIS_HOST, - port: process.env.REDIS_PORT, - password: process.env.REDIS_PASSWORD, - enableOfflineQueue: false, - }, + connection: new IORedis(REDIS_OPTIONS) }; diff --git a/src/depvis-next/pages/api/import/index.ts b/src/depvis-next/pages/api/import/index.ts index 36f6e76..8e12a88 100644 --- a/src/depvis-next/pages/api/import/index.ts +++ b/src/depvis-next/pages/api/import/index.ts @@ -52,11 +52,11 @@ export default async function handler(req, res) { //Clear vuln queue emptyQueue(GetVulnQueue); - const job = await ImportQueue.add(body.projectName, { + const job = await ImportQueue.add(body.projectName.toString(), { sbom: result.sbom, projectName: body.projectName, projectVersion: body.projectVersion, - } as ImportSbomJobData); + }); const response: ImportResult = { jobId: job.id, isError: false }; return res.status(200).json(response); } catch (err) { From 6068a283d4f05ccb8625801feb18e698f165d508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Fri, 10 Feb 2023 10:56:20 +0100 Subject: [PATCH 13/33] Exclude local .env files --- src/depvis-next/.dockerignore | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/depvis-next/.dockerignore b/src/depvis-next/.dockerignore index 72e9aa4..bfe7df6 100644 --- a/src/depvis-next/.dockerignore +++ b/src/depvis-next/.dockerignore @@ -4,4 +4,9 @@ node_modules npm-debug.log README.md .next -.git \ No newline at end of file +.git + +.env.development.local +.env.test.local +.env.production.local +.env.local From e553b0f779449cca780da10fddf638c766b342cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Sat, 18 Feb 2023 01:21:48 +0100 Subject: [PATCH 14/33] WIP: - upload shows progress bar - more resiliant upload form - upload gets its own snug id - project can be selected by URL - minor visual changes --- .../components/Error/NoProjectFoundError.tsx | 11 ++ .../components/Import/ImportForm.tsx | 158 +++++++++--------- .../components/Import/ImportResult.tsx | 57 ++++--- src/depvis-next/components/Import/types.ts | 5 + .../components/Layout/Workspace.tsx | 28 +++- src/depvis-next/helpers/ApolloClientHelper.ts | 5 +- src/depvis-next/helpers/DbDataHelper.ts | 7 +- src/depvis-next/helpers/ImportSbomHelper.ts | 140 +++++++++++----- src/depvis-next/pages/api/import/index.ts | 48 +++++- src/depvis-next/pages/api/import/status.ts | 42 ++++- src/depvis-next/pages/toolbox.tsx | 16 +- src/depvis-next/pages/upload.tsx | 50 +++++- src/depvis-next/queues/ImportQueue.ts | 8 +- src/depvis-next/styles/custom.css | 4 + 14 files changed, 394 insertions(+), 185 deletions(-) create mode 100644 src/depvis-next/components/Error/NoProjectFoundError.tsx create mode 100644 src/depvis-next/components/Import/types.ts diff --git a/src/depvis-next/components/Error/NoProjectFoundError.tsx b/src/depvis-next/components/Error/NoProjectFoundError.tsx new file mode 100644 index 0000000..879120e --- /dev/null +++ b/src/depvis-next/components/Error/NoProjectFoundError.tsx @@ -0,0 +1,11 @@ +import Link from 'next/link'; +import { Container } from 'react-bootstrap'; + +export default function NoProjectFoundError(props) { + return ( + +

No project found!

+ Upload SBOM + + ); +} diff --git a/src/depvis-next/components/Import/ImportForm.tsx b/src/depvis-next/components/Import/ImportForm.tsx index 76bdea7..ccc773a 100644 --- a/src/depvis-next/components/Import/ImportForm.tsx +++ b/src/depvis-next/components/Import/ImportForm.tsx @@ -1,17 +1,18 @@ +import { useRouter } from 'next/router'; import { useState } from 'react'; -import { Alert, Button, Container, Form } from 'react-bootstrap'; +import { Alert, Button, Container, Form, Row } from 'react-bootstrap'; import { ImportResult } from './ImportResult'; +import { ImportFormData } from './types'; const allowedExtensionsRegex = /(\.json|\.xml)$/i; -const ImportForm = () => { - const [file, setFile] = useState(''); +const ImportForm = (props) => { + const { onSubmitCallback } = props; + const [file, setFile] = useState(undefined); const [preview, setPreview] = useState(''); - const [validated, setValidated] = useState(false); - const [isSubmitted, setIsSubmitted] = useState(false); - const [jobId, setJobId] = useState(null); - const [projectName, setProjectName] = useState(''); - const [projectVersion, setProjectVersion] = useState('1.0.0'); + const [validated, setValidated] = useState(false); + const [projectName, setProjectName] = useState(''); + const [projectVersion, setProjectVersion] = useState('1.0.0'); const handleFiles = (e: any) => { const files = e.target.files; @@ -19,8 +20,9 @@ const ImportForm = () => { const file = files[0]; console.log(file); if (!allowedExtensionsRegex.exec(file.name)) { - alert('This extension is not allowed!'); - setFile(''); + setFile(undefined); + setValidated(true); + e.target.value = ''; return; } setFile(file); @@ -35,89 +37,83 @@ const ImportForm = () => { } setValidated(true); - const body = { + const body: ImportFormData = { projectName: projectName, projectVersion: projectVersion, sbom: await file.text(), }; - console.log(body); - const res = await fetch('/api/import', { - body: await JSON.stringify(body), - headers: { 'Content-Type': 'application/json' }, - method: 'POST', - }); - const json = await res.json(); - setJobId(json.jobId); - setIsSubmitted(true); + await onSubmitCallback(body); }; const handlePreview = async (e: any) => { e.preventDefault(); - file && setPreview(await file.text()); + if (!file) return; + preview ? setPreview(undefined) : setPreview(await file.text()); }; - return isSubmitted ? ( - - ) : ( - - - All data currently stored in DB will be overwritten. - -
- - Project name - { - setProjectName(e.target.value); - }} - value={projectName} - > - Please select any XML / JSON file with SBOM. - - - Project version - { - setProjectVersion(e.target.value); - }} - value={projectVersion} - > - Please select any XML / JSON file with SBOM. - - - SBOM File - { - handleFiles(e); - }} - > - Please select any XML / JSON file with SBOM. - - - -
- - + return ( + +
+ + Project name + { + setProjectName(e.target.value); + }} + value={projectName} + > + Please select any XML / JSON file with SBOM. + + + Project version + { + setProjectVersion(e.target.value); + }} + value={projectVersion} + > + Please select any XML / JSON file with SBOM. + + + SBOM File + { + handleFiles(e); + }} + > + Please select any XML / JSON file with SBOM. + + + +
{preview && ( - -

Preview - {file.name}

- 
{preview}
- + + Contents of file {file.name} +
+ 
{preview}
+ )} - + ); }; diff --git a/src/depvis-next/components/Import/ImportResult.tsx b/src/depvis-next/components/Import/ImportResult.tsx index 44b1f29..9aeae00 100644 --- a/src/depvis-next/components/Import/ImportResult.tsx +++ b/src/depvis-next/components/Import/ImportResult.tsx @@ -1,53 +1,52 @@ import Link from 'next/link'; import { useRouter } from 'next/router'; import { useEffect, useState } from 'react'; -import { Container, Spinner } from 'react-bootstrap'; - -const jobStatesRunning = ['active', 'waiting', 'paused', 'delayed']; +import { Alert, Container, ProgressBar, Row, Spinner } from 'react-bootstrap'; +import { ImportStatusReponse } from '../../pages/api/import/status'; +const fetchInterval = 500; export const ImportResult = (props) => { - const [status, setStatus] = useState(''); - const [isRunning, setIsRunning] = useState(true); - + const [response, setResponse] = useState(undefined); + const { jobId } = props; const router = useRouter(); const getJobStatus = async (id) => { - console.log({ id: id, ir: isRunning }); const res = await fetch(`/api/import/status?id=${id}`); - const json = await res.json(); - setStatus(json.status); - setIsRunning(jobStatesRunning.includes(json.status)); + setResponse(await res.json()); }; useEffect(() => { - if (!props.jobId || !isRunning) { + if (!jobId || (response && !response.continueQuery)) { return; } const interval = setInterval(() => { - getJobStatus(props.jobId); - }, 500); + getJobStatus(jobId); + }, fetchInterval); return () => clearInterval(interval); - }, [isRunning]); + }, [response]); - if (!isRunning && status == 'completed') { - router.push('/'); + if (!response) return <>; + if (response.status == 'completed') { + const url = response.projectName ? `/?projectName=${response.projectName}` : '/'; + router.push(url); } return ( - -

Your file was successfully submited and it will be parsed now!

+ +

Importing project {response.projectName}

+

{response.progress && response.progress.message}

This can take several minutes depending on the size of the input.

- {isRunning ? ( - - - Loading... - - + {response.continueQuery ? ( + ) : ( -

- - Completed (status {status}) Go to the main app! - -

+ + {response.status == 'completed' ? ( + Import completed! Redirecting to your graph... + ) : ( + + {response.status} - {response.message} + + )} + )} ); diff --git a/src/depvis-next/components/Import/types.ts b/src/depvis-next/components/Import/types.ts new file mode 100644 index 0000000..8287735 --- /dev/null +++ b/src/depvis-next/components/Import/types.ts @@ -0,0 +1,5 @@ +export type ImportFormData = { + projectName: string; + projectVersion: string; + sbom: string; +}; diff --git a/src/depvis-next/components/Layout/Workspace.tsx b/src/depvis-next/components/Layout/Workspace.tsx index 43c021f..da6047f 100644 --- a/src/depvis-next/components/Layout/Workspace.tsx +++ b/src/depvis-next/components/Layout/Workspace.tsx @@ -1,4 +1,6 @@ import { useLazyQuery, useQuery } from '@apollo/client'; +import Link from 'next/link'; +import { useRouter } from 'next/router'; import { useCallback, useEffect, useState } from 'react'; import { Container, Row } from 'react-bootstrap'; import { @@ -12,6 +14,7 @@ import ComponentDetails from '../Details/ComponentDetails'; import Details from '../Details/Details'; import VulnerabilityDetails from '../Details/VulnerabilityDetails'; import Dropdown from '../Dropdown/Dropdown'; +import NoProjectFoundError from '../Error/NoProjectFoundError'; import { GraphConfig } from '../Graph/GraphConfig'; import GraphControl from '../GraphControl/GraphControl'; import ImportForm from '../Import/ImportForm'; @@ -39,13 +42,12 @@ const Workspace = () => { const [getGraphData, { loading, error, data }] = useLazyQuery(getAllComponentsQuery); const { data: projects, loading: projectsLoading } = useQuery(getProjectsQuery, { onCompleted: (data) => { - if (data.projects.length > 0) { - setSelectedProject(data.projects[0].id); - } else { - setSelectedProject(undefined); - } + const project = selectProject(data.projects, router.query.projectName); + console.log(project); + setSelectedProject(project); }, }); + const router = useRouter(); useEffect(() => { if (data) { @@ -57,7 +59,7 @@ const Workspace = () => { if (selectedProject) { console.log('Getting data'); - getGraphData({ variables: { projectId: selectedProject } }); + getGraphData({ variables: { projectId: selectedProject }, pollInterval: 1000 }); } }, [selectedProject]); useEffect(() => { @@ -104,12 +106,17 @@ const Workspace = () => { ); if (projectsLoading) return ; - if (!selectedProject) return ; + if (!selectedProject) return ; return ( - setSelectedProject(e)} /> + setSelectedProject(e)} + default={selectedProject} + /> handleSelectedSearchResult(obj)} /> { }; export default Workspace; + +const selectProject = (projects, queryProjectName) => { + if (!queryProjectName || projects.filter((p) => p.name === queryProjectName).length == 0) return projects[0].id; + return projects.filter((p) => p.name === queryProjectName)[0].id; +}; diff --git a/src/depvis-next/helpers/ApolloClientHelper.ts b/src/depvis-next/helpers/ApolloClientHelper.ts index d13978a..ac9f765 100644 --- a/src/depvis-next/helpers/ApolloClientHelper.ts +++ b/src/depvis-next/helpers/ApolloClientHelper.ts @@ -8,10 +8,9 @@ import { gqlUrlPath } from '../pages/api/graphql'; */ export const createApolloClient = () => { if (!process.env.NEXT_PUBLIC_SERVER_URI) { - console.error("No server URI was provided, using default connection") + console.error('No server URI was provided, using default connection'); } - - const uri = urlJoin(process.env.NEXT_PUBLIC_SERVER_URI || "http://localhost:3000", gqlUrlPath); + const uri = urlJoin(process.env.NEXT_PUBLIC_SERVER_URI || 'http://localhost:3000', gqlUrlPath); console.log(`Creating GQL Client (connection to ${uri})`); const link = new HttpLink({ uri: uri, diff --git a/src/depvis-next/helpers/DbDataHelper.ts b/src/depvis-next/helpers/DbDataHelper.ts index 673d1d7..9d9eb70 100644 --- a/src/depvis-next/helpers/DbDataHelper.ts +++ b/src/depvis-next/helpers/DbDataHelper.ts @@ -33,9 +33,7 @@ async function sendMutation(mutation: any, variables?: Object) { return res; } -export async function ProjectExists(projectName: string) { - console.log(projectName); - +export async function TryGetProjectByName(projectName: string) { const query = gql` query Project($projectName: String!) { projects(where: { name_CONTAINS: $projectName }) { @@ -46,8 +44,7 @@ export async function ProjectExists(projectName: string) { } `; const data = await sendQuery(query, { projectName: projectName }); - console.log(data); - return data.projects.length > 0; + return data.projects; } export async function ComponentExists(componentName: string) { diff --git a/src/depvis-next/helpers/ImportSbomHelper.ts b/src/depvis-next/helpers/ImportSbomHelper.ts index d2d599d..40bdd87 100644 --- a/src/depvis-next/helpers/ImportSbomHelper.ts +++ b/src/depvis-next/helpers/ImportSbomHelper.ts @@ -5,55 +5,82 @@ import { processBatch } from './BatchHelper'; import { CreateComponents, CreateProject, + CreateUpdateVulnerability, UpdateComponentDependencies, UpdateProjectDependencies, } from './DbDataHelper'; -export async function ImportSbom(bom: any, projectName, projectVersion) { - // Prepare main component if exists - const mainComponentParsed = bom.metadata.component; - const mainComponent: Component | undefined = mainComponentParsed - ? { - type: mainComponentParsed.type, - name: mainComponentParsed.name, - purl: mainComponentParsed.purl || `${mainComponentParsed.name}@${mainComponentParsed.version}`, - version: mainComponentParsed.version, - author: mainComponentParsed.author, - publisher: mainComponentParsed.publisher, - } - : undefined; +export async function ImportSbom(bom: any, projectName, projectVersion, updateProgressCallback) { + try { + // Prepare main component if exists + const updateProgress = async (percent, message) => { + console.log(message); + + await updateProgressCallback({ message: message, percent: percent }); + console.log(message); + }; + await updateProgress(1, 'Parsing main component'); + const mainComponentParsed = bom.metadata.component; + const mainComponent: Component | undefined = mainComponentParsed + ? { + type: mainComponentParsed.type, + name: mainComponentParsed.name, + purl: mainComponentParsed.purl || `${mainComponentParsed.name}@${mainComponentParsed.version}`, + version: mainComponentParsed.version, + author: mainComponentParsed.author, + publisher: mainComponentParsed.publisher, + } + : undefined; + await updateProgress(2, 'Preparing project'); + // Prepare project + let project: Project = { + name: projectName || bom.metadata.component.name, + version: projectVersion || bom.metadata.component.version || '1.0.0', + date: bom.metadata.timestamp || '1970-01-01', + }; + console.log(project); + await updateProgress(3, 'Preparing dependencies'); + // Prepare dependencies + let dependencies = GetDependencies(bom.dependencies.dependency); - // Prepare project - let project: Project = { - name: projectName || bom.metadata.component.name, - version: projectVersion || bom.metadata.component.version || '1.0.0', - date: bom.metadata.timestamp || '1970-01-01', - }; - console.log(project); + // Currently there is no support for managing older projects - we first need to clear the DB + //await DeleteAllData(); + await updateProgress(4, 'Creating project in DB'); + // Create all objects in DB + const projectResponse = await CreateProject(project); + const projectId = projectResponse.createProjects.projects[0].id; - // Prepare dependencies - let dependencies = GetDependencies(bom.dependencies.dependency); + await updateProgress(3, 'Preparing components'); + // Prepare components + let components: [Component] = GetComponents(bom); + mainComponent && components.push(mainComponent); + console.log(components); - // Currently there is no support for managing older projects - we first need to clear the DB - //await DeleteAllData(); - // Create all objects in DB - const projectResponse = await CreateProject(project); - const projectId = projectResponse.createProjects.projects[0].id; + await updateProgress(10, 'Creating components'); + await CreateComponents(components, projectId); + await updateProgress(50, 'Updating dependencies'); - // Prepare components - let components: [Component] = GetComponents(bom); - mainComponent && components.push(mainComponent); - console.log(components); + await UpdateProjectDependencies(projectId, [mainComponent]); + await UpdateComponentDependencies(dependencies, projectId); + await updateProgress(70, 'Fetching vulnerabilities'); - await CreateComponents(components, projectId); - await UpdateProjectDependencies(projectId, [mainComponent]); - await UpdateComponentDependencies(dependencies, projectId); + //Vulnerabilities + const purlList = components.map((c) => { + return c.purl; + }); + const r = await processBatch(purlList, VulnFetcherHandler); + await updateProgress(90, 'Creating vulnerabilities in DB'); - //Vulnerabilities - const purlList = components.map((c) => { - return c.purl; - }); - await processBatch(purlList, VulnFetcherHandler); + r.forEach(async (component) => { + if (component.vulnerabilities.length > 0) { + console.log('Creating %d vulns for %s', component.vulnerabilities.length, component.purl); + await CreateUpdateVulnerability(component.purl, component.vulnerabilities); + } + }); + await updateProgress(90, 'Creating vulnerabilities in DB'); + } catch { + console.error('Recovery needed'); + } } function GetComponents(bom: any) { let components = bom.components.component; @@ -90,3 +117,38 @@ function GetDependencies(dependencies: any) { }); return res; } + +/** + * Function will find project with highest version number in format 1.2.3 + * Note: all project must have the same version length + * @param projects List of projects + * @returns The largest project + */ +export function getLatestProjectVersion(projects) { + return projects.reduce((highestVersionObj, currentObj) => { + if (!highestVersionObj || compareVersions(currentObj.version, highestVersionObj.version) > 0) { + return currentObj; + } else { + return highestVersionObj; + } + }, undefined); +} + +/** + * Function will parse the versions from format '1.2.3' and compare them + * @param version1 version of project 1 + * @param version2 version of project 2 + * @returns 1 if version1 > version2, -1 if version1 < version2, 0 if they are equal + */ +export function compareVersions(version1: string, version2: string): number { + const arr1 = version1.split('.').map((num) => parseInt(num, 10)); + const arr2 = version2.split('.').map((num) => parseInt(num, 10)); + const num1 = parseInt(arr1.join(''), 10); + const num2 = parseInt(arr2.join(''), 10); + if (num1 > num2) { + return 1; + } else if (num1 < num2) { + return -1; + } + return 0; +} diff --git a/src/depvis-next/pages/api/import/index.ts b/src/depvis-next/pages/api/import/index.ts index 20702ef..a2e5752 100644 --- a/src/depvis-next/pages/api/import/index.ts +++ b/src/depvis-next/pages/api/import/index.ts @@ -1,6 +1,7 @@ import { Queue } from 'bullmq'; import { XMLParser } from 'fast-xml-parser'; -import { ImportSbom } from '../../../helpers/ImportSbomHelper'; +import { TryGetProjectByName } from '../../../helpers/DbDataHelper'; +import { compareVersions, getLatestProjectVersion, ImportSbom } from '../../../helpers/ImportSbomHelper'; import { defaultBullConfig, emptyQueue } from '../../../helpers/QueueHelper'; import { GetVulnQueueName } from '../../../queues/GetVulnQueue'; import { ImportQueueName, ImportSbomJobData } from '../../../queues/ImportQueue'; @@ -38,12 +39,29 @@ type ImportRequestBody = { export default async function handler(req, res) { try { - // if (req.headers['content-type'] !== 'application/xml') { - // res.status(500).json({ error: "Content-type must be 'application/xml'" }); - // return; - // } - const body = req.body as ImportRequestBody; - console.log('Recieved body: %s', body.projectName); + //Validate request + if (req.headers['content-type'] !== 'application/json') { + res.status(500).json({ error: "Content-type must be 'application/json'" }); + return; + } + const body: ImportRequestBody = req.body; + if (!validateImportRequestBody(body)) { + return res.status(400).json({ isError: true, error: 'Request body is malformed!' }); + } + console.log(body.projectName); + + const projects = await TryGetProjectByName(body.projectName); + console.log(projects); + if (projects.length != 0) { + const highestVersionProject = getLatestProjectVersion(projects); + if (compareVersions(body.projectVersion, highestVersionProject.version) != 1) { + return res + .status(400) + .json({ isError: true, error: `Project must have higher version than ${highestVersionProject.version}` }); + } + } + + // Parse sbom const result = await parseXml(body.sbom); if (result.isError) { console.log('Import failed with %s', result.errorMessage); @@ -52,11 +70,15 @@ export default async function handler(req, res) { //Clear vuln queue emptyQueue(GetVulnQueue); + + // Create new job const job = await ImportQueue.add(body.projectName.toString(), { sbom: result.sbom, projectName: body.projectName, projectVersion: body.projectVersion, - }); + }); + + //Return response const response: ImportResult = { jobId: job.id, isError: false }; return res.status(200).json(response); } catch (err) { @@ -65,6 +87,11 @@ export default async function handler(req, res) { } } +function validateImportRequestBody(body: ImportRequestBody) { + if (!body || !body.projectName || !body.projectVersion || !body.sbom) return false; + return true; +} + // Function validates that object contains required properties function validateSbomXml(parsedXml): ImportResult { const sbom = parsedXml.bom; @@ -83,6 +110,11 @@ function validateSbomXml(parsedXml): ImportResult { isError: true, errorMessage: "Validation failed - Missing 'components' parameter in the file.", }; + if (!sbom.dependencies) + return { + isError: true, + errorMessage: "Validation failed - Missing 'dependencies' parameter in the file.", + }; return { isError: false, sbom: sbom }; } diff --git a/src/depvis-next/pages/api/import/status.ts b/src/depvis-next/pages/api/import/status.ts index 70b5496..5d41571 100644 --- a/src/depvis-next/pages/api/import/status.ts +++ b/src/depvis-next/pages/api/import/status.ts @@ -5,12 +5,42 @@ import { ImportQueueName } from '../../../queues/ImportQueue'; //Bull queue const ImportQueue = new Queue(ImportQueueName, defaultBullConfig); +export type ImportStatusReponse = { + status: string; + progress?: any; + message: string; + continueQuery: boolean; + projectName?: string; +}; export default async function handler(req, res) { if (req.method !== 'GET') return res.status(405); - if (!('id' in req.query && !isNaN(req.query.id))) - return res.status(400).json({ - error: "'id' parameter is missing or its value is not a number", - }); - const result = await await ImportQueue.getJob(req.query.id); - return res.status(200).json({ status: await result.getState() }); + if (!('id' in req.query && !isNaN(req.query.id))) { + const response: ImportStatusReponse = { + status: 'Server Error', + message: 'The jobId was not provided or the value was null!', + continueQuery: false, + }; + return res.status(400).json(response); + } + try { + const result = await await ImportQueue.getJob(req.query.id); + const status = await result.getState(); + const projectName = result.data.projectName; + const response: ImportStatusReponse = { + status: status, + progress: result.progress, + message: '', + continueQuery: result.finishedOn === undefined, + projectName: projectName, + }; + return res.status(200).json(response); + } catch (err) { + console.error(err); + const response: ImportStatusReponse = { + status: 'Server Error', + message: 'There was an error while processing the request. See server log for more information.', + continueQuery: false, + }; + return res.status(500).json(response); + } } diff --git a/src/depvis-next/pages/toolbox.tsx b/src/depvis-next/pages/toolbox.tsx index e2622c4..5ff2f71 100644 --- a/src/depvis-next/pages/toolbox.tsx +++ b/src/depvis-next/pages/toolbox.tsx @@ -1,6 +1,8 @@ +import Link from 'next/link'; +import { useRouter } from 'next/router'; import { PackageURL } from 'packageurl-js'; -import { useState } from 'react'; -import { Button, Col, Container, Row, Stack } from 'react-bootstrap'; +import { useEffect, useState } from 'react'; +import { Button, Col, Container, Form, ProgressBar, Row, Stack } from 'react-bootstrap'; import { ParsePurl } from '../components/Toolbox/ParsePurl'; const Toolbox = () => { @@ -14,6 +16,12 @@ const Toolbox = () => { const res = await fetch('/api/vuln'); console.log(res); }; + + const router = useRouter(); + useEffect(() => { + console.log(router); + console.log(router.query); + }, [router]); return ( @@ -31,6 +39,10 @@ const Toolbox = () => { + Query param is {router.query.test} + + + ); }; diff --git a/src/depvis-next/pages/upload.tsx b/src/depvis-next/pages/upload.tsx index 3c26929..8421e00 100644 --- a/src/depvis-next/pages/upload.tsx +++ b/src/depvis-next/pages/upload.tsx @@ -1,6 +1,54 @@ +import Link from 'next/link'; +import { useRouter } from 'next/router'; +import { useState } from 'react'; +import { Alert, Container } from 'react-bootstrap'; import ImportForm from '../components/Import/ImportForm'; +import { ImportResult } from '../components/Import/ImportResult'; + +const ImportApiUrl = '/api/import'; const Upload = () => { - return ; + const router = useRouter(); + const [serverResponse, setServerResponse] = useState(); + + const handleSubmit = async (data) => { + const res = await fetch(ImportApiUrl, { + body: await JSON.stringify(data), + headers: { 'Content-Type': 'application/json' }, + method: 'POST', + }); + if (res.status == 200) { + const json = await res.json(); + router.push(`?jobId=${json.jobId}`, undefined, { shallow: true }); + } else { + setServerResponse(await res.json()); + } + }; + + if (router.query.jobId) { + return ; + } + return ( + + {serverResponse && Form can not be submitted - {serverResponse.error}} + +

How to create SBOM for project

+
    +
  1. + Go to CycloneDX Tool Center and find a tool to + generate SBOM according to your programming language +
    2. +
  2. Create the SBOM file according to the documentation for a given tool - Choose XML as output file.
    3. +
  3. Upload generated file here
    4. +
+ + { + console.log(data); + handleSubmit(data); + }} + /> + + ); }; export default Upload; diff --git a/src/depvis-next/queues/ImportQueue.ts b/src/depvis-next/queues/ImportQueue.ts index e74785c..4cc17e4 100644 --- a/src/depvis-next/queues/ImportQueue.ts +++ b/src/depvis-next/queues/ImportQueue.ts @@ -13,9 +13,11 @@ export type ImportSbomJobData = { const worker = new Worker( ImportQueueName, async (job) => { - const test = job.data; - console.log(test); - const res = ImportSbom(test.sbom, test.projectName, test.projectVersion); + const updateProgress = async (input) => { + await job.updateProgress(input); + }; + const data = job.data; + const res = await ImportSbom(data.sbom, data.projectName, data.projectVersion, updateProgress); return res; }, defaultBullConfig diff --git a/src/depvis-next/styles/custom.css b/src/depvis-next/styles/custom.css index f633830..64214e9 100644 --- a/src/depvis-next/styles/custom.css +++ b/src/depvis-next/styles/custom.css @@ -32,3 +32,7 @@ overflow: hidden; white-space: nowrap; } + +.import-preview { + max-height: 25rem; +} From b59575aaadb1751a2844fabe6281eafcff44d30c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Sat, 18 Feb 2023 21:34:00 +0100 Subject: [PATCH 15/33] Create DB contstraints --- src/depvis-next/pages/api/graphql.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/depvis-next/pages/api/graphql.ts b/src/depvis-next/pages/api/graphql.ts index ace55d9..5fefb9b 100644 --- a/src/depvis-next/pages/api/graphql.ts +++ b/src/depvis-next/pages/api/graphql.ts @@ -27,8 +27,10 @@ const driver = neo4j.driver( async function handler(req, res) { const neoSchema = new Neo4jGraphQL({ typeDefs, driver }); + const schema = await neoSchema.getSchema(); + await neoSchema.assertIndexesAndConstraints({ options: { create: true } }); const apolloServer = new ApolloServer({ - schema: await neoSchema.getSchema(), + schema: schema, introspection: NEO4J_CONFIG.introspection, plugins: NEO4J_CONFIG.plugins, }); From 720b33b76926ba3e2d659e63b13b7c78c17f2d89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Sat, 18 Feb 2023 21:34:20 +0100 Subject: [PATCH 16/33] Introduce ProjectVersion type in DB --- src/depvis-next/types/gqlTypes.ts | 9 +++++++-- src/depvis-next/types/project.ts | 13 ++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/depvis-next/types/gqlTypes.ts b/src/depvis-next/types/gqlTypes.ts index 9818825..d7878cf 100644 --- a/src/depvis-next/types/gqlTypes.ts +++ b/src/depvis-next/types/gqlTypes.ts @@ -7,7 +7,7 @@ import { gql } from 'apollo-server-micro'; export const typeDefs = gql` type Component { id: ID @id - project: [Project!]! @relationship(type: "BELONGS_TO", direction: OUT) + project: [ProjectVersion!]! @relationship(type: "BELONGS_TO", direction: OUT) purl: String name: String type: String @@ -51,7 +51,11 @@ export const typeDefs = gql` type Project { id: ID @id - name: String + name: String @unique + versions: [ProjectVersion!]! @relationship(type: "HAS_VERSION", direction: OUT) + } + + type ProjectVersion { version: String allComponents: [Component!]! @cypher( @@ -75,6 +79,7 @@ export const typeDefs = gql` ) component: [Component!]! @relationship(type: "DEPENDS_ON", direction: OUT) date: Date + project: [Project!]! @relationship(type: "HAS_VERSION", direction: IN) } type Reference { diff --git a/src/depvis-next/types/project.ts b/src/depvis-next/types/project.ts index bc49162..e8ae8a3 100644 --- a/src/depvis-next/types/project.ts +++ b/src/depvis-next/types/project.ts @@ -1,4 +1,4 @@ -import { Component } from "./component"; +import { Component } from './component'; export type Project = { /** @@ -9,6 +9,13 @@ export type Project = { * Friendly name */ name: string; + /** + * List of versions related to a given project + */ + versions?: [ProjectVersion]; +}; + +export type ProjectVersion = { /** * Project version */ @@ -21,4 +28,8 @@ export type Project = { * Date when SBOM was created for a given project */ date?: Date; + /** + * Represents to which project a given version belongs + */ + project: Project; }; From e168ee0ae5cfda4ac36aee45defe31ea6f84435e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Sat, 18 Feb 2023 23:23:50 +0100 Subject: [PATCH 17/33] Update types --- src/depvis-next/types/component.ts | 4 ++++ src/depvis-next/types/gqlTypes.ts | 10 ++++++---- src/depvis-next/types/project.ts | 12 +++++++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/depvis-next/types/component.ts b/src/depvis-next/types/component.ts index 0e3740d..3e787c6 100644 --- a/src/depvis-next/types/component.ts +++ b/src/depvis-next/types/component.ts @@ -52,3 +52,7 @@ export type Component = { */ project?: Project; }; + +export type ComponentDto = Component & { + projectVersion: any; +}; diff --git a/src/depvis-next/types/gqlTypes.ts b/src/depvis-next/types/gqlTypes.ts index d7878cf..c183b41 100644 --- a/src/depvis-next/types/gqlTypes.ts +++ b/src/depvis-next/types/gqlTypes.ts @@ -7,7 +7,7 @@ import { gql } from 'apollo-server-micro'; export const typeDefs = gql` type Component { id: ID @id - project: [ProjectVersion!]! @relationship(type: "BELONGS_TO", direction: OUT) + projectVersion: ProjectVersion @relationship(type: "BELONGS_TO", direction: OUT) purl: String name: String type: String @@ -47,6 +47,7 @@ export const typeDefs = gql` cvssVector: String cwe: String references: [Reference!]! @relationship(type: "HAS_REFERENCE", direction: OUT) + affectedComponents: [Component!]! @relationship(type: "HAS_VULNERABILITY", direction: IN) } type Project { @@ -56,11 +57,12 @@ export const typeDefs = gql` } type ProjectVersion { + id: ID @id version: String allComponents: [Component!]! @cypher( statement: """ - MATCH (this)-->(c)-[:DEPENDS_ON*]->(c2) + MATCH (this)-->(c)-[:DEPENDS_ON*0..]->(c2) WITH collect(c) + collect(c2) as all UNWIND all as single RETURN distinct single @@ -69,7 +71,7 @@ export const typeDefs = gql` allVulnerableComponents: [Component!]! @cypher( statement: """ - MATCH a=(v:Vulnerability)<-[:HAS_VULNERABILITY]-(c1:Component)<-[:DEPENDS_ON*]-(c2)<-[:DEPENDS_ON]-(this) + MATCH a=(v:Vulnerability)<-[:HAS_VULNERABILITY]-(c1:Component)<-[:DEPENDS_ON*0..]-(c2)<-[:DEPENDS_ON]-(this) WITH NODES(a) AS nodes UNWIND nodes AS n WITH n @@ -79,7 +81,7 @@ export const typeDefs = gql` ) component: [Component!]! @relationship(type: "DEPENDS_ON", direction: OUT) date: Date - project: [Project!]! @relationship(type: "HAS_VERSION", direction: IN) + project: Project! @relationship(type: "HAS_VERSION", direction: IN) } type Reference { diff --git a/src/depvis-next/types/project.ts b/src/depvis-next/types/project.ts index e8ae8a3..28fec08 100644 --- a/src/depvis-next/types/project.ts +++ b/src/depvis-next/types/project.ts @@ -15,7 +15,7 @@ export type Project = { versions?: [ProjectVersion]; }; -export type ProjectVersion = { +type ProjectVersionBase = { /** * Project version */ @@ -28,8 +28,18 @@ export type ProjectVersion = { * Date when SBOM was created for a given project */ date?: Date; +}; + +export type ProjectVersion = ProjectVersionBase & { /** * Represents to which project a given version belongs */ project: Project; }; + +export type ProjectVersionDto = ProjectVersionBase & { + /** + * Represents graphql connection + */ + project: any; +}; From 0907c5898baf89445605df4e942f10595649a7ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Sat, 18 Feb 2023 23:24:07 +0100 Subject: [PATCH 18/33] WIP: rewrite DBDataHelper --- src/depvis-next/helpers/DbDataHelper.ts | 39 ++++--- src/depvis-next/helpers/DbDataProvider.ts | 134 ++++++++++++++++++++++ 2 files changed, 159 insertions(+), 14 deletions(-) create mode 100644 src/depvis-next/helpers/DbDataProvider.ts diff --git a/src/depvis-next/helpers/DbDataHelper.ts b/src/depvis-next/helpers/DbDataHelper.ts index 9d9eb70..6042f9e 100644 --- a/src/depvis-next/helpers/DbDataHelper.ts +++ b/src/depvis-next/helpers/DbDataHelper.ts @@ -1,6 +1,6 @@ import { DocumentNode, gql } from '@apollo/client'; import { randomBytes } from 'crypto'; -import { Component } from '../types/component'; +import { Component, ComponentDto } from '../types/component'; import { Project } from '../types/project'; import { Vulnerability } from '../types/vulnerability'; import { createApolloClient } from './ApolloClientHelper'; @@ -13,16 +13,16 @@ const chunkSize = 100; * @returns Data from successful query * @throws Error if there were some error during fetch */ -async function sendQuery(query: DocumentNode, variables?: Object) { +export async function sendGQLQuery(query: DocumentNode, variables?: Object) { const client = createApolloClient(); const res = await client.query({ query: query, variables: variables }); if (res.errors) { throw Error(res.errors.toString()); } - return res.data; + return res; } -async function sendMutation(mutation: any, variables?: Object) { +export async function sendGQLMutation(mutation: any, variables?: Object) { console.log(`Sending mutation ${mutation} with variables ${await JSON.stringify(variables)}`); const client = createApolloClient(); const res = await client.mutate({ mutation: mutation, variables: variables }); @@ -43,7 +43,7 @@ export async function TryGetProjectByName(projectName: string) { } } `; - const data = await sendQuery(query, { projectName: projectName }); + const { data } = await sendGQLQuery(query, { projectName: projectName }); return data.projects; } @@ -57,7 +57,7 @@ export async function ComponentExists(componentName: string) { } } `; - const data = await sendQuery(query, { componentName: componentName }); + const { data } = await sendGQLQuery(query, { componentName: componentName }); return data.projects.length > 0; } @@ -79,7 +79,7 @@ export async function CreateComponents(components: [Component?], projectId: stri for (let i = 0; i < components.length; i += chunkSize) { const chunk = components.slice(i, i + chunkSize); const chunkWithProjectId = AddProjectToComponents(chunk, projectId); - const { data } = await sendMutation(mutation, { components: chunkWithProjectId }); + const { data } = await sendGQLMutation(mutation, { components: chunkWithProjectId }); res.concat(data.components); } return res; @@ -109,7 +109,7 @@ export async function CreateProject(project: Project) { } } `; - const { data } = await sendMutation(mutation, { project: [project] }); + const { data } = await sendGQLMutation(mutation, { project: [project] }); return data; } @@ -126,7 +126,7 @@ export async function GetVulnerability(vulnerabilityId) { } } `; - const data = await sendQuery(query, { vulnerabilityId: vulnerabilityId }); + const { data } = await sendGQLQuery(query, { vulnerabilityId: vulnerabilityId }); return data.vulnerabilities; } export async function CreateUpdateVulnerability(purl: string, vulnerabilities: [Vulnerability?]) { @@ -160,7 +160,7 @@ export async function CreateUpdateVulnerability(purl: string, vulnerabilities: [ const vulnArray = newVulnerabilities.map((v) => { return { node: PrepareVulnAsGQL(v) }; }); - const { data } = await sendMutation(mutation, { + const { data } = await sendGQLMutation(mutation, { purl: purl, vuln_array: vulnArray, }); @@ -199,7 +199,7 @@ export async function UpdateProjectDependencies(projectId: string, components: [ } `; - const { data } = await sendMutation(mutation, { + const { data } = await sendGQLMutation(mutation, { projectId: projectId, componentsPurl: components.map((c) => { return { purl: c.purl }; @@ -222,7 +222,7 @@ export async function UpdateComponentDependencies(dependencies, projectId: strin ${chunkMutation} } `; - const { data } = await sendMutation(mutation, { projectId: projectId }); + const { data } = await sendGQLMutation(mutation, { projectId: projectId }); console.log(data); } } @@ -234,7 +234,7 @@ export async function GetComponents() { } } `; - const data = await sendQuery(query); + const { data } = await sendGQLQuery(query); return data; } @@ -283,6 +283,17 @@ export async function DeleteAllData() { } } `; - const { data } = await sendMutation(mutation); + const { data } = await sendGQLMutation(mutation); console.log(data); } + +export function AddProjectVersionConnectProject(projectId: string) { + return { connect: { where: { node: { id: projectId } } } }; +} + +export function CreateComponentsConnectProjectVersion(components: [ComponentDto], projectVersionId: string) { + const ConnectProjectVersion = { connect: { where: { node: { id: projectVersionId } } } }; + return components.map((c) => { + return { ...c, projectVersion: ConnectProjectVersion }; + }); +} diff --git a/src/depvis-next/helpers/DbDataProvider.ts b/src/depvis-next/helpers/DbDataProvider.ts new file mode 100644 index 0000000..d50aa51 --- /dev/null +++ b/src/depvis-next/helpers/DbDataProvider.ts @@ -0,0 +1,134 @@ +import { gql } from '@apollo/client'; +import { Component } from '../types/component'; +import { Project, ProjectVersion, ProjectVersionDto } from '../types/project'; +import { createApolloClient } from './ApolloClientHelper'; +import { sendGQLQuery, sendGQLMutation, AddProjectVersionConnectProject } from './DbDataHelper'; + +/** + * Function will create new project with optional first component + * @param project Project data + * @returns ID of newly created project + */ +export async function CreateProject(project: Project) { + if (!project) return; + const mutation = gql` + mutation CreateProject($project: [ProjectCreateInput!]!) { + createProjects(input: $project) { + projects { + id + } + } + } + `; + const { data } = await sendGQLMutation(mutation, { project: [project] }); + return data; +} + +/** + * Get all projects that match provided name + * @param projectName Project name + * @returns List of projects that match given name + */ +export async function GetProjectByName(projectName: string) { + const query = gql` + query Project($projectName: String!) { + projects(where: { name: $projectName }) { + id + name + versions { + id + version + } + } + } + `; + const { data } = await sendGQLQuery(query, { projectName: projectName }); + return data.projects; +} + +/** + * Get project by Id + * @param projectId Project Id + * @returns List of projects that match given Id + */ +export async function GetProjectById(projectId: string) { + const query = gql` + query Project($projectName: String!) { + projects(where: { id: $projectId }) { + id + name + versions { + id + version + } + } + } + `; + const { data } = await sendGQLQuery(query, { projectId: projectId }); + return data.projects; +} + +/** + * Creates new version for a given project + * @param projectId Project ID + * @param version new version identificator + * @returns ID of the new project version + */ +export async function AddProjectVersion(projectId, version: string) { + if (!projectId || !version) { + console.log('AddProjectComponent is missing some inputs! %s, %s', projectId, version); + return; + } + const mutation = gql` + mutation AddProjectVersion($projectVersion: ProjectVersionCreateInput!) { + createProjectVersions(input: [$projectVersion]) { + info { + nodesCreated + } + projectVersions { + id + } + } + } + `; + const projectVersion: ProjectVersionDto = { version: version, project: AddProjectVersionConnectProject(projectId) }; + const { data } = await sendGQLMutation(mutation, { projectVersion: projectVersion }); + return data.projectVersions.id; +} + +/** + * Deletes project version with all its connected components, references and vulnerabilities + * @param projectVersionId ID of the project version + */ +export async function DeleteProjectVersion(projectVersionId: string) { + const mutation = gql` + mutation DeleteProjectVersion($projectVersionId: ID) { + deleteProjectVersions(where: { id: $projectVersionId }, delete: { component: { where: {} } }) { + nodesDeleted + } + } + `; + const { data } = await sendGQLMutation(mutation, { projectVersionId: projectVersionId }); +} + +export async function CreateComponents(components: [Component?], projectVersionId: string) { + if (!components || components.length == 0) { + console.log('CreateComponents - No components provided!'); + return; + } + const mutation = gql` + mutation CreateComponents($components: [ComponentCreateInput!]!) { + createComponents(input: $components) { + info { + nodesCreated + } + components { + id + name + purl + } + } + } + `; + const { data } = await sendGQLMutation(mutation, { components: projectVersionId }); +} From 88a7bc70f733b75742e20f66d47ddca0724e589b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Sat, 18 Feb 2023 23:24:21 +0100 Subject: [PATCH 19/33] WIP: Rewrite import --- src/depvis-next/helpers/ImportSbomHelper.ts | 104 +++++++++++++------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/src/depvis-next/helpers/ImportSbomHelper.ts b/src/depvis-next/helpers/ImportSbomHelper.ts index 40bdd87..7b22586 100644 --- a/src/depvis-next/helpers/ImportSbomHelper.ts +++ b/src/depvis-next/helpers/ImportSbomHelper.ts @@ -3,23 +3,28 @@ import { Project } from '../types/project'; import { VulnFetcherHandler } from '../vulnerability-mgmt/VulnFetcherHandler'; import { processBatch } from './BatchHelper'; import { - CreateComponents, - CreateProject, CreateUpdateVulnerability, + TryGetProjectByName, UpdateComponentDependencies, UpdateProjectDependencies, } from './DbDataHelper'; +import { CreateProject, GetProjectById, GetProjectByName } from './DbDataProvider'; -export async function ImportSbom(bom: any, projectName, projectVersion, updateProgressCallback) { +type ProjectInput = { + name: string; + id: string; +}; + +export async function ImportSbom(bom: any, projectInput: ProjectInput, projectVersion, updateProgressCallback) { try { // Prepare main component if exists const updateProgress = async (percent, message) => { console.log(message); - await updateProgressCallback({ message: message, percent: percent }); console.log(message); }; - await updateProgress(1, 'Parsing main component'); + const projectId = GetProjectId(projectInput); + const projectVersionId = await updateProgress(1, 'Parsing main component'); const mainComponentParsed = bom.metadata.component; const mainComponent: Component | undefined = mainComponentParsed ? { @@ -42,42 +47,36 @@ export async function ImportSbom(bom: any, projectName, projectVersion, updatePr await updateProgress(3, 'Preparing dependencies'); // Prepare dependencies let dependencies = GetDependencies(bom.dependencies.dependency); - // Currently there is no support for managing older projects - we first need to clear the DB //await DeleteAllData(); await updateProgress(4, 'Creating project in DB'); // Create all objects in DB - const projectResponse = await CreateProject(project); - const projectId = projectResponse.createProjects.projects[0].id; - - await updateProgress(3, 'Preparing components'); - // Prepare components - let components: [Component] = GetComponents(bom); - mainComponent && components.push(mainComponent); - console.log(components); - - await updateProgress(10, 'Creating components'); - await CreateComponents(components, projectId); - await updateProgress(50, 'Updating dependencies'); - - await UpdateProjectDependencies(projectId, [mainComponent]); - await UpdateComponentDependencies(dependencies, projectId); - await updateProgress(70, 'Fetching vulnerabilities'); - - //Vulnerabilities - const purlList = components.map((c) => { - return c.purl; - }); - const r = await processBatch(purlList, VulnFetcherHandler); - await updateProgress(90, 'Creating vulnerabilities in DB'); - - r.forEach(async (component) => { - if (component.vulnerabilities.length > 0) { - console.log('Creating %d vulns for %s', component.vulnerabilities.length, component.purl); - await CreateUpdateVulnerability(component.purl, component.vulnerabilities); - } - }); - await updateProgress(90, 'Creating vulnerabilities in DB'); + // const projectResponse = await CreateProject(project); + // const projectId = projectResponse.createProjects.projects[0].id; + // await updateProgress(3, 'Preparing components'); + // // Prepare components + // let components: [Component] = GetComponents(bom); + // mainComponent && components.push(mainComponent); + // console.log(components); + // await updateProgress(10, 'Creating components'); + // await CreateComponents(components, projectId); + // await updateProgress(50, 'Updating dependencies'); + // await UpdateProjectDependencies(projectId, [mainComponent]); + // await UpdateComponentDependencies(dependencies, projectId); + // await updateProgress(70, 'Fetching vulnerabilities'); + // //Vulnerabilities + // const purlList = components.map((c) => { + // return c.purl; + // }); + // const r = await processBatch(purlList, VulnFetcherHandler); + // await updateProgress(90, 'Creating vulnerabilities in DB'); + // r.forEach(async (component) => { + // if (component.vulnerabilities.length > 0) { + // console.log('Creating %d vulns for %s', component.vulnerabilities.length, component.purl); + // await CreateUpdateVulnerability(component.purl, component.vulnerabilities); + // } + // }); + // await updateProgress(90, 'Creating vulnerabilities in DB'); } catch { console.error('Recovery needed'); } @@ -152,3 +151,34 @@ export function compareVersions(version1: string, version2: string): number { } return 0; } + +async function GetProjectId(project: ProjectInput) { + const { name, id } = project; + //We know exact ID + let existingProjects = []; + if (id) { + existingProjects = await GetProjectById(id); + } else { + existingProjects = await GetProjectByName(name); + } + // Project does not exist yet, so we need to create it first + if (existingProjects.length == 0) { + console.log('Creating new project with name %s', name); + const newProject: Project = { name: name }; + const currentProjectId = await CreateProject(newProject); + return currentProjectId; + } + // Project exist, so we just return it + // Because there might be more projects, return the first one + const currentProjectId = existingProjects[0].id; + if (existingProjects.length > 1) { + console.warn( + 'Multiple project was found for the name %s\nReturning the first one with id %s', + name, + existingProjects[0].id + ); + } + return currentProjectId; +} + +async function GetProjectVersionId(projectId) {} From cb519615b1f627c114bb7bed1db234c6b87a00d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Mon, 20 Feb 2023 21:10:36 +0100 Subject: [PATCH 20/33] Update types --- src/depvis-next/types/component.ts | 6 +++--- src/depvis-next/types/project.ts | 4 ++-- src/depvis-next/types/vulnerability.ts | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/depvis-next/types/component.ts b/src/depvis-next/types/component.ts index 3e787c6..fec561a 100644 --- a/src/depvis-next/types/component.ts +++ b/src/depvis-next/types/component.ts @@ -38,15 +38,15 @@ export type Component = { /** * List of all components that are direct dependencies for given component */ - dependsOn?: [Component]; + dependsOn?: Component[]; /** * References to external resources */ - references?: [Reference]; + references?: Reference[]; /** * List of vulnerabilities for given component */ - vulnerabilities?: [Vulnerability]; + vulnerabilities?: Vulnerability[]; /** * Reference to a project that contains given component */ diff --git a/src/depvis-next/types/project.ts b/src/depvis-next/types/project.ts index 28fec08..86ceea5 100644 --- a/src/depvis-next/types/project.ts +++ b/src/depvis-next/types/project.ts @@ -12,7 +12,7 @@ export type Project = { /** * List of versions related to a given project */ - versions?: [ProjectVersion]; + versions?: ProjectVersion[]; }; type ProjectVersionBase = { @@ -23,7 +23,7 @@ type ProjectVersionBase = { /** * List of all components in a given project */ - components?: [Component]; + components?: Component[]; /** * Date when SBOM was created for a given project */ diff --git a/src/depvis-next/types/vulnerability.ts b/src/depvis-next/types/vulnerability.ts index f14870c..4d4674f 100644 --- a/src/depvis-next/types/vulnerability.ts +++ b/src/depvis-next/types/vulnerability.ts @@ -1,4 +1,4 @@ -import { Reference } from "./reference"; +import { Reference } from './reference'; export type Vulnerability = { /** @@ -40,5 +40,5 @@ export type Vulnerability = { /** * References to additional resources */ - references?: [Reference]; + references?: Reference[]; }; From ed4dbe1464b5942522535a3f3e2e8a1973cb626d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Groman?= Date: Mon, 20 Feb 2023 22:04:10 +0100 Subject: [PATCH 21/33] WIP: continue with rewrite --- src/depvis-next/helpers/DbDataProvider.ts | 29 +++-- src/depvis-next/helpers/ImportSbomHelper.ts | 133 ++++++++++++++------ src/depvis-next/types/project.ts | 4 + 3 files changed, 120 insertions(+), 46 deletions(-) diff --git a/src/depvis-next/helpers/DbDataProvider.ts b/src/depvis-next/helpers/DbDataProvider.ts index d50aa51..d0bdeba 100644 --- a/src/depvis-next/helpers/DbDataProvider.ts +++ b/src/depvis-next/helpers/DbDataProvider.ts @@ -3,25 +3,31 @@ import { Component } from '../types/component'; import { Project, ProjectVersion, ProjectVersionDto } from '../types/project'; import { createApolloClient } from './ApolloClientHelper'; import { sendGQLQuery, sendGQLMutation, AddProjectVersionConnectProject } from './DbDataHelper'; +import { ProjectVersionInput } from './ImportSbomHelper'; /** * Function will create new project with optional first component * @param project Project data - * @returns ID of newly created project + * @returns List containing new project object */ -export async function CreateProject(project: Project) { +export async function CreateProject(project: Project): Promise { if (!project) return; const mutation = gql` mutation CreateProject($project: [ProjectCreateInput!]!) { createProjects(input: $project) { projects { id + name + versions { + id + version + } } } } `; const { data } = await sendGQLMutation(mutation, { project: [project] }); - return data; + return data.projects[0]; } /** @@ -29,7 +35,7 @@ export async function CreateProject(project: Project) { * @param projectName Project name * @returns List of projects that match given name */ -export async function GetProjectByName(projectName: string) { +export async function GetProjectByName(projectName: string): Promise { const query = gql` query Project($projectName: String!) { projects(where: { name: $projectName }) { @@ -51,7 +57,7 @@ export async function GetProjectByName(projectName: string) { * @param projectId Project Id * @returns List of projects that match given Id */ -export async function GetProjectById(projectId: string) { +export async function GetProjectById(projectId: string): Promise { const query = gql` query Project($projectName: String!) { projects(where: { id: $projectId }) { @@ -74,7 +80,8 @@ export async function GetProjectById(projectId: string) { * @param version new version identificator * @returns ID of the new project version */ -export async function AddProjectVersion(projectId, version: string) { +export async function CreateProjectVersion(projectId, projectVersionInput: ProjectVersionInput): Promise { + const { version, date } = projectVersionInput; if (!projectId || !version) { console.log('AddProjectComponent is missing some inputs! %s, %s', projectId, version); return; @@ -91,7 +98,11 @@ export async function AddProjectVersion(projectId, version: string) { } } `; - const projectVersion: ProjectVersionDto = { version: version, project: AddProjectVersionConnectProject(projectId) }; + const projectVersion: ProjectVersionDto = { + version: version, + project: AddProjectVersionConnectProject(projectId), + date: new Date(date), + }; const { data } = await sendGQLMutation(mutation, { projectVersion: projectVersion }); return data.projectVersions.id; } @@ -99,8 +110,9 @@ export async function AddProjectVersion(projectId, version: string) { /** * Deletes project version with all its connected components, references and vulnerabilities * @param projectVersionId ID of the project version + * @returns number of deleted nodes */ -export async function DeleteProjectVersion(projectVersionId: string) { +export async function DeleteProjectVersion(projectVersionId: string): Promise { const mutation = gql` mutation DeleteProjectVersion($projectVersionId: ID) { deleteProjectVersions(where: { id: $projectVersionId }, delete: { component: { where: {} } }) { @@ -109,6 +121,7 @@ export async function DeleteProjectVersion(projectVersionId: string) { } `; const { data } = await sendGQLMutation(mutation, { projectVersionId: projectVersionId }); + return data.nodesDeleted; } export async function CreateComponents(components: [Component?], projectVersionId: string) { diff --git a/src/depvis-next/helpers/ImportSbomHelper.ts b/src/depvis-next/helpers/ImportSbomHelper.ts index 7b22586..6974a78 100644 --- a/src/depvis-next/helpers/ImportSbomHelper.ts +++ b/src/depvis-next/helpers/ImportSbomHelper.ts @@ -1,5 +1,5 @@ import { Component } from '../types/component'; -import { Project } from '../types/project'; +import { Project, ProjectVersion } from '../types/project'; import { VulnFetcherHandler } from '../vulnerability-mgmt/VulnFetcherHandler'; import { processBatch } from './BatchHelper'; import { @@ -8,23 +8,48 @@ import { UpdateComponentDependencies, UpdateProjectDependencies, } from './DbDataHelper'; -import { CreateProject, GetProjectById, GetProjectByName } from './DbDataProvider'; +import { + CreateProject, + CreateProjectVersion, + DeleteProjectVersion, + GetProjectById, + GetProjectByName, +} from './DbDataProvider'; type ProjectInput = { name: string; id: string; }; -export async function ImportSbom(bom: any, projectInput: ProjectInput, projectVersion, updateProgressCallback) { +export type ProjectVersionInput = { + version: string; + date: string; +}; + +const NotKnownPlaceholder = "n/a" + +export async function ImportSbom(bom: any, projectInput: ProjectInput, projectVersion: string, updateProgressCallback) { try { - // Prepare main component if exists + /** + * Simple wrapper function that is responsible for updating status for job worker + * @param percent Status in percent (0-100) + * @param message Short description what is happening + */ const updateProgress = async (percent, message) => { console.log(message); await updateProgressCallback({ message: message, percent: percent }); console.log(message); }; - const projectId = GetProjectId(projectInput); - const projectVersionId = await updateProgress(1, 'Parsing main component'); + + // Find project information on backend + const project = await GetProject(projectInput); + const projectVersionInput: ProjectVersionInput = { + version: projectVersion || bom.metadata.component ? bom.metadata.component.version || NotKnownPlaceholder, + date: bom.metadata.timestamp || Date.now().toLocaleString() + } + const projectVersionId = GetProjectVersionId(project, projectVersionInput); + + // Create components for new version const mainComponentParsed = bom.metadata.component; const mainComponent: Component | undefined = mainComponentParsed ? { @@ -36,36 +61,26 @@ export async function ImportSbom(bom: any, projectInput: ProjectInput, projectVe publisher: mainComponentParsed.publisher, } : undefined; - await updateProgress(2, 'Preparing project'); - // Prepare project - let project: Project = { - name: projectName || bom.metadata.component.name, - version: projectVersion || bom.metadata.component.version || '1.0.0', - date: bom.metadata.timestamp || '1970-01-01', - }; - console.log(project); + await updateProgress(3, 'Preparing dependencies'); // Prepare dependencies let dependencies = GetDependencies(bom.dependencies.dependency); - // Currently there is no support for managing older projects - we first need to clear the DB - //await DeleteAllData(); await updateProgress(4, 'Creating project in DB'); // Create all objects in DB - // const projectResponse = await CreateProject(project); - // const projectId = projectResponse.createProjects.projects[0].id; - // await updateProgress(3, 'Preparing components'); - // // Prepare components - // let components: [Component] = GetComponents(bom); - // mainComponent && components.push(mainComponent); - // console.log(components); - // await updateProgress(10, 'Creating components'); + await updateProgress(3, 'Preparing components'); + // Prepare components + let components: Component[] = GetComponents(bom); + mainComponent && components.push(mainComponent); + await updateProgress(10, 'Creating components'); + // TODO: rewrite this using component IDs, not purl + // await CreateComponents(components, projectId); // await updateProgress(50, 'Updating dependencies'); // await UpdateProjectDependencies(projectId, [mainComponent]); // await UpdateComponentDependencies(dependencies, projectId); - // await updateProgress(70, 'Fetching vulnerabilities'); - // //Vulnerabilities - // const purlList = components.map((c) => { + await updateProgress(70, 'Fetching vulnerabilities'); + //Vulnerabilities + // const purlList = components.map((c) => { // return c.purl; // }); // const r = await processBatch(purlList, VulnFetcherHandler); @@ -120,11 +135,11 @@ function GetDependencies(dependencies: any) { /** * Function will find project with highest version number in format 1.2.3 * Note: all project must have the same version length - * @param projects List of projects + * @param ProjectVersionList List of projects * @returns The largest project */ -export function getLatestProjectVersion(projects) { - return projects.reduce((highestVersionObj, currentObj) => { +export function getLatestProjectVersion(ProjectVersionList: ProjectVersion[]): ProjectVersion { + return ProjectVersionList.reduce((highestVersionObj, currentObj) => { if (!highestVersionObj || compareVersions(currentObj.version, highestVersionObj.version) > 0) { return currentObj; } else { @@ -152,7 +167,14 @@ export function compareVersions(version1: string, version2: string): number { return 0; } -async function GetProjectId(project: ProjectInput) { +/** + * Function will return correct project. + * If the project exists, it will return corresponding object with additional info + * If the project does not exists, it will be created + * @param project Project Input data + * @returns Project + */ +async function GetProject(project: ProjectInput): Promise { const { name, id } = project; //We know exact ID let existingProjects = []; @@ -164,21 +186,56 @@ async function GetProjectId(project: ProjectInput) { // Project does not exist yet, so we need to create it first if (existingProjects.length == 0) { console.log('Creating new project with name %s', name); - const newProject: Project = { name: name }; - const currentProjectId = await CreateProject(newProject); - return currentProjectId; + const newProjectObj: Project = { name: name }; + const newProject = await CreateProject(newProjectObj); + console.log('Project created with id: %s', newProject!.id); + return newProject; } // Project exist, so we just return it // Because there might be more projects, return the first one - const currentProjectId = existingProjects[0].id; + const currentProject = existingProjects[0]; if (existingProjects.length > 1) { console.warn( 'Multiple project was found for the name %s\nReturning the first one with id %s', name, - existingProjects[0].id + currentProject.id ); } - return currentProjectId; + return currentProject; } -async function GetProjectVersionId(projectId) {} +/** + * Function will return Id of correct Project Version. + * If the same version already exists, it will be removed, + * then a new version will be created. + * @param project Project object + * @param projectVersion Version represented as string, e.g. "1.0.0" + * @returns ProjectVersion Id + */ +async function GetProjectVersionId(project: Project, projectVersionInput: ProjectVersionInput) { + const {version, date } = projectVersionInput; + if (!project || !version) { + throw Error('Invalid information - missing project or project version'); + } + + if (!project.versions || project.versions.length == 0) { + } + + //Version already exists + const existingProjectVersion = project.versions.find((pv) => { + pv.version === version; + }); + if (existingProjectVersion) { + console.log( + 'Version %s for project %s already exists with id %s, it will be removed.', + version, + project.name, + existingProjectVersion.id + ); + await DeleteProjectVersion(existingProjectVersion.id); + } + + const newVersionId = await CreateProjectVersion(project.id, version); + console.log('New version for project %s created with id %s', project.name, newVersionId); + return newVersionId; +} diff --git a/src/depvis-next/types/project.ts b/src/depvis-next/types/project.ts index 86ceea5..14d8187 100644 --- a/src/depvis-next/types/project.ts +++ b/src/depvis-next/types/project.ts @@ -16,6 +16,10 @@ export type Project = { }; type ProjectVersionBase = { + /** + * Project Version Id + */ + id?: string; /** * Project version */ From f79224b3b985247539a94652b21bdeb9c15ec139 Mon Sep 17 00:00:00 2001 From: Matej Groman Date: Wed, 22 Feb 2023 15:21:02 +0100 Subject: [PATCH 22/33] WIP: Fix some bugs with submission --- .../components/Import/ImportForm.tsx | 2 +- src/depvis-next/helpers/DbDataHelper.ts | 11 ++++++++--- src/depvis-next/helpers/DbDataProvider.ts | 9 +++++---- src/depvis-next/helpers/ImportSbomHelper.ts | 17 ++++++----------- src/depvis-next/pages/api/import/index.ts | 10 +++++----- src/depvis-next/queues/ImportQueue.ts | 2 +- 6 files changed, 26 insertions(+), 25 deletions(-) diff --git a/src/depvis-next/components/Import/ImportForm.tsx b/src/depvis-next/components/Import/ImportForm.tsx index ccc773a..3997cac 100644 --- a/src/depvis-next/components/Import/ImportForm.tsx +++ b/src/depvis-next/components/Import/ImportForm.tsx @@ -12,7 +12,7 @@ const ImportForm = (props) => { const [preview, setPreview] = useState(''); const [validated, setValidated] = useState(false); const [projectName, setProjectName] = useState(''); - const [projectVersion, setProjectVersion] = useState('1.0.0'); + const [projectVersion, setProjectVersion] = useState('1.0.1'); const handleFiles = (e: any) => { const files = e.target.files; diff --git a/src/depvis-next/helpers/DbDataHelper.ts b/src/depvis-next/helpers/DbDataHelper.ts index 6042f9e..1ae8b32 100644 --- a/src/depvis-next/helpers/DbDataHelper.ts +++ b/src/depvis-next/helpers/DbDataHelper.ts @@ -14,6 +14,8 @@ const chunkSize = 100; * @throws Error if there were some error during fetch */ export async function sendGQLQuery(query: DocumentNode, variables?: Object) { + console.log(`GQL Query: ${query} with variables ${await JSON.stringify(variables)}`); + console.dir({query: query, variables: variables}) const client = createApolloClient(); const res = await client.query({ query: query, variables: variables }); if (res.errors) { @@ -22,8 +24,9 @@ export async function sendGQLQuery(query: DocumentNode, variables?: Object) { return res; } -export async function sendGQLMutation(mutation: any, variables?: Object) { - console.log(`Sending mutation ${mutation} with variables ${await JSON.stringify(variables)}`); +export async function sendGQLMutation(mutation: DocumentNode, variables?: Object) { + console.log(`GQL Mutation: ${mutation} with variables ${await JSON.stringify(variables)}`); + console.dir({mutation: mutation, variables: variables}) const client = createApolloClient(); const res = await client.mutate({ mutation: mutation, variables: variables }); if (res.errors) { @@ -39,7 +42,9 @@ export async function TryGetProjectByName(projectName: string) { projects(where: { name_CONTAINS: $projectName }) { id name - version + versions { + version + } } } `; diff --git a/src/depvis-next/helpers/DbDataProvider.ts b/src/depvis-next/helpers/DbDataProvider.ts index d0bdeba..1781e85 100644 --- a/src/depvis-next/helpers/DbDataProvider.ts +++ b/src/depvis-next/helpers/DbDataProvider.ts @@ -1,7 +1,6 @@ import { gql } from '@apollo/client'; import { Component } from '../types/component'; import { Project, ProjectVersion, ProjectVersionDto } from '../types/project'; -import { createApolloClient } from './ApolloClientHelper'; import { sendGQLQuery, sendGQLMutation, AddProjectVersionConnectProject } from './DbDataHelper'; import { ProjectVersionInput } from './ImportSbomHelper'; @@ -27,7 +26,7 @@ export async function CreateProject(project: Project): Promise { } `; const { data } = await sendGQLMutation(mutation, { project: [project] }); - return data.projects[0]; + return data.createProjects.projects[0]; } /** @@ -48,6 +47,7 @@ export async function GetProjectByName(projectName: string): Promise } } `; + console.log(projectName) const { data } = await sendGQLQuery(query, { projectName: projectName }); return data.projects; } @@ -104,7 +104,8 @@ export async function CreateProjectVersion(projectId, projectVersionInput: Proje date: new Date(date), }; const { data } = await sendGQLMutation(mutation, { projectVersion: projectVersion }); - return data.projectVersions.id; + console.log(data) + return data.createProjectVersions.projectVersions[0].id; } /** @@ -124,7 +125,7 @@ export async function DeleteProjectVersion(projectVersionId: string): Promise Date: Wed, 22 Feb 2023 21:31:20 +0100 Subject: [PATCH 23/33] WIP: Can create components --- src/depvis-next/.vscode/launch.json | 2 +- src/depvis-next/helpers/BatchHelper.ts | 27 ++++--- src/depvis-next/helpers/DbDataHelper.ts | 10 ++- src/depvis-next/helpers/DbDataProvider.ts | 14 +++- src/depvis-next/helpers/ImportSbomHelper.ts | 89 ++++++++++++--------- src/depvis-next/pages/api/vuln/index.ts | 22 ++--- 6 files changed, 93 insertions(+), 71 deletions(-) diff --git a/src/depvis-next/.vscode/launch.json b/src/depvis-next/.vscode/launch.json index 2065274..943b870 100644 --- a/src/depvis-next/.vscode/launch.json +++ b/src/depvis-next/.vscode/launch.json @@ -5,7 +5,7 @@ "name": "Next.js: debug server-side", "type": "node-terminal", "request": "launch", - "command": "npm run dev" + "command": "npm run dev --trace-warnings" }, { "name": "Next.js: debug client-side", diff --git a/src/depvis-next/helpers/BatchHelper.ts b/src/depvis-next/helpers/BatchHelper.ts index 90b2a53..bf13b4e 100644 --- a/src/depvis-next/helpers/BatchHelper.ts +++ b/src/depvis-next/helpers/BatchHelper.ts @@ -5,18 +5,27 @@ * @param chunkSize How many items are used per one batch. Default is 10 * @returns Concatenated results of each function call. */ -export async function processBatch( +export async function processBatchAsync( inputList: any[], fn: Function, - chunkSize: number = 10 -) { + chunkSize: number = 10, + updateProgress: Function = undefined +): Promise { if (!inputList) return; let res = []; - for (let i = 0; i < inputList.length; i += chunkSize) { - const chunk = inputList.slice(i, i + chunkSize); - const chunkRes = await fn(chunk); - res = res.concat(chunkRes); + try { + for (let i = 0; i < inputList.length; i += chunkSize) { + if (updateProgress) { + updateProgress((i / inputList.length) * 100, 'Creating components'); + } + const chunk = inputList.slice(i, i + chunkSize); + const chunkRes = await fn(chunk); + res = res.concat(chunkRes); + } + } catch (error) { + console.error('While processing batch, error was encountered!'); + console.error(error); } - console.log("Total items processed %d", res.length); - return res; + console.log('Total items processed %d', res.length); + return res as T; } diff --git a/src/depvis-next/helpers/DbDataHelper.ts b/src/depvis-next/helpers/DbDataHelper.ts index 1ae8b32..a8aaee8 100644 --- a/src/depvis-next/helpers/DbDataHelper.ts +++ b/src/depvis-next/helpers/DbDataHelper.ts @@ -15,7 +15,7 @@ const chunkSize = 100; */ export async function sendGQLQuery(query: DocumentNode, variables?: Object) { console.log(`GQL Query: ${query} with variables ${await JSON.stringify(variables)}`); - console.dir({query: query, variables: variables}) + console.dir({ query: query, variables: variables }); const client = createApolloClient(); const res = await client.query({ query: query, variables: variables }); if (res.errors) { @@ -26,7 +26,7 @@ export async function sendGQLQuery(query: DocumentNode, variables?: Object) { export async function sendGQLMutation(mutation: DocumentNode, variables?: Object) { console.log(`GQL Mutation: ${mutation} with variables ${await JSON.stringify(variables)}`); - console.dir({mutation: mutation, variables: variables}) + console.dir({ mutation: mutation, variables: variables }); const client = createApolloClient(); const res = await client.mutate({ mutation: mutation, variables: variables }); if (res.errors) { @@ -302,3 +302,9 @@ export function CreateComponentsConnectProjectVersion(components: [ComponentDto] return { ...c, projectVersion: ConnectProjectVersion }; }); } + +export function AddComponentsConnectProjectVersion(components: Component[], projectVersionId: string) { + return components.map((c) => { + return { ...c, projectVersion: { connect: { where: { node: { id: projectVersionId } } } } }; + }); +} diff --git a/src/depvis-next/helpers/DbDataProvider.ts b/src/depvis-next/helpers/DbDataProvider.ts index 1781e85..54126b0 100644 --- a/src/depvis-next/helpers/DbDataProvider.ts +++ b/src/depvis-next/helpers/DbDataProvider.ts @@ -1,7 +1,12 @@ import { gql } from '@apollo/client'; import { Component } from '../types/component'; import { Project, ProjectVersion, ProjectVersionDto } from '../types/project'; -import { sendGQLQuery, sendGQLMutation, AddProjectVersionConnectProject } from './DbDataHelper'; +import { + sendGQLQuery, + sendGQLMutation, + AddProjectVersionConnectProject, + AddComponentsConnectProjectVersion, +} from './DbDataHelper'; import { ProjectVersionInput } from './ImportSbomHelper'; /** @@ -47,7 +52,7 @@ export async function GetProjectByName(projectName: string): Promise } } `; - console.log(projectName) + console.log(projectName); const { data } = await sendGQLQuery(query, { projectName: projectName }); return data.projects; } @@ -104,7 +109,7 @@ export async function CreateProjectVersion(projectId, projectVersionInput: Proje date: new Date(date), }; const { data } = await sendGQLMutation(mutation, { projectVersion: projectVersion }); - console.log(data) + console.log(data); return data.createProjectVersions.projectVersions[0].id; } @@ -144,5 +149,6 @@ export async function CreateComponents(components: Component[], projectVersionId } } `; - const { data } = await sendGQLMutation(mutation, { components: projectVersionId }); + const componentsWithConnect = AddComponentsConnectProjectVersion(components, projectVersionId); + const { data } = await sendGQLMutation(mutation, { components: componentsWithConnect }); } diff --git a/src/depvis-next/helpers/ImportSbomHelper.ts b/src/depvis-next/helpers/ImportSbomHelper.ts index 75f95bd..5bf9b0e 100644 --- a/src/depvis-next/helpers/ImportSbomHelper.ts +++ b/src/depvis-next/helpers/ImportSbomHelper.ts @@ -1,13 +1,14 @@ import { Component } from '../types/component'; import { Project, ProjectVersion } from '../types/project'; import { VulnFetcherHandler } from '../vulnerability-mgmt/VulnFetcherHandler'; -import { processBatch } from './BatchHelper'; +import { processBatchAsync } from './BatchHelper'; import { CreateProject, CreateProjectVersion, DeleteProjectVersion, GetProjectById, GetProjectByName, + CreateComponents, } from './DbDataProvider'; type ProjectInput = { @@ -20,54 +21,50 @@ export type ProjectVersionInput = { date: string; }; -const NotKnownPlaceholder = "n/a" +const NotKnownPlaceholder = 'n/a'; export async function ImportSbom(bom: any, projectInput: ProjectInput, projectVersion: string, updateProgressCallback) { - try { - /** - * Simple wrapper function that is responsible for updating status for job worker - * @param percent Status in percent (0-100) - * @param message Short description what is happening - */ - const updateProgress = async (percent, message) => { - console.log(message); - await updateProgressCallback({ message: message, percent: percent }); - console.log(message); - }; + /** + * Simple wrapper function that is responsible for updating status for job worker + * @param percent Status in percent (0-100) + * @param message Short description what is happening + */ + const updateProgress = async (percent, message) => { + console.log(message); + await updateProgressCallback({ message: message, percent: percent }); + console.log(message); + }; + + const importInfo = { + project: undefined, + projectVersion: undefined, + createdComponents: [], + createdVulnerabilitiesIds: [], + }; + try { // Find project information on backend - const project = await GetProject(projectInput); + await updateProgress(0, 'Creating new project version'); + importInfo.project = await GetProject(projectInput); + const projectVersionInput: ProjectVersionInput = { version: projectVersion || bom.metadata.component ? bom.metadata.component.version : NotKnownPlaceholder, - date: bom.metadata.timestamp || Date.now().toLocaleString() - } - const projectVersionId = GetProjectVersionId(project, projectVersionInput); + date: bom.metadata.timestamp || Date.now().toLocaleString(), + }; + importInfo.projectVersion = await GetProjectVersionId(importInfo.project, projectVersionInput); // Create components for new version - const mainComponentParsed = bom.metadata.component; - const mainComponent: Component | undefined = mainComponentParsed - ? { - type: mainComponentParsed.type, - name: mainComponentParsed.name, - purl: mainComponentParsed.purl || `${mainComponentParsed.name}@${mainComponentParsed.version}`, - version: mainComponentParsed.version, - author: mainComponentParsed.author, - publisher: mainComponentParsed.publisher, - } - : undefined; - - await updateProgress(3, 'Preparing dependencies'); + // Prepare dependencies let dependencies = GetDependencies(bom.dependencies.dependency); - await updateProgress(4, 'Creating project in DB'); // Create all objects in DB - await updateProgress(3, 'Preparing components'); // Prepare components - let components: Component[] = GetComponents(bom); - mainComponent && components.push(mainComponent); - await updateProgress(10, 'Creating components'); + const components: Component[] = GetComponents(bom); + importInfo.createdComponents = await processBatchAsync(components, CreateComponents, 5, updateProgress); + console.log('Components created'); + console.log(importInfo); // TODO: rewrite this using component IDs, not purl - + // await CreateComponents(components, projectId); // await updateProgress(50, 'Updating dependencies'); // await UpdateProjectDependencies(projectId, [mainComponent]); @@ -88,11 +85,11 @@ export async function ImportSbom(bom: any, projectInput: ProjectInput, projectVe // await updateProgress(90, 'Creating vulnerabilities in DB'); } catch (error) { console.error('Recovery needed'); - console.error(error) + console.error(error); } } function GetComponents(bom: any) { - let components = bom.components.component; + let components: any[] = bom.components.component; // Component data transformation components = components.map((c) => { return { @@ -104,6 +101,8 @@ function GetComponents(bom: any) { publisher: c.publisher, }; }); + + bom.metadata.component && components.push(createMainComponent(bom.metadata.component)); return components; } @@ -208,7 +207,7 @@ async function GetProject(project: ProjectInput): Promise { * @returns ProjectVersion Id */ async function GetProjectVersionId(project: Project, projectVersionInput: ProjectVersionInput) { - const {version, date } = projectVersionInput; + const { version, date } = projectVersionInput; if (!project || !version) { throw Error('Invalid information - missing project or project version'); } @@ -234,3 +233,15 @@ async function GetProjectVersionId(project: Project, projectVersionInput: Projec console.log('New version for project %s created with id %s', project.name, newVersionId); return newVersionId; } + +function createMainComponent(inputComponent) { + const purl = inputComponent.purl || `${inputComponent.name}@${inputComponent.version}`; + return { + type: inputComponent.type, + name: inputComponent.name, + purl: purl, + version: inputComponent.version, + author: inputComponent.author, + publisher: inputComponent.publisher, + }; +} diff --git a/src/depvis-next/pages/api/vuln/index.ts b/src/depvis-next/pages/api/vuln/index.ts index 829738b..1e5e87d 100644 --- a/src/depvis-next/pages/api/vuln/index.ts +++ b/src/depvis-next/pages/api/vuln/index.ts @@ -1,26 +1,16 @@ -import { processBatch } from "../../../helpers/BatchHelper"; -import { - CreateUpdateVulnerability, - GetComponents, -} from "../../../helpers/DbDataHelper"; -import { VulnFetcherHandler } from "../../../vulnerability-mgmt/VulnFetcherHandler"; +import { processBatchAsync } from '../../../helpers/BatchHelper'; +import { CreateUpdateVulnerability, GetComponents } from '../../../helpers/DbDataHelper'; +import { VulnFetcherHandler } from '../../../vulnerability-mgmt/VulnFetcherHandler'; export default async function handler(req, res) { const { components } = await GetComponents(); const purlList = components.map((c) => c.purl); - const r = await processBatch(purlList, VulnFetcherHandler, 100); + const r = await processBatchAsync(purlList, VulnFetcherHandler, 100); //Use queue here r.forEach(async (component) => { if (component.vulnerabilities.length > 0) { - console.log( - "Creating %d vulns for %s", - component.vulnerabilities.length, - component.purl - ); - await CreateUpdateVulnerability( - component.purl, - component.vulnerabilities - ); + console.log('Creating %d vulns for %s', component.vulnerabilities.length, component.purl); + await CreateUpdateVulnerability(component.purl, component.vulnerabilities); } }); res.status(200).json(r); From 0c3f016bdaf0639f22cb9fd4455f4b9752265347 Mon Sep 17 00:00:00 2001 From: Matej Groman Date: Fri, 24 Feb 2023 10:47:51 +0100 Subject: [PATCH 24/33] WIP: probably broken --- src/depvis-next/helpers/DbDataHelper.ts | 13 ++++++++++ src/depvis-next/helpers/DbDataProvider.ts | 30 +++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/depvis-next/helpers/DbDataHelper.ts b/src/depvis-next/helpers/DbDataHelper.ts index 1ae8b32..f8f5bf7 100644 --- a/src/depvis-next/helpers/DbDataHelper.ts +++ b/src/depvis-next/helpers/DbDataHelper.ts @@ -302,3 +302,16 @@ export function CreateComponentsConnectProjectVersion(components: [ComponentDto] return { ...c, projectVersion: ConnectProjectVersion }; }); } + +export function BuildAddDependencyQuery(dependencies: any[], projectVersionId: string) { + return dependencies.map((d) => { + if (!d.dependsOn) return; //No dependency + return {where: {purl: d.purl, projectVersion: {id: projectVersionId} }, connect: {dependsOn: {where: { node: {OR: getDependencyWherePurlPart(d.dependsOn)}}}} + }}) +} + +function getDependencyWherePurlPart(dependsOn: any[], projectVersionId) { + return dependsOn.map((d) => { + return {purl: d.purl, projectVersion: {id: projectVersionId}}; + }); +} \ No newline at end of file diff --git a/src/depvis-next/helpers/DbDataProvider.ts b/src/depvis-next/helpers/DbDataProvider.ts index 1781e85..8a0926c 100644 --- a/src/depvis-next/helpers/DbDataProvider.ts +++ b/src/depvis-next/helpers/DbDataProvider.ts @@ -1,7 +1,7 @@ import { gql } from '@apollo/client'; import { Component } from '../types/component'; import { Project, ProjectVersion, ProjectVersionDto } from '../types/project'; -import { sendGQLQuery, sendGQLMutation, AddProjectVersionConnectProject } from './DbDataHelper'; +import { sendGQLQuery, sendGQLMutation, AddProjectVersionConnectProject, CreateComponentsConnectProjectVersion, BuildAddDependencyQuery } from './DbDataHelper'; import { ProjectVersionInput } from './ImportSbomHelper'; /** @@ -144,5 +144,31 @@ export async function CreateComponents(components: Component[], projectVersionId } } `; - const { data } = await sendGQLMutation(mutation, { components: projectVersionId }); + const { data } = await sendGQLMutation(mutation, { components: CreateComponentsConnectProjectVersion(components, projectVersionId) }); } + +export async function updateComponentDependency(dependencies: any[], projectVersionId: string) { + if (!dependencies || dependencies.length == 0) { + console.log('Updating dependencies - No dependencies provided!'); + return; + } + + const mutation = gql` + mutation UpdateDependencies($where: ComponentWhere, $connect: ComponentConnectInput) { + name: updateComponents( + where: $where + connect: $connect + ) + { + info { + relationshipsCreated + } + } + } + ` + const dependencyQueryList: any[] = BuildAddDependencyQuery(dependencies, projectVersionId) + for (let index = 0; index < dependencyQueryList.length; index++) { + const { data } = await sendGQLMutation(mutation, {where: dependencyQueryList[index].where, connect: dependencyQueryList[index].connect}); + console.log("Created %s relationships", data.info.relationshipsCreated) + } +} \ No newline at end of file From 2dfbe45bcb903ae9ef2fbeba78098f1f6aeaf02d Mon Sep 17 00:00:00 2001 From: Matej Groman Date: Tue, 7 Mar 2023 17:20:07 +0100 Subject: [PATCH 25/33] Working --- src/depvis-next/helpers/ApolloClientHelper.ts | 17 +- src/depvis-next/helpers/BatchHelper.ts | 21 ++- src/depvis-next/helpers/DbDataHelper.ts | 160 +++++++++++++----- src/depvis-next/helpers/DbDataProvider.ts | 108 +++++++++--- src/depvis-next/helpers/GraphHelper.ts | 41 ++--- src/depvis-next/helpers/ImportSbomHelper.ts | 129 +++++++++----- src/depvis-next/pages/api/graphql.ts | 1 + src/depvis-next/pages/api/import/index.ts | 70 +++++--- src/depvis-next/pages/api/vuln/index.ts | 24 ++- src/depvis-next/pages/upload.tsx | 38 +++-- src/depvis-next/queues/ImportQueue.ts | 23 ++- 11 files changed, 435 insertions(+), 197 deletions(-) diff --git a/src/depvis-next/helpers/ApolloClientHelper.ts b/src/depvis-next/helpers/ApolloClientHelper.ts index ac9f765..73c063d 100644 --- a/src/depvis-next/helpers/ApolloClientHelper.ts +++ b/src/depvis-next/helpers/ApolloClientHelper.ts @@ -1,6 +1,6 @@ -import { ApolloClient, HttpLink, InMemoryCache } from '@apollo/client'; -import urlJoin from 'url-join'; -import { gqlUrlPath } from '../pages/api/graphql'; +import { ApolloClient, HttpLink, InMemoryCache } from "@apollo/client"; +import urlJoin from "url-join"; +import { gqlUrlPath } from "../pages/api/graphql"; /** * Function responsible for initialization of new Apollo Client used for GraphQL @@ -8,14 +8,17 @@ import { gqlUrlPath } from '../pages/api/graphql'; */ export const createApolloClient = () => { if (!process.env.NEXT_PUBLIC_SERVER_URI) { - console.error('No server URI was provided, using default connection'); + console.error("No server URI was provided, using default connection"); } - const uri = urlJoin(process.env.NEXT_PUBLIC_SERVER_URI || 'http://localhost:3000', gqlUrlPath); - console.log(`Creating GQL Client (connection to ${uri})`); + const uri = urlJoin( + process.env.NEXT_PUBLIC_SERVER_URI || "http://localhost:3000", + gqlUrlPath + ); + // console.log(`Creating GQL Client (connection to ${uri})`); const link = new HttpLink({ uri: uri, fetchOptions: { - mode: 'cors', + mode: "cors", }, }); return new ApolloClient({ diff --git a/src/depvis-next/helpers/BatchHelper.ts b/src/depvis-next/helpers/BatchHelper.ts index bf13b4e..926e42a 100644 --- a/src/depvis-next/helpers/BatchHelper.ts +++ b/src/depvis-next/helpers/BatchHelper.ts @@ -5,27 +5,34 @@ * @param chunkSize How many items are used per one batch. Default is 10 * @returns Concatenated results of each function call. */ + +export type processBatchOptions = { + fnArg2?: any; + updateProgressFn?: Function; + message?: any; + chunkSize: number; +}; export async function processBatchAsync( inputList: any[], fn: Function, - chunkSize: number = 10, - updateProgress: Function = undefined + options: processBatchOptions ): Promise { if (!inputList) return; let res = []; + const chunkSize = options.chunkSize || 10; try { for (let i = 0; i < inputList.length; i += chunkSize) { - if (updateProgress) { - updateProgress((i / inputList.length) * 100, 'Creating components'); + if (options.updateProgressFn) { + options.updateProgressFn((i / inputList.length) * 100, options.message); } const chunk = inputList.slice(i, i + chunkSize); - const chunkRes = await fn(chunk); + const chunkRes = await fn(chunk, options.fnArg2); res = res.concat(chunkRes); } } catch (error) { - console.error('While processing batch, error was encountered!'); + console.error("While processing batch, error was encountered!"); console.error(error); } - console.log('Total items processed %d', res.length); + console.log("Total items processed %d", res.length); return res as T; } diff --git a/src/depvis-next/helpers/DbDataHelper.ts b/src/depvis-next/helpers/DbDataHelper.ts index 427beca..9259fd5 100644 --- a/src/depvis-next/helpers/DbDataHelper.ts +++ b/src/depvis-next/helpers/DbDataHelper.ts @@ -1,9 +1,9 @@ -import { DocumentNode, gql } from '@apollo/client'; -import { randomBytes } from 'crypto'; -import { Component, ComponentDto } from '../types/component'; -import { Project } from '../types/project'; -import { Vulnerability } from '../types/vulnerability'; -import { createApolloClient } from './ApolloClientHelper'; +import { DocumentNode, gql } from "@apollo/client"; +import { randomBytes } from "crypto"; +import { Component, ComponentDto } from "../types/component"; +import { Project } from "../types/project"; +import { Vulnerability } from "../types/vulnerability"; +import { createApolloClient } from "./ApolloClientHelper"; const chunkSize = 100; /** @@ -14,7 +14,9 @@ const chunkSize = 100; * @throws Error if there were some error during fetch */ export async function sendGQLQuery(query: DocumentNode, variables?: Object) { - console.log(`GQL Query: ${query} with variables ${await JSON.stringify(variables)}`); + console.log( + `GQL Query: ${query} with variables ${await JSON.stringify(variables)}` + ); console.dir({ query: query, variables: variables }); const client = createApolloClient(); const res = await client.query({ query: query, variables: variables }); @@ -24,8 +26,15 @@ export async function sendGQLQuery(query: DocumentNode, variables?: Object) { return res; } -export async function sendGQLMutation(mutation: DocumentNode, variables?: Object) { - console.log(`GQL Mutation: ${mutation} with variables ${await JSON.stringify(variables)}`); +export async function sendGQLMutation( + mutation: DocumentNode, + variables?: Object +) { + console.log( + `GQL Mutation: ${mutation} with variables ${await JSON.stringify( + variables + )}` + ); console.dir({ mutation: mutation, variables: variables }); const client = createApolloClient(); const res = await client.mutate({ mutation: mutation, variables: variables }); @@ -66,7 +75,10 @@ export async function ComponentExists(componentName: string) { return data.projects.length > 0; } -export async function CreateComponents(components: [Component?], projectId: string) { +export async function CreateComponents( + components: [Component?], + projectId: string +) { if (!components || components.length == 0) return; const mutation = gql` mutation CreateComponent($components: [ComponentCreateInput!]!) { @@ -84,7 +96,9 @@ export async function CreateComponents(components: [Component?], projectId: stri for (let i = 0; i < components.length; i += chunkSize) { const chunk = components.slice(i, i + chunkSize); const chunkWithProjectId = AddProjectToComponents(chunk, projectId); - const { data } = await sendGQLMutation(mutation, { components: chunkWithProjectId }); + const { data } = await sendGQLMutation(mutation, { + components: chunkWithProjectId, + }); res.concat(data.components); } return res; @@ -92,9 +106,11 @@ export async function CreateComponents(components: [Component?], projectId: stri function AddProjectToComponents(components: Component[], projectId: string) { const res = components.map((c) => { - return { ...c, project: { connect: { where: { node: { id: projectId } } } } }; + return { + ...c, + project: { connect: { where: { node: { id: projectId } } } }, + }; }); - console.log(res); return res; } @@ -119,7 +135,7 @@ export async function CreateProject(project: Project) { } function generateName() { - return 'gql' + randomBytes(8).toString('hex'); + return "gql" + randomBytes(8).toString("hex"); } export async function GetVulnerability(vulnerabilityId) { @@ -131,11 +147,16 @@ export async function GetVulnerability(vulnerabilityId) { } } `; - const { data } = await sendGQLQuery(query, { vulnerabilityId: vulnerabilityId }); + const { data } = await sendGQLQuery(query, { + vulnerabilityId: vulnerabilityId, + }); return data.vulnerabilities; } -export async function CreateUpdateVulnerability(purl: string, vulnerabilities: [Vulnerability?]) { - console.log('Creating vulnerability for package %s', purl); +export async function CreateUpdateVulnerability( + purl: string, + vulnerabilities: [Vulnerability?] +) { + console.log("Creating vulnerability for package %s", purl); if (!vulnerabilities || vulnerabilities.length == 0 || !purl) return; const vulnExistsList = await Promise.all( @@ -144,15 +165,26 @@ export async function CreateUpdateVulnerability(purl: string, vulnerabilities: [ return getVuln.length == 0; }) ); - const newVulnerabilities = vulnerabilities.filter((_v, index) => vulnExistsList[index]); + const newVulnerabilities = vulnerabilities.filter( + (_v, index) => vulnExistsList[index] + ); if (newVulnerabilities.length == 0) { - console.log('All vulnerabilities for component %s already exists in DB', purl); + console.log( + "All vulnerabilities for component %s already exists in DB", + purl + ); return; } const mutation = gql` - mutation CreateVulnerability($purl: String, $vuln_array: [ComponentVulnerabilitiesCreateFieldInput!]) { - updateComponents(where: { purl: $purl }, update: { vulnerabilities: { create: $vuln_array } }) { + mutation CreateVulnerability( + $purl: String + $vuln_array: [ComponentVulnerabilitiesCreateFieldInput!] + ) { + updateComponents( + where: { purl: $purl } + update: { vulnerabilities: { create: $vuln_array } } + ) { info { nodesCreated relationshipsCreated @@ -169,7 +201,7 @@ export async function CreateUpdateVulnerability(purl: string, vulnerabilities: [ purl: purl, vuln_array: vulnArray, }); - console.log(data); + return data; } function PrepareVulnAsGQL(vuln: Vulnerability) { @@ -188,13 +220,21 @@ function PrepareVulnAsGQL(vuln: Vulnerability) { return { ...vuln, references: refs }; } -export async function UpdateProjectDependencies(projectId: string, components: [Component?]) { +export async function UpdateProjectDependencies( + projectId: string, + components: [Component?] +) { if (!projectId || components.length == 0) return; const mutation = gql` - mutation UpdateProjectDependencies($projectId: ID, $componentsPurl: [ComponentWhere!]!) { + mutation UpdateProjectDependencies( + $projectId: ID + $componentsPurl: [ComponentWhere!]! + ) { updateProjects( where: { id: $projectId } - update: { component: { connect: { where: { node: { OR: $componentsPurl } } } } } + update: { + component: { connect: { where: { node: { OR: $componentsPurl } } } } + } ) { __typename info { @@ -212,23 +252,30 @@ export async function UpdateProjectDependencies(projectId: string, components: [ }); } -export async function UpdateComponentDependencies(dependencies, projectId: string) { +export async function UpdateComponentDependencies( + dependencies, + projectId: string +) { if (dependencies == null || dependencies.length == 0) return; for (let i = 0; i < dependencies.length; i += chunkSize) { //TODO: rewrite using variables const chunk = dependencies.slice(i, i + chunkSize); const chunkMutation = chunk .map((dependency) => { - return getComponentUpdateGQLQuery(dependency, dependency.dependsOn, generateName()); + return getComponentUpdateGQLQuery( + dependency, + dependency.dependsOn, + generateName() + ); }) - .join('\n'); + .join("\n"); const mutation = gql` mutation UpdateComponents ($projectId: ID){ ${chunkMutation} } `; const { data } = await sendGQLMutation(mutation, { projectId: projectId }); - console.log(data); + return data; } } export async function GetComponents() { @@ -247,10 +294,14 @@ function getComponentWherePurlPart(array: [Component?]) { const res = array.map((c) => { return `{purl: \"${c.purl}\", project_SINGLE: {id: $projectId}}`; }); - return `[${res.join(',')}]`; + return `[${res.join(",")}]`; } -function getComponentUpdateGQLQuery(dependency, dependsOn, name = 'updateComponent') { +function getComponentUpdateGQLQuery( + dependency, + dependsOn, + name = "updateComponent" +) { const mutation_content = getComponentWherePurlPart(dependsOn); const mutation_part = `${name}: updateComponents( @@ -267,7 +318,6 @@ function getComponentUpdateGQLQuery(dependency, dependsOn, name = 'updateCompone relationshipsCreated } }`; - console.log(mutation_part); return mutation_part; } @@ -286,40 +336,68 @@ export async function DeleteAllData() { deleteVulnerabilities(where: {}) { nodesDeleted } + deleteProjectVersions(where: {}) { + nodesDeleted + } } `; const { data } = await sendGQLMutation(mutation); - console.log(data); + return data; } export function AddProjectVersionConnectProject(projectId: string) { return { connect: { where: { node: { id: projectId } } } }; } -export function CreateComponentsConnectProjectVersion(components: [ComponentDto], projectVersionId: string) { - const ConnectProjectVersion = { connect: { where: { node: { id: projectVersionId } } } }; +export function CreateComponentsConnectProjectVersion( + components: [ComponentDto], + projectVersionId: string +) { + const ConnectProjectVersion = { + connect: { where: { node: { id: projectVersionId } } }, + }; return components.map((c) => { return { ...c, projectVersion: ConnectProjectVersion }; }); } -export function AddComponentsConnectProjectVersion(components: Component[], projectVersionId: string) { +export function AddComponentsConnectProjectVersion( + components: Component[], + projectVersionId: string +) { return components.map((c) => { - return { ...c, projectVersion: { connect: { where: { node: { id: projectVersionId } } } } }; + return { + ...c, + projectVersion: { + connect: { where: { node: { id: projectVersionId } } }, + }, + }; }); } -export function BuildAddDependencyQuery(dependencies: any[], projectVersionId: string) { +export function BuildAddDependencyQuery( + dependencies: any[], + projectVersionId: string +) { return dependencies.map((d) => { if (!d.dependsOn) return; //No dependency return { where: { purl: d.purl, projectVersion: { id: projectVersionId } }, - connect: { dependsOn: { where: { node: { OR: getDependencyWherePurlPart(d.dependsOn, projectVersionId) } } } }, + connect: { + dependsOn: { + where: { + node: { + AND: getDependencyWherePurlPart(d.dependsOn, projectVersionId), + }, + }, + }, + }, }; }); } function getDependencyWherePurlPart(dependsOn: any[], projectVersionId) { - return dependsOn.map((d) => { - return { purl: d.purl, projectVersion: { id: projectVersionId } }; + const purls = dependsOn.map((d) => { + return d.purl; }); + return { purl_IN: purls, projectVersion: { id: projectVersionId } }; } diff --git a/src/depvis-next/helpers/DbDataProvider.ts b/src/depvis-next/helpers/DbDataProvider.ts index 279bce2..58b2988 100644 --- a/src/depvis-next/helpers/DbDataProvider.ts +++ b/src/depvis-next/helpers/DbDataProvider.ts @@ -1,14 +1,14 @@ -import { gql } from '@apollo/client'; -import { Component } from '../types/component'; -import { Project, ProjectVersion, ProjectVersionDto } from '../types/project'; +import { gql } from "@apollo/client"; +import { Component } from "../types/component"; +import { Project, ProjectVersion, ProjectVersionDto } from "../types/project"; import { sendGQLQuery, sendGQLMutation, AddProjectVersionConnectProject, BuildAddDependencyQuery, AddComponentsConnectProjectVersion, -} from './DbDataHelper'; -import { ProjectVersionInput } from './ImportSbomHelper'; +} from "./DbDataHelper"; +import { ProjectVersionInput } from "./ImportSbomHelper"; /** * Function will create new project with optional first component @@ -40,7 +40,9 @@ export async function CreateProject(project: Project): Promise { * @param projectName Project name * @returns List of projects that match given name */ -export async function GetProjectByName(projectName: string): Promise { +export async function GetProjectByName( + projectName: string +): Promise { const query = gql` query Project($projectName: String!) { projects(where: { name: $projectName }) { @@ -53,7 +55,6 @@ export async function GetProjectByName(projectName: string): Promise } } `; - console.log(projectName); const { data } = await sendGQLQuery(query, { projectName: projectName }); return data.projects; } @@ -86,10 +87,17 @@ export async function GetProjectById(projectId: string): Promise { * @param version new version identificator * @returns ID of the new project version */ -export async function CreateProjectVersion(projectId, projectVersionInput: ProjectVersionInput): Promise { +export async function CreateProjectVersion( + projectId, + projectVersionInput: ProjectVersionInput +): Promise { const { version, date } = projectVersionInput; if (!projectId || !version) { - console.log('AddProjectComponent is missing some inputs! %s, %s', projectId, version); + console.log( + "AddProjectComponent is missing some inputs! %s, %s", + projectId, + version + ); return; } const mutation = gql` @@ -109,8 +117,9 @@ export async function CreateProjectVersion(projectId, projectVersionInput: Proje project: AddProjectVersionConnectProject(projectId), date: new Date(date), }; - const { data } = await sendGQLMutation(mutation, { projectVersion: projectVersion }); - console.log(data); + const { data } = await sendGQLMutation(mutation, { + projectVersion: projectVersion, + }); return data.createProjectVersions.projectVersions[0].id; } @@ -119,21 +128,31 @@ export async function CreateProjectVersion(projectId, projectVersionInput: Proje * @param projectVersionId ID of the project version * @returns number of deleted nodes */ -export async function DeleteProjectVersion(projectVersionId: string): Promise { +export async function DeleteProjectVersion( + projectVersionId: string +): Promise { const mutation = gql` mutation DeleteProjectVersion($projectVersionId: ID) { - deleteProjectVersions(where: { id: $projectVersionId }, delete: { component: { where: {} } }) { + deleteProjectVersions( + where: { id: $projectVersionId } + delete: { component: { where: {} } } + ) { nodesDeleted } } `; - const { data } = await sendGQLMutation(mutation, { projectVersionId: projectVersionId }); + const { data } = await sendGQLMutation(mutation, { + projectVersionId: projectVersionId, + }); return data.nodesDeleted; } -export async function CreateComponents(components: Component[], projectVersionId: string) { +export async function CreateComponents( + components: Component[], + projectVersionId: string +) { if (!components || components.length == 0) { - console.log('CreateComponents - No components provided!'); + console.log("CreateComponents - No components provided!"); return; } const mutation = gql` @@ -150,18 +169,47 @@ export async function CreateComponents(components: Component[], projectVersionId } } `; - const componentsWithConnect = AddComponentsConnectProjectVersion(components, projectVersionId); - const { data } = await sendGQLMutation(mutation, { components: componentsWithConnect }); + const componentsWithConnect = AddComponentsConnectProjectVersion( + components, + projectVersionId + ); + const { data } = await sendGQLMutation(mutation, { + components: componentsWithConnect, + }); } -export async function updateComponentDependency(dependencies: any[], projectVersionId: string) { +export async function updateComponentDependency( + dependencies: any[], + projectVersionId: string, + mainComponentPurl: string +) { if (!dependencies || dependencies.length == 0) { - console.log('Updating dependencies - No dependencies provided!'); + console.log("Updating dependencies - No dependencies provided!"); return; } - + const mainComponentsMutation = gql` + mutation UpdateProjectVersion($projectVersionId: ID, $purl: String) { + updateProjectVersions( + where: { id: $projectVersionId } + connect: { + component: { + where: { + node: { purl: $purl, projectVersion: { id: $projectVersionId } } + } + } + } + ) { + info { + relationshipsCreated + } + } + } + `; const mutation = gql` - mutation UpdateDependencies($where: ComponentWhere, $connect: ComponentConnectInput) { + mutation UpdateDependencies( + $where: ComponentWhere + $connect: ComponentConnectInput + ) { name: updateComponents(where: $where, connect: $connect) { info { relationshipsCreated @@ -169,12 +217,24 @@ export async function updateComponentDependency(dependencies: any[], projectVers } } `; - const dependencyQueryList: any[] = BuildAddDependencyQuery(dependencies, projectVersionId); + // Connect main component + const mainComponentRes = await sendGQLMutation(mainComponentsMutation, { + projectVersionId: projectVersionId, + purl: mainComponentPurl, + }); + const dependencyQueryList: any[] = BuildAddDependencyQuery( + dependencies, + projectVersionId + ); for (let index = 0; index < dependencyQueryList.length; index++) { const { data } = await sendGQLMutation(mutation, { where: dependencyQueryList[index].where, connect: dependencyQueryList[index].connect, }); - console.log('Created %s relationships', data.info.relationshipsCreated); + console.log(data); + console.log( + "Created %s relationships", + data.name.info.relationshipsCreated + ); } } diff --git a/src/depvis-next/helpers/GraphHelper.ts b/src/depvis-next/helpers/GraphHelper.ts index 1445f29..0541eb6 100644 --- a/src/depvis-next/helpers/GraphHelper.ts +++ b/src/depvis-next/helpers/GraphHelper.ts @@ -1,4 +1,4 @@ -import { gql } from '@apollo/client'; +import { gql } from "@apollo/client"; /** * Function responsible for transforming the data to format that can be visualized @@ -61,10 +61,8 @@ export const formatData = (components) => { }); //Filter out links that are not connected links = links.filter((l) => { - console.log(l); if (l.toVuln || nodes.find((n) => n.id === l.target)) return true; }); - console.log({ node: nodes, links: links }); return { nodes, links }; }; @@ -129,22 +127,27 @@ export const getProjectsQuery = gql` } } `; -const componentColor = '#005f73'; -const vulnColor = '#ee9b00'; -const otherColor = '#001219'; -const severeVulnColor = '#bb3e03'; -const systemComponent = '#0f0f0f'; +const componentColor = "#005f73"; +const vulnColor = "#ee9b00"; +const otherColor = "#001219"; +const severeVulnColor = "#bb3e03"; +const systemComponent = "#0f0f0f"; export const getNodeColor = (node) => { if (!node) return otherColor; - if (node.__typename === 'Vulnerability') return node.cvssScore > 5 ? severeVulnColor : vulnColor; - if (node.selected) return '#6500ff'; - if (node.name && node.name.toLowerCase().includes('system')) return systemComponent; - if (node.__typename === 'Component') return componentColor; + if (node.__typename === "Vulnerability") + return node.cvssScore > 5 ? severeVulnColor : vulnColor; + if (node.selected) return "#6500ff"; + if (node.name && node.name.toLowerCase().includes("system")) + return systemComponent; + if (node.__typename === "Component") return componentColor; return otherColor; }; -const getNodeTier = (score: number, tresholds: { 1: 10; 0.75: 7; 0.5: 5; 0.25: 3; 0.1: 1 }) => { +const getNodeTier = ( + score: number, + tresholds: { 1: 10; 0.75: 7; 0.5: 5; 0.25: 3; 0.1: 1 } +) => { const limits = Object.keys(tresholds); limits.forEach((l) => { if (score < Number.parseFloat(l)) return tresholds[l]; @@ -162,13 +165,13 @@ export const getNodeValue = (node) => { */ function genGraphTree() { return { - nodes: [{ id: 'A' }, { id: 'B' }, { id: 'C' }, { id: 'D' }], + nodes: [{ id: "A" }, { id: "B" }, { id: "C" }, { id: "D" }], links: [ - { source: 'A', target: 'B' }, - { source: 'A', target: 'C' }, - { source: 'B', target: 'D' }, - { source: 'D', target: 'C' }, - { source: 'C', target: 'B' }, + { source: "A", target: "B" }, + { source: "A", target: "C" }, + { source: "B", target: "D" }, + { source: "D", target: "C" }, + { source: "C", target: "B" }, ], }; } diff --git a/src/depvis-next/helpers/ImportSbomHelper.ts b/src/depvis-next/helpers/ImportSbomHelper.ts index 5bf9b0e..90015a2 100644 --- a/src/depvis-next/helpers/ImportSbomHelper.ts +++ b/src/depvis-next/helpers/ImportSbomHelper.ts @@ -1,7 +1,7 @@ -import { Component } from '../types/component'; -import { Project, ProjectVersion } from '../types/project'; -import { VulnFetcherHandler } from '../vulnerability-mgmt/VulnFetcherHandler'; -import { processBatchAsync } from './BatchHelper'; +import { Component } from "../types/component"; +import { Project, ProjectVersion } from "../types/project"; +import { VulnFetcherHandler } from "../vulnerability-mgmt/VulnFetcherHandler"; +import { processBatchAsync } from "./BatchHelper"; import { CreateProject, CreateProjectVersion, @@ -9,7 +9,8 @@ import { GetProjectById, GetProjectByName, CreateComponents, -} from './DbDataProvider'; + updateComponentDependency, +} from "./DbDataProvider"; type ProjectInput = { name?: string; @@ -21,20 +22,23 @@ export type ProjectVersionInput = { date: string; }; -const NotKnownPlaceholder = 'n/a'; +const NotKnownPlaceholder = "n/a"; -export async function ImportSbom(bom: any, projectInput: ProjectInput, projectVersion: string, updateProgressCallback) { +export async function ImportSbom( + bom: any, + projectInput: ProjectInput, + projectVersion: string, + updateProgressCallback +) { /** * Simple wrapper function that is responsible for updating status for job worker * @param percent Status in percent (0-100) * @param message Short description what is happening */ const updateProgress = async (percent, message) => { - console.log(message); await updateProgressCallback({ message: message, percent: percent }); - console.log(message); }; - + console.log("Here the project version is: %s", projectVersion); const importInfo = { project: undefined, projectVersion: undefined, @@ -44,32 +48,54 @@ export async function ImportSbom(bom: any, projectInput: ProjectInput, projectVe try { // Find project information on backend - await updateProgress(0, 'Creating new project version'); + await updateProgress(0, "Creating new project version"); importInfo.project = await GetProject(projectInput); - + const tmpProjectVersion = projectVersion + ? projectVersion + : bom.metadata.component + ? (bom.metadata.component.version as string) + : NotKnownPlaceholder; const projectVersionInput: ProjectVersionInput = { - version: projectVersion || bom.metadata.component ? bom.metadata.component.version : NotKnownPlaceholder, + version: tmpProjectVersion, date: bom.metadata.timestamp || Date.now().toLocaleString(), }; - importInfo.projectVersion = await GetProjectVersionId(importInfo.project, projectVersionInput); + importInfo.projectVersion = await GetProjectVersionId( + importInfo.project, + projectVersionInput + ); + console.log(importInfo); // Create components for new version // Prepare dependencies let dependencies = GetDependencies(bom.dependencies.dependency); + // Create all objects in DB // Prepare components - const components: Component[] = GetComponents(bom); - importInfo.createdComponents = await processBatchAsync(components, CreateComponents, 5, updateProgress); - console.log('Components created'); - console.log(importInfo); - // TODO: rewrite this using component IDs, not purl + const mainComponent: Component = createMainComponent( + bom.metadata.component + ); + let components: Component[] = GetComponents(bom); + components.push(mainComponent); - // await CreateComponents(components, projectId); - // await updateProgress(50, 'Updating dependencies'); + importInfo.createdComponents = await processBatchAsync( + components, + CreateComponents, + { + chunkSize: 5, + updateProgressFn: updateProgress, + message: "Updating components", + fnArg2: importInfo.projectVersion, + } + ); + const dependenciesResult = await updateComponentDependency( + dependencies, + importInfo.projectVersion, + mainComponent.purl + ); // await UpdateProjectDependencies(projectId, [mainComponent]); // await UpdateComponentDependencies(dependencies, projectId); - await updateProgress(70, 'Fetching vulnerabilities'); + await updateProgress(70, "Fetching vulnerabilities"); //Vulnerabilities // const purlList = components.map((c) => { // return c.purl; @@ -84,7 +110,7 @@ export async function ImportSbom(bom: any, projectInput: ProjectInput, projectVe // }); // await updateProgress(90, 'Creating vulnerabilities in DB'); } catch (error) { - console.error('Recovery needed'); + console.error("Recovery needed"); console.error(error); } } @@ -101,8 +127,6 @@ function GetComponents(bom: any) { publisher: c.publisher, }; }); - - bom.metadata.component && components.push(createMainComponent(bom.metadata.component)); return components; } @@ -132,9 +156,14 @@ function GetDependencies(dependencies: any) { * @param ProjectVersionList List of projects * @returns The largest project */ -export function getLatestProjectVersion(ProjectVersionList: ProjectVersion[]): ProjectVersion { +export function getLatestProjectVersion( + ProjectVersionList: ProjectVersion[] +): ProjectVersion { return ProjectVersionList.reduce((highestVersionObj, currentObj) => { - if (!highestVersionObj || compareVersions(currentObj.version, highestVersionObj.version) > 0) { + if ( + !highestVersionObj || + compareVersions(currentObj.version, highestVersionObj.version) > 0 + ) { return currentObj; } else { return highestVersionObj; @@ -149,10 +178,10 @@ export function getLatestProjectVersion(ProjectVersionList: ProjectVersion[]): P * @returns 1 if version1 > version2, -1 if version1 < version2, 0 if they are equal */ export function compareVersions(version1: string, version2: string): number { - const arr1 = version1.split('.').map((num) => parseInt(num, 10)); - const arr2 = version2.split('.').map((num) => parseInt(num, 10)); - const num1 = parseInt(arr1.join(''), 10); - const num2 = parseInt(arr2.join(''), 10); + const arr1 = version1.split(".").map((num) => parseInt(num, 10)); + const arr2 = version2.split(".").map((num) => parseInt(num, 10)); + const num1 = parseInt(arr1.join(""), 10); + const num2 = parseInt(arr2.join(""), 10); if (num1 > num2) { return 1; } else if (num1 < num2) { @@ -179,10 +208,10 @@ async function GetProject(project: ProjectInput): Promise { } // Project does not exist yet, so we need to create it first if (existingProjects.length == 0) { - console.log('Creating new project with name %s', name); + console.log("Creating new project with name %s", name); const newProjectObj: Project = { name: name }; const newProject = await CreateProject(newProjectObj); - console.log('Project created with id: %s', newProject!.id); + console.log("Project created with id: %s", newProject!.id); return newProject; } // Project exist, so we just return it @@ -190,7 +219,7 @@ async function GetProject(project: ProjectInput): Promise { const currentProject = existingProjects[0]; if (existingProjects.length > 1) { console.warn( - 'Multiple project was found for the name %s\nReturning the first one with id %s', + "Multiple project was found for the name %s\nReturning the first one with id %s", name, currentProject.id ); @@ -206,36 +235,46 @@ async function GetProject(project: ProjectInput): Promise { * @param projectVersion Version represented as string, e.g. "1.0.0" * @returns ProjectVersion Id */ -async function GetProjectVersionId(project: Project, projectVersionInput: ProjectVersionInput) { +async function GetProjectVersionId( + project: Project, + projectVersionInput: ProjectVersionInput +) { const { version, date } = projectVersionInput; if (!project || !version) { - throw Error('Invalid information - missing project or project version'); + throw Error("Invalid information - missing project or project version"); } - if (!project.versions || project.versions.length == 0) { - } - - //Version already exists const existingProjectVersion = project.versions.find((pv) => { pv.version === version; }); if (existingProjectVersion) { console.log( - 'Version %s for project %s already exists with id %s, it will be removed.', + "Version %s for project %s already exists with id %s, it will be removed.", version, project.name, existingProjectVersion.id ); await DeleteProjectVersion(existingProjectVersion.id); } - - const newVersionId = await CreateProjectVersion(project.id, projectVersionInput); - console.log('New version for project %s created with id %s', project.name, newVersionId); + const newVersionId = await CreateProjectVersion( + project.id, + projectVersionInput + ); + console.log( + "New version (%s) for project %s created with id %s", + projectVersionInput.version, + project.name, + newVersionId + ); return newVersionId; } function createMainComponent(inputComponent) { - const purl = inputComponent.purl || `${inputComponent.name}@${inputComponent.version}`; + if (!inputComponent) { + throw Error("No main component was provided!"); + } + const purl = + inputComponent.purl || `${inputComponent.name}@${inputComponent.version}`; return { type: inputComponent.type, name: inputComponent.name, diff --git a/src/depvis-next/pages/api/graphql.ts b/src/depvis-next/pages/api/graphql.ts index 5fefb9b..911dbe9 100644 --- a/src/depvis-next/pages/api/graphql.ts +++ b/src/depvis-next/pages/api/graphql.ts @@ -43,6 +43,7 @@ async function handler(req, res) { const allowCors = (fn) => async (req, res) => { res.setHeader('Access-Control-Allow-Credentials', true); const origin = process.env.CORS_ORIGIN || 'http://localhost:3000'; + console.log(".env origin: %s", process.env.CORS_ORIGIN) res.setHeader('Access-Control-Allow-Origin', origin); res.setHeader('Access-Control-Allow-Methods', 'GET,OPTIONS,PATCH,DELETE,POST,PUT'); res.setHeader( diff --git a/src/depvis-next/pages/api/import/index.ts b/src/depvis-next/pages/api/import/index.ts index 87c740a..fb72ed5 100644 --- a/src/depvis-next/pages/api/import/index.ts +++ b/src/depvis-next/pages/api/import/index.ts @@ -1,22 +1,29 @@ -import { Queue } from 'bullmq'; -import { XMLParser } from 'fast-xml-parser'; -import { GetProjectByName } from '../../../helpers/DbDataProvider'; -import { compareVersions, getLatestProjectVersion, ImportSbom } from '../../../helpers/ImportSbomHelper'; -import { defaultBullConfig, emptyQueue } from '../../../helpers/QueueHelper'; -import { GetVulnQueueName } from '../../../queues/GetVulnQueue'; -import { ImportQueueName, ImportSbomJobData } from '../../../queues/ImportQueue'; +import { Queue } from "bullmq"; +import { XMLParser } from "fast-xml-parser"; +import { GetProjectByName } from "../../../helpers/DbDataProvider"; +import { + compareVersions, + getLatestProjectVersion, + ImportSbom, +} from "../../../helpers/ImportSbomHelper"; +import { defaultBullConfig, emptyQueue } from "../../../helpers/QueueHelper"; +import { GetVulnQueueName } from "../../../queues/GetVulnQueue"; +import { + ImportQueueName, + ImportSbomJobData, +} from "../../../queues/ImportQueue"; export const config = { api: { bodyParser: { - sizeLimit: '10mb', + sizeLimit: "10mb", }, }, }; const XMLParserOptions = { ignoreAttributes: false, - attributeNamePrefix: '', + attributeNamePrefix: "", ignoreDeclaration: true, }; @@ -40,37 +47,46 @@ type ImportRequestBody = { export default async function handler(req, res) { try { //Validate request - if (req.headers['content-type'] !== 'application/json') { - res.status(500).json({ error: "Content-type must be 'application/json'" }); + if (req.headers["content-type"] !== "application/json") { + res + .status(500) + .json({ error: "Content-type must be 'application/json'" }); return; } const body: ImportRequestBody = req.body; if (!validateImportRequestBody(body)) { - return res.status(400).json({ isError: true, error: 'Request body is malformed!' }); + return res + .status(400) + .json({ isError: true, error: "Request body is malformed!" }); } - console.log(body.projectName); + console.log("Received following body: %s", JSON.stringify(body)); const projects = await GetProjectByName(body.projectName); console.log(projects); if (projects.length != 0 && projects[0].versions) { - const highestVersionProject = getLatestProjectVersion(projects[0].versions); - if (compareVersions(body.projectVersion, highestVersionProject.version) != 1) { - return res - .status(400) - .json({ isError: true, error: `Project must have higher version than ${highestVersionProject.version}` }); + const highestVersionProject = getLatestProjectVersion( + projects[0].versions + ); + if ( + compareVersions(body.projectVersion, highestVersionProject.version) != 1 + ) { + return res.status(400).json({ + isError: true, + error: `Project must have higher version than ${highestVersionProject.version}`, + }); } } // Parse sbom const result = await parseXml(body.sbom); if (result.isError) { - console.log('Import failed with %s', result.errorMessage); + console.log("Import failed with %s", result.errorMessage); return res.status(400).json(result); } //Clear vuln queue emptyQueue(GetVulnQueue); - + console.log("Creating new job with following input: %s", body); // Create new job const job = await ImportQueue.add(body.projectName.toString(), { sbom: result.sbom, @@ -83,12 +99,13 @@ export default async function handler(req, res) { return res.status(200).json(response); } catch (err) { console.error(err); - return res.status(500).json({ error: 'failed to load data', content: err }); + return res.status(500).json({ error: "failed to load data", content: err }); } } function validateImportRequestBody(body: ImportRequestBody) { - if (!body || !body.projectName || !body.projectVersion || !body.sbom) return false; + if (!body || !body.projectName || !body.projectVersion || !body.sbom) + return false; return true; } @@ -103,17 +120,20 @@ function validateSbomXml(parsedXml): ImportResult { if (!sbom.metadata) return { isError: true, - errorMessage: "Validation failed - Missing 'metadata' parameter in the file.", + errorMessage: + "Validation failed - Missing 'metadata' parameter in the file.", }; if (!sbom.components) return { isError: true, - errorMessage: "Validation failed - Missing 'components' parameter in the file.", + errorMessage: + "Validation failed - Missing 'components' parameter in the file.", }; if (!sbom.dependencies) return { isError: true, - errorMessage: "Validation failed - Missing 'dependencies' parameter in the file.", + errorMessage: + "Validation failed - Missing 'dependencies' parameter in the file.", }; return { isError: false, sbom: sbom }; diff --git a/src/depvis-next/pages/api/vuln/index.ts b/src/depvis-next/pages/api/vuln/index.ts index 1e5e87d..fda8265 100644 --- a/src/depvis-next/pages/api/vuln/index.ts +++ b/src/depvis-next/pages/api/vuln/index.ts @@ -1,16 +1,28 @@ -import { processBatchAsync } from '../../../helpers/BatchHelper'; -import { CreateUpdateVulnerability, GetComponents } from '../../../helpers/DbDataHelper'; -import { VulnFetcherHandler } from '../../../vulnerability-mgmt/VulnFetcherHandler'; +import { processBatchAsync } from "../../../helpers/BatchHelper"; +import { + CreateUpdateVulnerability, + GetComponents, +} from "../../../helpers/DbDataHelper"; +import { VulnFetcherHandler } from "../../../vulnerability-mgmt/VulnFetcherHandler"; export default async function handler(req, res) { const { components } = await GetComponents(); const purlList = components.map((c) => c.purl); - const r = await processBatchAsync(purlList, VulnFetcherHandler, 100); + const r = await processBatchAsync(purlList, VulnFetcherHandler, { + chunkSize: 100, + }); //Use queue here r.forEach(async (component) => { if (component.vulnerabilities.length > 0) { - console.log('Creating %d vulns for %s', component.vulnerabilities.length, component.purl); - await CreateUpdateVulnerability(component.purl, component.vulnerabilities); + console.log( + "Creating %d vulns for %s", + component.vulnerabilities.length, + component.purl + ); + await CreateUpdateVulnerability( + component.purl, + component.vulnerabilities + ); } }); res.status(200).json(r); diff --git a/src/depvis-next/pages/upload.tsx b/src/depvis-next/pages/upload.tsx index 8421e00..5705b97 100644 --- a/src/depvis-next/pages/upload.tsx +++ b/src/depvis-next/pages/upload.tsx @@ -1,11 +1,11 @@ -import Link from 'next/link'; -import { useRouter } from 'next/router'; -import { useState } from 'react'; -import { Alert, Container } from 'react-bootstrap'; -import ImportForm from '../components/Import/ImportForm'; -import { ImportResult } from '../components/Import/ImportResult'; +import Link from "next/link"; +import { useRouter } from "next/router"; +import { useState } from "react"; +import { Alert, Container } from "react-bootstrap"; +import ImportForm from "../components/Import/ImportForm"; +import { ImportResult } from "../components/Import/ImportResult"; -const ImportApiUrl = '/api/import'; +const ImportApiUrl = "/api/import"; const Upload = () => { const router = useRouter(); @@ -14,8 +14,8 @@ const Upload = () => { const handleSubmit = async (data) => { const res = await fetch(ImportApiUrl, { body: await JSON.stringify(data), - headers: { 'Content-Type': 'application/json' }, - method: 'POST', + headers: { "Content-Type": "application/json" }, + method: "POST", }); if (res.status == 200) { const json = await res.json(); @@ -30,21 +30,31 @@ const Upload = () => { } return ( - {serverResponse && Form can not be submitted - {serverResponse.error}} + {serverResponse && ( + + Form can not be submitted - {serverResponse.error} + + )}

How to create SBOM for project

  1. - Go to CycloneDX Tool Center and find a tool to - generate SBOM according to your programming language + Go to{" "} + + CycloneDX Tool Center + {" "} + and find a tool to generate SBOM according to your programming + language +
    2. +
  2. + Create the SBOM file according to the documentation for a given tool + - Choose XML as output file.
    3. -
  3. Create the SBOM file according to the documentation for a given tool - Choose XML as output file.
  4. Upload generated file here
 { - console.log(data); handleSubmit(data); }} /> diff --git a/src/depvis-next/queues/ImportQueue.ts b/src/depvis-next/queues/ImportQueue.ts index 06fb1a0..bb7976e 100644 --- a/src/depvis-next/queues/ImportQueue.ts +++ b/src/depvis-next/queues/ImportQueue.ts @@ -1,8 +1,8 @@ -import { Worker } from 'bullmq'; -import { ImportSbom } from '../helpers/ImportSbomHelper'; -import { defaultBullConfig } from '../helpers/QueueHelper'; +import { Worker } from "bullmq"; +import { ImportSbom } from "../helpers/ImportSbomHelper"; +import { defaultBullConfig } from "../helpers/QueueHelper"; -export const ImportQueueName = 'import-queue'; +export const ImportQueueName = "import-queue"; export type ImportSbomJobData = { projectName: string; @@ -17,17 +17,22 @@ const worker = new Worker( await job.updateProgress(input); }; const data = job.data; - const res = await ImportSbom(data.sbom, {name: data.projectName }, data.projectVersion, updateProgress); + const res = await ImportSbom( + data.sbom, + { name: data.projectName }, + data.projectVersion, + updateProgress + ); return res; }, defaultBullConfig ); -worker.on('failed', (job, error) => { - console.log('Job %d failed with error %s', job.id, error.message); +worker.on("failed", (job, error) => { + console.log("Job %d failed with error %s", job.id, error.message); console.error(error); }); -worker.on('completed', (job) => { - console.log('Job %d completed successfully!', job.id); +worker.on("completed", (job) => { + console.log("Job %d completed successfully!", job.id); }); From 10f7321720e4004cf8354a8c46b5e7fc12838769 Mon Sep 17 00:00:00 2001 From: Matej Groman Date: Wed, 8 Mar 2023 18:47:31 +0100 Subject: [PATCH 26/33] Working import + vulnerabilities --- src/depvis-next/helpers/DbDataHelper.ts | 45 +++-------------- src/depvis-next/helpers/DbDataProvider.ts | 53 +++++++++++++++++++++ src/depvis-next/helpers/ImportSbomHelper.ts | 35 ++++++++------ src/depvis-next/pages/api/graphql.ts | 44 +++++++++-------- src/depvis-next/types/gqlTypes.ts | 24 ++++++---- 5 files changed, 120 insertions(+), 81 deletions(-) diff --git a/src/depvis-next/helpers/DbDataHelper.ts b/src/depvis-next/helpers/DbDataHelper.ts index 9259fd5..4e1cb43 100644 --- a/src/depvis-next/helpers/DbDataHelper.ts +++ b/src/depvis-next/helpers/DbDataHelper.ts @@ -152,59 +152,28 @@ export async function GetVulnerability(vulnerabilityId) { }); return data.vulnerabilities; } -export async function CreateUpdateVulnerability( - purl: string, - vulnerabilities: [Vulnerability?] -) { - console.log("Creating vulnerability for package %s", purl); - if (!vulnerabilities || vulnerabilities.length == 0 || !purl) return; - - const vulnExistsList = await Promise.all( - vulnerabilities.map(async (v) => { - const getVuln = await GetVulnerability(v.id); - return getVuln.length == 0; - }) - ); - const newVulnerabilities = vulnerabilities.filter( - (_v, index) => vulnExistsList[index] - ); - if (newVulnerabilities.length == 0) { - console.log( - "All vulnerabilities for component %s already exists in DB", - purl - ); - return; - } +export async function CreateVulnerability(vulnList: Vulnerability[]) { const mutation = gql` - mutation CreateVulnerability( - $purl: String - $vuln_array: [ComponentVulnerabilitiesCreateFieldInput!] - ) { - updateComponents( - where: { purl: $purl } - update: { vulnerabilities: { create: $vuln_array } } - ) { + mutation CreateVulnerability($input: [VulnerabilityCreateInput!]!) { + createVulnerabilities(input: $input) { info { nodesCreated - relationshipsCreated } } } `; - //Tranform for mutation compatibility - const vulnArray = newVulnerabilities.map((v) => { - return { node: PrepareVulnAsGQL(v) }; + const input = vulnList.map((v) => { + return VulnToGQL(v); }); const { data } = await sendGQLMutation(mutation, { - purl: purl, - vuln_array: vulnArray, + input: input, }); return data; } -function PrepareVulnAsGQL(vuln: Vulnerability) { +export function VulnToGQL(vuln: Vulnerability) { const refs = vuln.references ? { connectOrCreate: [ diff --git a/src/depvis-next/helpers/DbDataProvider.ts b/src/depvis-next/helpers/DbDataProvider.ts index 58b2988..85841dd 100644 --- a/src/depvis-next/helpers/DbDataProvider.ts +++ b/src/depvis-next/helpers/DbDataProvider.ts @@ -1,12 +1,15 @@ import { gql } from "@apollo/client"; import { Component } from "../types/component"; import { Project, ProjectVersion, ProjectVersionDto } from "../types/project"; +import { Vulnerability } from "../types/vulnerability"; import { sendGQLQuery, sendGQLMutation, AddProjectVersionConnectProject, BuildAddDependencyQuery, AddComponentsConnectProjectVersion, + GetVulnerability, + CreateVulnerability as CreateVulnerabilities, } from "./DbDataHelper"; import { ProjectVersionInput } from "./ImportSbomHelper"; @@ -238,3 +241,53 @@ export async function updateComponentDependency( ); } } + +export async function CreateUpdateVulnerability( + purl: string, + vulnerabilities: Vulnerability[] +) { + console.log("Creating vulnerability for package %s", purl); + if (!vulnerabilities || vulnerabilities.length == 0 || !purl) return; + + const vulnExistsList = await Promise.all( + vulnerabilities.map(async (v) => { + const getVuln = await GetVulnerability(v.id); + return getVuln.length == 0; + }) + ); + const newVulnerabilities = vulnerabilities.filter( + (_v, index) => vulnExistsList[index] + ); + const newVulnsUnique = newVulnerabilities.filter( + (v, index, arr) => arr.indexOf(v) == index + ); + //Create new vulnerabilities + await CreateVulnerabilities(newVulnsUnique); + + const mutation = gql` + mutation CreateVulnerability( + $purl: String + $vuln_array: [ComponentVulnerabilitiesConnectFieldInput!] + ) { + updateComponents( + where: { purl: $purl } + connect: { vulnerabilities: $vuln_array } + ) { + info { + nodesCreated + relationshipsCreated + } + } + } + `; + + //Connect vulnerabilities to component + const vulnArray = vulnerabilities.map((v) => { + return { where: { node: { id: v.id } } }; + }); + const { data } = await sendGQLMutation(mutation, { + purl: purl, + vuln_array: vulnArray, + }); + return data; +} diff --git a/src/depvis-next/helpers/ImportSbomHelper.ts b/src/depvis-next/helpers/ImportSbomHelper.ts index 90015a2..2a21033 100644 --- a/src/depvis-next/helpers/ImportSbomHelper.ts +++ b/src/depvis-next/helpers/ImportSbomHelper.ts @@ -10,6 +10,7 @@ import { GetProjectByName, CreateComponents, updateComponentDependency, + CreateUpdateVulnerability, } from "./DbDataProvider"; type ProjectInput = { @@ -93,22 +94,28 @@ export async function ImportSbom( importInfo.projectVersion, mainComponent.purl ); - // await UpdateProjectDependencies(projectId, [mainComponent]); - // await UpdateComponentDependencies(dependencies, projectId); await updateProgress(70, "Fetching vulnerabilities"); //Vulnerabilities - // const purlList = components.map((c) => { - // return c.purl; - // }); - // const r = await processBatch(purlList, VulnFetcherHandler); - // await updateProgress(90, 'Creating vulnerabilities in DB'); - // r.forEach(async (component) => { - // if (component.vulnerabilities.length > 0) { - // console.log('Creating %d vulns for %s', component.vulnerabilities.length, component.purl); - // await CreateUpdateVulnerability(component.purl, component.vulnerabilities); - // } - // }); - // await updateProgress(90, 'Creating vulnerabilities in DB'); + const purlList = components.map((c) => { + return c.purl; + }); + const r = await processBatchAsync(purlList, VulnFetcherHandler, { + chunkSize: 10, + }); + await updateProgress(90, "Creating vulnerabilities in DB"); + r.forEach(async (component) => { + if (component.vulnerabilities.length > 0) { + console.log( + "Creating %d vulns for %s", + component.vulnerabilities.length, + component.purl + ); + await CreateUpdateVulnerability( + component.purl, + component.vulnerabilities + ); + } + }); } catch (error) { console.error("Recovery needed"); console.error(error); diff --git a/src/depvis-next/pages/api/graphql.ts b/src/depvis-next/pages/api/graphql.ts index 911dbe9..980298e 100644 --- a/src/depvis-next/pages/api/graphql.ts +++ b/src/depvis-next/pages/api/graphql.ts @@ -1,23 +1,25 @@ -import { Neo4jGraphQL } from '@neo4j/graphql'; -import { ApolloServerPluginLandingPageGraphQLPlayground } from 'apollo-server-core'; -import { ApolloServer } from 'apollo-server-micro'; -import neo4j from 'neo4j-driver'; -import { env } from 'process'; -import { typeDefs } from '../../types/gqlTypes'; +import { Neo4jGraphQL } from "@neo4j/graphql"; +import { ApolloServerPluginLandingPageGraphQLPlayground } from "apollo-server-core"; +import { ApolloServer } from "apollo-server-micro"; +import neo4j from "neo4j-driver"; +import { env } from "process"; +import { typeDefs } from "../../types/gqlTypes"; -export const gqlUrlPath = '/api/graphql'; -const IsGQLDevToolsEnabled = env.GQL_ALLOW_DEV_TOOLS === 'true'; +export const gqlUrlPath = "/api/graphql"; +const IsGQLDevToolsEnabled = env.GQL_ALLOW_DEV_TOOLS === "true"; /** * Neo4j configuration settings * Use .env file to provide custom values! */ const NEO4J_CONFIG = { - neo4jHost: env.NEO4J_HOST || 'neo4j://localhost:7687', - neo4jUsername: env.NEO4J_USER || 'neo4j', - neo4jPassword: env.NEO4J_PASSWORD || '', + neo4jHost: env.NEO4J_HOST || "neo4j://localhost:7687", + neo4jUsername: env.NEO4J_USER || "neo4j", + neo4jPassword: env.NEO4J_PASSWORD || "", introspection: IsGQLDevToolsEnabled, - plugins: IsGQLDevToolsEnabled ? [ApolloServerPluginLandingPageGraphQLPlayground] : [], + plugins: IsGQLDevToolsEnabled + ? [ApolloServerPluginLandingPageGraphQLPlayground] + : [], }; const driver = neo4j.driver( @@ -41,16 +43,18 @@ async function handler(req, res) { } const allowCors = (fn) => async (req, res) => { - res.setHeader('Access-Control-Allow-Credentials', true); - const origin = process.env.CORS_ORIGIN || 'http://localhost:3000'; - console.log(".env origin: %s", process.env.CORS_ORIGIN) - res.setHeader('Access-Control-Allow-Origin', origin); - res.setHeader('Access-Control-Allow-Methods', 'GET,OPTIONS,PATCH,DELETE,POST,PUT'); + res.setHeader("Access-Control-Allow-Credentials", true); + const origin = process.env.CORS_ORIGIN || "http://localhost:3000"; + res.setHeader("Access-Control-Allow-Origin", origin); res.setHeader( - 'Access-Control-Allow-Headers', - 'X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, Content-Type, Date, X-Api-Version' + "Access-Control-Allow-Methods", + "GET,OPTIONS,PATCH,DELETE,POST,PUT" ); - if (req.method === 'OPTIONS') { + res.setHeader( + "Access-Control-Allow-Headers", + "X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, Content-Type, Date, X-Api-Version" + ); + if (req.method === "OPTIONS") { res.status(200).end(); return; } diff --git a/src/depvis-next/types/gqlTypes.ts b/src/depvis-next/types/gqlTypes.ts index c183b41..4ce53cf 100644 --- a/src/depvis-next/types/gqlTypes.ts +++ b/src/depvis-next/types/gqlTypes.ts @@ -1,4 +1,4 @@ -import { gql } from 'apollo-server-micro'; +import { gql } from "apollo-server-micro"; /** * Type definition for GraphQL server @@ -7,7 +7,8 @@ import { gql } from 'apollo-server-micro'; export const typeDefs = gql` type Component { id: ID @id - projectVersion: ProjectVersion @relationship(type: "BELONGS_TO", direction: OUT) + projectVersion: ProjectVersion + @relationship(type: "BELONGS_TO", direction: OUT) purl: String name: String type: String @@ -23,8 +24,10 @@ export const typeDefs = gql` """ ) dependsOn: [Component!]! @relationship(type: "DEPENDS_ON", direction: OUT) - references: [Reference!]! @relationship(type: "HAS_REFERENCE", direction: OUT) - vulnerabilities: [Vulnerability!]! @relationship(type: "HAS_VULNERABILITY", direction: OUT) + references: [Reference!]! + @relationship(type: "HAS_REFERENCE", direction: OUT) + vulnerabilities: [Vulnerability!]! + @relationship(type: "HAS_VULNERABILITY", direction: OUT) indirectVulnCount: Int @cypher( statement: """ @@ -38,22 +41,25 @@ export const typeDefs = gql` type Vulnerability { id: String @unique - cve: String @unique - ghsa: String @unique + cve: String + ghsa: String name: String description: String affectedVersions: String cvssScore: Float cvssVector: String cwe: String - references: [Reference!]! @relationship(type: "HAS_REFERENCE", direction: OUT) - affectedComponents: [Component!]! @relationship(type: "HAS_VULNERABILITY", direction: IN) + references: [Reference!]! + @relationship(type: "HAS_REFERENCE", direction: OUT) + affectedComponents: [Component!]! + @relationship(type: "HAS_VULNERABILITY", direction: IN) } type Project { id: ID @id name: String @unique - versions: [ProjectVersion!]! @relationship(type: "HAS_VERSION", direction: OUT) + versions: [ProjectVersion!]! + @relationship(type: "HAS_VERSION", direction: OUT) } type ProjectVersion { From d0eb41ded006ba53fb8385f6fdb9e7d07a6a804f Mon Sep 17 00:00:00 2001 From: Matej Groman Date: Mon, 13 Mar 2023 16:27:53 +0100 Subject: [PATCH 27/33] WIP: working upload + visualization --- docker-compose.yml | 7 +- .../components/Details/ComponentDetails.tsx | 25 ++- .../components/Dropdown/Dropdown.tsx | 37 ++-- .../components/Layout/Workspace.tsx | 161 ----------------- .../components/Workspace/ProjectSelector.tsx | 100 ++++++++++ .../Workspace/ProjectVersionSelector.tsx | 59 ++++++ .../components/Workspace/Workspace.tsx | 171 ++++++++++++++++++ src/depvis-next/helpers/DbDataProvider.ts | 27 ++- src/depvis-next/helpers/GraphHelper.ts | 20 +- src/depvis-next/helpers/ImportSbomHelper.ts | 6 +- src/depvis-next/pages/api/vuln/index.ts | 6 +- src/depvis-next/pages/index.tsx | 2 +- src/depvis-next/pages/toolbox.tsx | 30 ++- src/depvis-next/queues/GetVulnQueue.ts | 18 +- src/depvis-next/types/component.ts | 13 +- 15 files changed, 459 insertions(+), 223 deletions(-) delete mode 100644 src/depvis-next/components/Layout/Workspace.tsx create mode 100644 src/depvis-next/components/Workspace/ProjectSelector.tsx create mode 100644 src/depvis-next/components/Workspace/ProjectVersionSelector.tsx create mode 100644 src/depvis-next/components/Workspace/Workspace.tsx diff --git a/docker-compose.yml b/docker-compose.yml index fb24bfa..f2812dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,8 @@ -version: '3' +version: "3" services: neo4j: - image: neo4j:4.4.7 + image: neo4j:latest ports: - 7474:7474 - 7687:7687 @@ -22,6 +22,7 @@ services: redis-cache: image: redis:latest command: redis-server --requirepass ${REDIS_PASSWORD} + restart: unless-stopped volumes: - cache:/data ports: @@ -31,7 +32,7 @@ services: redis-commander: image: rediscommander/redis-commander:latest - restart: always + restart: unless-stopped environment: REDIS_HOSTS: redis-cache REDIS_HOST: redis-cache diff --git a/src/depvis-next/components/Details/ComponentDetails.tsx b/src/depvis-next/components/Details/ComponentDetails.tsx index 07df282..a89fb49 100644 --- a/src/depvis-next/components/Details/ComponentDetails.tsx +++ b/src/depvis-next/components/Details/ComponentDetails.tsx @@ -1,12 +1,14 @@ -import { gql, useQuery } from '@apollo/client'; -import { Container } from 'react-bootstrap'; -import { GetComponentRepositoryURL } from '../../helpers/WorkspaceHelper'; -import Loading from '../Loading/Loading'; -import { DL, DLItem } from './DescriptionList'; +import { gql, useQuery } from "@apollo/client"; +import { Container } from "react-bootstrap"; +import { GetComponentRepositoryURL } from "../../helpers/WorkspaceHelper"; +import Loading from "../Loading/Loading"; +import { DL, DLItem } from "./DescriptionList"; const getComponentDetailsQuery = gql` query componentDetails($componentPurl: String, $projectId: ID) { - components(where: { purl: $componentPurl, project_SINGLE: { id: $projectId } }) { + components( + where: { purl: $componentPurl, projectVersion: { id: $projectId } } + ) { name purl author @@ -36,13 +38,15 @@ const ComponentDetails = (props) => { }; if (loading) return ; if (!data.components[0]) { - console.error('No data found when querying backend! Below is Apollo query result'); + console.error( + "No data found when querying backend! Below is Apollo query result" + ); console.error({ data: data, error: error }); return No data found!; } const component = data.components[0]; return ( - +

{component.name}

@@ -51,7 +55,10 @@ const ComponentDetails = (props) => { - + ( diff --git a/src/depvis-next/components/Dropdown/Dropdown.tsx b/src/depvis-next/components/Dropdown/Dropdown.tsx index 85b90b2..34671bf 100644 --- a/src/depvis-next/components/Dropdown/Dropdown.tsx +++ b/src/depvis-next/components/Dropdown/Dropdown.tsx @@ -1,24 +1,37 @@ -import { useState } from 'react'; -import { Container, Form } from 'react-bootstrap'; +import { useEffect, useState } from "react"; +import { Container, Form } from "react-bootstrap"; +export type DropdownItem = { + displayName: string; + id: string; +}; + +/** + * Dropdown component, renders set of options and fires a callback on change + * @param props one of title, onChange, defaultValue, options, disabled + * @returns + */ export default function Dropdown(props) { - const [selected, setSelected] = useState(props.default); + const { title, onChange, defaultValue, options, disabled } = props; + const [selectedId, setSelectedId] = useState(defaultValue); return (
- {props.title && {props.title}} + {title && {title}} { - props.onChange(e.target.value); - setSelected(e.target.value); + setSelectedId(e.target.value); + onChange(e.target.value); }} > - {props.options.map((v, i) => ( - - ))} + {options && + options.map((v: DropdownItem, i: number) => ( + + ))}
diff --git a/src/depvis-next/components/Layout/Workspace.tsx b/src/depvis-next/components/Layout/Workspace.tsx deleted file mode 100644 index da6047f..0000000 --- a/src/depvis-next/components/Layout/Workspace.tsx +++ /dev/null @@ -1,161 +0,0 @@ -import { useLazyQuery, useQuery } from '@apollo/client'; -import Link from 'next/link'; -import { useRouter } from 'next/router'; -import { useCallback, useEffect, useState } from 'react'; -import { Container, Row } from 'react-bootstrap'; -import { - formatData, - getAllComponentsQuery, - getNodeColor, - getNodeValue, - getProjectsQuery, -} from '../../helpers/GraphHelper'; -import ComponentDetails from '../Details/ComponentDetails'; -import Details from '../Details/Details'; -import VulnerabilityDetails from '../Details/VulnerabilityDetails'; -import Dropdown from '../Dropdown/Dropdown'; -import NoProjectFoundError from '../Error/NoProjectFoundError'; -import { GraphConfig } from '../Graph/GraphConfig'; -import GraphControl from '../GraphControl/GraphControl'; -import ImportForm from '../Import/ImportForm'; -import Loading from '../Loading/Loading'; -import Search from '../Search/Search'; -import GraphContainer from './GraphContainer'; -import Sidebar from './Sidebar'; - -const defaultGraphConfig: GraphConfig = { - zoomLevel: 1, - color: getNodeColor, - label: 'id', - linkDirectionalArrowLength: 5, - linkDirectionalRelPos: 0, - linkLength: 10, - nodeVal: getNodeValue, - showOnlyVulnerable: false, -}; - -const Workspace = () => { - const [node, setNode] = useState(undefined); - const [graphConfig, setGraphConfig] = useState(defaultGraphConfig); - const [graphData, setGraphData] = useState({ nodes: [], links: [] }); - const [selectedProject, setSelectedProject] = useState(); - const [getGraphData, { loading, error, data }] = useLazyQuery(getAllComponentsQuery); - const { data: projects, loading: projectsLoading } = useQuery(getProjectsQuery, { - onCompleted: (data) => { - const project = selectProject(data.projects, router.query.projectName); - console.log(project); - setSelectedProject(project); - }, - }); - const router = useRouter(); - - useEffect(() => { - if (data) { - setGraphData(formatData(data.projects[0].allComponents)); - } - }, [data]); - useEffect(() => { - console.log(`Detected change, val ${selectedProject}`); - - if (selectedProject) { - console.log('Getting data'); - getGraphData({ variables: { projectId: selectedProject }, pollInterval: 1000 }); - } - }, [selectedProject]); - useEffect(() => { - handleShowOnlyVulnerableToggle(); - }, [graphConfig]); - - const handleNodeClick = (node) => { - setNode(node); - }; - - const handleShowOnlyVulnerableToggle = () => { - if (!data) return; - if (graphConfig.showOnlyVulnerable) { - setGraphData(formatData(data.projects[0].allVulnerableComponents)); - } else { - setGraphData(formatData(data.projects[0].allComponents)); - } - }; - - const handleSelectedSearchResult = (object) => { - setNode(object); - }; - - const handleNodeValToggle = (e) => { - if (typeof graphConfig.nodeVal === 'function') { - setGraphConfig({ ...graphConfig, nodeVal: 1 }); - console.log(graphConfig); - } else { - setGraphConfig({ ...graphConfig, nodeVal: getNodeValue }); - } - }; - - const paintRing = useCallback( - (currNode, ctx) => { - if (node) { - // add ring just for highlighted nodes - ctx.beginPath(); - ctx.arc(node.x, node.y, node.size | 1, 0, 2 * Math.PI, false); - ctx.fillStyle = currNode === node ? 'red' : 'orange'; - ctx.fill(); - } - }, - [node] - ); - - if (projectsLoading) return ; - if (!selectedProject) return ; - return ( - - - - setSelectedProject(e)} - default={selectedProject} - /> - - handleSelectedSearchResult(obj)} /> - { - setGraphConfig(val); - }} - onRefetchGraphClick={() => { - getGraphData({ - variables: { projectId: selectedProject }, - fetchPolicy: 'no-cache', - }); - }} - /> - {node && node.__typename === 'Component' && ( - - )} - {node && node.__typename === 'Vulnerability' && } - -
- - {!loading && ( - handleNodeClick(node)} - graphConfig={graphConfig} - /> - )} - - - ); -}; - -export default Workspace; - -const selectProject = (projects, queryProjectName) => { - if (!queryProjectName || projects.filter((p) => p.name === queryProjectName).length == 0) return projects[0].id; - return projects.filter((p) => p.name === queryProjectName)[0].id; -}; diff --git a/src/depvis-next/components/Workspace/ProjectSelector.tsx b/src/depvis-next/components/Workspace/ProjectSelector.tsx new file mode 100644 index 0000000..784418b --- /dev/null +++ b/src/depvis-next/components/Workspace/ProjectSelector.tsx @@ -0,0 +1,100 @@ +import { useQuery } from "@apollo/client"; +import { useRouter } from "next/router"; +import { useEffect, useState } from "react"; +import { Button, Container } from "react-bootstrap"; +import { getProjectsQuery } from "../../helpers/GraphHelper"; +import Dropdown, { DropdownItem } from "../Dropdown/Dropdown"; + +type ProjectSelectorProps = { + onProjectVersionSelect?: (projectVersionId: DropdownItem) => void; +}; +const ProjectSelector = (props: ProjectSelectorProps) => { + const { onProjectVersionSelect } = props; + const [project, setProject] = useState(); + const [projectVersion, setProjectVersion] = useState(); + const router = useRouter(); + const { data: projects, loading: projectsLoading } = useQuery( + getProjectsQuery, + { + onCompleted: (data) => { + const project = selectProject(data.projects, router.query.projectName); + console.log("Selected project %s", project.id); + setProject({ id: project.id, displayName: project.name }); + const version = selectVersion(project, router.query.projectVersion); + console.log("Selected version %s", version.id); + + setProjectVersion(version); + }, + } + ); + + // useEffect(() => { + // if (projectVersion) { + // onProjectVersionSelect(projectVersion); + // } + // }, [projectVersion]); + const getProjectVersions = (project) => { + const projectObj = projects.projects.filter((p) => p.id === project.id)[0]; + return projectObj.versions.map((v) => { + return { id: v.id, displayName: v.version }; + }); + }; + + const handleProjectVersionSelect = (e: any) => { + console.log({ selectedVersion: e }); + onProjectVersionSelect && onProjectVersionSelect(e.id); + setProjectVersion(e); + }; + return ( + + {!projectsLoading && ( + <> + { + return { id: p.id, displayName: p.name }; + })} + onChange={(e) => { + setProjectVersion(undefined); + setProject(e); + }} + default={project} + /> + handleProjectVersionSelect(e)} + default={projectVersion} + /> + + )} + + ); +}; + +const selectProject = (projects, queryProjectName) => { + console.log({ projects: projects, query: queryProjectName }); + if ( + !queryProjectName || + projects.filter((p) => p.name === queryProjectName).length == 0 + ) + console.log("Fuck you"); + return projects[0]; + return projects.find((p) => p.name === queryProjectName); +}; + +const selectVersion = (project, queryProjectVersion) => { + if (project.versions.length == 0) { + console.log("Project %s does not have any versions", project.id); + return; + } + if ( + !queryProjectVersion || + project.versions.filter((v) => v.version === queryProjectVersion).length == + 0 + ) { + return project.versions[0]; + } + return project.versions.filter((v) => v.version === queryProjectVersion)[0]; +}; +export default ProjectSelector; diff --git a/src/depvis-next/components/Workspace/ProjectVersionSelector.tsx b/src/depvis-next/components/Workspace/ProjectVersionSelector.tsx new file mode 100644 index 0000000..514cdb7 --- /dev/null +++ b/src/depvis-next/components/Workspace/ProjectVersionSelector.tsx @@ -0,0 +1,59 @@ +import { useQuery } from "@apollo/client"; +import { useRouter } from "next/router"; +import { useEffect, useState } from "react"; +import { Container } from "react-bootstrap"; +import { getProjectVersionsQuery } from "../../helpers/GraphHelper"; +import Dropdown, { DropdownItem } from "../Dropdown/Dropdown"; + +type ProjectSelectorProps = { + onProjectVersionSelect?: (projectVersionId: string) => void; +}; +const ProjectVersionSelector = (props: ProjectSelectorProps) => { + const { onProjectVersionSelect } = props; + const [projectVersion, setProjectVersion] = useState(); + const router = useRouter(); + const { data: projects, loading: projectsLoading } = useQuery( + getProjectVersionsQuery, + { + onCompleted: (data) => { + setProjectVersion(data.projectVersions[0].id); + }, + } + ); + + useEffect(() => { + if (projectVersion) { + console.log("Fire"); + onProjectVersionSelect(projectVersion); + } + }, [projectVersion]); + + const handleProjectVersionSelect = (e: any) => { + setProjectVersion(e); + }; + + const transformDefault = () => { + const pv = projects.projectVersions.find((pv) => pv.id == projectVersion); + console.log({ projV: projectVersion, pv: pv }); + return { + id: projectVersion, + displayName: `${pv.name} v${pv.version}`, + }; + }; + return ( + + {!projectsLoading && ( + { + return { id: p.id, displayName: `${p.project.name} v${p.version}` }; + })} + onChange={(e) => handleProjectVersionSelect(e)} + defaultValue={projectVersion} + /> + )} + + ); +}; + +export default ProjectVersionSelector; diff --git a/src/depvis-next/components/Workspace/Workspace.tsx b/src/depvis-next/components/Workspace/Workspace.tsx new file mode 100644 index 0000000..7c8bae8 --- /dev/null +++ b/src/depvis-next/components/Workspace/Workspace.tsx @@ -0,0 +1,171 @@ +import { useLazyQuery, useQuery } from "@apollo/client"; +import Link from "next/link"; +import { useRouter } from "next/router"; +import { useCallback, useEffect, useState } from "react"; +import { Container, Row } from "react-bootstrap"; +import { + formatData, + getAllComponentsQuery, + getNodeColor, + getNodeValue, + getProjectsQuery, +} from "../../helpers/GraphHelper"; +import ComponentDetails from "../Details/ComponentDetails"; +import Details from "../Details/Details"; +import VulnerabilityDetails from "../Details/VulnerabilityDetails"; +import NoProjectFoundError from "../Error/NoProjectFoundError"; +import { GraphConfig } from "../Graph/GraphConfig"; +import GraphControl from "../GraphControl/GraphControl"; +import ImportForm from "../Import/ImportForm"; +import Loading from "../Loading/Loading"; +import Search from "../Search/Search"; +import GraphContainer from "../Layout/GraphContainer"; +import Sidebar from "../Layout/Sidebar"; +import ProjectSelector from "./ProjectSelector"; +import { DropdownItem } from "../Dropdown/Dropdown"; +import ProjectVersionSelector from "./ProjectVersionSelector"; + +const defaultGraphConfig: GraphConfig = { + zoomLevel: 1, + color: getNodeColor, + label: "id", + linkDirectionalArrowLength: 5, + linkDirectionalRelPos: 0, + linkLength: 10, + nodeVal: getNodeValue, + showOnlyVulnerable: false, +}; + +const Workspace = () => { + const [node, setNode] = useState(undefined); + const [graphConfig, setGraphConfig] = + useState(defaultGraphConfig); + const [graphData, setGraphData] = useState({ nodes: [], links: [] }); + const [selectedProjectVersion, setSelectedProjectVersion] = useState< + string | undefined + >(); + const [getGraphData, { loading, error, data }] = useLazyQuery( + getAllComponentsQuery + ); + + const router = useRouter(); + + useEffect(() => { + if (data) { + console.log(data); + setGraphData(formatData(data.projectVersions[0].allComponents)); + } + }, [data]); + useEffect(() => { + console.log(`Detected change, val ${selectedProjectVersion}`); + + if (selectedProjectVersion) { + console.log("Getting data"); + getGraphData({ + variables: { projectVersionId: selectedProjectVersion }, + pollInterval: 1000, + }); + } + }, [selectedProjectVersion]); + useEffect(() => { + handleShowOnlyVulnerableToggle(); + }, [graphConfig]); + + const handleNodeClick = (node) => { + setNode(node); + }; + + const handleShowOnlyVulnerableToggle = () => { + if (!data) return; + if (graphConfig.showOnlyVulnerable) { + setGraphData(formatData(data.projectVersions[0].allVulnerableComponents)); + } else { + setGraphData(formatData(data.projectVersions[0].allComponents)); + } + }; + + const handleSelectedSearchResult = (object) => { + setNode(object); + }; + + const handleNodeValToggle = (e) => { + if (typeof graphConfig.nodeVal === "function") { + setGraphConfig({ ...graphConfig, nodeVal: 1 }); + console.log(graphConfig); + } else { + setGraphConfig({ ...graphConfig, nodeVal: getNodeValue }); + } + }; + + const paintRing = useCallback( + (currNode, ctx) => { + if (node) { + // add ring just for highlighted nodes + ctx.beginPath(); + ctx.arc(node.x, node.y, node.size | 1, 0, 2 * Math.PI, false); + ctx.fillStyle = currNode === node ? "red" : "orange"; + ctx.fill(); + } + }, + [node] + ); + + const handleProjectVersion = (projectVersion: string) => { + setSelectedProjectVersion(projectVersion); + }; + + if (!selectedProjectVersion) + return ( + + ); + return ( + + + + + handleSelectedSearchResult(obj)} + /> + { + setGraphConfig(val); + }} + onRefetchGraphClick={() => { + getGraphData({ + variables: { projectId: selectedProjectVersion }, + fetchPolicy: "no-cache", + }); + }} + /> + {node && node.__typename === "Component" && ( + + )} + {node && node.__typename === "Vulnerability" && ( + + )} + +
+ + {!loading && ( + handleNodeClick(node)} + graphConfig={graphConfig} + /> + )} + + + ); +}; + +export default Workspace; diff --git a/src/depvis-next/helpers/DbDataProvider.ts b/src/depvis-next/helpers/DbDataProvider.ts index 85841dd..972f84f 100644 --- a/src/depvis-next/helpers/DbDataProvider.ts +++ b/src/depvis-next/helpers/DbDataProvider.ts @@ -1,5 +1,5 @@ import { gql } from "@apollo/client"; -import { Component } from "../types/component"; +import { Component, Dependency } from "../types/component"; import { Project, ProjectVersion, ProjectVersionDto } from "../types/project"; import { Vulnerability } from "../types/vulnerability"; import { @@ -150,6 +150,12 @@ export async function DeleteProjectVersion( return data.nodesDeleted; } +/** + * Creates components in database and connects them with projectVersion + * @param components List of components + * @param projectVersionId ID of the project version + * @returns List of created components + */ export async function CreateComponents( components: Component[], projectVersionId: string @@ -179,10 +185,18 @@ export async function CreateComponents( const { data } = await sendGQLMutation(mutation, { components: componentsWithConnect, }); + return data.createComponents.components; } +/** + * Adds dependency relationship for components and connects main component to project version + * @param dependencies List of dependencies + * @param projectVersionId ID of the project version + * @param mainComponentPurl purl of the main component that will be connected to projectVersion + * @returns number of dependencies created + */ export async function updateComponentDependency( - dependencies: any[], + dependencies: Dependency[], projectVersionId: string, mainComponentPurl: string ) { @@ -229,17 +243,16 @@ export async function updateComponentDependency( dependencies, projectVersionId ); + let dependencyCount = 0; for (let index = 0; index < dependencyQueryList.length; index++) { const { data } = await sendGQLMutation(mutation, { where: dependencyQueryList[index].where, connect: dependencyQueryList[index].connect, }); - console.log(data); - console.log( - "Created %s relationships", - data.name.info.relationshipsCreated - ); + dependencyCount += data.name.info.relationshipsCreated; } + console.log("Created %s relationships", dependencyCount); + return dependencyCount; } export async function CreateUpdateVulnerability( diff --git a/src/depvis-next/helpers/GraphHelper.ts b/src/depvis-next/helpers/GraphHelper.ts index 0541eb6..bf59229 100644 --- a/src/depvis-next/helpers/GraphHelper.ts +++ b/src/depvis-next/helpers/GraphHelper.ts @@ -67,8 +67,8 @@ export const formatData = (components) => { }; export const getAllComponentsQuery = gql` - query getProjectComponents($projectId: ID) { - projects(where: { id: $projectId }) { + query getProjectComponents($projectVersionId: ID) { + projectVersions(where: { id: $projectVersionId }) { allVulnerableComponents { id name @@ -123,7 +123,23 @@ export const getProjectsQuery = gql` projects { id name + versions { + id + version + } + } + } +`; + +export const getProjectVersionsQuery = gql` + { + projectVersions { + id version + project { + name + id + } } } `; diff --git a/src/depvis-next/helpers/ImportSbomHelper.ts b/src/depvis-next/helpers/ImportSbomHelper.ts index 2a21033..9ee7b8c 100644 --- a/src/depvis-next/helpers/ImportSbomHelper.ts +++ b/src/depvis-next/helpers/ImportSbomHelper.ts @@ -1,4 +1,4 @@ -import { Component } from "../types/component"; +import { Component, Dependency } from "../types/component"; import { Project, ProjectVersion } from "../types/project"; import { VulnFetcherHandler } from "../vulnerability-mgmt/VulnFetcherHandler"; import { processBatchAsync } from "./BatchHelper"; @@ -137,9 +137,9 @@ function GetComponents(bom: any) { return components; } -function GetDependencies(dependencies: any) { +function GetDependencies(dependencies: any): Dependency[] { if (!dependencies) return; - const res = dependencies + const res: Dependency[] = dependencies .map((d) => { if (d.dependency != undefined) { if (!(d.dependency instanceof Array)) d.dependency = [d.dependency]; diff --git a/src/depvis-next/pages/api/vuln/index.ts b/src/depvis-next/pages/api/vuln/index.ts index fda8265..e693098 100644 --- a/src/depvis-next/pages/api/vuln/index.ts +++ b/src/depvis-next/pages/api/vuln/index.ts @@ -1,8 +1,6 @@ import { processBatchAsync } from "../../../helpers/BatchHelper"; -import { - CreateUpdateVulnerability, - GetComponents, -} from "../../../helpers/DbDataHelper"; +import { GetComponents } from "../../../helpers/DbDataHelper"; +import { CreateUpdateVulnerability } from "../../../helpers/DbDataProvider"; import { VulnFetcherHandler } from "../../../vulnerability-mgmt/VulnFetcherHandler"; export default async function handler(req, res) { const { components } = await GetComponents(); diff --git a/src/depvis-next/pages/index.tsx b/src/depvis-next/pages/index.tsx index 4e91549..75f6d88 100644 --- a/src/depvis-next/pages/index.tsx +++ b/src/depvis-next/pages/index.tsx @@ -1,4 +1,4 @@ -import Workspace from '../components/Layout/Workspace'; +import Workspace from "../components/Workspace/Workspace"; const index = () => { return ; diff --git a/src/depvis-next/pages/toolbox.tsx b/src/depvis-next/pages/toolbox.tsx index 5ff2f71..4fb97ed 100644 --- a/src/depvis-next/pages/toolbox.tsx +++ b/src/depvis-next/pages/toolbox.tsx @@ -1,19 +1,28 @@ -import Link from 'next/link'; -import { useRouter } from 'next/router'; -import { PackageURL } from 'packageurl-js'; -import { useEffect, useState } from 'react'; -import { Button, Col, Container, Form, ProgressBar, Row, Stack } from 'react-bootstrap'; -import { ParsePurl } from '../components/Toolbox/ParsePurl'; +import Link from "next/link"; +import { useRouter } from "next/router"; +import { PackageURL } from "packageurl-js"; +import { useEffect, useState } from "react"; +import { + Button, + Col, + Container, + Form, + ProgressBar, + Row, + Stack, +} from "react-bootstrap"; +import { ParsePurl } from "../components/Toolbox/ParsePurl"; +import ProjectSelector from "../components/Workspace/ProjectSelector"; const Toolbox = () => { - const [purlString, setPurlString] = useState(''); - const [purlOutput, setPurlOutput] = useState(''); + const [purlString, setPurlString] = useState(""); + const [purlOutput, setPurlOutput] = useState(""); const handlePurl = async () => { setPurlOutput(await JSON.stringify(PackageURL.fromString(purlString))); console.log(purlOutput); }; const handleVuln = async () => { - const res = await fetch('/api/vuln'); + const res = await fetch("/api/vuln"); console.log(res); }; @@ -43,6 +52,9 @@ const Toolbox = () => { + + + ); }; diff --git a/src/depvis-next/queues/GetVulnQueue.ts b/src/depvis-next/queues/GetVulnQueue.ts index e39ab81..076041c 100644 --- a/src/depvis-next/queues/GetVulnQueue.ts +++ b/src/depvis-next/queues/GetVulnQueue.ts @@ -1,8 +1,8 @@ -import { Job, Worker } from 'bullmq'; -import { CreateUpdateVulnerability } from '../helpers/DbDataHelper'; -import { defaultBullConfig } from '../helpers/QueueHelper'; -import { VulnFetcherHandler } from '../vulnerability-mgmt/VulnFetcherHandler'; -export const GetVulnQueueName = 'get-vuln-queue'; +import { Job, Worker } from "bullmq"; +import { CreateUpdateVulnerability } from "../helpers/DbDataProvider"; +import { defaultBullConfig } from "../helpers/QueueHelper"; +import { VulnFetcherHandler } from "../vulnerability-mgmt/VulnFetcherHandler"; +export const GetVulnQueueName = "get-vuln-queue"; const worker = new Worker( GetVulnQueueName, @@ -15,11 +15,11 @@ const worker = new Worker( defaultBullConfig ); -worker.on('failed', (job, error) => { - console.log('Job %d failed with error %s', job.id, error.message); +worker.on("failed", (job, error) => { + console.log("Job %d failed with error %s", job.id, error.message); console.error(error); }); -worker.on('completed', (job) => { - console.log('Job %d completed successfully!', job.id); +worker.on("completed", (job) => { + console.log("Job %d completed successfully!", job.id); }); diff --git a/src/depvis-next/types/component.ts b/src/depvis-next/types/component.ts index fec561a..331401d 100644 --- a/src/depvis-next/types/component.ts +++ b/src/depvis-next/types/component.ts @@ -1,6 +1,6 @@ -import { Project } from './project'; -import { Reference } from './reference'; -import { Vulnerability } from './vulnerability'; +import { Project } from "./project"; +import { Reference } from "./reference"; +import { Vulnerability } from "./vulnerability"; export type Component = { /** @@ -56,3 +56,10 @@ export type Component = { export type ComponentDto = Component & { projectVersion: any; }; + +export type Dependency = { + purl: string; + dependsOn: { + purl: string; + }[]; +}; From 7664e5ca2f3a04f0ca828312cdbf7c24d1616d75 Mon Sep 17 00:00:00 2001 From: Matej Groman Date: Fri, 17 Mar 2023 19:21:34 +0100 Subject: [PATCH 28/33] Graph cleanup --- .../components/Graph/NoSSRGraph.tsx | 100 +++--------------- .../components/Layout/GraphContainer.tsx | 20 ++-- .../components/NavBar/MainNavBar.tsx | 9 +- .../Workspace/ProjectVersionSelector.tsx | 5 + .../components/Workspace/Workspace.tsx | 18 ++-- 5 files changed, 47 insertions(+), 105 deletions(-) diff --git a/src/depvis-next/components/Graph/NoSSRGraph.tsx b/src/depvis-next/components/Graph/NoSSRGraph.tsx index 5434b26..3875db5 100644 --- a/src/depvis-next/components/Graph/NoSSRGraph.tsx +++ b/src/depvis-next/components/Graph/NoSSRGraph.tsx @@ -1,6 +1,6 @@ -import { forceCollide, forceManyBody, forceX, forceY } from 'd3-force'; -import { useCallback, useEffect, useRef } from 'react'; -import ReactForceGraph2d, { ForceGraphMethods } from 'react-force-graph-2d'; +import { forceCollide, forceManyBody, forceX, forceY } from "d3-force"; +import { useEffect, useRef } from "react"; +import ReactForceGraph2d, { ForceGraphMethods } from "react-force-graph-2d"; export default function NoSSRGraph(props) { const graphRef = useRef(); @@ -9,97 +9,23 @@ export default function NoSSRGraph(props) { const r = graphRef.current; if (props.selectedNode) { console.log(props.selectedNode); - r.centerAt(props.selectedNode.x, props.selectedNode.y, 500); - const zoom_level = 50 / (props.selectedNode.dependsOnCount || 30); - graphRef.current.zoom(zoom_level, 500); + r.centerAt(props.selectedNode.x, props.selectedNode.y, 1000); } r.d3Force( - 'collide', + "collide", forceCollide(10) .radius((node) => node.size | 1) .strength(0.1) ); - r.d3Force('x', forceX()); - r.d3Force('y', forceY()); - r.d3Force('link') - .distance((link) => link.source.size + link.target.size + props.linkLength) + r.d3Force("x", forceX()); + r.d3Force("y", forceY()); + r.d3Force("link") + .distance( + (link) => link.source.size + link.target.size + props.linkLength + ) .strength(1); - r.d3Force('charge', forceManyBody().strength(-50)); + r.d3Force("charge", forceManyBody().strength(-50)); }, [props, graphRef]); - return ( - { - // ctx.fillStyle = color; - // const bckgDimensions = 1; - // bckgDimensions && ctx.fillRect(5, 5, 10, 10); - // }} - /> - ); + return ; } -// const my_map = (value, x1, y1, x2, y2) => -// ((value - x1) * (y2 - x2)) / (y1 - x1) + x2; - -// export default function NoSSRGraph(props) { -// const maxDepsCount = 262; -// const graphRef = useRef(); -// const f = new SonatypeOSSVulnProvider(); - -// useEffect(() => { -// const r = graphRef.current; -// r.zoomToFit(); -// // r.d3Force("collide", forceCollide(4)); -// // r.d3Force("x", forceX()); -// // r.d3Force("y", forceY()); -// // r.d3Force("link") -// // .distance( -// // (link) => -// // 1 * (link.source.dependsOnCount + link.target.dependsOnCount * 2) -// // ) -// // // .distance((link) => 1000) -// // .strength(1); -// // r.d3Force("charge", forceManyBody().strength(-70)); -// }, []); -// const handleClick = useCallback( -// (node) => { -// // Aim at node from outside it -// graphRef.current.centerAt(node.x, node.y, 500); -// let zoom_level = 3 - node.dependsOnCount / 100; -// if (zoom_level < 0.5) zoom_level = 0.5; -// graphRef.current.zoom(zoom_level, 500); -// console.log(node.neighbors); -// // f.fetchInfo(node.id); -// }, -// [graphRef] -// ); -// return ( -// "after"} -// nodeCanvasObject={(node, ctx, globalScale) => { -// ctx.canvas.width = 500; -// ctx.canvas.height = 500; -// // const label = node.id as string; //`${node.name} | ${node.dependsOnCount}`; -// // //const fontSize = my_map(node.dependsOnCount, 0, maxDepsCount, 5, 22); -// // ctx.font = "12px Sans-Serif"; //`${fontSize}px Sans-Serif`; -// // ctx.fillStyle = "#121212"; -// // ctx.textAlign = "center"; -// // ctx.textBaseline = "middle"; -// // ctx.fillText(label, node.x, node.y); -// }} -// {...props} -// linkDirectionalArrowLength={5} -// linkDirectionalArrowRelPos={3} -// nodeAutoColorBy={"id"} //nodeAutoColorBy={"__typename"} -// nodeLabel={"name"} -// // nodeVal={"dependsOnCount"} -// // backgroundColor={'#f0f0f0'} -// linkColor={"#ff0000"} -// // onNodeClick={(n) => { -// // handleClick(n); -// // }} -// /> -// ); -// } diff --git a/src/depvis-next/components/Layout/GraphContainer.tsx b/src/depvis-next/components/Layout/GraphContainer.tsx index 380acc4..9f66a25 100644 --- a/src/depvis-next/components/Layout/GraphContainer.tsx +++ b/src/depvis-next/components/Layout/GraphContainer.tsx @@ -1,10 +1,10 @@ -import { useEffect, useRef, useState } from 'react'; -import { Col } from 'react-bootstrap'; -import { ForceGraphMethods } from 'react-force-graph-2d'; -import { getNodeColor, getNodeValue } from '../../helpers/GraphHelper'; -import { GraphConfig } from '../Graph/GraphConfig'; -import NoSSRGraphWrapper from '../Graph/NoSSRGraphWrapper'; -import Loading from '../Loading/Loading'; +import { useEffect, useRef, useState } from "react"; +import { Col } from "react-bootstrap"; +import { ForceGraphMethods } from "react-force-graph-2d"; +import { getNodeColor, getNodeValue } from "../../helpers/GraphHelper"; +import { GraphConfig } from "../Graph/GraphConfig"; +import NoSSRGraphWrapper from "../Graph/NoSSRGraphWrapper"; +import Loading from "../Loading/Loading"; const GraphContainer = (props) => { const [graphDimensions, setGraphDimensions] = useState({ width: 0, @@ -21,9 +21,9 @@ const GraphContainer = (props) => { useEffect(() => { setSize(); - window.addEventListener('resize', setSize); + window.addEventListener("resize", setSize); return () => { - window.removeEventListener('resize', setSize); + window.removeEventListener("resize", setSize); }; }, []); @@ -34,7 +34,7 @@ const GraphContainer = (props) => { {props.isLoading && } {!props.isLoading && ( 'after'} + nodeCanvasObjectMode={() => "after"} linkLength={graphConfig.linkLength} {...props} width={graphDimensions.width} diff --git a/src/depvis-next/components/NavBar/MainNavBar.tsx b/src/depvis-next/components/NavBar/MainNavBar.tsx index 265ef36..a2467bc 100644 --- a/src/depvis-next/components/NavBar/MainNavBar.tsx +++ b/src/depvis-next/components/NavBar/MainNavBar.tsx @@ -1,4 +1,5 @@ import Link from "next/link"; +import { useRouter } from "next/router"; import { Button } from "react-bootstrap"; import Container from "react-bootstrap/Container"; import Nav from "react-bootstrap/Nav"; @@ -7,6 +8,12 @@ import NavDropdown from "react-bootstrap/NavDropdown"; import { DeleteAllData } from "../../helpers/DbDataHelper"; import { gqlUrlPath } from "../../pages/api/graphql"; const MainNavbar = () => { + const router = useRouter(); + + const handleDeleteAllData = async () => { + await DeleteAllData(); + router.reload(); + }; return ( @@ -30,7 +37,7 @@ const MainNavbar = () => { className="mx-3" variant="danger" onClick={() => { - DeleteAllData(); + handleDeleteAllData(); }} > Delete all data in DB diff --git a/src/depvis-next/components/Workspace/ProjectVersionSelector.tsx b/src/depvis-next/components/Workspace/ProjectVersionSelector.tsx index 514cdb7..d48d2a4 100644 --- a/src/depvis-next/components/Workspace/ProjectVersionSelector.tsx +++ b/src/depvis-next/components/Workspace/ProjectVersionSelector.tsx @@ -4,6 +4,7 @@ import { useEffect, useState } from "react"; import { Container } from "react-bootstrap"; import { getProjectVersionsQuery } from "../../helpers/GraphHelper"; import Dropdown, { DropdownItem } from "../Dropdown/Dropdown"; +import NoProjectFoundError from "../Error/NoProjectFoundError"; type ProjectSelectorProps = { onProjectVersionSelect?: (projectVersionId: string) => void; @@ -40,6 +41,10 @@ const ProjectVersionSelector = (props: ProjectSelectorProps) => { displayName: `${pv.name} v${pv.version}`, }; }; + + if (!projectsLoading && projects.projectVersions.length == 0) { + return ; + } return ( {!projectsLoading && ( diff --git a/src/depvis-next/components/Workspace/Workspace.tsx b/src/depvis-next/components/Workspace/Workspace.tsx index 7c8bae8..e68c5c4 100644 --- a/src/depvis-next/components/Workspace/Workspace.tsx +++ b/src/depvis-next/components/Workspace/Workspace.tsx @@ -99,13 +99,17 @@ const Workspace = () => { const paintRing = useCallback( (currNode, ctx) => { - if (node) { - // add ring just for highlighted nodes - ctx.beginPath(); - ctx.arc(node.x, node.y, node.size | 1, 0, 2 * Math.PI, false); - ctx.fillStyle = currNode === node ? "red" : "orange"; - ctx.fill(); - } + ctx.beginPath(); + ctx.arc( + currNode.x, + currNode.y, + (Math.sqrt(currNode.size) * 4 + 1) | 1, + 0, + 2 * Math.PI, + false + ); + ctx.fillStyle = currNode === node ? "red" : ""; + ctx.fill(); }, [node] ); From f0c629dab1dcec9082db7f9eda8459b17cf85558 Mon Sep 17 00:00:00 2001 From: Matej Groman Date: Sat, 18 Mar 2023 14:42:58 +0100 Subject: [PATCH 29/33] FInalize import - progress bar --- .../components/Import/ImportResult.tsx | 40 ++++--- src/depvis-next/helpers/BatchHelper.ts | 2 +- src/depvis-next/helpers/DbDataProvider.ts | 4 +- src/depvis-next/helpers/ImportSbomHelper.ts | 102 +++++++++++++----- 4 files changed, 109 insertions(+), 39 deletions(-) diff --git a/src/depvis-next/components/Import/ImportResult.tsx b/src/depvis-next/components/Import/ImportResult.tsx index 9aeae00..9cca651 100644 --- a/src/depvis-next/components/Import/ImportResult.tsx +++ b/src/depvis-next/components/Import/ImportResult.tsx @@ -1,8 +1,7 @@ -import Link from 'next/link'; -import { useRouter } from 'next/router'; -import { useEffect, useState } from 'react'; -import { Alert, Container, ProgressBar, Row, Spinner } from 'react-bootstrap'; -import { ImportStatusReponse } from '../../pages/api/import/status'; +import { useRouter } from "next/router"; +import { useEffect, useState } from "react"; +import { Alert, Container, ProgressBar, Row, Spinner } from "react-bootstrap"; +import { ImportStatusReponse } from "../../pages/api/import/status"; const fetchInterval = 500; export const ImportResult = (props) => { @@ -26,21 +25,36 @@ export const ImportResult = (props) => { }, [response]); if (!response) return <>; - if (response.status == 'completed') { - const url = response.projectName ? `/?projectName=${response.projectName}` : '/'; + if (response.status == "completed") { + const url = response.projectName + ? `/?projectName=${response.projectName}` + : "/"; router.push(url); } + + const getPercent = () => { + if (!response.progress || !response.progress.percent) return 0; + return response.progress!.percent; + }; return ( -

Importing project {response.projectName}

-

{response.progress && response.progress.message}

-

This can take several minutes depending on the size of the input.

+

+ Importing project {response.projectName} +

+

+ {response.progress && response.progress.message} +

+

+ This can take several minutes depending on the size of the input. +

{response.continueQuery ? ( - + ) : ( - {response.status == 'completed' ? ( - Import completed! Redirecting to your graph... + {response.status == "completed" ? ( + + Import completed! Redirecting to your graph... + ) : ( {response.status} - {response.message} diff --git a/src/depvis-next/helpers/BatchHelper.ts b/src/depvis-next/helpers/BatchHelper.ts index 926e42a..2187aa7 100644 --- a/src/depvis-next/helpers/BatchHelper.ts +++ b/src/depvis-next/helpers/BatchHelper.ts @@ -23,7 +23,7 @@ export async function processBatchAsync( try { for (let i = 0; i < inputList.length; i += chunkSize) { if (options.updateProgressFn) { - options.updateProgressFn((i / inputList.length) * 100, options.message); + options.updateProgressFn(i / inputList.length); } const chunk = inputList.slice(i, i + chunkSize); const chunkRes = await fn(chunk, options.fnArg2); diff --git a/src/depvis-next/helpers/DbDataProvider.ts b/src/depvis-next/helpers/DbDataProvider.ts index 972f84f..fb3fe4f 100644 --- a/src/depvis-next/helpers/DbDataProvider.ts +++ b/src/depvis-next/helpers/DbDataProvider.ts @@ -198,7 +198,8 @@ export async function CreateComponents( export async function updateComponentDependency( dependencies: Dependency[], projectVersionId: string, - mainComponentPurl: string + mainComponentPurl: string, + progressUpdateFn: Function ) { if (!dependencies || dependencies.length == 0) { console.log("Updating dependencies - No dependencies provided!"); @@ -250,6 +251,7 @@ export async function updateComponentDependency( connect: dependencyQueryList[index].connect, }); dependencyCount += data.name.info.relationshipsCreated; + progressUpdateFn(index / dependencies.length); } console.log("Created %s relationships", dependencyCount); return dependencyCount; diff --git a/src/depvis-next/helpers/ImportSbomHelper.ts b/src/depvis-next/helpers/ImportSbomHelper.ts index 9ee7b8c..e4e68e4 100644 --- a/src/depvis-next/helpers/ImportSbomHelper.ts +++ b/src/depvis-next/helpers/ImportSbomHelper.ts @@ -31,25 +31,32 @@ export async function ImportSbom( projectVersion: string, updateProgressCallback ) { - /** - * Simple wrapper function that is responsible for updating status for job worker - * @param percent Status in percent (0-100) - * @param message Short description what is happening - */ - const updateProgress = async (percent, message) => { - await updateProgressCallback({ message: message, percent: percent }); - }; - console.log("Here the project version is: %s", projectVersion); const importInfo = { project: undefined, projectVersion: undefined, createdComponents: [], createdVulnerabilitiesIds: [], + currentPhase: ImportPhase.Init, + }; + + /** + * Simple wrapper function that is responsible for updating status for job worker + * @param percent Status in percent (0-100) + */ + const updateProgress = async (percent) => { + const obj = { + message: importInfo.currentPhase, + percent: CalculateProgress(importInfo.currentPhase, percent), + }; + console.log(obj); + await updateProgressCallback(obj); }; try { - // Find project information on backend - await updateProgress(0, "Creating new project version"); + // Get project & project version + // Both project and project version will be created if it does not exist yet + importInfo.currentPhase = ImportPhase.CreateProject; + await updateProgress(0); importInfo.project = await GetProject(projectInput); const tmpProjectVersion = projectVersion ? projectVersion @@ -64,46 +71,51 @@ export async function ImportSbom( importInfo.project, projectVersionInput ); - console.log(importInfo); - - // Create components for new version + updateProgress(5); - // Prepare dependencies + // Prepare necessary objects - if it fails, no objects in DB are created yet let dependencies = GetDependencies(bom.dependencies.dependency); - - // Create all objects in DB - // Prepare components const mainComponent: Component = createMainComponent( bom.metadata.component ); let components: Component[] = GetComponents(bom); components.push(mainComponent); + // Create components in DB + importInfo.currentPhase = ImportPhase.CreateComponents; + importInfo.createdComponents = await processBatchAsync( components, CreateComponents, { chunkSize: 5, updateProgressFn: updateProgress, - message: "Updating components", fnArg2: importInfo.projectVersion, } ); + + // Connect dependencies + importInfo.currentPhase = ImportPhase.ConnectDependencies; const dependenciesResult = await updateComponentDependency( dependencies, importInfo.projectVersion, - mainComponent.purl + mainComponent.purl, + updateProgress ); - await updateProgress(70, "Fetching vulnerabilities"); - //Vulnerabilities + + // Find vulnerabilities + importInfo.currentPhase = ImportPhase.FindVulnerabilities; const purlList = components.map((c) => { return c.purl; }); const r = await processBatchAsync(purlList, VulnFetcherHandler, { chunkSize: 10, + updateProgressFn: updateProgress, }); - await updateProgress(90, "Creating vulnerabilities in DB"); - r.forEach(async (component) => { + + // Create or connect vulnerabilities + r.forEach(async (component, index) => { + updateProgress(index / r.length); if (component.vulnerabilities.length > 0) { console.log( "Creating %d vulns for %s", @@ -116,9 +128,13 @@ export async function ImportSbom( ); } }); + importInfo.currentPhase = ImportPhase.Completed; } catch (error) { console.error("Recovery needed"); console.error(error); + if (importInfo.projectVersion) { + await DeleteProjectVersion(importInfo.projectVersion); + } } } function GetComponents(bom: any) { @@ -291,3 +307,41 @@ function createMainComponent(inputComponent) { publisher: inputComponent.publisher, }; } + +enum ImportPhase { + CreateProject = "Creating project version", + CreateComponents = "Creating components", + ConnectDependencies = "Connecting dependencies", + FindVulnerabilities = "Looking for vulnerabilities", + CreateVulnerabilities = "Adding vulnerabilities", + Completed = "Import is completed", + Init = "Starting import", +} + +const CalculateProgress = (importPhase: ImportPhase, phasePercent: number) => { + switch (importPhase) { + case ImportPhase.CreateProject: + return CalculateProgressFn(0, 5, phasePercent); + case ImportPhase.CreateComponents: + return CalculateProgressFn(5, 40, phasePercent); + case ImportPhase.ConnectDependencies: + return CalculateProgressFn(40, 60, phasePercent); + case ImportPhase.FindVulnerabilities: + return CalculateProgressFn(60, 85, phasePercent); + case ImportPhase.CreateVulnerabilities: + return CalculateProgressFn(85, 100, phasePercent); + default: + return 0; + } +}; + +const CalculateProgressFn = ( + offset: number, + maximum: number, + percent: number +): number => { + const multiplier = percent > 100 ? 100 : percent; + const result = offset + (maximum - offset) * multiplier; + if (result < 0) return 0; + return Number.parseInt(result.toFixed(0)); +}; From 228d49dd221d7dede55422f8cf4ab0c1222f47b8 Mon Sep 17 00:00:00 2001 From: Matej Groman Date: Sun, 19 Mar 2023 10:24:52 +0100 Subject: [PATCH 30/33] Colors improvement --- .../components/Workspace/Workspace.tsx | 3 +- src/depvis-next/helpers/GraphHelper.ts | 41 +++++++++++++------ src/depvis-next/types/colorPalette.ts | 8 ++++ 3 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 src/depvis-next/types/colorPalette.ts diff --git a/src/depvis-next/components/Workspace/Workspace.tsx b/src/depvis-next/components/Workspace/Workspace.tsx index e68c5c4..2252dcf 100644 --- a/src/depvis-next/components/Workspace/Workspace.tsx +++ b/src/depvis-next/components/Workspace/Workspace.tsx @@ -24,6 +24,7 @@ import Sidebar from "../Layout/Sidebar"; import ProjectSelector from "./ProjectSelector"; import { DropdownItem } from "../Dropdown/Dropdown"; import ProjectVersionSelector from "./ProjectVersionSelector"; +import { graphSelectedNode } from "../../types/colorPalette"; const defaultGraphConfig: GraphConfig = { zoomLevel: 1, @@ -108,7 +109,7 @@ const Workspace = () => { 2 * Math.PI, false ); - ctx.fillStyle = currNode === node ? "red" : ""; + ctx.fillStyle = currNode === node ? graphSelectedNode : ""; ctx.fill(); }, [node] diff --git a/src/depvis-next/helpers/GraphHelper.ts b/src/depvis-next/helpers/GraphHelper.ts index bf59229..a7cf9d2 100644 --- a/src/depvis-next/helpers/GraphHelper.ts +++ b/src/depvis-next/helpers/GraphHelper.ts @@ -1,4 +1,13 @@ import { gql } from "@apollo/client"; +import { + graphExcludedNode, + graphNode, + graphUIGrey, + vulnerabilityCriticalColor, + vulnerabilityHighColor, + vulnerabilityLowColor, + vulnerabilityMediumColor, +} from "../types/colorPalette"; /** * Function responsible for transforming the data to format that can be visualized @@ -143,21 +152,29 @@ export const getProjectVersionsQuery = gql` } } `; -const componentColor = "#005f73"; -const vulnColor = "#ee9b00"; -const otherColor = "#001219"; -const severeVulnColor = "#bb3e03"; -const systemComponent = "#0f0f0f"; +// const componentColor = "#005f73"; +// const vulnColor = "#ee9b00"; +// const otherColor = "#001219"; +// const severeVulnColor = "#bb3e03"; +// const systemComponent = "#0f0f0f"; +const vulnerabilityColorByCVSS = (cvssScore: number) => { + if (cvssScore >= 9) return vulnerabilityCriticalColor; + if (cvssScore >= 7) return vulnerabilityHighColor; + if (cvssScore >= 4) return vulnerabilityMediumColor; + return vulnerabilityLowColor; +}; + +const nodeExcludeRegex = new RegExp( + process.env.NEXT_PUBLIC_GRAPH_EXCLUDED_REGEX +); export const getNodeColor = (node) => { - if (!node) return otherColor; + if (!node) return graphUIGrey; if (node.__typename === "Vulnerability") - return node.cvssScore > 5 ? severeVulnColor : vulnColor; - if (node.selected) return "#6500ff"; - if (node.name && node.name.toLowerCase().includes("system")) - return systemComponent; - if (node.__typename === "Component") return componentColor; - return otherColor; + return vulnerabilityColorByCVSS(node.cvssScore); + if (node.name && nodeExcludeRegex.test(node.name)) return graphExcludedNode; + if (node.__typename === "Component") return graphNode; + return graphUIGrey; }; const getNodeTier = ( diff --git a/src/depvis-next/types/colorPalette.ts b/src/depvis-next/types/colorPalette.ts new file mode 100644 index 0000000..531b564 --- /dev/null +++ b/src/depvis-next/types/colorPalette.ts @@ -0,0 +1,8 @@ +export const vulnerabilityCriticalColor = "#931621"; +export const vulnerabilityHighColor = "#D45113"; +export const vulnerabilityMediumColor = "#FB8500"; +export const vulnerabilityLowColor = "#FFB703"; +export const graphSelectedNode = "#00ABE7"; +export const graphNode = "#054A91"; +export const graphExcludedNode = "#81A4CD"; +export const graphUIGrey = "#423E3B"; From a36f5650020a383f8f6308b20d75d1066d61234c Mon Sep 17 00:00:00 2001 From: Matej Groman Date: Sun, 19 Mar 2023 10:51:25 +0100 Subject: [PATCH 31/33] Vulnerability detail improvement --- .../Details/VulnerabilityDetails.tsx | 65 +++++++++++++++---- src/depvis-next/helpers/GraphHelper.ts | 2 +- 2 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/depvis-next/components/Details/VulnerabilityDetails.tsx b/src/depvis-next/components/Details/VulnerabilityDetails.tsx index 2b2f6dd..1cd643b 100644 --- a/src/depvis-next/components/Details/VulnerabilityDetails.tsx +++ b/src/depvis-next/components/Details/VulnerabilityDetails.tsx @@ -1,7 +1,10 @@ -import { gql, useQuery } from '@apollo/client'; -import { Badge, Container } from 'react-bootstrap'; -import Loading from '../Loading/Loading'; -import { DL, DLItem } from './DescriptionList'; +import { gql, useQuery } from "@apollo/client"; +import Link from "next/link"; +import { Badge, Container } from "react-bootstrap"; +import urlJoin from "url-join"; +import { vulnerabilityColorByCVSS } from "../../helpers/GraphHelper"; +import Loading from "../Loading/Loading"; +import { DL, DLItem } from "./DescriptionList"; const getVulnerabilityDetailsQuery = gql` query vulnerabilityDetails($vulnerabilityId: String) { @@ -38,31 +41,71 @@ const VulnerabilityDetails = (props) => { )); }; + const getCvssScoreLabel = (cvssScore) => { + if (cvssScore >= 9) return "Critical"; + if (cvssScore >= 7) return "High"; + if (cvssScore >= 4) return "Medium"; + return "Low"; + }; const renderCvss = (cvssScore) => { - return 5 ? 'danger' : 'warning'}>{cvssScore}; + return ( + + {cvssScore} - {getCvssScoreLabel(cvssScore)} + + ); + }; + + const renderCvssVectorAsLink = (cvssVector) => { + const cvssPortalUrl = "https://www.first.org/cvss/calculator/3.1"; + const url = `${cvssPortalUrl}#${cvssVector}`; + return ( + + {cvssVector} + + ); }; if (loading) return ; if (!data.vulnerabilities[0]) { - console.error('No data found when querying backend! Below is Apollo query result'); + console.error( + "No data found when querying backend! Below is Apollo query result" + ); console.error({ data: data, error: error }); return No data found!; } const vulnerability = data.vulnerabilities[0]; return ( - +

{vulnerability.name}

- - + + - + - +
); diff --git a/src/depvis-next/helpers/GraphHelper.ts b/src/depvis-next/helpers/GraphHelper.ts index a7cf9d2..2ed5bcf 100644 --- a/src/depvis-next/helpers/GraphHelper.ts +++ b/src/depvis-next/helpers/GraphHelper.ts @@ -158,7 +158,7 @@ export const getProjectVersionsQuery = gql` // const severeVulnColor = "#bb3e03"; // const systemComponent = "#0f0f0f"; -const vulnerabilityColorByCVSS = (cvssScore: number) => { +export const vulnerabilityColorByCVSS = (cvssScore: number) => { if (cvssScore >= 9) return vulnerabilityCriticalColor; if (cvssScore >= 7) return vulnerabilityHighColor; if (cvssScore >= 4) return vulnerabilityMediumColor; From 1d9543c65f5ad3caa43d21a0209462a33e63da04 Mon Sep 17 00:00:00 2001 From: Matej Groman Date: Sun, 19 Mar 2023 10:54:50 +0100 Subject: [PATCH 32/33] Link Improvement --- src/depvis-next/components/Details/VulnerabilityDetails.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/depvis-next/components/Details/VulnerabilityDetails.tsx b/src/depvis-next/components/Details/VulnerabilityDetails.tsx index 1cd643b..6ec8823 100644 --- a/src/depvis-next/components/Details/VulnerabilityDetails.tsx +++ b/src/depvis-next/components/Details/VulnerabilityDetails.tsx @@ -64,7 +64,7 @@ const VulnerabilityDetails = (props) => { const cvssPortalUrl = "https://www.first.org/cvss/calculator/3.1"; const url = `${cvssPortalUrl}#${cvssVector}`; return ( - + {cvssVector} ); From c2155620e7aa8a378020b5faf81ee20a9c7b8b1a Mon Sep 17 00:00:00 2001 From: Matej Groman Date: Sun, 19 Mar 2023 12:26:39 +0100 Subject: [PATCH 33/33] Different endpoint for FE /BE --- src/depvis-next/helpers/ApolloClientHelper.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/depvis-next/helpers/ApolloClientHelper.ts b/src/depvis-next/helpers/ApolloClientHelper.ts index 73c063d..abf67ec 100644 --- a/src/depvis-next/helpers/ApolloClientHelper.ts +++ b/src/depvis-next/helpers/ApolloClientHelper.ts @@ -7,14 +7,14 @@ import { gqlUrlPath } from "../pages/api/graphql"; * @returns ApolloClient object */ export const createApolloClient = () => { - if (!process.env.NEXT_PUBLIC_SERVER_URI) { + const serverUri = !(typeof window === "object") //decide if using browser or console + ? process.env.BACKEND_SERVER_URI + : process.env.NEXT_PUBLIC_SERVER_URI; + if (!serverUri) { console.error("No server URI was provided, using default connection"); } - const uri = urlJoin( - process.env.NEXT_PUBLIC_SERVER_URI || "http://localhost:3000", - gqlUrlPath - ); - // console.log(`Creating GQL Client (connection to ${uri})`); + const uri = urlJoin(serverUri || "http://localhost:3000", gqlUrlPath); + console.log(`Creating GQL Client (connection to ${uri})`); const link = new HttpLink({ uri: uri, fetchOptions: {