Merge pull request #64 from RHEcosystemAppEng/match-manifest-versions

feat: add functionality of matching manifest versions for python' pip and golang'  go modules.

 Jira Ticket: https://issues.redhat.com/browse/APPENG-2062

Author: zvigrinberg

src/providers/golang_gomodules.js

@@ -3,7 +3,7 @@ import { execSync } from "node:child_process"
 import fs from 'node:fs'
 import os from "node:os";
 import {EOL} from "os";
-import { getCustomPath } from "../tools.js";
+import {getCustom, getCustomPath} from "../tools.js";
 import path from 'node:path'
 import Sbom from '../sbom.js'
 import {PackageURL} from 'packageurl-js'
@@ -174,6 +174,93 @@ function enforceRemovingIgnoredDepsInCaseOfAutomaticVersionUpdate(ignoredDeps, s
 	})
 }
 
+/**
+ *
+ * @param {[string]} lines - array of lines of go.mod manifest
+ * @param {string} goMod - content of go.mod manifest
+ * @return {[string]} all dependencies from go.mod file as array
+ */
+function collectAllDepsFromManifest(lines, goMod) {
+	let result
+	// collect all deps that starts with require keyword
+
+	result = lines.filter((line) => line.trim().startsWith("require") && !line.includes("(")).map((dep) => dep.substring("require".length).trim())
+
+
+
+	// collect all deps that are inside `require` blocks
+	let currentSegmentOfGoMod = goMod
+	let requirePositionObject = decideRequireBlockIndex(currentSegmentOfGoMod)
+	while(requirePositionObject.index > -1) {
+		let depsInsideRequirementsBlock = currentSegmentOfGoMod.substring(requirePositionObject.index + requirePositionObject.startingOffeset).trim();
+		let endOfBlockIndex = depsInsideRequirementsBlock.indexOf(")")
+		let currentIndex = 0
+		while(currentIndex < endOfBlockIndex)
+		{
+			let endOfLinePosition = depsInsideRequirementsBlock.indexOf(EOL, currentIndex);
+			let dependency = depsInsideRequirementsBlock.substring(currentIndex, endOfLinePosition)
+			result.push(dependency.trim())
+			currentIndex = endOfLinePosition + 1
+		}
+		currentSegmentOfGoMod = currentSegmentOfGoMod.substring(endOfBlockIndex + 1).trim()
+		requirePositionObject = decideRequireBlockIndex(currentSegmentOfGoMod)
+	}
+
+	function decideRequireBlockIndex(goMod) {
+		let object = {}
+		let index = goMod.indexOf("require(")
+		object.startingOffeset = "require(".length
+		if (index === -1)
+		{
+			index = goMod.indexOf("require (")
+			object.startingOffeset = "require (".length
+			if(index === -1)
+			{
+				index = goMod.indexOf("require  (")
+				object.startingOffeset = "require  (".length
+			}
+		}
+		object.index = index
+		return object
+	}
+	return result
+}
+
+/**
+ *
+ * @param {string} rootElementName the rootElementName element of go mod graph, to compare only direct deps from go mod graph against go.mod manifest
+ * @param{[string]} goModGraphOutputRows the goModGraphOutputRows from go mod graph' output
+ * @param {string }manifest path to go.mod manifest on file system
+ * @private
+ */
+function performManifestVersionsCheck(rootElementName, goModGraphOutputRows, manifest) {
+	let goMod = fs.readFileSync(manifest).toString().trim()
+	let lines = goMod.split(EOL);
+	let comparisonLines = goModGraphOutputRows.filter((line)=> line.startsWith(rootElementName)).map((line)=> getChildVertexFromEdge(line))
+	let manifestDeps = collectAllDepsFromManifest(lines,goMod)
+	try {
+		comparisonLines.forEach((dependency) => {
+			let parts = dependency.split("@")
+			let version = parts[1]
+			let depName = parts[0]
+			manifestDeps.forEach(dep => {
+				let components = dep.trim().split(" ");
+				let currentDepName = components[0]
+				let currentVersion = components[1]
+				if (currentDepName === depName) {
+					if (currentVersion !== version) {
+						throw new Error(`versions mismatch for dependency name ${depName}, manifest version=${currentVersion}, installed Version=${version}, if you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting - MATCH_MANIFEST_VERSIONS=false`)
+					}
+				}
+			})
+		})
+	}
+	catch(error) {
+		console.error("Can't continue with analysis")
+		throw error
+	}
+}
+
 /**
  * Create SBOM json string for go Module.
  * @param {string} manifest - path for go.mod
@@ -201,6 +288,12 @@ function getSBOM(manifest, opts = {}, includeTransitive) {
 	let sbom = new Sbom();
 	let rows = goGraphOutput.split(EOL);
 	let root = getParentVertexFromEdge(rows[0])
+	let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS","false");
+	if(matchManifestVersions === "true") {
+		{
+			performManifestVersionsCheck(root, rows, manifest)
+		}
+	}
 	let mainModule = toPurl(root, "@", undefined)
 	sbom.addRoot(mainModule)
 

src/providers/python_controller.js

@@ -2,6 +2,7 @@ import {execSync} from "node:child_process";
 import fs from "node:fs";
 import path from 'node:path';
 import {EOL} from "os";
+import {getCustom} from "../tools.js";
 
 
 /** @typedef {{name: string, version: string, dependencies: DependencyEntry[]}} DependencyEntry */
@@ -126,6 +127,7 @@ export default class Python_controller {
 			}
 		}).toString();
 		let allPipShowDeps = pipShowOutput.split( EOL +"---" + EOL);
+		let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS","true");
 		let linesOfRequirements = fs.readFileSync(this.pathToRequirements).toString().split(EOL).filter( (line) => !line.startsWith("#")).map(line => line.trim())
 		let CachedEnvironmentDeps = {}
 		allPipShowDeps.forEach( (record) => {
@@ -135,6 +137,31 @@ export default class Python_controller {
 			CachedEnvironmentDeps[dependencyName.replace("_","-")] = record
 		})
 		linesOfRequirements.forEach( (dep) => {
+			// if matchManifestVersions setting is turned on , then
+			if(matchManifestVersions === "true")
+			{
+				let dependencyName
+				let manifestVersion
+				let installedVersion
+				let doubleEqualSignPosition
+				if(dep.includes("=="))
+				{
+					doubleEqualSignPosition = dep.indexOf("==")
+					manifestVersion = dep.substring(doubleEqualSignPosition + 2).trim()
+					if(manifestVersion.includes("#"))
+					{
+						let hashCharIndex = manifestVersion.indexOf("#");
+						manifestVersion = manifestVersion.substring(0,hashCharIndex)
+					}
+					dependencyName = getDependencyName(dep)
+					installedVersion = getDependencyVersion(CachedEnvironmentDeps[dependencyName.toLowerCase()])
+					if(manifestVersion.trim() !== installedVersion.trim())
+					{
+						throw new Error(`Can't continue with analysis - versions mismatch for dependency name ${dependencyName}, manifest version=${manifestVersion}, installed Version=${installedVersion}, if you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting - MATCH_MANIFEST_VERSIONS=false`)
+					}
+
+				}
+			}
 			bringAllDependencies(dependencies,getDependencyName(dep),CachedEnvironmentDeps,includeTransitive)
 		})
 		dependencies.sort((dep1,dep2) =>{

test/providers/golang_gomodules.test.js

@@ -29,7 +29,6 @@ suite('testing the golang-go-modules data provider', () => {
 			let expectedSbom = fs.readFileSync(`test/providers/tst_manifests/golang/${testCase}/expected_sbom_stack_analysis.json`,).toString()
 			expectedSbom = JSON.stringify(JSON.parse(expectedSbom))
 			// invoke sut stack analysis for scenario manifest
-
 			let providedDataForStack = await golangGoModules.provideStack(`test/providers/tst_manifests/golang/${testCase}/go.mod`)
 			// new(year: number, month: number, date?: number, hours?: number, minutes?: number, seconds?: number, ms?: number): Date
 

test/providers/python_pip.test.js

@@ -67,7 +67,7 @@ suite('testing the python-pip data provider', () => {
 
 }).beforeAll(() => clock = sinon.useFakeTimers(new Date('2023-10-01T00:00:00.000Z'))).afterAll(()=> clock.restore());
 
-suite('testing the python-pip data provider', () => {
+suite('testing the python-pip data provider with virtual environment', () => {
 	[
 		"pip_requirements_virtual_env_txt_no_ignore",
 		"pip_requirements_virtual_env_with_ignore"
@@ -93,7 +93,7 @@ suite('testing the python-pip data provider', () => {
 			// 	content: expectedSbom
 			// })
 			// these test cases takes ~2500-2700 ms each pr >10000 in CI (for the first test-case)
-		}).timeout(process.env.GITHUB_ACTIONS ? 30000 : 15000)
+		}).timeout(process.env.GITHUB_ACTIONS ? 60000 : 30000)
 
 
 	})