Fixed jf ca for npm while cvs enabled

build/testdata/npm/projectCvs/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "cvs-test-project",
+  "version": "1.0.0",
+  "description": "Test project for CVS (Curation Version Substitution)",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "lightweight": "^0.1.0",
+    "lodash": "999.999.999"
+  }
+}
+

build/utils/npm.go

@@ -8,9 +8,11 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/jfrog/gofrog/crypto"
+	"github.com/jfrog/gofrog/log"
 
 	"golang.org/x/exp/slices"
 
@@ -22,6 +24,11 @@ import (
 
 const npmInstallCommand = "install"
 
+// TransitiveDepMarker is appended to version for transitive blocked packages
+// Uses ":" prefix so it becomes an extra segment that's ignored in parsing but detectable
+// e.g., "5.1.6:TRANSITIVE" -> splits to ["name", "5.1.6", "TRANSITIVE"]
+const TransitiveDepMarker = ":TRANSITIVE"
+
 // CalculateNpmDependenciesList gets an npm project's dependencies.
 func CalculateNpmDependenciesList(executablePath, srcPath, moduleId string, npmParams NpmTreeDepListParam, calculateChecksums bool, log utils.Log) ([]entities.Dependency, error) {
 	if log == nil {
@@ -91,6 +98,15 @@ type dependencyInfo struct {
 	*npmLsDependency
 }
 
+type DependencyInfo = dependencyInfo
+
+// NotFoundPackage represents a package not found during npm install (404/ETARGET).
+type NotFoundPackage struct {
+	Name         string
+	Version      string
+	IsTransitive bool // True if this is a transitive dependency
+}
+
 // Run 'npm list ...' command and parse the returned result to create a dependencies map of.
 // The dependencies map looks like name:version -> entities.Dependency.
 func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmListParams NpmTreeDepListParam, log utils.Log, skipInstall bool) (map[string]*dependencyInfo, error) {
@@ -125,6 +141,176 @@ func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmListP
 	})
 }
 
+// CalculateDependenciesMapWithCvs calculates dependencies.
+// It retries npm install when packages are blocked, collecting blocked packages for audit.
+// this code is used by 'jf ca'.
+func CalculateDependenciesMapWithCvs(executablePath, srcPath, moduleId string, npmListParams NpmTreeDepListParam, log utils.Log, skipInstall bool) (map[string]*DependencyInfo, []NotFoundPackage, error) {
+	dependenciesMap := make(map[string]*DependencyInfo)
+	npmVersion, err := GetNpmVersion(executablePath, log)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	data, blockedPackages, err := runNpmLsWithCurationSupport(executablePath, srcPath, npmListParams, log, npmVersion)
+	if err != nil {
+		return nil, blockedPackages, err
+	}
+
+	parseFunc := parseNpmLsDependencyFunc(npmVersion)
+	err = jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) {
+		if string(key) == "dependencies" {
+			err = parseDependencies(value, []string{moduleId}, dependenciesMap, parseFunc, log)
+		}
+		return err
+	})
+
+	return dependenciesMap, blockedPackages, err
+}
+
+func runNpmLsWithCurationSupport(executablePath, srcPath string, npmListParams NpmTreeDepListParam, log utils.Log, npmVersion *version.Version) ([]byte, []NotFoundPackage, error) {
+	tempDir, err := utils.CreateTempDir()
+	if err != nil {
+		return nil, nil, err
+	}
+	defer func() {
+		_ = utils.RemoveTempDir(tempDir)
+	}()
+
+	if err := utils.CopyDir(srcPath, tempDir, true, []string{"node_modules"}); err != nil {
+		return nil, nil, err
+	}
+	blockedPackages, err := runNpmInstallWithRetry(executablePath, tempDir, npmListParams.InstallCommandArgs, npmListParams.Args, log, npmVersion)
+	if err != nil {
+		return nil, blockedPackages, err
+	}
+
+	// Check if package-lock.json exists before running npm ls --package-lock-only
+	packageLockPath := filepath.Join(tempDir, "package-lock.json")
+	packageLockExists, _ := utils.IsFileExists(packageLockPath, false)
+	if !packageLockExists {
+		log.Warn("No package-lock.json found. Cannot run npm ls --package-lock-only.")
+		// Return empty JSON with just the blocked packages
+		data := []byte("{}")
+		if len(blockedPackages) > 0 {
+			data = addNotFoundPackagesToNpmLsOutput(data, blockedPackages)
+		}
+		return data, blockedPackages, nil
+	}
+
+	npmListParams.Args = append(npmListParams.Args, "--json", "--all", "--long", "--package-lock-only")
+	data, errData, err := RunNpmCmd(executablePath, tempDir, AppendNpmCommand(npmListParams.Args, "ls"), log)
+	if err != nil {
+		log.Warn(err.Error())
+	} else if len(errData) > 0 {
+		log.Warn("Encountered some issues while running 'npm ls' command:\n" + strings.TrimSpace(string(errData)))
+	}
+
+	// Add blocked packages to the dependency tree for auditing
+	if len(blockedPackages) > 0 {
+		data = addNotFoundPackagesToNpmLsOutput(data, blockedPackages)
+	}
+
+	return data, blockedPackages, nil
+}
+
+func addNotFoundPackagesToNpmLsOutput(data []byte, blockedPackages []NotFoundPackage) []byte {
+	var npmLsOutput map[string]interface{}
+	if err := json.Unmarshal(data, &npmLsOutput); err != nil {
+		log.Debug("Failed to add blocked packages to npm ls output: unable to parse JSON")
+		return data
+	}
+	deps, ok := npmLsOutput["dependencies"].(map[string]interface{})
+	if !ok {
+		deps = make(map[string]interface{})
+		npmLsOutput["dependencies"] = deps
+	}
+	for _, bp := range blockedPackages {
+		version := bp.Version
+		if bp.IsTransitive {
+			version = bp.Version + TransitiveDepMarker
+		}
+		deps[bp.Name] = map[string]interface{}{
+			"version": version,
+			"missing": true,
+		}
+	}
+	result, err := json.Marshal(npmLsOutput)
+	if err != nil {
+		log.Debug("Failed to add blocked packages to npm ls output: unable to marshal JSON")
+		return data
+	}
+	return result
+}
+
+// runNpmInstallWithRetry runs npm install in the given directory (should be temp dir).
+// If a package is blocked (404), it removes it from package.json and retries.
+func runNpmInstallWithRetry(executablePath, workDir string, npmInstallCommandArgs, npmArgs []string, log utils.Log, npmVersion *version.Version) ([]NotFoundPackage, error) {
+	pkgJsonPath := filepath.Join(workDir, "package.json")
+	var blocked []NotFoundPackage
+	var lastPkg string
+
+	for {
+		err := installPackageLock(executablePath, workDir, npmInstallCommandArgs, npmArgs, log, npmVersion)
+		if err == nil {
+			return blocked, nil
+		}
+
+		name, version, found := ParseNpmNotFoundError(err)
+		if !found {
+			return blocked, err
+		}
+
+		pkgId := name + "@" + version
+		if pkgId == lastPkg {
+			// Package couldn't be removed (transitive dependency)
+			log.Debug(fmt.Sprintf("Package %s is not found (transitive dependency). Adding to blocked packages and continuing...", pkgId))
+			blocked = append(blocked, NotFoundPackage{Name: name, Version: version, IsTransitive: true})
+			return blocked, nil
+		}
+		lastPkg = pkgId
+
+		log.Info(fmt.Sprintf("Retrying without package %s...", pkgId))
+		blocked = append(blocked, NotFoundPackage{Name: name, Version: version, IsTransitive: false})
+		if err := removePackageFromPackageJson(pkgJsonPath, name); err != nil {
+			return blocked, err
+		}
+	}
+}
+
+var npmNotFoundPattern = regexp.MustCompile(`No matching version found for\s+(@?[\w./-]+)@([\d][\w._-]*)`)
+
+func ParseNpmNotFoundError(err error) (name, version string, found bool) {
+	if err == nil {
+		return "", "", false
+	}
+	if matches := npmNotFoundPattern.FindStringSubmatch(err.Error()); len(matches) >= 3 {
+		return matches[1], strings.TrimSuffix(matches[2], "."), true
+	}
+	return "", "", false
+}
+
+func removePackageFromPackageJson(packageJsonPath, packageName string) error {
+	data, err := os.ReadFile(packageJsonPath)
+	if err != nil {
+		return err
+	}
+	var pkg map[string]interface{}
+	if err := json.Unmarshal(data, &pkg); err != nil {
+		return err
+	}
+	depTypes := []string{"dependencies", "devDependencies", "peerDependencies", "optionalDependencies"}
+	for _, depType := range depTypes {
+		if deps, ok := pkg[depType].(map[string]interface{}); ok {
+			delete(deps, packageName)
+		}
+	}
+	newData, err := json.MarshalIndent(pkg, "", "  ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(packageJsonPath, newData, 0644)
+}
+
 func runNpmLsWithNodeModules(executablePath, srcPath string, npmArgs []string, log utils.Log) (data []byte) {
 	npmArgs = append(npmArgs, "--json", "--all", "--long")
 	data, errData, err := RunNpmCmd(executablePath, srcPath, AppendNpmCommand(npmArgs, "ls"), log)

build/utils/npm_test.go

@@ -271,6 +271,34 @@ func TestCalculateDependenciesMapWithProhibitedInstallation(t *testing.T) {
 	assert.True(t, errors.As(err, &installForbiddenErr))
 }
 
+// TestCalculateDependenciesMapWithCvs tests the full CVS flow with a real npm project.
+func TestCalculateDependenciesMapWithCvs(t *testing.T) {
+	npmVersion, _, err := GetNpmVersionAndExecPath(logger)
+	require.NoError(t, err)
+	if !npmVersion.AtLeast("7.0.0") {
+		t.Skip("Running on npm v7 and above only, skipping...")
+	}
+
+	path, cleanup := tests.CreateTestProject(t, filepath.Join("..", "testdata", "npm", "projectCvs"))
+	defer cleanup()
+
+	dependencies, blockedPackages, err := CalculateDependenciesMapWithCvs(
+		"npm", path, "cvs-test-project",
+		NpmTreeDepListParam{Args: []string{"--cache=" + filepath.Join(path, "tmpcache")}, IgnoreNodeModules: true, OverwritePackageLock: true},
+		logger, false,
+	)
+	assert.NoError(t, err)
+	assert.Len(t, blockedPackages, 1)
+	assert.Equal(t, "lodash", blockedPackages[0].Name)
+	assert.Equal(t, "999.999.999", blockedPackages[0].Version)
+
+	_, hasLightweight := dependencies["lightweight:0.1.0"]
+	assert.True(t, hasLightweight, "package 'lightweight' should be in dependencies")
+
+	_, hasBlocked := dependencies["lodash:999.999.999"]
+	assert.True(t, hasBlocked, "Blocked package should be added to dependencies for audit")
+}
+
 func getExpectedRespForTestDependencyPackageLockOnly() map[string]*dependencyInfo {
 	return map[string]*dependencyInfo{
 		"underscore:1.13.6": {
@@ -484,6 +512,51 @@ func TestFilterUniqueArgs(t *testing.T) {
 	}
 }
 
+// TestParseNpmNotFoundError tests parsing "No matching version found for pkg@version" error messages.
+func TestParseNpmNotFoundError(t *testing.T) {
+	name, version, found := ParseNpmNotFoundError(errors.New("No matching version found for lodash@4.17.21"))
+	assert.True(t, found)
+	assert.Equal(t, "lodash", name)
+	assert.Equal(t, "4.17.21", version)
+
+	name, version, found = ParseNpmNotFoundError(errors.New("No matching version found for @angular/core@15.0.0"))
+	assert.True(t, found)
+	assert.Equal(t, "@angular/core", name)
+	assert.Equal(t, "15.0.0", version)
+
+	_, _, found = ParseNpmNotFoundError(errors.New("npm ERR! code ENOTFOUND"))
+	assert.False(t, found)
+
+	_, _, found = ParseNpmNotFoundError(nil)
+	assert.False(t, found)
+}
+
+// TestAddNotFoundPackagesToNpmLsOutput tests adding blocked packages to npm ls JSON output.
+func TestAddNotFoundPackagesToNpmLsOutput(t *testing.T) {
+	npmLsOutput := `{"name": "test-project", "dependencies": {"express": {"version": "4.18.0"}}}`
+	blockedPackages := []NotFoundPackage{{Name: "lodash", Version: "4.17.21"}}
+
+	result := addNotFoundPackagesToNpmLsOutput([]byte(npmLsOutput), blockedPackages)
+
+	var output map[string]interface{}
+	err := json.Unmarshal(result, &output)
+	assert.NoError(t, err)
+
+	deps, ok := output["dependencies"].(map[string]interface{})
+	assert.True(t, ok, "dependencies should be a map")
+
+	express, ok := deps["express"].(map[string]interface{})
+	assert.True(t, ok, "express should be a map")
+	assert.Equal(t, "4.18.0", express["version"])
+
+	lodash, ok := deps["lodash"].(map[string]interface{})
+	assert.True(t, ok, "lodash should be a map")
+	assert.Equal(t, "4.17.21", lodash["version"])
+	assert.Equal(t, true, lodash["missing"])
+
+	assert.Equal(t, 2, len(deps))
+}
+
 func TestParseDependenciesEdgeCases(t *testing.T) {
 	testcases := []struct {
 		name                string