From ce6948be669a642884ba55623abfde3a04940e5a Mon Sep 17 00:00:00 2001 From: Bassel Mbariky Date: Thu, 22 Jan 2026 12:07:12 +0200 Subject: [PATCH 1/5] Fixed jf ca for npm while cvs enabled --- build/testdata/npm/projectCvs/package.json | 16 +++ build/utils/npm.go | 150 +++++++++++++++++++++ build/utils/npm_test.go | 70 ++++++++++ 3 files changed, 236 insertions(+) create mode 100644 build/testdata/npm/projectCvs/package.json diff --git a/build/testdata/npm/projectCvs/package.json b/build/testdata/npm/projectCvs/package.json new file mode 100644 index 00000000..db1fab97 --- /dev/null +++ b/build/testdata/npm/projectCvs/package.json @@ -0,0 +1,16 @@ +{ + "name": "cvs-test-project", + "version": "1.0.0", + "description": "Test project for CVS (Curation Version Substitution)", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "lightweight": "^0.1.0", + "lodash": "999.999.999" + } +} + diff --git a/build/utils/npm.go b/build/utils/npm.go index 5a7a524d..60e89733 100644 --- a/build/utils/npm.go +++ b/build/utils/npm.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "github.com/jfrog/gofrog/crypto" @@ -91,6 +92,14 @@ type dependencyInfo struct { *npmLsDependency } +type DependencyInfo = dependencyInfo + +// NotFoundPackage represents a package not found during npm install (404/ETARGET). +type NotFoundPackage struct { + Name string + Version string +} + // Run 'npm list ...' command and parse the returned result to create a dependencies map of. // The dependencies map looks like name:version -> entities.Dependency. func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmListParams NpmTreeDepListParam, log utils.Log, skipInstall bool) (map[string]*dependencyInfo, error) { @@ -125,6 +134,147 @@ func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmListP }) } +// CalculateDependenciesMapWithCvs calculates dependencies. +// It retries npm install when packages are blocked, collecting blocked packages for audit. +// this code is used by 'jf ca'. +func CalculateDependenciesMapWithCvs(executablePath, srcPath, moduleId string, npmListParams NpmTreeDepListParam, log utils.Log, skipInstall bool) (map[string]*DependencyInfo, []NotFoundPackage, error) { + dependenciesMap := make(map[string]*DependencyInfo) + npmVersion, err := GetNpmVersion(executablePath, log) + if err != nil { + return nil, nil, err + } + + data, blockedPackages, err := runNpmLsWithCurationSupport(executablePath, srcPath, npmListParams, log, npmVersion) + if err != nil { + return nil, blockedPackages, err + } + + parseFunc := parseNpmLsDependencyFunc(npmVersion) + err = jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) { + if string(key) == "dependencies" { + err = parseDependencies(value, []string{moduleId}, dependenciesMap, parseFunc, log) + } + return err + }) + + return dependenciesMap, blockedPackages, err +} + +func runNpmLsWithCurationSupport(executablePath, srcPath string, npmListParams NpmTreeDepListParam, log utils.Log, npmVersion *version.Version) ([]byte, []NotFoundPackage, error) { + tempDir, err := utils.CreateTempDir() + if err != nil { + return nil, nil, err + } + defer utils.RemoveTempDir(tempDir) + + if err := utils.CopyDir(srcPath, tempDir, true, []string{"node_modules"}); err != nil { + return nil, nil, err + } + blockedPackages, err := runNpmInstallWithRetry(executablePath, tempDir, npmListParams.InstallCommandArgs, npmListParams.Args, log, npmVersion) + if err != nil { + return nil, blockedPackages, err + } + + npmListParams.Args = append(npmListParams.Args, "--json", "--all", "--long", "--package-lock-only") + data, errData, err := RunNpmCmd(executablePath, tempDir, AppendNpmCommand(npmListParams.Args, "ls"), log) + if err != nil { + log.Warn(err.Error()) + } else if len(errData) > 0 { + log.Warn("Encountered some issues while running 'npm ls' command:\n" + strings.TrimSpace(string(errData))) + } + + // Add blocked packages to the dependency tree for auditing + if len(blockedPackages) > 0 { + data = addNotFoundPackagesToNpmLsOutput(data, blockedPackages) + } + + return data, blockedPackages, nil +} + +func addNotFoundPackagesToNpmLsOutput(data []byte, blockedPackages []NotFoundPackage) []byte { + var npmLsOutput map[string]interface{} + if err := json.Unmarshal(data, &npmLsOutput); err != nil { + return data + } + deps, ok := npmLsOutput["dependencies"].(map[string]interface{}) + if !ok { + deps = make(map[string]interface{}) + npmLsOutput["dependencies"] = deps + } + for _, bp := range blockedPackages { + deps[bp.Name] = map[string]interface{}{ + "version": bp.Version, + "missing": true, + } + } + result, _ := json.Marshal(npmLsOutput) + return result +} + +// runNpmInstallWithRetry runs npm install in the given directory (should be temp dir). +// If a package is blocked (404), it removes it from package.json and retries. +func runNpmInstallWithRetry(executablePath, workDir string, npmInstallCommandArgs, npmArgs []string, log utils.Log, npmVersion *version.Version) ([]NotFoundPackage, error) { + pkgJsonPath := filepath.Join(workDir, "package.json") + var blocked []NotFoundPackage + var lastPkg string + + for { + err := installPackageLock(executablePath, workDir, npmInstallCommandArgs, npmArgs, log, npmVersion) + if err == nil { + return blocked, nil + } + + name, version, found := ParseNpmNotFoundError(err) + if !found { + return blocked, err + } + + pkgId := name + "@" + version + if pkgId == lastPkg { + return blocked, fmt.Errorf("package %s is not found", pkgId) + } + lastPkg = pkgId + + log.Info(fmt.Sprintf("Retrying without package %s...", pkgId)) + blocked = append(blocked, NotFoundPackage{Name: name, Version: version}) + removePackageFromPackageJson(pkgJsonPath, name) + } +} + +var npmNotFoundPattern = regexp.MustCompile(`No matching version found for\s+(@?[\w./-]+)@([\d][\w._-]*)`) + +func ParseNpmNotFoundError(err error) (name, version string, found bool) { + if err == nil { + return "", "", false + } + if matches := npmNotFoundPattern.FindStringSubmatch(err.Error()); len(matches) >= 3 { + return matches[1], strings.TrimSuffix(matches[2], "."), true + } + return "", "", false +} + +func removePackageFromPackageJson(packageJsonPath, packageName string) error { + data, err := os.ReadFile(packageJsonPath) + if err != nil { + return err + } + var pkg map[string]interface{} + if err := json.Unmarshal(data, &pkg); err != nil { + return err + } + depTypes := []string{"dependencies", "devDependencies", "peerDependencies", "optionalDependencies"} + for _, depType := range depTypes { + if deps, ok := pkg[depType].(map[string]interface{}); ok { + delete(deps, packageName) + } + } + newData, err := json.MarshalIndent(pkg, "", " ") + if err != nil { + return err + } + return os.WriteFile(packageJsonPath, newData, 0644) +} + func runNpmLsWithNodeModules(executablePath, srcPath string, npmArgs []string, log utils.Log) (data []byte) { npmArgs = append(npmArgs, "--json", "--all", "--long") data, errData, err := RunNpmCmd(executablePath, srcPath, AppendNpmCommand(npmArgs, "ls"), log) diff --git a/build/utils/npm_test.go b/build/utils/npm_test.go index 4f0c3d42..ed179f53 100644 --- a/build/utils/npm_test.go +++ b/build/utils/npm_test.go @@ -271,6 +271,34 @@ func TestCalculateDependenciesMapWithProhibitedInstallation(t *testing.T) { assert.True(t, errors.As(err, &installForbiddenErr)) } +// TestCalculateDependenciesMapWithCvs tests the full CVS flow with a real npm project. +func TestCalculateDependenciesMapWithCvs(t *testing.T) { + npmVersion, _, err := GetNpmVersionAndExecPath(logger) + require.NoError(t, err) + if !npmVersion.AtLeast("7.0.0") { + t.Skip("Running on npm v7 and above only, skipping...") + } + + path, cleanup := tests.CreateTestProject(t, filepath.Join("..", "testdata", "npm", "projectCvs")) + defer cleanup() + + dependencies, blockedPackages, err := CalculateDependenciesMapWithCvs( + "npm", path, "cvs-test-project", + NpmTreeDepListParam{Args: []string{"--cache=" + filepath.Join(path, "tmpcache")}, IgnoreNodeModules: true, OverwritePackageLock: true}, + logger, false, + ) + assert.NoError(t, err) + assert.Len(t, blockedPackages, 1) + assert.Equal(t, "lodash", blockedPackages[0].Name) + assert.Equal(t, "999.999.999", blockedPackages[0].Version) + + _, hasLightweight := dependencies["lightweight:0.1.0"] + assert.True(t, hasLightweight, "package 'lightweight' should be in dependencies") + + _, hasBlocked := dependencies["lodash:999.999.999"] + assert.True(t, hasBlocked, "Blocked package should be added to dependencies for audit") +} + func getExpectedRespForTestDependencyPackageLockOnly() map[string]*dependencyInfo { return map[string]*dependencyInfo{ "underscore:1.13.6": { @@ -484,6 +512,48 @@ func TestFilterUniqueArgs(t *testing.T) { } } +// TestParseNpmNotFoundError tests parsing "No matching version found for pkg@version" error messages. +func TestParseNpmNotFoundError(t *testing.T) { + name, version, found := ParseNpmNotFoundError(errors.New("No matching version found for lodash@4.17.21")) + assert.True(t, found) + assert.Equal(t, "lodash", name) + assert.Equal(t, "4.17.21", version) + + name, version, found = ParseNpmNotFoundError(errors.New("No matching version found for @angular/core@15.0.0")) + assert.True(t, found) + assert.Equal(t, "@angular/core", name) + assert.Equal(t, "15.0.0", version) + + _, _, found = ParseNpmNotFoundError(errors.New("npm ERR! code ENOTFOUND")) + assert.False(t, found) + + _, _, found = ParseNpmNotFoundError(nil) + assert.False(t, found) +} + +// TestAddNotFoundPackagesToNpmLsOutput tests adding blocked packages to npm ls JSON output. +func TestAddNotFoundPackagesToNpmLsOutput(t *testing.T) { + npmLsOutput := `{"name": "test-project", "dependencies": {"express": {"version": "4.18.0"}}}` + blockedPackages := []NotFoundPackage{{Name: "lodash", Version: "4.17.21"}} + + result := addNotFoundPackagesToNpmLsOutput([]byte(npmLsOutput), blockedPackages) + + var output map[string]interface{} + err := json.Unmarshal(result, &output) + assert.NoError(t, err) + + deps := output["dependencies"].(map[string]interface{}) + + express := deps["express"].(map[string]interface{}) + assert.Equal(t, "4.18.0", express["version"]) + + lodash := deps["lodash"].(map[string]interface{}) + assert.Equal(t, "4.17.21", lodash["version"]) + assert.Equal(t, true, lodash["missing"]) + + assert.Equal(t, 2, len(deps)) +} + func TestParseDependenciesEdgeCases(t *testing.T) { testcases := []struct { name string From 552acd85b725f7d705750eb2d9669892045726db Mon Sep 17 00:00:00 2001 From: Bassel Mbariky Date: Thu, 22 Jan 2026 12:20:53 +0200 Subject: [PATCH 2/5] Fix gosec G104: handle error from removePackageFromPackageJson --- build/utils/npm.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/utils/npm.go b/build/utils/npm.go index 60e89733..3b5c097e 100644 --- a/build/utils/npm.go +++ b/build/utils/npm.go @@ -237,7 +237,9 @@ func runNpmInstallWithRetry(executablePath, workDir string, npmInstallCommandArg log.Info(fmt.Sprintf("Retrying without package %s...", pkgId)) blocked = append(blocked, NotFoundPackage{Name: name, Version: version}) - removePackageFromPackageJson(pkgJsonPath, name) + if err := removePackageFromPackageJson(pkgJsonPath, name); err != nil { + return blocked, err + } } } From 8dbf687a93b0fc2b8c1441871fddee8799e33075 Mon Sep 17 00:00:00 2001 From: Bassel Mbariky Date: Thu, 22 Jan 2026 12:28:45 +0200 Subject: [PATCH 3/5] Fix linter errors and add debug messages for blocked packages --- build/utils/npm.go | 12 ++++++++++-- build/utils/npm_test.go | 9 ++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/build/utils/npm.go b/build/utils/npm.go index 3b5c097e..29eab33b 100644 --- a/build/utils/npm.go +++ b/build/utils/npm.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/jfrog/gofrog/crypto" + "github.com/jfrog/gofrog/log" "golang.org/x/exp/slices" @@ -165,7 +166,9 @@ func runNpmLsWithCurationSupport(executablePath, srcPath string, npmListParams N if err != nil { return nil, nil, err } - defer utils.RemoveTempDir(tempDir) + defer func() { + _ = utils.RemoveTempDir(tempDir) + }() if err := utils.CopyDir(srcPath, tempDir, true, []string{"node_modules"}); err != nil { return nil, nil, err @@ -194,6 +197,7 @@ func runNpmLsWithCurationSupport(executablePath, srcPath string, npmListParams N func addNotFoundPackagesToNpmLsOutput(data []byte, blockedPackages []NotFoundPackage) []byte { var npmLsOutput map[string]interface{} if err := json.Unmarshal(data, &npmLsOutput); err != nil { + log.Debug("Failed to add blocked packages to npm ls output: unable to parse JSON") return data } deps, ok := npmLsOutput["dependencies"].(map[string]interface{}) @@ -207,7 +211,11 @@ func addNotFoundPackagesToNpmLsOutput(data []byte, blockedPackages []NotFoundPac "missing": true, } } - result, _ := json.Marshal(npmLsOutput) + result, err := json.Marshal(npmLsOutput) + if err != nil { + log.Debug("Failed to add blocked packages to npm ls output: unable to marshal JSON") + return data + } return result } diff --git a/build/utils/npm_test.go b/build/utils/npm_test.go index ed179f53..0436e731 100644 --- a/build/utils/npm_test.go +++ b/build/utils/npm_test.go @@ -542,12 +542,15 @@ func TestAddNotFoundPackagesToNpmLsOutput(t *testing.T) { err := json.Unmarshal(result, &output) assert.NoError(t, err) - deps := output["dependencies"].(map[string]interface{}) + deps, ok := output["dependencies"].(map[string]interface{}) + assert.True(t, ok, "dependencies should be a map") - express := deps["express"].(map[string]interface{}) + express, ok := deps["express"].(map[string]interface{}) + assert.True(t, ok, "express should be a map") assert.Equal(t, "4.18.0", express["version"]) - lodash := deps["lodash"].(map[string]interface{}) + lodash, ok := deps["lodash"].(map[string]interface{}) + assert.True(t, ok, "lodash should be a map") assert.Equal(t, "4.17.21", lodash["version"]) assert.Equal(t, true, lodash["missing"]) From 36f2ed85b845017a122f099c625e0ead8a85e9bb Mon Sep 17 00:00:00 2001 From: Bassel Mbariky Date: Wed, 4 Feb 2026 12:54:08 +0200 Subject: [PATCH 4/5] Handle transitive not-found packages in npm curation support --- build/utils/npm.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/build/utils/npm.go b/build/utils/npm.go index 29eab33b..efc3c766 100644 --- a/build/utils/npm.go +++ b/build/utils/npm.go @@ -178,6 +178,19 @@ func runNpmLsWithCurationSupport(executablePath, srcPath string, npmListParams N return nil, blockedPackages, err } + // Check if package-lock.json exists before running npm ls --package-lock-only + packageLockPath := filepath.Join(tempDir, "package-lock.json") + packageLockExists, _ := utils.IsFileExists(packageLockPath, false) + if !packageLockExists { + log.Warn("No package-lock.json found. Cannot run npm ls --package-lock-only.") + // Return empty JSON with just the blocked packages + data := []byte("{}") + if len(blockedPackages) > 0 { + data = addNotFoundPackagesToNpmLsOutput(data, blockedPackages) + } + return data, blockedPackages, nil + } + npmListParams.Args = append(npmListParams.Args, "--json", "--all", "--long", "--package-lock-only") data, errData, err := RunNpmCmd(executablePath, tempDir, AppendNpmCommand(npmListParams.Args, "ls"), log) if err != nil { @@ -239,7 +252,10 @@ func runNpmInstallWithRetry(executablePath, workDir string, npmInstallCommandArg pkgId := name + "@" + version if pkgId == lastPkg { - return blocked, fmt.Errorf("package %s is not found", pkgId) + // Package couldn't be removed (transitive dependency) + // Just add to blocked and continue, don't fail + log.Debug(fmt.Sprintf("Package %s is not found (transitive dependency). Adding to blocked packages and continuing...", pkgId)) + return blocked, nil } lastPkg = pkgId From 1bfb2f5bbd43ba638af1698b4f13cd449700d5b6 Mon Sep 17 00:00:00 2001 From: Bassel Mbariky Date: Wed, 4 Feb 2026 14:22:38 +0200 Subject: [PATCH 5/5] Add transitive dependency marker for curation audit --- build/utils/npm.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/build/utils/npm.go b/build/utils/npm.go index efc3c766..62406043 100644 --- a/build/utils/npm.go +++ b/build/utils/npm.go @@ -24,6 +24,11 @@ import ( const npmInstallCommand = "install" +// TransitiveDepMarker is appended to version for transitive blocked packages +// Uses ":" prefix so it becomes an extra segment that's ignored in parsing but detectable +// e.g., "5.1.6:TRANSITIVE" -> splits to ["name", "5.1.6", "TRANSITIVE"] +const TransitiveDepMarker = ":TRANSITIVE" + // CalculateNpmDependenciesList gets an npm project's dependencies. func CalculateNpmDependenciesList(executablePath, srcPath, moduleId string, npmParams NpmTreeDepListParam, calculateChecksums bool, log utils.Log) ([]entities.Dependency, error) { if log == nil { @@ -97,8 +102,9 @@ type DependencyInfo = dependencyInfo // NotFoundPackage represents a package not found during npm install (404/ETARGET). type NotFoundPackage struct { - Name string - Version string + Name string + Version string + IsTransitive bool // True if this is a transitive dependency } // Run 'npm list ...' command and parse the returned result to create a dependencies map of. @@ -219,8 +225,12 @@ func addNotFoundPackagesToNpmLsOutput(data []byte, blockedPackages []NotFoundPac npmLsOutput["dependencies"] = deps } for _, bp := range blockedPackages { + version := bp.Version + if bp.IsTransitive { + version = bp.Version + TransitiveDepMarker + } deps[bp.Name] = map[string]interface{}{ - "version": bp.Version, + "version": version, "missing": true, } } @@ -253,14 +263,14 @@ func runNpmInstallWithRetry(executablePath, workDir string, npmInstallCommandArg pkgId := name + "@" + version if pkgId == lastPkg { // Package couldn't be removed (transitive dependency) - // Just add to blocked and continue, don't fail log.Debug(fmt.Sprintf("Package %s is not found (transitive dependency). Adding to blocked packages and continuing...", pkgId)) + blocked = append(blocked, NotFoundPackage{Name: name, Version: version, IsTransitive: true}) return blocked, nil } lastPkg = pkgId log.Info(fmt.Sprintf("Retrying without package %s...", pkgId)) - blocked = append(blocked, NotFoundPackage{Name: name, Version: version}) + blocked = append(blocked, NotFoundPackage{Name: name, Version: version, IsTransitive: false}) if err := removePackageFromPackageJson(pkgJsonPath, name); err != nil { return blocked, err }