diff --git a/build/testdata/npm/projectCvs/package.json b/build/testdata/npm/projectCvs/package.json new file mode 100644 index 00000000..db1fab97 --- /dev/null +++ b/build/testdata/npm/projectCvs/package.json @@ -0,0 +1,16 @@ +{ + "name": "cvs-test-project", + "version": "1.0.0", + "description": "Test project for CVS (Curation Version Substitution)", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "lightweight": "^0.1.0", + "lodash": "999.999.999" + } +} + diff --git a/build/utils/npm.go b/build/utils/npm.go index 5a7a524d..62406043 100644 --- a/build/utils/npm.go +++ b/build/utils/npm.go @@ -8,9 +8,11 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "github.com/jfrog/gofrog/crypto" + "github.com/jfrog/gofrog/log" "golang.org/x/exp/slices" @@ -22,6 +24,11 @@ import ( const npmInstallCommand = "install" +// TransitiveDepMarker is appended to version for transitive blocked packages +// Uses ":" prefix so it becomes an extra segment that's ignored in parsing but detectable +// e.g., "5.1.6:TRANSITIVE" -> splits to ["name", "5.1.6", "TRANSITIVE"] +const TransitiveDepMarker = ":TRANSITIVE" + // CalculateNpmDependenciesList gets an npm project's dependencies. func CalculateNpmDependenciesList(executablePath, srcPath, moduleId string, npmParams NpmTreeDepListParam, calculateChecksums bool, log utils.Log) ([]entities.Dependency, error) { if log == nil { @@ -91,6 +98,15 @@ type dependencyInfo struct { *npmLsDependency } +type DependencyInfo = dependencyInfo + +// NotFoundPackage represents a package not found during npm install (404/ETARGET). +type NotFoundPackage struct { + Name string + Version string + IsTransitive bool // True if this is a transitive dependency +} + // Run 'npm list ...' command and parse the returned result to create a dependencies map of. // The dependencies map looks like name:version -> entities.Dependency. func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmListParams NpmTreeDepListParam, log utils.Log, skipInstall bool) (map[string]*dependencyInfo, error) { @@ -125,6 +141,176 @@ func CalculateDependenciesMap(executablePath, srcPath, moduleId string, npmListP }) } +// CalculateDependenciesMapWithCvs calculates dependencies. +// It retries npm install when packages are blocked, collecting blocked packages for audit. +// this code is used by 'jf ca'. +func CalculateDependenciesMapWithCvs(executablePath, srcPath, moduleId string, npmListParams NpmTreeDepListParam, log utils.Log, skipInstall bool) (map[string]*DependencyInfo, []NotFoundPackage, error) { + dependenciesMap := make(map[string]*DependencyInfo) + npmVersion, err := GetNpmVersion(executablePath, log) + if err != nil { + return nil, nil, err + } + + data, blockedPackages, err := runNpmLsWithCurationSupport(executablePath, srcPath, npmListParams, log, npmVersion) + if err != nil { + return nil, blockedPackages, err + } + + parseFunc := parseNpmLsDependencyFunc(npmVersion) + err = jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) { + if string(key) == "dependencies" { + err = parseDependencies(value, []string{moduleId}, dependenciesMap, parseFunc, log) + } + return err + }) + + return dependenciesMap, blockedPackages, err +} + +func runNpmLsWithCurationSupport(executablePath, srcPath string, npmListParams NpmTreeDepListParam, log utils.Log, npmVersion *version.Version) ([]byte, []NotFoundPackage, error) { + tempDir, err := utils.CreateTempDir() + if err != nil { + return nil, nil, err + } + defer func() { + _ = utils.RemoveTempDir(tempDir) + }() + + if err := utils.CopyDir(srcPath, tempDir, true, []string{"node_modules"}); err != nil { + return nil, nil, err + } + blockedPackages, err := runNpmInstallWithRetry(executablePath, tempDir, npmListParams.InstallCommandArgs, npmListParams.Args, log, npmVersion) + if err != nil { + return nil, blockedPackages, err + } + + // Check if package-lock.json exists before running npm ls --package-lock-only + packageLockPath := filepath.Join(tempDir, "package-lock.json") + packageLockExists, _ := utils.IsFileExists(packageLockPath, false) + if !packageLockExists { + log.Warn("No package-lock.json found. Cannot run npm ls --package-lock-only.") + // Return empty JSON with just the blocked packages + data := []byte("{}") + if len(blockedPackages) > 0 { + data = addNotFoundPackagesToNpmLsOutput(data, blockedPackages) + } + return data, blockedPackages, nil + } + + npmListParams.Args = append(npmListParams.Args, "--json", "--all", "--long", "--package-lock-only") + data, errData, err := RunNpmCmd(executablePath, tempDir, AppendNpmCommand(npmListParams.Args, "ls"), log) + if err != nil { + log.Warn(err.Error()) + } else if len(errData) > 0 { + log.Warn("Encountered some issues while running 'npm ls' command:\n" + strings.TrimSpace(string(errData))) + } + + // Add blocked packages to the dependency tree for auditing + if len(blockedPackages) > 0 { + data = addNotFoundPackagesToNpmLsOutput(data, blockedPackages) + } + + return data, blockedPackages, nil +} + +func addNotFoundPackagesToNpmLsOutput(data []byte, blockedPackages []NotFoundPackage) []byte { + var npmLsOutput map[string]interface{} + if err := json.Unmarshal(data, &npmLsOutput); err != nil { + log.Debug("Failed to add blocked packages to npm ls output: unable to parse JSON") + return data + } + deps, ok := npmLsOutput["dependencies"].(map[string]interface{}) + if !ok { + deps = make(map[string]interface{}) + npmLsOutput["dependencies"] = deps + } + for _, bp := range blockedPackages { + version := bp.Version + if bp.IsTransitive { + version = bp.Version + TransitiveDepMarker + } + deps[bp.Name] = map[string]interface{}{ + "version": version, + "missing": true, + } + } + result, err := json.Marshal(npmLsOutput) + if err != nil { + log.Debug("Failed to add blocked packages to npm ls output: unable to marshal JSON") + return data + } + return result +} + +// runNpmInstallWithRetry runs npm install in the given directory (should be temp dir). +// If a package is blocked (404), it removes it from package.json and retries. +func runNpmInstallWithRetry(executablePath, workDir string, npmInstallCommandArgs, npmArgs []string, log utils.Log, npmVersion *version.Version) ([]NotFoundPackage, error) { + pkgJsonPath := filepath.Join(workDir, "package.json") + var blocked []NotFoundPackage + var lastPkg string + + for { + err := installPackageLock(executablePath, workDir, npmInstallCommandArgs, npmArgs, log, npmVersion) + if err == nil { + return blocked, nil + } + + name, version, found := ParseNpmNotFoundError(err) + if !found { + return blocked, err + } + + pkgId := name + "@" + version + if pkgId == lastPkg { + // Package couldn't be removed (transitive dependency) + log.Debug(fmt.Sprintf("Package %s is not found (transitive dependency). Adding to blocked packages and continuing...", pkgId)) + blocked = append(blocked, NotFoundPackage{Name: name, Version: version, IsTransitive: true}) + return blocked, nil + } + lastPkg = pkgId + + log.Info(fmt.Sprintf("Retrying without package %s...", pkgId)) + blocked = append(blocked, NotFoundPackage{Name: name, Version: version, IsTransitive: false}) + if err := removePackageFromPackageJson(pkgJsonPath, name); err != nil { + return blocked, err + } + } +} + +var npmNotFoundPattern = regexp.MustCompile(`No matching version found for\s+(@?[\w./-]+)@([\d][\w._-]*)`) + +func ParseNpmNotFoundError(err error) (name, version string, found bool) { + if err == nil { + return "", "", false + } + if matches := npmNotFoundPattern.FindStringSubmatch(err.Error()); len(matches) >= 3 { + return matches[1], strings.TrimSuffix(matches[2], "."), true + } + return "", "", false +} + +func removePackageFromPackageJson(packageJsonPath, packageName string) error { + data, err := os.ReadFile(packageJsonPath) + if err != nil { + return err + } + var pkg map[string]interface{} + if err := json.Unmarshal(data, &pkg); err != nil { + return err + } + depTypes := []string{"dependencies", "devDependencies", "peerDependencies", "optionalDependencies"} + for _, depType := range depTypes { + if deps, ok := pkg[depType].(map[string]interface{}); ok { + delete(deps, packageName) + } + } + newData, err := json.MarshalIndent(pkg, "", " ") + if err != nil { + return err + } + return os.WriteFile(packageJsonPath, newData, 0644) +} + func runNpmLsWithNodeModules(executablePath, srcPath string, npmArgs []string, log utils.Log) (data []byte) { npmArgs = append(npmArgs, "--json", "--all", "--long") data, errData, err := RunNpmCmd(executablePath, srcPath, AppendNpmCommand(npmArgs, "ls"), log) diff --git a/build/utils/npm_test.go b/build/utils/npm_test.go index 4f0c3d42..0436e731 100644 --- a/build/utils/npm_test.go +++ b/build/utils/npm_test.go @@ -271,6 +271,34 @@ func TestCalculateDependenciesMapWithProhibitedInstallation(t *testing.T) { assert.True(t, errors.As(err, &installForbiddenErr)) } +// TestCalculateDependenciesMapWithCvs tests the full CVS flow with a real npm project. +func TestCalculateDependenciesMapWithCvs(t *testing.T) { + npmVersion, _, err := GetNpmVersionAndExecPath(logger) + require.NoError(t, err) + if !npmVersion.AtLeast("7.0.0") { + t.Skip("Running on npm v7 and above only, skipping...") + } + + path, cleanup := tests.CreateTestProject(t, filepath.Join("..", "testdata", "npm", "projectCvs")) + defer cleanup() + + dependencies, blockedPackages, err := CalculateDependenciesMapWithCvs( + "npm", path, "cvs-test-project", + NpmTreeDepListParam{Args: []string{"--cache=" + filepath.Join(path, "tmpcache")}, IgnoreNodeModules: true, OverwritePackageLock: true}, + logger, false, + ) + assert.NoError(t, err) + assert.Len(t, blockedPackages, 1) + assert.Equal(t, "lodash", blockedPackages[0].Name) + assert.Equal(t, "999.999.999", blockedPackages[0].Version) + + _, hasLightweight := dependencies["lightweight:0.1.0"] + assert.True(t, hasLightweight, "package 'lightweight' should be in dependencies") + + _, hasBlocked := dependencies["lodash:999.999.999"] + assert.True(t, hasBlocked, "Blocked package should be added to dependencies for audit") +} + func getExpectedRespForTestDependencyPackageLockOnly() map[string]*dependencyInfo { return map[string]*dependencyInfo{ "underscore:1.13.6": { @@ -484,6 +512,51 @@ func TestFilterUniqueArgs(t *testing.T) { } } +// TestParseNpmNotFoundError tests parsing "No matching version found for pkg@version" error messages. +func TestParseNpmNotFoundError(t *testing.T) { + name, version, found := ParseNpmNotFoundError(errors.New("No matching version found for lodash@4.17.21")) + assert.True(t, found) + assert.Equal(t, "lodash", name) + assert.Equal(t, "4.17.21", version) + + name, version, found = ParseNpmNotFoundError(errors.New("No matching version found for @angular/core@15.0.0")) + assert.True(t, found) + assert.Equal(t, "@angular/core", name) + assert.Equal(t, "15.0.0", version) + + _, _, found = ParseNpmNotFoundError(errors.New("npm ERR! code ENOTFOUND")) + assert.False(t, found) + + _, _, found = ParseNpmNotFoundError(nil) + assert.False(t, found) +} + +// TestAddNotFoundPackagesToNpmLsOutput tests adding blocked packages to npm ls JSON output. +func TestAddNotFoundPackagesToNpmLsOutput(t *testing.T) { + npmLsOutput := `{"name": "test-project", "dependencies": {"express": {"version": "4.18.0"}}}` + blockedPackages := []NotFoundPackage{{Name: "lodash", Version: "4.17.21"}} + + result := addNotFoundPackagesToNpmLsOutput([]byte(npmLsOutput), blockedPackages) + + var output map[string]interface{} + err := json.Unmarshal(result, &output) + assert.NoError(t, err) + + deps, ok := output["dependencies"].(map[string]interface{}) + assert.True(t, ok, "dependencies should be a map") + + express, ok := deps["express"].(map[string]interface{}) + assert.True(t, ok, "express should be a map") + assert.Equal(t, "4.18.0", express["version"]) + + lodash, ok := deps["lodash"].(map[string]interface{}) + assert.True(t, ok, "lodash should be a map") + assert.Equal(t, "4.17.21", lodash["version"]) + assert.Equal(t, true, lodash["missing"]) + + assert.Equal(t, 2, len(deps)) +} + func TestParseDependenciesEdgeCases(t *testing.T) { testcases := []struct { name string