diff --git a/packages/scan/src/__tests__/dependencies.test.ts b/packages/scan/src/__tests__/dependencies.test.ts new file mode 100644 index 0000000..3bce8b0 --- /dev/null +++ b/packages/scan/src/__tests__/dependencies.test.ts @@ -0,0 +1,218 @@ +import { describe, expect, it } from 'vitest'; +import { LOCKFILE_PARSERS } from '../node/dependencies'; + +/** + * These parsers exist because the previous ones did not, and the absence read + * as a clean scan. So the assertion that matters in every case below is the + * same: a real lockfile must not parse to an empty list. + */ + +const parse = (file: string, content: string) => { + const parser = LOCKFILE_PARSERS[file]; + if (!parser) throw new Error(`no parser registered for ${file}`); + return parser(content); +}; + +describe('pnpm-lock.yaml', () => { + it('reads v9 quoted keys', () => { + const lock = `lockfileVersion: '9.0' + +importers: + + .: + dependencies: + lodash: + specifier: ^4.17.21 + version: 4.17.21 + +packages: + + '@babel/code-frame@7.24.7': + resolution: {integrity: sha512-aaa} + + lodash@4.17.21: + resolution: {integrity: sha512-bbb} + +snapshots: + + '@babel/code-frame@7.24.7': {} +`; + expect(parse('pnpm-lock.yaml', lock)).toEqual([ + { name: '@babel/code-frame', version: '7.24.7' }, + { name: 'lodash', version: '4.17.21' }, + ]); + }); + + it('reads v6 leading-slash keys and drops the peer suffix', () => { + const lock = `lockfileVersion: '6.0' + +packages: + + /@babel/core@7.24.7: + resolution: {integrity: sha512-aaa} + + /react-dom@18.3.1(react@18.3.1): + resolution: {integrity: sha512-bbb} +`; + expect(parse('pnpm-lock.yaml', lock)).toEqual([ + { name: '@babel/core', version: '7.24.7' }, + { name: 'react-dom', version: '18.3.1' }, + ]); + }); + + it('reads v5 slash-separated versions, including underscore peer suffixes', () => { + const lock = `lockfileVersion: 5.4 + +packages: + + /@babel/core/7.24.7: + resolution: {integrity: sha512-aaa} + + /react-dom/17.0.2_react@17.0.2: + resolution: {integrity: sha512-bbb} +`; + expect(parse('pnpm-lock.yaml', lock)).toEqual([ + { name: '@babel/core', version: '7.24.7' }, + { name: 'react-dom', version: '17.0.2' }, + ]); + }); + + it('keeps underscores that belong to the package name', () => { + const lock = `lockfileVersion: '9.0' + +packages: + + my_package@1.2.3: + resolution: {integrity: sha512-aaa} +`; + expect(parse('pnpm-lock.yaml', lock)).toEqual([{ name: 'my_package', version: '1.2.3' }]); + }); + + it('ignores entries outside the packages block', () => { + const lock = `lockfileVersion: '9.0' + +importers: + + apps/cli: + dependencies: + commander: + specifier: ^12.0.0 + version: 12.1.0 +`; + expect(parse('pnpm-lock.yaml', lock)).toEqual([]); + }); +}); + +describe('yarn.lock', () => { + it('reads the v1 dialect', () => { + const lock = `# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4": + version "7.24.7" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.24.7.tgz#aaa" + integrity sha512-aaa + +lodash@^4.17.20: + version "4.17.21" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#bbb" +`; + expect(parse('yarn.lock', lock)).toEqual([ + { name: '@babel/code-frame', version: '7.24.7' }, + { name: 'lodash', version: '4.17.21' }, + ]); + }); + + it('reads the berry dialect and skips its metadata block', () => { + const lock = `# This file is generated by running "yarn install" inside your project. + +__metadata: + version: 8 + cacheKey: 10c0 + +"lodash@npm:^4.17.20": + version: 4.17.21 + resolution: "lodash@npm:4.17.21" + languageName: node +`; + expect(parse('yarn.lock', lock)).toEqual([{ name: 'lodash', version: '4.17.21' }]); + }); + + it('skips workspace entries that resolve to no published version', () => { + const lock = `"my-app@workspace:.": + version: 0.0.0-use.local + resolution: "my-app@workspace:." +`; + expect(parse('yarn.lock', lock)).toEqual([]); + }); +}); + +describe('Pipfile.lock', () => { + it('reads both sections and strips the == specifier', () => { + const lock = JSON.stringify({ + _meta: { hash: { sha256: 'aaa' } }, + default: { + requests: { version: '==2.32.3' }, + 'some-vcs-dep': { git: 'https://example.com/x.git', ref: 'abc123' }, + }, + develop: { + pytest: { version: '==8.3.2' }, + }, + }); + expect(parse('Pipfile.lock', lock)).toEqual([ + { name: 'requests', version: '2.32.3' }, + { name: 'pytest', version: '8.3.2' }, + ]); + }); +}); + +describe('requirements.txt', () => { + it('keeps PEP 440 pre- and post-release pins', () => { + const txt = `requests==2.32.3 +django==5.0rc1 +urllib3==2.2.2.post1 +`; + expect(parse('requirements.txt', txt)).toEqual([ + { name: 'requests', version: '2.32.3' }, + { name: 'django', version: '5.0rc1' }, + { name: 'urllib3', version: '2.2.2.post1' }, + ]); + }); + + it('handles extras, markers, comments and flag lines', () => { + const txt = `# pinned for prod +-r base.txt +--index-url https://example.com/simple +celery[redis]==5.4.0 +uvicorn==0.30.6 ; python_version >= "3.11" +flask # unpinned, unusable +django>=4.0 +`; + expect(parse('requirements.txt', txt)).toEqual([ + { name: 'celery', version: '5.4.0' }, + { name: 'uvicorn', version: '0.30.6' }, + ]); + }); + + it('drops wildcard pins, which are ranges rather than versions', () => { + expect(parse('requirements.txt', 'django==4.2.*\n')).toEqual([]); + }); +}); + +describe('package-lock.json', () => { + it('reads v3 nested paths down to the package name', () => { + const lock = JSON.stringify({ + lockfileVersion: 3, + packages: { + '': { name: 'root', version: '1.0.0' }, + 'node_modules/lodash': { version: '4.17.21' }, + 'node_modules/a/node_modules/semver': { version: '7.6.3' }, + }, + }); + expect(parse('package-lock.json', lock)).toEqual([ + { name: 'lodash', version: '4.17.21' }, + { name: 'semver', version: '7.6.3' }, + ]); + }); +}); diff --git a/packages/scan/src/node/dependencies.ts b/packages/scan/src/node/dependencies.ts index a4e1f94..c0bf7b9 100644 --- a/packages/scan/src/node/dependencies.ts +++ b/packages/scan/src/node/dependencies.ts @@ -6,6 +6,18 @@ * boundary: the pattern rules are deterministic and fast, this is neither, and * CI wants to choose. `runScan` keeps calling it (the daemon has always * included advisories); the CLI asks for it with `--deps`. + * + * The lockfile list and the parser used to disagree. `pnpm-lock.yaml`, + * `yarn.lock` and `Pipfile.lock` were listed as supported, matched by + * filename, and then fell through a parser that only implemented + * `package-lock.json` and `requirements.txt` — so they resolved to zero + * dependencies and the scan reported nothing. Not "could not parse this + * lockfile": nothing, which reads exactly like a clean result. A pnpm + * repository (this one included) got a dependency scan that never asked OSV a + * single question. + * + * So every entry in `LOCKFILES` now names the function that parses it. Adding + * a filename without a parser is a type error rather than a silent hole. */ import { existsSync, readFileSync } from 'node:fs'; @@ -19,13 +31,18 @@ interface OsvVulnerability { severity?: Array<{ type: string; score: string }>; } -const LOCKFILES: readonly { file: string; ecosystem: string }[] = [ - { file: 'package-lock.json', ecosystem: 'npm' }, - { file: 'pnpm-lock.yaml', ecosystem: 'npm' }, - { file: 'yarn.lock', ecosystem: 'npm' }, - { file: 'requirements.txt', ecosystem: 'PyPI' }, - { file: 'Pipfile.lock', ecosystem: 'PyPI' }, -]; +export interface LockedDependency { + name: string; + version: string; +} + +type LockfileParser = (content: string) => LockedDependency[]; + +interface LockfileSupport { + file: string; + ecosystem: string; + parse: LockfileParser; +} /** Cap per lockfile, so a first run on a neglected tree summarises rather than floods OSV. */ const MAX_DEPS_PER_LOCKFILE = 50; @@ -33,17 +50,34 @@ const MAX_DEPS_PER_LOCKFILE = 50; export async function scanDependencies(targetPath: string): Promise { const findings: ScanFinding[] = []; - for (const { file, ecosystem } of LOCKFILES) { + for (const { file, ecosystem, parse } of LOCKFILES) { const lockPath = join(targetPath, file); if (!existsSync(lockPath)) continue; - let deps: Array<{ name: string; version: string }>; + let deps: LockedDependency[]; try { - deps = parseDependencies(lockPath, file); + deps = dedupe(parse(readFileSync(lockPath, 'utf-8'))); } catch { continue; } + // A lockfile that parsed to nothing is a parser problem, not an empty + // project — real lockfiles have entries. Say so rather than moving on, + // because the alternative is a scan that reports clean without asking. + if (deps.length === 0) { + findings.push(incompleteFinding(file, 'No dependencies could be read from this lockfile.')); + continue; + } + + if (deps.length > MAX_DEPS_PER_LOCKFILE) { + findings.push( + incompleteFinding( + file, + `Only the first ${MAX_DEPS_PER_LOCKFILE} of ${deps.length} locked packages were checked against OSV.`, + ), + ); + } + for (const dep of deps.slice(0, MAX_DEPS_PER_LOCKFILE)) { let vulns: OsvVulnerability[]; try { @@ -73,6 +107,26 @@ export async function scanDependencies(targetPath: string): Promise { - const deps: Array<{ name: string; version: string }> = []; +/** + * One query per distinct name@version. Lockfiles repeat a package once per + * dependent, and a 900-line `yarn.lock` can resolve to a few hundred unique + * packages — the duplicates would spend the per-file cap on questions already + * asked. + */ +function dedupe(deps: LockedDependency[]): LockedDependency[] { + const seen = new Set(); + const unique: LockedDependency[] = []; + for (const dep of deps) { + const key = `${dep.name}@${dep.version}`; + if (seen.has(key)) continue; + seen.add(key); + unique.push(dep); + } + return unique; +} + +/** + * Split `name@version` where the name may itself begin with `@`. + * + * `@babel/core@7.24.7` has two `@`, and only the last one separates. Anything + * without a separating `@` is a spec this parser does not understand, and is + * dropped rather than guessed at. + */ +function splitNameVersion(spec: string): LockedDependency | null { + const at = spec.lastIndexOf('@'); + if (at <= 0) return null; + const name = spec.slice(0, at); + const version = spec.slice(at + 1); + if (!name || !version) return null; + return { name, version }; +} - if (filename === 'package-lock.json') { - const lock = JSON.parse(readFileSync(lockPath, 'utf-8')) as { - packages?: Record; - dependencies?: Record; - }; - const packages = lock.packages ?? lock.dependencies ?? {}; - for (const [key, value] of Object.entries(packages)) { - const name = key.replace(/^node_modules\//, ''); - const version = value?.version; - if (name && version && !name.startsWith('.')) deps.push({ name, version }); +/** + * A resolved version, or null if the string is a range, a tag or a URL. + * + * OSV answers questions about exact versions. `^7.0.0`, `next` and + * `github:owner/repo` are not versions, and sending them produces a confident + * empty answer — the worst possible result, because it looks like "no known + * advisories". + */ +function exactVersion(raw: string): string | null { + const version = raw.trim().replace(/^[=v]+/, ''); + return /^[0-9][0-9a-zA-Z.+-]*$/.test(version) ? version : null; +} + +function parsePackageLock(content: string): LockedDependency[] { + const lock = JSON.parse(content) as { + packages?: Record; + dependencies?: Record; + }; + const packages = lock.packages ?? lock.dependencies ?? {}; + const deps: LockedDependency[] = []; + for (const [key, value] of Object.entries(packages)) { + // v2+ keys are paths (`node_modules/a/node_modules/b`); the package is the + // last segment, not the whole path. + const name = key.replace(/^.*node_modules\//, ''); + const version = value?.version; + if (name && version && !name.startsWith('.')) deps.push({ name, version }); + } + return deps; +} + +/** + * pnpm writes its resolved tree as keys under `packages:`, and has changed the + * shape of those keys three times: + * + * v5 `/@babel/core/7.24.7:` + * v6 `/@babel/core@7.24.7:` + * v9 `'@babel/core@7.24.7':` + * + * All three are handled by normalising the key rather than by detecting the + * lockfile version, because a scanner that recognises only the version it was + * written against fails silently on the next one — which is the failure this + * file exists to fix. + * + * Peer-dependency suffixes (`(react@18.3.1)`, `_react@17.0.2`) are stripped: + * they describe the resolution context, not the package. + */ +function parsePnpmLock(content: string): LockedDependency[] { + const deps: LockedDependency[] = []; + let inPackages = false; + + for (const line of content.split('\n')) { + if (/^[a-zA-Z]/.test(line)) { + inPackages = line.startsWith('packages:'); + continue; + } + if (!inPackages) continue; + + // Entries sit exactly one level in. Deeper lines are the entry's fields. + const match = /^ {2}(?! )(.+):\s*$/.exec(line); + if (!match?.[1]) continue; + + let key = match[1].trim().replace(/^['"]|['"]$/g, ''); + key = key.replace(/^\//, ''); + // Parentheses cannot occur in a package name, so the v6/v9 peer suffix can + // be cut from the whole key. + key = key.replace(/\(.*$/, ''); + + // v5 separated the version with a slash. It is split here rather than + // rewritten into the `@` form, because a v5 peer suffix (`_react@17.0.2`) + // contains an `@` of its own and would capture the generic splitter. + let name: string; + let rawVersion: string; + const slashed = /^(@?[^@]+)\/([0-9][^/]*)$/.exec(key); + if (slashed?.[1] && slashed[2]) { + name = slashed[1]; + rawVersion = slashed[2]; + } else { + const dep = splitNameVersion(key); + if (!dep) continue; + name = dep.name; + rawVersion = dep.version; + } + + // The v5 peer suffix is an underscore, which *is* legal in a package name + // — so it is only ever stripped from the version, never from the name. + const version = exactVersion(rawVersion.replace(/_.*$/, '')); + if (version) deps.push({ name, version }); + } + + return deps; +} + +/** + * Both yarn dialects, which differ in punctuation rather than structure: + * + * v1 `"@babel/core@^7.0.0":` then ` version "7.24.7"` + * berry `"@babel/core@npm:^7.0.0":` then ` version: 7.24.7` + * + * The header carries the name and the indented `version` carries the truth, so + * the parser holds the last header and waits. + */ +function parseYarnLock(content: string): LockedDependency[] { + const deps: LockedDependency[] = []; + let pendingName: string | null = null; + + for (const line of content.split('\n')) { + if (line.startsWith('#') || line.trim() === '') continue; + + if (!/^\s/.test(line)) { + pendingName = null; + const header = line.replace(/:\s*$/, ''); + // Multiple specs share one entry; they name the same package, so the + // first is enough. + const first = header.split(',')[0]?.trim().replace(/^['"]|['"]$/g, ''); + if (!first) continue; + // `__metadata:` and other berry bookkeeping blocks are not packages. + if (!first.includes('@') || first === '__metadata') continue; + // Anything resolved by a non-registry protocol is local to this + // checkout — the workspace root resolves to `0.0.0-use.local`, which is + // a well-formed version of a package no registry has ever heard of. + // Asking OSV about it spends the per-file cap on a guaranteed miss. + if (/@(?:workspace|file|link|portal|exec|patch):/.test(first)) continue; + const dep = splitNameVersion(first.replace(/@npm:/, '@')); + if (dep) pendingName = dep.name; + continue; } - return deps; + + if (!pendingName) continue; + const version = /^\s+version:?\s+["']?([^"'\s]+)["']?\s*$/.exec(line); + if (!version?.[1]) continue; + const exact = exactVersion(version[1]); + if (exact) deps.push({ name: pendingName, version: exact }); + pendingName = null; } - if (filename === 'requirements.txt') { - for (const line of readFileSync(lockPath, 'utf-8').split('\n')) { - const match = /^([a-zA-Z0-9_.-]+)==([0-9.]+)/.exec(line); - if (match?.[1] && match[2]) deps.push({ name: match[1], version: match[2] }); + return deps; +} + +/** Pipenv's lockfile is JSON, with versions written as the `==x.y.z` specifier. */ +function parsePipfileLock(content: string): LockedDependency[] { + const lock = JSON.parse(content) as Record>; + const deps: LockedDependency[] = []; + + for (const section of ['default', 'develop']) { + const packages = lock[section]; + if (!packages || typeof packages !== 'object') continue; + for (const [name, value] of Object.entries(packages)) { + // VCS and file entries carry a ref instead of a version. There is no + // version to ask OSV about, so they are skipped. + const version = exactVersion(String(value?.version ?? '').replace(/^==/, '')); + if (name && version) deps.push({ name, version }); } } return deps; } +/** + * Only `==` pins are usable. A range gives OSV nothing to answer against, and + * the previous `([0-9.]+)` version pattern also silently dropped every + * pre-release and post-release pin PEP 440 allows (`2.0.0rc1`, `1.4.post1`), + * which are ordinary in pinned production requirements. + */ +function parseRequirementsTxt(content: string): LockedDependency[] { + const deps: LockedDependency[] = []; + + for (const raw of content.split('\n')) { + // Environment markers and comments both describe the pin rather than + // being part of it. + const line = raw.split('#')[0]?.split(';')[0]?.trim(); + if (!line || line.startsWith('-')) continue; + + const match = /^([a-zA-Z0-9._-]+)\s*(?:\[[^\]]*\])?\s*==\s*([^\s,]+)/.exec(line); + if (!match?.[1] || !match[2]) continue; + // A wildcard pin (`==1.4.*`) is a range wearing an equals sign. + const version = exactVersion(match[2]); + if (version) deps.push({ name: match[1], version }); + } + + return deps; +} + +const LOCKFILES: readonly LockfileSupport[] = [ + { file: 'package-lock.json', ecosystem: 'npm', parse: parsePackageLock }, + { file: 'pnpm-lock.yaml', ecosystem: 'npm', parse: parsePnpmLock }, + { file: 'yarn.lock', ecosystem: 'npm', parse: parseYarnLock }, + { file: 'requirements.txt', ecosystem: 'PyPI', parse: parseRequirementsTxt }, + { file: 'Pipfile.lock', ecosystem: 'PyPI', parse: parsePipfileLock }, +]; + +/** Exposed so the parsers can be tested without a network or a fixture tree. */ +export const LOCKFILE_PARSERS: Readonly> = Object.fromEntries( + LOCKFILES.map((entry) => [entry.file, entry.parse]), +); + /** Package names and versions are validated before they reach the query body. */ function isValidPackageName(name: string): boolean { return /^[@a-zA-Z0-9_.\-/]{1,214}$/.test(name);