Detect Changes based on Lockfile

.github/actions/src/__tests__/lockfiles.test.ts

@@ -0,0 +1,23 @@
+import { describe, expect, it, vi } from 'vitest';
+import * as utils from '../lockfiles.js';
+
+vi.mock(import('../gitRoot.js'), () => ({
+  gitRoot: 'root'
+}));
+
+describe(utils.extractPackageName, () => {
+  it('works with packages that start with @', () => {
+    expect(utils.extractPackageName('@sourceacademy/tab-Rune@workspace:^'))
+      .toEqual('@sourceacademy/tab-Rune');
+  });
+
+  it('works with regular package names', () => {
+    expect(utils.extractPackageName('lodash@npm:^4.17.20'))
+      .toEqual('lodash');
+  });
+
+  it('throws on invalid package name', () => {
+    expect(() => utils.extractPackageName('something weird'))
+      .toThrowError('Invalid package name: something weird');
+  });
+});

.github/actions/src/__tests__/sample_why.txt

@@ -0,0 +1,13 @@
+{"value":"@sourceacademy/bundle-unittest@workspace:src/bundles/unittest","children":{"lodash@npm:4.17.21":{"locator":"lodash@npm:4.17.21","descriptor":"lodash@npm:^4.17.21"}}}
+{"value":"@sourceacademy/markdown-plugin-directory-tree@virtual:f0562030a653a107496800a80ececfadfd737a6a260bbd2572ca4e42de1d6b97749197bb1c41d0d71c9f18ab009789560666041994678fe1aba63902e9fb15bd#workspace:lib/markdown-tree","children":{"lodash@npm:4.17.21":{"locator":"lodash@npm:4.17.21","descriptor":"lodash@npm:^4.17.21"}}}
+{"value":"@sourceacademy/markdown-plugin-directory-tree@workspace:lib/markdown-tree","children":{"lodash@npm:4.17.21":{"locator":"lodash@npm:4.17.21","descriptor":"lodash@npm:^4.17.21"}}}
+{"value":"@sourceacademy/modules-buildtools@workspace:lib/buildtools","children":{"lodash@npm:4.17.21":{"locator":"lodash@npm:4.17.21","descriptor":"lodash@npm:^4.17.21"}}}
+{"value":"@sourceacademy/modules-github-actions@workspace:.github/actions","children":{"lodash@npm:4.17.21":{"locator":"lodash@npm:4.17.21","descriptor":"lodash@npm:^4.17.21"}}}
+{"value":"@sourceacademy/modules-repotools@virtual:1448f4d06c0cf02cc986bce0139b31c0f3c95af1573e69673bc11ad2928a2fc505818a40d3f1caf168c35ce00c20238d52dfdba1940c59c81fd5df1b08ba49d6#workspace:lib/repotools","children":{"lodash@npm:4.17.21":{"locator":"lodash@npm:4.17.21","descriptor":"lodash@npm:^4.17.21"}}}
+{"value":"@sourceacademy/modules-repotools@virtual:420fa510eaf06edbb20ddb72fbcd454e937e7e266b8c6b7a55ab314e0562f2688175809f2b44c78563a0467da21b6d34688fe0020ecef5e669997fe518ee56e1#workspace:lib/repotools","children":{"lodash@npm:4.17.21":{"locator":"lodash@npm:4.17.21","descriptor":"lodash@npm:^4.17.21"}}}
+{"value":"@sourceacademy/modules-repotools@virtual:72a684de9912e8b42da7a581a5478d78ad73e0a505830e61e9b2fbc990fcc10719f71ef5373c26313c28b9295f1381b0d95a4d0ab2ae6e33671c2ba25a5c1dc2#workspace:lib/repotools","children":{"lodash@npm:4.17.21":{"locator":"lodash@npm:4.17.21","descriptor":"lodash@npm:^4.17.21"}}}
+{"value":"@sourceacademy/modules-repotools@workspace:lib/repotools","children":{"lodash@npm:4.17.21":{"locator":"lodash@npm:4.17.21","descriptor":"lodash@npm:^4.17.21"}}}
+{"value":"@sourceacademy/modules@workspace:.","children":{"lodash@npm:4.17.21":{"locator":"lodash@npm:4.17.21","descriptor":"lodash@npm:^4.17.21"}}}
+{"value":"archiver-utils@npm:5.0.2","children":{"lodash@npm:4.17.21":{"locator":"lodash@npm:4.17.21","descriptor":"lodash@npm:^4.17.15"}}}
+{"value":"js-slang@npm:1.0.85","children":{"lodash@npm:4.17.21":{"locator":"lodash@npm:4.17.21","descriptor":"lodash@npm:^4.17.21"}}}
+{"value":"source-academy-wabt@npm:1.1.3","children":{"lodash@npm:4.17.21":{"locator":"lodash@npm:4.17.21","descriptor":"lodash@npm:^4.17.21"}}}

.github/actions/src/info/__tests__/index.test.ts

@@ -4,7 +4,7 @@ import pathlib from 'path';
 import * as core from '@actions/core';
 import { describe, expect, test, vi } from 'vitest';
 import * as git from '../../commons.js';
-import * as lockfiles from '../../lockfiles/index.js';
+import * as lockfiles from '../../lockfiles.js';
 import { getAllPackages, getRawPackages, main } from '../index.js';
 
 const mockedCheckChanges = vi.spyOn(git, 'checkDirForChanges');

.github/actions/src/info/index.ts

@@ -6,7 +6,7 @@ import type { SummaryTableRow } from '@actions/core/lib/summary.js';
 import packageJson from '../../../../package.json' with { type: 'json' };
 import { checkDirForChanges, type PackageRecord, type RawPackageRecord } from '../commons.js';
 import { gitRoot } from '../gitRoot.js';
-import { getPackagesWithResolutionChanges, hasLockFileChanged } from '../lockfiles/index.js';
+import { getPackagesWithResolutionChanges, hasLockFileChanged } from '../lockfiles.js';
 import { topoSortPackages } from './sorter.js';
 
 const packageNameRE = /^@sourceacademy\/(.+?)-(.+)$/u;
@@ -16,7 +16,7 @@ const packageNameRE = /^@sourceacademy\/(.+?)-(.+)$/u;
  * an unprocessed format
  */
 export async function getRawPackages(gitRoot: string, maxDepth?: number) {
-  let packagesWithResolutionChanges: Set<string> | null = null;
+  let packagesWithResolutionChanges: string[] | null = null;
 
   // If there are lock file changes we need to set hasChanges to true for
   // that package even if that package's directory has no changes
@@ -43,7 +43,7 @@ export async function getRawPackages(gitRoot: string, maxDepth?: number) {
 
             output[packageJson.name] = {
               directory: currentDir,
-              hasChanges: packagesWithResolutionChanges?.has(packageJson.name) ?? hasChanges,
+              hasChanges: hasChanges || !!packagesWithResolutionChanges?.includes(packageJson.name),
               package: packageJson
             };
           } catch (error) {

.github/actions/src/lockfiles.ts

@@ -0,0 +1,173 @@
+import fs from 'fs/promises';
+import pathlib from 'path';
+import * as core from '@actions/core';
+import { getExecOutput } from '@actions/exec';
+import memoize from 'lodash/memoize.js';
+import { extractPkgsFromYarnLockV2 } from 'snyk-nodejs-lockfile-parser';
+import { gitRoot } from './gitRoot.js';
+
+const packageNameRE = /^(.+)@.+$/;
+
+/**
+ * Lockfile specifications come in the form of package_name@resolution, but
+ * we only want the package name. This function extracts that package name,
+ * accounting for the fact that package names might start with '@'
+ */
+export function extractPackageName(raw: string) {
+  const match = packageNameRE.exec(raw);
+  if (!match) {
+    throw new Error(`Invalid package name: ${raw}`);
+  }
+
+  return match[1];
+}
+
+/**
+ * Parses and lockfile's contents and extracts all the different dependencies and
+ * versions
+ */
+function processLockFileText(contents: string) {
+  const lockFile = extractPkgsFromYarnLockV2(contents);
+  const mappings = new Set<string>();
+  for (const [pkgSpecifier, { resolution }] of Object.entries(lockFile)) {
+    mappings.add(`${pkgSpecifier} -> ${resolution}`);
+  }
+  return mappings;
+}
+
+/**
+ * Retrieves the contents of the lockfile in the repo
+ */
+async function getCurrentLockFile() {
+  const lockFilePath = pathlib.join(gitRoot, 'yarn.lock');
+  const contents = await fs.readFile(lockFilePath, 'utf-8');
+  return processLockFileText(contents);
+}
+
+/**
+ * Retrieves the contents of the lockfile on the master branch
+ */
+async function getMasterLockFile() {
+  const { stdout, stderr, exitCode } = await getExecOutput(
+    'git',
+    [
+      '--no-pager',
+      'show',
+      'origin/master:yarn.lock'
+    ],
+    { silent: true }
+  );
+
+  if (exitCode !== 0) {
+    core.error(stderr);
+    throw new Error('git show exited with non-zero error-code');
+  }
+
+  return processLockFileText(stdout);
+}
+
+interface ResolutionSpec { pkgSpecifier: string, pkgName: string }
+
+/**
+ * Parsed output entry returned by `yarn why`
+ */
+interface YarnWhyOutput {
+  value: string;
+  children: {
+    [locator: string]: {
+      locator: string;
+      descriptor: string;
+    };
+  };
+}
+
+/**
+ * Run `yarn why <package_name>` to see why a package is included
+ * Don't use recursive (-R) since we want to build the graph ourselves
+ * @function
+ */
+const runYarnWhy = memoize(async (pkgName: string) => {
+  // Memoize the call so that we don't need to call yarn why multiple times for each package
+  const { stdout: output, exitCode, stderr } = await getExecOutput('yarn', ['why', pkgName, '--json'], { silent: true });
+  if (exitCode !== 0) {
+    core.error(stderr);
+    throw new Error(`yarn why for ${pkgName} failed!`);
+  }
+
+  return output.split('\n').reduce<YarnWhyOutput[]>((res, each) => {
+    each = each.trim();
+    if (each === '') return res;
+
+    const pkg = JSON.parse(each) as YarnWhyOutput;
+    return [...res, pkg];
+  }, []);
+})
+
+/**
+ * Determines the names of the packages that have changed versions
+ */
+export async function getPackagesWithResolutionChanges() {
+  const [currentLockFileMappings, masterLockFileMappings] = await Promise.all([
+    getCurrentLockFile(),
+    getMasterLockFile()
+  ]);
+
+  const changes = new Set(masterLockFileMappings);
+  for (const edge of currentLockFileMappings) {
+    changes.delete(edge);
+  }
+
+  const frontier: ResolutionSpec[] = [];
+  const changedDeps = new Set<string>();
+  for (const edge of changes) {
+    const [pkgSpecifier] = edge.split(' -> ');
+    changedDeps.add(pkgSpecifier);
+    frontier.push({ 
+      pkgSpecifier,
+      pkgName: extractPackageName(pkgSpecifier)
+    });
+  }
+
+  while (frontier.length > 0) {
+    const { pkgName, pkgSpecifier } = frontier.shift()!;
+
+    const reasons = await runYarnWhy(pkgName);
+    reasons.forEach(pkg => {
+      if (changedDeps.has(pkg.value)) {
+        // If we've already added this pkg specifier, don't need to explore it again
+        return;
+      }
+
+      const childrenSpecifiers = Object.values(pkg.children).map(({ descriptor }) => descriptor);
+      if (!childrenSpecifiers.includes(pkgSpecifier)) return;
+
+      frontier.push({ pkgSpecifier: pkg.value, pkgName: extractPackageName(pkg.value) });
+      changedDeps.add(pkg.value);
+    });
+  }
+
+  core.info('=== Summary of dirty monorepo packages ===\n');
+  const pkgsToRebuild = [...changedDeps].filter(pkgSpecifier => pkgSpecifier.includes('@workspace:'));
+  for (const pkgName of pkgsToRebuild) {
+    core.info(`- ${pkgName}`);
+  }
+
+  return pkgsToRebuild.map(extractPackageName);
+}
+
+/**
+ * Returns `true` if there are changes present in the given directory relative to
+ * the master branch\
+ * Used to determine, particularly for libraries, if running tests and tsc are necessary
+ */
+export const hasLockFileChanged = memoize(async () => {
+  const { exitCode } = await getExecOutput(
+    'git --no-pager diff --quiet origin/master -- yarn.lock',
+    [],
+    {
+      failOnStdErr: false,
+      ignoreReturnCode: true
+    }
+  );
+  return exitCode !== 0;
+});