Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial setup for concurrent workers #46

Merged
merged 10 commits into from
Dec 11, 2024
Merged

Initial setup for concurrent workers #46

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
20e9014
Initial setup for concurrent workers
erbesharat Dec 8, 2024
1652470
Add logs for timer
erbesharat Dec 8, 2024
4f09e12
Prettier lint
erbesharat Dec 8, 2024
27c8b6e
Fix worker script issues and await in api
erbesharat Dec 9, 2024
377a506
Lint
erbesharat Dec 9, 2024
5692d52
Fix worker in dev mode
erbesharat Dec 9, 2024
6af7709
Remove unneeded awaits
erbesharat Dec 9, 2024
25b0769
Lint and format
erbesharat Dec 9, 2024
9c76c2a
Ignore worker.js file in eslint
erbesharat Dec 9, 2024
2ebd17b
remove class from worker for readability
florianbgt Dec 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

29 changes: 18 additions & 11 deletions packages/cli/src/api/split.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import fs from "fs";
import path from "path";
import { z } from "zod";
import DependencyTreeManager from "../dependencyManager/dependencyManager";
import { Group } from "../dependencyManager/types";
import { cleanupOutputDir, createOutputDir } from "../helper/file";
import SplitRunner from "../splitRunner/splitRunner";
import { splitSchema } from "./helpers/validation";
import { z } from "zod";
import { Group } from "../dependencyManager/types";

export function split(payload: z.infer<typeof splitSchema>) {
console.time("split command");
Expand Down Expand Up @@ -33,15 +33,22 @@ export function split(payload: z.infer<typeof splitSchema>) {
const targetDir = path.dirname(payload.entrypointPath);
const annotationDirectory = path.join(outputDir, index.toString());

files.forEach((file) => {
const relativeFileNamePath = path.relative(targetDir, file.path);
const destinationPath = path.join(
annotationDirectory,
relativeFileNamePath,
);
fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
fs.writeFileSync(destinationPath, file.sourceCode, "utf8");
});
files
.then((files) => {
files.forEach((file) => {
const relativeFileNamePath = path.relative(targetDir, file.path);
const destinationPath = path.join(
annotationDirectory,
relativeFileNamePath,
);
fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
fs.writeFileSync(destinationPath, file.sourceCode, "utf8");
});
})
.catch((error) => {
console.error(error);
throw error;
});
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The response is sent before the worker or done with the jobs.
Should wait until the worker are done before sending the response

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated the code to wait for the response 🟢

});

// Store the processed annotations in the output directory
Expand Down
29 changes: 18 additions & 11 deletions packages/cli/src/commands/split.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import path from "path";
import fs from "fs";
import path from "path";
import DependencyTreeManager from "../dependencyManager/dependencyManager";
import { Group } from "../dependencyManager/types";
import { cleanupOutputDir, createOutputDir } from "../helper/file";
import SplitRunner from "../splitRunner/splitRunner";
import { Group } from "../dependencyManager/types";

export default function splitCommandHandler(
entrypointPath: string, // Path to the entrypoint file
Expand All @@ -27,15 +27,22 @@ export default function splitCommandHandler(
const targetDir = path.dirname(entrypointPath);
const annotationDirectory = path.join(outputDir, index.toString());

files.forEach((file) => {
const relativeFileNamePath = path.relative(targetDir, file.path);
const destinationPath = path.join(
annotationDirectory,
relativeFileNamePath,
);
fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
fs.writeFileSync(destinationPath, file.sourceCode, "utf8");
});
files
.then((files) => {
files.forEach((file) => {
const relativeFileNamePath = path.relative(targetDir, file.path);
const destinationPath = path.join(
annotationDirectory,
relativeFileNamePath,
);
fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
fs.writeFileSync(destinationPath, file.sourceCode, "utf8");
});
})
.catch((error) => {
console.error(error);
throw error;
});
});

// Store the processed annotations in the output directory
Expand Down
2 changes: 1 addition & 1 deletion packages/cli/src/languagesPlugins/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Group } from "../dependencyManager/types";
import Parser from "tree-sitter";
import { Group } from "../dependencyManager/types";

export interface DepImportIdentifier {
// Specific to each programing languages. Used by the language plugins.
Expand Down
229 changes: 21 additions & 208 deletions packages/cli/src/splitRunner/splitRunner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,231 +1,44 @@
import { Group } from "../dependencyManager/types";
import { removeIndexesFromSourceCode } from "../helper/file";
import path from "path";
import { Worker } from "worker_threads";
import DependencyTreeManager from "../dependencyManager/dependencyManager";
import { Group } from "../dependencyManager/types";
import { File } from "./types";
import Parser from "tree-sitter";
import assert from "assert";
import { getLanguagePlugin } from "../languagesPlugins";
import { DepExport } from "../languagesPlugins/types";

class SplitRunner {
private dependencyTreeManager: DependencyTreeManager;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's delete this class, become kind of useless now. We can do a simple method to replace it.
Maybe in another PR

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#48

private entrypointPath: string;
private group: Group;
private files: File[];

constructor(dependencyTreeManager: DependencyTreeManager, group: Group) {
this.dependencyTreeManager = dependencyTreeManager;
this.entrypointPath = dependencyTreeManager.dependencyTree.path;
this.group = group;
this.files = dependencyTreeManager.getFiles();
}

#removeAnnotationFromOtherGroups() {
this.files = this.files.map((file) => {
const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path);

const updatedSourceCode = languagePlugin.removeAnnotationFromOtherGroups(
file.sourceCode,
this.group,
);
return { ...file, sourceCode: updatedSourceCode };
});
}

#getExportMap() {
const exportMap = new Map<string, DepExport[]>();

this.files.forEach((file) => {
const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path);

const tree = languagePlugin.parser.parse(file.sourceCode);

const exports = languagePlugin.getExports(tree.rootNode);

exportMap.set(file.path, exports);
});

return exportMap;
}

#removeInvalidImportsAndUsages(exportMap: Map<string, DepExport[]>) {
this.files = this.files.map((file) => {
const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path);

const updatedSourceCode = languagePlugin.cleanupInvalidImports(
file.path,
file.sourceCode,
exportMap,
);

return { ...file, sourceCode: updatedSourceCode };
});
}

#removeUnusedImports() {
this.files = this.files.map((file) => {
const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path);
async run(): Promise<File[]> {
console.info(`Splitting group: ${this.group.name}`);
console.time("Total Splitting Time");

const updatedSourceCode = languagePlugin.cleanupUnusedImports(
file.path,
file.sourceCode,
);

return { ...file, sourceCode: updatedSourceCode };
const worker = new Worker(path.resolve(__dirname, "worker.js"), {
workerData: {
entrypointPath: this.dependencyTreeManager.dependencyTree.path,
group: this.group,
files: this.dependencyTreeManager.getFiles(),
},
});
}

#removeUnusedFiles() {
let fileRemoved = true;
while (fileRemoved) {
fileRemoved = false;

// We always want to keep the entrypoint file.
// It will never be imported anywhere, so we add it now.
const filesToKeep = new Set<string>();
filesToKeep.add(this.dependencyTreeManager.dependencyTree.path);

this.files.forEach((file) => {
const languagePlugin = getLanguagePlugin(
this.entrypointPath,
file.path,
);

const tree = languagePlugin.parser.parse(file.sourceCode);

const imports = languagePlugin.getImports(file.path, tree.rootNode);

imports.forEach((depImport) => {
if (depImport.isExternal || !depImport.source) {
// Ignore external dependencies
return;
}

filesToKeep.add(depImport.source);
});
});

const previousFilesLength = this.files.length;

this.files = this.files.filter((file) => {
return filesToKeep.has(file.path);
return new Promise<File[]>((resolve, reject) => {
worker.on("message", (updatedFiles: File[]) => {
console.timeEnd("Total Splitting Time");
resolve(updatedFiles);
});

if (this.files.length !== previousFilesLength) {
fileRemoved = true;
}
}
}

#removeUnusedExports(exportMap: Map<string, DepExport[]>) {
let exportDeleted = true;
while (exportDeleted) {
exportDeleted = false;

// const usedExportMap = new Map<string, Export>();

this.files = this.files.map((file) => {
const languagePlugin = getLanguagePlugin(
this.entrypointPath,
file.path,
);

const tree = languagePlugin.parser.parse(file.sourceCode);

const imports = languagePlugin.getImports(file.path, tree.rootNode);

imports.forEach((depImport) => {
if (depImport.isExternal || !depImport.source) {
// Ignore external dependencies
return;
}

// for each import, reconstruct the export map
const depExport = exportMap.get(depImport.source);
if (!depExport) {
throw new Error("Export not found");
}

// check named imports
});

return file;
});
}
// TODO
// Step 1, create variable to track which export is used
// Step 2, iterate over all file imports. If the import is used, mark the export as used
// Step 3, iterate over each file, and remove the unused exports

// Repeat above step until no more unused exports are found
assert(exportMap);
}

#removeErrors() {
this.files = this.files.map((file) => {
const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path);

const tree = languagePlugin.parser.parse(file.sourceCode);

const indexesToRemove: { startIndex: number; endIndex: number }[] = [];

const query = new Parser.Query(
languagePlugin.parser.getLanguage(),
"(ERROR) @error",
);
const errorCaptures = query.captures(tree.rootNode);
errorCaptures.forEach((capture) => {
indexesToRemove.push({
startIndex: capture.node.startIndex,
endIndex: capture.node.endIndex,
});
worker.on("error", reject);
worker.on("exit", (code) => {
if (code !== 0) {
reject(new Error(`Worker stopped with exit code ${code}`));
}
});

const updatedSourceCode = removeIndexesFromSourceCode(
file.sourceCode,
indexesToRemove,
);

return { ...file, sourceCode: updatedSourceCode };
});
}

run() {
console.info("\n");
console.time("Splitting");

console.time("remove annotation from other groups");
this.#removeAnnotationFromOtherGroups();
console.timeEnd("remove annotation from other groups");

console.time("Get export map");
const exportMap = this.#getExportMap();
console.timeEnd("Get export map");

console.time("Remove invalid imports and usages");
this.#removeInvalidImportsAndUsages(exportMap);
console.timeEnd("Remove invalid imports and usages");

console.time("Remove unused imports");
this.#removeUnusedImports();
console.timeEnd("Remove unused imports");

console.time("Remove unused files");
this.#removeUnusedFiles();
console.timeEnd("Remove unused files");

console.time("Remove unused exports");
this.#removeUnusedExports(exportMap);
console.timeEnd("Remove unused exports");

console.time("Remove errors");
this.#removeErrors();
console.timeEnd("Remove errors");

console.timeEnd("Splitting");

return this.files;
}
}

export default SplitRunner;
Loading
Loading