Allow processing directories, reports may contain warnings

Author: rien

lib/src/dolos.ts

@@ -6,13 +6,11 @@ import { Result } from "./lib/util/result";
 import { csvParse, DSVRowString } from "d3-dsv";
 import * as path from "path";
 import { Tokenizer } from "./lib/tokenizer/tokenizer";
-import { default as fsWithCallbacks, constants } from "fs";
+import { constants, default as fsWithCallbacks } from "fs";
 import { spawnSync as spawn } from "child_process";
 import { tmpdir } from "os";
-import {
-  Language,
-  LanguagePicker
-} from "./lib/util/language";
+import { Language, LanguagePicker } from "./lib/util/language";
+
 const fs = fsWithCallbacks.promises;
 
 
@@ -31,8 +29,22 @@ export class Dolos {
   }
 
   private async fromDirectory(dirPath: string): Promise<Result<File[]>> {
-    const entries = await fs.readdir(dirPath, { withFileTypes: true });
-    const files = entries.filter(ent => ent.isFile()).map(ent => File.fromPath(ent.name));
+    const dirs = [dirPath];
+    const files = [];
+
+    let i = 0;
+
+    while(i < dirs.length) {
+      for (const entry of await fs.readdir(dirs[i], { withFileTypes: true })) {
+        if (entry.isDirectory()) {
+          dirs.push(path.join(dirs[i], entry.name));
+        } else if (entry.isFile()) {
+          files.push(File.fromPath(path.join(dirs[i], entry.name)));
+        }
+      }
+      i += 1;
+    }
+
     return await Result.all(files);
   }
 
@@ -46,10 +58,10 @@ export class Dolos {
         throw new Error(`Unzipping failed with exit status ${ status }, stderr: \n${stderr}`);
       }
       const infoPath = path.join(tmpDir, "info.csv");
-      if (await fs.access(infoPath, constants.R_OK).catch(() => false)) {
-        return await this.fromDirectory(tmpDir);
-      } else {
+      if (await fs.access(infoPath, constants.R_OK).then(() => true).catch(() => false)) {
         return await this.fromCSV(infoPath);
+      } else {
+        return await this.fromDirectory(tmpDir);
       }
     } finally {
       await fs.rm(tmpDir, { recursive: true });
@@ -121,13 +133,19 @@ export class Dolos {
       this.index = new Index(this.tokenizer, this.options);
     }
 
+    const warnings = [];
+    let filteredFiles;
     if (this.languageDetected) {
-      const filteredFiles = files.filter(file => this.language?.extensionMatches(file.path));
+      filteredFiles = files.filter(file => this.language?.extensionMatches(file.path));
       const diff = files.length - filteredFiles.length;
       if (diff > 0) {
-        console.warn(`Ignoring ${diff} files because the file extension did not match the detected language ${this.language!.name}. Specify the language explicitly to avoid this.`);
+        warnings.push(
+          `The language of the files was detected as ${this.language?.name} ` +
+          `but ${diff} files were ignored because they did not have a matching extension.`
+        );
       }
-      files = filteredFiles;
+    } else {
+      filteredFiles = files;
     }
 
     if (files.length < 2) {
@@ -138,6 +156,18 @@ export class Dolos {
         "be present in 100% of the files. This option does only" +
         "make sense when comparing more than two files.");
     }
-    return this.index.compareFiles(files, nameCandidate);
+
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const tokenizedFiles = filteredFiles.map(f => this.tokenizer!.tokenizeFile(f));
+    const fingerprints = await this.index.createMatches(tokenizedFiles);
+
+    return new Report(
+      this.options,
+      this.language,
+      tokenizedFiles,
+      fingerprints,
+      nameCandidate,
+      warnings
+    );
   }
 }

lib/src/lib/analyze/report.ts

@@ -10,6 +10,7 @@ type Hash = number;
 export interface Metadata extends DolosOptions {
   languageDetected: boolean;
   createdAt: string;
+  warnings: string[];
 }
 
 export class Report {
@@ -28,6 +29,7 @@ export class Report {
     public readonly files: TokenizedFile[],
     fingerprints: Map<Hash, SharedFingerprint>,
     name?: string,
+    public readonly warnings: string[] = [],
   ) {
     if (this.options.maxFingerprintCount != null) {
       this.kgramMaxFileOccurrences = this.options.maxFingerprintCount;
@@ -87,6 +89,7 @@ export class Report {
       createdAt: this.createdAt,
       language: this.language?.name ?? null,
       languageDetected: this.options.language == undefined,
+      warnings: this.warnings,
     };
   }
 }

lib/src/lib/util/language.ts

@@ -153,7 +153,7 @@ export class LanguagePicker {
 
     if (language == undefined) {
       throw new LanguageError(
-        `Could not detect language based on extension.`
+        "Could not detect language based on extension."
       );
     }
 

lib/src/test/dolos.test.ts

@@ -255,8 +255,9 @@ test("should read ZIP-files without info.csv", async t => {
   const report = await dolos.analyzePaths(["../samples/javascript/simple-dataset-no-csv.zip"]);
 
   t.is(4, report.files.length);
-  t.is(report.name, "simple-dataset");
+  t.is(report.name, "simple-dataset-no-csv");
   t.is(report.metadata()["reportName"], "simple-dataset-no-csv");
+  t.is(report.metadata()["warnings"].length, 1);
 
   const pairs = report.allPairs();
   t.is(6, pairs.length);
@@ -271,3 +272,19 @@ test("empty files should match 0%", async t => {
   t.is(0, pairs[0].overlap);
   t.is(0, pairs[0].longest);
 });
+
+test("should generate warning when not all files match detected language", async t => {
+  const dolos = new Dolos();
+
+  const report = await dolos.analyzePaths([
+    "../samples/javascript/sample.js",
+    "../samples/javascript/copied_function.js",
+    "../samples/java/Caesar.java"
+  ]);
+
+  t.is(report.metadata()["warnings"].length, 1);
+  t.is(2, report.files.length);
+
+  const pairs = report.allPairs();
+  t.is(1, pairs.length);
+});

lib/src/test/tokenizer.test.ts

@@ -53,6 +53,8 @@ test("language picker should throw an error for unknown extension", t => {
   t.throws(() => new LanguagePicker().detectLanguage([new File("unknown.extension", "")]));
 });
 
-test("language picker should throw an error for different languages", t => {
-  t.throws(() => new LanguagePicker().detectLanguage([new File("file.py", ""), new File("file.js", "")]));
+test("language picker should detect most common language", t => {
+  const files = [new File("file.py", ""), new File("otherfile.py", ""), new File("file.js", "")];
+  const detected = new LanguagePicker().detectLanguage(files);
+  t.deepEqual(detected.name, "python");
 });