|
| 1 | +import { readFiles, readPath } from "./reader.js"; |
| 2 | + |
| 3 | +import { ExtraInfo, File, Result } from "@dodona/dolos-core"; |
| 4 | +import { csvParse, DSVRowString } from "d3-dsv"; |
| 5 | + |
| 6 | +import { constants } from "node:fs"; |
| 7 | +import fs from "node:fs/promises"; |
| 8 | +import path from "node:path"; |
| 9 | +import { spawnSync as spawn } from "node:child_process"; |
| 10 | +import { tmpdir } from "node:os"; |
| 11 | + |
| 12 | +export class Dataset { |
| 13 | + constructor( |
| 14 | + public name: string, |
| 15 | + public files: File[], |
| 16 | + public ignore?: File) { |
| 17 | + } |
| 18 | + |
| 19 | + private static async fromDirectory(dirPath: string): Promise<Result<File[]>> { |
| 20 | + const dirs = [dirPath]; |
| 21 | + const files = []; |
| 22 | + |
| 23 | + let i = 0; |
| 24 | + |
| 25 | + while(i < dirs.length) { |
| 26 | + for (const entry of await fs.readdir(dirs[i], { withFileTypes: true })) { |
| 27 | + if (entry.isDirectory()) { |
| 28 | + dirs.push(path.join(dirs[i], entry.name)); |
| 29 | + } else if (entry.isFile()) { |
| 30 | + files.push(readPath(path.join(dirs[i], entry.name))); |
| 31 | + } |
| 32 | + } |
| 33 | + i += 1; |
| 34 | + } |
| 35 | + |
| 36 | + return await Result.all(files); |
| 37 | + } |
| 38 | + |
| 39 | + private static async setIgnoredFile(resolvedFiles: File[], ignore?: string): Promise<File | undefined> { |
| 40 | + const ignoredFiles = resolvedFiles.filter(file => file.extra?.ignored === "true"); |
| 41 | + if (ignoredFiles.length > 1) { |
| 42 | + throw new Error( |
| 43 | + "More than one file has the ignored field set to true. " + |
| 44 | + "Only one template/boilerplate code file is allowed at this moment." |
| 45 | + ); |
| 46 | + } |
| 47 | + else if (ignore) { |
| 48 | + return (await readPath(ignore)).ok(); |
| 49 | + } |
| 50 | + return ignoredFiles.length === 1 ? ignoredFiles[0] : undefined; |
| 51 | + } |
| 52 | + |
| 53 | + |
| 54 | + private static async fromZIP( |
| 55 | + zipPath: string, |
| 56 | + ignore?: string |
| 57 | + ): Promise<Dataset> { |
| 58 | + const tmpDir = await fs.mkdtemp(path.join(tmpdir(), "dolos-unzip-")); |
| 59 | + try { |
| 60 | + const { status, error, stderr } = spawn("unzip", [zipPath, "-d", tmpDir]); |
| 61 | + if (error) { |
| 62 | + throw error; |
| 63 | + } else if (status != 0) { |
| 64 | + throw new Error(`Unzipping failed with exit status ${ status }, stderr: \n${stderr}`); |
| 65 | + } |
| 66 | + const infoPath = path.join(tmpDir, "info.csv"); |
| 67 | + if (await fs.access(infoPath, constants.R_OK).then(() => true).catch(() => false)) { |
| 68 | + const dataset = await Dataset.fromCSV(infoPath, ignore); |
| 69 | + if (dataset) { |
| 70 | + dataset.name = path.basename(zipPath, ".zip"); |
| 71 | + return dataset; |
| 72 | + } |
| 73 | + else { |
| 74 | + throw new Error("Failed to process files"); |
| 75 | + } |
| 76 | + } else { |
| 77 | + const files = (await this.fromDirectory(tmpDir)).ok(); |
| 78 | + const ignoredFile = undefined; |
| 79 | + const nameCandidate = path.basename(zipPath, ".zip"); |
| 80 | + return new Dataset(nameCandidate, files, ignoredFile); |
| 81 | + } |
| 82 | + } finally { |
| 83 | + await fs.rm(tmpDir, { recursive: true }); |
| 84 | + } |
| 85 | + } |
| 86 | + |
| 87 | + |
| 88 | + private static async fromCSV( |
| 89 | + infoPath: string, |
| 90 | + ignore?: string |
| 91 | + ): Promise<Dataset> { |
| 92 | + const dirname = path.dirname(infoPath); |
| 93 | + try { |
| 94 | + const csv_files = csvParse((await fs.readFile(infoPath)).toString()) |
| 95 | + .map((row: DSVRowString) => ({ |
| 96 | + filename: row.filename as string, |
| 97 | + fullName: row.full_name as string, |
| 98 | + id: row.id as string, |
| 99 | + status: row.status as string, |
| 100 | + submissionID: row.submission_id as string, |
| 101 | + nameEN: row.name_en as string, |
| 102 | + nameNL: row.name_nl as string, |
| 103 | + exerciseID: row.exercise_id as string, |
| 104 | + createdAt: new Date(row.created_at as string), |
| 105 | + labels: row.label as string || row.labels as string, |
| 106 | + ignored: row.ignored as string |
| 107 | + })) |
| 108 | + .map((row: ExtraInfo) => readPath(path.join(dirname, row.filename), row)); |
| 109 | + const resolvedFiles = await Result.all(csv_files); |
| 110 | + const ignoredFile = await this.setIgnoredFile(resolvedFiles.ok(), ignore); |
| 111 | + const files = resolvedFiles.ok().filter(file => file.extra?.ignored !== "true"); |
| 112 | + const nameCandidate = path.dirname(infoPath).split(path.sep).pop() || "undefined"; |
| 113 | + return new Dataset(nameCandidate, files, ignoredFile); |
| 114 | + } catch(e) { |
| 115 | + throw new Error("The given '.csv'-file could not be opened"); |
| 116 | + } |
| 117 | + } |
| 118 | + |
| 119 | + |
| 120 | + public static async create(paths: string[], ignore?: string): Promise<Dataset> { |
| 121 | + let resolvedIgnoredFile = null; |
| 122 | + let resolvedFiles = null; |
| 123 | + let nameCandidate = "undefined"; |
| 124 | + |
| 125 | + if (paths.length == 1) { |
| 126 | + const inputFile = paths[0]; |
| 127 | + if (inputFile.toLowerCase().endsWith(".zip")) { |
| 128 | + return Dataset.fromZIP(inputFile, ignore); |
| 129 | + } else if (inputFile.toLowerCase().endsWith(".csv")) { |
| 130 | + return Dataset.fromCSV(inputFile, ignore); |
| 131 | + } else { |
| 132 | + throw new Error("You gave one input file, but it is not a CSV file or a ZIP archive."); |
| 133 | + } |
| 134 | + } else { |
| 135 | + resolvedFiles = (await readFiles(paths)).ok(); |
| 136 | + resolvedIgnoredFile = await this.setIgnoredFile(resolvedFiles, ignore); |
| 137 | + nameCandidate = path.basename(paths[0]) + " & " + path.basename(paths[1]); |
| 138 | + return new Dataset(nameCandidate, resolvedFiles, resolvedIgnoredFile); |
| 139 | + } |
| 140 | + } |
| 141 | +} |
0 commit comments