refactor: backend database migrations (@fehmer)

backend/__migration__/funboxConfig.ts

@@ -0,0 +1,78 @@
+import { Collection, Db } from "mongodb";
+import { Migration } from "./types";
+import type { DBConfig } from "../src/dal/config";
+
+export default class FunboxConfig implements Migration {
+  private configCollection!: Collection<DBConfig>;
+  private filter = {
+    $or: [
+      { "config.funbox": { $type: 2, $not: { $type: 4 } } },
+      { "config.customLayoutfluid": { $type: 2, $not: { $type: 4 } } },
+    ],
+  };
+  private collectionName = "configs";
+
+  name: string = "FunboxConfig";
+
+  async setup(db: Db): Promise<void> {
+    this.configCollection = db.collection(this.collectionName);
+  }
+  async getRemainingCount(): Promise<number> {
+    return this.configCollection.countDocuments(this.filter);
+  }
+
+  async migrate({ batchSize }: { batchSize: number }): Promise<number> {
+    await this.configCollection
+      .aggregate([
+        { $match: this.filter },
+        { $limit: batchSize },
+        //don't use projection
+        {
+          $addFields: {
+            "config.funbox": {
+              $cond: {
+                if: {
+                  $and: [
+                    { $ne: ["$config.funbox", null] },
+                    { $ne: [{ $type: "$config.funbox" }, "array"] },
+                  ],
+                },
+                // eslint-disable-next-line no-thenable
+                then: {
+                  $cond: {
+                    if: { $eq: ["$config.funbox", "none"] },
+                    // eslint-disable-next-line no-thenable
+                    then: [],
+                    else: { $split: ["$config.funbox", "#"] },
+                  },
+                },
+                else: "$config.funbox",
+              },
+            },
+            "config.customLayoutfluid": {
+              $cond: {
+                if: {
+                  $and: [
+                    { $ne: ["$config.customLayoutfluid", null] },
+                    { $ne: [{ $type: "$config.customLayoutfluid" }, "array"] },
+                  ],
+                },
+                // eslint-disable-next-line no-thenable
+                then: { $split: ["$config.customLayoutfluid", "#"] },
+                else: "$config.customLayoutfluid",
+              },
+            },
+          },
+        },
+        {
+          $merge: {
+            into: this.collectionName,
+            on: "_id",
+            whenMatched: "merge",
+          },
+        },
+      ])
+      .toArray();
+    return batchSize; //TODO hmmm....
+  }
+}

backend/__migration__/funboxResult.ts

@@ -0,0 +1,49 @@
+import { Collection, Db } from "mongodb";
+import { Migration } from "./types";
+import type { DBResult } from "../src/utils/result";
+
+export default class FunboxResult implements Migration {
+  private resultCollection!: Collection<DBResult>;
+  private filter = { funbox: { $type: 2, $not: { $type: 4 } } }; //string, not array of strings
+  private collectionName = "results";
+  name: string = "FunboxResult";
+
+  async setup(db: Db): Promise<void> {
+    this.resultCollection = db.collection(this.collectionName);
+  }
+  async getRemainingCount(): Promise<number> {
+    return this.resultCollection.countDocuments(this.filter);
+  }
+
+  async migrate({ batchSize }: { batchSize: number }): Promise<number> {
+    const pipeline = this.resultCollection.aggregate([
+      { $match: this.filter },
+      { $sort: { timestamp: 1 } },
+      { $limit: batchSize },
+      { $project: { _id: 1, timestamp: 1, funbox: 1 } },
+
+      {
+        $addFields: {
+          funbox: {
+            $cond: {
+              if: { $eq: ["$funbox", "none"] },
+              // eslint-disable-next-line no-thenable
+              then: [],
+              else: { $split: ["$funbox", "#"] },
+            },
+          },
+        },
+      },
+      {
+        $merge: {
+          into: this.collectionName,
+          on: "_id",
+          whenMatched: "merge",
+        },
+      },
+    ]);
+    await pipeline.toArray();
+
+    return batchSize; //TODO hmmm....
+  }
+}

backend/__migration__/index.ts

@@ -0,0 +1,151 @@
+import "dotenv/config";
+import * as DB from "../src/init/db";
+import { Db } from "mongodb";
+import readlineSync from "readline-sync";
+import funboxResult from "./funboxResult";
+import testActivity from "./testActivity";
+import { Migration } from "./types";
+
+const batchSize = 100_000;
+let appRunning = true;
+let db: Db | undefined;
+const migrations = {
+  //funboxConfig: new funboxConfig(), // not ready yet
+  funboxResult: new funboxResult(),
+  testActivity: new testActivity(),
+};
+
+const delay = 1_000;
+
+const sleep = (durationMs): Promise<void> => {
+  return new Promise((resolve) => setTimeout(resolve, durationMs));
+};
+
+process.on("SIGINT", () => {
+  console.log("\nshutting down...");
+  appRunning = false;
+});
+
+if (require.main === module) {
+  void main();
+}
+
+function getMigrationToRun(): Migration {
+  //read all files in files directory, then use readlint sync keyInSelect to select a migration to run
+  const migrationNames = Object.keys(migrations);
+  const selectedMigration = readlineSync.keyInSelect(
+    migrationNames,
+    "Select migration to run"
+  );
+  if (selectedMigration === -1) {
+    console.log("No migration selected");
+    process.exit(0);
+  }
+  const migrationName = migrationNames[selectedMigration];
+  console.log("Selected migration:", migrationName);
+
+  const migration = migrations[migrationName];
+  if (migration === undefined) {
+    throw new Error("Migration not found");
+  }
+  console.log("Migration found:", migration.name);
+  return migration;
+}
+
+async function main(): Promise<void> {
+  try {
+    console.log(
+      `Connecting to database ${process.env["DB_NAME"]} on ${process.env["DB_URI"]}...`
+    );
+
+    const migration = getMigrationToRun();
+
+    if (
+      !readlineSync.keyInYN(`Ready to start migration  ${migration.name} ?`)
+    ) {
+      appRunning = false;
+    }
+
+    if (appRunning) {
+      await DB.connect();
+      console.log("Connected to database");
+      db = DB.getDb();
+      if (db === undefined) {
+        throw Error("db connection failed");
+      }
+
+      console.log(`Running migration ${migration.name}`);
+
+      await migrate(migration);
+    }
+
+    console.log(`\nMigration ${appRunning ? "done" : "aborted"}.`);
+  } catch (e) {
+    console.log("error occured:", { e });
+  } finally {
+    await DB.close();
+  }
+}
+
+export async function migrate(migration: Migration): Promise<void> {
+  await migration.setup(db as Db);
+
+  const remainingCount = await migration.getRemainingCount();
+  if (remainingCount === 0) {
+    console.log("No documents to migrate.");
+    return;
+  } else {
+    console.log("Documents to migrate:", remainingCount);
+  }
+
+  console.log(
+    `Migrating ~${remainingCount} documents using batchSize=${batchSize}`
+  );
+
+  let count = 0;
+  const start = new Date().valueOf();
+
+  do {
+    const t0 = Date.now();
+
+    const migratedCount = await migration.migrate({ batchSize });
+
+    //progress tracker
+    count += migratedCount;
+    updateProgress(remainingCount, count, start, Date.now() - t0);
+    if (delay) {
+      await sleep(delay);
+    }
+  } while (remainingCount - count > 0 && appRunning);
+
+  if (appRunning) {
+    updateProgress(100, 100, start, 0);
+    const left = await migration.getRemainingCount();
+    if (left !== 0) {
+      console.log(
+        `After migration there are ${left} unmigrated documents left. You might want to run the migration again.`
+      );
+    }
+  }
+}
+function updateProgress(
+  all: number,
+  current: number,
+  start: number,
+  previousBatchSizeTime: number
+): void {
+  const percentage = (current / all) * 100;
+  const timeLeft = Math.round(
+    (((new Date().valueOf() - start) / percentage) * (100 - percentage)) / 1000
+  );
+
+  process.stdout.clearLine?.(0);
+  process.stdout.cursorTo?.(0);
+  process.stdout.write(
+    `Previous batch took ${Math.round(previousBatchSizeTime)}ms (~${
+      previousBatchSizeTime / batchSize
+    }ms per document) ${Math.round(
+      percentage
+    )}% done, estimated time left ${timeLeft} seconds.`
+  );
+}