Add migration script to/from legacy history to generic history

Author: omkar-ethz

migrations/20251014092142-legacy-history-to-generic-history.js

@@ -0,0 +1,67 @@
+/* eslint-disable @typescript-eslint/no-require-imports */
+const convertObsoleteHistoryToGenericHistory =
+  require("../dist/datasets/utils/history.util").convertObsoleteHistoryToGenericHistory;
+
+const convertGenericHistoriesToObsoleteHistories =
+  require("../dist/datasets/utils/history.util").convertGenericHistoriesToObsoleteHistories;
+
+module.exports = {
+  /**
+   * @param db {import('mongodb').Db}
+   * @param client {import('mongodb').MongoClient}
+   * @returns {Promise<void>}
+   */
+  async up(db, client) {
+    for await (const dataset of db
+      .collection("Dataset")
+      .find({ history: { $exists: true, $type: "array", $ne: [] } })) {
+      console.log(
+        `Migrating history for dataset ${dataset._id}. Entries: ${dataset.history.length}`,
+      );
+      dataset.history.forEach(async (entry) => {
+        const genericHistory = convertObsoleteHistoryToGenericHistory(
+          entry,
+          dataset._id,
+        );
+        console.log(`Inserting history entry for dataset ${dataset._id}`);
+        result = await db.collection("History").insertOne(genericHistory);
+      });
+    }
+    await db.collection("Dataset").updateMany({}, { $unset: { history: "" } });
+  },
+
+  /**
+   * @param db {import('mongodb').Db}
+   * @param client {import('mongodb').MongoClient}
+   * @returns {Promise<void>}
+   */
+  async down(db, client) {
+    for await (const dataset of db.collection("Dataset").find({})) {
+      console.log(`Rolling back history for dataset ${dataset._id}`);
+      const genericHistories = await db
+        .collection("History")
+        .find(
+          { documentId: dataset._id, subsystem: "Dataset" },
+          { sort: { timestamp: "desc" } },
+        )
+        .toArray();
+      console.log(
+        `Found ${genericHistories.length} history entries for dataset ${dataset._id}`,
+      );
+      dataset.$clone = () => dataset; // Mock the $clone method, as this is not a mongoose document
+      const obsoleteHistories = convertGenericHistoriesToObsoleteHistories(
+        genericHistories,
+        dataset,
+      );
+      console.log(
+        `Inserting obsolete history entries for dataset ${dataset._id}`,
+      );
+      await db
+        .collection("Dataset")
+        .updateOne(
+          { _id: dataset._id },
+          { $set: { history: obsoleteHistories } },
+        );
+    }
+  },
+};

package.json

@@ -11,8 +11,8 @@
     "build": "nest build",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
     "format:check": "prettier --check \"src/**/*.ts\" \"test/**/*.ts\"",
-    "migrate:db:up": "node ./node_modules/migrate-mongo/bin/migrate-mongo.js up",
-    "migrate:db:down": "node ./node_modules/migrate-mongo/bin/migrate-mongo.js down",
+    "migrate:db:up": "npm run build && node ./node_modules/migrate-mongo/bin/migrate-mongo.js up",
+    "migrate:db:down": "npm run build && node ./node_modules/migrate-mongo/bin/migrate-mongo.js down",
     "migrate:db:status": "node ./node_modules/migrate-mongo/bin/migrate-mongo.js status",
     "start": "nest start",
     "start:dev": "nest start --watch",

src/datasets/utils/history.util.spec.ts

@@ -1,6 +1,8 @@
+import { HistoryClass } from "src/datasets/schemas/history.schema";
 import {
   convertGenericHistoriesToObsoleteHistories,
   convertGenericHistoryToObsoleteHistory,
+  convertObsoleteHistoryToGenericHistory,
 } from "./history.util";
 import {
   DatasetClass,
@@ -9,6 +11,63 @@ import {
 import { GenericHistory } from "../../common/schemas/generic-history.schema";
 
 describe("History Utility Functions", () => {
+  it("should convert obsolete history to generic history", () => {
+    const obsoleteHistory: HistoryClass = {
+      updatedAt: new Date("2023-10-01T12:00:00Z"),
+      updatedBy: "user123",
+      isPublished: {
+        currentValue: true,
+        previousValue: false,
+      },
+      datasetlifecycle: {
+        currentValue: {
+          publishedOn: new Date("2023-10-01T12:00:00Z"),
+          archivable: true,
+          retrievable: true,
+        },
+        previousValue: {
+          archivable: false,
+          retrievable: true,
+          publishable: false,
+          archiveRetentionTime: new Date("2031-10-01T12:00:00Z"),
+          dateOfPublishing: new Date("2024-10-01T12:00:00Z"),
+          isOnCentralDisk: true,
+          archiveStatusMessage: "datasetOnArchiveDisk",
+          retrieveStatusMessage: "",
+          retrieveIntegrityCheck: false,
+        },
+      },
+      _id: "",
+    };
+    const documentId = "pid123";
+    const genericHistory = convertObsoleteHistoryToGenericHistory(
+      obsoleteHistory,
+      documentId,
+    );
+
+    expect(genericHistory).toEqual({
+      subsystem: "Dataset",
+      documentId: "pid123",
+      user: "user123",
+      operation: "update",
+      timestamp: new Date("2023-10-01T12:00:00Z"),
+      before: {
+        isPublished: false,
+        datasetlifecycle: {
+          publishedOn: undefined,
+          archivable: false,
+        },
+      },
+      after: {
+        datasetlifecycle: {
+          publishedOn: new Date("2023-10-01T12:00:00Z"),
+          archivable: true,
+        },
+        isPublished: true,
+      },
+    });
+  });
+
   it("should convert generic history to obsolete history", () => {
     const genericHistory: GenericHistory = {
       subsystem: "Dataset",

src/datasets/utils/history.util.ts

@@ -7,6 +7,58 @@ import {
 
 const IGNORE_FIELDS = ["updatedAt", "updatedBy", "_id"];
 
+export function convertObsoleteHistoryToGenericHistory(
+  history: HistoryClass,
+  documentId: string,
+): GenericHistory {
+  const result: GenericHistory = {
+    subsystem: "Dataset",
+    documentId: documentId,
+    user: history.updatedBy,
+    operation: "update",
+    timestamp: history.updatedAt,
+    before: {},
+    after: {},
+  };
+  const changeList = Object.keys(history).filter(
+    (key) => !IGNORE_FIELDS.includes(key),
+  );
+  for (const key of changeList) {
+    if (
+      !history[key] ||
+      !history[key].hasOwnProperty("previousValue") ||
+      !history[key].hasOwnProperty("currentValue")
+    ) {
+      continue;
+    }
+    const fieldChange = history[key] as {
+      previousValue: unknown;
+      currentValue: unknown;
+    };
+    if (key === "datasetlifecycle") {
+      const currentValue = fieldChange.currentValue as Record<string, unknown>;
+      const previousValue = fieldChange.previousValue as Record<
+        string,
+        unknown
+      >;
+      // only retain the intersection of keys in currentValue and previousValue and whose value has changed. drop all others
+      const prunedPreviousValue: Record<string, unknown> = {};
+      const prunedCurrentValue: Record<string, unknown> = {};
+      for (const subKey of Object.keys(currentValue)) {
+        if (currentValue[subKey] !== previousValue[subKey]) {
+          prunedPreviousValue[subKey] = previousValue[subKey];
+          prunedCurrentValue[subKey] = currentValue[subKey];
+        }
+      }
+      fieldChange.previousValue = prunedPreviousValue;
+      fieldChange.currentValue = prunedCurrentValue;
+    }
+    result.before![key] = fieldChange.previousValue;
+    result.after![key] = fieldChange.currentValue;
+  }
+  return result;
+}
+
 // Given a dataset snapshot and a history entry, reconstruct the obsolete history entry
 export function convertGenericHistoryToObsoleteHistory(
   history: GenericHistory,