From b4e9c3d36f019a464eebbae01225ab0b69262a68 Mon Sep 17 00:00:00 2001
From: Spencer Bliven <spencer.bliven@gmail.com>
Date: Mon, 6 Oct 2025 16:45:35 +0200
Subject: [PATCH 01/26] fix: provide datasets to all actions (#2253)

* Load context.datasets globally for all actions

* Fix exception handling during actions

* Improve job-template-simplified.html email

* Update tests

{{datasets}} is now empty rather than undefined/error for jobs with not datasetList.
---
 .../job-template-simplified.html              |  22 ++--
 src/common/utils.ts                           |   6 +
 src/config/job-config/actions/actionutils.ts  |  50 ++++----
 .../actions/switchaction/switchaction.spec.ts | 117 +++++++++++++-----
 .../actions/switchaction/switchaction.ts      |  35 +-----
 .../validateaction/validateaction.service.ts  |   5 +-
 .../validateaction/validateaction.spec.ts     |  23 ++--
 .../actions/validateaction/validateaction.ts  |  31 +++--
 src/config/job-config/jobconfig.interface.ts  |   4 +-
 src/jobs/jobs.controller.utils.ts             |  17 ++-
 10 files changed, 172 insertions(+), 138 deletions(-)
diff --git a/src/common/email-templates/job-template-simplified.html b/src/common/email-templates/job-template-simplified.html
index 3aa422ba6..6dedf8f3c 100644
--- a/src/common/email-templates/job-template-simplified.html
+++ b/src/common/email-templates/job-template-simplified.html
@@ -1,4 +1,8 @@
-{{#with job}}
+{{!--
+  Minimal job email template.
+
+  This is primarily meant as a starting point for customization.
+--}}
 <html>
 <head>
     <style type="text/css">
@@ -57,19 +61,18 @@
 <body>
     <div class="container">
         <div>
-            Your {{type}} job has been submitted and will be processed as soon as possible.<br>
-            You will be notified by email as soon as the job is completed.
+            Your {{job.type}} job has been submitted. It currently has status "{{job.statusCode}}".
         </div>
         <div class="job-id-container">
-            <b>Job id:</b> {{id}}
+            <b>Job id:</b> {{job.id}}
         </div>
-        {{#if jobParams.datasetIds}}
-        <p><b>Job will be perfomed on the following dataset(s):</b></p>
+        {{#if datasets}}
+        <p><b>The job will be perfomed on the following dataset(s):</b></p>
         <table>
-            {{#each jobParams.datasetIds}}
+            {{#each datasets}}
             <tr style="background-color: lightblue;">
                 <td class="key">
-                    {{this}}
+                    {{pid}}
                 </td>
             </tr>
             {{/each}}
@@ -83,5 +86,4 @@
         </div>
     </div>
 </body>
-</html>
-{{/with}}
+</html>
\ No newline at end of file
diff --git a/src/common/utils.ts b/src/common/utils.ts
index 6cf342db2..e590844cb 100644
--- a/src/common/utils.ts
+++ b/src/common/utils.ts
@@ -1199,6 +1199,12 @@ export function oneOrMore<T>(x: T[] | T): T[] {
   return Array.isArray(x) ? x : [x];
 }
 
+/**
+ * Make a single property K of T optional
+ */
+export type MakeOptional<T, K extends keyof T> = Omit<T, K> &
+  Partial<Pick<T, K>>;
+
 /**
  * Helper function to generate HttpExceptions
  * @param message error message
diff --git a/src/config/job-config/actions/actionutils.ts b/src/config/job-config/actions/actionutils.ts
index 7724947d7..74ee33eba 100644
--- a/src/config/job-config/actions/actionutils.ts
+++ b/src/config/job-config/actions/actionutils.ts
@@ -9,7 +9,8 @@ import { DatasetsService } from "src/datasets/datasets.service";
 import { DatasetClass } from "src/datasets/schemas/dataset.schema";
 import { DatasetListDto } from "src/jobs/dto/dataset-list.dto";
 import { JobParams } from "src/jobs/types/job-types.enum";
-import { JobTemplateContext } from "../jobconfig.interface";
+import { JobDto, JobValidateContext } from "../jobconfig.interface";
+import { MakeOptional } from "src/common/utils";
 
 export type JSONData = JSONPathOptions["json"];
 
@@ -54,10 +55,10 @@ export function toObject(json: JSONData | HasToObject): JSONData {
  * @returns The list of datasets, which are also stored as context.datasets
  */
 
-export async function loadDatasets(
+export async function loadDatasets<T extends JobDto>(
   datasetsService: DatasetsService,
-  context: JobTemplateContext,
-): Promise<DatasetClass[]> {
+  context: MakeOptional<JobValidateContext<T>, "datasets">,
+): Promise<JobValidateContext<T>> {
   if (!context.datasets) {
     // Require datasetList
     let datasetList: DatasetListDto[] = [];
@@ -77,32 +78,35 @@ export async function loadDatasets(
         JobParams.DatasetList
       ] as DatasetListDto[];
     } else {
-      throw makeHttpException(
-        `'jobParams.${JobParams.DatasetList}' is required.`,
-      );
+      datasetList = [];
     }
+    if (datasetList.length > 0) {
+      const datasetIds = datasetList.map((x) => x.pid);
 
-    const datasetIds = datasetList.map((x) => x.pid);
-
-    // Load linked datasets
-    const filter = {
-      where: {
-        pid: {
-          $in: datasetIds,
+      // Load linked datasets
+      const filter = {
+        where: {
+          pid: {
+            $in: datasetIds,
+          },
         },
-      },
-    };
+      };
 
-    const result = await datasetsService.findAll(filter);
-    if (result.length != datasetIds.length) {
-      throw new NotFoundException(
-        `Unable to get a dataset. (${JSON.stringify(datasetIds)})`,
-      );
+      const result = await datasetsService.findAll(filter);
+      if (result.length != datasetIds.length) {
+        throw new NotFoundException(
+          `Unable to get a dataset. (${JSON.stringify(datasetIds)})`,
+        );
+      }
+      context.datasets = result.map(toObject) as DatasetClass[];
+    } else {
+      context.datasets = [];
     }
-    context.datasets = result.map(toObject) as DatasetClass[];
   }
-  return context.datasets;
+  // Destructuring only necessary for type checking
+  return { ...context, datasets: context.datasets };
 }
+
 /**
  * Resolves DatasetsService from a moduleRef.
  *
diff --git a/src/config/job-config/actions/switchaction/switchaction.spec.ts b/src/config/job-config/actions/switchaction/switchaction.spec.ts
index 5c4f2a728..56653c96f 100644
--- a/src/config/job-config/actions/switchaction/switchaction.spec.ts
+++ b/src/config/job-config/actions/switchaction/switchaction.spec.ts
@@ -202,7 +202,12 @@ describe("SwitchAction", () => {
         },
       };
 
-      const context = { request: jobDto, job: jobDto as JobClass, env: {} };
+      const context = {
+        request: jobDto,
+        job: jobDto as JobClass,
+        datasets: [],
+        env: {},
+      };
       await expect(action.validate(context)).resolves.toBeUndefined();
       await expect(action.perform(context)).resolves.toBeUndefined();
 
@@ -217,7 +222,12 @@ describe("SwitchAction", () => {
         },
       };
 
-      const context = { request: jobDto, job: jobDto as JobClass, env: {} };
+      const context = {
+        request: jobDto,
+        job: jobDto as JobClass,
+        datasets: [],
+        env: {},
+      };
       await expect(action.validate(context)).resolves.toBeUndefined();
       await expect(action.perform(context)).resolves.toBeUndefined();
 
@@ -232,7 +242,12 @@ describe("SwitchAction", () => {
         },
       };
 
-      const context = { request: jobDto, job: jobDto as JobClass, env: {} };
+      const context = {
+        request: jobDto,
+        job: jobDto as JobClass,
+        datasets: [],
+        env: {},
+      };
       await expect(action.validate(context)).resolves.toBeUndefined();
       await expect(action.perform(context)).resolves.toBeUndefined();
 
@@ -247,7 +262,12 @@ describe("SwitchAction", () => {
         },
       };
 
-      const context = { request: jobDto, job: jobDto as JobClass, env: {} };
+      const context = {
+        request: jobDto,
+        job: jobDto as JobClass,
+        datasets: [],
+        env: {},
+      };
       await expect(action.validate(context)).resolves.toBeUndefined();
       await expect(action.perform(context)).resolves.toBeUndefined();
 
@@ -262,7 +282,12 @@ describe("SwitchAction", () => {
         },
       };
 
-      const context = { request: jobDto, job: jobDto as JobClass, env: {} };
+      const context = {
+        request: jobDto,
+        job: jobDto as JobClass,
+        datasets: [],
+        env: {},
+      };
       await expect(action.validate(context)).resolves.toBeUndefined();
       await expect(action.perform(context)).resolves.toBeUndefined();
 
@@ -277,7 +302,12 @@ describe("SwitchAction", () => {
         },
       };
 
-      const context = { request: jobDto, job: jobDto as JobClass, env: {} };
+      const context = {
+        request: jobDto,
+        job: jobDto as JobClass,
+        datasets: [],
+        env: {},
+      };
       await expect(action.validate(context)).resolves.toBeUndefined();
       await expect(action.perform(context)).resolves.toBeUndefined();
 
@@ -336,30 +366,30 @@ describe("SwitchAction", () => {
       expect(action["validate"]).toBeDefined();
     });
 
-    it("should fail without a dataset", async () => {
-      findAll.mockResolvedValue([mockDataset]);
+    it("should call default without a dataset", async () => {
+      // Not specifying any datasets evaluates to undefined, which matches the default case
+      const datasets = [mockDataset];
+      findAll.mockResolvedValue(datasets);
       const jobDto: CreateJobDto = {
         type: "test",
         jobParams: {},
       };
 
-      const context = { request: jobDto, job: jobDto as JobClass, env: {} };
-      await expect(action.validate(context))
-        .rejects.toThrow
-        //"'jobParams.datasetList' is required.",
-        ();
-      await expect(action.perform(context))
-        .rejects.toThrow
-        //"'jobParams.datasetList' is required.",
-        ();
-      expect(caseString.validate).not.toHaveBeenCalled();
-      expect(caseString.perform).not.toHaveBeenCalled();
-      expect(caseDefault.validate).not.toHaveBeenCalled();
-      expect(caseDefault.perform).not.toHaveBeenCalled();
+      const context = {
+        request: jobDto,
+        job: jobDto as JobClass,
+        datasets: [],
+        env: {},
+      };
+      await expect(action.validate(context)).resolves.toBeUndefined();
+      await expect(action.perform(context)).resolves.toBeUndefined();
+
+      expectCalled(caseDefault);
     });
 
     it("should match archivable==true", async () => {
-      findAll.mockResolvedValue([mockDataset]);
+      const datasets = [mockDataset];
+      findAll.mockResolvedValue(datasets);
       const jobDto: CreateJobDto = {
         type: "test",
         jobParams: {
@@ -372,7 +402,12 @@ describe("SwitchAction", () => {
         },
       };
 
-      const context = { request: jobDto, job: jobDto as JobClass, env: {} };
+      const context = {
+        request: jobDto,
+        job: jobDto as JobClass,
+        datasets: datasets,
+        env: {},
+      };
       await expect(action.validate(context)).resolves.toBeUndefined();
       await expect(action.perform(context)).resolves.toBeUndefined();
 
@@ -380,7 +415,7 @@ describe("SwitchAction", () => {
     });
 
     it("shouldn't match archivable==false", async () => {
-      findAll.mockResolvedValue([
+      const datasets = [
         {
           ...mockDataset,
           datasetlifecycle: {
@@ -388,7 +423,8 @@ describe("SwitchAction", () => {
             archivable: false,
           },
         },
-      ]);
+      ];
+      findAll.mockResolvedValue(datasets);
       const jobDto: CreateJobDto = {
         type: "test",
         jobParams: {
@@ -401,7 +437,12 @@ describe("SwitchAction", () => {
         },
       };
 
-      const context = { request: jobDto, job: jobDto as JobClass, env: {} };
+      const context = {
+        request: jobDto,
+        job: jobDto as JobClass,
+        datasets: datasets,
+        env: {},
+      };
       await expect(action.validate(context)).resolves.toBeUndefined();
       await expect(action.perform(context)).resolves.toBeUndefined();
 
@@ -409,13 +450,14 @@ describe("SwitchAction", () => {
     });
 
     it("should pass with multiple consistent datasets", async () => {
-      findAll.mockResolvedValue([
+      const datasets = [
         mockDataset,
         {
           ...mockDataset,
           pid: "otherPid",
         },
-      ]);
+      ];
+      findAll.mockResolvedValue(datasets);
       const jobDto: CreateJobDto = {
         type: "test",
         jobParams: {
@@ -432,7 +474,12 @@ describe("SwitchAction", () => {
         },
       };
 
-      const context = { request: jobDto, job: jobDto as JobClass, env: {} };
+      const context = {
+        request: jobDto,
+        job: jobDto as JobClass,
+        datasets: datasets,
+        env: {},
+      };
       await expect(action.validate(context)).resolves.toBeUndefined();
       await expect(action.perform(context)).resolves.toBeUndefined();
 
@@ -440,7 +487,7 @@ describe("SwitchAction", () => {
     });
 
     it("should fail with ambiguous datasets", async () => {
-      findAll.mockResolvedValue([
+      const datasets = [
         mockDataset,
         {
           ...mockDataset,
@@ -450,7 +497,8 @@ describe("SwitchAction", () => {
             archivable: false,
           },
         },
-      ]);
+      ];
+      findAll.mockResolvedValue(datasets);
       const jobDto: CreateJobDto = {
         type: "test",
         jobParams: {
@@ -467,7 +515,12 @@ describe("SwitchAction", () => {
         },
       };
 
-      const context = { request: jobDto, job: jobDto as JobClass, env: {} };
+      const context = {
+        request: jobDto,
+        job: jobDto as JobClass,
+        datasets: datasets,
+        env: {},
+      };
       await expect(action.validate(context)).rejects.toThrow(
         "Ambiguous value for 'datasets[*].datasetlifecycle.archivable' (2 distinct results).'",
       );
diff --git a/src/config/job-config/actions/switchaction/switchaction.ts b/src/config/job-config/actions/switchaction/switchaction.ts
index 6c2dac783..7927ee10c 100644
--- a/src/config/job-config/actions/switchaction/switchaction.ts
+++ b/src/config/job-config/actions/switchaction/switchaction.ts
@@ -18,15 +18,9 @@ import {
   SwitchPhase,
 } from "./switchaction.interface";
 import { ModuleRef } from "@nestjs/core";
-import {
-  JSONData,
-  loadDatasets,
-  resolveDatasetService,
-  toObject,
-} from "../actionutils";
+import { JSONData } from "../actionutils";
 import { makeHttpException } from "src/common/utils";
 import { HttpStatus, Logger } from "@nestjs/common";
-import { DatasetClass } from "src/datasets/schemas/dataset.schema";
 
 /**
  * A Case gets matched against some target property.
@@ -232,31 +226,6 @@ export class SwitchJobAction<Dto extends JobDto> implements JobAction<Dto> {
     return creators;
   }
 
-  /**
-   * Load context.datasets if needed
-   *
-   * Throws an HTTP exception if no datasets are associated with this job. Datasets are
-   * never available during the validate phase of an update job.
-   * @param context Current job context
-   */
-  async loadDatasets(context: JobTemplateContext): Promise<void> {
-    if (context.datasets !== undefined) {
-      return;
-    }
-
-    // Guess if we need to load datasets
-    const needDatasets = this.property.includes("datasets");
-    if (!needDatasets) {
-      return;
-    }
-
-    const datasetsService = await resolveDatasetService(this.moduleRef);
-    const datasets = await loadDatasets(datasetsService, context);
-
-    // flatten mongo documents to JSON objects
-    context.datasets = datasets.map(toObject) as DatasetClass[];
-  }
-
   /**
    * Validate the current request
    * @param dto Job DTO
@@ -265,7 +234,6 @@ export class SwitchJobAction<Dto extends JobDto> implements JobAction<Dto> {
     if (this.phase !== SwitchPhase.Validate && this.phase !== SwitchPhase.All) {
       return;
     }
-    await this.loadDatasets(context);
     const actions = await this.resolveActions(context);
     return await validateActions(actions, context);
   }
@@ -274,7 +242,6 @@ export class SwitchJobAction<Dto extends JobDto> implements JobAction<Dto> {
     if (this.phase !== SwitchPhase.Perform && this.phase !== SwitchPhase.All) {
       return;
     }
-    await this.loadDatasets(context);
     const actions = await this.resolveActions(context);
     return await performActions(actions, context);
   }
diff --git a/src/config/job-config/actions/validateaction/validateaction.service.ts b/src/config/job-config/actions/validateaction/validateaction.service.ts
index 1417171e4..e853ba533 100644
--- a/src/config/job-config/actions/validateaction/validateaction.service.ts
+++ b/src/config/job-config/actions/validateaction/validateaction.service.ts
@@ -10,7 +10,6 @@ import {
   isValidateJobActionOptions,
 } from "./validateaction.interface";
 import { CreateJobDto } from "src/jobs/dto/create-job.dto";
-import { ModuleRef } from "@nestjs/core";
 
 @Injectable()
 export class ValidateJobActionCreator implements JobActionCreator<JobDto> {
@@ -30,7 +29,7 @@ export class ValidateJobActionCreator implements JobActionCreator<JobDto> {
 export class ValidateCreateJobActionCreator
   implements JobActionCreator<CreateJobDto>
 {
-  constructor(private moduleRef: ModuleRef) {}
+  constructor() {}
 
   public create<Options extends JobActionOptions>(options: Options) {
     if (!isValidateCreateJobActionOptions(options)) {
@@ -38,6 +37,6 @@ export class ValidateCreateJobActionCreator
         `Invalid options for validate action: ${JSON.stringify(options)}`,
       );
     }
-    return new ValidateCreateJobAction(this.moduleRef, options);
+    return new ValidateCreateJobAction(options);
   }
 }
diff --git a/src/config/job-config/actions/validateaction/validateaction.spec.ts b/src/config/job-config/actions/validateaction/validateaction.spec.ts
index 7f6b40a7c..a326d2472 100644
--- a/src/config/job-config/actions/validateaction/validateaction.spec.ts
+++ b/src/config/job-config/actions/validateaction/validateaction.spec.ts
@@ -115,7 +115,7 @@ describe("ValidateAction", () => {
         },
       };
 
-      const context = { request: dto, env: {} };
+      const context = { request: dto, env: {}, datasets: [] };
       await expect(action.validate(context)).resolves.toBeUndefined();
     });
 
@@ -130,7 +130,7 @@ describe("ValidateAction", () => {
         },
       };
 
-      const context = { request: dto, env: {} };
+      const context = { request: dto, env: {}, datasets: [] };
       await expect(action.validate(context)).rejects.toThrow(
         "Invalid request. Invalid value for 'jobParams'",
       );
@@ -147,7 +147,7 @@ describe("ValidateAction", () => {
         },
       };
 
-      const context = { request: dto, env: {} };
+      const context = { request: dto, env: {}, datasets: [] };
       await expect(action.validate(context)).rejects.toThrow(
         "Invalid request. Invalid value for 'jobParams.stringVal",
       );
@@ -164,7 +164,7 @@ describe("ValidateAction", () => {
         },
       };
 
-      const context = { request: dto, env: {} };
+      const context = { request: dto, env: {}, datasets: [] };
       await expect(action.validate(context)).rejects.toThrow(
         "Invalid request. Invalid value for 'jobParams.numberVal'",
       );
@@ -181,7 +181,7 @@ describe("ValidateAction", () => {
         },
       };
 
-      const context = { request: dto, env: {} };
+      const context = { request: dto, env: {}, datasets: [] };
       await expect(action.validate(context)).rejects.toThrow(
         "Invalid request. Requires 'jobParams.requiredArray[*]'",
       );
@@ -197,7 +197,7 @@ describe("ValidateAction", () => {
           requiredArray: [],
         },
       };
-      const context = { request: dto, env: {} };
+      const context = { request: dto, env: {}, datasets: [] };
       await expect(action.validate(context)).rejects.toThrow(
         "Invalid request. Requires 'jobParams.requiredArray[*]'",
       );
@@ -214,7 +214,7 @@ describe("ValidateAction", () => {
         },
       };
 
-      const context = { request: dto, env: {} };
+      const context = { request: dto, env: {}, datasets: [] };
       await expect(action.validate(context)).rejects.toThrow(
         "Invalid request. Invalid value for 'jobParams.requiredArray[*]'",
       );
@@ -267,9 +267,6 @@ describe("ValidateAction", () => {
     it("should be configured successfully", async () => {
       expect(action).toBeDefined();
       expect(action["datasets"]).toBeDefined();
-      expect(action["moduleRef"]).toBeDefined();
-      const datasetsService = action["moduleRef"].resolve(DatasetsService);
-      expect(datasetsService).toBeDefined();
     });
 
     it("should fail without a dataset", async () => {
@@ -278,7 +275,7 @@ describe("ValidateAction", () => {
         jobParams: {},
       };
 
-      const context = { request: dto, env: {} };
+      const context = { request: dto, env: {}, datasets: [] };
       await expect(action.validate(context)).rejects.toThrow(
         "'jobParams.datasetList' is required.",
       );
@@ -289,7 +286,7 @@ describe("ValidateAction", () => {
         ...mockCreateDto,
       };
 
-      const context = { request: dto, env: {} };
+      const context = { request: dto, env: {}, datasets: [mockDataset] };
       await expect(action.validate(context)).resolves.toBeUndefined();
     });
 
@@ -303,7 +300,7 @@ describe("ValidateAction", () => {
       unarchivableDataset.datasetlifecycle.archivable = false;
 
       findAll.mockResolvedValueOnce([unarchivableDataset]);
-      const context = { request: dto, env: {} };
+      const context = { request: dto, env: {}, datasets: [mockDataset] };
       await expect(action.validate(context)).rejects.toThrow(
         "Invalid request. Invalid value for 'datasetlifecycle.archivable'",
       );
diff --git a/src/config/job-config/actions/validateaction/validateaction.ts b/src/config/job-config/actions/validateaction/validateaction.ts
index 4114c222d..80093537d 100644
--- a/src/config/job-config/actions/validateaction/validateaction.ts
+++ b/src/config/job-config/actions/validateaction/validateaction.ts
@@ -12,15 +12,9 @@ import {
   ValidateJobActionOptions,
 } from "./validateaction.interface";
 import { CreateJobDto } from "src/jobs/dto/create-job.dto";
-import { ModuleRef } from "@nestjs/core";
-import {
-  toObject,
-  resolveDatasetService,
-  loadDatasets,
-  HasToObject,
-  JSONData,
-} from "../actionutils";
+import { toObject, HasToObject, JSONData } from "../actionutils";
 import { makeHttpException } from "src/common/utils";
+import { JobParams } from "src/jobs/types/job-types.enum";
 
 /**
  * Validates the job for the presence of required fields. Can also check types or
@@ -177,11 +171,7 @@ export class ValidateJobAction<T extends JobDto> implements JobAction<T> {
 export class ValidateCreateJobAction extends ValidateJobAction<CreateJobDto> {
   private datasets?: Record<string, ValidateFunction>;
 
-  constructor(
-    private moduleRef: ModuleRef,
-    options: ValidateCreateJobActionOptions,
-    ajv?: Ajv,
-  ) {
+  constructor(options: ValidateCreateJobActionOptions, ajv?: Ajv) {
     ajv =
       ajv ||
       new Ajv({
@@ -204,16 +194,23 @@ export class ValidateCreateJobAction extends ValidateJobAction<CreateJobDto> {
   async validate(context: JobValidateContext<CreateJobDto>): Promise<void> {
     // Validate this.requests
     await super.validate(context);
+
+    // Validate this.datasets. Requires datasets to be present.
     if (!this.datasets) {
       return;
     }
 
-    // Validate this.datasets
-    const datasetsService = await resolveDatasetService(this.moduleRef);
-    const datasets = await loadDatasets(datasetsService, context);
+    if (!context.datasets || context.datasets.length == 0) {
+      throw makeHttpException(
+        `'jobParams.${JobParams.DatasetList}' is required.`,
+      );
+    }
 
+    // Validate this.datasets
     await Promise.all(
-      datasets.map((dataset) => this.validateJson(dataset, this.datasets!)),
+      context.datasets.map((dataset) =>
+        this.validateJson(dataset, this.datasets!),
+      ),
     );
   }
 }
diff --git a/src/config/job-config/jobconfig.interface.ts b/src/config/job-config/jobconfig.interface.ts
index 7d3bd49b2..d0213b8f2 100644
--- a/src/config/job-config/jobconfig.interface.ts
+++ b/src/config/job-config/jobconfig.interface.ts
@@ -51,7 +51,7 @@ export interface JobOperationOptions {
 export interface JobValidateContext<DtoType extends JobDto> {
   request: DtoType;
   job?: JobClass; // Either the previous value (validate) or the new value (perform)
-  datasets?: DatasetClass[]; // Should be set lazily when needed
+  datasets: DatasetClass[]; // Should be set lazily when needed
   env: Record<string, string | undefined>;
 }
 /**
@@ -153,7 +153,7 @@ export async function performActions<DtoType extends JobDto>(
       if (err instanceof HttpException) {
         throw err;
       }
-      makeHttpException(
+      throw makeHttpException(
         `Invalid job input. Job '${context.job.type}' unable to perform action '${action.getActionType()}' due to ${err}`,
       );
     });
diff --git a/src/jobs/jobs.controller.utils.ts b/src/jobs/jobs.controller.utils.ts
index 205340590..b4c3f9d8c 100644
--- a/src/jobs/jobs.controller.utils.ts
+++ b/src/jobs/jobs.controller.utils.ts
@@ -41,6 +41,7 @@ import {
   PartialOutputWithJobIdDto,
 } from "./dto/output-job-v4.dto";
 import { toObject } from "src/config/job-config/actions/actionutils";
+import { loadDatasets } from "src/config/job-config/actions/actionutils";
 
 @Injectable()
 export class JobsControllerUtils {
@@ -652,12 +653,16 @@ export class JobsControllerUtils {
     // Allow actions to validate DTO
     const jobConfig = this.getJobTypeConfiguration(createJobDto.type);
     const validateContext = { request: createJobDto, env: process.env };
-    await validateActions(jobConfig.create.actions, validateContext);
+    const contextWithDatasets = await loadDatasets(
+      this.datasetsService,
+      validateContext,
+    );
+    await validateActions(jobConfig.create.actions, contextWithDatasets);
     // Create actual job in database
     const createdJobInstance = await this.jobsService.create(jobInstance);
     // Perform the action that is specified in the create portion of the job configuration
     const performContext = {
-      ...validateContext,
+      ...contextWithDatasets,
       job: toObject(createdJobInstance) as JobClass,
     };
     await performActions(jobConfig.create.actions, performContext);
@@ -707,7 +712,11 @@ export class JobsControllerUtils {
       job: currentJob,
       env: process.env,
     };
-    await validateActions(jobConfig.update.actions, validateContext);
+    const contextWithDatasets = await loadDatasets(
+      this.datasetsService,
+      validateContext,
+    );
+    await validateActions(jobConfig.update.actions, contextWithDatasets);
 
     const updateJobDtoForService =
       request.headers["content-type"] === "application/merge-patch+json"
@@ -722,7 +731,7 @@ export class JobsControllerUtils {
     // Perform the action that is specified in the update portion of the job configuration
     if (updatedJob !== null) {
       await this.checkConfigVersion(jobConfig, updatedJob);
-      const performContext = { ...validateContext, job: updatedJob };
+      const performContext = { ...contextWithDatasets, job: updatedJob };
       await performActions(jobConfig.update.actions, performContext);
     }
     return updatedJob;

From 56cd0eac41443922e3f62afdf3cebbc6ece021f9 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 Oct 2025 15:12:19 +0000
Subject: [PATCH 02/26] build(deps-dev): bump @nestjs/schematics in the nestjs
 group

Bumps the nestjs group with 1 update: [@nestjs/schematics](https://github.com/nestjs/schematics).


Updates `@nestjs/schematics` from 11.0.7 to 11.0.8
- [Release notes](https://github.com/nestjs/schematics/releases)
- [Changelog](https://github.com/nestjs/schematics/blob/master/.release-it.json)
- [Commits](https://github.com/nestjs/schematics/compare/11.0.7...11.0.8)

---
updated-dependencies:
- dependency-name: "@nestjs/schematics"
  dependency-version: 11.0.8
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: nestjs
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 104 ++++++++++++++++++++++++++++++++--------------
 1 file changed, 72 insertions(+), 32 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 5782a026f..ac78ba032 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -3463,15 +3463,15 @@
       }
     },
     "node_modules/@nestjs/schematics": {
-      "version": "11.0.7",
-      "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-11.0.7.tgz",
-      "integrity": "sha512-t8dNYYMwEeEsrlwc2jbkfwCfXczq4AeNEgx1KVQuJ6wYibXk0ZbXbPdfp8scnEAaQv1grpncNV5gWgzi7ZwbvQ==",
+      "version": "11.0.8",
+      "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-11.0.8.tgz",
+      "integrity": "sha512-HKunkzfBYLpNyL/qP5wu0OBKVPrISJLnrB4r6S53fT99pEvopDcJAeIuznSAD1Dx1njUqpbTR/uGyD0xL1y0nw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@angular-devkit/core": "19.2.15",
-        "@angular-devkit/schematics": "19.2.15",
-        "comment-json": "4.2.5",
+        "@angular-devkit/core": "19.2.17",
+        "@angular-devkit/schematics": "19.2.17",
+        "comment-json": "4.4.1",
         "jsonc-parser": "3.3.1",
         "pluralize": "8.0.0"
       },
@@ -3479,6 +3479,63 @@
         "typescript": ">=4.8.2"
       }
     },
+    "node_modules/@nestjs/schematics/node_modules/@angular-devkit/core": {
+      "version": "19.2.17",
+      "resolved": "https://registry.npmjs.org/@angular-devkit/core/-/core-19.2.17.tgz",
+      "integrity": "sha512-Ah008x2RJkd0F+NLKqIpA34/vUGwjlprRCkvddjDopAWRzYn6xCkz1Tqwuhn0nR1Dy47wTLKYD999TYl5ONOAQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "ajv": "8.17.1",
+        "ajv-formats": "3.0.1",
+        "jsonc-parser": "3.3.1",
+        "picomatch": "4.0.2",
+        "rxjs": "7.8.1",
+        "source-map": "0.7.4"
+      },
+      "engines": {
+        "node": "^18.19.1 || ^20.11.1 || >=22.0.0",
+        "npm": "^6.11.0 || ^7.5.6 || >=8.0.0",
+        "yarn": ">= 1.13.0"
+      },
+      "peerDependencies": {
+        "chokidar": "^4.0.0"
+      },
+      "peerDependenciesMeta": {
+        "chokidar": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@nestjs/schematics/node_modules/@angular-devkit/schematics": {
+      "version": "19.2.17",
+      "resolved": "https://registry.npmjs.org/@angular-devkit/schematics/-/schematics-19.2.17.tgz",
+      "integrity": "sha512-ADfbaBsrG8mBF6Mfs+crKA/2ykB8AJI50Cv9tKmZfwcUcyAdmTr+vVvhsBCfvUAEokigSsgqgpYxfkJVxhJYeg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@angular-devkit/core": "19.2.17",
+        "jsonc-parser": "3.3.1",
+        "magic-string": "0.30.17",
+        "ora": "5.4.1",
+        "rxjs": "7.8.1"
+      },
+      "engines": {
+        "node": "^18.19.1 || ^20.11.1 || >=22.0.0",
+        "npm": "^6.11.0 || ^7.5.6 || >=8.0.0",
+        "yarn": ">= 1.13.0"
+      }
+    },
+    "node_modules/@nestjs/schematics/node_modules/rxjs": {
+      "version": "7.8.1",
+      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz",
+      "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "tslib": "^2.1.0"
+      }
+    },
     "node_modules/@nestjs/swagger": {
       "version": "11.2.0",
       "resolved": "https://registry.npmjs.org/@nestjs/swagger/-/swagger-11.2.0.tgz",
@@ -6085,7 +6142,8 @@
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/array-timsort/-/array-timsort-1.0.3.tgz",
       "integrity": "sha512-/+3GRL7dDAGEfM6TseQk/U+mi18TU2Ms9I3UlLdUMhz2hbvGNTKdj9xniwXfUqgYhHxRx0+8UnKkvlNwVU+cWQ==",
-      "dev": true
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/asap": {
       "version": "2.0.6",
@@ -7152,16 +7210,15 @@
       }
     },
     "node_modules/comment-json": {
-      "version": "4.2.5",
-      "resolved": "https://registry.npmjs.org/comment-json/-/comment-json-4.2.5.tgz",
-      "integrity": "sha512-bKw/r35jR3HGt5PEPm1ljsQQGyCrR8sFGNiN5L+ykDHdpO8Smxkrkla9Yi6NkQyUrb8V54PGhfMs6NrIwtxtdw==",
+      "version": "4.4.1",
+      "resolved": "https://registry.npmjs.org/comment-json/-/comment-json-4.4.1.tgz",
+      "integrity": "sha512-r1To31BQD5060QdkC+Iheai7gHwoSZobzunqkf2/kQ6xIAfJyrKNAFUwdKvkK7Qgu7pVTKQEa7ok7Ed3ycAJgg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "array-timsort": "^1.0.3",
         "core-util-is": "^1.0.3",
-        "esprima": "^4.0.1",
-        "has-own-prop": "^2.0.0",
-        "repeat-string": "^1.6.1"
+        "esprima": "^4.0.1"
       },
       "engines": {
         "node": ">= 6"
@@ -7357,7 +7414,8 @@
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
       "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==",
-      "dev": true
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/cors": {
       "version": "2.8.5",
@@ -9206,15 +9264,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/has-own-prop": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/has-own-prop/-/has-own-prop-2.0.0.tgz",
-      "integrity": "sha512-Pq0h+hvsVm6dDEa8x82GnLSYHOzNDt7f0ddFa3FqcQlgzEiptPqL+XrOJNavjOzSYiYWIrgeVYYgGlLmnxwilQ==",
-      "dev": true,
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/has-symbols": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
@@ -13531,15 +13580,6 @@
         "node": ">= 0.10"
       }
     },
-    "node_modules/repeat-string": {
-      "version": "1.6.1",
-      "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
-      "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10"
-      }
-    },
     "node_modules/require-directory": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",

From 983b5b00620f193baef46208dfd0ce3c22451626 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 Oct 2025 15:25:18 +0000
Subject: [PATCH 03/26] build(deps-dev): bump @types/node from 24.5.2 to 24.7.0

Bumps [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node) from 24.5.2 to 24.7.0.
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node)

---
updated-dependencies:
- dependency-name: "@types/node"
  dependency-version: 24.7.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index ac78ba032..8c0359e28 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -4896,12 +4896,12 @@
       "dev": true
     },
     "node_modules/@types/node": {
-      "version": "24.5.2",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.5.2.tgz",
-      "integrity": "sha512-FYxk1I7wPv3K2XBaoyH2cTnocQEu8AOZ60hPbsyukMPLv5/5qr7V1i8PLHdl6Zf87I+xZXFvPCXYjiTFq+YSDQ==",
+      "version": "24.7.0",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.7.0.tgz",
+      "integrity": "sha512-IbKooQVqUBrlzWTi79E8Fw78l8k1RNtlDDNWsFZs7XonuQSJ8oNYfEeclhprUldXISRMLzBpILuKgPlIxm+/Yw==",
       "license": "MIT",
       "dependencies": {
-        "undici-types": "~7.12.0"
+        "undici-types": "~7.14.0"
       }
     },
     "node_modules/@types/node-fetch": {
@@ -15339,9 +15339,9 @@
       }
     },
     "node_modules/undici-types": {
-      "version": "7.12.0",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.12.0.tgz",
-      "integrity": "sha512-goOacqME2GYyOZZfb5Lgtu+1IDmAlAEu5xnD3+xTzS10hT0vzpf0SPjkXwAw9Jm+4n/mQGDP3LO8CPbYROeBfQ==",
+      "version": "7.14.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.14.0.tgz",
+      "integrity": "sha512-QQiYxHuyZ9gQUIrmPo3IA+hUl4KYk8uSA7cHrcKd/l3p1OTpZcM0Tbp9x7FAtXdAYhlasd60ncPpgu6ihG6TOA==",
       "license": "MIT"
     },
     "node_modules/universalify": {

From 4ce864db17767396456e630a64e9ca6f711e4801 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 Oct 2025 15:39:59 +0000
Subject: [PATCH 04/26] build(deps-dev): bump @types/nodemailer from 7.0.1 to
 7.0.2

Bumps [@types/nodemailer](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/nodemailer) from 7.0.1 to 7.0.2.
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/nodemailer)

---
updated-dependencies:
- dependency-name: "@types/nodemailer"
  dependency-version: 7.0.2
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 8c0359e28..52be56f8b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -4916,9 +4916,9 @@
       }
     },
     "node_modules/@types/nodemailer": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-7.0.1.tgz",
-      "integrity": "sha512-UfHAghPmGZVzaL8x9y+mKZMWyHC399+iq0MOmya5tIyenWX3lcdSb60vOmp0DocR6gCDTYTozv/ULQnREyyjkg==",
+      "version": "7.0.2",
+      "resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-7.0.2.tgz",
+      "integrity": "sha512-Zo6uOA9157WRgBk/ZhMpTQ/iCWLMk7OIs/Q9jvHarMvrzUUP/MDdPHL2U1zpf57HrrWGv4nYQn5uIxna0xY3xw==",
       "dev": true,
       "license": "MIT",
       "dependencies": {

From 1a602c0b644b117a328ffb59082cbcf84ca5b79f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 Oct 2025 15:49:52 +0000
Subject: [PATCH 05/26] build(deps): bump mathjs from 14.8.1 to 14.8.2

Bumps [mathjs](https://github.com/josdejong/mathjs) from 14.8.1 to 14.8.2.
- [Changelog](https://github.com/josdejong/mathjs/blob/develop/HISTORY.md)
- [Commits](https://github.com/josdejong/mathjs/commits)

---
updated-dependencies:
- dependency-name: mathjs
  dependency-version: 14.8.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 52be56f8b..3a88f2834 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -11300,9 +11300,9 @@
       }
     },
     "node_modules/mathjs": {
-      "version": "14.8.1",
-      "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-14.8.1.tgz",
-      "integrity": "sha512-7UDitJFsXoPbFdK2V7S6uJp7krx9B5bQSzAvCnh7ecsceSK4w0PBbEt9B3IRg9lHqzjk3YJNT1fZg/BNK+4CQw==",
+      "version": "14.8.2",
+      "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-14.8.2.tgz",
+      "integrity": "sha512-QbiGLoNwd/UazrpWTn3nnmIo51Es4X93cbKoKJP6/SBcNOX9sOOGu5ZTC7bI7SuK2Uy0oIhVKFyezG6LuAxrfQ==",
       "license": "Apache-2.0",
       "dependencies": {
         "@babel/runtime": "^7.26.10",

From c3a232a62053153902801ae3a7831a3d98afaa97 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 Oct 2025 15:59:54 +0000
Subject: [PATCH 06/26] build(deps-dev): bump @eslint/js from 9.36.0 to 9.37.0

Bumps [@eslint/js](https://github.com/eslint/eslint/tree/HEAD/packages/js) from 9.36.0 to 9.37.0.
- [Release notes](https://github.com/eslint/eslint/releases)
- [Commits](https://github.com/eslint/eslint/commits/v9.37.0/packages/js)

---
updated-dependencies:
- dependency-name: "@eslint/js"
  dependency-version: 9.37.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 3a88f2834..f21c62448 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1904,9 +1904,9 @@
       "dev": true
     },
     "node_modules/@eslint/js": {
-      "version": "9.36.0",
-      "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.36.0.tgz",
-      "integrity": "sha512-uhCbYtYynH30iZErszX78U+nR3pJU3RHGQ57NXy5QupD4SBVwDeU8TNBy+MjMngc1UyIW9noKqsRqfjQTBU2dw==",
+      "version": "9.37.0",
+      "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.37.0.tgz",
+      "integrity": "sha512-jaS+NJ+hximswBG6pjNX0uEJZkrT0zwpVi3BA3vX22aFGjJjmgSTSmPpZCRKmoBL5VY/M6p0xsSJx7rk7sy5gg==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -8242,6 +8242,19 @@
         "url": "https://opencollective.com/eslint"
       }
     },
+    "node_modules/eslint/node_modules/@eslint/js": {
+      "version": "9.36.0",
+      "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.36.0.tgz",
+      "integrity": "sha512-uhCbYtYynH30iZErszX78U+nR3pJU3RHGQ57NXy5QupD4SBVwDeU8TNBy+MjMngc1UyIW9noKqsRqfjQTBU2dw==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+      },
+      "funding": {
+        "url": "https://eslint.org/donate"
+      }
+    },
     "node_modules/eslint/node_modules/ajv": {
       "version": "6.12.6",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",

From a56b41e31d19b8313e69d0952429dbd87e0907b7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 Oct 2025 16:10:03 +0000
Subject: [PATCH 07/26] build(deps): bump dotenv from 17.2.2 to 17.2.3

Bumps [dotenv](https://github.com/motdotla/dotenv) from 17.2.2 to 17.2.3.
- [Changelog](https://github.com/motdotla/dotenv/blob/master/CHANGELOG.md)
- [Commits](https://github.com/motdotla/dotenv/compare/v17.2.2...v17.2.3)

---
updated-dependencies:
- dependency-name: dotenv
  dependency-version: 17.2.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index f21c62448..d495da582 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -7744,9 +7744,9 @@
       }
     },
     "node_modules/dotenv": {
-      "version": "17.2.2",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.2.tgz",
-      "integrity": "sha512-Sf2LSQP+bOlhKWWyhFsn0UsfdK/kCWRv1iuA2gXAwt3dyNabr6QSj00I2V10pidqz69soatm9ZwZvpQMTIOd5Q==",
+      "version": "17.2.3",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz",
+      "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==",
       "license": "BSD-2-Clause",
       "engines": {
         "node": ">=12"

From 8fe0dcfd10a94a9e92f4c659c4810bb99ee5b080 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 Oct 2025 16:22:39 +0000
Subject: [PATCH 08/26] build(deps-dev): bump mocha from 11.7.2 to 11.7.4

Bumps [mocha](https://github.com/mochajs/mocha) from 11.7.2 to 11.7.4.
- [Release notes](https://github.com/mochajs/mocha/releases)
- [Changelog](https://github.com/mochajs/mocha/blob/main/CHANGELOG.md)
- [Commits](https://github.com/mochajs/mocha/compare/v11.7.2...v11.7.4)

---
updated-dependencies:
- dependency-name: mocha
  dependency-version: 11.7.4
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index d495da582..aa97daa95 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -9696,6 +9696,16 @@
         "node": ">=0.12.0"
       }
     },
+    "node_modules/is-path-inside": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz",
+      "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/is-plain-obj": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz",
@@ -12060,9 +12070,9 @@
       }
     },
     "node_modules/mocha": {
-      "version": "11.7.2",
-      "resolved": "https://registry.npmjs.org/mocha/-/mocha-11.7.2.tgz",
-      "integrity": "sha512-lkqVJPmqqG/w5jmmFtiRvtA2jkDyNVUcefFJKb2uyX4dekk8Okgqop3cgbFiaIvj8uCRJVTP5x9dfxGyXm2jvQ==",
+      "version": "11.7.4",
+      "resolved": "https://registry.npmjs.org/mocha/-/mocha-11.7.4.tgz",
+      "integrity": "sha512-1jYAaY8x0kAZ0XszLWu14pzsf4KV740Gld4HXkhNTXwcHx4AUEDkPzgEHg9CM5dVcW+zv036tjpsEbLraPJj4w==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -12074,6 +12084,7 @@
         "find-up": "^5.0.0",
         "glob": "^10.4.5",
         "he": "^1.2.0",
+        "is-path-inside": "^3.0.3",
         "js-yaml": "^4.1.0",
         "log-symbols": "^4.1.0",
         "minimatch": "^9.0.5",

From bb1b16e0fd98ecb6cef1532f081960f4a4fc479a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 Oct 2025 16:32:38 +0000
Subject: [PATCH 09/26] build(deps-dev): bump eslint from 9.36.0 to 9.37.0

Bumps [eslint](https://github.com/eslint/eslint) from 9.36.0 to 9.37.0.
- [Release notes](https://github.com/eslint/eslint/releases)
- [Commits](https://github.com/eslint/eslint/compare/v9.36.0...v9.37.0)

---
updated-dependencies:
- dependency-name: eslint
  dependency-version: 9.37.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 50 +++++++++++++++++++----------------------------
 1 file changed, 20 insertions(+), 30 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index aa97daa95..070f09393 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1824,19 +1824,22 @@
       }
     },
     "node_modules/@eslint/config-helpers": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.3.1.tgz",
-      "integrity": "sha512-xR93k9WhrDYpXHORXpxVL5oHj3Era7wo6k/Wd8/IsQNnZUTzkGS29lyn3nAT05v6ltUuTFVCCYDEGfy2Or/sPA==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.0.tgz",
+      "integrity": "sha512-WUFvV4WoIwW8Bv0KeKCIIEgdSiFOsulyN0xrMu+7z43q/hkOLXjvb5u7UC9jDxvRzcrbEmuZBX5yJZz1741jog==",
       "dev": true,
       "license": "Apache-2.0",
+      "dependencies": {
+        "@eslint/core": "^0.16.0"
+      },
       "engines": {
         "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
       }
     },
     "node_modules/@eslint/core": {
-      "version": "0.15.2",
-      "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.15.2.tgz",
-      "integrity": "sha512-78Md3/Rrxh83gCxoUc0EiciuOHsIITzLy53m3d9UyiW8y9Dj2D29FeETqyKA+BRK76tnTp6RXWb3pCay8Oyomg==",
+      "version": "0.16.0",
+      "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.16.0.tgz",
+      "integrity": "sha512-nmC8/totwobIiFcGkDza3GIKfAw1+hLiYVrh3I1nIomQ8PEr5cxg34jnkmGawul/ep52wGRAcyeDCNtWKSOj4Q==",
       "dev": true,
       "license": "Apache-2.0",
       "dependencies": {
@@ -1926,13 +1929,13 @@
       }
     },
     "node_modules/@eslint/plugin-kit": {
-      "version": "0.3.5",
-      "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.3.5.tgz",
-      "integrity": "sha512-Z5kJ+wU3oA7MMIqVR9tyZRtjYPr4OC004Q4Rw7pgOKUOKkJfZ3O24nz3WYfGRpMDNmcOi3TwQOmgm7B7Tpii0w==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.0.tgz",
+      "integrity": "sha512-sB5uyeq+dwCWyPi31B2gQlVlo+j5brPlWx4yZBrEaRo/nhdDE8Xke1gsGgtiBdaBTxuTkceLVuVt/pclrasb0A==",
       "dev": true,
       "license": "Apache-2.0",
       "dependencies": {
-        "@eslint/core": "^0.15.2",
+        "@eslint/core": "^0.16.0",
         "levn": "^0.4.1"
       },
       "engines": {
@@ -8105,20 +8108,20 @@
       }
     },
     "node_modules/eslint": {
-      "version": "9.36.0",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.36.0.tgz",
-      "integrity": "sha512-hB4FIzXovouYzwzECDcUkJ4OcfOEkXTv2zRY6B9bkwjx/cprAq0uvm1nl7zvQ0/TsUk0zQiN4uPfJpB9m+rPMQ==",
+      "version": "9.37.0",
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.37.0.tgz",
+      "integrity": "sha512-XyLmROnACWqSxiGYArdef1fItQd47weqB7iwtfr9JHwRrqIXZdcFMvvEcL9xHCmL0SNsOvF0c42lWyM1U5dgig==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.8.0",
         "@eslint-community/regexpp": "^4.12.1",
         "@eslint/config-array": "^0.21.0",
-        "@eslint/config-helpers": "^0.3.1",
-        "@eslint/core": "^0.15.2",
+        "@eslint/config-helpers": "^0.4.0",
+        "@eslint/core": "^0.16.0",
         "@eslint/eslintrc": "^3.3.1",
-        "@eslint/js": "9.36.0",
-        "@eslint/plugin-kit": "^0.3.5",
+        "@eslint/js": "9.37.0",
+        "@eslint/plugin-kit": "^0.4.0",
         "@humanfs/node": "^0.16.6",
         "@humanwhocodes/module-importer": "^1.0.1",
         "@humanwhocodes/retry": "^0.4.2",
@@ -8242,19 +8245,6 @@
         "url": "https://opencollective.com/eslint"
       }
     },
-    "node_modules/eslint/node_modules/@eslint/js": {
-      "version": "9.36.0",
-      "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.36.0.tgz",
-      "integrity": "sha512-uhCbYtYynH30iZErszX78U+nR3pJU3RHGQ57NXy5QupD4SBVwDeU8TNBy+MjMngc1UyIW9noKqsRqfjQTBU2dw==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
-      },
-      "funding": {
-        "url": "https://eslint.org/donate"
-      }
-    },
     "node_modules/eslint/node_modules/ajv": {
       "version": "6.12.6",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",

From 9507eb13eb031348d292847c4422f143a70e31e7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 Oct 2025 16:42:32 +0000
Subject: [PATCH 10/26] build(deps-dev): bump typescript from 5.9.2 to 5.9.3

Bumps [typescript](https://github.com/microsoft/TypeScript) from 5.9.2 to 5.9.3.
- [Release notes](https://github.com/microsoft/TypeScript/releases)
- [Changelog](https://github.com/microsoft/TypeScript/blob/main/azure-pipelines.release-publish.yml)
- [Commits](https://github.com/microsoft/TypeScript/compare/v5.9.2...v5.9.3)

---
updated-dependencies:
- dependency-name: typescript
  dependency-version: 5.9.3
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 070f09393..cfbe17bcd 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -15282,9 +15282,9 @@
       "license": "MIT"
     },
     "node_modules/typescript": {
-      "version": "5.9.2",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.2.tgz",
-      "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==",
+      "version": "5.9.3",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
+      "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
       "dev": true,
       "license": "Apache-2.0",
       "bin": {

From 336bad4d924f7ffb331024281d553381ac80c54d Mon Sep 17 00:00:00 2001
From: Jay <a331998513@gmail.com>
Date: Wed, 8 Oct 2025 11:08:18 +0200
Subject: [PATCH 11/26] build(deps): downgrade uuid from 13.0.0 to 11.1.0
 (#2266)

---
 package-lock.json | 11 +++++------
 package.json      |  2 +-
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index cfbe17bcd..6fa499092 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -56,7 +56,7 @@
         "rimraf": "^6.0.1",
         "rxjs": "^7.5.7",
         "swagger-ui-express": "^5.0.0",
-        "uuid": "^13.0.0"
+        "uuid": "^11.1.0"
       },
       "devDependencies": {
         "@eslint/eslintrc": "^3.1.0",
@@ -15477,16 +15477,15 @@
       }
     },
     "node_modules/uuid": {
-      "version": "13.0.0",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.0.tgz",
-      "integrity": "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==",
+      "version": "11.1.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz",
+      "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==",
       "funding": [
         "https://github.com/sponsors/broofa",
         "https://github.com/sponsors/ctavan"
       ],
-      "license": "MIT",
       "bin": {
-        "uuid": "dist-node/bin/uuid"
+        "uuid": "dist/esm/bin/uuid"
       }
     },
     "node_modules/v8-compile-cache-lib": {
diff --git a/package.json b/package.json
index 847b4add4..ed9aeb28a 100644
--- a/package.json
+++ b/package.json
@@ -78,7 +78,7 @@
     "rimraf": "^6.0.1",
     "rxjs": "^7.5.7",
     "swagger-ui-express": "^5.0.0",
-    "uuid": "^13.0.0"
+    "uuid": "^11.1.0"
   },
   "overrides": {
     "pac-resolver": {

From 80ab0a14e44488600415fdd835fa63ae3aeb1782 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 8 Oct 2025 09:09:45 +0000
Subject: [PATCH 12/26] build(deps): bump mongoose from 8.18.2 to 8.19.0

Bumps [mongoose](https://github.com/Automattic/mongoose) from 8.18.2 to 8.19.0.
- [Release notes](https://github.com/Automattic/mongoose/releases)
- [Changelog](https://github.com/Automattic/mongoose/blob/master/CHANGELOG.md)
- [Commits](https://github.com/Automattic/mongoose/compare/8.18.2...8.19.0)

---
updated-dependencies:
- dependency-name: mongoose
  dependency-version: 8.19.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 6fa499092..f959e68e3 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -3068,9 +3068,10 @@
       "integrity": "sha512-4aErSrCR/On/e5G2hDP0wjooqDdauzEbIq8hIkIe5pXV0rtWJZvdCEKL0ykZxex+IxIwBp0eGeV48hQN07dXtw=="
     },
     "node_modules/@mongodb-js/saslprep": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.2.2.tgz",
-      "integrity": "sha512-EB0O3SCSNRUFk66iRCpI+cXzIjdswfCs7F6nOC3RAGJ7xr5YhaicvsRwJ9eyzYvYRlCSDUO/c7g4yNulxKC1WA==",
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.3.1.tgz",
+      "integrity": "sha512-6nZrq5kfAz0POWyhljnbWQQJQ5uT8oE2ddX303q1uY0tWsivWKgBDXBBvuFPwOqRRalXJuVO9EjOdVtuhLX0zg==",
+      "license": "MIT",
       "dependencies": {
         "sparse-bitfield": "^3.0.3"
       }
@@ -11358,7 +11359,8 @@
     "node_modules/memory-pager": {
       "version": "1.5.0",
       "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
-      "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg=="
+      "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
+      "license": "MIT"
     },
     "node_modules/mensch": {
       "version": "0.3.4",
@@ -12173,14 +12175,14 @@
       }
     },
     "node_modules/mongodb": {
-      "version": "6.18.0",
-      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.18.0.tgz",
-      "integrity": "sha512-fO5ttN9VC8P0F5fqtQmclAkgXZxbIkYRTUi1j8JO6IYwvamkhtYDilJr35jOPELR49zqCJgXZWwCtW7B+TM8vQ==",
+      "version": "6.20.0",
+      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.20.0.tgz",
+      "integrity": "sha512-Tl6MEIU3K4Rq3TSHd+sZQqRBoGlFsOgNrH5ltAcFBV62Re3Fd+FcaVf8uSEQFOJ51SDowDVttBTONMfoYWrWlQ==",
       "license": "Apache-2.0",
       "dependencies": {
-        "@mongodb-js/saslprep": "^1.1.9",
+        "@mongodb-js/saslprep": "^1.3.0",
         "bson": "^6.10.4",
-        "mongodb-connection-string-url": "^3.0.0"
+        "mongodb-connection-string-url": "^3.0.2"
       },
       "engines": {
         "node": ">=16.20.1"
@@ -12191,7 +12193,7 @@
         "gcp-metadata": "^5.2.0",
         "kerberos": "^2.0.1",
         "mongodb-client-encryption": ">=6.0.0 <7",
-        "snappy": "^7.2.2",
+        "snappy": "^7.3.2",
         "socks": "^2.7.1"
       },
       "peerDependenciesMeta": {
@@ -12228,14 +12230,14 @@
       }
     },
     "node_modules/mongoose": {
-      "version": "8.18.2",
-      "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.18.2.tgz",
-      "integrity": "sha512-gA6GFlshOHUdNyw9OQTmMLSGzVOPbcbjaSZ1dvR5iMp668N2UUznTuzgTY6V6Q41VtBc4kmL/qqML1RNgXB5Fg==",
+      "version": "8.19.1",
+      "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.19.1.tgz",
+      "integrity": "sha512-oB7hGQJn4f8aebqE7mhE54EReb5cxVgpCxQCQj0K/cK3q4J3Tg08nFP6sM52nJ4Hlm8jsDnhVYpqIITZUAhckQ==",
       "license": "MIT",
       "dependencies": {
         "bson": "^6.10.4",
         "kareem": "2.6.3",
-        "mongodb": "~6.18.0",
+        "mongodb": "~6.20.0",
         "mpath": "0.9.0",
         "mquery": "5.0.0",
         "ms": "2.1.3",
@@ -14360,6 +14362,7 @@
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
       "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==",
+      "license": "MIT",
       "dependencies": {
         "memory-pager": "^1.0.2"
       }

From 32fe42cc91c29fa5e096b5ec44e8b44883283968 Mon Sep 17 00:00:00 2001
From: Carlo Minotti <50220438+minottic@users.noreply.github.com>
Date: Wed, 8 Oct 2025 16:51:29 +0200
Subject: [PATCH 13/26] fix: pass URL directly to include qwargs (#2268)

---
 migrate-mongo-config.js | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/migrate-mongo-config.js b/migrate-mongo-config.js
index e731c925e..621db781a 100644
--- a/migrate-mongo-config.js
+++ b/migrate-mongo-config.js
@@ -5,16 +5,9 @@ if (!MONGODB_URI) {
   throw new Error("Environment variable MONGODB_URI not set");
 }
 
-const lastSlashIndex = MONGODB_URI.lastIndexOf("/");
-// NOTE: Get the database url and name from the MONGODB_URI and use them instead of defining new variables.
-const DATABASE_URL = MONGODB_URI.substring(0, lastSlashIndex);
-const DATABASE_NAME = MONGODB_URI.substring(lastSlashIndex + 1);
-
 const config = {
   mongodb: {
-    url: DATABASE_URL,
-
-    databaseName: DATABASE_NAME,
+    url: MONGODB_URI,
 
     options: {
       useNewUrlParser: true, // removes a deprecation warning when connecting

From a1d7def9668869319ebda23eff0009011a200f5d Mon Sep 17 00:00:00 2001
From: abdimo101 <abdi.abdulle@outlook.dk>
Date: Thu, 9 Oct 2025 11:49:46 +0200
Subject: [PATCH 14/26] docs: add note explaining duplicated V4 in SDK methods
 (#2267)

* added a note explaining the situation

* make comment concise

---------

Co-authored-by: Abdi Abdulle <abdi.abdulle@ess.eu>
Co-authored-by: junjiequan <a331998513@gmail.com>
---
 src/attachments/attachments.v4.controller.ts              | 5 +++++
 src/datasets/datasets-public.v4.controller.ts             | 5 +++++
 src/datasets/datasets.v4.controller.ts                    | 5 +++++
 src/jobs/jobs.v4.controller.ts                            | 5 +++++
 src/origdatablocks/origdatablocks-public.v4.controller.ts | 5 +++++
 src/origdatablocks/origdatablocks.v4.controller.ts        | 5 +++++
 src/published-data/published-data.v4.controller.ts        | 5 +++++
 7 files changed, 35 insertions(+)

diff --git a/src/attachments/attachments.v4.controller.ts b/src/attachments/attachments.v4.controller.ts
index 2810b98eb..c06b76ea2 100644
--- a/src/attachments/attachments.v4.controller.ts
+++ b/src/attachments/attachments.v4.controller.ts
@@ -60,6 +60,11 @@ import {
 
 @ApiBearerAuth()
 @ApiTags("attachments v4")
+/* NOTE: Generated SDK method names include "V4" twice:
+ *  - From the controller class name (AttachmentsV4Controller)
+ *  - From the route version (`version: '4'`)
+ * This is intentional for versioned routing.
+ */
 @Controller({ path: "attachments", version: "4" })
 export class AttachmentsV4Controller {
   constructor(
diff --git a/src/datasets/datasets-public.v4.controller.ts b/src/datasets/datasets-public.v4.controller.ts
index 10cf861d9..76821b9e0 100644
--- a/src/datasets/datasets-public.v4.controller.ts
+++ b/src/datasets/datasets-public.v4.controller.ts
@@ -37,6 +37,11 @@ import { AllowAny } from "src/auth/decorators/allow-any.decorator";
 
 @ApiExtraModels(HistoryClass, TechniqueClass, RelationshipClass)
 @ApiTags("datasets public v4")
+/* NOTE: Generated SDK method names include "V4" twice:
+ *  - From the controller class name (DatasetsPublicV4Controller)
+ *  - From the route version (`version: '4'`)
+ * This is intentional for versioned routing.
+ */
 @Controller({ path: "datasets/public", version: "4" })
 export class DatasetsPublicV4Controller {
   constructor(private datasetsService: DatasetsService) {}
diff --git a/src/datasets/datasets.v4.controller.ts b/src/datasets/datasets.v4.controller.ts
index fd9513890..35c6b6a47 100644
--- a/src/datasets/datasets.v4.controller.ts
+++ b/src/datasets/datasets.v4.controller.ts
@@ -97,6 +97,11 @@ import { TechniqueClass } from "./schemas/technique.schema";
   RelationshipClass,
 )
 @ApiTags("datasets v4")
+/* NOTE: Generated SDK method names include "V4" twice:
+ *  - From the controller class name (DatasetsV4Controller)
+ *  - From the route version (`version: '4'`)
+ * This is intentional for versioned routing.
+ */
 @Controller({ path: "datasets", version: "4" })
 export class DatasetsV4Controller {
   constructor(
diff --git a/src/jobs/jobs.v4.controller.ts b/src/jobs/jobs.v4.controller.ts
index 35ecd0bdf..045f41570 100644
--- a/src/jobs/jobs.v4.controller.ts
+++ b/src/jobs/jobs.v4.controller.ts
@@ -47,6 +47,11 @@ import { ALLOWED_JOB_KEYS, ALLOWED_JOB_FILTER_KEYS } from "./types/job-lookup";
 
 @ApiBearerAuth()
 @ApiTags("jobs v4")
+/* NOTE: Generated SDK method names include "V4" twice:
+ *  - From the controller class name (JobsV4Controller)
+ *  - From the route version (`version: '4'`)
+ * This is intentional for versioned routing.
+ */
 @Controller({ path: "jobs", version: "4" })
 export class JobsV4Controller {
   constructor(
diff --git a/src/origdatablocks/origdatablocks-public.v4.controller.ts b/src/origdatablocks/origdatablocks-public.v4.controller.ts
index dd12c8544..6b5166b82 100644
--- a/src/origdatablocks/origdatablocks-public.v4.controller.ts
+++ b/src/origdatablocks/origdatablocks-public.v4.controller.ts
@@ -27,6 +27,11 @@ import { FilterValidationPipe } from "src/common/pipes/filter-validation.pipe";
 import { AllowAny } from "src/auth/decorators/allow-any.decorator";
 
 @ApiTags("origdatablocks public v4")
+/* NOTE: Generated SDK method names include "V4" twice:
+ *  - From the controller class name (OrigDatablocksPublicV4Controller)
+ *  - From the route version (`version: '4'`)
+ * This is intentional for versioned routing.
+ */
 @Controller({ path: "origdatablocks/public", version: "4" })
 export class OrigDatablocksPublicV4Controller {
   constructor(private readonly origDatablocksService: OrigDatablocksService) {}
diff --git a/src/origdatablocks/origdatablocks.v4.controller.ts b/src/origdatablocks/origdatablocks.v4.controller.ts
index f9515d427..f76518fe5 100644
--- a/src/origdatablocks/origdatablocks.v4.controller.ts
+++ b/src/origdatablocks/origdatablocks.v4.controller.ts
@@ -70,6 +70,11 @@ import { FilterValidationPipe } from "src/common/pipes/filter-validation.pipe";
 
 @ApiBearerAuth()
 @ApiTags("origdatablocks v4")
+/* NOTE: Generated SDK method names include "V4" twice:
+ *  - From the controller class name (OrigDatablocksV4Controller)
+ *  - From the route version (`version: '4'`)
+ * This is intentional for versioned routing.
+ */
 @Controller({ path: "origdatablocks", version: "4" })
 export class OrigDatablocksV4Controller {
   constructor(
diff --git a/src/published-data/published-data.v4.controller.ts b/src/published-data/published-data.v4.controller.ts
index feeb91f77..81cfd51a1 100644
--- a/src/published-data/published-data.v4.controller.ts
+++ b/src/published-data/published-data.v4.controller.ts
@@ -60,6 +60,11 @@ import { RegisteredFilterPipe } from "./pipes/registered.pipe";
 
 @ApiBearerAuth()
 @ApiTags("published data v4")
+/* NOTE: Generated SDK method names include "V4" twice:
+ *  - From the controller class name (PublishedDataV4Controller)
+ *  - From the route version (`version: '4'`)
+ * This is intentional for versioned routing.
+ */
 @Controller({ path: "publisheddata", version: "4" })
 export class PublishedDataV4Controller {
   constructor(

From acecc7617b9d3aff727aca8f1e6dad5da78e7cb6 Mon Sep 17 00:00:00 2001
From: abdimo101 <abdi.abdulle@outlook.dk>
Date: Mon, 13 Oct 2025 10:06:00 +0200
Subject: [PATCH 15/26] feat(config): add the numberOfDatasets in
 defaultProposalsListSettings & labelsLocalization (#2255)

adds the numberOfDatasets in defaultProposalsListSettings inside config file
---
 src/config/frontend.config.json | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/config/frontend.config.json b/src/config/frontend.config.json
index 0fe2df27f..76965a454 100644
--- a/src/config/frontend.config.json
+++ b/src/config/frontend.config.json
@@ -275,6 +275,11 @@
         "name": "pi_lastname",
         "enabled": true,
         "width": 200
+      },
+      {
+        "name": "numberOfDatasets",
+        "width": 150,
+        "enabled": true
       }
     ],
     "filters": [
@@ -333,7 +338,8 @@
       "abstract": "Abstract",
       "pi_lastname": "PI Last Name",
       "type": "Type",
-      "instrumentIds": "Instrument"
+      "instrumentIds": "Instrument",
+      "numberOfDatasets": "Number of Datasets"
     }
   },
   "dateFormat": "yyyy-MM-dd HH:mm",

From c979ae8384c66999a753b9eb09becaa4e6b62fed Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Oct 2025 15:24:33 +0000
Subject: [PATCH 16/26] build(deps-dev): bump @typescript-eslint/parser from
 8.45.0 to 8.46.0

Bumps [@typescript-eslint/parser](https://github.com/typescript-eslint/typescript-eslint/tree/HEAD/packages/parser) from 8.45.0 to 8.46.0.
- [Release notes](https://github.com/typescript-eslint/typescript-eslint/releases)
- [Changelog](https://github.com/typescript-eslint/typescript-eslint/blob/main/packages/parser/CHANGELOG.md)
- [Commits](https://github.com/typescript-eslint/typescript-eslint/commits/v8.46.0/packages/parser)

---
updated-dependencies:
- dependency-name: "@typescript-eslint/parser"
  dependency-version: 8.46.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 158 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 151 insertions(+), 7 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index f959e68e3..4514d98dd 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -5129,16 +5129,16 @@
       }
     },
     "node_modules/@typescript-eslint/parser": {
-      "version": "8.45.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.45.0.tgz",
-      "integrity": "sha512-TGf22kon8KW+DeKaUmOibKWktRY8b2NSAZNdtWh798COm1NWx8+xJ6iFBtk3IvLdv6+LGLJLRlyhrhEDZWargQ==",
+      "version": "8.46.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.46.0.tgz",
+      "integrity": "sha512-n1H6IcDhmmUEG7TNVSspGmiHHutt7iVKtZwRppD7e04wha5MrkV1h3pti9xQLcCMt6YWsncpoT0HMjkH1FNwWQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@typescript-eslint/scope-manager": "8.45.0",
-        "@typescript-eslint/types": "8.45.0",
-        "@typescript-eslint/typescript-estree": "8.45.0",
-        "@typescript-eslint/visitor-keys": "8.45.0",
+        "@typescript-eslint/scope-manager": "8.46.0",
+        "@typescript-eslint/types": "8.46.0",
+        "@typescript-eslint/typescript-estree": "8.46.0",
+        "@typescript-eslint/visitor-keys": "8.46.0",
         "debug": "^4.3.4"
       },
       "engines": {
@@ -5153,6 +5153,150 @@
         "typescript": ">=4.8.4 <6.0.0"
       }
     },
+    "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/project-service": {
+      "version": "8.46.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.46.0.tgz",
+      "integrity": "sha512-OEhec0mH+U5Je2NZOeK1AbVCdm0ChyapAyTeXVIYTPXDJ3F07+cu87PPXcGoYqZ7M9YJVvFnfpGg1UmCIqM+QQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@typescript-eslint/tsconfig-utils": "^8.46.0",
+        "@typescript-eslint/types": "^8.46.0",
+        "debug": "^4.3.4"
+      },
+      "engines": {
+        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
+      },
+      "peerDependencies": {
+        "typescript": ">=4.8.4 <6.0.0"
+      }
+    },
+    "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/scope-manager": {
+      "version": "8.46.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.46.0.tgz",
+      "integrity": "sha512-lWETPa9XGcBes4jqAMYD9fW0j4n6hrPtTJwWDmtqgFO/4HF4jmdH/Q6wggTw5qIT5TXjKzbt7GsZUBnWoO3dqw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@typescript-eslint/types": "8.46.0",
+        "@typescript-eslint/visitor-keys": "8.46.0"
+      },
+      "engines": {
+        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
+      }
+    },
+    "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/tsconfig-utils": {
+      "version": "8.46.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.46.0.tgz",
+      "integrity": "sha512-WrYXKGAHY836/N7zoK/kzi6p8tXFhasHh8ocFL9VZSAkvH956gfeRfcnhs3xzRy8qQ/dq3q44v1jvQieMFg2cw==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
+      },
+      "peerDependencies": {
+        "typescript": ">=4.8.4 <6.0.0"
+      }
+    },
+    "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/types": {
+      "version": "8.46.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.46.0.tgz",
+      "integrity": "sha512-bHGGJyVjSE4dJJIO5yyEWt/cHyNwga/zXGJbJJ8TiO01aVREK6gCTu3L+5wrkb1FbDkQ+TKjMNe9R/QQQP9+rA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
+      }
+    },
+    "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/typescript-estree": {
+      "version": "8.46.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.46.0.tgz",
+      "integrity": "sha512-ekDCUfVpAKWJbRfm8T1YRrCot1KFxZn21oV76v5Fj4tr7ELyk84OS+ouvYdcDAwZL89WpEkEj2DKQ+qg//+ucg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@typescript-eslint/project-service": "8.46.0",
+        "@typescript-eslint/tsconfig-utils": "8.46.0",
+        "@typescript-eslint/types": "8.46.0",
+        "@typescript-eslint/visitor-keys": "8.46.0",
+        "debug": "^4.3.4",
+        "fast-glob": "^3.3.2",
+        "is-glob": "^4.0.3",
+        "minimatch": "^9.0.4",
+        "semver": "^7.6.0",
+        "ts-api-utils": "^2.1.0"
+      },
+      "engines": {
+        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
+      },
+      "peerDependencies": {
+        "typescript": ">=4.8.4 <6.0.0"
+      }
+    },
+    "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/visitor-keys": {
+      "version": "8.46.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.46.0.tgz",
+      "integrity": "sha512-FrvMpAK+hTbFy7vH5j1+tMYHMSKLE6RzluFJlkFNKD0p9YsUT75JlBSmr5so3QRzvMwU5/bIEdeNrxm8du8l3Q==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@typescript-eslint/types": "8.46.0",
+        "eslint-visitor-keys": "^4.2.1"
+      },
+      "engines": {
+        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/typescript-eslint"
+      }
+    },
+    "node_modules/@typescript-eslint/parser/node_modules/brace-expansion": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
+      "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "balanced-match": "^1.0.0"
+      }
+    },
+    "node_modules/@typescript-eslint/parser/node_modules/minimatch": {
+      "version": "9.0.5",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
+      "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "brace-expansion": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/@typescript-eslint/project-service": {
       "version": "8.45.0",
       "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.45.0.tgz",

From 7a27f812bbe385bd553f46e918b325b07fb403d3 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Oct 2025 15:37:37 +0000
Subject: [PATCH 17/26] build(deps-dev): bump @types/node from 24.7.0 to 24.7.2

Bumps [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node) from 24.7.0 to 24.7.2.
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node)

---
updated-dependencies:
- dependency-name: "@types/node"
  dependency-version: 24.7.2
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 4514d98dd..4666f0e97 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -4900,9 +4900,9 @@
       "dev": true
     },
     "node_modules/@types/node": {
-      "version": "24.7.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.7.0.tgz",
-      "integrity": "sha512-IbKooQVqUBrlzWTi79E8Fw78l8k1RNtlDDNWsFZs7XonuQSJ8oNYfEeclhprUldXISRMLzBpILuKgPlIxm+/Yw==",
+      "version": "24.7.2",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-24.7.2.tgz",
+      "integrity": "sha512-/NbVmcGTP+lj5oa4yiYxxeBjRivKQ5Ns1eSZeB99ExsEQ6rX5XYU1Zy/gGxY/ilqtD4Etx9mKyrPxZRetiahhA==",
       "license": "MIT",
       "dependencies": {
         "undici-types": "~7.14.0"

From 4302dc07d938b7a5b813c0f1171e7cf092f15a3a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Oct 2025 15:47:40 +0000
Subject: [PATCH 18/26] build(deps-dev): bump ts-jest from 29.4.4 to 29.4.5

Bumps [ts-jest](https://github.com/kulshekhar/ts-jest) from 29.4.4 to 29.4.5.
- [Release notes](https://github.com/kulshekhar/ts-jest/releases)
- [Changelog](https://github.com/kulshekhar/ts-jest/blob/main/CHANGELOG.md)
- [Commits](https://github.com/kulshekhar/ts-jest/compare/v29.4.4...v29.4.5)

---
updated-dependencies:
- dependency-name: ts-jest
  dependency-version: 29.4.5
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 4666f0e97..61811aa39 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -14245,9 +14245,9 @@
       }
     },
     "node_modules/semver": {
-      "version": "7.7.2",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz",
-      "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==",
+      "version": "7.7.3",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz",
+      "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==",
       "license": "ISC",
       "bin": {
         "semver": "bin/semver.js"
@@ -15187,9 +15187,9 @@
       }
     },
     "node_modules/ts-jest": {
-      "version": "29.4.4",
-      "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.4.tgz",
-      "integrity": "sha512-ccVcRABct5ZELCT5U0+DZwkXMCcOCLi2doHRrKy1nK/s7J7bch6TzJMsrY09WxgUUIP/ITfmcDS8D2yl63rnXw==",
+      "version": "29.4.5",
+      "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.5.tgz",
+      "integrity": "sha512-HO3GyiWn2qvTQA4kTgjDcXiMwYQt68a1Y8+JuLRVpdIzm+UOLSHgl/XqR4c6nzJkq5rOkjc02O2I7P7l/Yof0Q==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -15199,7 +15199,7 @@
         "json5": "^2.2.3",
         "lodash.memoize": "^4.1.2",
         "make-error": "^1.3.6",
-        "semver": "^7.7.2",
+        "semver": "^7.7.3",
         "type-fest": "^4.41.0",
         "yargs-parser": "^21.1.1"
       },

From 44315f596b8638a06645287c7f022c23c1183bb7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Oct 2025 15:58:05 +0000
Subject: [PATCH 19/26] build(deps-dev): bump @typescript-eslint/eslint-plugin

Bumps [@typescript-eslint/eslint-plugin](https://github.com/typescript-eslint/typescript-eslint/tree/HEAD/packages/eslint-plugin) from 8.45.0 to 8.46.0.
- [Release notes](https://github.com/typescript-eslint/typescript-eslint/releases)
- [Changelog](https://github.com/typescript-eslint/typescript-eslint/blob/main/packages/eslint-plugin/CHANGELOG.md)
- [Commits](https://github.com/typescript-eslint/typescript-eslint/commits/v8.46.0/packages/eslint-plugin)

---
updated-dependencies:
- dependency-name: "@typescript-eslint/eslint-plugin"
  dependency-version: 8.46.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 package-lock.json | 216 ++++++++--------------------------------------
 1 file changed, 36 insertions(+), 180 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 61811aa39..5165e1b36 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -5089,17 +5089,17 @@
       "dev": true
     },
     "node_modules/@typescript-eslint/eslint-plugin": {
-      "version": "8.45.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.45.0.tgz",
-      "integrity": "sha512-HC3y9CVuevvWCl/oyZuI47dOeDF9ztdMEfMH8/DW/Mhwa9cCLnK1oD7JoTVGW/u7kFzNZUKUoyJEqkaJh5y3Wg==",
+      "version": "8.46.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.46.0.tgz",
+      "integrity": "sha512-hA8gxBq4ukonVXPy0OKhiaUh/68D0E88GSmtC1iAEnGaieuDi38LhS7jdCHRLi6ErJBNDGCzvh5EnzdPwUc0DA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "@eslint-community/regexpp": "^4.10.0",
-        "@typescript-eslint/scope-manager": "8.45.0",
-        "@typescript-eslint/type-utils": "8.45.0",
-        "@typescript-eslint/utils": "8.45.0",
-        "@typescript-eslint/visitor-keys": "8.45.0",
+        "@typescript-eslint/scope-manager": "8.46.0",
+        "@typescript-eslint/type-utils": "8.46.0",
+        "@typescript-eslint/utils": "8.46.0",
+        "@typescript-eslint/visitor-keys": "8.46.0",
         "graphemer": "^1.4.0",
         "ignore": "^7.0.0",
         "natural-compare": "^1.4.0",
@@ -5113,7 +5113,7 @@
         "url": "https://opencollective.com/typescript-eslint"
       },
       "peerDependencies": {
-        "@typescript-eslint/parser": "^8.45.0",
+        "@typescript-eslint/parser": "^8.46.0",
         "eslint": "^8.57.0 || ^9.0.0",
         "typescript": ">=4.8.4 <6.0.0"
       }
@@ -5153,7 +5153,7 @@
         "typescript": ">=4.8.4 <6.0.0"
       }
     },
-    "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/project-service": {
+    "node_modules/@typescript-eslint/project-service": {
       "version": "8.46.0",
       "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.46.0.tgz",
       "integrity": "sha512-OEhec0mH+U5Je2NZOeK1AbVCdm0ChyapAyTeXVIYTPXDJ3F07+cu87PPXcGoYqZ7M9YJVvFnfpGg1UmCIqM+QQ==",
@@ -5175,7 +5175,7 @@
         "typescript": ">=4.8.4 <6.0.0"
       }
     },
-    "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/scope-manager": {
+    "node_modules/@typescript-eslint/scope-manager": {
       "version": "8.46.0",
       "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.46.0.tgz",
       "integrity": "sha512-lWETPa9XGcBes4jqAMYD9fW0j4n6hrPtTJwWDmtqgFO/4HF4jmdH/Q6wggTw5qIT5TXjKzbt7GsZUBnWoO3dqw==",
@@ -5193,7 +5193,7 @@
         "url": "https://opencollective.com/typescript-eslint"
       }
     },
-    "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/tsconfig-utils": {
+    "node_modules/@typescript-eslint/tsconfig-utils": {
       "version": "8.46.0",
       "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.46.0.tgz",
       "integrity": "sha512-WrYXKGAHY836/N7zoK/kzi6p8tXFhasHh8ocFL9VZSAkvH956gfeRfcnhs3xzRy8qQ/dq3q44v1jvQieMFg2cw==",
@@ -5210,160 +5210,16 @@
         "typescript": ">=4.8.4 <6.0.0"
       }
     },
-    "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/types": {
-      "version": "8.46.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.46.0.tgz",
-      "integrity": "sha512-bHGGJyVjSE4dJJIO5yyEWt/cHyNwga/zXGJbJJ8TiO01aVREK6gCTu3L+5wrkb1FbDkQ+TKjMNe9R/QQQP9+rA==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
-      }
-    },
-    "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/typescript-estree": {
-      "version": "8.46.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.46.0.tgz",
-      "integrity": "sha512-ekDCUfVpAKWJbRfm8T1YRrCot1KFxZn21oV76v5Fj4tr7ELyk84OS+ouvYdcDAwZL89WpEkEj2DKQ+qg//+ucg==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@typescript-eslint/project-service": "8.46.0",
-        "@typescript-eslint/tsconfig-utils": "8.46.0",
-        "@typescript-eslint/types": "8.46.0",
-        "@typescript-eslint/visitor-keys": "8.46.0",
-        "debug": "^4.3.4",
-        "fast-glob": "^3.3.2",
-        "is-glob": "^4.0.3",
-        "minimatch": "^9.0.4",
-        "semver": "^7.6.0",
-        "ts-api-utils": "^2.1.0"
-      },
-      "engines": {
-        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
-      },
-      "peerDependencies": {
-        "typescript": ">=4.8.4 <6.0.0"
-      }
-    },
-    "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/visitor-keys": {
+    "node_modules/@typescript-eslint/type-utils": {
       "version": "8.46.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.46.0.tgz",
-      "integrity": "sha512-FrvMpAK+hTbFy7vH5j1+tMYHMSKLE6RzluFJlkFNKD0p9YsUT75JlBSmr5so3QRzvMwU5/bIEdeNrxm8du8l3Q==",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.46.0.tgz",
+      "integrity": "sha512-hy+lvYV1lZpVs2jRaEYvgCblZxUoJiPyCemwbQZ+NGulWkQRy0HRPYAoef/CNSzaLt+MLvMptZsHXHlkEilaeg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "@typescript-eslint/types": "8.46.0",
-        "eslint-visitor-keys": "^4.2.1"
-      },
-      "engines": {
-        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
-      }
-    },
-    "node_modules/@typescript-eslint/parser/node_modules/brace-expansion": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
-      "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "balanced-match": "^1.0.0"
-      }
-    },
-    "node_modules/@typescript-eslint/parser/node_modules/minimatch": {
-      "version": "9.0.5",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
-      "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==",
-      "dev": true,
-      "license": "ISC",
-      "dependencies": {
-        "brace-expansion": "^2.0.1"
-      },
-      "engines": {
-        "node": ">=16 || 14 >=14.17"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
-    "node_modules/@typescript-eslint/project-service": {
-      "version": "8.45.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.45.0.tgz",
-      "integrity": "sha512-3pcVHwMG/iA8afdGLMuTibGR7pDsn9RjDev6CCB+naRsSYs2pns5QbinF4Xqw6YC/Sj3lMrm/Im0eMfaa61WUg==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@typescript-eslint/tsconfig-utils": "^8.45.0",
-        "@typescript-eslint/types": "^8.45.0",
-        "debug": "^4.3.4"
-      },
-      "engines": {
-        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
-      },
-      "peerDependencies": {
-        "typescript": ">=4.8.4 <6.0.0"
-      }
-    },
-    "node_modules/@typescript-eslint/scope-manager": {
-      "version": "8.45.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.45.0.tgz",
-      "integrity": "sha512-clmm8XSNj/1dGvJeO6VGH7EUSeA0FMs+5au/u3lrA3KfG8iJ4u8ym9/j2tTEoacAffdW1TVUzXO30W1JTJS7dA==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@typescript-eslint/types": "8.45.0",
-        "@typescript-eslint/visitor-keys": "8.45.0"
-      },
-      "engines": {
-        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
-      }
-    },
-    "node_modules/@typescript-eslint/tsconfig-utils": {
-      "version": "8.45.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.45.0.tgz",
-      "integrity": "sha512-aFdr+c37sc+jqNMGhH+ajxPXwjv9UtFZk79k8pLoJ6p4y0snmYpPA52GuWHgt2ZF4gRRW6odsEj41uZLojDt5w==",
-      "dev": true,
-      "license": "MIT",
-      "engines": {
-        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/typescript-eslint"
-      },
-      "peerDependencies": {
-        "typescript": ">=4.8.4 <6.0.0"
-      }
-    },
-    "node_modules/@typescript-eslint/type-utils": {
-      "version": "8.45.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.45.0.tgz",
-      "integrity": "sha512-bpjepLlHceKgyMEPglAeULX1vixJDgaKocp0RVJ5u4wLJIMNuKtUXIczpJCPcn2waII0yuvks/5m5/h3ZQKs0A==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "@typescript-eslint/types": "8.45.0",
-        "@typescript-eslint/typescript-estree": "8.45.0",
-        "@typescript-eslint/utils": "8.45.0",
+        "@typescript-eslint/typescript-estree": "8.46.0",
+        "@typescript-eslint/utils": "8.46.0",
         "debug": "^4.3.4",
         "ts-api-utils": "^2.1.0"
       },
@@ -5380,9 +5236,9 @@
       }
     },
     "node_modules/@typescript-eslint/types": {
-      "version": "8.45.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.45.0.tgz",
-      "integrity": "sha512-WugXLuOIq67BMgQInIxxnsSyRLFxdkJEJu8r4ngLR56q/4Q5LrbfkFRH27vMTjxEK8Pyz7QfzuZe/G15qQnVRA==",
+      "version": "8.46.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.46.0.tgz",
+      "integrity": "sha512-bHGGJyVjSE4dJJIO5yyEWt/cHyNwga/zXGJbJJ8TiO01aVREK6gCTu3L+5wrkb1FbDkQ+TKjMNe9R/QQQP9+rA==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -5394,16 +5250,16 @@
       }
     },
     "node_modules/@typescript-eslint/typescript-estree": {
-      "version": "8.45.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.45.0.tgz",
-      "integrity": "sha512-GfE1NfVbLam6XQ0LcERKwdTTPlLvHvXXhOeUGC1OXi4eQBoyy1iVsW+uzJ/J9jtCz6/7GCQ9MtrQ0fml/jWCnA==",
+      "version": "8.46.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.46.0.tgz",
+      "integrity": "sha512-ekDCUfVpAKWJbRfm8T1YRrCot1KFxZn21oV76v5Fj4tr7ELyk84OS+ouvYdcDAwZL89WpEkEj2DKQ+qg//+ucg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@typescript-eslint/project-service": "8.45.0",
-        "@typescript-eslint/tsconfig-utils": "8.45.0",
-        "@typescript-eslint/types": "8.45.0",
-        "@typescript-eslint/visitor-keys": "8.45.0",
+        "@typescript-eslint/project-service": "8.46.0",
+        "@typescript-eslint/tsconfig-utils": "8.46.0",
+        "@typescript-eslint/types": "8.46.0",
+        "@typescript-eslint/visitor-keys": "8.46.0",
         "debug": "^4.3.4",
         "fast-glob": "^3.3.2",
         "is-glob": "^4.0.3",
@@ -5449,16 +5305,16 @@
       }
     },
     "node_modules/@typescript-eslint/utils": {
-      "version": "8.45.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.45.0.tgz",
-      "integrity": "sha512-bxi1ht+tLYg4+XV2knz/F7RVhU0k6VrSMc9sb8DQ6fyCTrGQLHfo7lDtN0QJjZjKkLA2ThrKuCdHEvLReqtIGg==",
+      "version": "8.46.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.46.0.tgz",
+      "integrity": "sha512-nD6yGWPj1xiOm4Gk0k6hLSZz2XkNXhuYmyIrOWcHoPuAhjT9i5bAG+xbWPgFeNR8HPHHtpNKdYUXJl/D3x7f5g==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.7.0",
-        "@typescript-eslint/scope-manager": "8.45.0",
-        "@typescript-eslint/types": "8.45.0",
-        "@typescript-eslint/typescript-estree": "8.45.0"
+        "@typescript-eslint/scope-manager": "8.46.0",
+        "@typescript-eslint/types": "8.46.0",
+        "@typescript-eslint/typescript-estree": "8.46.0"
       },
       "engines": {
         "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
@@ -5473,13 +5329,13 @@
       }
     },
     "node_modules/@typescript-eslint/visitor-keys": {
-      "version": "8.45.0",
-      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.45.0.tgz",
-      "integrity": "sha512-qsaFBA3e09MIDAGFUrTk+dzqtfv1XPVz8t8d1f0ybTzrCY7BKiMC5cjrl1O/P7UmHsNyW90EYSkU/ZWpmXelag==",
+      "version": "8.46.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.46.0.tgz",
+      "integrity": "sha512-FrvMpAK+hTbFy7vH5j1+tMYHMSKLE6RzluFJlkFNKD0p9YsUT75JlBSmr5so3QRzvMwU5/bIEdeNrxm8du8l3Q==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "@typescript-eslint/types": "8.45.0",
+        "@typescript-eslint/types": "8.46.0",
         "eslint-visitor-keys": "^4.2.1"
       },
       "engines": {

From 41517349057436d0082b6af90a8511f4195adc26 Mon Sep 17 00:00:00 2001
From: Spencer Bliven <spencer.bliven@gmail.com>
Date: Mon, 13 Oct 2025 20:13:34 +0200
Subject: [PATCH 20/26] feat: allow url actions to ignore errors from remote
 calls (#2271)

Add 'ignoreErrors' option to url action.

Also adds the url response to logs.
---
 src/common/utils.ts                           | 16 ++++++++
 .../actions/urlaction/urlaction.interface.ts  | 21 ++--------
 .../actions/urlaction/urlaction.spec.ts       | 35 +++++++++++++++--
 .../job-config/actions/urlaction/urlaction.ts | 38 ++++++++++++-------
 4 files changed, 77 insertions(+), 33 deletions(-)

diff --git a/src/common/utils.ts b/src/common/utils.ts
index e590844cb..881433699 100644
--- a/src/common/utils.ts
+++ b/src/common/utils.ts
@@ -1205,6 +1205,22 @@ export function oneOrMore<T>(x: T[] | T): T[] {
 export type MakeOptional<T, K extends keyof T> = Omit<T, K> &
   Partial<Pick<T, K>>;
 
+/**
+ * Type guard for Record<string, string>
+ * @param obj
+ * @returns
+ */
+export function isStringRecord(obj: unknown): obj is Record<string, string> {
+  if (typeof obj !== "object" || obj === null) {
+    return false;
+  }
+  const rec = obj as Record<string, string>;
+
+  return Object.keys(rec).every(
+    (key) => typeof key === "string" && typeof rec[key] === "string",
+  );
+}
+
 /**
  * Helper function to generate HttpExceptions
  * @param message error message
diff --git a/src/config/job-config/actions/urlaction/urlaction.interface.ts b/src/config/job-config/actions/urlaction/urlaction.interface.ts
index 677abf209..ddc3e78fb 100644
--- a/src/config/job-config/actions/urlaction/urlaction.interface.ts
+++ b/src/config/job-config/actions/urlaction/urlaction.interface.ts
@@ -1,3 +1,4 @@
+import { isStringRecord } from "src/common/utils";
 export const actionType = "url";
 
 export interface URLJobActionOptions {
@@ -6,6 +7,7 @@ export interface URLJobActionOptions {
   method?: "GET" | "POST" | "PUT" | "DELETE";
   headers?: Record<string, string>;
   body?: unknown;
+  ignoreErrors?: boolean;
 }
 
 /**
@@ -24,22 +26,7 @@ export function isURLJobActionOptions(
     typeof opts.url === "string" &&
     (opts.method === undefined ||
       ["GET", "POST", "PUT", "DELETE"].includes(opts.method)) &&
-    (opts.headers === undefined || isStringRecord(opts.headers))
-  );
-}
-
-/**
- * Type guard for Record<string, string>
- * @param obj
- * @returns
- */
-function isStringRecord(obj: unknown): obj is Record<string, string> {
-  if (typeof obj !== "object" || obj === null) {
-    return false;
-  }
-  const rec = obj as Record<string, string>;
-
-  return Object.keys(rec).every(
-    (key) => typeof key === "string" && typeof rec[key] === "string",
+    (opts.headers === undefined || isStringRecord(opts.headers)) &&
+    (opts.ignoreErrors === undefined || typeof opts.ignoreErrors === "boolean")
   );
 }
diff --git a/src/config/job-config/actions/urlaction/urlaction.spec.ts b/src/config/job-config/actions/urlaction/urlaction.spec.ts
index d1d95ac1a..ed8104248 100644
--- a/src/config/job-config/actions/urlaction/urlaction.spec.ts
+++ b/src/config/job-config/actions/urlaction/urlaction.spec.ts
@@ -39,7 +39,7 @@ describe("URLJobAction", () => {
       text: jest.fn().mockResolvedValue("OK"),
     });
 
-    const context = { request: job, job, env: process.env };
+    const context = { request: job, job, env: process.env, datasets: [] };
     await action.perform(context);
 
     expect(global.fetch).toHaveBeenCalledWith(
@@ -63,9 +63,9 @@ describe("URLJobAction", () => {
       text: jest.fn().mockResolvedValue("Internal Server Error"),
     });
 
-    const context = { request: job, job, env: process.env };
+    const context = { request: job, job, env: process.env, datasets: [] };
     await expect(action.perform(context)).rejects.toThrow(
-      "Got response: Internal Server Error",
+      "A remote URL call failed with response: Internal Server Error",
     );
 
     expect(global.fetch).toHaveBeenCalledWith(
@@ -80,4 +80,33 @@ describe("URLJobAction", () => {
       },
     );
   });
+
+  it("should ignore errors if the ignoreError is set", async () => {
+    const job = { id: "12345" } as JobClass;
+    (global.fetch as jest.Mock).mockResolvedValue({
+      ok: false,
+      status: 500,
+      text: jest.fn().mockResolvedValue("Internal Server Error"),
+    });
+
+    const actionIgnore = new URLJobAction<CreateJobDto>({
+      ...config,
+      ignoreErrors: true,
+    });
+
+    const context = { request: job, job, env: process.env, datasets: [] };
+    await expect(actionIgnore.perform(context)).resolves.toBeUndefined();
+
+    expect(global.fetch).toHaveBeenCalledWith(
+      "http://localhost:3000/api/v3/health?jobid=12345",
+      {
+        method: "GET",
+        headers: {
+          accept: "application/json",
+          Authorization: "Bearer TheAuthToken",
+        },
+        body: "This is the body.",
+      },
+    );
+  });
 });
diff --git a/src/config/job-config/actions/urlaction/urlaction.ts b/src/config/job-config/actions/urlaction/urlaction.ts
index a2bb0363a..d181c7c6e 100644
--- a/src/config/job-config/actions/urlaction/urlaction.ts
+++ b/src/config/job-config/actions/urlaction/urlaction.ts
@@ -15,6 +15,7 @@ export class URLJobAction<T extends JobDto> implements JobAction<T> {
   private method = "GET";
   private headerTemplates?: Record<string, TemplateJob> = {};
   private bodyTemplate?: TemplateJob;
+  private ignoreErrors = false;
 
   getActionType(): string {
     return actionType;
@@ -37,20 +38,27 @@ export class URLJobAction<T extends JobDto> implements JobAction<T> {
       body: this.bodyTemplate ? this.bodyTemplate(context) : undefined,
     });
 
-    Logger.log(
-      `(Job ${context.job.id}) Request for ${url} returned ${response.status}`,
-      "URLAction",
-    );
-    if (!response.ok) {
-      const text = await response.text();
-      Logger.error(`(Job ${context.job.id}) Got response: ${text}`);
-      throw new HttpException(
-        {
-          status: response.status,
-          message: `Got response: ${text}`,
-        },
-        response.status,
+    const text = await response.text();
+    if (response.ok) {
+      Logger.log(
+        `(Job ${context.job.id}) Request for ${url} returned ${response.status}. Response: ${text}`,
+        "URLAction",
       );
+    } else {
+      Logger.error(
+        `(Job ${context.job.id}) Request for ${url} returned ${response.status}. Response: ${text}`,
+        "URLAction",
+      );
+
+      if (!this.ignoreErrors) {
+        throw new HttpException(
+          {
+            status: response.status,
+            message: `A remote URL call failed with response: ${text}`,
+          },
+          response.status,
+        );
+      }
     }
   }
 
@@ -85,5 +93,9 @@ export class URLJobAction<T extends JobDto> implements JobAction<T> {
     if (options["body"]) {
       this.bodyTemplate = compileJobTemplate(options["body"]);
     }
+
+    if (options["ignoreErrors"]) {
+      this.ignoreErrors = options.ignoreErrors;
+    }
   }
 }

From 1952d6b7736005025c73404c5f4dddfb8d59a518 Mon Sep 17 00:00:00 2001
From: Carlo Minotti <50220438+minottic@users.noreply.github.com>
Date: Tue, 14 Oct 2025 10:54:00 +0200
Subject: [PATCH 21/26] fix: change hardcoded admin to ADMIN_GROUPS (#2272)

---
 src/jobs/jobs.controller.utils.ts |  2 +-
 test/JobsV3.js                    | 35 +++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/src/jobs/jobs.controller.utils.ts b/src/jobs/jobs.controller.utils.ts
index b4c3f9d8c..7bb29463e 100644
--- a/src/jobs/jobs.controller.utils.ts
+++ b/src/jobs/jobs.controller.utils.ts
@@ -570,7 +570,7 @@ export class JobsControllerUtils {
     // check if the user can create this job
     const canCreate =
       (ability.can(Action.JobCreateAny, JobClass) &&
-        user.currentGroups.includes("admin")) ||
+        user.currentGroups.some((g) => this.accessGroups?.admin.includes(g))) ||
       (ability.can(Action.JobCreateAny, JobClass) && datasetsNoAccess == 0) ||
       ability.can(Action.JobCreateOwner, jobInstance) ||
       (ability.can(Action.JobCreateConfiguration, jobInstance) &&
diff --git a/test/JobsV3.js b/test/JobsV3.js
index 520c55cde..ad5933526 100644
--- a/test/JobsV3.js
+++ b/test/JobsV3.js
@@ -1229,6 +1229,41 @@ describe("1191: Jobs: Test Backwards Compatibility", () => {
         res.body.should.have.property("ownerGroup").to.be.equal("group5");
       });
   });
+
+  it("0440: Add via /api/v3 an anonymous job with custom jobStatusMessage, as a user from ADMIN_GROUPS: adminingestor", async () => {
+    jobCreateDtoByAdmin = {
+      ...jobDatasetPublic,
+      datasetList: [{ pid: datasetPid1, files: [] }],
+      jobStatusMessage: "custom_message"
+    };
+
+    return request(appUrl)
+      .post("/api/v3/Jobs")
+      .send(jobCreateDtoByAdmin)
+      .set("Accept", "application/json")
+      .set({ Authorization: `Bearer ${accessTokenAdminIngestor}` })
+      .expect(TestData.EntryCreatedStatusCode)
+      .expect("Content-Type", /json/)
+      .then((res) => {
+        res.body.should.have.property("id");
+        res.body.should.have.property("creationTime");
+        res.body.should.have.property("type").and.be.string;
+        res.body.should.have
+          .property("jobStatusMessage")
+          .to.be.equal("custom_message");
+        res.body.should.have
+          .property("datasetList")
+          .that.deep.equals(jobCreateDtoByAdmin.datasetList);
+        res.body.should.have.property("jobParams").that.deep.equals({});
+        res.body.should.have
+          .property("emailJobInitiator")
+          .to.be.equal(TestData.Accounts["adminIngestor"]["email"]);
+        res.body.should.not.have.property("ownerUser");
+        res.body.should.not.have.property("executionTime");
+        encodedJobOwnedByAdmin = encodeURIComponent(res.body["id"]);
+      });
+  });
+
   describe("1192: Jobs: Test datasetDetails backwards Compatibility", () => {
     before(async () => {
       const newJob = {

From 6d279030872c2923925f02ec353f134fe6d3f2a4 Mon Sep 17 00:00:00 2001
From: abdimo101 <abdi.abdulle@outlook.dk>
Date: Wed, 15 Oct 2025 09:42:26 +0200
Subject: [PATCH 22/26] fix(dataset): ensure that fields required for the
 lookups are included (#2212)

* ensures fields required for the lookups are included

* changed comments and eslint fix

* API test added

* ensures fields required for the lookups are included

* changed comments and eslint fix

* API test added

* modified parsePipelineProjection to add include field to the projection

* fixed failing tests

* changes to fix failing tests + added a new api test

* eslint fix

* removed .only from api test

* fixed api test

* removed fields inside scope and added some requested changes

* reverted back some changes to only fix the bug, also added error handling for path collision and a warning in swagger

* eslint fix

* fixed failing api tests

---------

Co-authored-by: Carlo Minotti <50220438+minottic@users.noreply.github.com>
---
 src/datasets/datasets.service.ts       | 67 +++++++++++++++++------
 src/datasets/datasets.v4.controller.ts |  5 +-
 test/DatasetV4.js                      | 73 +++++++++++++++++++++-----
 test/DatasetV4Public.js                | 12 ++---
 4 files changed, 121 insertions(+), 36 deletions(-)

diff --git a/src/datasets/datasets.service.ts b/src/datasets/datasets.service.ts
index 985dfc43a..1a6f6f9ad 100644
--- a/src/datasets/datasets.service.ts
+++ b/src/datasets/datasets.service.ts
@@ -1,4 +1,10 @@
-import { Inject, Injectable, NotFoundException, Scope } from "@nestjs/common";
+import {
+  BadRequestException,
+  Inject,
+  Injectable,
+  NotFoundException,
+  Scope,
+} from "@nestjs/common";
 import { ConfigService } from "@nestjs/config";
 import { REQUEST } from "@nestjs/core";
 import { InjectModel } from "@nestjs/mongoose";
@@ -77,6 +83,7 @@ export class DatasetsService {
       this.extractRelationsAndScopes(datasetLookupFields);
 
     const scopes = relationsAndScopes.scopes;
+    const addedRelations: string[] = [];
     for (const field of relationsAndScopes.relations) {
       const fieldValue = structuredClone(DATASET_LOOKUP_FIELDS[field]);
       if (!fieldValue) continue;
@@ -97,7 +104,7 @@ export class DatasetsService {
         includePipeline.push({ $limit: scope.limits.limit });
       if (scope?.limits?.sort) {
         const sort = parsePipelineSort(scope.limits.sort);
-        pipeline.push({ $sort: sort });
+        includePipeline.push({ $sort: sort });
       }
 
       if (includePipeline.length > 0)
@@ -106,7 +113,9 @@ export class DatasetsService {
         ).concat(includePipeline);
 
       pipeline.push(fieldValue);
+      addedRelations.push(field);
     }
+    return addedRelations;
   }
 
   private extractRelationsAndScopes(
@@ -183,7 +192,7 @@ export class DatasetsService {
     filter: IDatasetFiltersV4<DatasetDocument, IDatasetFields>,
   ): Promise<PartialOutputDatasetDto[]> {
     const whereFilter: FilterQuery<DatasetDocument> = filter.where ?? {};
-    const fieldsProjection = (filter.fields ?? []) as string[];
+    let fieldsProjection = (filter.fields ?? []) as string[];
     const limits = filter.limits ?? {
       limit: 10,
       skip: 0,
@@ -191,7 +200,13 @@ export class DatasetsService {
     };
 
     const pipeline: PipelineStage[] = [{ $match: whereFilter }];
-    this.addLookupFields(pipeline, filter.include);
+    const addedRelations = this.addLookupFields(pipeline, filter.include);
+
+    if (Array.isArray(fieldsProjection) && fieldsProjection.length > 0) {
+      fieldsProjection = Array.from(
+        new Set([...fieldsProjection, ...addedRelations]),
+      );
+    }
 
     if (!isEmpty(fieldsProjection)) {
       const projection = parsePipelineProjection(fieldsProjection);
@@ -206,12 +221,18 @@ export class DatasetsService {
     pipeline.push({ $skip: limits.skip || 0 });
 
     pipeline.push({ $limit: limits.limit || 10 });
+    try {
+      const data = await this.datasetModel
+        .aggregate<PartialOutputDatasetDto>(pipeline)
+        .exec();
 
-    const data = await this.datasetModel
-      .aggregate<PartialOutputDatasetDto>(pipeline)
-      .exec();
-
-    return data;
+      return data;
+    } catch (error) {
+      if (error instanceof Error) {
+        throw new BadRequestException(error.message);
+      }
+      throw new BadRequestException("An unknown error occurred");
+    }
   }
 
   async fullquery(
@@ -312,13 +333,22 @@ export class DatasetsService {
     filter: FilterQuery<DatasetDocument>,
   ): Promise<OutputDatasetDto | null> {
     const whereFilter: FilterQuery<DatasetDocument> = filter.where ?? {};
-    const fieldsProjection: string[] = filter.fields ?? {};
+    let fieldsProjection: string[] = filter.fields ?? {};
+
     const limits: QueryOptions<DatasetDocument> = filter.limits ?? {
       skip: 0,
       sort: { createdAt: "desc" },
     };
 
     const pipeline: PipelineStage[] = [{ $match: whereFilter }];
+    const addedRelations = this.addLookupFields(pipeline, filter.include);
+
+    if (Array.isArray(fieldsProjection)) {
+      fieldsProjection = Array.from(
+        new Set([...fieldsProjection, ...addedRelations]),
+      );
+    }
+
     if (!isEmpty(fieldsProjection)) {
       const projection = parsePipelineProjection(fieldsProjection);
       pipeline.push({ $project: projection });
@@ -331,13 +361,18 @@ export class DatasetsService {
 
     pipeline.push({ $skip: limits.skip || 0 });
 
-    this.addLookupFields(pipeline, filter.include);
-
-    const [data] = await this.datasetModel
-      .aggregate<OutputDatasetDto | undefined>(pipeline)
-      .exec();
+    try {
+      const [data] = await this.datasetModel
+        .aggregate<OutputDatasetDto | undefined>(pipeline)
+        .exec();
 
-    return data || null;
+      return data || null;
+    } catch (error) {
+      if (error instanceof Error) {
+        throw new BadRequestException(error.message);
+      }
+      throw new BadRequestException("An unknown error occurred");
+    }
   }
 
   async count(
diff --git a/src/datasets/datasets.v4.controller.ts b/src/datasets/datasets.v4.controller.ts
index 35c6b6a47..3cd0fa7ff 100644
--- a/src/datasets/datasets.v4.controller.ts
+++ b/src/datasets/datasets.v4.controller.ts
@@ -422,7 +422,8 @@ export class DatasetsV4Controller {
   })
   @ApiQuery({
     name: "filter",
-    description: "Database filters to apply when retrieving datasets",
+    description:
+      "Database filters to apply when retrieving datasets <br> ⚠️ Do not include both a parent field (e.g. 'proposals') and its subfields (e.g. 'proposals.type') in the same request, as this will cause a path collision error. ",
     required: false,
     type: String,
     content: getSwaggerDatasetFilterContent(),
@@ -593,7 +594,7 @@ export class DatasetsV4Controller {
   @ApiOperation({
     summary: "It returns the first dataset found.",
     description:
-      "It returns the first dataset of the ones that matches the filter provided. The list returned can be modified by providing a filter.",
+      "It returns the first dataset of the ones that matches the filter provided. The list returned can be modified by providing a filter.<br> ⚠️ Do not include both a parent field (e.g. 'proposals') and its subfields (e.g. 'proposals.type') in the same request, as this will cause a path collision error.",
   })
   @ApiQuery({
     name: "filter",
diff --git a/test/DatasetV4.js b/test/DatasetV4.js
index 0ad127ae7..58b849b85 100644
--- a/test/DatasetV4.js
+++ b/test/DatasetV4.js
@@ -612,7 +612,6 @@ describe("2500: Datasets v4 tests", () => {
         .then((res) => {
           res.body.should.be.a("array");
           const [firstDataset] = res.body;
-
           firstDataset.should.have.property("pid");
           firstDataset.should.have.property("instruments");
           firstDataset.should.have.property("proposals");
@@ -703,12 +702,12 @@ describe("2500: Datasets v4 tests", () => {
             dataset.should.have.property("datasetName");
             dataset.should.have.property("pid");
             dataset.should.not.have.property("description");
-            dataset.should.not.have.property("instruments");
-            dataset.should.not.have.property("proposals");
-            dataset.should.not.have.property("datablocks");
-            dataset.should.not.have.property("attachments");
-            dataset.should.not.have.property("origdatablocks");
-            dataset.should.not.have.property("samples");
+            dataset.should.have.property("instruments");
+            dataset.should.have.property("proposals");
+            dataset.should.have.property("datablocks");
+            dataset.should.have.property("attachments");
+            dataset.should.have.property("origdatablocks");
+            dataset.should.have.property("samples");
 
             dataset.datasetName.should.match(/Dataset/i);
           });
@@ -907,7 +906,33 @@ describe("2500: Datasets v4 tests", () => {
         });
     });
 
-    it("0303: should fetch dataset relation fields if provided in the filter", async () => {
+    it("0303: should fetch specific dataset fields only if fields is provided in the filter with relationships", async () => {
+      const filter = {
+        include: ["instruments", "proposals"],
+        fields: ["datasetName"],
+      };
+
+      return request(appUrl)
+        .get(`/api/v4/datasets/findOne`)
+        .query({ filter: JSON.stringify(filter) })
+        .auth(accessTokenAdminIngestor, { type: "bearer" })
+        .expect(TestData.SuccessfulGetStatusCode)
+        .expect("Content-Type", /json/)
+        .then((res) => {
+          res.body.should.be.a("object");
+
+          res.body.should.have.property("datasetName");
+
+          res.body.should.not.have.property("description");
+          res.body.should.not.have.property("pid");
+
+          res.body.should.have.property("instruments");
+          res.body.should.have.property("proposals");
+          res.body.should.not.have.property("datablocks");
+        });
+    });
+
+    it("0304: should fetch dataset relation fields if provided in the filter", async () => {
       const filter = {
         include: ["instruments", "proposals"],
       };
@@ -928,7 +953,7 @@ describe("2500: Datasets v4 tests", () => {
         });
     });
 
-    it("0304: should fetch all dataset relation fields if provided in the filter", async () => {
+    it("0305: should fetch all dataset relation fields if provided in the filter", async () => {
       const filter = {
         include: ["all"],
       };
@@ -952,7 +977,7 @@ describe("2500: Datasets v4 tests", () => {
         });
     });
 
-    it("0305: should be able to fetch the dataset providing where filter", async () => {
+    it("0306: should be able to fetch the dataset providing where filter", async () => {
       const filter = {
         where: {
           datasetName: {
@@ -974,7 +999,7 @@ describe("2500: Datasets v4 tests", () => {
         });
     });
 
-    it("0306: should be able to fetch a dataset providing all allowed filters together", async () => {
+    it("0307: should be able to fetch a dataset providing all allowed filters together", async () => {
       const filter = {
         where: {
           datasetName: {
@@ -1017,7 +1042,7 @@ describe("2500: Datasets v4 tests", () => {
         });
     });
 
-    it("0307: should not be able to provide filters that are not allowed", async () => {
+    it("0308: should not be able to provide filters that are not allowed", async () => {
       const filter = {
         customField: { datasetName: "test" },
       };
@@ -1029,6 +1054,30 @@ describe("2500: Datasets v4 tests", () => {
         .expect(TestData.BadRequestStatusCode)
         .expect("Content-Type", /json/);
     });
+
+    it("0309: should throw BadRequest when subfields within the embedded documents are used in the fields projection", async () => {
+      const filter = {
+        include: ["all"],
+        fields: [
+          "datasetName",
+          "attachments.thumbnail",
+          "attachments.relationships.targetType",
+          "origdatablocks.dataFileList.path",
+        ],
+      };
+
+      return request(appUrl)
+        .get(`/api/v4/datasets/findOne`)
+        .query({ filter: JSON.stringify(filter) })
+        .auth(accessTokenAdminIngestor, { type: "bearer" })
+        .expect(TestData.BadRequestStatusCode)
+        .expect("Content-Type", /json/)
+        .then((res) => {
+          res.body.should.be.a("object");
+          res.body.should.have.property("message");
+          res.body.message.should.match(/Invalid \$project :: caused by :: Path collision at origdatablocks/);
+        });
+    });
   });
 
   describe("Datasets v4 count tests", () => {
diff --git a/test/DatasetV4Public.js b/test/DatasetV4Public.js
index 797fa93e1..c07173991 100644
--- a/test/DatasetV4Public.js
+++ b/test/DatasetV4Public.js
@@ -269,12 +269,12 @@ describe("2600: Datasets v4 public endpoints tests", () => {
             dataset.should.have.property("datasetName");
             dataset.should.have.property("pid");
             dataset.should.not.have.property("description");
-            dataset.should.not.have.property("instruments");
-            dataset.should.not.have.property("proposals");
-            dataset.should.not.have.property("datablocks");
-            dataset.should.not.have.property("attachments");
-            dataset.should.not.have.property("origdatablocks");
-            dataset.should.not.have.property("samples");
+            dataset.should.have.property("instruments");
+            dataset.should.have.property("proposals");
+            dataset.should.have.property("datablocks");
+            dataset.should.have.property("attachments");
+            dataset.should.have.property("origdatablocks");
+            dataset.should.have.property("samples");
 
             dataset.datasetName.should.match(/Dataset/i);
           });

From 993c33fe1bb0fd426750d80a43a6d6e1a4e024b2 Mon Sep 17 00:00:00 2001
From: fpotier <58594145+fpotier@users.noreply.github.com>
Date: Wed, 15 Oct 2025 15:15:27 +0200
Subject: [PATCH 23/26] fix: respect the `PublishedData.status` when creating
 with /api/v3 (#2249)

Creating a PublishedData with the v3 api was not respecting the `status`
in the DTO and always creating the instance with
`PublishedDataStatus.PRIVATE`
---
 .../published-data.controller.spec.ts         |  42 ++++++
 .../published-data.controller.ts              | 123 +++++++++++-------
 2 files changed, 116 insertions(+), 49 deletions(-)

diff --git a/src/published-data/published-data.controller.spec.ts b/src/published-data/published-data.controller.spec.ts
index fbe2c4889..84ba776ef 100644
--- a/src/published-data/published-data.controller.spec.ts
+++ b/src/published-data/published-data.controller.spec.ts
@@ -5,6 +5,7 @@ import { AttachmentsService } from "src/attachments/attachments.service";
 import { CaslAbilityFactory } from "src/casl/casl-ability.factory";
 import { DatasetsService } from "src/datasets/datasets.service";
 import { ProposalsService } from "src/proposals/proposals.service";
+import { PublishedDataStatus } from "./interfaces/published-data.interface";
 import { PublishedDataController } from "./published-data.controller";
 import { PublishedDataService } from "./published-data.service";
 
@@ -43,4 +44,45 @@ describe("PublishedDataController", () => {
   it("should be defined", () => {
     expect(controller).toBeDefined();
   });
+
+  it("should result in PublishedDataStatus.REGISTERED", () => {
+    expect(controller.convertObsoleteStatusToCurrent("registered")).toEqual(
+      PublishedDataStatus.REGISTERED,
+    );
+  });
+  it("should result in PublishedDataStatus.PRIVATE", () => {
+    expect(
+      controller.convertObsoleteStatusToCurrent("pending_registration"),
+    ).toEqual(PublishedDataStatus.PRIVATE);
+  });
+  it("should result in PublishedDataStatus.PRIVATE", () => {
+    expect(controller.convertObsoleteStatusToCurrent("invalid_status")).toEqual(
+      PublishedDataStatus.PRIVATE,
+    );
+  });
+  it("should result in 'pending_registration'", () => {
+    expect(controller.convertCurrentStatusToObsolete(undefined)).toEqual(
+      "pending_registration",
+    );
+  });
+  it("should result in 'pending_registration'", () => {
+    expect(
+      controller.convertCurrentStatusToObsolete(PublishedDataStatus.PRIVATE),
+    ).toEqual("pending_registration");
+  });
+  it("should result in 'pending_registration'", () => {
+    expect(
+      controller.convertCurrentStatusToObsolete(PublishedDataStatus.PUBLIC),
+    ).toEqual("pending_registration");
+  });
+  it("should result in 'registered'", () => {
+    expect(
+      controller.convertCurrentStatusToObsolete(PublishedDataStatus.REGISTERED),
+    ).toEqual("registered");
+  });
+  it("should result in 'registered'", () => {
+    expect(
+      controller.convertCurrentStatusToObsolete(PublishedDataStatus.AMENDED),
+    ).toEqual("registered");
+  });
 });
diff --git a/src/published-data/published-data.controller.ts b/src/published-data/published-data.controller.ts
index 67288ec0a..bfd9bb07f 100644
--- a/src/published-data/published-data.controller.ts
+++ b/src/published-data/published-data.controller.ts
@@ -1,25 +1,22 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
+import { HttpService } from "@nestjs/axios";
 import {
-  Controller,
-  Get,
-  Post,
+  BadRequestException,
   Body,
-  Patch,
-  Param,
+  Controller,
   Delete,
-  UseGuards,
-  Query,
+  Get,
   HttpException,
   HttpStatus,
+  Logger,
   NotFoundException,
-  BadRequestException,
+  Param,
+  Patch,
+  Post,
+  Query,
+  UseGuards,
 } from "@nestjs/common";
-import { PublishedDataService } from "./published-data.service";
-import { CreatePublishedDataDto } from "./dto/create-published-data.dto";
-import {
-  PartialUpdatePublishedDataDto,
-  UpdatePublishedDataDto,
-} from "./dto/update-published-data.dto";
+import { ConfigService } from "@nestjs/config";
 import {
   ApiBearerAuth,
   ApiBody,
@@ -29,43 +26,47 @@ import {
   ApiResponse,
   ApiTags,
 } from "@nestjs/swagger";
-import { PoliciesGuard } from "src/casl/guards/policies.guard";
-import { CheckPolicies } from "src/casl/decorators/check-policies.decorator";
-import { AppAbility } from "src/casl/casl-ability.factory";
+import { FilterQuery, QueryOptions } from "mongoose";
+import { firstValueFrom } from "rxjs";
+import { AttachmentsService } from "src/attachments/attachments.service";
+import { AllowAny } from "src/auth/decorators/allow-any.decorator";
 import { Action } from "src/casl/action.enum";
+import { AppAbility } from "src/casl/casl-ability.factory";
+import { CheckPolicies } from "src/casl/decorators/check-policies.decorator";
+import { PoliciesGuard } from "src/casl/guards/policies.guard";
+import { FilterPipe } from "src/common/pipes/filter.pipe";
+import { handleAxiosRequestError } from "src/common/utils";
+import { DatasetsService } from "src/datasets/datasets.service";
+import { DatasetClass } from "src/datasets/schemas/dataset.schema";
+import { ProposalsService } from "src/proposals/proposals.service";
+import { CreatePublishedDataDto } from "./dto/create-published-data.dto";
+import { CreatePublishedDataV4Dto } from "./dto/create-published-data.v4.dto";
+import { PublishedDataObsoleteDto } from "./dto/published-data.obsolete.dto";
 import {
-  PublishedData,
-  PublishedDataDocument,
-} from "./schemas/published-data.schema";
+  PartialUpdatePublishedDataDto,
+  UpdatePublishedDataDto,
+} from "./dto/update-published-data.dto";
+import {
+  PartialUpdatePublishedDataV4Dto,
+  UpdatePublishedDataV4Dto,
+} from "./dto/update-published-data.v4.dto";
 import {
-  ICount,
   FormPopulateData,
+  ICount,
   IPublishedDataFilters,
   IRegister,
   PublishedDataStatus,
 } from "./interfaces/published-data.interface";
-import { AllowAny } from "src/auth/decorators/allow-any.decorator";
 import {
   IdToDoiPipe,
   RegisteredFilterPipe,
   RegisteredPipe,
 } from "./pipes/registered.pipe";
-import { FilterQuery, QueryOptions } from "mongoose";
-import { DatasetsService } from "src/datasets/datasets.service";
-import { ProposalsService } from "src/proposals/proposals.service";
-import { AttachmentsService } from "src/attachments/attachments.service";
-import { HttpService } from "@nestjs/axios";
-import { ConfigService } from "@nestjs/config";
-import { firstValueFrom } from "rxjs";
-import { handleAxiosRequestError } from "src/common/utils";
-import { DatasetClass } from "src/datasets/schemas/dataset.schema";
-import { CreatePublishedDataV4Dto } from "./dto/create-published-data.v4.dto";
+import { PublishedDataService } from "./published-data.service";
 import {
-  PartialUpdatePublishedDataV4Dto,
-  UpdatePublishedDataV4Dto,
-} from "./dto/update-published-data.v4.dto";
-import { PublishedDataObsoleteDto } from "./dto/published-data.obsolete.dto";
-import { FilterPipe } from "src/common/pipes/filter.pipe";
+  PublishedData,
+  PublishedDataDocument,
+} from "./schemas/published-data.schema";
 
 @ApiBearerAuth()
 @ApiTags("published data")
@@ -80,6 +81,34 @@ export class PublishedDataController {
     private readonly publishedDataService: PublishedDataService,
   ) {}
 
+  convertObsoleteStatusToCurrent(obsoleteStatus: string): PublishedDataStatus {
+    switch (obsoleteStatus) {
+      case "registered":
+        return PublishedDataStatus.REGISTERED;
+      case "pending_registration":
+        return PublishedDataStatus.PRIVATE;
+      default:
+        Logger.error(
+          `Unknown PublishedData.status '${obsoleteStatus}' defaulting to PublishedDataStatus.PRIVATE`,
+        );
+        return PublishedDataStatus.PRIVATE;
+    }
+  }
+
+  convertCurrentStatusToObsolete(
+    currentStatus: PublishedDataStatus | undefined,
+  ): string {
+    switch (currentStatus) {
+      case undefined:
+      case PublishedDataStatus.PRIVATE:
+      case PublishedDataStatus.PUBLIC:
+        return "pending_registration";
+      case PublishedDataStatus.REGISTERED:
+      case PublishedDataStatus.AMENDED:
+        return "registered";
+    }
+  }
+
   convertObsoleteToCurrentSchema(
     inputObsoletePublishedData:
       | CreatePublishedDataDto
@@ -168,11 +197,13 @@ export class PublishedDataController {
       propertiesModifier.datasetPids = inputObsoletePublishedData.pidArray;
     }
 
-    if ("status" in inputObsoletePublishedData) {
-      propertiesModifier.status =
-        inputObsoletePublishedData.status === "registered"
-          ? "registered"
-          : "private";
+    if (
+      "status" in inputObsoletePublishedData &&
+      typeof inputObsoletePublishedData.status === "string"
+    ) {
+      propertiesModifier.status = this.convertObsoleteStatusToCurrent(
+        inputObsoletePublishedData.status,
+      );
     }
 
     let outputPublishedData:
@@ -183,19 +214,16 @@ export class PublishedDataController {
     if (inputObsoletePublishedData instanceof CreatePublishedDataDto) {
       outputPublishedData = {
         ...propertiesModifier,
-        status: PublishedDataStatus.PRIVATE,
       } as CreatePublishedDataV4Dto;
     } else if (inputObsoletePublishedData instanceof UpdatePublishedDataDto) {
       outputPublishedData = {
         ...propertiesModifier,
-        status: inputObsoletePublishedData.status,
       } as UpdatePublishedDataV4Dto;
     } else if (
       inputObsoletePublishedData instanceof PartialUpdatePublishedDataDto
     ) {
       outputPublishedData = {
         ...propertiesModifier,
-        status: inputObsoletePublishedData.status,
       } as PartialUpdatePublishedDataV4Dto;
     }
 
@@ -241,10 +269,7 @@ export class PublishedDataController {
         inputPublishedData.metadata?.relatedIdentifiers as object[]
       )?.map((identifier: any) => identifier.relatedIdentifier),
       pidArray: inputPublishedData.datasetPids,
-      status:
-        inputPublishedData.status === "registered"
-          ? "registered"
-          : "pending_registration",
+      status: this.convertCurrentStatusToObsolete(inputPublishedData.status),
     };
 
     return propertiesModifier;

From 624d42c189d5c50bdfcd3b46c9a0259591ecde5d Mon Sep 17 00:00:00 2001
From: Omkar Zade <omkar.zade@psi.ch>
Date: Thu, 16 Oct 2025 11:29:56 +0200
Subject: [PATCH 24/26] feat: add backwards compatible history to v3/datasets
 response (#2282)

* Try add backwards compatible history array to /v3 datasets outputs

* Add adapters to/from GenericHistory to HistoryClass

* Do not mutate currentDataset in convertToObsoleteHistories function

* Only add datasetlifecycle to previousValue if included in the snapshot

* Address PR comments: move history-util to datasets/

* Address PR comment: remove obsolete to generic history converter

---------

Co-authored-by: Carlo Minotti <50220438+minottic@users.noreply.github.com>
---
 src/datasets/datasets.controller.spec.ts      |   4 +
 src/datasets/datasets.controller.ts           |  37 +--
 src/datasets/datasets.module.ts               |   2 +
 src/datasets/datasets.service.spec.ts         |   1 -
 src/datasets/datasets.service.ts              |  14 +-
 .../dto/output-dataset-obsolete.dto.ts        |  13 ++
 src/datasets/schemas/history.schema.ts        |   1 +
 src/datasets/utils/history.util.spec.ts       | 214 ++++++++++++++++++
 src/datasets/utils/history.util.ts            |  60 +++++
 src/history/history.module.ts                 |   1 +
 test/TestData.js                              |   1 +
 11 files changed, 328 insertions(+), 20 deletions(-)
 create mode 100644 src/datasets/utils/history.util.spec.ts
 create mode 100644 src/datasets/utils/history.util.ts

diff --git a/src/datasets/datasets.controller.spec.ts b/src/datasets/datasets.controller.spec.ts
index 72bf6a09a..d605f06a8 100644
--- a/src/datasets/datasets.controller.spec.ts
+++ b/src/datasets/datasets.controller.spec.ts
@@ -4,6 +4,7 @@ import { DatablocksService } from "src/datablocks/datablocks.service";
 import { OrigDatablocksService } from "src/origdatablocks/origdatablocks.service";
 import { DatasetsController } from "./datasets.controller";
 import { DatasetsService } from "./datasets.service";
+import { HistoryService } from "src/history/history.service";
 import { LogbooksService } from "src/logbooks/logbooks.service";
 import { CaslAbilityFactory } from "src/casl/casl-ability.factory";
 import { ConfigModule } from "@nestjs/config";
@@ -20,6 +21,8 @@ class LogbooksServiceMock {}
 
 class CaslAbilityFactoryMock {}
 
+class HistoryServiceMock {}
+
 describe("DatasetsController", () => {
   let controller: DatasetsController;
 
@@ -34,6 +37,7 @@ describe("DatasetsController", () => {
         { provide: DatasetsService, useClass: DatasetsServiceMock },
         { provide: OrigDatablocksService, useClass: OrigDatablocksServiceMock },
         { provide: CaslAbilityFactory, useClass: CaslAbilityFactoryMock },
+        { provide: HistoryService, useClass: HistoryServiceMock },
       ],
     }).compile();
 
diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts
index 1ddc872c7..69c1539f2 100644
--- a/src/datasets/datasets.controller.ts
+++ b/src/datasets/datasets.controller.ts
@@ -113,6 +113,8 @@ import { LifecycleClass } from "./schemas/lifecycle.schema";
 import { RelationshipClass } from "./schemas/relationship.schema";
 import { TechniqueClass } from "./schemas/technique.schema";
 import { DatasetType } from "./types/dataset-type.enum";
+import { HistoryService } from "src/history/history.service";
+import { convertGenericHistoriesToObsoleteHistories } from "src/datasets/utils/history.util";
 
 @ApiBearerAuth()
 @ApiExtraModels(
@@ -134,6 +136,7 @@ export class DatasetsController {
     private caslAbilityFactory: CaslAbilityFactory,
     private logbooksService: LogbooksService,
     private configService: ConfigService,
+    private historyService: HistoryService,
   ) {
     this.accessGroups =
       this.configService.get<AccessGroupsType>("accessGroups");
@@ -578,9 +581,9 @@ export class DatasetsController {
     return outputDataset;
   }
 
-  convertCurrentToObsoleteSchema(
-    inputDataset: DatasetClass | null,
-  ): OutputDatasetObsoleteDto {
+  async convertCurrentToObsoleteSchema(
+    inputDataset: DatasetDocument | null,
+  ): Promise<OutputDatasetObsoleteDto> {
     const propertiesModifier: Record<string, unknown> = {};
     if (inputDataset) {
       if ("proposalIds" in inputDataset && inputDataset.proposalIds?.length) {
@@ -613,6 +616,14 @@ export class DatasetsController {
             inputDataset.principalInvestigators[0];
         }
       }
+
+      propertiesModifier.history = convertGenericHistoriesToObsoleteHistories(
+        await this.historyService.find({
+          documentId: inputDataset._id,
+          subsystem: "Dataset",
+        }),
+        inputDataset,
+      );
     }
 
     const outputDataset: OutputDatasetObsoleteDto = {
@@ -702,7 +713,7 @@ export class DatasetsController {
       ) as CreateDatasetDto;
       const createdDataset = await this.datasetsService.create(datasetDto);
       const outputObsoleteDatasetDto =
-        this.convertCurrentToObsoleteSchema(createdDataset);
+        await this.convertCurrentToObsoleteSchema(createdDataset);
 
       return outputObsoleteDatasetDto;
     } catch (error) {
@@ -911,8 +922,8 @@ export class DatasetsController {
     let outputDatasets: OutputDatasetObsoleteDto[] = [];
     if (datasets && datasets.length > 0) {
       const includeFilters = mergedFilters.include ?? [];
-      outputDatasets = datasets.map((dataset) =>
-        this.convertCurrentToObsoleteSchema(dataset),
+      outputDatasets = await Promise.all(
+        datasets.map((dataset) => this.convertCurrentToObsoleteSchema(dataset)),
       );
       await Promise.all(
         outputDatasets.map(async (dataset) => {
@@ -1038,8 +1049,8 @@ export class DatasetsController {
     let outputDatasets: OutputDatasetObsoleteDto[] = [];
 
     if (datasets && datasets.length > 0) {
-      outputDatasets = datasets.map((dataset) =>
-        this.convertCurrentToObsoleteSchema(dataset),
+      outputDatasets = await Promise.all(
+        datasets.map((dataset) => this.convertCurrentToObsoleteSchema(dataset)),
       );
     }
 
@@ -1347,7 +1358,7 @@ export class DatasetsController {
     description: "Dataset not found",
   })
   async findById(@Req() request: Request, @Param("pid") id: string) {
-    const dataset = this.convertCurrentToObsoleteSchema(
+    const dataset = await this.convertCurrentToObsoleteSchema(
       await this.checkPermissionsForDatasetObsolete(request, id),
     );
 
@@ -1458,7 +1469,7 @@ export class DatasetsController {
       validatedUpdateDatasetObsoleteDto,
     ) as UpdateDatasetDto;
 
-    const res = this.convertCurrentToObsoleteSchema(
+    const res = await this.convertCurrentToObsoleteSchema(
       await this.datasetsService.findByIdAndUpdate(pid, updateDatasetDto),
     );
     return res;
@@ -1568,7 +1579,7 @@ export class DatasetsController {
       updateDatasetDto as UpdateDatasetDto,
     );
 
-    return this.convertCurrentToObsoleteSchema(outputDatasetDto);
+    return await this.convertCurrentToObsoleteSchema(outputDatasetDto);
   }
 
   // GET /datasets/:id/datasetlifecycle
@@ -1601,7 +1612,7 @@ export class DatasetsController {
     description: "Dataset not found",
   })
   async findLifecycleById(@Req() request: Request, @Param("pid") id: string) {
-    const dataset = this.convertCurrentToObsoleteSchema(
+    const dataset = await this.convertCurrentToObsoleteSchema(
       await this.checkPermissionsForDatasetObsolete(request, id),
     );
     return dataset?.datasetlifecycle;
@@ -1681,7 +1692,7 @@ export class DatasetsController {
       throw new ForbiddenException("Unauthorized to update this dataset");
     }
 
-    const res = this.convertCurrentToObsoleteSchema(
+    const res = await this.convertCurrentToObsoleteSchema(
       await this.datasetsService.findByIdAndUpdate(pid, foundDataset),
     );
     return res.datasetlifecycle;
diff --git a/src/datasets/datasets.module.ts b/src/datasets/datasets.module.ts
index ff6a12465..c1c6df8d7 100644
--- a/src/datasets/datasets.module.ts
+++ b/src/datasets/datasets.module.ts
@@ -23,6 +23,7 @@ import {
 import { ConfigModule, ConfigService } from "@nestjs/config";
 import { applyHistoryPluginOnce } from "src/common/mongoose/plugins/history.plugin.util";
 import { ProposalsModule } from "src/proposals/proposals.module";
+import { HistoryModule } from "src/history/history.module";
 
 @Module({
   imports: [
@@ -31,6 +32,7 @@ import { ProposalsModule } from "src/proposals/proposals.module";
     DatablocksModule,
     OrigDatablocksModule,
     InitialDatasetsModule,
+    HistoryModule,
     ElasticSearchModule,
     ProposalsModule,
     forwardRef(() => LogbooksModule),
diff --git a/src/datasets/datasets.service.spec.ts b/src/datasets/datasets.service.spec.ts
index da0554482..ba73de5ba 100644
--- a/src/datasets/datasets.service.spec.ts
+++ b/src/datasets/datasets.service.spec.ts
@@ -45,7 +45,6 @@ const mockDataset: DatasetClass = {
   license: "string",
   version: "string",
   isPublished: false,
-  history: [],
   datasetlifecycle: {
     id: "testId",
     archivable: true,
diff --git a/src/datasets/datasets.service.ts b/src/datasets/datasets.service.ts
index 1a6f6f9ad..bdb37064c 100644
--- a/src/datasets/datasets.service.ts
+++ b/src/datasets/datasets.service.ts
@@ -172,7 +172,9 @@ export class DatasetsService {
     return savedDataset;
   }
 
-  async findAll(filter: FilterQuery<DatasetDocument>): Promise<DatasetClass[]> {
+  async findAll(
+    filter: FilterQuery<DatasetDocument>,
+  ): Promise<DatasetDocument[]> {
     const whereFilter: RootFilterQuery<DatasetDocument> = filter.where ?? {};
     const fieldsProjection: ProjectionType<DatasetDocument> =
       filter.fields ?? {};
@@ -238,7 +240,7 @@ export class DatasetsService {
   async fullquery(
     filter: IFilters<DatasetDocument, IDatasetFields>,
     extraWhereClause: FilterQuery<DatasetDocument> = {},
-  ): Promise<DatasetClass[] | null> {
+  ): Promise<DatasetDocument[] | null> {
     let datasets;
 
     const filterQuery: FilterQuery<DatasetDocument> =
@@ -322,7 +324,7 @@ export class DatasetsService {
 
   async findOne(
     filter: FilterQuery<DatasetDocument>,
-  ): Promise<DatasetClass | null> {
+  ): Promise<DatasetDocument | null> {
     const whereFilter: FilterQuery<DatasetDocument> = filter.where ?? {};
     const fieldsProjection: FilterQuery<DatasetDocument> = filter.fields ?? {};
 
@@ -400,7 +402,7 @@ export class DatasetsService {
   async findByIdAndReplace(
     id: string,
     updateDatasetDto: UpdateDatasetDto,
-  ): Promise<DatasetClass> {
+  ): Promise<DatasetDocument> {
     const username = (this.request.user as JWTUser).username;
     const existingDataset = await this.datasetModel.findOne({ pid: id }).exec();
 
@@ -444,7 +446,7 @@ export class DatasetsService {
     updateDatasetDto:
       | PartialUpdateDatasetDto
       | PartialUpdateDatasetWithHistoryDto,
-  ): Promise<DatasetClass | null> {
+  ): Promise<DatasetDocument | null> {
     const existingDataset = await this.datasetModel.findOne({ pid: id }).exec();
     // check if we were able to find the dataset
     if (!existingDataset) {
@@ -476,7 +478,7 @@ export class DatasetsService {
   }
 
   // DELETE dataset
-  async findByIdAndDelete(id: string): Promise<DatasetClass | null> {
+  async findByIdAndDelete(id: string): Promise<DatasetDocument | null> {
     if (this.ESClient) {
       await this.ESClient.deleteDocument(id);
     }
diff --git a/src/datasets/dto/output-dataset-obsolete.dto.ts b/src/datasets/dto/output-dataset-obsolete.dto.ts
index d367241a6..56e86dcb8 100644
--- a/src/datasets/dto/output-dataset-obsolete.dto.ts
+++ b/src/datasets/dto/output-dataset-obsolete.dto.ts
@@ -13,6 +13,7 @@ import { OrigDatablock } from "src/origdatablocks/schemas/origdatablock.schema";
 import { Datablock } from "src/datablocks/schemas/datablock.schema";
 import { DatasetType } from "../types/dataset-type.enum";
 import { OutputAttachmentV3Dto } from "src/attachments/dto-obsolete/output-attachment.v3.dto";
+import { HistoryClass } from "../schemas/history.schema";
 
 export class OutputDatasetObsoleteDto extends UpdateDatasetObsoleteDto {
   @ApiProperty({
@@ -200,6 +201,18 @@ export class OutputDatasetObsoleteDto extends UpdateDatasetObsoleteDto {
   @Type(() => Datablock)
   datablocks?: Datablock[];
 
+  @ApiProperty({
+    type: "array",
+    items: { $ref: getSchemaPath(HistoryClass) },
+    required: false,
+    description:
+      "List of changes made to this dataset record. For compatibility with legacy backend",
+  })
+  @IsOptional()
+  @IsArray()
+  @Type(() => HistoryClass)
+  history?: HistoryClass[];
+
   @ApiProperty({
     type: String,
     required: true,
diff --git a/src/datasets/schemas/history.schema.ts b/src/datasets/schemas/history.schema.ts
index 6210f1675..b4d88b31f 100644
--- a/src/datasets/schemas/history.schema.ts
+++ b/src/datasets/schemas/history.schema.ts
@@ -39,6 +39,7 @@ export class HistoryClass {
    * After the migration, we should look in to change the structure
    * to a new one that can be better validated
    */
+  [key: string]: unknown;
 
   @ApiProperty({
     type: Date,
diff --git a/src/datasets/utils/history.util.spec.ts b/src/datasets/utils/history.util.spec.ts
new file mode 100644
index 000000000..c2af6076e
--- /dev/null
+++ b/src/datasets/utils/history.util.spec.ts
@@ -0,0 +1,214 @@
+import {
+  convertGenericHistoriesToObsoleteHistories,
+  convertGenericHistoryToObsoleteHistory,
+} from "./history.util";
+import {
+  DatasetClass,
+  DatasetDocument,
+} from "src/datasets/schemas/dataset.schema";
+import { GenericHistory } from "../../common/schemas/generic-history.schema";
+
+describe("History Utility Functions", () => {
+  it("should convert generic history to obsolete history", () => {
+    const genericHistory: GenericHistory = {
+      subsystem: "Dataset",
+      documentId: "pid123",
+      user: "user123",
+      operation: "update",
+      timestamp: new Date("2023-10-01T12:00:00Z"),
+      before: {
+        isPublished: false,
+        datasetlifecycle: {
+          publishedOn: undefined,
+          archivable: false,
+        },
+      },
+      after: {
+        datasetlifecycle: {
+          publishedOn: new Date("2023-10-01T12:00:00Z"),
+          archivable: true,
+        },
+        isPublished: true,
+      },
+    };
+
+    const currentDataset: Partial<DatasetClass> = {
+      isPublished: true,
+      datasetlifecycle: {
+        publishedOn: new Date("2023-10-01T12:00:00Z"),
+        archivable: true,
+        retrievable: true,
+        publishable: false,
+        archiveRetentionTime: new Date("2031-10-01T12:00:00Z"),
+        dateOfPublishing: new Date("2024-10-01T12:00:00Z"),
+        isOnCentralDisk: true,
+        archiveStatusMessage: "datasetOnArchiveDisk",
+        retrieveStatusMessage: "",
+        retrieveIntegrityCheck: false,
+      },
+    };
+    const obsoleteHistory = convertGenericHistoryToObsoleteHistory(
+      genericHistory,
+      currentDataset,
+    );
+
+    expect(obsoleteHistory).toEqual({
+      updatedAt: new Date("2023-10-01T12:00:00Z"),
+      updatedBy: "user123",
+      isPublished: {
+        previousValue: false,
+        currentValue: true,
+      },
+      datasetlifecycle: {
+        previousValue: {
+          publishedOn: undefined,
+          archivable: false,
+          retrievable: true,
+          publishable: false,
+          archiveRetentionTime: new Date("2031-10-01T12:00:00Z").toISOString(),
+          dateOfPublishing: new Date("2024-10-01T12:00:00Z").toISOString(),
+          isOnCentralDisk: true,
+          archiveStatusMessage: "datasetOnArchiveDisk",
+          retrieveStatusMessage: "",
+          retrieveIntegrityCheck: false,
+        },
+        currentValue: {
+          publishedOn: new Date("2023-10-01T12:00:00Z").toISOString(),
+          archivable: true,
+        },
+      },
+      _id: "",
+    });
+  });
+
+  it("should convert history list to obsolete histories", () => {
+    const genericHistories: GenericHistory[] = [
+      {
+        subsystem: "Dataset",
+        documentId: "pid123",
+        user: "user123",
+        operation: "update",
+        timestamp: new Date("2023-10-02T12:00:00Z"),
+        before: {
+          isPublished: false,
+          datasetlifecycle: {
+            publishedOn: undefined,
+            archivable: false,
+          },
+        },
+        after: {
+          datasetlifecycle: {
+            publishedOn: new Date("2023-10-02T12:00:00Z"),
+            archivable: true,
+          },
+          isPublished: true,
+        },
+      },
+      {
+        subsystem: "Dataset",
+        documentId: "pid123",
+        user: "user456",
+        operation: "update",
+        timestamp: new Date("2023-10-01T12:00:00Z"),
+        before: {
+          isPublished: false,
+          datasetlifecycle: {
+            publishedOn: undefined,
+            archivable: false,
+          },
+        },
+        after: {
+          datasetlifecycle: {
+            publishedOn: new Date("2023-10-01T12:00:00Z"),
+            archivable: false,
+          },
+          isPublished: false,
+        },
+      },
+    ];
+
+    const currentDataset: Partial<DatasetDocument> = {
+      isPublished: true,
+      datasetlifecycle: {
+        publishedOn: new Date("2023-10-02T12:00:00Z"),
+        archivable: true,
+        retrievable: true,
+        publishable: false,
+        archiveRetentionTime: new Date("2031-10-01T12:00:00Z"),
+        dateOfPublishing: new Date("2024-10-01T12:00:00Z"),
+        isOnCentralDisk: true,
+        archiveStatusMessage: "datasetOnArchiveDisk",
+        retrieveStatusMessage: "",
+        retrieveIntegrityCheck: false,
+      },
+    };
+
+    currentDataset.$clone = () => currentDataset as DatasetDocument; // Mock the $clone method
+
+    const obsoleteHistories = convertGenericHistoriesToObsoleteHistories(
+      genericHistories,
+      currentDataset as DatasetDocument,
+    );
+
+    expect(obsoleteHistories).toEqual([
+      {
+        updatedAt: new Date("2023-10-02T12:00:00Z"),
+        updatedBy: "user123",
+        isPublished: {
+          previousValue: false,
+          currentValue: true,
+        },
+        datasetlifecycle: {
+          previousValue: {
+            publishedOn: undefined,
+            archivable: false,
+            retrievable: true,
+            publishable: false,
+            archiveRetentionTime: new Date(
+              "2031-10-01T12:00:00Z",
+            ).toISOString(),
+            dateOfPublishing: new Date("2024-10-01T12:00:00Z").toISOString(),
+            isOnCentralDisk: true,
+            archiveStatusMessage: "datasetOnArchiveDisk",
+            retrieveStatusMessage: "",
+            retrieveIntegrityCheck: false,
+          },
+          currentValue: {
+            publishedOn: new Date("2023-10-02T12:00:00Z").toISOString(),
+            archivable: true,
+          },
+        },
+        _id: "",
+      },
+      {
+        updatedAt: new Date("2023-10-01T12:00:00Z"),
+        updatedBy: "user456",
+        isPublished: {
+          previousValue: false,
+          currentValue: false,
+        },
+        datasetlifecycle: {
+          previousValue: {
+            publishedOn: undefined,
+            archivable: false,
+            retrievable: true,
+            publishable: false,
+            archiveRetentionTime: new Date(
+              "2031-10-01T12:00:00Z",
+            ).toISOString(),
+            dateOfPublishing: new Date("2024-10-01T12:00:00Z").toISOString(),
+            isOnCentralDisk: true,
+            archiveStatusMessage: "datasetOnArchiveDisk",
+            retrieveStatusMessage: "",
+            retrieveIntegrityCheck: false,
+          },
+          currentValue: {
+            publishedOn: new Date("2023-10-01T12:00:00Z").toISOString(),
+            archivable: false,
+          },
+        },
+        _id: "",
+      },
+    ]);
+  });
+});
diff --git a/src/datasets/utils/history.util.ts b/src/datasets/utils/history.util.ts
new file mode 100644
index 000000000..64d6c73c7
--- /dev/null
+++ b/src/datasets/utils/history.util.ts
@@ -0,0 +1,60 @@
+import { HistoryClass } from "src/datasets/schemas/history.schema";
+import { GenericHistory } from "../../common/schemas/generic-history.schema";
+import {
+  DatasetClass,
+  DatasetDocument,
+} from "src/datasets/schemas/dataset.schema";
+
+const IGNORE_FIELDS = ["updatedAt", "updatedBy", "_id"];
+
+// Given a dataset snapshot and a history entry, reconstruct the obsolete history entry
+export function convertGenericHistoryToObsoleteHistory(
+  history: GenericHistory,
+  datasetSnapshot: Partial<DatasetClass>,
+): HistoryClass {
+  const result: HistoryClass = {
+    updatedAt: history.timestamp,
+    updatedBy: history.user ?? "",
+    _id: "",
+  };
+  for (const field in history.before) {
+    if (IGNORE_FIELDS.includes(field)) {
+      continue;
+    }
+    if (field === "datasetlifecycle" && datasetSnapshot.datasetlifecycle) {
+      history.before[field] = {
+        ...JSON.parse(JSON.stringify(datasetSnapshot.datasetlifecycle)),
+        ...(history.before[field] as Record<string, unknown>),
+      };
+      history.after![field] = JSON.parse(
+        JSON.stringify(history.after![field] as Record<string, unknown>),
+      );
+    }
+    result[field] = {
+      previousValue: history.before[field],
+      currentValue: history.after?.[field],
+    };
+  }
+  return result;
+}
+
+// starting from the latest dataset, replay the history entries in reverse order to reconstruct the obsolete history entries
+export function convertGenericHistoriesToObsoleteHistories(
+  histories: GenericHistory[],
+  currentDataset: DatasetDocument,
+): HistoryClass[] {
+  currentDataset = currentDataset.$clone();
+  const result: HistoryClass[] = [];
+  for (const history of histories) {
+    const obsoleteHistory = convertGenericHistoryToObsoleteHistory(
+      history,
+      currentDataset,
+    );
+    for (const key of Object.keys(history.before || {})) {
+      (currentDataset as unknown as Record<string, unknown>)[key] =
+        history.before?.[key];
+    }
+    result.push(obsoleteHistory);
+  }
+  return result;
+}
diff --git a/src/history/history.module.ts b/src/history/history.module.ts
index 0600ac105..d9dd2d86f 100644
--- a/src/history/history.module.ts
+++ b/src/history/history.module.ts
@@ -22,5 +22,6 @@ import { AuthModule } from "../auth/auth.module";
   ],
   controllers: [HistoryController],
   providers: [HistoryService],
+  exports: [HistoryService],
 })
 export class HistoryModule {}
diff --git a/test/TestData.js b/test/TestData.js
index bbcf808a8..ef4263f39 100644
--- a/test/TestData.js
+++ b/test/TestData.js
@@ -449,6 +449,7 @@ const TestData = {
       }
     },
     sharedWith: [],
+    history: "JEST_ANY",
     size: 0,
     sourceFolder: "/iramjet/tif",
     startTime: "JEST_ANY",

From d57621b075a36dfaa128a6076e2cbe495849cf96 Mon Sep 17 00:00:00 2001
From: Omkar Zade <omkar.zade@psi.ch>
Date: Thu, 16 Oct 2025 13:58:01 +0200
Subject: [PATCH 25/26] feat: add migration script to/from legacy history to
 generic history (#2286)

* Add migration script to/from legacy history to generic history

* Use insertMany genericHistories instead of repeated insertOnes

* Address PR comment: filter on unmigrated datasets in the down migration
---
 ...92142-legacy-history-to-generic-history.js | 65 +++++++++++++++++++
 package.json                                  |  4 +-
 src/datasets/utils/history.util.spec.ts       | 59 +++++++++++++++++
 src/datasets/utils/history.util.ts            | 52 +++++++++++++++
 4 files changed, 178 insertions(+), 2 deletions(-)
 create mode 100644 migrations/20251014092142-legacy-history-to-generic-history.js

diff --git a/migrations/20251014092142-legacy-history-to-generic-history.js b/migrations/20251014092142-legacy-history-to-generic-history.js
new file mode 100644
index 000000000..b06049658
--- /dev/null
+++ b/migrations/20251014092142-legacy-history-to-generic-history.js
@@ -0,0 +1,65 @@
+/* eslint-disable @typescript-eslint/no-require-imports */
+const convertObsoleteHistoryToGenericHistory =
+  require("../dist/datasets/utils/history.util").convertObsoleteHistoryToGenericHistory;
+
+const convertGenericHistoriesToObsoleteHistories =
+  require("../dist/datasets/utils/history.util").convertGenericHistoriesToObsoleteHistories;
+
+module.exports = {
+  /**
+   * @param db {import('mongodb').Db}
+   * @param client {import('mongodb').MongoClient}
+   * @returns {Promise<void>}
+   */
+  async up(db, client) {
+    for await (const dataset of db
+      .collection("Dataset")
+      .find({ history: { $exists: true, $type: "array", $ne: [] } })) {
+      console.log(
+        `Migrating history for dataset ${dataset._id}. Entries: ${dataset.history.length}`,
+      );
+      const genericHistories = dataset.history.map((entry) =>
+        convertObsoleteHistoryToGenericHistory(entry, dataset._id),
+      );
+      result = await db.collection("History").insertMany(genericHistories);
+    }
+    await db.collection("Dataset").updateMany({}, { $unset: { history: "" } });
+  },
+
+  /**
+   * @param db {import('mongodb').Db}
+   * @param client {import('mongodb').MongoClient}
+   * @returns {Promise<void>}
+   */
+  async down(db, client) {
+    for await (const dataset of db
+      .collection("Dataset")
+      .find({ history: { $exists: false } })) {
+      console.log(`Rolling back history for dataset ${dataset._id}`);
+      const genericHistories = await db
+        .collection("History")
+        .find(
+          { documentId: dataset._id, subsystem: "Dataset" },
+          { sort: { timestamp: "desc" } },
+        )
+        .toArray();
+      console.log(
+        `Found ${genericHistories.length} history entries for dataset ${dataset._id}`,
+      );
+      dataset.$clone = () => dataset; // Mock the $clone method, as this is not a mongoose document
+      const obsoleteHistories = convertGenericHistoriesToObsoleteHistories(
+        genericHistories,
+        dataset,
+      );
+      console.log(
+        `Inserting obsolete history entries for dataset ${dataset._id}`,
+      );
+      await db
+        .collection("Dataset")
+        .updateOne(
+          { _id: dataset._id },
+          { $set: { history: obsoleteHistories } },
+        );
+    }
+  },
+};
diff --git a/package.json b/package.json
index ed9aeb28a..fd558a548 100644
--- a/package.json
+++ b/package.json
@@ -11,8 +11,8 @@
     "build": "nest build",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
     "format:check": "prettier --check \"src/**/*.ts\" \"test/**/*.ts\"",
-    "migrate:db:up": "node ./node_modules/migrate-mongo/bin/migrate-mongo.js up",
-    "migrate:db:down": "node ./node_modules/migrate-mongo/bin/migrate-mongo.js down",
+    "migrate:db:up": "npm run build && node ./node_modules/migrate-mongo/bin/migrate-mongo.js up",
+    "migrate:db:down": "npm run build && node ./node_modules/migrate-mongo/bin/migrate-mongo.js down",
     "migrate:db:status": "node ./node_modules/migrate-mongo/bin/migrate-mongo.js status",
     "start": "nest start",
     "start:dev": "nest start --watch",
diff --git a/src/datasets/utils/history.util.spec.ts b/src/datasets/utils/history.util.spec.ts
index c2af6076e..aad19f363 100644
--- a/src/datasets/utils/history.util.spec.ts
+++ b/src/datasets/utils/history.util.spec.ts
@@ -1,6 +1,8 @@
+import { HistoryClass } from "src/datasets/schemas/history.schema";
 import {
   convertGenericHistoriesToObsoleteHistories,
   convertGenericHistoryToObsoleteHistory,
+  convertObsoleteHistoryToGenericHistory,
 } from "./history.util";
 import {
   DatasetClass,
@@ -9,6 +11,63 @@ import {
 import { GenericHistory } from "../../common/schemas/generic-history.schema";
 
 describe("History Utility Functions", () => {
+  it("should convert obsolete history to generic history", () => {
+    const obsoleteHistory: HistoryClass = {
+      updatedAt: new Date("2023-10-01T12:00:00Z"),
+      updatedBy: "user123",
+      isPublished: {
+        currentValue: true,
+        previousValue: false,
+      },
+      datasetlifecycle: {
+        currentValue: {
+          publishedOn: new Date("2023-10-01T12:00:00Z"),
+          archivable: true,
+          retrievable: true,
+        },
+        previousValue: {
+          archivable: false,
+          retrievable: true,
+          publishable: false,
+          archiveRetentionTime: new Date("2031-10-01T12:00:00Z"),
+          dateOfPublishing: new Date("2024-10-01T12:00:00Z"),
+          isOnCentralDisk: true,
+          archiveStatusMessage: "datasetOnArchiveDisk",
+          retrieveStatusMessage: "",
+          retrieveIntegrityCheck: false,
+        },
+      },
+      _id: "",
+    };
+    const documentId = "pid123";
+    const genericHistory = convertObsoleteHistoryToGenericHistory(
+      obsoleteHistory,
+      documentId,
+    );
+
+    expect(genericHistory).toEqual({
+      subsystem: "Dataset",
+      documentId: "pid123",
+      user: "user123",
+      operation: "update",
+      timestamp: new Date("2023-10-01T12:00:00Z"),
+      before: {
+        isPublished: false,
+        datasetlifecycle: {
+          publishedOn: undefined,
+          archivable: false,
+        },
+      },
+      after: {
+        datasetlifecycle: {
+          publishedOn: new Date("2023-10-01T12:00:00Z"),
+          archivable: true,
+        },
+        isPublished: true,
+      },
+    });
+  });
+
   it("should convert generic history to obsolete history", () => {
     const genericHistory: GenericHistory = {
       subsystem: "Dataset",
diff --git a/src/datasets/utils/history.util.ts b/src/datasets/utils/history.util.ts
index 64d6c73c7..b0c582a23 100644
--- a/src/datasets/utils/history.util.ts
+++ b/src/datasets/utils/history.util.ts
@@ -7,6 +7,58 @@ import {
 
 const IGNORE_FIELDS = ["updatedAt", "updatedBy", "_id"];
 
+export function convertObsoleteHistoryToGenericHistory(
+  history: HistoryClass,
+  documentId: string,
+): GenericHistory {
+  const result: GenericHistory = {
+    subsystem: "Dataset",
+    documentId: documentId,
+    user: history.updatedBy,
+    operation: "update",
+    timestamp: history.updatedAt,
+    before: {},
+    after: {},
+  };
+  const changeList = Object.keys(history).filter(
+    (key) => !IGNORE_FIELDS.includes(key),
+  );
+  for (const key of changeList) {
+    if (
+      !history[key] ||
+      !history[key].hasOwnProperty("previousValue") ||
+      !history[key].hasOwnProperty("currentValue")
+    ) {
+      continue;
+    }
+    const fieldChange = history[key] as {
+      previousValue: unknown;
+      currentValue: unknown;
+    };
+    if (key === "datasetlifecycle") {
+      const currentValue = fieldChange.currentValue as Record<string, unknown>;
+      const previousValue = fieldChange.previousValue as Record<
+        string,
+        unknown
+      >;
+      // only retain the intersection of keys in currentValue and previousValue and whose value has changed. drop all others
+      const prunedPreviousValue: Record<string, unknown> = {};
+      const prunedCurrentValue: Record<string, unknown> = {};
+      for (const subKey of Object.keys(currentValue)) {
+        if (currentValue[subKey] !== previousValue[subKey]) {
+          prunedPreviousValue[subKey] = previousValue[subKey];
+          prunedCurrentValue[subKey] = currentValue[subKey];
+        }
+      }
+      fieldChange.previousValue = prunedPreviousValue;
+      fieldChange.currentValue = prunedCurrentValue;
+    }
+    result.before![key] = fieldChange.previousValue;
+    result.after![key] = fieldChange.currentValue;
+  }
+  return result;
+}
+
 // Given a dataset snapshot and a history entry, reconstruct the obsolete history entry
 export function convertGenericHistoryToObsoleteHistory(
   history: GenericHistory,

From e1c024a9c7731e2353edc235f77ef4bd0f1ebb2b Mon Sep 17 00:00:00 2001
From: Omkar Zade <omkar.zade@psi.ch>
Date: Thu, 16 Oct 2025 16:50:13 +0200
Subject: [PATCH 26/26] fix: convert all object fields from obsolete to generic
 history (#2288)

Convert all object fields from obsolete to generic history
---
 src/datasets/utils/history.util.spec.ts | 74 +++++++++++++++++++++++--
 src/datasets/utils/history.util.ts      | 19 +++----
 2 files changed, 76 insertions(+), 17 deletions(-)

diff --git a/src/datasets/utils/history.util.spec.ts b/src/datasets/utils/history.util.spec.ts
index aad19f363..5af8291c7 100644
--- a/src/datasets/utils/history.util.spec.ts
+++ b/src/datasets/utils/history.util.spec.ts
@@ -37,6 +37,64 @@ describe("History Utility Functions", () => {
           retrieveIntegrityCheck: false,
         },
       },
+      scientificMetadata: {
+        currentValue: {
+          runNumber: 484,
+          scanAxis0Name: ["phi"],
+          scanName: "run0484_diamond_D2FG",
+          scan_parameters: {
+            expected_total_number_of_steps: 7,
+            name: ["phi2"],
+            scan_name: "run0484_diamond_D2FG",
+          },
+          scan_readbacks: [
+            [73.369784942746],
+            [73.36755732624964],
+            [73.3680595489329],
+            [73.36857790677979],
+            [73.3690102872549],
+            [73.36911958727225],
+            [73.37012839122846],
+          ],
+          scan_readbacks_raw: [[], [], [], [], [], [], []],
+          scan_step_info: [
+            {
+              step_number: 1,
+            },
+            {
+              step_number: 2,
+            },
+          ],
+        },
+        previousValue: {
+          runNumber: 0,
+          scanAxis0Name: ["phi"],
+          scanName: "run0484_diamond_D2FG",
+          scan_parameters: {
+            expected_total_number_of_steps: 7,
+            name: ["phi"],
+            scan_name: "run0484_diamond_D2FG",
+          },
+          scan_readbacks: [
+            [73.369784942746],
+            [73.36755732624964],
+            [73.3680595489329],
+            [73.36857790677979],
+            [73.3690102872549],
+            [73.36911958727225],
+            [73.37012839122846],
+          ],
+          scan_readbacks_raw: [[], [], [], [], [], [], []],
+          scan_step_info: [
+            {
+              step_number: 1,
+            },
+            {
+              step_number: 2,
+            },
+          ],
+        },
+      },
       _id: "",
     };
     const documentId = "pid123";
@@ -44,7 +102,6 @@ describe("History Utility Functions", () => {
       obsoleteHistory,
       documentId,
     );
-
     expect(genericHistory).toEqual({
       subsystem: "Dataset",
       documentId: "pid123",
@@ -53,17 +110,22 @@ describe("History Utility Functions", () => {
       timestamp: new Date("2023-10-01T12:00:00Z"),
       before: {
         isPublished: false,
-        datasetlifecycle: {
-          publishedOn: undefined,
-          archivable: false,
+        datasetlifecycle: { archivable: false, publishedOn: undefined },
+        scientificMetadata: {
+          runNumber: 0,
+          scan_parameters: { name: ["phi"] },
         },
       },
       after: {
+        isPublished: true,
         datasetlifecycle: {
-          publishedOn: new Date("2023-10-01T12:00:00Z"),
           archivable: true,
+          publishedOn: new Date("2023-10-01T12:00:00.000Z"),
+        },
+        scientificMetadata: {
+          runNumber: 484,
+          scan_parameters: { name: ["phi2"] },
         },
-        isPublished: true,
       },
     });
   });
diff --git a/src/datasets/utils/history.util.ts b/src/datasets/utils/history.util.ts
index b0c582a23..eb163c0b3 100644
--- a/src/datasets/utils/history.util.ts
+++ b/src/datasets/utils/history.util.ts
@@ -4,6 +4,7 @@ import {
   DatasetClass,
   DatasetDocument,
 } from "src/datasets/schemas/dataset.schema";
+import { computeDeltaWithOriginals } from "src/common/utils/delta.util";
 
 const IGNORE_FIELDS = ["updatedAt", "updatedBy", "_id"];
 
@@ -35,23 +36,19 @@ export function convertObsoleteHistoryToGenericHistory(
       previousValue: unknown;
       currentValue: unknown;
     };
-    if (key === "datasetlifecycle") {
+    if (typeof fieldChange.previousValue == "object") {
       const currentValue = fieldChange.currentValue as Record<string, unknown>;
       const previousValue = fieldChange.previousValue as Record<
         string,
         unknown
       >;
       // only retain the intersection of keys in currentValue and previousValue and whose value has changed. drop all others
-      const prunedPreviousValue: Record<string, unknown> = {};
-      const prunedCurrentValue: Record<string, unknown> = {};
-      for (const subKey of Object.keys(currentValue)) {
-        if (currentValue[subKey] !== previousValue[subKey]) {
-          prunedPreviousValue[subKey] = previousValue[subKey];
-          prunedCurrentValue[subKey] = currentValue[subKey];
-        }
-      }
-      fieldChange.previousValue = prunedPreviousValue;
-      fieldChange.currentValue = prunedCurrentValue;
+      const { delta, originals } = computeDeltaWithOriginals(previousValue, {
+        ...previousValue,
+        ...currentValue,
+      });
+      fieldChange.previousValue = originals;
+      fieldChange.currentValue = delta;
     }
     result.before![key] = fieldChange.previousValue;
     result.after![key] = fieldChange.currentValue;
