Add metadataTypes endpoint - readonly; available for everyone

Author: Ingvord

src/datasets/datasets.controller.ts

@@ -888,6 +888,29 @@ export class DatasetsController {
     return this.datasetsService.metadataKeys(parsedFilters);
   }
 
+  // GET /datasets/metadataTypes
+  // @UseGuards(PoliciesGuard)
+  // @CheckPolicies((ability: AppAbility) =>
+  //   ability.can(Action.DatasetRead, DatasetClass),
+  // )
+  @Get("/metadataTypes")
+  @ApiOperation({
+    summary: "Retrieve all available scientific metadata field types.",
+    description:
+      "This endpoint returns the field names and their corresponding data types from the `scientificMetadata` fields of all datasets in the system. It aggregates all the unique field names and their associated types from the dataset collection.",
+  })
+  @ApiResponse({
+    status: 200,
+    type: Array,
+    description:
+      "Returns an array of objects where each object contains a metadata field name and its associated type.",
+  })
+  async metadataTypes(): Promise<any[]> {
+    const result = this.datasetsService.getScientificMetadataTypes();
+
+    return result;
+  }
+
   // GET /datasets/findOne
   @UseGuards(PoliciesGuard)
   @CheckPolicies("datasets", (ability: AppAbility) =>

src/datasets/datasets.service.ts

@@ -476,6 +476,43 @@ export class DatasetsService {
     }
   }
 
+  async getScientificMetadataTypes() {
+    // TODO performance research required, say 1K records, 100K and 1M
+    return this.datasetModel.aggregate([
+      {
+        // Step 1: Convert the scientificMetadata object into an array of key-value pairs
+        $project: {
+          scientificMetadataArray: { $objectToArray: "$scientificMetadata" },
+        },
+      },
+      {
+        // Step 2: Unwind the array to treat each field individually
+        $unwind: "$scientificMetadataArray",
+      },
+      {
+        // Step 3: Group by the field name and accumulate types
+        $group: {
+          _id: "$scientificMetadataArray.k", // Group by field name (key)
+          types: { $addToSet: { $type: "$scientificMetadataArray.v.value" } }, // Add the data type of the value
+        },
+      },
+      {
+        // Step 4: Convert the array of types into a string for each field name
+        $project: {
+          _id: 0, // Don't include the default _id
+          metadataKey: "$_id",
+          metadataType: {
+            $cond: {
+              if: { $eq: [{ $size: "$types" }, 1] }, // If only one type is present
+              then: { $arrayElemAt: ["$types", 0] }, // Use the type
+              else: "mixed", // If there are multiple types, mark it as 'mixed'
+            },
+          },
+        },
+      },
+    ]);
+  }
+
   async isElasticSearchDBEmpty() {
     if (!this.ESClient) return;
     const count = await this.ESClient.getCount();