fix(dataset): ensure that fields required for the lookups are included

src/attachments/attachments.service.ts

@@ -65,7 +65,7 @@ export class AttachmentsService {
 
     const pipeline: PipelineStage[] = [{ $match: whereFilter }];
     if (!isEmpty(fieldsProjection)) {
-      const projection = parsePipelineProjection(fieldsProjection);
+      const projection = parsePipelineProjection(fieldsProjection, filter);
       pipeline.push({ $project: projection });
     }
 

src/attachments/attachments.v4.service.ts

@@ -51,7 +51,7 @@ export class AttachmentsV4Service {
 
     const pipeline: PipelineStage[] = [{ $match: whereFilter }];
     if (!isEmpty(fieldsProjection)) {
-      const projection = parsePipelineProjection(fieldsProjection);
+      const projection = parsePipelineProjection(fieldsProjection, filter);
       pipeline.push({ $project: projection });
     }
 

src/common/utils.ts

@@ -11,6 +11,15 @@ import {
 } from "./interfaces/common.interface";
 import { ScientificRelation } from "./scientific-relation.enum";
 import { DatasetType } from "src/datasets/types/dataset-type.enum";
+import { DatasetDocument } from "src/datasets/schemas/dataset.schema";
+import { AttachmentDocument } from "src/attachments/schemas/attachment.schema";
+import { JobDocument } from "src/jobs/schemas/job.schema";
+import { OrigDatablockDocument } from "src/origdatablocks/schemas/origdatablock.schema";
+import { DatasetLookupKeysEnum } from "src/datasets/types/dataset-lookup";
+import { AttachmentLookupKeysEnum } from "src/attachments/types/attachment-lookup";
+import { OrigDatablockLookupKeysEnum } from "src/origdatablocks/types/origdatablock-lookup";
+import { JobLookupKeysEnum } from "src/jobs/types/job-lookup";
+import { InstrumentDocument } from "src/instruments/schemas/instrument.schema";
 
 // add Å to mathjs accepted units as equivalent to angstrom
 const isAlphaOriginal = Unit.isValidAlpha;
@@ -414,8 +423,33 @@ export const parsePipelineSort = (sort: Record<string, "asc" | "desc">) => {
   return pipelineSort;
 };
 
-export const parsePipelineProjection = (fieldsProjection: string[]) => {
+export const parsePipelineProjection = (
+  fieldsProjection: string[],
+  filter: FilterQuery<
+    | DatasetDocument
+    | AttachmentDocument
+    | OrigDatablockDocument
+    | JobDocument
+    | InstrumentDocument
+  >,
+) => {
   const pipelineProjection: Record<string, boolean> = {};
+  const embeddedKeys = [
+    "datasets",
+    "origdatablocks",
+    "datablocks",
+    "jobs",
+    "attachments",
+    "instruments",
+    "proposals",
+    "samples",
+  ];
+
+  const embeddedDocumentNames = new Set(
+    fieldsProjection
+      .filter((field) => field.includes("."))
+      .map((field) => field.split(".")[0]),
+  );
 
   if (!Array.isArray(fieldsProjection)) {
     throw new HttpException("fields must be an array", HttpStatus.BAD_REQUEST);
@@ -424,6 +458,27 @@ export const parsePipelineProjection = (fieldsProjection: string[]) => {
     pipelineProjection[field] = true;
   });
 
+  if (filter.include) {
+    filter.include.forEach(
+      (
+        field:
+          | DatasetLookupKeysEnum
+          | AttachmentLookupKeysEnum
+          | OrigDatablockLookupKeysEnum
+          | JobLookupKeysEnum,
+      ) => {
+        if (field === "all") {
+          embeddedKeys.forEach((key) => {
+            if (!embeddedDocumentNames.has(key)) {
+              pipelineProjection[key] = true;
+            }
+          });
+        } else if (!embeddedDocumentNames.has(String(field))) {
+          pipelineProjection[String(field)] = true;
+        }
+      },
+    );
+  }
   return pipelineProjection;
 };
 

src/datasets/datasets.service.ts

@@ -133,7 +133,7 @@ export class DatasetsService {
     this.addLookupFields(pipeline, filter.include);
 
     if (!isEmpty(fieldsProjection)) {
-      const projection = parsePipelineProjection(fieldsProjection);
+      const projection = parsePipelineProjection(fieldsProjection, filter);
       pipeline.push({ $project: projection });
     }
 
@@ -258,8 +258,10 @@ export class DatasetsService {
     };
 
     const pipeline: PipelineStage[] = [{ $match: whereFilter }];
+    this.addLookupFields(pipeline, filter.include);
+
     if (!isEmpty(fieldsProjection)) {
-      const projection = parsePipelineProjection(fieldsProjection);
+      const projection = parsePipelineProjection(fieldsProjection, filter);
       pipeline.push({ $project: projection });
     }
 
@@ -270,8 +272,6 @@ export class DatasetsService {
 
     pipeline.push({ $skip: limits.skip || 0 });
 
-    this.addLookupFields(pipeline, filter.include);
-
     const [data] = await this.datasetModel
       .aggregate<OutputDatasetDto | undefined>(pipeline)
       .exec();

src/jobs/jobs.service.ts

@@ -169,7 +169,7 @@ export class JobsService {
 
     if (fieldsProjection && fieldsProjection.length > 0) {
       const projectionFields = [...fieldsProjection];
-      const projection = parsePipelineProjection(projectionFields);
+      const projection = parsePipelineProjection(projectionFields, filter);
       pipeline.push({ $project: projection });
     } else {
       // remove field datasetIds

src/origdatablocks/origdatablocks.service.ts

@@ -125,7 +125,7 @@ export class OrigDatablocksService {
     this.addLookupFields(pipeline, filter.include);
 
     if (!isEmpty(fieldsProjection)) {
-      const projection = parsePipelineProjection(fieldsProjection);
+      const projection = parsePipelineProjection(fieldsProjection, filter);
       pipeline.push({ $project: projection });
     }
 
@@ -160,7 +160,7 @@ export class OrigDatablocksService {
     this.addLookupFields(pipeline, filter.include);
 
     if (!isEmpty(fieldsProjection)) {
-      const projection = parsePipelineProjection(fieldsProjection);
+      const projection = parsePipelineProjection(fieldsProjection, filter);
       pipeline.push({ $project: projection });
     }
 
@@ -230,7 +230,7 @@ export class OrigDatablocksService {
 
     const pipeline: PipelineStage[] = [{ $match: whereFilter }];
     if (!isEmpty(fieldsProjection)) {
-      const projection = parsePipelineProjection(fieldsProjection);
+      const projection = parsePipelineProjection(fieldsProjection, filter);
       pipeline.push({ $project: projection });
     }
 

test/DatasetV4.js

@@ -594,12 +594,12 @@ describe("2500: Datasets v4 tests", () => {
             dataset.should.have.property("datasetName");
             dataset.should.have.property("pid");
             dataset.should.not.have.property("description");
-            dataset.should.not.have.property("instruments");
-            dataset.should.not.have.property("proposals");
-            dataset.should.not.have.property("datablocks");
-            dataset.should.not.have.property("attachments");
-            dataset.should.not.have.property("origdatablocks");
-            dataset.should.not.have.property("samples");
+            dataset.should.have.property("instruments");
+            dataset.should.have.property("proposals");
+            dataset.should.have.property("datablocks");
+            dataset.should.have.property("attachments");
+            dataset.should.have.property("origdatablocks");
+            dataset.should.have.property("samples");
 
             dataset.datasetName.should.match(/Dataset/i);
           });
@@ -626,7 +626,7 @@ describe("2500: Datasets v4 tests", () => {
         },
       };
       const malformedJson = JSON.stringify(filter).replace("}", "},");
-      
+
       return request(appUrl)
         .get(`/api/v4/datasets`)
         .query({ filter: malformedJson })
@@ -721,7 +721,33 @@ describe("2500: Datasets v4 tests", () => {
         });
     });
 
-    it("0303: should fetch dataset relation fields if provided in the filter", async () => {
+    it("0303: should fetch specific dataset fields only if fields is provided in the filter with relationships", async () => {
+      const filter = {
+        include: ["instruments", "proposals"],
+        fields: ["datasetName"],
+      };
+
+      return request(appUrl)
+        .get(`/api/v4/datasets/findOne`)
+        .query({ filter: JSON.stringify(filter) })
+        .auth(accessTokenAdminIngestor, { type: "bearer" })
+        .expect(TestData.SuccessfulGetStatusCode)
+        .expect("Content-Type", /json/)
+        .then((res) => {
+          res.body.should.be.a("object");
+
+          res.body.should.have.property("datasetName");
+
+          res.body.should.not.have.property("description");
+          res.body.should.not.have.property("pid");
+
+          res.body.should.have.property("instruments");
+          res.body.should.have.property("proposals");
+          res.body.should.not.have.property("datablocks");
+        });
+    });
+
+    it("0304: should fetch dataset relation fields if provided in the filter", async () => {
       const filter = {
         include: ["instruments", "proposals"],
       };
@@ -742,7 +768,7 @@ describe("2500: Datasets v4 tests", () => {
         });
     });
 
-    it("0304: should fetch all dataset relation fields if provided in the filter", async () => {
+    it("0305: should fetch all dataset relation fields if provided in the filter", async () => {
       const filter = {
         include: ["all"],
       };
@@ -766,7 +792,7 @@ describe("2500: Datasets v4 tests", () => {
         });
     });
 
-    it("0305: should be able to fetch the dataset providing where filter", async () => {
+    it("0306: should be able to fetch the dataset providing where filter", async () => {
       const filter = {
         where: {
           datasetName: {
@@ -788,7 +814,7 @@ describe("2500: Datasets v4 tests", () => {
         });
     });
 
-    it("0306: should be able to fetch a dataset providing all allowed filters together", async () => {
+    it("0307: should be able to fetch a dataset providing all allowed filters together", async () => {
       const filter = {
         where: {
           datasetName: {
@@ -831,7 +857,7 @@ describe("2500: Datasets v4 tests", () => {
         });
     });
 
-    it("0307: should not be able to provide filters that are not allowed", async () => {
+    it("0308: should not be able to provide filters that are not allowed", async () => {
       const filter = {
         customField: { datasetName: "test" },
       };
@@ -843,6 +869,54 @@ describe("2500: Datasets v4 tests", () => {
         .expect(TestData.BadRequestStatusCode)
         .expect("Content-Type", /json/);
     });
+
+    it("0309: should only return the field and its subfields within the embedded documents even when 'all' is used under include", async () => {
+      const filter = {
+        include: ["all"],
+        fields: [
+          "datasetName",
+          "attachments.thumbnail",
+          "attachments.relationships.targetType",
+          "origdatablocks.dataFileList.path",
+        ],
+      };
+
+      return request(appUrl)
+        .get(`/api/v4/datasets/findOne`)
+        .query({ filter: JSON.stringify(filter) })
+        .auth(accessTokenAdminIngestor, { type: "bearer" })
+        .expect(TestData.SuccessfulGetStatusCode)
+        .expect("Content-Type", /json/)
+        .then((res) => {
+          console.log("RES BODY ", JSON.stringify(res.body, null, 2));
+          res.body.should.be.a("object");
+
+          res.body.should.have.property("datasetName");
+
+          res.body.should.not.have.property("description");
+          res.body.should.not.have.property("pid");
+
+          res.body.attachments.forEach((attachment) => {
+            attachment.should.have.property("thumbnail");
+            attachment.should.not.have.property("caption");
+
+            attachment.relationships.forEach((relationship) => {
+              relationship.should.have.property("targetType");
+              relationship.should.not.have.property("relationType");
+            });
+          });
+
+          res.body.origdatablocks.forEach((origDatablock) => {
+            origDatablock.should.have.property("dataFileList");
+            origDatablock.should.not.have.property("size");
+
+            origDatablock.dataFileList.forEach((dataFile) => {
+              dataFile.should.have.property("path");
+              dataFile.should.not.have.property("size");
+            });
+          });
+        });
+    });
   });
 
   describe("Datasets v4 count tests", () => {
@@ -1038,7 +1112,7 @@ describe("2500: Datasets v4 tests", () => {
 
     it("0604: should be able to partially update dataset's scientific metadata field", () => {
       const updatedDataset = {
-        datasetlifecycle:{storageLocation:"new location"},
+        datasetlifecycle: { storageLocation: "new location" },
         scientificMetadata: {
           with_unit_and_value_si: {
             value: 600,
@@ -1070,7 +1144,9 @@ describe("2500: Datasets v4 tests", () => {
             value: 111,
             unit: "",
           });
-          res.body.datasetlifecycle.should.have.property("storageLocation").and.equal("new location");
+          res.body.datasetlifecycle.should.have
+            .property("storageLocation")
+            .and.equal("new location");
         });
     });
 
@@ -1457,7 +1533,9 @@ describe("2500: Datasets v4 tests", () => {
         .then((res) => {
           res.body.should.be.a("object");
           res.body.should.be.deep.include(updatedDataset);
-          res.body.should.have.property("storageLocation").and.equal("new location");
+          res.body.should.have
+            .property("storageLocation")
+            .and.equal("new location");
         });
     });
 

test/DatasetV4Public.js

@@ -269,12 +269,12 @@ describe("2600: Datasets v4 public endpoints tests", () => {
             dataset.should.have.property("datasetName");
             dataset.should.have.property("pid");
             dataset.should.not.have.property("description");
-            dataset.should.not.have.property("instruments");
-            dataset.should.not.have.property("proposals");
-            dataset.should.not.have.property("datablocks");
-            dataset.should.not.have.property("attachments");
-            dataset.should.not.have.property("origdatablocks");
-            dataset.should.not.have.property("samples");
+            dataset.should.have.property("instruments");
+            dataset.should.have.property("proposals");
+            dataset.should.have.property("datablocks");
+            dataset.should.have.property("attachments");
+            dataset.should.have.property("origdatablocks");
+            dataset.should.have.property("samples");
 
             dataset.datasetName.should.match(/Dataset/i);
           });