diff --git a/apps/research-service/src/app.module.ts b/apps/research-service/src/app.module.ts index 117fa8e8..cf5eb69b 100644 --- a/apps/research-service/src/app.module.ts +++ b/apps/research-service/src/app.module.ts @@ -9,10 +9,30 @@ import { BoundingBoxController } from "./bounding-boxes/bounding-box.controller" import { BoundingBoxService } from "./bounding-boxes/bounding-box.service"; import { ValidationController } from "./validations/validation.controller"; import { ValidationService } from "./validations/validation.service"; +import { ValidationProcessor } from "./validations/validation.processor"; import { DataApiModule } from "@terramatch-microservices/data-api"; +import { BullModule } from "@nestjs/bullmq"; +import { ConfigModule, ConfigService } from "@nestjs/config"; @Module({ - imports: [SentryModule.forRoot(), CommonModule, HealthModule, DataApiModule], + imports: [ + SentryModule.forRoot(), + CommonModule, + HealthModule, + DataApiModule, + ConfigModule.forRoot({ isGlobal: true }), + BullModule.forRootAsync({ + imports: [ConfigModule], + inject: [ConfigService], + useFactory: (configService: ConfigService) => ({ + connection: { + host: configService.get("REDIS_HOST"), + port: configService.get("REDIS_PORT") + } + }) + }), + BullModule.registerQueue({ name: "validation" }) + ], controllers: [SitePolygonsController, BoundingBoxController, ValidationController], providers: [ { @@ -21,7 +41,8 @@ import { DataApiModule } from "@terramatch-microservices/data-api"; }, SitePolygonsService, BoundingBoxService, - ValidationService + ValidationService, + ValidationProcessor ] }) export class AppModule {} diff --git a/apps/research-service/src/validations/dto/site-validation-request.dto.ts b/apps/research-service/src/validations/dto/site-validation-request.dto.ts new file mode 100644 index 00000000..4823ee82 --- /dev/null +++ b/apps/research-service/src/validations/dto/site-validation-request.dto.ts @@ -0,0 +1,22 @@ +import { ApiProperty } from "@nestjs/swagger"; +import { IsArray, IsIn, IsOptional } from "class-validator"; +import { VALIDATION_TYPES, ValidationType } from "@terramatch-microservices/database/constants"; +import { CreateDataDto, JsonApiBodyDto } from "@terramatch-microservices/common/util/json-api-update-dto"; + +export class SiteValidationRequestAttributes { + @ApiProperty({ + enum: VALIDATION_TYPES, + isArray: true, + required: false, + description: + "Array of validation types to run on all polygons in the site. If not provided or empty, all validation types will be run." + }) + @IsOptional() + @IsArray() + @IsIn(VALIDATION_TYPES, { each: true }) + validationTypes?: ValidationType[]; +} + +export class SiteValidationRequestBody extends JsonApiBodyDto( + class SiteValidationRequestData extends CreateDataDto("validations", SiteValidationRequestAttributes) {} +) {} diff --git a/apps/research-service/src/validations/dto/validation-criteria.dto.ts b/apps/research-service/src/validations/dto/validation-criteria.dto.ts index 44b53678..dd949422 100644 --- a/apps/research-service/src/validations/dto/validation-criteria.dto.ts +++ b/apps/research-service/src/validations/dto/validation-criteria.dto.ts @@ -4,12 +4,6 @@ import { VALIDATION_CRITERIA_IDS, CriteriaId } from "@terramatch-microservices/d @JsonApiDto({ type: "validationCriterias" }) export class ValidationCriteriaDto { - @ApiProperty({ - description: "The polygon UUID that was validated (optional for historic data)", - required: false - }) - polygonUuid?: string; - @ApiProperty({ description: "The validation criteria ID", enum: VALIDATION_CRITERIA_IDS diff --git a/apps/research-service/src/validations/dto/validation-request.dto.ts b/apps/research-service/src/validations/dto/validation-request.dto.ts index 348a2498..7e0a4627 100644 --- a/apps/research-service/src/validations/dto/validation-request.dto.ts +++ b/apps/research-service/src/validations/dto/validation-request.dto.ts @@ -1,27 +1,31 @@ import { ApiProperty } from "@nestjs/swagger"; -import { IsArray, IsString, ArrayMinSize, IsIn } from "class-validator"; +import { IsArray, ArrayMinSize, IsIn, IsOptional } from "class-validator"; import { VALIDATION_TYPES, ValidationType } from "@terramatch-microservices/database/constants"; +import { CreateDataDto, JsonApiBodyDto } from "@terramatch-microservices/common/util/json-api-update-dto"; -export class ValidationRequestDto { +export class ValidationRequestAttributes { @ApiProperty({ description: "Array of polygon UUIDs to validate", example: ["7631be34-bbe0-4e1e-b4fe-592677dc4b50", "d6502d4c-dfd6-461e-af62-21a0ec2f3e65"], - type: [String], - isArray: true + isArray: true, + type: String }) @IsArray() @ArrayMinSize(1) - @IsString({ each: true }) polygonUuids: string[]; @ApiProperty({ enum: VALIDATION_TYPES, - name: "validationTypes[]", isArray: true, - required: true, - description: "Array of validation types to run" + required: false, + description: "Array of validation types to run. If not provided or empty, all validation types will be run." }) + @IsOptional() @IsArray() @IsIn(VALIDATION_TYPES, { each: true }) - validationTypes: ValidationType[]; + validationTypes?: ValidationType[]; } + +export class ValidationRequestBody extends JsonApiBodyDto( + class ValidationRequestData extends CreateDataDto("validations", ValidationRequestAttributes) {} +) {} diff --git a/apps/research-service/src/validations/dto/validation-summary.dto.ts b/apps/research-service/src/validations/dto/validation-summary.dto.ts new file mode 100644 index 00000000..942acd7a --- /dev/null +++ b/apps/research-service/src/validations/dto/validation-summary.dto.ts @@ -0,0 +1,52 @@ +import { ApiProperty } from "@nestjs/swagger"; +import { JsonApiDto } from "@terramatch-microservices/common/decorators"; +import { ValidationType } from "@terramatch-microservices/database/constants"; + +export class ValidationTypeSummary { + @ApiProperty({ + description: "Number of polygons that passed this validation" + }) + valid: number; + + @ApiProperty({ + description: "Number of polygons that failed this validation" + }) + invalid: number; +} + +@JsonApiDto({ type: "validationSummaries" }) +export class ValidationSummaryDto { + @ApiProperty({ + description: "The UUID of the site that was validated", + example: "7631be34-bbe0-4e1e-b4fe-592677dc4b50" + }) + siteUuid: string; + + @ApiProperty({ + description: "Total number of polygons in the site" + }) + totalPolygons: number; + + @ApiProperty({ + description: "Number of polygons that were validated" + }) + validatedPolygons: number; + + @ApiProperty({ + description: "Summary of validation results by validation type", + type: "object", + additionalProperties: { + type: "object", + properties: { + valid: { type: "number" }, + invalid: { type: "number" } + } + } + }) + validationSummary: Record; + + @ApiProperty({ + description: "When the validation was completed" + }) + completedAt: Date; +} diff --git a/apps/research-service/src/validations/dto/validation.dto.ts b/apps/research-service/src/validations/dto/validation.dto.ts index a68d268d..26278d6b 100644 --- a/apps/research-service/src/validations/dto/validation.dto.ts +++ b/apps/research-service/src/validations/dto/validation.dto.ts @@ -8,7 +8,7 @@ export class ValidationDto { description: "The UUID of the polygon that was validated", example: "d6502d4c-dfd6-461e-af62-21a0ec2f3e65" }) - polygonId: string; + polygonUuid: string; @ApiProperty({ description: "List of validation criteria results for this polygon", diff --git a/apps/research-service/src/validations/validation.controller.spec.ts b/apps/research-service/src/validations/validation.controller.spec.ts index dc1778cb..f2253487 100644 --- a/apps/research-service/src/validations/validation.controller.spec.ts +++ b/apps/research-service/src/validations/validation.controller.spec.ts @@ -5,14 +5,20 @@ import { ValidationDto } from "./dto/validation.dto"; import { populateDto } from "@terramatch-microservices/common/dto/json-api-attributes"; import { serialize } from "@terramatch-microservices/common/util/testing"; import { SiteValidationQueryDto } from "./dto/site-validation-query.dto"; -import { ValidationRequestDto } from "./dto/validation-request.dto"; +import { ValidationRequestBody } from "./dto/validation-request.dto"; +import { SiteValidationRequestBody } from "./dto/site-validation-request.dto"; +import { getQueueToken } from "@nestjs/bullmq"; +import { BadRequestException, NotFoundException } from "@nestjs/common"; +import { DelayedJob } from "@terramatch-microservices/database/entities"; +import { ValidationType } from "@terramatch-microservices/database/constants"; +import { Site } from "@terramatch-microservices/database/entities"; describe("ValidationController", () => { let controller: ValidationController; const sampleValidation = new ValidationDto(); populateDto(sampleValidation, { - polygonId: "7631be34-bbe0-4e1e-b4fe-592677dc4b50", + polygonUuid: "7631be34-bbe0-4e1e-b4fe-592677dc4b50", criteriaList: [ { criteriaId: 4, @@ -31,7 +37,7 @@ describe("ValidationController", () => { const siteValidation1 = new ValidationDto(); populateDto(siteValidation1, { - polygonId: "polygon-uuid-123", + polygonUuid: "polygon-uuid-123", criteriaList: [ { criteriaId: 4, @@ -44,7 +50,7 @@ describe("ValidationController", () => { const siteValidation2 = new ValidationDto(); populateDto(siteValidation2, { - polygonId: "polygon-uuid-456", + polygonUuid: "polygon-uuid-456", criteriaList: [ { criteriaId: 8, @@ -61,42 +67,43 @@ describe("ValidationController", () => { validations: [siteValidation1, siteValidation2], total: 2 }), - validatePolygons: jest.fn().mockResolvedValue({ - results: [ - { - polygonUuid: "polygon-1", - criteriaId: 4, - valid: true, - createdAt: new Date("2025-01-08T22:15:15.000Z"), - extraInfo: null - }, - { - polygonUuid: "polygon-1", - criteriaId: 8, - valid: false, - createdAt: new Date("2025-01-08T22:15:15.000Z"), - extraInfo: { spikes: [], spikeCount: 0 } - }, - { - polygonUuid: "polygon-2", - criteriaId: 4, - valid: true, - createdAt: new Date("2025-01-08T22:15:15.000Z"), - extraInfo: null - } - ] - }) + validatePolygonsBatch: jest.fn().mockResolvedValue(undefined), + getSitePolygonUuids: jest.fn().mockResolvedValue(["polygon-1", "polygon-2"]) + }; + + const mockQueue = { + add: jest.fn() }; beforeEach(async () => { jest.clearAllMocks(); + jest.spyOn(DelayedJob, "create").mockResolvedValue({ + id: 1, + uuid: "job-uuid-123", + name: "", + totalContent: 0, + processedContent: 0, + progressMessage: "", + metadata: {}, + save: jest.fn().mockResolvedValue(undefined) + } as unknown as DelayedJob); + + jest.spyOn(Site, "findOne").mockResolvedValue({ + id: 1, + name: "Test Site" + } as unknown as Site); + const module: TestingModule = await Test.createTestingModule({ controllers: [ValidationController], providers: [ { provide: ValidationService, useValue: mockValidationService + }, + { + provide: getQueueToken("validation"), + useValue: mockQueue } ] }).compile(); @@ -146,54 +153,125 @@ describe("ValidationController", () => { expect(mockValidationService.getSiteValidations).toHaveBeenCalledWith(siteUuid, 10, 3, undefined); }); + + it("should throw BadRequestException for invalid criteriaId", async () => { + const query = { criteriaId: "0" }; + await expect(controller.getSiteValidation(siteUuid, query as unknown as SiteValidationQueryDto)).rejects.toThrow( + BadRequestException + ); + }); + + it("should throw BadRequestException for non-integer criteriaId", async () => { + const query = { criteriaId: "1.5" }; + await expect(controller.getSiteValidation(siteUuid, query as unknown as SiteValidationQueryDto)).rejects.toThrow( + BadRequestException + ); + }); + + it("should use criteriaId when provided", async () => { + const query = { criteriaId: 4 }; + await controller.getSiteValidation(siteUuid, query as unknown as SiteValidationQueryDto); + expect(mockValidationService.getSiteValidations).toHaveBeenCalledWith(siteUuid, 100, 1, 4); + }); }); describe("createPolygonValidations", () => { it("should create polygon validations and return proper JSON API format", async () => { - const request: ValidationRequestDto = { - polygonUuids: ["polygon-1", "polygon-2"], - validationTypes: ["SELF_INTERSECTION", "SPIKES"] + const request: ValidationRequestBody = { + data: { + type: "validations", + attributes: { + polygonUuids: ["polygon-1", "polygon-2"], + validationTypes: ["SELF_INTERSECTION", "SPIKES"] + } + } }; + const polygon1Validation = new ValidationDto(); + populateDto(polygon1Validation, { + polygonUuid: "polygon-1", + criteriaList: [ + { + criteriaId: 4, + valid: true, + createdAt: new Date("2025-01-08T22:15:15.000Z"), + extraInfo: null + }, + { + criteriaId: 8, + valid: false, + createdAt: new Date("2025-01-08T22:15:15.000Z"), + extraInfo: { spikes: [], spikeCount: 0 } + } + ] + }); + + const polygon2Validation = new ValidationDto(); + populateDto(polygon2Validation, { + polygonUuid: "polygon-2", + criteriaList: [ + { + criteriaId: 4, + valid: true, + createdAt: new Date("2025-01-08T22:15:15.000Z"), + extraInfo: null + } + ] + }); + + mockValidationService.getPolygonValidation + .mockResolvedValueOnce(polygon1Validation) + .mockResolvedValueOnce(polygon2Validation); + const result = serialize(await controller.createPolygonValidations(request)); - expect(mockValidationService.validatePolygons).toHaveBeenCalledWith(request); + expect(mockValidationService.validatePolygonsBatch).toHaveBeenCalledWith( + ["polygon-1", "polygon-2"], + ["SELF_INTERSECTION", "SPIKES"] + ); + expect(mockValidationService.getPolygonValidation).toHaveBeenCalledTimes(2); + expect(mockValidationService.getPolygonValidation).toHaveBeenCalledWith("polygon-1"); + expect(mockValidationService.getPolygonValidation).toHaveBeenCalledWith("polygon-2"); expect(result.data).toBeDefined(); expect(Array.isArray(result.data)).toBe(true); expect(result.data).toHaveLength(2); const dataArray = result.data as unknown as Array<{ id: string; - attributes: { polygonId: string; criteriaList: unknown[] }; + attributes: { polygonUuid: string; criteriaList: unknown[] }; }>; const polygon1Data = dataArray.find(item => item.id === "polygon-1"); const polygon2Data = dataArray.find(item => item.id === "polygon-2"); expect(polygon1Data).toBeDefined(); if (polygon1Data != null) { - expect(polygon1Data.attributes.polygonId).toBe("polygon-1"); + expect(polygon1Data.attributes.polygonUuid).toBe("polygon-1"); expect(polygon1Data.attributes.criteriaList).toHaveLength(2); } expect(polygon2Data).toBeDefined(); if (polygon2Data != null) { - expect(polygon2Data.attributes.polygonId).toBe("polygon-2"); + expect(polygon2Data.attributes.polygonUuid).toBe("polygon-2"); expect(polygon2Data.attributes.criteriaList).toHaveLength(1); } }); - it("should handle results with null polygonUuid", async () => { - mockValidationService.validatePolygons.mockResolvedValueOnce({ - results: [ - { - polygonUuid: null, - criteriaId: 4, - valid: true, - createdAt: new Date("2025-01-08T22:15:15.000Z"), - extraInfo: null - }, + it("should handle single polygon validation", async () => { + const request: ValidationRequestBody = { + data: { + type: "validations", + attributes: { + polygonUuids: ["polygon-1"], + validationTypes: ["SELF_INTERSECTION"] + } + } + }; + + const polygon1Validation = new ValidationDto(); + populateDto(polygon1Validation, { + polygonUuid: "polygon-1", + criteriaList: [ { - polygonUuid: "polygon-1", criteriaId: 4, valid: true, createdAt: new Date("2025-01-08T22:15:15.000Z"), @@ -202,13 +280,11 @@ describe("ValidationController", () => { ] }); - const request: ValidationRequestDto = { - polygonUuids: ["polygon-1"], - validationTypes: ["SELF_INTERSECTION"] - }; + mockValidationService.getPolygonValidation.mockResolvedValueOnce(polygon1Validation); const result = serialize(await controller.createPolygonValidations(request)); + expect(mockValidationService.validatePolygonsBatch).toHaveBeenCalledWith(["polygon-1"], ["SELF_INTERSECTION"]); expect(result.data).toBeDefined(); if (Array.isArray(result.data)) { expect(result.data).toHaveLength(1); @@ -221,15 +297,21 @@ describe("ValidationController", () => { }); it("should create polygon validations with DATA_COMPLETENESS validation type", async () => { - const request: ValidationRequestDto = { - polygonUuids: ["polygon-1"], - validationTypes: ["DATA_COMPLETENESS"] + const request: ValidationRequestBody = { + data: { + type: "validations", + attributes: { + polygonUuids: ["polygon-1"], + validationTypes: ["DATA_COMPLETENESS"] + } + } }; - mockValidationService.validatePolygons.mockResolvedValueOnce({ - results: [ + const polygon1Validation = new ValidationDto(); + populateDto(polygon1Validation, { + polygonUuid: "polygon-1", + criteriaList: [ { - polygonUuid: "polygon-1", criteriaId: 14, valid: false, createdAt: new Date("2025-01-08T22:15:15.000Z"), @@ -244,40 +326,48 @@ describe("ValidationController", () => { ] }); + mockValidationService.getPolygonValidation.mockResolvedValueOnce(polygon1Validation); + const result = serialize(await controller.createPolygonValidations(request)); - expect(mockValidationService.validatePolygons).toHaveBeenCalledWith(request); + expect(mockValidationService.validatePolygonsBatch).toHaveBeenCalledWith(["polygon-1"], ["DATA_COMPLETENESS"]); expect(result.data).toBeDefined(); if (Array.isArray(result.data)) { expect(result.data).toHaveLength(1); const dataArray = result.data as unknown as Array<{ id: string; - attributes: { polygonId: string; criteriaList: unknown[] }; + attributes: { polygonUuid: string; criteriaList: unknown[] }; }>; const polygonData = dataArray[0]; - expect(polygonData.attributes.polygonId).toBe("polygon-1"); + expect(polygonData.attributes.polygonUuid).toBe("polygon-1"); expect(polygonData.attributes.criteriaList).toHaveLength(1); } else { const singleData = result.data as unknown as { id: string; - attributes: { polygonId: string; criteriaList: unknown[] }; + attributes: { polygonUuid: string; criteriaList: unknown[] }; }; - expect(singleData.attributes.polygonId).toBe("polygon-1"); + expect(singleData.attributes.polygonUuid).toBe("polygon-1"); expect(singleData.attributes.criteriaList).toHaveLength(1); } }); it("should create polygon validations with PLANT_START_DATE validation type", async () => { - const request: ValidationRequestDto = { - polygonUuids: ["polygon-1"], - validationTypes: ["PLANT_START_DATE"] + const request: ValidationRequestBody = { + data: { + type: "validations", + attributes: { + polygonUuids: ["polygon-1"], + validationTypes: ["PLANT_START_DATE"] + } + } }; - mockValidationService.validatePolygons.mockResolvedValueOnce({ - results: [ + const polygon1Validation = new ValidationDto(); + populateDto(polygon1Validation, { + polygonUuid: "polygon-1", + criteriaList: [ { - polygonUuid: "polygon-1", criteriaId: 15, valid: false, createdAt: new Date("2025-01-08T22:15:15.000Z"), @@ -293,40 +383,48 @@ describe("ValidationController", () => { ] }); + mockValidationService.getPolygonValidation.mockResolvedValueOnce(polygon1Validation); + const result = serialize(await controller.createPolygonValidations(request)); - expect(mockValidationService.validatePolygons).toHaveBeenCalledWith(request); + expect(mockValidationService.validatePolygonsBatch).toHaveBeenCalledWith(["polygon-1"], ["PLANT_START_DATE"]); expect(result.data).toBeDefined(); if (Array.isArray(result.data)) { expect(result.data).toHaveLength(1); const dataArray = result.data as unknown as Array<{ id: string; - attributes: { polygonId: string; criteriaList: unknown[] }; + attributes: { polygonUuid: string; criteriaList: unknown[] }; }>; const polygonData = dataArray[0]; - expect(polygonData.attributes.polygonId).toBe("polygon-1"); + expect(polygonData.attributes.polygonUuid).toBe("polygon-1"); expect(polygonData.attributes.criteriaList).toHaveLength(1); } else { const singleData = result.data as unknown as { id: string; - attributes: { polygonId: string; criteriaList: unknown[] }; + attributes: { polygonUuid: string; criteriaList: unknown[] }; }; - expect(singleData.attributes.polygonId).toBe("polygon-1"); + expect(singleData.attributes.polygonUuid).toBe("polygon-1"); expect(singleData.attributes.criteriaList).toHaveLength(1); } }); it("should create polygon validations with POLYGON_SIZE validation type", async () => { - const request: ValidationRequestDto = { - polygonUuids: ["polygon-1"], - validationTypes: ["POLYGON_SIZE"] + const request: ValidationRequestBody = { + data: { + type: "validations", + attributes: { + polygonUuids: ["polygon-1"], + validationTypes: ["POLYGON_SIZE"] + } + } }; - mockValidationService.validatePolygons.mockResolvedValueOnce({ - results: [ + const polygon1Validation = new ValidationDto(); + populateDto(polygon1Validation, { + polygonUuid: "polygon-1", + criteriaList: [ { - polygonUuid: "polygon-1", criteriaId: 6, valid: false, createdAt: new Date("2025-01-08T22:15:15.000Z"), @@ -338,40 +436,48 @@ describe("ValidationController", () => { ] }); + mockValidationService.getPolygonValidation.mockResolvedValueOnce(polygon1Validation); + const result = serialize(await controller.createPolygonValidations(request)); - expect(mockValidationService.validatePolygons).toHaveBeenCalledWith(request); + expect(mockValidationService.validatePolygonsBatch).toHaveBeenCalledWith(["polygon-1"], ["POLYGON_SIZE"]); expect(result.data).toBeDefined(); if (Array.isArray(result.data)) { expect(result.data).toHaveLength(1); const dataArray = result.data as unknown as Array<{ id: string; - attributes: { polygonId: string; criteriaList: unknown[] }; + attributes: { polygonUuid: string; criteriaList: unknown[] }; }>; const polygonData = dataArray[0]; - expect(polygonData.attributes.polygonId).toBe("polygon-1"); + expect(polygonData.attributes.polygonUuid).toBe("polygon-1"); expect(polygonData.attributes.criteriaList).toHaveLength(1); } else { const singleData = result.data as unknown as { id: string; - attributes: { polygonId: string; criteriaList: unknown[] }; + attributes: { polygonUuid: string; criteriaList: unknown[] }; }; - expect(singleData.attributes.polygonId).toBe("polygon-1"); + expect(singleData.attributes.polygonUuid).toBe("polygon-1"); expect(singleData.attributes.criteriaList).toHaveLength(1); } }); it("should create polygon validations with ESTIMATED_AREA validation type", async () => { - const request: ValidationRequestDto = { - polygonUuids: ["polygon-1"], - validationTypes: ["ESTIMATED_AREA"] + const request: ValidationRequestBody = { + data: { + type: "validations", + attributes: { + polygonUuids: ["polygon-1"], + validationTypes: ["ESTIMATED_AREA"] + } + } }; - mockValidationService.validatePolygons.mockResolvedValueOnce({ - results: [ + const polygon1Validation = new ValidationDto(); + populateDto(polygon1Validation, { + polygonUuid: "polygon-1", + criteriaList: [ { - polygonUuid: "polygon-1", criteriaId: 12, valid: true, createdAt: new Date("2025-01-08T22:15:15.000Z"), @@ -391,28 +497,79 @@ describe("ValidationController", () => { ] }); + mockValidationService.getPolygonValidation.mockResolvedValueOnce(polygon1Validation); + const result = serialize(await controller.createPolygonValidations(request)); - expect(mockValidationService.validatePolygons).toHaveBeenCalledWith(request); + expect(mockValidationService.validatePolygonsBatch).toHaveBeenCalledWith(["polygon-1"], ["ESTIMATED_AREA"]); expect(result.data).toBeDefined(); if (Array.isArray(result.data)) { expect(result.data).toHaveLength(1); const dataArray = result.data as unknown as Array<{ id: string; - attributes: { polygonId: string; criteriaList: unknown[] }; + attributes: { polygonUuid: string; criteriaList: unknown[] }; }>; const polygonData = dataArray[0]; - expect(polygonData.attributes.polygonId).toBe("polygon-1"); + expect(polygonData.attributes.polygonUuid).toBe("polygon-1"); expect(polygonData.attributes.criteriaList).toHaveLength(1); } else { const singleData = result.data as unknown as { id: string; - attributes: { polygonId: string; criteriaList: unknown[] }; + attributes: { polygonUuid: string; criteriaList: unknown[] }; }; - expect(singleData.attributes.polygonId).toBe("polygon-1"); + expect(singleData.attributes.polygonUuid).toBe("polygon-1"); expect(singleData.attributes.criteriaList).toHaveLength(1); } }); }); + + describe("createSiteValidation", () => { + const siteUuid = "site-uuid-123"; + const mockRequest = { authenticatedUserId: 1 }; + + it("should create a site validation job", async () => { + const request: SiteValidationRequestBody = { + data: { + type: "validations", + attributes: { + validationTypes: ["SELF_INTERSECTION", "SPIKES"] as ValidationType[] + } + } + }; + const result = serialize(await controller.createSiteValidation(siteUuid, request, mockRequest)); + + expect(mockValidationService.getSitePolygonUuids).toHaveBeenCalledWith(siteUuid); + expect(mockQueue.add).toHaveBeenCalledWith("siteValidation", { + siteUuid, + validationTypes: ["SELF_INTERSECTION", "SPIKES"], + delayedJobId: 1 + }); + expect(result.data).toBeDefined(); + }); + + it("should throw NotFoundException when site has no polygons", async () => { + mockValidationService.getSitePolygonUuids.mockResolvedValueOnce([]); + const request: SiteValidationRequestBody = { + data: { + type: "validations", + attributes: { + validationTypes: ["SELF_INTERSECTION"] as ValidationType[] + } + } + }; + await expect(controller.createSiteValidation(siteUuid, request, mockRequest)).rejects.toThrow(NotFoundException); + }); + + it("should use all validation types when none provided", async () => { + const request: SiteValidationRequestBody = { + data: { + type: "validations", + attributes: {} + } + }; + await controller.createSiteValidation(siteUuid, request, mockRequest); + expect(mockQueue.add).toHaveBeenCalledWith("siteValidation", expect.objectContaining({ siteUuid })); + }); + }); }); diff --git a/apps/research-service/src/validations/validation.controller.ts b/apps/research-service/src/validations/validation.controller.ts index b14c38ba..9b1f91e7 100644 --- a/apps/research-service/src/validations/validation.controller.ts +++ b/apps/research-service/src/validations/validation.controller.ts @@ -1,19 +1,38 @@ -import { BadRequestException, Controller, Get, NotFoundException, Param, Query, Post, Body } from "@nestjs/common"; +import { + BadRequestException, + Controller, + Get, + NotFoundException, + Param, + Query, + Post, + Body, + Request +} from "@nestjs/common"; import { ApiOperation, ApiTags } from "@nestjs/swagger"; import { ValidationService } from "./validation.service"; import { ValidationDto } from "./dto/validation.dto"; -import { ValidationRequestDto } from "./dto/validation-request.dto"; -import { ValidationCriteriaDto } from "./dto/validation-criteria.dto"; +import { ValidationRequestBody } from "./dto/validation-request.dto"; +import { ValidationSummaryDto } from "./dto/validation-summary.dto"; +import { SiteValidationRequestBody } from "./dto/site-validation-request.dto"; import { ExceptionResponse, JsonApiResponse } from "@terramatch-microservices/common/decorators"; import { buildJsonApi, getStableRequestQuery } from "@terramatch-microservices/common/util"; import { MAX_PAGE_SIZE } from "@terramatch-microservices/common/util/paginated-query.builder"; import { SiteValidationQueryDto } from "./dto/site-validation-query.dto"; -import { CriteriaId } from "@terramatch-microservices/database/constants"; +import { CriteriaId, VALIDATION_TYPES } from "@terramatch-microservices/database/constants"; +import { InjectQueue } from "@nestjs/bullmq"; +import { Queue } from "bullmq"; +import { DelayedJob, Site } from "@terramatch-microservices/database/entities"; +import { DelayedJobDto } from "@terramatch-microservices/common/dto/delayed-job.dto"; +import { populateDto } from "@terramatch-microservices/common/dto/json-api-attributes"; @Controller("validations/v3") @ApiTags("Validations") export class ValidationController { - constructor(private readonly validationService: ValidationService) {} + constructor( + private readonly validationService: ValidationService, + @InjectQueue("validation") private readonly validationQueue: Queue + ) {} @Get("polygons/:polygonUuid") @ApiOperation({ @@ -60,7 +79,7 @@ export class ValidationController { return validations .reduce( - (document, validation) => document.addData(validation.polygonId, validation).document, + (document, validation) => document.addData(validation.polygonUuid, validation).document, buildJsonApi(ValidationDto) ) .addIndex({ @@ -82,32 +101,86 @@ export class ValidationController { @ExceptionResponse(BadRequestException, { description: "Invalid validation request" }) - async createPolygonValidations(@Body() request: ValidationRequestDto) { - const validationResponse = await this.validationService.validatePolygons(request); + async createPolygonValidations(@Body() payload: ValidationRequestBody) { + const request = payload.data.attributes; - const document = buildJsonApi(ValidationDto); + const validationTypes = + request.validationTypes == null || request.validationTypes.length === 0 + ? [...VALIDATION_TYPES] + : request.validationTypes; - const resultsByPolygon = new Map(); - - for (const result of validationResponse.results) { - if (result.polygonUuid != null) { - if (!resultsByPolygon.has(result.polygonUuid)) { - resultsByPolygon.set(result.polygonUuid, []); - } - const criteriaList = resultsByPolygon.get(result.polygonUuid); - if (criteriaList != null) { - criteriaList.push(result); - } - } - } + await this.validationService.validatePolygonsBatch(request.polygonUuids, validationTypes); - for (const [polygonUuid, criteriaList] of resultsByPolygon) { - const validation = new ValidationDto(); - validation.polygonId = polygonUuid; - validation.criteriaList = criteriaList; + const document = buildJsonApi(ValidationDto); + + for (const polygonUuid of request.polygonUuids) { + const validation = await this.validationService.getPolygonValidation(polygonUuid); document.addData(polygonUuid, validation); } return document; } + + @Post("sites/:siteUuid/validation") + @ApiOperation({ + operationId: "createSiteValidation", + summary: "Start asynchronous validation for all polygons in a site" + }) + @JsonApiResponse([ValidationSummaryDto, DelayedJobDto]) + @ExceptionResponse(NotFoundException, { + description: "Site not found or has no polygons" + }) + @ExceptionResponse(BadRequestException, { + description: "Invalid validation request" + }) + async createSiteValidation( + @Param("siteUuid") siteUuid: string, + @Body() payload: SiteValidationRequestBody, + @Request() { authenticatedUserId } + ) { + const request = payload.data.attributes; + + const polygonUuids = await this.validationService.getSitePolygonUuids(siteUuid); + + if (polygonUuids.length === 0) { + throw new NotFoundException(`No polygons found for site ${siteUuid}`); + } + + const site = await Site.findOne({ + where: { uuid: siteUuid }, + attributes: ["id", "name"] + }); + + if (site == null) { + throw new NotFoundException(`Site with UUID ${siteUuid} not found`); + } + + const validationTypes = + request.validationTypes == null || request.validationTypes.length === 0 + ? VALIDATION_TYPES + : request.validationTypes; + + const delayedJob = await DelayedJob.create({ + isAcknowledged: false, + name: "Polygon Validation", + totalContent: polygonUuids.length, + processedContent: 0, + progressMessage: "Starting validation...", + createdBy: authenticatedUserId, + metadata: { + entity_id: site.id, + entity_type: "App\\Models\\V2\\Sites\\Site", + entity_name: site.name + } + } as DelayedJob); + + await this.validationQueue.add("siteValidation", { + siteUuid, + validationTypes, + delayedJobId: delayedJob.id + }); + + const delayedJobDto = populateDto(new DelayedJobDto(), delayedJob); + return buildJsonApi(DelayedJobDto).addData(delayedJob.uuid, delayedJobDto); + } } diff --git a/apps/research-service/src/validations/validation.processor.ts b/apps/research-service/src/validations/validation.processor.ts new file mode 100644 index 00000000..167b01bb --- /dev/null +++ b/apps/research-service/src/validations/validation.processor.ts @@ -0,0 +1,93 @@ +import { Processor, WorkerHost } from "@nestjs/bullmq"; +import { Job } from "bullmq"; +import { TMLogger } from "@terramatch-microservices/common/util/tm-logger"; +import { DelayedJob } from "@terramatch-microservices/database/entities"; +import { ValidationService } from "./validation.service"; +import { ValidationType } from "@terramatch-microservices/database/constants"; + +export interface SiteValidationJobData { + siteUuid: string; + validationTypes: ValidationType[]; + delayedJobId: number; +} + +@Processor("validation") +export class ValidationProcessor extends WorkerHost { + private readonly logger = new TMLogger(ValidationProcessor.name); + + constructor(private readonly validationService: ValidationService) { + super(); + } + + async process(job: Job) { + const { siteUuid, validationTypes, delayedJobId } = job.data; + + try { + const polygonUuids = await this.validationService.getSitePolygonUuids(siteUuid); + + if (polygonUuids.length === 0) { + await DelayedJob.update( + { + status: "failed", + statusCode: 404, + payload: { error: `No polygons found for site ${siteUuid}` } + }, + { where: { id: delayedJobId } } + ); + return; + } + + await DelayedJob.update( + { + totalContent: polygonUuids.length, + processedContent: 0, + progressMessage: `Starting validation of ${polygonUuids.length} polygons...` + }, + { where: { id: delayedJobId } } + ); + + const batchSize = 50; + let processed = 0; + + for (let i = 0; i < polygonUuids.length; i += batchSize) { + const batch = polygonUuids.slice(i, i + batchSize); + + await this.validationService.validatePolygonsBatch(batch, validationTypes); + + processed += batch.length; + + const progressPercentage = Math.floor((processed / polygonUuids.length) * 100); + await DelayedJob.update( + { + processedContent: processed, + progressMessage: `Running ${processed} out of ${polygonUuids.length} polygons (${progressPercentage}%)` + }, + { where: { id: delayedJobId } } + ); + } + + const summary = await this.validationService.generateValidationSummary(siteUuid, validationTypes); + + await DelayedJob.update( + { + status: "succeeded", + statusCode: 200, + payload: summary, + progressMessage: `Completed validation of ${polygonUuids.length} polygons` + }, + { where: { id: delayedJobId } } + ); + } catch (error) { + this.logger.error(`Error processing site validation for site ${siteUuid}:`, error); + + await DelayedJob.update( + { + status: "failed", + statusCode: 500, + payload: { error: error instanceof Error ? error.message : "Unknown error occurred" } + }, + { where: { id: delayedJobId } } + ); + } + } +} diff --git a/apps/research-service/src/validations/validation.service.spec.ts b/apps/research-service/src/validations/validation.service.spec.ts index 0d5d7f29..7102b80c 100644 --- a/apps/research-service/src/validations/validation.service.spec.ts +++ b/apps/research-service/src/validations/validation.service.spec.ts @@ -11,10 +11,12 @@ import { } from "@terramatch-microservices/database/entities"; import { Literal } from "sequelize/types/utils"; import { ValidationType } from "@terramatch-microservices/database/constants"; +import { Op } from "sequelize"; interface MockCriteriaSite { update: jest.MockedFunction<(data: { valid: boolean; extraInfo: object | null }) => Promise>; save: jest.MockedFunction<() => Promise>; + destroy: jest.MockedFunction<() => Promise>; } interface MockSelfIntersectionValidator { @@ -39,7 +41,8 @@ jest.mock("@terramatch-microservices/database/entities", () => ({ } }, CriteriaSite: jest.fn().mockImplementation(() => ({ - save: jest.fn() + save: jest.fn(), + destroy: jest.fn() })), CriteriaSiteHistoric: jest.fn().mockImplementation(() => ({ save: jest.fn() @@ -48,11 +51,14 @@ jest.mock("@terramatch-microservices/database/entities", () => ({ findAndCountAll: jest.fn(), findAll: jest.fn(), findOne: jest.fn(), - sum: jest.fn() + save: jest.fn(), + sum: jest.fn(), + update: jest.fn() }, Site: { findAll: jest.fn(), - uuidsSubquery: jest.fn() + uuidsSubquery: jest.fn(), + findOne: jest.fn() }, Project: { findByPk: jest.fn() @@ -93,6 +99,7 @@ describe("ValidationService", () => { jest.clearAllMocks(); (PolygonGeometry.sequelize?.query as jest.Mock)?.mockResolvedValue([{ is_simple: true }]); + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([]); mockSelfIntersectionValidator = { validatePolygon: jest.fn(), @@ -139,7 +146,7 @@ describe("ValidationService", () => { }); expect(result).toBeDefined(); - expect(result.polygonId).toBe(polygonUuid); + expect(result.polygonUuid).toBe(polygonUuid); expect(result.criteriaList).toHaveLength(3); expect(result.criteriaList[0]).toEqual({ criteriaId: 3, @@ -169,7 +176,7 @@ describe("ValidationService", () => { const result = await service.getPolygonValidation(polygonUuid); - expect(result.polygonId).toBe(polygonUuid); + expect(result.polygonUuid).toBe(polygonUuid); expect(result.criteriaList).toEqual([]); }); }); @@ -226,9 +233,9 @@ describe("ValidationService", () => { expect(result.validations).toHaveLength(2); expect(result.total).toBe(2); - expect(result.validations[0].polygonId).toBe(polygonUuid1); + expect(result.validations[0].polygonUuid).toBe(polygonUuid1); expect(result.validations[0].criteriaList).toHaveLength(2); - expect(result.validations[1].polygonId).toBe(polygonUuid2); + expect(result.validations[1].polygonUuid).toBe(polygonUuid2); expect(result.validations[1].criteriaList).toHaveLength(1); }); @@ -285,7 +292,8 @@ describe("ValidationService", () => { describe("validatePolygons", () => { const mockCriteriaSite: MockCriteriaSite = { update: jest.fn(), - save: jest.fn() + save: jest.fn(), + destroy: jest.fn() }; beforeEach(() => { @@ -306,14 +314,12 @@ describe("ValidationService", () => { expect(result.results).toHaveLength(2); expect(result.results[0]).toEqual({ - polygonUuid: "uuid-1", criteriaId: 4, valid: true, createdAt: expect.any(Date), extraInfo: null }); expect(result.results[1]).toEqual({ - polygonUuid: "uuid-2", criteriaId: 4, valid: false, createdAt: expect.any(Date), @@ -345,7 +351,6 @@ describe("ValidationService", () => { expect(result.results).toHaveLength(1); expect(result.results[0]).toEqual({ - polygonUuid: "uuid-1", criteriaId: 8, valid: false, createdAt: expect.any(Date), @@ -395,7 +400,6 @@ describe("ValidationService", () => { expect(result.results).toHaveLength(1); expect(result.results[0]).toEqual({ - polygonUuid: "uuid-1", criteriaId: 14, valid: false, createdAt: expect.any(Date), @@ -436,7 +440,6 @@ describe("ValidationService", () => { expect(result.results).toHaveLength(1); expect(result.results[0]).toEqual({ - polygonUuid: "uuid-1", criteriaId: 15, valid: true, createdAt: expect.any(Date), @@ -472,7 +475,6 @@ describe("ValidationService", () => { expect(result.results).toHaveLength(1); expect(result.results[0]).toEqual({ - polygonUuid: "uuid-1", criteriaId: 6, valid: true, createdAt: expect.any(Date), @@ -517,7 +519,6 @@ describe("ValidationService", () => { expect(result.results).toHaveLength(1); expect(result.results[0]).toEqual({ - polygonUuid: "uuid-1", criteriaId: 12, valid: true, createdAt: expect.any(Date), @@ -604,7 +605,8 @@ describe("ValidationService", () => { valid: false, extraInfo: { old: "data" }, update: jest.fn(), - save: jest.fn() + save: jest.fn(), + destroy: jest.fn() }; (CriteriaSite.findOne as jest.Mock).mockResolvedValue(existingCriteria); @@ -613,10 +615,8 @@ describe("ValidationService", () => { await service.validatePolygons(request); expect(CriteriaSiteHistoric).toHaveBeenCalled(); - expect(existingCriteria.update).toHaveBeenCalledWith({ - valid: true, - extraInfo: null - }); + expect(existingCriteria.destroy).toHaveBeenCalled(); + expect(CriteriaSite).toHaveBeenCalledTimes(1); // Once for new record }); it("should handle validator errors gracefully", async () => { @@ -699,7 +699,339 @@ describe("ValidationService", () => { expect(result.total).toBe(1); // Only polygon-1 has criteriaId=4 with valid=false expect(result.validations).toHaveLength(1); - expect(result.validations[0].polygonId).toBe("polygon-1"); + expect(result.validations[0].polygonUuid).toBe("polygon-1"); + }); + }); + + describe("getSitePolygonUuids", () => { + it("should return polygon UUIDs for a site", async () => { + (Site.findOne as jest.Mock).mockResolvedValue({ uuid: "site-uuid" }); + (SitePolygon.findAll as jest.Mock).mockResolvedValue([ + { polygonUuid: "polygon-1" }, + { polygonUuid: "polygon-2" } + ]); + + const result = await service.getSitePolygonUuids("site-uuid"); + + expect(result).toEqual(["polygon-1", "polygon-2"]); + }); + + it("should throw NotFoundException when site not found", async () => { + (Site.findOne as jest.Mock).mockResolvedValue(null); + await expect(service.getSitePolygonUuids("non-existent")).rejects.toThrow(NotFoundException); + }); + + it("should filter out empty polygon UUIDs", async () => { + (Site.findOne as jest.Mock).mockResolvedValue({ uuid: "site-uuid" }); + (SitePolygon.findAll as jest.Mock).mockResolvedValue([ + { polygonUuid: "polygon-1" }, + { polygonUuid: "" }, + { polygonUuid: null } + ]); + + const result = await service.getSitePolygonUuids("site-uuid"); + + expect(result).toEqual(["polygon-1"]); + }); + }); + + describe("validatePolygonsBatch", () => { + it("should use batch validation when available", async () => { + mockSelfIntersectionValidator.validatePolygons.mockResolvedValue([ + { polygonUuid: "uuid-1", valid: true, extraInfo: null }, + { polygonUuid: "uuid-2", valid: false, extraInfo: null } + ]); + + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([]); + (CriteriaSite as jest.MockedClass).bulkCreate = jest.fn().mockResolvedValue(undefined); + + await service.validatePolygonsBatch(["uuid-1", "uuid-2"], ["SELF_INTERSECTION"]); + + expect(mockSelfIntersectionValidator.validatePolygons).toHaveBeenCalledWith(["uuid-1", "uuid-2"]); + }); + + it("should throw BadRequestException for duplicate results from validator", async () => { + mockSelfIntersectionValidator.validatePolygons.mockResolvedValue([ + { polygonUuid: "uuid-1", valid: true, extraInfo: null }, + { polygonUuid: "uuid-1", valid: false, extraInfo: null } + ]); + + await expect(service.validatePolygonsBatch(["uuid-1"], ["SELF_INTERSECTION"])).rejects.toThrow( + BadRequestException + ); + }); + + it("should fall back to single validation when batch not available", async () => { + (VALIDATORS as Record).POLYGON_SIZE = { validatePolygon: jest.fn() }; + ( + (VALIDATORS as Record).POLYGON_SIZE as { validatePolygon: jest.Mock } + ).validatePolygon.mockResolvedValue({ + valid: true, + extraInfo: null + }); + + (SitePolygon.findOne as jest.Mock).mockResolvedValue({ calcArea: 500 }); + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([]); + (CriteriaSite as jest.MockedClass).bulkCreate = jest.fn().mockResolvedValue(undefined); + + await service.validatePolygonsBatch(["uuid-1"], ["POLYGON_SIZE"]); + + expect( + ((VALIDATORS as Record).POLYGON_SIZE as { validatePolygon: jest.Mock }).validatePolygon + ).toHaveBeenCalledWith("uuid-1"); + }); + }); + + describe("saveValidationResultsBatch", () => { + beforeEach(() => { + (CriteriaSite as jest.MockedClass).bulkCreate = jest.fn().mockResolvedValue(undefined); + (CriteriaSite as jest.MockedClass).destroy = jest.fn().mockResolvedValue(undefined); + (CriteriaSiteHistoric as jest.MockedClass).bulkCreate = jest + .fn() + .mockResolvedValue(undefined); + }); + + it("should handle empty results", async () => { + await service.saveValidationResultsBatch([]); + expect(CriteriaSite.findAll).not.toHaveBeenCalled(); + }); + + it("should create new records when none exist", async () => { + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([]); + + await service.saveValidationResultsBatch([ + { polygonUuid: "uuid-1", criteriaId: 4, valid: true, extraInfo: null } + ]); + + expect(CriteriaSite.bulkCreate).toHaveBeenCalledWith( + [{ polygonId: "uuid-1", criteriaId: 4, valid: true, extraInfo: null }], + { validate: true } + ); + }); + + it("should update existing records and create historic records", async () => { + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([ + { id: 1, polygonId: "uuid-1", criteriaId: 4, valid: false, extraInfo: { old: "data" } } + ]); + + await service.saveValidationResultsBatch([ + { polygonUuid: "uuid-1", criteriaId: 4, valid: true, extraInfo: null } + ]); + + expect(CriteriaSiteHistoric.bulkCreate).toHaveBeenCalled(); + expect(CriteriaSite.destroy).toHaveBeenCalled(); + expect(CriteriaSite.bulkCreate).toHaveBeenCalled(); + }); + + it("should deduplicate results with same polygon and criteria", async () => { + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([]); + + await service.saveValidationResultsBatch([ + { polygonUuid: "uuid-1", criteriaId: 4, valid: true, extraInfo: null }, + { polygonUuid: "uuid-1", criteriaId: 4, valid: false, extraInfo: null } + ]); + + expect(CriteriaSite.bulkCreate).toHaveBeenCalledWith( + [{ polygonId: "uuid-1", criteriaId: 4, valid: false, extraInfo: null }], + { validate: true } + ); + }); + }); + + describe("generateValidationSummary", () => { + it("should generate validation summary for a site", async () => { + (Site.findOne as jest.Mock).mockResolvedValue({ uuid: "site-uuid" }); + (SitePolygon.findAll as jest.Mock).mockResolvedValue([ + { polygonUuid: "polygon-1" }, + { polygonUuid: "polygon-2" } + ]); + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([{ valid: true }, { valid: false }]); + + const result = await service.generateValidationSummary("site-uuid", ["SELF_INTERSECTION"]); + + expect(result.siteUuid).toBe("site-uuid"); + expect(result.totalPolygons).toBe(2); + expect(result.validatedPolygons).toBe(2); + expect(result.validationSummary.SELF_INTERSECTION).toEqual({ valid: 1, invalid: 1 }); + }); + + it("should throw NotFoundException when site has no polygons", async () => { + (Site.findOne as jest.Mock).mockResolvedValue({ uuid: "site-uuid" }); + (SitePolygon.findAll as jest.Mock).mockResolvedValue([]); + + await expect(service.generateValidationSummary("site-uuid", ["SELF_INTERSECTION"])).rejects.toThrow( + NotFoundException + ); + }); + }); + + describe("updateSitePolygonValidityBatch", () => { + it("should handle empty polygon UUIDs array", async () => { + await ( + service as unknown as { updateSitePolygonValidityBatch: (uuids: string[]) => Promise } + ).updateSitePolygonValidityBatch([]); + + expect(SitePolygon.findAll).not.toHaveBeenCalled(); + }); + + it("should handle no SitePolygons found", async () => { + (SitePolygon.findAll as jest.Mock).mockResolvedValue([]); + + await ( + service as unknown as { updateSitePolygonValidityBatch: (uuids: string[]) => Promise } + ).updateSitePolygonValidityBatch(["polygon-1"]); + + expect(SitePolygon.findAll).toHaveBeenCalledWith({ + where: { + polygonUuid: { [Op.in]: ["polygon-1"] }, + isActive: true + } + }); + expect(CriteriaSite.findAll).not.toHaveBeenCalled(); + }); + + it("should set validation status to null when no criteria exist", async () => { + const mockSitePolygon = { + id: 1, + polygonUuid: "polygon-1", + validationStatus: "passed" + }; + + (SitePolygon.findAll as jest.Mock).mockResolvedValue([mockSitePolygon]); + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([]); + (SitePolygon.update as jest.Mock).mockResolvedValue(undefined); + + await ( + service as unknown as { updateSitePolygonValidityBatch: (uuids: string[]) => Promise } + ).updateSitePolygonValidityBatch(["polygon-1"]); + + expect(SitePolygon.update).toHaveBeenCalledWith({ validationStatus: null }, { where: { id: 1 } }); + }); + + it("should set validation status to 'passed' when all criteria pass", async () => { + const mockSitePolygon = { + id: 1, + polygonUuid: "polygon-1", + validationStatus: null + }; + + (SitePolygon.findAll as jest.Mock).mockResolvedValue([mockSitePolygon]); + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([ + { polygonId: "polygon-1", criteriaId: 3, valid: true }, + { polygonId: "polygon-1", criteriaId: 4, valid: true } + ]); + (SitePolygon.update as jest.Mock).mockResolvedValue(undefined); + + await ( + service as unknown as { updateSitePolygonValidityBatch: (uuids: string[]) => Promise } + ).updateSitePolygonValidityBatch(["polygon-1"]); + + expect(SitePolygon.update).toHaveBeenCalledWith({ validationStatus: "passed" }, { where: { id: 1 } }); + }); + + it("should set validation status to 'failed' when non-excluded criteria fail", async () => { + const mockSitePolygon = { + id: 1, + polygonUuid: "polygon-1", + validationStatus: "passed" + }; + + (SitePolygon.findAll as jest.Mock).mockResolvedValue([mockSitePolygon]); + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([ + { polygonId: "polygon-1", criteriaId: 3, valid: true }, // Non-excluded, passing + { polygonId: "polygon-1", criteriaId: 4, valid: false } // Non-excluded, failing + ]); + (SitePolygon.update as jest.Mock).mockResolvedValue(undefined); + + await ( + service as unknown as { updateSitePolygonValidityBatch: (uuids: string[]) => Promise } + ).updateSitePolygonValidityBatch(["polygon-1"]); + + expect(SitePolygon.update).toHaveBeenCalledWith({ validationStatus: "failed" }, { where: { id: 1 } }); + }); + + it("should set validation status to 'partial' when only excluded criteria fail", async () => { + const mockSitePolygon = { + id: 1, + polygonUuid: "polygon-1", + validationStatus: "passed" + }; + + (SitePolygon.findAll as jest.Mock).mockResolvedValue([mockSitePolygon]); + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([ + { polygonId: "polygon-1", criteriaId: 7, valid: false }, // WITHIN_COUNTRY - excluded criteria, failing + { polygonId: "polygon-1", criteriaId: 12, valid: false } // ESTIMATED_AREA - excluded criteria, failing + ]); + (SitePolygon.update as jest.Mock).mockResolvedValue(undefined); + + await ( + service as unknown as { updateSitePolygonValidityBatch: (uuids: string[]) => Promise } + ).updateSitePolygonValidityBatch(["polygon-1"]); + + expect(SitePolygon.update).toHaveBeenCalledWith({ validationStatus: "partial" }, { where: { id: 1 } }); + }); + + it("should set validation status to 'failed' when both excluded and non-excluded criteria fail", async () => { + const mockSitePolygon = { + id: 1, + polygonUuid: "polygon-1", + validationStatus: "passed" + }; + + (SitePolygon.findAll as jest.Mock).mockResolvedValue([mockSitePolygon]); + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([ + { polygonId: "polygon-1", criteriaId: 7, valid: false }, // WITHIN_COUNTRY - excluded criteria, failing + { polygonId: "polygon-1", criteriaId: 4, valid: false } // SELF_INTERSECTION - non-excluded criteria, failing + ]); + (SitePolygon.update as jest.Mock).mockResolvedValue(undefined); + + await ( + service as unknown as { updateSitePolygonValidityBatch: (uuids: string[]) => Promise } + ).updateSitePolygonValidityBatch(["polygon-1"]); + + expect(SitePolygon.update).toHaveBeenCalledWith({ validationStatus: "failed" }, { where: { id: 1 } }); + }); + + it("should not update when validation status does not need updating", async () => { + const mockSitePolygon = { + id: 1, + polygonUuid: "polygon-1", + validationStatus: "passed" + }; + + (SitePolygon.findAll as jest.Mock).mockResolvedValue([mockSitePolygon]); + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([ + { polygonId: "polygon-1", criteriaId: 3, valid: true }, + { polygonId: "polygon-1", criteriaId: 4, valid: true } + ]); + + await ( + service as unknown as { updateSitePolygonValidityBatch: (uuids: string[]) => Promise } + ).updateSitePolygonValidityBatch(["polygon-1"]); + + expect(SitePolygon.update).not.toHaveBeenCalled(); + }); + + it("should handle multiple polygons in batch", async () => { + const mockSitePolygons = [ + { id: 1, polygonUuid: "polygon-1", validationStatus: "passed" }, + { id: 2, polygonUuid: "polygon-2", validationStatus: null } + ]; + + (SitePolygon.findAll as jest.Mock).mockResolvedValue(mockSitePolygons); + (CriteriaSite.findAll as jest.Mock).mockResolvedValue([ + { polygonId: "polygon-1", criteriaId: 4, valid: false }, // polygon-1 fails + { polygonId: "polygon-2", criteriaId: 4, valid: true } // polygon-2 passes + ]); + (SitePolygon.update as jest.Mock).mockResolvedValue(undefined); + + await ( + service as unknown as { updateSitePolygonValidityBatch: (uuids: string[]) => Promise } + ).updateSitePolygonValidityBatch(["polygon-1", "polygon-2"]); + + expect(SitePolygon.update).toHaveBeenCalledTimes(2); + expect(SitePolygon.update).toHaveBeenCalledWith({ validationStatus: "failed" }, { where: { id: 1 } }); + expect(SitePolygon.update).toHaveBeenCalledWith({ validationStatus: "passed" }, { where: { id: 2 } }); }); }); }); diff --git a/apps/research-service/src/validations/validation.service.ts b/apps/research-service/src/validations/validation.service.ts index 12dedd57..97a51ccb 100644 --- a/apps/research-service/src/validations/validation.service.ts +++ b/apps/research-service/src/validations/validation.service.ts @@ -3,11 +3,13 @@ import { CriteriaSite, CriteriaSiteHistoric, PolygonGeometry, - SitePolygon + SitePolygon, + Site } from "@terramatch-microservices/database/entities"; import { ValidationDto } from "./dto/validation.dto"; -import { ValidationRequestDto } from "./dto/validation-request.dto"; +import { ValidationRequestAttributes } from "./dto/validation-request.dto"; import { ValidationResponseDto, ValidationCriteriaDto } from "./dto/validation-criteria.dto"; +import { ValidationSummaryDto, ValidationTypeSummary } from "./dto/validation-summary.dto"; import { populateDto } from "@terramatch-microservices/common/dto/json-api-attributes"; import { MAX_PAGE_SIZE } from "@terramatch-microservices/common/util/paginated-query.builder"; import { groupBy } from "lodash"; @@ -20,7 +22,28 @@ import { EstimatedAreaValidator } from "./validators/estimated-area.validator"; import { OverlappingValidator } from "./validators/overlapping.validator"; import { WithinCountryValidator } from "./validators/within-country.validator"; import { Validator } from "./validators/validator.interface"; -import { ValidationType, VALIDATION_CRITERIA_IDS, CriteriaId } from "@terramatch-microservices/database/constants"; +import { + ValidationType, + VALIDATION_CRITERIA_IDS, + VALIDATION_TYPES, + CriteriaId, + EXCLUDED_VALIDATION_CRITERIA +} from "@terramatch-microservices/database/constants"; +import { Op } from "sequelize"; + +type ValidationResult = { + polygonUuid: string; + criteriaId: CriteriaId; + valid: boolean; + extraInfo: object | null; +}; + +type CriteriaRecord = { + polygonId: string; + criteriaId: CriteriaId; + valid: boolean; + extraInfo: object | null; +}; export const VALIDATORS: Record = { SELF_INTERSECTION: new SelfIntersectionValidator(), @@ -60,7 +83,7 @@ export class ValidationService { const dto = new ValidationDto(); return populateDto(dto, { - polygonId: polygonUuid, + polygonUuid, criteriaList }); } @@ -127,7 +150,7 @@ export class ValidationService { const validations = paginatedPolygonIds.map(polygonId => populateDto(new ValidationDto(), { - polygonId, + polygonUuid: polygonId, criteriaList: (criteriaByPolygon[polygonId] ?? []).map(criteria => ({ criteriaId: criteria.criteriaId, valid: criteria.valid, @@ -143,11 +166,15 @@ export class ValidationService { }; } - async validatePolygons(request: ValidationRequestDto): Promise { + /** + * @deprecated Use validatePolygonsBatch instead to avoid duplicate results with batch validators + */ + async validatePolygons(request: ValidationRequestAttributes): Promise { const results: ValidationCriteriaDto[] = []; + const validationTypes = request.validationTypes ?? [...VALIDATION_TYPES]; for (const polygonUuid of request.polygonUuids) { - for (const validationType of request.validationTypes) { + for (const validationType of validationTypes) { const validator = VALIDATORS[validationType]; if (validator == null) { throw new BadRequestException(`Unknown validation type: ${validationType}`); @@ -159,7 +186,6 @@ export class ValidationService { await this.saveValidationResult(polygonUuid, criteriaId, validationResult.valid, validationResult.extraInfo); results.push({ - polygonUuid: polygonUuid, criteriaId: criteriaId, valid: validationResult.valid, createdAt: new Date(), @@ -195,18 +221,274 @@ export class ValidationService { historicRecord.valid = existingCriteria.valid; historicRecord.extraInfo = existingCriteria.extraInfo; await historicRecord.save(); + await existingCriteria.destroy(); + } + + const newRecord = new CriteriaSite(); + newRecord.polygonId = polygonUuid; + newRecord.criteriaId = criteriaId; + newRecord.valid = valid; + newRecord.extraInfo = extraInfo; + await newRecord.save(); + } + + async getSitePolygonUuids(siteUuid: string): Promise { + const site = await Site.findOne({ + where: { uuid: siteUuid }, + attributes: ["uuid"] + }); + + if (site === null) { + throw new NotFoundException(`Site with UUID ${siteUuid} not found`); + } + + const sitePolygons = await SitePolygon.findAll({ + where: { + siteUuid, + polygonUuid: { [Op.ne]: "" }, + isActive: true, + deletedAt: null + }, + attributes: ["polygonUuid"] + }); + + return sitePolygons.map(sp => sp.polygonUuid).filter(uuid => uuid != null && uuid !== "") as string[]; + } + + async validatePolygonsBatch(polygonUuids: string[], validationTypes: ValidationType[]): Promise { + const validationResults: ValidationResult[] = []; + + for (const validationType of validationTypes) { + const validator = VALIDATORS[validationType]; + if (validator == null) { + throw new BadRequestException(`Unknown validation type: ${validationType}`); + } + + const criteriaId = this.getCriteriaIdForValidationType(validationType); + + if (validator.validatePolygons != null) { + const batchResults = await validator.validatePolygons(polygonUuids); + + const seenPolygons = new Set(); + for (const result of batchResults) { + if (seenPolygons.has(result.polygonUuid)) { + throw new BadRequestException( + `Duplicate result from ${validationType} validator for polygon ${result.polygonUuid}` + ); + } + seenPolygons.add(result.polygonUuid); - await existingCriteria.update({ - valid, - extraInfo + validationResults.push({ + polygonUuid: result.polygonUuid, + criteriaId, + valid: result.valid, + extraInfo: result.extraInfo + }); + } + } else { + for (const polygonUuid of polygonUuids) { + const validationResult = await validator.validatePolygon(polygonUuid); + validationResults.push({ + polygonUuid, + criteriaId, + valid: validationResult.valid, + extraInfo: validationResult.extraInfo + }); + } + } + } + + await this.saveValidationResultsBatch(validationResults); + } + + async saveValidationResultsBatch(results: ValidationResult[]): Promise { + if (results.length === 0) { + return; + } + + const polygonIds = [...new Set(results.map(r => r.polygonUuid))]; + const criteriaIds = [...new Set(results.map(r => r.criteriaId))]; + + const existingCriteria = await CriteriaSite.findAll({ + where: { + polygonId: polygonIds, + criteriaId: criteriaIds + }, + attributes: ["id", "polygonId", "criteriaId", "valid", "extraInfo"] + }); + + const existingMap = new Map(); + for (const criteria of existingCriteria) { + const key = `${criteria.polygonId}_${criteria.criteriaId}`; + existingMap.set(key, criteria); + } + + const deduplicatedResults = new Map(); + for (const result of results) { + const key = `${result.polygonUuid}_${result.criteriaId}`; + deduplicatedResults.set(key, result); + } + + const historicRecords: CriteriaRecord[] = []; + const recordsToCreate: CriteriaRecord[] = []; + const recordsToDelete: number[] = []; + + for (const [key, result] of deduplicatedResults.entries()) { + const existing = existingMap.get(key); + + if (existing != null) { + historicRecords.push({ + polygonId: existing.polygonId, + criteriaId: existing.criteriaId, + valid: existing.valid, + extraInfo: existing.extraInfo + }); + recordsToDelete.push(existing.id); + } + + recordsToCreate.push({ + polygonId: result.polygonUuid, + criteriaId: result.criteriaId, + valid: result.valid, + extraInfo: result.extraInfo }); - } else { - const newRecord = new CriteriaSite(); - newRecord.polygonId = polygonUuid; - newRecord.criteriaId = criteriaId; - newRecord.valid = valid; - newRecord.extraInfo = extraInfo; - await newRecord.save(); + } + + if (historicRecords.length > 0) { + await CriteriaSiteHistoric.bulkCreate(historicRecords as never, { + validate: true + }); + } + + if (recordsToDelete.length > 0) { + await CriteriaSite.destroy({ + where: { + id: recordsToDelete + } + }); + } + + if (recordsToCreate.length > 0) { + await CriteriaSite.bulkCreate(recordsToCreate as never, { + validate: true + }); + + const affectedPolygonUuids = [...new Set(recordsToCreate.map(r => r.polygonId))]; + await this.updateSitePolygonValidityBatch(affectedPolygonUuids); + } + } + + async generateValidationSummary(siteUuid: string, validationTypes: ValidationType[]): Promise { + const polygonUuids = await this.getSitePolygonUuids(siteUuid); + + if (polygonUuids.length === 0) { + throw new NotFoundException(`No polygons found for site with UUID ${siteUuid}`); + } + + const validationSummary: Record = {} as Record< + ValidationType, + ValidationTypeSummary + >; + + for (const validationType of validationTypes) { + const criteriaId = this.getCriteriaIdForValidationType(validationType); + + const criteriaData = await CriteriaSite.findAll({ + where: { + polygonId: { [Op.in]: polygonUuids }, + criteriaId + }, + attributes: ["valid"] + }); + + const validCount = criteriaData.filter(c => c.valid === true).length; + const invalidCount = criteriaData.filter(c => c.valid === false).length; + + validationSummary[validationType] = { + valid: validCount, + invalid: invalidCount + }; + } + + return { + siteUuid, + totalPolygons: polygonUuids.length, + validatedPolygons: polygonUuids.length, + validationSummary, + completedAt: new Date() + }; + } + + private async updateSitePolygonValidityBatch(polygonUuids: string[]): Promise { + if (polygonUuids.length === 0) { + return; + } + + const sitePolygons = await SitePolygon.findAll({ + where: { + polygonUuid: { [Op.in]: polygonUuids }, + isActive: true + } + }); + + if (sitePolygons.length === 0) { + return; + } + + const allCriteria = await CriteriaSite.findAll({ + where: { polygonId: { [Op.in]: polygonUuids } }, + attributes: ["polygonId", "criteriaId", "valid"] + }); + + const criteriaByPolygon = new Map(); + for (const criteria of allCriteria) { + if (!criteriaByPolygon.has(criteria.polygonId)) { + criteriaByPolygon.set(criteria.polygonId, []); + } + const criteriaList = criteriaByPolygon.get(criteria.polygonId); + if (criteriaList != null) { + criteriaList.push(criteria); + } + } + + const excludedCriteriaSet = new Set(EXCLUDED_VALIDATION_CRITERIA as number[]); + const updates: Array<{ id: number; validationStatus: string | null }> = []; + + for (const sitePolygon of sitePolygons) { + if (sitePolygon.polygonUuid == null) continue; + + const polygonCriteria = criteriaByPolygon.get(sitePolygon.polygonUuid) ?? []; + let newValidationStatus: string | null; + + if (polygonCriteria.length === 0) { + newValidationStatus = null; + } else { + const hasAnyFailing = polygonCriteria.some(c => c.valid === false); + + if (!hasAnyFailing) { + newValidationStatus = "passed"; + } else { + const nonExcludedCriteria = polygonCriteria.filter(c => !excludedCriteriaSet.has(c.criteriaId)); + const hasFailingNonExcluded = nonExcludedCriteria.some(c => c.valid === false); + + newValidationStatus = hasFailingNonExcluded ? "failed" : "partial"; + } + } + + if (sitePolygon.validationStatus !== newValidationStatus) { + updates.push({ + id: sitePolygon.id, + validationStatus: newValidationStatus + }); + } + } + + if (updates.length > 0) { + await Promise.all( + updates.map(update => + SitePolygon.update({ validationStatus: update.validationStatus }, { where: { id: update.id } }) + ) + ); } } } diff --git a/apps/research-service/src/validations/validators/estimated-area.validator.ts b/apps/research-service/src/validations/validators/estimated-area.validator.ts index 9b5f8dd5..34c99839 100644 --- a/apps/research-service/src/validations/validators/estimated-area.validator.ts +++ b/apps/research-service/src/validations/validators/estimated-area.validator.ts @@ -1,6 +1,7 @@ import { SitePolygon, Site, Project } from "@terramatch-microservices/database/entities"; import { Validator, ValidationResult, PolygonValidationResult } from "./validator.interface"; import { NotFoundException } from "@nestjs/common"; +import { Op } from "sequelize"; interface EstimatedAreaResult extends ValidationResult { extraInfo: { @@ -170,6 +171,10 @@ export class EstimatedAreaValidator implements Validator { } private async calculateSiteAreaSum(siteUuid: string): Promise { + if (siteUuid == null || siteUuid === "") { + return 0; + } + const result = await SitePolygon.sum("calcArea", { where: { siteUuid, isActive: true } }); @@ -179,7 +184,7 @@ export class EstimatedAreaValidator implements Validator { private async calculateProjectAreaSum(projectId: number): Promise { const result = await SitePolygon.sum("calcArea", { where: { - siteUuid: Site.uuidsSubquery(projectId), + siteUuid: { [Op.in]: Site.uuidsSubquery(projectId) }, isActive: true } }); diff --git a/apps/research-service/src/validations/validators/overlapping.validator.spec.ts b/apps/research-service/src/validations/validators/overlapping.validator.spec.ts index 1e0abceb..fb32af63 100644 --- a/apps/research-service/src/validations/validators/overlapping.validator.spec.ts +++ b/apps/research-service/src/validations/validators/overlapping.validator.spec.ts @@ -1,7 +1,7 @@ import { Test, TestingModule } from "@nestjs/testing"; import { OverlappingValidator } from "./overlapping.validator"; import { SitePolygon, PolygonGeometry } from "@terramatch-microservices/database/entities"; -import { NotFoundException, InternalServerErrorException } from "@nestjs/common"; +import { NotFoundException, InternalServerErrorException, BadRequestException } from "@nestjs/common"; interface MockTransaction { commit: jest.Mock; @@ -474,4 +474,76 @@ describe("OverlappingValidator", () => { expect(result[0].polyUuid).toBe(testUuids.polygon2); }); }); + + describe("validatePolygons", () => { + it("should return empty array for empty input", async () => { + const result = await validator.validatePolygons([]); + expect(result).toEqual([]); + }); + + it("should validate multiple polygons in batch", async () => { + const mockSitePolygons = [ + { polygonUuid: testUuids.polygon1, site: { projectId: testProjectId } }, + { polygonUuid: testUuids.polygon2, site: { projectId: testProjectId } } + ]; + + mockSitePolygonFindAll.mockResolvedValueOnce(mockSitePolygons as unknown as SitePolygon[]); + mockSitePolygonFindAll.mockResolvedValueOnce(mockSitePolygons as unknown as SitePolygon[]); + + const mockTransactionInstance = { commit: jest.fn(), rollback: jest.fn() }; + mockTransaction.mockResolvedValueOnce(mockTransactionInstance); + mockCheckBoundingBoxIntersections.mockResolvedValueOnce([]); + + const result = await validator.validatePolygons([testUuids.polygon1, testUuids.polygon2]); + + expect(result).toHaveLength(2); + expect(result[0].polygonUuid).toBe(testUuids.polygon1); + expect(result[1].polygonUuid).toBe(testUuids.polygon2); + }); + + it("should throw BadRequestException for duplicate polygon UUIDs", async () => { + await expect(validator.validatePolygons([testUuids.polygon1, testUuids.polygon1])).rejects.toThrow( + BadRequestException + ); + }); + + it("should handle polygon not found in batch", async () => { + mockSitePolygonFindAll.mockResolvedValueOnce([]); + const result = await validator.validatePolygons([testUuids.polygon1]); + expect(result[0].valid).toBe(false); + }); + + it("should filter self-intersections in batch", async () => { + const mockSitePolygons = [ + { polygonUuid: testUuids.polygon1, site: { projectId: testProjectId } }, + { polygonUuid: testUuids.polygon2, site: { projectId: testProjectId } } + ]; + + mockSitePolygonFindAll.mockResolvedValueOnce(mockSitePolygons as unknown as SitePolygon[]); + mockSitePolygonFindAll.mockResolvedValueOnce(mockSitePolygons as unknown as SitePolygon[]); + + const mockTransactionInstance = { commit: jest.fn(), rollback: jest.fn() }; + mockTransaction.mockResolvedValueOnce(mockTransactionInstance); + mockCheckBoundingBoxIntersections.mockResolvedValueOnce([ + { targetUuid: testUuids.polygon1, candidateUuid: testUuids.polygon1 } + ]); + mockCheckGeometryIntersections.mockResolvedValueOnce([ + { + targetUuid: testUuids.polygon1, + candidateUuid: testUuids.polygon1, + candidateName: "A", + siteName: "Test", + targetArea: 1000, + candidateArea: 1000, + intersectionArea: 1000, + intersectionLatitude: 35.0 + } + ]); + + const result = await validator.validatePolygons([testUuids.polygon1, testUuids.polygon2]); + + expect(result[0].valid).toBe(true); + expect(result[1].valid).toBe(true); + }); + }); }); diff --git a/apps/research-service/src/validations/validators/overlapping.validator.ts b/apps/research-service/src/validations/validators/overlapping.validator.ts index 12e11cf4..56d85db9 100644 --- a/apps/research-service/src/validations/validators/overlapping.validator.ts +++ b/apps/research-service/src/validations/validators/overlapping.validator.ts @@ -1,6 +1,6 @@ import { PolygonGeometry, SitePolygon, Site } from "@terramatch-microservices/database/entities"; -import { Validator, ValidationResult } from "./validator.interface"; -import { NotFoundException, InternalServerErrorException } from "@nestjs/common"; +import { Validator, ValidationResult, PolygonValidationResult } from "./validator.interface"; +import { NotFoundException, InternalServerErrorException, BadRequestException } from "@nestjs/common"; import { Transaction } from "sequelize"; interface OverlapInfo { @@ -53,6 +53,92 @@ export class OverlappingValidator implements Validator { }; } + async validatePolygons(polygonUuids: string[]): Promise { + if (polygonUuids.length === 0) { + return []; + } + + const uniquePolygonUuids = [...new Set(polygonUuids)]; + if (uniquePolygonUuids.length < polygonUuids.length) { + throw new BadRequestException( + `OverlappingValidator received ${polygonUuids.length - uniquePolygonUuids.length} duplicate polygon UUIDs` + ); + } + + const projectPolygonsMap = await this.getProjectPolygonsBatch(uniquePolygonUuids); + const polygonsByProject = new Map(); + const allProjectPolygonsMap = new Map>(); + + for (const [polygonUuid, projectData] of projectPolygonsMap.entries()) { + if (projectData != null) { + const projectId = projectData.projectId; + + if (!polygonsByProject.has(projectId)) { + polygonsByProject.set(projectId, []); + allProjectPolygonsMap.set(projectId, new Set()); + } + + polygonsByProject.get(projectId)?.push(polygonUuid); + const projectPolygonsSet = allProjectPolygonsMap.get(projectId); + if (projectPolygonsSet != null) { + projectPolygonsSet.add(polygonUuid); + projectData.relatedPolygonUuids.forEach(uuid => projectPolygonsSet.add(uuid)); + } + } + } + + const allIntersections: IntersectionQueryResult[] = []; + for (const [projectId, projectPolygonUuids] of polygonsByProject.entries()) { + const allPolygonsInProjectSet = allProjectPolygonsMap.get(projectId); + const allPolygonsInProject = allPolygonsInProjectSet != null ? Array.from(allPolygonsInProjectSet) : []; + + if (allPolygonsInProject.length > 1) { + const intersections = await this.checkIntersections(projectPolygonUuids, allPolygonsInProject); + + const validIntersections = intersections.filter( + intersection => intersection.targetUuid !== intersection.candidateUuid + ); + + allIntersections.push(...validIntersections); + } + } + const results: PolygonValidationResult[] = []; + for (const polygonUuid of uniquePolygonUuids) { + const projectData = projectPolygonsMap.get(polygonUuid); + + if (projectData == null) { + results.push({ + polygonUuid, + valid: false, + extraInfo: { error: "Polygon not found or has no associated project" } + }); + continue; + } + + const allPolygonsInProjectSet = allProjectPolygonsMap.get(projectData.projectId); + const allPolygonsInProject = allPolygonsInProjectSet != null ? Array.from(allPolygonsInProjectSet) : []; + const otherPolygonsCount = allPolygonsInProject.filter(uuid => uuid !== polygonUuid).length; + + if (otherPolygonsCount === 0) { + results.push({ + polygonUuid, + valid: true, + extraInfo: null + }); + continue; + } + + const overlaps = this.buildOverlapInfo(allIntersections, polygonUuid); + results.push({ + polygonUuid, + valid: overlaps.length === 0, + extraInfo: overlaps.length > 0 ? overlaps : null + }); + } + + return results; + } + private async getProjectPolygons( polygonUuid: string ): Promise<{ projectId: number; relatedPolygonUuids: string[] } | null> { @@ -96,6 +182,71 @@ export class OverlappingValidator implements Validator { return { projectId, relatedPolygonUuids }; } + private async getProjectPolygonsBatch( + polygonUuids: string[] + ): Promise> { + const sitePolygons = await SitePolygon.findAll({ + where: { polygonUuid: polygonUuids, isActive: true }, + include: [ + { + model: Site, + as: "site", + required: true, + attributes: ["projectId"] + } + ], + attributes: ["polygonUuid", "siteUuid"] + }); + + const projectIds = [...new Set(sitePolygons.map(sp => sp.site?.projectId).filter(id => id != null))] as number[]; + + if (projectIds.length === 0) { + return new Map(polygonUuids.map(uuid => [uuid, null])); + } + const allProjectSitePolygons = await SitePolygon.findAll({ + where: { isActive: true }, + include: [ + { + model: Site, + as: "site", + required: true, + where: { projectId: projectIds }, + attributes: ["projectId"] + } + ], + attributes: ["polygonUuid"] + }); + + const polygonsByProjectId = new Map(); + for (const sp of allProjectSitePolygons) { + const projectId = sp.site?.projectId; + if (projectId != null && sp.polygonUuid != null) { + if (!polygonsByProjectId.has(projectId)) { + polygonsByProjectId.set(projectId, []); + } + polygonsByProjectId.get(projectId)?.push(sp.polygonUuid); + } + } + const resultMap = new Map(); + + for (const polygonUuid of polygonUuids) { + const sitePolygon = sitePolygons.find(sp => sp.polygonUuid === polygonUuid); + + if (sitePolygon?.site?.projectId == null) { + resultMap.set(polygonUuid, null); + continue; + } + + const projectId = sitePolygon.site.projectId; + const allProjectPolygons = polygonsByProjectId.get(projectId) ?? []; + const relatedPolygonUuids = allProjectPolygons.filter(uuid => uuid !== polygonUuid); + + resultMap.set(polygonUuid, { projectId, relatedPolygonUuids }); + } + + return resultMap; + } + private async checkIntersections( targetUuids: string[], candidateUuids: string[] @@ -128,10 +279,6 @@ export class OverlappingValidator implements Validator { const bboxTargets = [...new Set(bboxFilteredResults.map(r => r.targetUuid))]; const bboxCandidates = [...new Set(bboxFilteredResults.map(r => r.candidateUuid))]; - // as the area in this validator is only for information purposes - // Using fixed 35° latitude for area conversion to avoid performance impact of ST_Centroid() - // 35° provides balance for most planting regions - const intersectionResults = await PolygonGeometry.checkGeometryIntersections( bboxTargets, bboxCandidates, diff --git a/apps/research-service/src/validations/validators/spikes.validator.ts b/apps/research-service/src/validations/validators/spikes.validator.ts index e19d82fc..43e20a40 100644 --- a/apps/research-service/src/validations/validators/spikes.validator.ts +++ b/apps/research-service/src/validations/validators/spikes.validator.ts @@ -22,6 +22,10 @@ interface SpikeDetectionResult extends ValidationResult { } export class SpikesValidator implements Validator { + private readonly SPIKE_ANGLE_THRESHOLD = 10; + + private readonly SPIKE_RATIO_THRESHOLD = 5; + async validatePolygon(polygonUuid: string): Promise { const geoJson = await PolygonGeometry.getGeoJSONParsed(polygonUuid); if (geoJson == null) { @@ -71,33 +75,68 @@ export class SpikesValidator implements Validator { private detectSpikes(geometry: GeoJSONGeometry): number[][] { const spikes: number[][] = []; - if (geometry.type == "Polygon" || geometry.type == "MultiPolygon") { - const coordinates = geometry.type == "Polygon" ? geometry.coordinates[0] : geometry.coordinates[0][0]; + if (geometry.type === "Polygon") { + for (const ring of geometry.coordinates) { + spikes.push(...this.detectSpikesInRing(ring)); + } + } else if (geometry.type === "MultiPolygon") { + for (const polygon of geometry.coordinates) { + for (const ring of polygon) { + spikes.push(...this.detectSpikesInRing(ring)); + } + } + } + + return spikes; + } - const numVertices = coordinates.length; - let totalDistance = 0; + private detectSpikesInRing(coordinates: number[][]): number[][] { + const spikes: number[][] = []; + const n = coordinates.length; - for (let i = 0; i < numVertices - 1; i++) { - totalDistance += this.calculateDistance(coordinates[i], coordinates[i + 1]); - } + if (n < 4) return spikes; - for (let i = 0; i < numVertices - 1; i++) { - const distance1 = this.calculateDistance(coordinates[i], coordinates[(i + 1) % numVertices]); - const distance2 = this.calculateDistance( - coordinates[(i + 1) % numVertices], - coordinates[(i + 2) % numVertices] - ); - const combinedDistance = distance1 + distance2; + for (let i = 1; i < n - 1; i++) { + const prev = coordinates[i - 1]; + const current = coordinates[i]; + const next = coordinates[i + 1]; - if (combinedDistance > 0.6 * totalDistance) { - spikes.push(coordinates[(i + 1) % numVertices]); - } + const angle = this.calculateAngle(prev, current, next); + + const d1 = this.calculateDistance(prev, current); + const d2 = this.calculateDistance(current, next); + const baseDistance = this.calculateDistance(prev, next); + + const isSharpAngle = angle < this.SPIKE_ANGLE_THRESHOLD; + const isSkinny = baseDistance > 0 && (d1 + d2) / baseDistance > this.SPIKE_RATIO_THRESHOLD; + + if (isSharpAngle && isSkinny) { + spikes.push(current); } } return spikes; } + private calculateAngle(p1: number[], p2: number[], p3: number[]): number { + const v1 = [p1[0] - p2[0], p1[1] - p2[1]]; + const v2 = [p3[0] - p2[0], p3[1] - p2[1]]; + + const dot = v1[0] * v2[0] + v1[1] * v2[1]; + + const mag1 = Math.sqrt(v1[0] * v1[0] + v1[1] * v1[1]); + const mag2 = Math.sqrt(v2[0] * v2[0] + v2[1] * v2[1]); + + if (mag1 === 0 || mag2 === 0) return 180; + + const cosAngle = dot / (mag1 * mag2); + const clampedCos = Math.max(-1, Math.min(1, cosAngle)); + const angleRad = Math.acos(clampedCos); + const angleDeg = angleRad * (180 / Math.PI); + + return angleDeg; + } + private calculateDistance(point1: number[], point2: number[]): number { const lat1 = point1[1]; const lon1 = point1[0]; @@ -108,7 +147,9 @@ export class SpikesValidator implements Validator { const dist = Math.sin(this.deg2rad(lat1)) * Math.sin(this.deg2rad(lat2)) + Math.cos(this.deg2rad(lat1)) * Math.cos(this.deg2rad(lat2)) * Math.cos(this.deg2rad(theta)); - const acosDist = Math.acos(dist); + + const clampedDist = Math.max(-1, Math.min(1, dist)); + const acosDist = Math.acos(clampedDist); const rad2degDist = this.rad2deg(acosDist); const miles = rad2degDist * 60 * 1.1515; diff --git a/apps/research-service/src/validations/validators/validator.interface.ts b/apps/research-service/src/validations/validators/validator.interface.ts index 3f97ab33..7aa4e599 100644 --- a/apps/research-service/src/validations/validators/validator.interface.ts +++ b/apps/research-service/src/validations/validators/validator.interface.ts @@ -11,4 +11,5 @@ export interface PolygonValidationResult { export interface Validator { validatePolygon(polygonUuid: string): Promise; + validatePolygons?(polygonUuids: string[]): Promise; } diff --git a/apps/research-service/src/validations/validators/within-country.validator.spec.ts b/apps/research-service/src/validations/validators/within-country.validator.spec.ts index 85ef9d8a..a9f273f3 100644 --- a/apps/research-service/src/validations/validators/within-country.validator.spec.ts +++ b/apps/research-service/src/validations/validators/within-country.validator.spec.ts @@ -247,4 +247,83 @@ describe("WithinCountryValidator", () => { }); }); }); + + describe("validatePolygons", () => { + it("should return empty array for empty input", async () => { + const result = await validator.validatePolygons([]); + expect(result).toEqual([]); + }); + + it("should validate multiple polygons in batch", async () => { + const testPolygonUuid2 = "e3349d63-83ed-4df8-996c-2b79555385fa"; + const mockTransactionInstance: MockTransaction = { + commit: jest.fn(), + rollback: jest.fn() + }; + + const mockResults = [ + { polygonUuid: testPolygonUuid, polygonArea: 1000, intersectionArea: 800, country: "Cambodia" }, + { polygonUuid: testPolygonUuid2, polygonArea: 1000, intersectionArea: 900, country: "Cambodia" } + ]; + + ( + PolygonGeometry as unknown as { checkWithinCountryIntersectionBatch: jest.Mock } + ).checkWithinCountryIntersectionBatch = jest.fn(); + mockTransaction.mockResolvedValueOnce(mockTransactionInstance); + (PolygonGeometry.checkWithinCountryIntersectionBatch as jest.Mock).mockResolvedValueOnce(mockResults); + + const result = await validator.validatePolygons([testPolygonUuid, testPolygonUuid2]); + + expect(result).toHaveLength(2); + expect(result[0].polygonUuid).toBe(testPolygonUuid); + expect(result[0].valid).toBe(true); + expect(result[1].polygonUuid).toBe(testPolygonUuid2); + expect(result[1].valid).toBe(true); + }); + + it("should handle polygon not found in batch", async () => { + const mockTransactionInstance: MockTransaction = { + commit: jest.fn(), + rollback: jest.fn() + }; + + ( + PolygonGeometry as unknown as { checkWithinCountryIntersectionBatch: jest.Mock } + ).checkWithinCountryIntersectionBatch = jest.fn(); + mockTransaction.mockResolvedValueOnce(mockTransactionInstance); + (PolygonGeometry.checkWithinCountryIntersectionBatch as jest.Mock).mockResolvedValueOnce([]); + + const result = await validator.validatePolygons([testPolygonUuid]); + + expect(result).toHaveLength(1); + expect(result[0].valid).toBe(false); + }); + + it("should throw InternalServerErrorException when sequelize is not available in batch", async () => { + const originalSequelize = PolygonGeometry.sequelize; + (PolygonGeometry as unknown as { sequelize: null }).sequelize = null; + + await expect(validator.validatePolygons([testPolygonUuid])).rejects.toThrow(InternalServerErrorException); + + (PolygonGeometry as unknown as { sequelize: typeof originalSequelize }).sequelize = originalSequelize; + }); + + it("should handle database errors gracefully in batch", async () => { + const mockTransactionInstance: MockTransaction = { + commit: jest.fn(), + rollback: jest.fn() + }; + + ( + PolygonGeometry as unknown as { checkWithinCountryIntersectionBatch: jest.Mock } + ).checkWithinCountryIntersectionBatch = jest.fn(); + mockTransaction.mockResolvedValueOnce(mockTransactionInstance); + (PolygonGeometry.checkWithinCountryIntersectionBatch as jest.Mock).mockRejectedValueOnce( + new Error("Database error") + ); + + await expect(validator.validatePolygons([testPolygonUuid])).rejects.toThrow("Database error"); + expect(mockTransactionInstance.rollback).toHaveBeenCalled(); + }); + }); }); diff --git a/apps/research-service/src/validations/validators/within-country.validator.ts b/apps/research-service/src/validations/validators/within-country.validator.ts index 1e14f143..a236b3d2 100644 --- a/apps/research-service/src/validations/validators/within-country.validator.ts +++ b/apps/research-service/src/validations/validators/within-country.validator.ts @@ -1,5 +1,5 @@ import { PolygonGeometry } from "@terramatch-microservices/database/entities"; -import { Validator, ValidationResult } from "./validator.interface"; +import { Validator, ValidationResult, PolygonValidationResult } from "./validator.interface"; import { NotFoundException, InternalServerErrorException } from "@nestjs/common"; import { Transaction } from "sequelize"; @@ -32,6 +32,39 @@ export class WithinCountryValidator implements Validator { }; } + async validatePolygons(polygonUuids: string[]): Promise { + if (polygonUuids.length === 0) { + return []; + } + + const results = await this.getIntersectionDataBatch(polygonUuids); + const resultMap = new Map(results.filter(r => r != null).map(r => [r.polygonUuid, r])); + + return polygonUuids.map(polygonUuid => { + const result = resultMap.get(polygonUuid); + + if (result == null) { + return { + polygonUuid, + valid: false, + extraInfo: { error: "Polygon not found or has no associated project" } + }; + } + + const insidePercentage = Math.round((result.intersectionArea / result.polygonArea) * 100 * 100) / 100; + const valid = insidePercentage >= this.THRESHOLD_PERCENTAGE; + + return { + polygonUuid, + valid, + extraInfo: { + insidePercentage, + countryName: result.country + } + }; + }); + } + private async getIntersectionData(polygonUuid: string): Promise<{ polygonArea: number; intersectionArea: number; @@ -60,4 +93,36 @@ export class WithinCountryValidator implements Validator { } } } + + private async getIntersectionDataBatch(polygonUuids: string[]): Promise< + { + polygonUuid: string; + polygonArea: number; + intersectionArea: number; + country: string; + }[] + > { + if (PolygonGeometry.sequelize == null) { + throw new InternalServerErrorException("PolygonGeometry model is missing sequelize connection"); + } + + const transaction = await PolygonGeometry.sequelize.transaction({ + isolationLevel: Transaction.ISOLATION_LEVELS.READ_COMMITTED + }); + + let shouldCommit = true; + + try { + const results = await PolygonGeometry.checkWithinCountryIntersectionBatch(polygonUuids, transaction); + return results; + } catch (error) { + shouldCommit = false; + await transaction.rollback(); + throw error; + } finally { + if (shouldCommit) { + await transaction.commit(); + } + } + } } diff --git a/libs/database/src/lib/constants/validation-types.ts b/libs/database/src/lib/constants/validation-types.ts index e7a94f88..a89c8159 100644 --- a/libs/database/src/lib/constants/validation-types.ts +++ b/libs/database/src/lib/constants/validation-types.ts @@ -23,4 +23,10 @@ export const VALIDATION_CRITERIA_IDS = { PLANT_START_DATE: 15 } as const; +export const EXCLUDED_VALIDATION_CRITERIA = [ + VALIDATION_CRITERIA_IDS.WITHIN_COUNTRY, + VALIDATION_CRITERIA_IDS.ESTIMATED_AREA, + VALIDATION_CRITERIA_IDS.PLANT_START_DATE +]; + export type CriteriaId = (typeof VALIDATION_CRITERIA_IDS)[ValidationType]; diff --git a/libs/database/src/lib/entities/delayed-job.entity.ts b/libs/database/src/lib/entities/delayed-job.entity.ts index e5006078..1fb07079 100644 --- a/libs/database/src/lib/entities/delayed-job.entity.ts +++ b/libs/database/src/lib/entities/delayed-job.entity.ts @@ -17,6 +17,8 @@ import { JsonColumn } from "../decorators/json-column.decorator"; // references. interface Metadata { entity_name?: string; + entity_type?: string; + entity_id?: number; } @Table({ tableName: "delayed_jobs", underscored: true }) export class DelayedJob extends Model { diff --git a/libs/database/src/lib/entities/polygon-geometry.entity.ts b/libs/database/src/lib/entities/polygon-geometry.entity.ts index dc45a653..1faa4b30 100644 --- a/libs/database/src/lib/entities/polygon-geometry.entity.ts +++ b/libs/database/src/lib/entities/polygon-geometry.entity.ts @@ -229,6 +229,53 @@ export class PolygonGeometry extends Model { return results.length > 0 ? results[0] : null; } + static async checkWithinCountryIntersectionBatch( + polygonUuids: string[], + transaction?: Transaction + ): Promise< + { + polygonUuid: string; + polygonArea: number; + intersectionArea: number; + country: string; + }[] + > { + if (this.sequelize == null) { + throw new InternalServerErrorException("PolygonGeometry model is missing sequelize connection"); + } + + if (polygonUuids.length === 0) { + return []; + } + + return (await this.sequelize.query( + ` + SELECT + pg.uuid as polygonUuid, + ST_Area(pg.geom) as "polygonArea", + ST_Area(ST_Intersection(pg.geom, wcg.geometry)) as "intersectionArea", + wcg.country + FROM polygon_geometry pg + JOIN site_polygon sp ON sp.poly_id = pg.uuid AND sp.is_active = 1 + JOIN v2_sites s ON s.uuid = sp.site_id + JOIN v2_projects p ON p.id = s.project_id + JOIN world_countries_generalized wcg ON wcg.iso = p.country + WHERE pg.uuid IN (:polygonUuids) + AND ST_Area(pg.geom) > 0 + `, + { + replacements: { polygonUuids }, + type: QueryTypes.SELECT, + transaction + } + )) as { + polygonUuid: string; + polygonArea: number; + intersectionArea: number; + country: string; + }[]; + } + @PrimaryKey @AutoIncrement @Column(BIGINT.UNSIGNED)