diff --git a/jobConfig.example.yaml b/jobConfig.example.yaml index 652cef1f9..73e5b7ec5 100644 --- a/jobConfig.example.yaml +++ b/jobConfig.example.yaml @@ -82,14 +82,29 @@ jobs: request: jobParams.subject: type: string - - # Send an email. Requires email to be configured + - &email + # Send an email. Requires email to be configured actionType: email + ignoreErrors: true to: "{{{ job.contactEmail }}}" - subject: "[SciCat] {{ job.jobParams.subject }}" - bodyTemplateFile: demo_email.html + subject: |- + [SciCatLive] {{ default job.jobParams.subject "Job has been updated"}} + bodyTemplateFile: src/common/email-templates/job-template-simplified.html update: auth: "#jobOwnerUser" - actions: [] + actions: + - # Only send emails for updates to status 'finished*' + actionType: switch + phase: perform + property: job.statusCode + cases: + - regex: "/^finished/" + actions: + - *email + - actions: + - actionType: log + perform: "(Job {{job.id}} Skip email notification for status {{job.statusCode}}" + - jobType: url_demo create: @@ -97,6 +112,7 @@ jobs: actions: - # Call a REST endpoint actionType: url + ignoreErrors: true url: http://localhost:3000/api/v3/health?jobid={{ job.id }} method: GET headers: diff --git a/src/app.module.ts b/src/app.module.ts index 45ceb11e1..03beaac1e 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -23,14 +23,7 @@ import { InstrumentsModule } from "./instruments/instruments.module"; import { MailerModule } from "@nestjs-modules/mailer"; import { join } from "path"; import { HandlebarsAdapter } from "@nestjs-modules/mailer/dist/adapters/handlebars.adapter"; -import { - formatCamelCase, - unwrapJSON, - jsonify, - job_v3, - urlencode, - base64enc, -} from "./common/handlebars-helpers"; +import { handlebarsHelpers } from "./common/handlebars-helpers"; import { CommonModule } from "./common/common.module"; import { RabbitMQModule } from "./common/rabbitmq/rabbitmq.module"; import { EventEmitterModule } from "@nestjs/event-emitter"; @@ -131,15 +124,7 @@ import { MaskSensitiveDataInterceptorModule } from "./common/interceptors/mask-s }, template: { dir: join(__dirname, "./common/email-templates"), - adapter: new HandlebarsAdapter({ - unwrapJSON: unwrapJSON, - keyToWord: formatCamelCase, - eq: (a, b) => a === b, - jsonify: jsonify, - job_v3: job_v3, - urlencode: urlencode, - base64enc: base64enc, - }), + adapter: new HandlebarsAdapter(handlebarsHelpers), options: { strict: true, }, diff --git a/src/common/email-templates/job-template.spec.html b/src/common/email-templates/test-minimal-template.spec.html similarity index 50% rename from src/common/email-templates/job-template.spec.html rename to src/common/email-templates/test-minimal-template.spec.html index 5f795d7e5..4872e2fa4 100644 --- a/src/common/email-templates/job-template.spec.html +++ b/src/common/email-templates/test-minimal-template.spec.html @@ -1 +1,2 @@ +{{!-- This is a minimal template used for testing email templating. --}} Your {{job.type}} job with ID {{job.id}} has been completed successfully. \ No newline at end of file diff --git a/src/common/handlebars-helpers.ts b/src/common/handlebars-helpers.ts index 57a5c49b5..617f0cfcd 100644 --- a/src/common/handlebars-helpers.ts +++ b/src/common/handlebars-helpers.ts @@ -136,6 +136,8 @@ export const handlebarsHelpers = { unwrapJSON: unwrapJSON, keyToWord: formatCamelCase, eq: (a: unknown, b: unknown) => a === b, + matches: (query: string, regex: string) => RegExp(regex).test(query), + default: (query: unknown, def: unknown) => query || def, jsonify: jsonify, job_v3: job_v3, urlencode: urlencode, diff --git a/src/config/job-config/actions/emailaction/__snapshots__/emailaction.spec.ts.snap b/src/config/job-config/actions/emailaction/__snapshots__/emailaction.spec.ts.snap new file mode 100644 index 000000000..9f14feecf --- /dev/null +++ b/src/config/job-config/actions/emailaction/__snapshots__/emailaction.spec.ts.snap @@ -0,0 +1,84 @@ +// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing + +exports[`Email Template job-template-simplified.html should render correctly 1`] = ` +" + + + + +
+
+ Your archive job has been submitted. It currently has status "jobStarted". +
+
+ Job id: jobId123 +
+

The job will be perfomed on the following dataset(s):

+ + + + +
+ testPid +
+ +
+ This email was automatically generated by + SciCat. + Please do not reply. +
+
+ +" +`; diff --git a/src/config/job-config/actions/emailaction/emailaction.interface.ts b/src/config/job-config/actions/emailaction/emailaction.interface.ts index e9996ac32..b51205514 100644 --- a/src/config/job-config/actions/emailaction/emailaction.interface.ts +++ b/src/config/job-config/actions/emailaction/emailaction.interface.ts @@ -8,6 +8,7 @@ export interface EmailJobActionOptions extends JobActionOptions { from?: string; subject: string; bodyTemplateFile: string; + ignoreErrors?: boolean; } /** @@ -26,6 +27,7 @@ export function isEmailJobActionOptions( typeof opts.to === "string" && (opts.from === undefined || typeof opts.from === "string") && typeof opts.subject === "string" && - typeof opts.bodyTemplateFile === "string" + typeof opts.bodyTemplateFile === "string" && + (opts.ignoreErrors === undefined || typeof opts.ignoreErrors === "boolean") ); } diff --git a/src/config/job-config/actions/emailaction/emailaction.spec.ts b/src/config/job-config/actions/emailaction/emailaction.spec.ts index 57553855b..e4fd3143d 100644 --- a/src/config/job-config/actions/emailaction/emailaction.spec.ts +++ b/src/config/job-config/actions/emailaction/emailaction.spec.ts @@ -3,17 +3,112 @@ import { EmailJobActionOptions } from "./emailaction.interface"; import { JobClass } from "../../../../jobs/schemas/job.schema"; import { MailService } from "src/common/mail.service"; import { MailerService } from "@nestjs-modules/mailer"; +import { DatasetClass } from "src/datasets/schemas/dataset.schema"; +import { CreateJobDto } from "src/jobs/dto/create-job.dto"; jest.mock("src/common/mail.service"); jest.mock("@nestjs-modules/mailer"); +const mockDataset: DatasetClass = { + _id: "testId", + pid: "testPid", + owner: "testOwner", + ownerEmail: "testOwner@email.com", + instrumentIds: ["testInstrumentId"], + orcidOfOwner: "https://0000.0000.0000.0001", + contactEmail: "testContact@email.com", + sourceFolder: "/nfs/groups/beamlines/test/123456", + sourceFolderHost: "https://fileserver.site.com", + size: 1000000, + packedSize: 1000000, + numberOfFiles: 1, + numberOfFilesArchived: 1, + creationTime: new Date("2021-11-11T12:29:02.083Z"), + type: "raw", + validationStatus: "string", + keywords: [], + description: "Test dataset.", + datasetName: "Test Dataset", + classification: "string", + license: "string", + version: "string", + isPublished: false, + datasetlifecycle: { + id: "testId", + archivable: true, + retrievable: false, + publishable: true, + dateOfDiskPurging: new Date("2031-11-11T12:29:02.083Z"), + archiveRetentionTime: new Date("2031-11-11T12:29:02.083Z"), + dateOfPublishing: new Date("2024-11-11T12:29:02.083Z"), + publishedOn: new Date("2024-11-11T12:29:02.083Z"), + isOnCentralDisk: true, + archiveReturnMessage: {}, + retrieveReturnMessage: {}, + archiveStatusMessage: "string", + retrieveStatusMessage: "string", + exportedTo: "string", + retrieveIntegrityCheck: false, + }, + createdAt: new Date("2021-11-11T12:29:02.083Z"), + updatedAt: new Date("2021-11-11T12:29:02.083Z"), + endTime: new Date("2021-12-11T12:29:02.083Z"), + creationLocation: "test", + dataFormat: "Test Format", + scientificMetadata: {}, + proposalIds: ["ABCDEF"], + sampleIds: ["testSampleId"], + accessGroups: [], + createdBy: "test user", + ownerGroup: "test", + updatedBy: "test", + instrumentGroup: "test", +}; + +const mockJob: JobClass = { + id: "jobId123", + _id: "jobId123", + type: "archive", + statusCode: "jobStarted", + statusMessage: "Job started", + jobParams: { + datasetList: [ + { + pid: mockDataset.pid, + files: [], + }, + ], + }, + jobResultObject: {}, + ownerUser: "admin", + ownerGroup: "admin", + configVersion: "1.0", + createdBy: "admin", + updatedBy: "admin", + createdAt: new Date("2023-10-01T10:00:00Z"), + updatedAt: new Date("2023-10-01T10:00:00Z"), + accessGroups: [], + isPublished: false, +}; + +function jobToCreateDto(job: JobClass): CreateJobDto { + return { + type: job.type, + jobParams: job.jobParams, + ownerUser: job.ownerUser, + ownerGroup: job.ownerGroup, + contactEmail: job.contactEmail, + }; +} + describe("EmailJobAction", () => { const config: EmailJobActionOptions = { actionType: "email", to: "recipient@example.com", from: "sender@example.com", subject: "Job {{job.id}} completed", - bodyTemplateFile: "src/common/email-templates/job-template.spec.html", + bodyTemplateFile: + "src/common/email-templates/test-minimal-template.spec.html", }; const mailService = { @@ -36,7 +131,7 @@ describe("EmailJobAction", () => { type: "testemail", } as JobClass; - const context = { request: job, job, env: {} }; + const context = { request: job, job, env: {}, datasets: [] }; await action.perform(context); expect(mailService.sendMail).toHaveBeenCalledWith({ @@ -57,11 +152,37 @@ describe("EmailJobAction", () => { new Error("Email sending failed"), ); - const context = { request: job, job, env: {} }; + const context = { request: job, job, env: {}, datasets: [] }; await expect(action.perform(context)).rejects.toThrow( "Email sending failed", ); }); + + it("should ignore errors if the ignoreError is set", async () => { + const job = { + id: "12345", + type: "testemail", + } as JobClass; + + (mailService.sendMail as jest.Mock).mockRejectedValue( + new Error("Email sending failed"), + ); + + const actionIgnore = new EmailJobAction(mailService, { + ...config, + ignoreErrors: true, + }); + + const context = { request: job, job, env: {}, datasets: [] }; + await expect(actionIgnore.perform(context)).resolves.toBeUndefined(); + + expect(mailService.sendMail).toHaveBeenCalledWith({ + to: "recipient@example.com", + from: "sender@example.com", + subject: "Job 12345 completed", + html: "Your testemail job with ID 12345 has been completed successfully.", + }); + }); }); describe("EmailJobAction with default sender", () => { @@ -69,7 +190,8 @@ describe("EmailJobAction with default sender", () => { actionType: "email", to: "recipient@example.com", subject: "Job {{ job.id }} completed", - bodyTemplateFile: "src/common/email-templates/job-template.spec.html", + bodyTemplateFile: + "src/common/email-templates/test-minimal-template.spec.html", }; const mailService = { @@ -106,7 +228,7 @@ describe("EmailJobAction with default sender", () => { return mailerService.sendMail(mailOptions); }); - const context = { request: job, job, env: {} }; + const context = { request: job, job, env: {}, datasets: [] }; await action.perform(context); expect(mailerService.sendMail).toHaveBeenCalledWith({ @@ -117,3 +239,37 @@ describe("EmailJobAction with default sender", () => { }); }); }); + +describe("Email Template", () => { + const mailService = { + sendMail: jest.fn(), + } as unknown as MailService; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("job-template-simplified.html should render correctly", async () => { + const config: EmailJobActionOptions = { + actionType: "email", + to: "recipient@example.com", + from: "sender@example.com", + subject: "Job {{ job.id }}", + bodyTemplateFile: + "src/common/email-templates/job-template-simplified.html", + }; + const action = new EmailJobAction(mailService, config); + + const context = { + request: jobToCreateDto(mockJob), + job: mockJob, + env: {}, + datasets: [mockDataset], + }; + await action.perform(context); + + expect(mailService.sendMail).toHaveBeenCalled(); + const mail = (mailService.sendMail as jest.Mock).mock.calls.at(-1)[0]; + expect(mail.html).toMatchSnapshot(); + }); +}); diff --git a/src/config/job-config/actions/emailaction/emailaction.ts b/src/config/job-config/actions/emailaction/emailaction.ts index 81143a9ba..11a77db7b 100644 --- a/src/config/job-config/actions/emailaction/emailaction.ts +++ b/src/config/job-config/actions/emailaction/emailaction.ts @@ -23,6 +23,7 @@ export class EmailJobAction implements JobAction { private from?: string = undefined; private subjectTemplate: TemplateJob; private bodyTemplate: TemplateJob; + private ignoreErrors = false; getActionType(): string { return actionType; @@ -45,6 +46,10 @@ export class EmailJobAction implements JobAction { "utf8", ); this.bodyTemplate = compileJobTemplate(templateFile); + + if (options["ignoreErrors"]) { + this.ignoreErrors = options.ignoreErrors; + } } async perform(context: JobPerformContext) { @@ -53,17 +58,39 @@ export class EmailJobAction implements JobAction { "EmailJobAction", ); - // Fill templates - const mail: ISendMailOptions = { - to: this.toTemplate(context), - subject: this.subjectTemplate(context), - html: this.bodyTemplate(context), - }; - if (this.from) { - mail.from = this.from; + let mail: ISendMailOptions; + try { + // Fill templates + mail = { + to: this.toTemplate(context), + subject: this.subjectTemplate(context), + html: this.bodyTemplate(context), + }; + if (this.from) { + mail.from = this.from; + } + } catch (err) { + Logger.error( + `(Job ${context.job.id}) EmailJobAction: Template error: ${err}`, + "EmailJobAction", + ); + if (!this.ignoreErrors) { + throw err; + } + return; } - // Send the email - await this.mailService.sendMail(mail); + try { + // Send the email + await this.mailService.sendMail(mail); + } catch (err) { + Logger.error( + `(Job ${context.job.id}) EmailJobAction: Sending email failed: ${err}`, + "EmailJobAction", + ); + if (!this.ignoreErrors) { + throw err; + } + } } } diff --git a/src/config/job-config/actions/switchaction/switchaction.ts b/src/config/job-config/actions/switchaction/switchaction.ts index 7927ee10c..cdcba9bf8 100644 --- a/src/config/job-config/actions/switchaction/switchaction.ts +++ b/src/config/job-config/actions/switchaction/switchaction.ts @@ -85,6 +85,7 @@ class RegexCase extends Case { constructor( options: { regex: string; actions: JobActionOptions[] }, creators: Record>, + private propertyName: string, ) { super(options, creators); this.regex = this.parseRegex(options.regex); @@ -102,7 +103,7 @@ class RegexCase extends Case { public matches(target: JSONData) { if (typeof target !== "string") { throw makeHttpException( - `Property ${target} was expected to be a string.`, + `Property ${this.propertyName} was expected to be a string. Got ${target}`, ); } // regex match @@ -164,7 +165,7 @@ export class SwitchJobAction implements JobAction { if ("schema" in caseOptions) { return new SchemaCase(caseOptions, creators, ajvDefined); } else if ("regex" in caseOptions) { - return new RegexCase(caseOptions, creators); + return new RegexCase(caseOptions, creators, this.property); } else if ("match" in caseOptions) { return new MatchCase(caseOptions, creators); } else { diff --git a/src/config/job-config/actions/urlaction/urlaction.spec.ts b/src/config/job-config/actions/urlaction/urlaction.spec.ts index ed8104248..63bff7a1d 100644 --- a/src/config/job-config/actions/urlaction/urlaction.spec.ts +++ b/src/config/job-config/actions/urlaction/urlaction.spec.ts @@ -81,7 +81,7 @@ describe("URLJobAction", () => { ); }); - it("should ignore errors if the ignoreError is set", async () => { + it("should ignore http errors if the ignoreError is set", async () => { const job = { id: "12345" } as JobClass; (global.fetch as jest.Mock).mockResolvedValue({ ok: false, @@ -109,4 +109,31 @@ describe("URLJobAction", () => { }, ); }); + + it("should ignore exceptions if the ignoreError is set", async () => { + const job = { id: "12345" } as JobClass; + (global.fetch as jest.Mock).mockImplementation(() => { + throw new Error("Network error"); + }); + + const actionIgnore = new URLJobAction({ + ...config, + ignoreErrors: true, + }); + + const context = { request: job, job, env: process.env, datasets: [] }; + await expect(actionIgnore.perform(context)).resolves.toBeUndefined(); + + expect(global.fetch).toHaveBeenCalledWith( + "http://localhost:3000/api/v3/health?jobid=12345", + { + method: "GET", + headers: { + accept: "application/json", + Authorization: "Bearer TheAuthToken", + }, + body: "This is the body.", + }, + ); + }); }); diff --git a/src/config/job-config/actions/urlaction/urlaction.ts b/src/config/job-config/actions/urlaction/urlaction.ts index d181c7c6e..c270da58e 100644 --- a/src/config/job-config/actions/urlaction/urlaction.ts +++ b/src/config/job-config/actions/urlaction/urlaction.ts @@ -25,20 +25,58 @@ export class URLJobAction implements JobAction { const url = encodeURI(this.urlTemplate(context)); Logger.log(`(Job ${context.job.id}) Requesting ${url}`, "URLAction"); - const response = await fetch(url, { - method: this.method, - headers: this.headerTemplates - ? Object.fromEntries( - Object.entries(this.headerTemplates).map(([key, template]) => [ - key, - template(context), - ]), - ) - : undefined, - body: this.bodyTemplate ? this.bodyTemplate(context) : undefined, - }); + let msg; + try { + msg = { + method: this.method, + headers: this.headerTemplates + ? Object.fromEntries( + Object.entries(this.headerTemplates).map(([key, template]) => [ + key, + template(context), + ]), + ) + : undefined, + body: this.bodyTemplate ? this.bodyTemplate(context) : undefined, + }; + } catch (err) { + Logger.error( + `(Job ${context.job.id}) Templating error generating request for ${url}: ${err}`, + "URLAction", + ); + if (!this.ignoreErrors) { + throw err; + } + return; + } + + let response; + try { + response = await fetch(url, msg); + } catch (err) { + Logger.error( + `(Job ${context.job.id}) Network error requesting ${url}: ${err}`, + "URLAction", + ); + if (!this.ignoreErrors) { + throw err; + } + return; + } + + let text = "undefined"; + try { + text = await response.text(); + } catch (err) { + Logger.error( + `(Job ${context.job.id}) Error reading response text from ${url}: ${err}`, + "URLAction", + ); + if (!this.ignoreErrors) { + throw err; + } + } - const text = await response.text(); if (response.ok) { Logger.log( `(Job ${context.job.id}) Request for ${url} returned ${response.status}. Response: ${text}`, diff --git a/src/config/job-config/actions/validateaction/validateaction.spec.ts b/src/config/job-config/actions/validateaction/validateaction.spec.ts index a326d2472..868cfec5c 100644 --- a/src/config/job-config/actions/validateaction/validateaction.spec.ts +++ b/src/config/job-config/actions/validateaction/validateaction.spec.ts @@ -7,6 +7,7 @@ import { import { DatasetsService } from "src/datasets/datasets.service"; import { Test, TestingModule } from "@nestjs/testing"; import { ValidateCreateJobActionCreator } from "./validateaction.service"; +import { DatasetClass } from "src/datasets/schemas/dataset.schema"; const createJobBase = { type: "validate", @@ -15,7 +16,7 @@ const createJobBase = { contactEmail: "email@example.com", }; -const mockDataset = { +const mockDataset: DatasetClass = { _id: "testId", pid: "testPid", owner: "testOwner", @@ -39,7 +40,6 @@ const mockDataset = { license: "string", version: "string", isPublished: false, - history: [], datasetlifecycle: { id: "testId", archivable: true, @@ -60,7 +60,6 @@ const mockDataset = { createdAt: new Date("2021-11-11T12:29:02.083Z"), updatedAt: new Date("2021-11-11T12:29:02.083Z"), techniques: [], - principalInvestigator: "testInvestigator", endTime: new Date("2021-12-11T12:29:02.083Z"), creationLocation: "test", dataFormat: "Test Format", @@ -70,15 +69,8 @@ const mockDataset = { accessGroups: [], createdBy: "test user", ownerGroup: "test", - relationships: [], - sharedWith: [], updatedBy: "test", instrumentGroup: "test", - inputDatasets: [], - usedSoftware: [], - jobParameters: {}, - jobLogData: "", - comment: "", dataQualityMetrics: 1, }; @@ -297,7 +289,7 @@ describe("ValidateAction", () => { const unarchivableDataset = { ...mockDataset, }; - unarchivableDataset.datasetlifecycle.archivable = false; + unarchivableDataset.datasetlifecycle!.archivable = false; findAll.mockResolvedValueOnce([unarchivableDataset]); const context = { request: dto, env: {}, datasets: [mockDataset] }; diff --git a/src/config/job-config/handlebar-utils.ts b/src/config/job-config/handlebar-utils.ts index 2aea4d276..cad0c6b8a 100644 --- a/src/config/job-config/handlebar-utils.ts +++ b/src/config/job-config/handlebar-utils.ts @@ -1,21 +1,20 @@ import * as hb from "handlebars"; import { JobTemplateContext } from "./jobconfig.interface"; - -const jobTemplateOptions = { - allowedProtoProperties: { - id: true, - type: true, - statusCode: true, - statusMessage: true, - createdBy: true, - jobParams: true, - contactEmail: true, - }, - allowProtoPropertiesByDefault: false, // limit accessible fields for security -}; +import { handlebarsHelpers } from "src/common/handlebars-helpers"; export type TemplateJob = hb.TemplateDelegate; +/** + * Register all standard handlebars helpers + * + * This is normally handled by app.module.ts, but may need to be called manually by + * tests that bypass module loading. + */ +export function registerHelpers() { + Object.entries(handlebarsHelpers).forEach(([name, impl]) => + hb.registerHelper(name, impl), + ); +} /** * Standardizes compilation of handlebars templates * @@ -31,7 +30,6 @@ export function compileJobTemplate( const template: TemplateJob = hb.compile(input, options); return (context: JobTemplateContext, options?: RuntimeOptions) => { return template(context, { - ...jobTemplateOptions, ...options, }); }; diff --git a/src/jobs/jobs.controller.utils.ts b/src/jobs/jobs.controller.utils.ts index 7bb29463e..ea254e99a 100644 --- a/src/jobs/jobs.controller.utils.ts +++ b/src/jobs/jobs.controller.utils.ts @@ -731,7 +731,10 @@ export class JobsControllerUtils { // Perform the action that is specified in the update portion of the job configuration if (updatedJob !== null) { await this.checkConfigVersion(jobConfig, updatedJob); - const performContext = { ...contextWithDatasets, job: updatedJob }; + const performContext = { + ...contextWithDatasets, + job: toObject(updatedJob) as JobClass, + }; await performActions(jobConfig.update.actions, performContext); } return updatedJob;