|
| 1 | +import workerpool from "workerpool"; |
| 2 | +import os from "node:os"; |
| 3 | +import {fileURLToPath} from "node:url"; |
| 4 | +import {getLogger} from "@ui5/logger"; |
| 5 | +import {serializeData, deserializeData, serializeResources, FsMainThreadInterface} from "./threadUtils.js"; |
| 6 | +import {setTimeout as setTimeoutPromise} from "node:timers/promises"; |
| 7 | + |
| 8 | +const MIN_WORKERS = 2; |
| 9 | +const MAX_WORKERS = 4; |
| 10 | +const osCpus = os.cpus().length || 1; |
| 11 | +const maxWorkers = Math.max(Math.min(osCpus - 1, MAX_WORKERS), MIN_WORKERS); |
| 12 | + |
| 13 | +export default class WorkDispatcher { |
| 14 | + #log = getLogger("build:helpers:WorkDispatcher"); |
| 15 | + #activeBuilds = new Set(); |
| 16 | + #pool; |
| 17 | + static #ensureSingleton = false; |
| 18 | + static #instance; |
| 19 | + |
| 20 | + #getPool() { |
| 21 | + if (!this.#pool) { |
| 22 | + this.#log.verbose( |
| 23 | + `Creating workerpool with up to ${maxWorkers} workers (available CPU cores: ${osCpus})` |
| 24 | + ); |
| 25 | + const workerPath = fileURLToPath( |
| 26 | + new URL("./threadRunner.js", import.meta.url) |
| 27 | + ); |
| 28 | + this.#pool = workerpool.pool(workerPath, { |
| 29 | + workerType: "auto", |
| 30 | + maxWorkers, |
| 31 | + }); |
| 32 | + } |
| 33 | + return this.#pool; |
| 34 | + } |
| 35 | + |
| 36 | + constructor() { |
| 37 | + if (!WorkDispatcher.#ensureSingleton) { |
| 38 | + throw new Error( |
| 39 | + "WorkDispatcher is a singleton class. Use WorkDispatcher.getInstance()" |
| 40 | + ); |
| 41 | + } |
| 42 | + } |
| 43 | + |
| 44 | + static getInstance(buildRef) { |
| 45 | + if (!buildRef) { |
| 46 | + throw new Error(`A reference to the calling instance must be provided`); |
| 47 | + } |
| 48 | + if (!WorkDispatcher.#instance) { |
| 49 | + WorkDispatcher.#ensureSingleton = true; |
| 50 | + WorkDispatcher.#instance = new WorkDispatcher(); |
| 51 | + WorkDispatcher.#ensureSingleton = false; |
| 52 | + } |
| 53 | + |
| 54 | + WorkDispatcher.#instance.#registerActiveBuild(buildRef); |
| 55 | + |
| 56 | + return WorkDispatcher.#instance; |
| 57 | + } |
| 58 | + |
| 59 | + getProcessor(modulePath) { |
| 60 | + return { |
| 61 | + execute: async ({resources, options, reader}) => { |
| 62 | + const buildUpArgs = {modulePath, args: {options: await serializeData(options)}}; |
| 63 | + let toTransfer; |
| 64 | + let threadMessageHandler; |
| 65 | + let fsInterfaceMainPort; |
| 66 | + |
| 67 | + if (reader) { |
| 68 | + const {port1, port2} = new MessageChannel(); |
| 69 | + fsInterfaceMainPort = port1; |
| 70 | + buildUpArgs.args.fsInterfaceComPort = port2; |
| 71 | + toTransfer = {transfer: [port2]}; |
| 72 | + |
| 73 | + threadMessageHandler = new FsMainThreadInterface(reader); |
| 74 | + threadMessageHandler.startCommunication(fsInterfaceMainPort); |
| 75 | + } |
| 76 | + |
| 77 | + if (resources) { |
| 78 | + buildUpArgs.args.resources = await serializeResources(resources); |
| 79 | + } |
| 80 | + |
| 81 | + const result = await this.#getPool().exec("execProcessor", [buildUpArgs], toTransfer); |
| 82 | + |
| 83 | + if (reader) { |
| 84 | + threadMessageHandler.endCommunication(fsInterfaceMainPort); |
| 85 | + } |
| 86 | + |
| 87 | + return deserializeData(result); |
| 88 | + } |
| 89 | + }; |
| 90 | + } |
| 91 | + |
| 92 | + async cleanup(buildRef, force) { |
| 93 | + const attemptPoolTermination = async () => { |
| 94 | + if (this.#activeBuilds.size && !force) { |
| 95 | + this.#log.verbose( |
| 96 | + `Pool termination canceled. There are still ${this.#activeBuilds.size} active builds` |
| 97 | + ); |
| 98 | + return; |
| 99 | + } |
| 100 | + |
| 101 | + this.#log.verbose(`Attempting to terminate the workerpool...`); |
| 102 | + |
| 103 | + if (!this.#pool) { |
| 104 | + this.#log.verbose( |
| 105 | + "Pool termination requested, but a pool has not been initialized or has already been terminated." |
| 106 | + ); |
| 107 | + return; |
| 108 | + } |
| 109 | + |
| 110 | + // There are many stats that could be used, but these ones seem the most |
| 111 | + // convenient. When all the (available) workers are idle, then it's safe to terminate. |
| 112 | + // There are many stats that could be used, but these ones seem the most |
| 113 | + // convenient. When all the (available) workers are idle, then it's safe to terminate. |
| 114 | + let {idleWorkers, totalWorkers} = this.#pool.stats(); |
| 115 | + while (idleWorkers !== totalWorkers && !force) { |
| 116 | + await setTimeoutPromise(100); // Wait a bit workers to finish and try again |
| 117 | + ({idleWorkers, totalWorkers} = this.#pool.stats()); |
| 118 | + } |
| 119 | + |
| 120 | + return await this.terminateTasks(force); |
| 121 | + }; |
| 122 | + |
| 123 | + if (!buildRef) { |
| 124 | + throw new Error(`A reference to the calling instance must be provided`); |
| 125 | + } |
| 126 | + if (!this.#activeBuilds.has(buildRef)) { |
| 127 | + throw new Error(`The provided build reference is unknown`); |
| 128 | + } |
| 129 | + this.#activeBuilds.delete(buildRef); |
| 130 | + |
| 131 | + return await attemptPoolTermination(); |
| 132 | + } |
| 133 | + |
| 134 | + async terminateTasks(force) { |
| 135 | + if (!this.#pool) { |
| 136 | + this.#log.verbose( |
| 137 | + "Pool termination requested, but a pool has not been initialized or has already been terminated"); |
| 138 | + return; |
| 139 | + } |
| 140 | + |
| 141 | + this.#activeBuilds = []; |
| 142 | + const pool = this.#pool; |
| 143 | + this.#pool = null; |
| 144 | + return await pool.terminate(force); |
| 145 | + } |
| 146 | + |
| 147 | + #registerActiveBuild(instanceRef) { |
| 148 | + if (this.#activeBuilds.has(instanceRef)) { |
| 149 | + throw new Error(`Build already registered in Work Dispatcher. This should never happen`); |
| 150 | + } |
| 151 | + this.#activeBuilds.add(instanceRef); |
| 152 | + } |
| 153 | +} |
0 commit comments