From 04ce22c3fc4647b796a1dab20e9d1e8dbe78c968 Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Thu, 22 Jun 2023 20:49:06 +0200 Subject: [PATCH 1/4] [POC] ProjectBuilder: Build projects in parallel if possible Based on the required dependencies information provided by the tasks, determine which projects need to be build in sequence and which can be build in parallel. This can lead to a significantly faster build in some cases, depending on the number of dependencies and which tasks need to be executed. --- lib/build/ProjectBuilder.js | 89 +++++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 29 deletions(-) diff --git a/lib/build/ProjectBuilder.js b/lib/build/ProjectBuilder.js index 63f10aa1c..7483f196d 100644 --- a/lib/build/ProjectBuilder.js +++ b/lib/build/ProjectBuilder.js @@ -170,7 +170,7 @@ class ProjectBuilder { `while including any dependencies into the build result`); } - const projectBuildContexts = await this._createRequiredBuildContexts(requestedProjects); + const {projectBuildContexts, buildDependencies} = await this._createRequiredBuildContexts(requestedProjects); const cleanupSigHooks = this._registerCleanupSigHooks(); const fsTarget = resourceFactory.createAdapter({ fsBasePath: destPath, @@ -179,30 +179,55 @@ class ProjectBuilder { const queue = []; const alreadyBuilt = []; + let currentQueueEntry; // Create build queue based on graph depth-first search to ensure correct build order - await this._graph.traverseDepthFirst(async ({project}) => { + await this._graph.traverseDepthFirst(({project}) => { const projectName = project.getName(); const projectBuildContext = projectBuildContexts.get(projectName); if (projectBuildContext) { // Build context exists // => This project needs to be built or, in case it has already // been built, it's build result needs to be written out (if requested) - queue.push(projectBuildContext); + if (!currentQueueEntry) { + // Create new entry + currentQueueEntry = new Map(); + currentQueueEntry.set(projectName, projectBuildContext); + queue.push(currentQueueEntry); + } else { + // Check current entry and append project if no dependency exists + const buildDeps = buildDependencies.get(projectName); + const entryContainsDep = Array.from(currentQueueEntry.keys()).some((depName) => { + return buildDeps.has(depName); + }); + if (entryContainsDep) { + // Create a new entry + currentQueueEntry = new Map(); + currentQueueEntry.set(projectName, projectBuildContext); + queue.push(currentQueueEntry); + } else { + // Append project to current entry + currentQueueEntry.set(projectName, projectBuildContext); + } + } if (!projectBuildContext.requiresBuild()) { alreadyBuilt.push(projectName); } } }); - log.setProjects(queue.map((projectBuildContext) => { - return projectBuildContext.getProject().getName(); - })); + const projectNames = []; + for (const queueEntry of queue) { + for (const projectBuildContext of queueEntry.values()) { + projectNames.push(projectBuildContext.getProject().getName()); + } + } + log.setProjects(projectNames); if (queue.length > 1) { // Do not log if only the root project is being built - log.info(`Processing ${queue.length} projects`); + log.info(`Processing ${projectNames.length} projects`); if (alreadyBuilt.length) { log.info(` Reusing build results of ${alreadyBuilt.length} projects`); - log.info(` Building ${queue.length - alreadyBuilt.length} projects`); + log.info(` Building ${projectNames.length - alreadyBuilt.length} projects`); } if (log.isLevelEnabled("verbose")) { @@ -231,27 +256,31 @@ class ProjectBuilder { const startTime = process.hrtime(); try { const pWrites = []; - for (const projectBuildContext of queue) { - const projectName = projectBuildContext.getProject().getName(); - const projectType = projectBuildContext.getProject().getType(); - log.verbose(`Processing project ${projectName}...`); - - // Only build projects that are not already build (i.e. provide a matching build manifest) - if (alreadyBuilt.includes(projectName)) { - log.skipProjectBuild(projectName, projectType); - } else { - log.startProjectBuild(projectName, projectType); - await projectBuildContext.getTaskRunner().runTasks(); - log.endProjectBuild(projectName, projectType); - } - if (!requestedProjects.includes(projectName)) { - // Project has not been requested - // => Its resources shall not be part of the build result - continue; - } + for (const queueEntry of queue) { + const projectBuildContexts = Array.from(queueEntry.values()); + log.info(`Building ${projectBuildContexts.length} projects in parallel:`); + await Promise.all(projectBuildContexts.map(async (projectBuildContext) => { + const projectName = projectBuildContext.getProject().getName(); + const projectType = projectBuildContext.getProject().getType(); + log.verbose(`Processing project ${projectName}...`); + + // Only build projects that are not already build (i.e. provide a matching build manifest) + if (alreadyBuilt.includes(projectName)) { + log.skipProjectBuild(projectName, projectType); + } else { + log.startProjectBuild(projectName, projectType); + await projectBuildContext.getTaskRunner().runTasks(); + log.endProjectBuild(projectName, projectType); + } + if (!requestedProjects.includes(projectName)) { + // Project has not been requested + // => Its resources shall not be part of the build result + return; + } - log.verbose(`Writing out files...`); - pWrites.push(this._writeResults(projectBuildContext, fsTarget)); + log.verbose(`Writing out files...`); + pWrites.push(this._writeResults(projectBuildContext, fsTarget)); + })); } await Promise.all(pWrites); log.info(`Build succeeded in ${this._getElapsedTime(startTime)}`); @@ -270,6 +299,7 @@ class ProjectBuilder { })); const projectBuildContexts = new Map(); + const buildDependencies = new Map(); for (const projectName of requiredProjects) { log.verbose(`Creating build context for project ${projectName}...`); @@ -282,6 +312,7 @@ class ProjectBuilder { if (projectBuildContext.requiresBuild()) { const taskRunner = projectBuildContext.getTaskRunner(); const requiredDependencies = await taskRunner.getRequiredDependencies(); + buildDependencies.set(projectName, requiredDependencies); if (requiredDependencies.size === 0) { continue; @@ -302,7 +333,7 @@ class ProjectBuilder { } } - return projectBuildContexts; + return {projectBuildContexts, buildDependencies}; } async _getProjectFilter({ From b2e9c6739e69bf030f20b6289fe7ae2202c9276e Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Thu, 22 Jun 2023 21:29:20 +0200 Subject: [PATCH 2/4] Optimize determination for parallel processing slightly --- lib/build/ProjectBuilder.js | 93 +++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 51 deletions(-) diff --git a/lib/build/ProjectBuilder.js b/lib/build/ProjectBuilder.js index 7483f196d..b3749649d 100644 --- a/lib/build/ProjectBuilder.js +++ b/lib/build/ProjectBuilder.js @@ -170,64 +170,52 @@ class ProjectBuilder { `while including any dependencies into the build result`); } - const {projectBuildContexts, buildDependencies} = await this._createRequiredBuildContexts(requestedProjects); + const buildContexts = await this._createRequiredBuildContexts(requestedProjects); const cleanupSigHooks = this._registerCleanupSigHooks(); const fsTarget = resourceFactory.createAdapter({ fsBasePath: destPath, virBasePath: "/" }); - const queue = []; + const queue = []; // A queue of batches to process const alreadyBuilt = []; - let currentQueueEntry; - - // Create build queue based on graph depth-first search to ensure correct build order - await this._graph.traverseDepthFirst(({project}) => { - const projectName = project.getName(); - const projectBuildContext = projectBuildContexts.get(projectName); - if (projectBuildContext) { - // Build context exists - // => This project needs to be built or, in case it has already - // been built, it's build result needs to be written out (if requested) - if (!currentQueueEntry) { - // Create new entry - currentQueueEntry = new Map(); - currentQueueEntry.set(projectName, projectBuildContext); - queue.push(currentQueueEntry); - } else { - // Check current entry and append project if no dependency exists - const buildDeps = buildDependencies.get(projectName); - const entryContainsDep = Array.from(currentQueueEntry.keys()).some((depName) => { - return buildDeps.has(depName); - }); - if (entryContainsDep) { - // Create a new entry - currentQueueEntry = new Map(); - currentQueueEntry.set(projectName, projectBuildContext); - queue.push(currentQueueEntry); - } else { - // Append project to current entry - currentQueueEntry.set(projectName, projectBuildContext); - } + + function determineBatchNumber(depName, minBatchNumber) { + for (let i = queue.length - 1; i >= minBatchNumber; i--) { + if (queue[i].has(depName)) { + return i + 1; } - if (!projectBuildContext.requiresBuild()) { - alreadyBuilt.push(projectName); + } + return minBatchNumber; + } + + buildContexts.forEach(({projectBuildContext, requiredDependencies}, projectName) => { + // => This project needs to be built or, in case it has already + // been built, it's build result needs to be written out (if requested) + if (!projectBuildContext.requiresBuild()) { + alreadyBuilt.push(projectName); + } + let minBatchNumber = 0; + if (queue.length) { + for (const depName of requiredDependencies) { + minBatchNumber = determineBatchNumber(depName, minBatchNumber); } } - }); - const projectNames = []; - for (const queueEntry of queue) { - for (const projectBuildContext of queueEntry.values()) { - projectNames.push(projectBuildContext.getProject().getName()); + console.log(`${projectName}: Queued at batch #${minBatchNumber}`); + if (minBatchNumber === queue.length) { + // Add new batch to queue + queue.push(new Map()); } - } - log.setProjects(projectNames); + queue[minBatchNumber].set(projectName, projectBuildContext); + }); + + log.setProjects(Array.from(buildContexts.keys())); if (queue.length > 1) { // Do not log if only the root project is being built - log.info(`Processing ${projectNames.length} projects`); + log.info(`Processing ${buildContexts.size} projects`); if (alreadyBuilt.length) { log.info(` Reusing build results of ${alreadyBuilt.length} projects`); - log.info(` Building ${projectNames.length - alreadyBuilt.length} projects`); + log.info(` Building ${buildContexts.size - alreadyBuilt.length} projects`); } if (log.isLevelEnabled("verbose")) { @@ -256,8 +244,8 @@ class ProjectBuilder { const startTime = process.hrtime(); try { const pWrites = []; - for (const queueEntry of queue) { - const projectBuildContexts = Array.from(queueEntry.values()); + for (const batch of queue) { + const projectBuildContexts = Array.from(batch.values()); log.info(`Building ${projectBuildContexts.length} projects in parallel:`); await Promise.all(projectBuildContexts.map(async (projectBuildContext) => { const projectName = projectBuildContext.getProject().getName(); @@ -298,8 +286,7 @@ class ProjectBuilder { return requestedProjects.includes(projectName); })); - const projectBuildContexts = new Map(); - const buildDependencies = new Map(); + const buildContexts = new Map(); for (const projectName of requiredProjects) { log.verbose(`Creating build context for project ${projectName}...`); @@ -307,19 +294,23 @@ class ProjectBuilder { project: this._graph.getProject(projectName) }); - projectBuildContexts.set(projectName, projectBuildContext); + const projectInfo = { + projectBuildContext, + requiredDependencies: new Set() + }; + buildContexts.set(projectName, projectInfo); if (projectBuildContext.requiresBuild()) { const taskRunner = projectBuildContext.getTaskRunner(); const requiredDependencies = await taskRunner.getRequiredDependencies(); - buildDependencies.set(projectName, requiredDependencies); + projectInfo.requiredDependencies = requiredDependencies; if (requiredDependencies.size === 0) { continue; } // This project needs to be built and required dependencies to be built as well this._graph.getDependencies(projectName).forEach((depName) => { - if (projectBuildContexts.has(depName)) { + if (buildContexts.has(depName)) { // Build context already exists // => Dependency will be built return; @@ -333,7 +324,7 @@ class ProjectBuilder { } } - return {projectBuildContexts, buildDependencies}; + return buildContexts; } async _getProjectFilter({ From bcca6ffb58743593acc017f3bc761910bb631eb2 Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Thu, 22 Jun 2023 21:32:15 +0200 Subject: [PATCH 3/4] Add UI5_BUILD_PARALLEL env var to enable new behavior --- lib/build/ProjectBuilder.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/build/ProjectBuilder.js b/lib/build/ProjectBuilder.js index b3749649d..69229afd2 100644 --- a/lib/build/ProjectBuilder.js +++ b/lib/build/ProjectBuilder.js @@ -133,7 +133,7 @@ class ProjectBuilder { * @returns {Promise} Promise resolving once the build has finished */ async build({ - destPath, cleanDest = false, + destPath, cleanDest = false, parallel = !!process.env.UI5_BUILD_PARALLEL, includedDependencies = [], excludedDependencies = [], dependencyIncludes }) { @@ -195,10 +195,13 @@ class ProjectBuilder { if (!projectBuildContext.requiresBuild()) { alreadyBuilt.push(projectName); } - let minBatchNumber = 0; - if (queue.length) { - for (const depName of requiredDependencies) { - minBatchNumber = determineBatchNumber(depName, minBatchNumber); + let minBatchNumber = queue.length; + if (parallel) { + minBatchNumber = 0; + if (queue.length) { + for (const depName of requiredDependencies) { + minBatchNumber = determineBatchNumber(depName, minBatchNumber); + } } } From 1f5293f4775ff3d651e7c3f439bba6dc4ba0b688 Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Fri, 23 Jun 2023 15:04:01 +0200 Subject: [PATCH 4/4] Further optimize for maximum concurrency With this change the build order is no longer pre-determined but dynamic. Once all dependencies of a project (as defined by the task runner) are built, the project is built. Renamed env variable UI5_BUILD_PARALLEL to UI5_BUILD_CONCURRENT --- lib/build/ProjectBuilder.js | 151 +++++++++++++++++++++--------------- 1 file changed, 88 insertions(+), 63 deletions(-) diff --git a/lib/build/ProjectBuilder.js b/lib/build/ProjectBuilder.js index 69229afd2..71bb99f2a 100644 --- a/lib/build/ProjectBuilder.js +++ b/lib/build/ProjectBuilder.js @@ -121,6 +121,7 @@ class ProjectBuilder { * @param {object} parameters Parameters * @param {string} parameters.destPath Target path * @param {boolean} [parameters.cleanDest=false] Decides whether project should clean the target path before build + * @param {boolean} [parameters.concurrentBuild=false] Whether to build projects concurrently if possible * @param {Array.} [parameters.includedDependencies=[]] * List of names of projects to include in the build result * If the wildcard '*' is provided, all dependencies will be included in the build result. @@ -133,7 +134,7 @@ class ProjectBuilder { * @returns {Promise} Promise resolving once the build has finished */ async build({ - destPath, cleanDest = false, parallel = !!process.env.UI5_BUILD_PARALLEL, + destPath, cleanDest = false, concurrentBuild = !!process.env.UI5_BUILD_CONCURRENT, includedDependencies = [], excludedDependencies = [], dependencyIncludes }) { @@ -170,61 +171,46 @@ class ProjectBuilder { `while including any dependencies into the build result`); } - const buildContexts = await this._createRequiredBuildContexts(requestedProjects); + const projectContexts = await this._createRequiredBuildContexts(requestedProjects); const cleanupSigHooks = this._registerCleanupSigHooks(); const fsTarget = resourceFactory.createAdapter({ fsBasePath: destPath, virBasePath: "/" }); - const queue = []; // A queue of batches to process const alreadyBuilt = []; - function determineBatchNumber(depName, minBatchNumber) { - for (let i = queue.length - 1; i >= minBatchNumber; i--) { - if (queue[i].has(depName)) { - return i + 1; + const pendingDependencies = new Map(); + const pendingProjects = new Set(); + + // Start with a deep graph traversal to get a deterministic (and somewhat reasonable) build order + await this._graph.traverseDepthFirst(({project}) => { + const projectName = project.getName(); + const {projectBuildContext, requiredDependencies} = projectContexts.get(projectName); + if (projectBuildContext) { + // Build context exists + // => This project needs to be built or, in case it has already + // been built, it's build result needs to be written out (if requested) + pendingProjects.add(projectName); + pendingDependencies.set(projectName, new Set(requiredDependencies)); + + if (!projectBuildContext.requiresBuild()) { + alreadyBuilt.push(projectName); } } - return minBatchNumber; - } - - buildContexts.forEach(({projectBuildContext, requiredDependencies}, projectName) => { - // => This project needs to be built or, in case it has already - // been built, it's build result needs to be written out (if requested) - if (!projectBuildContext.requiresBuild()) { - alreadyBuilt.push(projectName); - } - let minBatchNumber = queue.length; - if (parallel) { - minBatchNumber = 0; - if (queue.length) { - for (const depName of requiredDependencies) { - minBatchNumber = determineBatchNumber(depName, minBatchNumber); - } - } - } - - console.log(`${projectName}: Queued at batch #${minBatchNumber}`); - if (minBatchNumber === queue.length) { - // Add new batch to queue - queue.push(new Map()); - } - queue[minBatchNumber].set(projectName, projectBuildContext); }); - log.setProjects(Array.from(buildContexts.keys())); - if (queue.length > 1) { // Do not log if only the root project is being built - log.info(`Processing ${buildContexts.size} projects`); + log.setProjects(Array.from(pendingProjects.keys())); + if (pendingProjects.size > 1) { // Do not log if only the root project is being built + log.info(`Processing ${pendingProjects.size} projects`); if (alreadyBuilt.length) { log.info(` Reusing build results of ${alreadyBuilt.length} projects`); - log.info(` Building ${buildContexts.size - alreadyBuilt.length} projects`); + log.info(` Building ${pendingProjects.size - alreadyBuilt.length} projects`); } if (log.isLevelEnabled("verbose")) { - log.verbose(` Required projects:`); - log.verbose(` ${queue - .map((projectBuildContext) => { + log.verbose(` Required projects:\n ${Array.from(projectContexts.values()) + .map(({projectBuildContext}) => { const projectName = projectBuildContext.getProject().getName(); let msg; if (alreadyBuilt.includes(projectName)) { @@ -240,39 +226,78 @@ class ProjectBuilder { } } + const projectsInProcess = new Set(); + function hasPendingDependencies(projectName) { + const pendingDeps = pendingDependencies.get(projectName); + for (const depName of pendingDeps) { + if (!pendingProjects.has(depName) && !projectsInProcess.has(depName)) { + pendingDeps.delete(depName); + } + } + + if (log.isLevelEnabled("verbose")) { + log.verbose(`${projectName} is waiting for: ${Array.from(pendingDeps.keys()).join(", ")}`); + } + return !!pendingDeps.size; + } + if (cleanDest) { log.info(`Cleaning target directory...`); await rimraf(destPath); } const startTime = process.hrtime(); try { + const pBuilds = new Map(); const pWrites = []; - for (const batch of queue) { - const projectBuildContexts = Array.from(batch.values()); - log.info(`Building ${projectBuildContexts.length} projects in parallel:`); - await Promise.all(projectBuildContexts.map(async (projectBuildContext) => { - const projectName = projectBuildContext.getProject().getName(); - const projectType = projectBuildContext.getProject().getType(); - log.verbose(`Processing project ${projectName}...`); - - // Only build projects that are not already build (i.e. provide a matching build manifest) - if (alreadyBuilt.includes(projectName)) { - log.skipProjectBuild(projectName, projectType); - } else { - log.startProjectBuild(projectName, projectType); - await projectBuildContext.getTaskRunner().runTasks(); - log.endProjectBuild(projectName, projectType); - } - if (!requestedProjects.includes(projectName)) { - // Project has not been requested - // => Its resources shall not be part of the build result - return; - } - log.verbose(`Writing out files...`); - pWrites.push(this._writeResults(projectBuildContext, fsTarget)); - })); + while (pendingProjects.size) { + const buildCount = pBuilds.length; + for (const projectName of pendingProjects) { + if (!hasPendingDependencies(projectName)) { + projectsInProcess.add(projectName); + pendingProjects.delete(projectName); + // => Build the project + pBuilds.set(projectName, (async ({projectBuildContext}) => { + const projectName = projectBuildContext.getProject().getName(); + const projectType = projectBuildContext.getProject().getType(); + + // Only build projects that are not already build (i.e. provide a matching build manifest) + if (alreadyBuilt.includes(projectName)) { + log.skipProjectBuild(projectName, projectType); + } else { + log.startProjectBuild(projectName, projectType); + await projectBuildContext.getTaskRunner().runTasks(); + log.endProjectBuild(projectName, projectType); + } + projectsInProcess.delete(projectName); + if (!requestedProjects.includes(projectName)) { + // Project has not been requested + // => Its resources shall not be part of the build result + return; + } + + log.verbose(`Writing out files...`); + pWrites.push(this._writeResults(projectBuildContext, fsTarget)); + return projectName; + })(projectContexts.get(projectName))); + + if (!concurrentBuild) { + // Do not add more builds + break; + } + } + } + if (buildCount === pBuilds.length || !concurrentBuild) { + log.verbose(`Waiting for a build to finish`); + // Either schedule a build or wait till any promise resolved + const finishedProjectName = await Promise.any(pBuilds.values()); + pBuilds.delete(finishedProjectName); + log.verbose(`${finishedProjectName} finished building. Builds in process: ${pBuilds.size}`); + } } + // Wait for any remaining builds + await Promise.all(pBuilds); + // Wait for the deferred writes await Promise.all(pWrites); log.info(`Build succeeded in ${this._getElapsedTime(startTime)}`); } catch (err) {