From 052e83ef35bac025e69187a372ec04f23afe570b Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 1 Nov 2024 08:19:12 +0100 Subject: [PATCH 1/6] split into bin, public, private Signed-off-by: Dennis Behm --- .../PackageBuildOutputs.groovy | 314 +++++++++++------- .../applicationDescriptorUtils.groovy | 2 +- 2 files changed, 189 insertions(+), 127 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 35e0be0a..9566d096 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -66,8 +66,11 @@ def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).p @Field def applicationDescriptor @Field def sbomUtilities @Field def rc = 0 -@Field String includeSubfolder = "include" + +@Field String includeSubfolder = "public" @Field String binSubfolder = "bin" +@Field String internalSubfolder = "private" +@Field def tempLoadDir def startTime = new Date() @@ -91,10 +94,6 @@ if (rc != 0) { // Enable file tagging BuildProperties.setProperty("dbb.file.tagging", "true") // Enable dbb file tagging -// Map of last level dataset qualifier to DBB CopyToHFS CopyMode. -def copyModeMap = parseCopyModeMap(props.copyModeMap) - - // Hashmap of BuildOutput to Record Map buildOutputsMap = new HashMap() @@ -260,8 +259,8 @@ props.buildReportOrder.each { buildReportFile -> if (applicationDescriptor && output.deployType.equals("OBJ")) { fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", member) } - // If the artifact is not an Object Deck or has no usage or its usage is not main - if ((output.deployType.equals("OBJ") && fileUsage && !fileUsage.equals("main")) || !output.deployType.equals("OBJ")) { + // Leave OBJs with main behind + if ((output.deployType.equals("OBJ") && fileUsage && (fileUsage.equals("service submodule") || fileUsage.equals("internal submodule")) ) || !output.deployType.equals("OBJ")) { datasetMembersCount++ String file = buildRecord.getFile() def dependencySetRecord = buildReport.getRecords().find { @@ -382,7 +381,7 @@ if (rc == 0) { def tarFile = "$props.workDir/${tarFileName}" //Create a temporary directory on zFS to copy the load modules from data sets to - def tempLoadDir = new File("$props.workDir/tempPackageDir") + tempLoadDir = new File("$props.workDir/tempPackageDir") !tempLoadDir.exists() ?: tempLoadDir.deleteDir() tempLoadDir.mkdirs() @@ -473,102 +472,88 @@ if (rc == 0) { } } - if (rc == 0) { - + if (rc == 0) { + println("*** Number of build outputs to package: ${buildOutputsMap.size()}") - - println("** Copy build outputs to temporary package directory '$tempLoadDir'") - - buildOutputsMap.each { deployableArtifact, info -> - String container = info.get("container") - String owningApplication = info.get("owningApplication") - Record record = info.get("record") - PropertiesRecord propertiesRecord = info.get("propertiesRecord") - DependencySetRecord dependencySetRecord = info.get("dependencySetRecord") + + // TODO: Add to properties + def shared = ["OBJ"] + def generated = ["GEN", "OBJ"] + + println("** Copy deployable outputs to temporary package directory '$tempLoadDir/$binSubfolder'") + deployableOutputs = buildOutputsMap.findAll { deployableArtifact, info -> + !(generated.contains(deployableArtifact.deployType) || shared.contains(deployableArtifact.deployType)) + } + copyArtifactsToUSS(deployableOutputs, binSubfolder) + + if (applicationDescriptor) { + + // build outputs that are mapped to a shared deployType and are flagged as 'service submodule' in the application descriptor + publicDerivedBuildOutputs = buildOutputsMap.findAll { deployableArtifact, info -> + fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) + shared.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("service submodule") + } - def relativeFilePath = "" - if (deployableArtifact.artifactType.equals("zFSFile")) { - relativeFilePath = "$binSubfolder" - } else { - if (deployableArtifact.deployType.equals("OBJ")) { - relativeFilePath = "$includeSubfolder/bin" - } else { - relativeFilePath = "$binSubfolder/${deployableArtifact.deployType}" - } + // build outputs that are mapped to a shared deployType and are flagged as 'internal submodule' in the application descriptor + internalDerivedBuildOutputs = buildOutputsMap.findAll { deployableArtifact, info -> + fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) + generated.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("internal submodule") } - - // define file name in USS - def fileName = deployableArtifact.file - - // add deployType to file name - if (props.addExtension && props.addExtension.toBoolean()) { - fileName = fileName + '.' + deployableArtifact.deployType + + println("** Copy public build outputs to temporary package directory '$tempLoadDir/$includeSubfolder'") + copyArtifactsToUSS(publicDerivedBuildOutputs, includeSubfolder) + + println("** Copy internal outputs to temporary package directory '$tempLoadDir/$internalSubfolder'") + copyArtifactsToUSS(internalDerivedBuildOutputs, internalSubfolder) + + if (deployableOutputs.size() + publicDerivedBuildOutputs.size() + internalDerivedBuildOutputs.size() != buildOutputsMap.size()) { + println("*! [WARNING] The number of copied artifacts does not match the identified build outputs ${buildOutputsMap.size()} . Most likely files got not assigned a correct field in the application descriptor.") } - def file = new File("$tempLoadDir/$relativeFilePath/$fileName") - - def (directory, relativeFileName) = extractDirectoryAndFile(file.toPath().toString()) - new File(directory).mkdirs() - - - if (deployableArtifact.artifactType.equals("zFSFile")) { - def originalFile = new File(container + "/" + deployableArtifact.file) - println "\tCopy '${originalFile.toPath()}' to '${file.toPath()}'" - try { - Files.copy(originalFile.toPath(), file.toPath(), StandardCopyOption.COPY_ATTRIBUTES); - } catch (IOException exception) { - println "!* [ERROR] Copy failed: an error occurred when copying '${originalFile.toPath()}' to '${file.toPath()}'" - rc = Math.max(rc, 1) + + // Checks if all binary interfaces (submodules) are in the archive or not + checkBinaryInterfaces(tempLoadDir, applicationDescriptor) + + + ArrayList publicIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "public") + ArrayList sharedIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "shared") + + + if (publicIncludeFiles && !publicIncludeFiles.isEmpty()) { + if (sharedIncludeFiles && !sharedIncludeFiles.isEmpty()) { + publicIncludeFiles.addAll(sharedIncludeFiles) } - } else if (deployableArtifact.artifactType.equals("DatasetMember")) { - // set copyMode based on last level qualifier - currentCopyMode = copyModeMap[container.replaceAll(/.*\.([^.]*)/, "\$1")] - if (currentCopyMode != null) { - if (ZFile.exists("//'$container(${deployableArtifact.file})'")) { - // Copy outputs to HFS - CopyToHFS copy = new CopyToHFS() - copy.setCopyMode(DBBConstants.CopyMode.valueOf(currentCopyMode)) - copy.setDataset(container) - - println "\tCopy '$container(${deployableArtifact.file})' to '$tempLoadDir/$relativeFilePath/$fileName' with DBB Copymode '$currentCopyMode'" - copy.dataset(container).member(deployableArtifact.file).file(file).execute() - - // Tagging binary files - if (currentCopyMode == CopyMode.BINARY || currentCopyMode == CopyMode.LOAD) { - StringBuffer stdout = new StringBuffer() - StringBuffer stderr = new StringBuffer() - Process process = "chtag -b $file".execute() - process.waitForProcessOutput(stdout, stderr) - if (stderr){ - println ("*! stderr : $stderr") - println ("*! stdout : $stdout") - } - } - - // Append record to Wazi Deploy Application Manifest - if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - wdManifestGeneratorUtilities.appendArtifactToManifest(deployableArtifact, "$relativeFilePath/$fileName", record, dependencySetRecord, propertiesRecord) + + println("*** Number of public source code interfaces to package: ${publicIncludeFiles.size()}") + println("** Copy all public source code interfaces from application repository to temporary package directory '$tempLoadDir'") + + publicIncludeFiles.forEach() { includeFile -> + Path includeFilePath = Paths.get("${props.applicationFolderPath}/${includeFile}") + Path targetIncludeFilePath = Paths.get("${tempLoadDir.getPath()}/${includeSubfolder}/src/${includeFilePath.getFileName()}") + if (props.verbose) println("\tCopy '${includeFilePath}' file to '${targetIncludeFilePath}'") + try { + //Create target parent folder if it doesn't exist + def targetIncludeFilesFolder = targetIncludeFilePath.getParent().toFile() + if (!targetIncludeFilesFolder.exists()) { + targetIncludeFilesFolder.mkdirs() } - - } else { - println "*! [ERROR] Copy failed: The file '$container(${deployableArtifact.file})' doesn't exist." - rc = Math.max(rc, 1) + Files.copy(includeFilePath, targetIncludeFilePath, COPY_ATTRIBUTES, REPLACE_EXISTING) + } catch (IOException exception) { + println "!* [ERROR] Copy failed: an error occurred when copying '${includeFilePath}' to '${targetIncludeFilePath}'" + rc = Math.max(rc, 1) } - } else { - println "*! [ERROR] Copy failed: The file '$container(${deployableArtifact.file})' could not be copied due to missing mapping." - rc = Math.max(rc, 1) } - } else if (deployableArtifact.artifactType.equals("DatasetMemberDelete")) { - // generate delete instruction for Wazi Deploy - if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { - wdManifestGeneratorUtilities.appendArtifactDeletionToManifest(deployableArtifact, "$relativeFilePath/$fileName", record, propertiesRecord) - } } - - if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { - sbomUtilities.addEntryToSBOM(deployableArtifact, info) + } else { + // grouping of build outputs for shared and generated source only available via Application Descriptor + skippedDeployableOutputs = buildOutputsMap.findAll { deployableArtifact, info -> + (generated.contains(deployableArtifact.deployType) || shared.contains(deployableArtifact.deployType)) + } + if (skippedDeployableOutputs) { + println("** Number of skipped build outputs: ${skippedDeployableOutputs.size()}") } } + } if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { @@ -624,33 +609,6 @@ if (rc == 0) { if (props.verbose) println("** Copy packaging properties config file to '$copiedPackagingPropertiesFilePath'") Files.copy(packagingPropertiesFilePath, copiedPackagingPropertiesFilePath, COPY_ATTRIBUTES, REPLACE_EXISTING) - if (applicationDescriptor) { - ArrayList publicIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "public") - ArrayList sharedIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "shared") - if (publicIncludeFiles && !publicIncludeFiles.isEmpty()) { - if (sharedIncludeFiles && !sharedIncludeFiles.isEmpty()) { - publicIncludeFiles.addAll(sharedIncludeFiles) - } - publicIncludeFiles.forEach() { includeFile -> - Path includeFilePath = Paths.get("${props.applicationFolderPath}/${includeFile}") - Path targetIncludeFilePath = Paths.get("${tempLoadDir.getPath()}/${includeSubfolder}/src/${includeFilePath.getFileName()}") - if (props.verbose) println("** Copy '${includeFilePath}' file to '${targetIncludeFilePath}'") - try { - //Create target parent folder if it doesn't exist - def targetIncludeFilesFolder = targetIncludeFilePath.getParent().toFile() - if (!targetIncludeFilesFolder.exists()) { - targetIncludeFilesFolder.mkdirs() - } - Files.copy(includeFilePath, targetIncludeFilePath, COPY_ATTRIBUTES, REPLACE_EXISTING) - } catch (IOException exception) { - println "!* [ERROR] Copy failed: an error occurred when copying '${includeFilePath}' to '${targetIncludeFilePath}'" - rc = Math.max(rc, 1) - } - } - } - // Checks if all binary interfaces (submodules) are in the archive or not - checkBinaryInterfaces(tempLoadDir, applicationDescriptor) - } if (props.owner) { def processCmd = [ @@ -751,6 +709,106 @@ if (rc > 0) { } System.exit(rc) +/** + * + */ + +def copyArtifactsToUSS(Map buildOutputsMap, String tarSubfolder) { + // local variables + def copyModeMap = parseCopyModeMap(props.copyModeMap) // Map of last level dataset qualifier to DBB CopyToHFS CopyMode. + + buildOutputsMap.each { deployableArtifact, info -> + String container = info.get("container") + String owningApplication = info.get("owningApplication") + Record record = info.get("record") + PropertiesRecord propertiesRecord = info.get("propertiesRecord") + DependencySetRecord dependencySetRecord = info.get("dependencySetRecord") + + def relativeFilePath = "" + if (deployableArtifact.artifactType.equals("zFSFile")) { + relativeFilePath = "$binSubfolder/uss" + } else { + if (deployableArtifact.deployType.equals("OBJ")) { + relativeFilePath = "$tarSubfolder/bin" + } else { + relativeFilePath = "$tarSubfolder/${deployableArtifact.deployType}" + } + } + + // define file name in USS + def fileName = deployableArtifact.file + + // add deployType to file name + if (props.addExtension && props.addExtension.toBoolean()) { + fileName = fileName + '.' + deployableArtifact.deployType + } + def file = new File("$tempLoadDir/$relativeFilePath/$fileName") + + def (directory, relativeFileName) = extractDirectoryAndFile(file.toPath().toString()) + new File(directory).mkdirs() + + + if (deployableArtifact.artifactType.equals("zFSFile")) { + def originalFile = new File(container + "/" + deployableArtifact.file) + println "\tCopy '${originalFile.toPath()}' to '${file.toPath()}'" + try { + Files.copy(originalFile.toPath(), file.toPath(), StandardCopyOption.COPY_ATTRIBUTES); + } catch (IOException exception) { + println "!* [ERROR] Copy failed: an error occurred when copying '${originalFile.toPath()}' to '${file.toPath()}'" + rc = Math.max(rc, 1) + } + } else if (deployableArtifact.artifactType.equals("DatasetMember")) { + // set copyMode based on last level qualifier + currentCopyMode = copyModeMap[container.replaceAll(/.*\.([^.]*)/, "\$1")] + if (currentCopyMode != null) { + if (ZFile.exists("//'$container(${deployableArtifact.file})'")) { + // Copy outputs to HFS + CopyToHFS copy = new CopyToHFS() + copy.setCopyMode(DBBConstants.CopyMode.valueOf(currentCopyMode)) + copy.setDataset(container) + + println "\tCopy '$container(${deployableArtifact.file})' to '$tempLoadDir/$relativeFilePath/$fileName' with DBB Copymode '$currentCopyMode'" + copy.dataset(container).member(deployableArtifact.file).file(file).execute() + + // Tagging binary files + if (currentCopyMode == CopyMode.BINARY || currentCopyMode == CopyMode.LOAD) { + StringBuffer stdout = new StringBuffer() + StringBuffer stderr = new StringBuffer() + Process process = "chtag -b $file".execute() + process.waitForProcessOutput(stdout, stderr) + if (stderr){ + println ("*! stderr : $stderr") + println ("*! stdout : $stdout") + } + } + + // Append record to Wazi Deploy Application Manifest + if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { + wdManifestGeneratorUtilities.appendArtifactToManifest(deployableArtifact, "$relativeFilePath/$fileName", record, dependencySetRecord, propertiesRecord) + } + + } else { + println "*! [ERROR] Copy failed: The file '$container(${deployableArtifact.file})' doesn't exist." + rc = Math.max(rc, 1) + } + } else { + println "*! [ERROR] Copy failed: The file '$container(${deployableArtifact.file})' could not be copied due to missing mapping." + rc = Math.max(rc, 1) + } + } else if (deployableArtifact.artifactType.equals("DatasetMemberDelete")) { + // generate delete instruction for Wazi Deploy + if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { + wdManifestGeneratorUtilities.appendArtifactDeletionToManifest(deployableArtifact, "$relativeFilePath/$fileName", record, propertiesRecord) + } + } + + + if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { + sbomUtilities.addEntryToSBOM(deployableArtifact, info) + } + } +} + /** * parse data set name and member name * @param fullname e.g. BLD.LOAD(PGM1) @@ -1054,18 +1112,22 @@ def checkBinaryInterfaces(File tempLoadDir, applicationDescriptor) { if (binaryInternalInterfaces) { binaryInterfaces.addAll(binaryInternalInterfaces) } - ArrayList missingBinaryInterfaces = new ArrayList() + binaryInterfaces.each { binaryInterface -> String sourceFileName = Paths.get(binaryInterface).getFileName().toString() - String expectedFileName = "$includeSubfolder/bin/" + sourceFileName.split("\\.")[0].toUpperCase() + ".OBJ" + String member = sourceFileName.split("\\.")[0].toUpperCase() + + // shared + String expectedFileName = "$includeSubfolder/bin/" + member + ".OBJ" File expectedInterface = new File("${tempLoadDir.getAbsolutePath()}/$expectedFileName") - if (!expectedInterface.exists()) { - missingBinaryInterfaces.add(expectedFileName) + // private + String expectedFileNamePrivate = "$internalSubfolder/bin/" + member + ".OBJ" + File expectedInterfacePrivate = new File("${tempLoadDir.getAbsolutePath()}/$expectedFileNamePrivate") + + if (!(expectedInterface.exists() || expectedInterfacePrivate.exists())) { // not found + println("*! [WARNING] Binary interface for $member not found in interface folders. Tar not exposing all interfaces.") } } - if (missingBinaryInterfaces.size() > 0) { - println("*! [WARNING] Some binary interfaces are missing in the current archive: ${missingBinaryInterfaces}.") - } } diff --git a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy index 53aab543..8c6477a4 100644 --- a/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/applicationDescriptorUtils.groovy @@ -283,7 +283,7 @@ def getFileUsage(ApplicationDescriptor applicationDescriptor, String sourceGroup println("*! [WARNING] Multiple files found matching '${name}'. Skipping search.") return null } else { - println("*! [WARNING] No file found matching '${name}'. Skipping search.") + println("*! [WARNING] No file reference found in application descripor matching '${name}'. Skipping search.") return null } } else { From 911428a61244ccf23d3032ad3d9b1d903cad18fb Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 1 Nov 2024 09:49:16 +0100 Subject: [PATCH 2/6] configurable properties Signed-off-by: Dennis Behm --- .../PackageBuildOutputs.groovy | 130 +++++++++++------- .../packageBuildOutputs.properties | 42 ++++++ 2 files changed, 121 insertions(+), 51 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 9566d096..8886f694 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -273,6 +273,8 @@ props.buildReportOrder.each { buildReportFile -> propertiesRecord: buildResultPropertiesRecord, dependencySetRecord: dependencySetRecord ]) + } else { + if (props.verbose) println("*! Build output ${output.dataset} with deployType '${output.deployType}' has been excluded from packaging.") } } } @@ -476,70 +478,82 @@ if (rc == 0) { println("*** Number of build outputs to package: ${buildOutputsMap.size()}") - // TODO: Add to properties - def shared = ["OBJ"] - def generated = ["GEN", "OBJ"] + def publicInterfacesDeployTypes + def derivedInterfacesDeployTypes + def processedArtifacts = 0 // used as a checksum that all files got categorized + + if (props.publicInterfaces) publicInterfacesDeployTypes = props.publicInterfaces.split(",") // split comma separated list into list + if (props.internalInterfaces) derivedInterfacesDeployTypes = props.internalInterfaces.split(",") // split comma separated list into list println("** Copy deployable outputs to temporary package directory '$tempLoadDir/$binSubfolder'") deployableOutputs = buildOutputsMap.findAll { deployableArtifact, info -> - !(generated.contains(deployableArtifact.deployType) || shared.contains(deployableArtifact.deployType)) + !((publicInterfacesDeployTypes && publicInterfacesDeployTypes.contains(deployableArtifact.deployType)) || (derivedInterfacesDeployTypes && derivedInterfacesDeployTypes.contains(deployableArtifact.deployType))) } copyArtifactsToUSS(deployableOutputs, binSubfolder) - - if (applicationDescriptor) { - - // build outputs that are mapped to a shared deployType and are flagged as 'service submodule' in the application descriptor - publicDerivedBuildOutputs = buildOutputsMap.findAll { deployableArtifact, info -> - fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) - shared.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("service submodule") + processedArtifacts += deployableOutputs.size() + + if (applicationDescriptor) { // application descriptor needs to be present for categorizing artifacts + + if (publicInterfacesDeployTypes) { + // build outputs that are mapped to a shared deployType and are flagged as 'service submodule' in the application descriptor + publicDerivedBuildOutputs = buildOutputsMap.findAll { deployableArtifact, info -> + fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) + publicInterfacesDeployTypes.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("service submodule") + } + println("** Copy public build outputs to temporary package directory '$tempLoadDir/$includeSubfolder'") + copyArtifactsToUSS(publicDerivedBuildOutputs, includeSubfolder) + processedArtifacts += publicDerivedBuildOutputs.size() } - - // build outputs that are mapped to a shared deployType and are flagged as 'internal submodule' in the application descriptor - internalDerivedBuildOutputs = buildOutputsMap.findAll { deployableArtifact, info -> - fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) - generated.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("internal submodule") + + if (derivedInterfacesDeployTypes) { + // build outputs that are mapped to a shared deployType and are flagged as 'internal submodule' in the application descriptor + internalDerivedBuildOutputs = buildOutputsMap.findAll { deployableArtifact, info -> + fileUsage = applicationDescriptorUtils.getFileUsageByType(applicationDescriptor, "Program", deployableArtifact.file) + derivedInterfacesDeployTypes.contains(deployableArtifact.deployType) && fileUsage && fileUsage.equals("internal submodule") + } + println("** Copy internal outputs to temporary package directory '$tempLoadDir/$internalSubfolder'") + copyArtifactsToUSS(internalDerivedBuildOutputs, internalSubfolder) + processedArtifacts += internalDerivedBuildOutputs.size() } - - println("** Copy public build outputs to temporary package directory '$tempLoadDir/$includeSubfolder'") - copyArtifactsToUSS(publicDerivedBuildOutputs, includeSubfolder) - println("** Copy internal outputs to temporary package directory '$tempLoadDir/$internalSubfolder'") - copyArtifactsToUSS(internalDerivedBuildOutputs, internalSubfolder) - - if (deployableOutputs.size() + publicDerivedBuildOutputs.size() + internalDerivedBuildOutputs.size() != buildOutputsMap.size()) { + if (processedArtifacts != buildOutputsMap.size()) { println("*! [WARNING] The number of copied artifacts does not match the identified build outputs ${buildOutputsMap.size()} . Most likely files got not assigned a correct field in the application descriptor.") } // Checks if all binary interfaces (submodules) are in the archive or not + println("** Validate if all derived interfaces declared in application descriptor got processed.") checkBinaryInterfaces(tempLoadDir, applicationDescriptor) - - - ArrayList publicIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "public") - ArrayList sharedIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "shared") - - if (publicIncludeFiles && !publicIncludeFiles.isEmpty()) { - if (sharedIncludeFiles && !sharedIncludeFiles.isEmpty()) { - publicIncludeFiles.addAll(sharedIncludeFiles) - } - - println("*** Number of public source code interfaces to package: ${publicIncludeFiles.size()}") - println("** Copy all public source code interfaces from application repository to temporary package directory '$tempLoadDir'") - - publicIncludeFiles.forEach() { includeFile -> - Path includeFilePath = Paths.get("${props.applicationFolderPath}/${includeFile}") - Path targetIncludeFilePath = Paths.get("${tempLoadDir.getPath()}/${includeSubfolder}/src/${includeFilePath.getFileName()}") - if (props.verbose) println("\tCopy '${includeFilePath}' file to '${targetIncludeFilePath}'") - try { - //Create target parent folder if it doesn't exist - def targetIncludeFilesFolder = targetIncludeFilePath.getParent().toFile() - if (!targetIncludeFilesFolder.exists()) { - targetIncludeFilesFolder.mkdirs() + if (props.publishInterfaces && props.publishInterfaces.toBoolean()) { + + ArrayList publicIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "public") + ArrayList sharedIncludeFiles = applicationDescriptorUtils.getFilesByTypeAndUsage(applicationDescriptor, "Include File", "shared") + + + if (publicIncludeFiles && !publicIncludeFiles.isEmpty()) { + if (sharedIncludeFiles && !sharedIncludeFiles.isEmpty()) { + publicIncludeFiles.addAll(sharedIncludeFiles) + } + + println("*** Number of public source code interfaces to package: ${publicIncludeFiles.size()}") + println("** Copy all public source code interfaces from application repository to temporary package directory '$tempLoadDir'") + + publicIncludeFiles.forEach() { includeFile -> + Path includeFilePath = Paths.get("${props.applicationFolderPath}/${includeFile}") + Path targetIncludeFilePath = Paths.get("${tempLoadDir.getPath()}/${includeSubfolder}/src/${includeFilePath.getFileName()}") + try { + //Create target parent folder if it doesn't exist + def targetIncludeFilesFolder = targetIncludeFilePath.getParent().toFile() + if (!targetIncludeFilesFolder.exists()) { + targetIncludeFilesFolder.mkdirs() + } + println("\tCopy '${includeFilePath}' file to '${targetIncludeFilePath}'") + runShellCmd("cp -Rf $includeFilePath $targetIncludeFilePath") + + } catch (IOException exception) { + println "!* [ERROR] Copy failed: an error occurred when copying '${includeFilePath}' to '${targetIncludeFilePath}'" + rc = Math.max(rc, 1) } - Files.copy(includeFilePath, targetIncludeFilePath, COPY_ATTRIBUTES, REPLACE_EXISTING) - } catch (IOException exception) { - println "!* [ERROR] Copy failed: an error occurred when copying '${includeFilePath}' to '${targetIncludeFilePath}'" - rc = Math.max(rc, 1) } } } @@ -950,7 +964,7 @@ def parseInput(String[] cliArgs){ if (opts.o) props.owner = opts.o - props.verbose = (opts.verb) ? 'true' : 'false' + if (opts.verb) props.verbose = 'true' if (opts.af) { props.applicationFolderPath = opts.af @@ -1125,7 +1139,7 @@ def checkBinaryInterfaces(File tempLoadDir, applicationDescriptor) { File expectedInterfacePrivate = new File("${tempLoadDir.getAbsolutePath()}/$expectedFileNamePrivate") if (!(expectedInterface.exists() || expectedInterfacePrivate.exists())) { // not found - println("*! [WARNING] Binary interface for $member not found in interface folders. Tar not exposing all interfaces.") + println("*! [WARNING] Binary interface for $member not found in interface folders. Tar not exposing all declared interfaces.") } } } @@ -1188,4 +1202,18 @@ def retrieveBuildResultProperty(PropertiesRecord buildResultPropertiesRecord, St return null } } +} + +// Methods +def runShellCmd(String cmd){ + StringBuffer resp = new StringBuffer() + StringBuffer error = new StringBuffer() + + Process process = cmd.execute() + process.waitForProcessOutput(resp, error) + if (error) { + String warningMsg = "*! Failed to execute shell command $cmd" + println(warningMsg) + println(error) + } } \ No newline at end of file diff --git a/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties b/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties index 90269dbf..4e9aa7a8 100644 --- a/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties +++ b/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties @@ -14,6 +14,48 @@ copyModeMap = ["COPYBOOK": "TEXT", "COPY": "TEXT", "DBRM": "BINARY", "LOAD": "LO # # deployTypesFilter= +# Comma-separated list of deployTypes that describe >public interfaces< +# that consuming applications pull into their build workspace as a build +# dependency. +# +# Along with the the usage definition 'service interface' in the +# 'Application Descriptor', these outputs will be made +# available in a dedicated sub folder of the tar file, that +# applications that need these modules as a build dependency +# can include into their build scope/workspace. +# +# These deployTypes are excluded from being added to the /bin folder +# that contains the deployable outputs. +# +# This is specifically useful for applications that provide object modules +# of statically called submodules to other applications. +# It does not effect modules that are called dynamically. +publicInterfaces=OBJ + +# Comma-separated list of deployTypes that describe derived build outputs +# that represent >private interfaces< that are required internally for by +# the application +# +# Along with the required usage definition in the +# 'Application Descriptor', these files will be made available +# in a dedicated sub folder of the tar file, that +# can then be included as a baseline package. +# +# These deployTypes are excluded from being added to the /bin folder +# that contains the deployable outputs. +# +# This is useful for applications with static calls to provide object modules +# of statically called submodules or generated source code such as +# generated copybooks for BMS or DCLgen files for subsequent builds. +internalInterfaces=OBJ,BMSCOPY + +# Publish source code interfaces that are declared as 'shared/public' in the +# 'Application Descriptor' into dedicated subfolder for consumption by +# applications that depend this application. +# The packaging process retrieves the files directly from the +# checked out Git repository. +publishInterfaces=true + # Comma-separated list of files/patterns from the USS build workspace. (Optional) # Please note that the cli option `includeLogs` overwrites this setting # Sample: includeLogs = *.log,*.xml From 502ca55896028d4247b974287c4b0f2d2785949d Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 1 Nov 2024 10:03:50 +0100 Subject: [PATCH 3/6] Fix tracing Signed-off-by: Dennis Behm --- Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 8886f694..ec4870fe 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -711,7 +711,8 @@ if (rc == 0) { GroovyObject artifactRepositoryHelpers = (GroovyObject) artifactRepositoryHelpersClass.newInstance() println ("** Upload package to Artifact Repository '$url'.") - artifactRepositoryHelpers.upload(url, tarFile as String, user, password, props.verbose.toBoolean(), httpClientVersion) + def tracing = (props.verbose && props.verbose.toBoolean()) ? true : false + artifactRepositoryHelpers.upload(url, tarFile as String, user, password, tracing, httpClientVersion) } } } From 6dcacce4fbbd4ea135b0d84888a75ada5f87cb42 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 1 Nov 2024 10:14:55 +0100 Subject: [PATCH 4/6] manage cases without processing the application descriptor Signed-off-by: Dennis Behm --- Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index ec4870fe..27d94fb9 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -561,7 +561,7 @@ if (rc == 0) { } else { // grouping of build outputs for shared and generated source only available via Application Descriptor skippedDeployableOutputs = buildOutputsMap.findAll { deployableArtifact, info -> - (generated.contains(deployableArtifact.deployType) || shared.contains(deployableArtifact.deployType)) + (derivedInterfacesDeployTypes.contains(deployableArtifact.deployType) || publicInterfacesDeployTypes.contains(deployableArtifact.deployType)) } if (skippedDeployableOutputs) { println("** Number of skipped build outputs: ${skippedDeployableOutputs.size()}") From 5ec84a8184283ad8903be07a7724deae69aa2dc2 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 1 Nov 2024 10:19:15 +0100 Subject: [PATCH 5/6] manage cases with old configuration file for package outputs Signed-off-by: Dennis Behm --- Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 27d94fb9..50e5b4c7 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -561,7 +561,7 @@ if (rc == 0) { } else { // grouping of build outputs for shared and generated source only available via Application Descriptor skippedDeployableOutputs = buildOutputsMap.findAll { deployableArtifact, info -> - (derivedInterfacesDeployTypes.contains(deployableArtifact.deployType) || publicInterfacesDeployTypes.contains(deployableArtifact.deployType)) + ((derivedInterfacesDeployTypes && derivedInterfacesDeployTypes.contains(deployableArtifact.deployType)) || (publicInterfacesDeployTypes && publicInterfacesDeployTypes.contains(deployableArtifact.deployType))) } if (skippedDeployableOutputs) { println("** Number of skipped build outputs: ${skippedDeployableOutputs.size()}") From 364b2ce23e7675e12d7feab91939ed163c8e5cbd Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 13 Nov 2024 18:16:20 +0100 Subject: [PATCH 6/6] document external dependencies and packageinfo Signed-off-by: Dennis Behm --- .../PackageBuildOutputs.groovy | 45 ++++++++++-- .../packageBuildOutputs.properties | 2 +- .../WaziDeployManifestGenerator.groovy | 68 ++++++++++++++++++- 3 files changed, 107 insertions(+), 8 deletions(-) diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 50e5b4c7..6d150524 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -98,7 +98,8 @@ BuildProperties.setProperty("dbb.file.tagging", "true") // Enable dbb file taggi Map buildOutputsMap = new HashMap() // Field to store default tarFileLabel (buildInfo.label) when cli argument tarFileName is not passed. -def String tarFileLabel = "Default" +@Field def String tarFileLabel = "Default" +@Field def String tarFileName = "" // Object to store scm information for Wazi Deploy Application Manifest file HashMap scmInfo = new HashMap() @@ -364,6 +365,8 @@ props.buildReportOrder.each { buildReportFile -> scmInfo.put("uri", "multipleBuildReports") } } + + } } @@ -378,8 +381,17 @@ if (rc == 0) { if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { wdManifestGeneratorUtilities.initWaziDeployManifestGenerator(props)// Wazi Deploy Application Manifest wdManifestGeneratorUtilities.setScmInfo(scmInfo) + if (props.externalDependenciesEvidences) { + File externalDependenciesEvidenceFile = new File("${props.externalDependenciesEvidences}") + if (externalDependenciesEvidenceFile.exists()){ + wdManifestGeneratorUtilities.setExternalDependencies(externalDependenciesEvidenceFile) + } else { + println("** External build dependencies file not found (${props.externalDependenciesEvidences}). Exiting.") + rc=4 + } + } } - def String tarFileName = (props.tarFileName) ? props.tarFileName : "${tarFileLabel}.tar" + tarFileName = (props.tarFileName) ? props.tarFileName : "${tarFileLabel}.tar" def tarFile = "$props.workDir/${tarFileName}" //Create a temporary directory on zFS to copy the load modules from data sets to @@ -574,10 +586,21 @@ if (rc == 0) { sbomUtilities.writeSBOM("$tempLoadDir/sbom.json", props.fileEncoding) } + if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean() && props.publish && props.publish.toBoolean() && rc == 0) { + HashMap packageInfo = new HashMap() + packageInfo.put("type", "artifactRepository") + packageInfo.put("name", props.versionName) + + packageUrl = computeAbsoluteRepositoryUrl() + if (packageUrl) packageInfo.put("uri", packageUrl) + + wdManifestGeneratorUtilities.setPackageInfo(packageInfo) + } + if (wdManifestGeneratorUtilities && props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean() && rc == 0) { // print application manifest - // wazideploy_manifest.yml is the default name of the manifest file + // wazideploy_manifest.yml is the default name of the manifest file wdManifestGeneratorUtilities.writeApplicationManifest(new File("$tempLoadDir/wazideploy_manifest.yml"), props.fileEncoding, props.verbose) } @@ -696,8 +719,7 @@ if (rc == 0) { //Set up the artifact repository information to publish the tar file if (props.publish && props.publish.toBoolean() && rc == 0){ // Configuring artifact repositoryHelper parms - def String remotePath = (props.versionName) ? (props.versionName + "/" + tarFileName) : (tarFileLabel + "/" + tarFileName) - def url = new URI(props.get('artifactRepository.url') + "/" + props.get('artifactRepository.repo') + "/" + (props.get('artifactRepository.directory') ? "${props.get('artifactRepository.directory')}/" : "") + remotePath).normalize().toString() // Normalized URL + def url = computeAbsoluteRepositoryUrl() def apiKey = props.'artifactRepository.user' def user = props.'artifactRepository.user' @@ -896,6 +918,7 @@ def parseInput(String[] cliArgs){ // Wazi Deploy Application Manifest generation cli.wd(longOpt:'generateWaziDeployAppManifest', 'Flag indicating to generate and add the Wazi Deploy Application Manifest file.') + cli.ed(longOpt:'externalDependenciesEvidences', args:1, argName:'externalDependenciesEvidences', 'File documenting the external dependencies that were provided to the build phase.') // Artifact repository options :: cli.p(longOpt:'publish', 'Flag to indicate package upload to the provided Artifact Repository server. (Optional)') @@ -972,6 +995,9 @@ def parseInput(String[] cliArgs){ if (opts.bp) props.baselinePackageFilePath = opts.bp } + // Track retrieved external dependencies + if (opts.ed) props.externalDependenciesEvidences = opts.ed + // default log encoding if not specified via config passed in via --properties if (!props.fileEncoding) props.fileEncoding = "IBM-1047" @@ -1110,6 +1136,15 @@ def parseCopyModeMap(String copyModeMapString) { return copyModeMap } +/* + * build package url + */ +def computeAbsoluteRepositoryUrl() { + def String remotePath = (props.versionName) ? (props.versionName + "/" + tarFileName) : (tarFileLabel + "/" + tarFileName) + def url = new URI(props.get('artifactRepository.url') + "/" + props.get('artifactRepository.repo') + "/" + (props.get('artifactRepository.directory') ? "${props.get('artifactRepository.directory')}/" : "") + remotePath).normalize().toString() // Normalized URL + return url +} + /* * checksInterfaces - Checks if all interfaces * (public/shared Include Files and submodules) diff --git a/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties b/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties index 4e9aa7a8..9cfa0d6f 100644 --- a/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties +++ b/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties @@ -70,7 +70,7 @@ addExtension=true # Boolean setting to define if the Wazi Deploy Application Manifest file should be generated # Please note that the cli option `generateWaziDeployAppManifest` can override this setting and activate it. # Default: false -generateWaziDeployAppManifest=false +generateWaziDeployAppManifest=true # Boolean setting to define if SBOM based on cycloneDX should be generated # Please note that the cli option `generateSBOM` can override this setting and activate it. diff --git a/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy b/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy index f82d4031..42ca3bfc 100644 --- a/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/WaziDeployManifestGenerator.groovy @@ -2,6 +2,7 @@ import groovy.transform.* import groovy.yaml.YamlBuilder import groovy.yaml.YamlSlurper import com.ibm.dbb.build.report.records.* +import com.ibm.dbb.dependency.* /* * This is a utility method to generate the Wazi Deploy Application Manifest file @@ -87,6 +88,21 @@ def appendArtifactToManifest(DeployableArtifact deployableArtifact, String path, } } } + + if (dependencySetRecord) { + + // init the dependency set object + DependencySet dependencySet = new DependencySet() + + dependencySetRecord.getAllDependencies().each { PhysicalDependency pDep -> + // add the dependencies to the arraylist + dependencySet.value.add(pDep) + } + + artifact.properties.add(dependencySet) + + } + // add type artifact.type = deployableArtifact.deployType @@ -207,6 +223,9 @@ def appendArtifactDeletionToManifest(DeployableArtifact deployableArtifact, Stri } +/** + * Set scm info for application manifest + */ def setScmInfo(HashMap scmInfoMap) { wdManifest.metadata.annotations.scmInfo = new ScmInfo() @@ -215,6 +234,29 @@ def setScmInfo(HashMap scmInfoMap) { } } +/** + * Set package info for application manifest + */ + +def setPackageInfo(HashMap packageInfoMap) { + wdManifest.metadata.annotations.packageInfo = new PackageInfo() + packageInfoMap.each { k, v -> + wdManifest.metadata.annotations.packageInfo."$k" = v + } +} + +/** + * Set external dependencies to metadata/annotations/external_dependencies + */ + +def setExternalDependencies(File externalDependenciesEvidenceFile) { + def yamlSlurper = new YamlSlurper() + if (externalDependenciesEvidenceFile.exists()) { + ArrayList externalDependenciesEvidences = yamlSlurper.parse(externalDependenciesEvidenceFile) + wdManifest.metadata.annotations.external_dependencies = externalDependenciesEvidences + } +} + /** * Write an Wazi Deploy Manifest a YAML file */ @@ -298,6 +340,8 @@ class Annotations { String creationTimestamp ScmInfo scmInfo PackageInfo packageInfo + ArrayList external_dependencies + ArrayList buildInfo } class ScmInfo { @@ -310,7 +354,7 @@ class ScmInfo { class PackageInfo { String name String description - Properties properties + //Properties properties String uri String type } @@ -326,4 +370,24 @@ class Artifact { class ElementProperty { String key String value -} \ No newline at end of file +} +class DependencySet{ + String key = "dependency_set" + ArrayList value = new ArrayList<>() +} + +/** + * * + * External Dependencies + */ + +class ExternalDependency { + String name + String type + HashSet properties = new HashSet<>() +} + +class Property { + String key + String value +}