diff --git a/Setup.sh b/Setup.sh index dcb0551..6277583 100755 --- a/Setup.sh +++ b/Setup.sh @@ -83,15 +83,20 @@ DBB_MODELER_APPMAPPINGS_DIR="$DBB_MODELER_WORK/applications-mappings" # Reference to the repository paths mapping file REPOSITORY_PATH_MAPPING_FILE=$DBB_MODELER_WORK/repositoryPathsMapping.yaml # Reference to the type mapping file -APPLICATION_MEMBER_TYPE_MAPPING=$DBB_MODELER_WORK/types.txt +APPLICATION_MEMBER_TYPE_MAPPING=$DBB_MODELER_WORK/typesMapping.yaml # Reference to the type configuration file to generate build configuration -TYPE_CONFIGURATIONS_FILE=$DBB_MODELER_WORK/typesConfigurations.yaml +TYPE_CONFIGURATIONS_FILE= # Input files and configuration # APPLICATION_DATASETS=DBEHM.MIG.COBOL,DBEHM.MIG.COPY,DBEHM.MIG.BMS APPLICATION_ARTIFACTS_HLQ=DBEHM.MIG # Scanning options SCAN_DATASET_MEMBERS=false SCAN_DATASET_MEMBERS_ENCODING=IBM-1047 + +# Build Framework to use. Either zBuilder or zAppBuild +BUILD_FRAMEWORK=zBuilder +# Build Framework to use. Either zBuilder or zAppBuild +DBB_ZBUILDER=/var/dbb/zBuilder # Reference to zAppBuild DBB_ZAPPBUILD=/var/dbb/dbb-zappbuild # Reference to DBB Community Repo @@ -129,7 +134,7 @@ PIPELINE_CI=1 # Arrays for configuration parameters, that will the Setup script will prompt the user for path_config_array=(DBB_MODELER_APPCONFIG_DIR DBB_MODELER_APPLICATION_DIR DBB_MODELER_LOGS DBB_MODELER_DEFAULT_GIT_CONFIG) -input_array=(DBB_MODELER_APPMAPPINGS_DIR REPOSITORY_PATH_MAPPING_FILE APPLICATION_MEMBER_TYPE_MAPPING TYPE_CONFIGURATIONS_FILE APPLICATION_ARTIFACTS_HLQ SCAN_DATASET_MEMBERS SCAN_DATASET_MEMBERS_ENCODING DBB_ZAPPBUILD DBB_COMMUNITY_REPO APPLICATION_DEFAULT_BRANCH INTERACTIVE_RUN PUBLISH_ARTIFACTS ARTIFACT_REPOSITORY_SERVER_URL ARTIFACT_REPOSITORY_USER ARTIFACT_REPOSITORY_PASSWORD ARTIFACT_REPOSITORY_SUFFIX PIPELINE_USER PIPELINE_USER_GROUP) +input_array=(DBB_MODELER_APPMAPPINGS_DIR REPOSITORY_PATH_MAPPING_FILE APPLICATION_MEMBER_TYPE_MAPPING TYPE_CONFIGURATIONS_FILE APPLICATION_ARTIFACTS_HLQ SCAN_DATASET_MEMBERS SCAN_DATASET_MEMBERS_ENCODING DBB_COMMUNITY_REPO APPLICATION_DEFAULT_BRANCH INTERACTIVE_RUN PUBLISH_ARTIFACTS ARTIFACT_REPOSITORY_SERVER_URL ARTIFACT_REPOSITORY_USER ARTIFACT_REPOSITORY_PASSWORD ARTIFACT_REPOSITORY_SUFFIX PIPELINE_USER PIPELINE_USER_GROUP) # Create work dir echo @@ -147,6 +152,34 @@ if [[ -z "$variable" || $variable =~ ^[Yy]$ ]]; then fi fi +if [ $rc -eq 0 ]; then + # Specify DBB Build Framework and related options + echo + echo "[SETUP] Specifying the Build Framework configuration" + read -p "Specify the type of the Build Framework to use with DBB ("zBuilder" or "zAppBuild") [default: ${BUILD_FRAMEWORK}]: " variable + if [ "$variable" ]; then + declare BUILD_FRAMEWORK="${variable}" + fi + TYPE_CONFIGURATIONS_FILE=$DBB_MODELER_WORK/typesConfigurations-$BUILD_FRAMEWORK.yaml + + if [ "$BUILD_FRAMEWORK" = "zBuilder" ]; then + read -p "Specify the location of the DBB zBuilder installation [default: ${DBB_ZBUILDER}]: " variable + if [ "$variable" ]; then + declare DBB_ZBUILDER="${variable}" + fi + elif [ "$BUILD_FRAMEWORK" = "zAppBuild" ]; then + read -p "Specify the location of the zAppBuild installation [default: ${DBB_ZAPPBUILD}]: " variable + if [ "$variable" ]; then + declare DBB_ZAPPBUILD="${variable}" + fi + else + rc=8 + ERRMSG="[ERROR] The value ${variable} was specified and is invalid (either 'zBuilder' or 'zAppBuild' are valid options). Exiting. rc="$rc + echo $ERRMSG + fi +fi + + if [ $rc -eq 0 ]; then # Specify DBB Metadatastore type and config echo @@ -161,9 +194,7 @@ if [ $rc -eq 0 ]; then if [ "$variable" ]; then declare DBB_MODELER_FILE_METADATA_STORE_DIR="${variable}" fi - fi - - if [ "$DBB_MODELER_METADATASTORE_TYPE" = "db2" ]; then + elif [ "$DBB_MODELER_METADATASTORE_TYPE" = "db2" ]; then read -p "Specify the location of the DBB Db2 Metadatastore configuration file [default: ${DBB_MODELER_DB2_METADATASTORE_CONFIG_FILE}]: " variable if [ "$variable" ]; then declare DBB_MODELER_DB2_METADATASTORE_CONFIG_FILE="${variable}" @@ -185,6 +216,10 @@ if [ $rc -eq 0 ]; then rm -rf $DBB_MODELER_WORK exit 1 fi + else + rc=8 + ERRMSG="[ERROR] The value ${variable} was specified and is invalid (either 'file' or 'db2' are valid options). Exiting. rc="$rc + echo $ERRMSG fi fi @@ -235,11 +270,15 @@ if [ $rc -eq 0 ]; then rc=$? fi if [ $rc -eq 0 ]; then - cp $DBB_MODELER_HOME/samples/types.txt $APPLICATION_MEMBER_TYPE_MAPPING + cp $DBB_MODELER_HOME/samples/typesMapping.yaml $APPLICATION_MEMBER_TYPE_MAPPING rc=$? fi if [ $rc -eq 0 ]; then - cp $DBB_MODELER_HOME/samples/typesConfigurations.yaml $TYPE_CONFIGURATIONS_FILE + if [ "$BUILD_FRAMEWORK" = "zBuilder" ]; then + cp $DBB_MODELER_HOME/samples/typesConfigurations-zBuilder.yaml $TYPE_CONFIGURATIONS_FILE + else + cp $DBB_MODELER_HOME/samples/typesConfigurations.yaml $TYPE_CONFIGURATIONS_FILE + fi rc=$? fi if [ $rc -eq 0 ]; then @@ -283,6 +322,12 @@ if [ $rc -eq 0 ]; then echo "${config}=${!config}" >> $DBB_GIT_MIGRATION_MODELER_CONFIG_FILE done + echo "" >> $DBB_GIT_MIGRATION_MODELER_CONFIG_FILE + echo "# DBB Git Migration Modeler - Build Framework configuration" >> $DBB_GIT_MIGRATION_MODELER_CONFIG_FILE + echo "BUILD_FRAMEWORK=${BUILD_FRAMEWORK}" >> $DBB_GIT_MIGRATION_MODELER_CONFIG_FILE + echo "DBB_ZBUILDER=${DBB_ZBUILDER}" >> $DBB_GIT_MIGRATION_MODELER_CONFIG_FILE + echo "DBB_ZAPPBUILD=${DBB_ZAPPBUILD}" >> $DBB_GIT_MIGRATION_MODELER_CONFIG_FILE + echo "" >> $DBB_GIT_MIGRATION_MODELER_CONFIG_FILE echo "# DBB Git Migration Modeler - DBB Metadatastore configuration" >> $DBB_GIT_MIGRATION_MODELER_CONFIG_FILE echo "DBB_MODELER_METADATASTORE_TYPE=${DBB_MODELER_METADATASTORE_TYPE}" >> $DBB_GIT_MIGRATION_MODELER_CONFIG_FILE diff --git a/samples/git-config/.gitattributes b/samples/git-config/.gitattributes index e005e38..7562051 100644 --- a/samples/git-config/.gitattributes +++ b/samples/git-config/.gitattributes @@ -33,7 +33,7 @@ server.xml zos-working-tree-encoding=iso8859-1 git-encoding=utf-8 *.plbck binary *.md zos-working-tree-encoding=IBM-1047 git-encoding=utf-8 *.yml zos-working-tree-encoding=IBM-1047 git-encoding=utf-8 -*.yaml zos-working-tree-encoding=IBM-1047 git-encoding=utf-8 +*.yaml zos-working-tree-encoding=utf-8 git-encoding=utf-8 *.md zos-working-tree-encoding=utf-8 git-encoding=utf-8 .gitattributes zos-working-tree-encoding=ISO8859-1 git-encoding=utf-8 diff --git a/samples/git-config/dbb-app.yaml b/samples/git-config/dbb-app.yaml new file mode 100644 index 0000000..34adab7 --- /dev/null +++ b/samples/git-config/dbb-app.yaml @@ -0,0 +1,22 @@ +--- +# +# Licensed materials - Property of IBM +# 5655-AC5 Copyright IBM Corp. 2024 +# All rights reserved +# US Government users restricted rights - Use, duplication or +# disclosure restricted by GSA ADP schedule contract with IBM Corp. +# +version: 1.0.0 +application: + name: + tasks: + # Variable overrides for the ImpactAnalysis task + - task: ImpactAnalysis + variables: + - name: impactQueryPatterns + value: + + - task: ScannerInit + variables: + - name: loadModuleExcludeList + value: "${SCEELKED}.*,${SDFHLOAD}.*" \ No newline at end of file diff --git a/samples/types.txt b/samples/types.txt deleted file mode 100644 index 926b29b..0000000 --- a/samples/types.txt +++ /dev/null @@ -1,4 +0,0 @@ -LGACDB01, CBLCICSDB2 -LGACDB02, CBLDB2 -LGACUS01, PLICICS - diff --git a/samples/typesConfigurations.yaml b/samples/typesConfigurations-zAppBuild.yaml similarity index 100% rename from samples/typesConfigurations.yaml rename to samples/typesConfigurations-zAppBuild.yaml diff --git a/samples/typesConfigurations-zBuilder.yaml b/samples/typesConfigurations-zBuilder.yaml new file mode 100644 index 0000000..856b5d7 --- /dev/null +++ b/samples/typesConfigurations-zBuilder.yaml @@ -0,0 +1,29 @@ +typesConfigurations: + - typeConfiguration: "PLI" + tasks: + - "Pli" + variables: + - name: compileParms + value: "PP(INCLUDE('ID(++INCLUDE)'))" + - name: linkedit + value: true + - typeConfiguration: "CBLDB2" + tasks: + - "Cobol" + variables: + - name: compileParms + value: "LIB,SOURCE,SQL" + - name: isSQL + value: true + - name: linkedit + value: true + - typeConfiguration: "CICS" + tasks: + - "Pli" + - "Cobol" + variables: + - name: isCICS + value: true + - name: compileParms + value: "${compileParms},CICS" + \ No newline at end of file diff --git a/samples/typesMapping.yaml b/samples/typesMapping.yaml new file mode 100644 index 0000000..ed45c91 --- /dev/null +++ b/samples/typesMapping.yaml @@ -0,0 +1,11 @@ +types: + - type: "CICS" + filePatterns: + - "LGACDB01" + - "LGACUS01" + - type: "CBLDB2" + filePatterns: + - "LGACDB.." + - type: "PLI" + filePatterns: + - "LGACUS01" diff --git a/src/groovy/extractApplications.groovy b/src/groovy/extractApplications.groovy index 086f4df..e466dc0 100644 --- a/src/groovy/extractApplications.groovy +++ b/src/groovy/extractApplications.groovy @@ -44,11 +44,12 @@ import java.text.DecimalFormat //////@Field HashMap> membersToDatasetsMap = new HashMap>() @Field HashMap membersToDatasetsMap = new HashMap() -// Types Configurations -@Field HashMap types +//@Field HashMap types // script properties @Field Properties props = new Properties() @Field repositoryPathsMapping +// Types Configurations +@Field typesMapping @Field Dmh5210 scanner HashMap storageRequirements = new HashMap() @@ -64,89 +65,90 @@ parseArgs(args) // Read the repository layout mapping file logger.logMessage("** Reading the Repository Layout Mapping definition.") if (props.REPOSITORY_PATH_MAPPING_FILE) { - File repositoryPathsMappingFile = new File(props.REPOSITORY_PATH_MAPPING_FILE) - if (!repositoryPathsMappingFile.exists()) { - logger.logMessage("*! [WARNING] The Repository Path Mapping file ${props.REPOSITORY_PATH_MAPPING_FILE} was not found. Exiting.") - System.exit(1) - } else { - def yamlSlurper = new groovy.yaml.YamlSlurper() - repositoryPathsMapping = yamlSlurper.parse(repositoryPathsMappingFile) - } + File repositoryPathsMappingFile = new File(props.REPOSITORY_PATH_MAPPING_FILE) + if (!repositoryPathsMappingFile.exists()) { + logger.logMessage("*! [WARNING] The Repository Path Mapping file ${props.REPOSITORY_PATH_MAPPING_FILE} was not found. Exiting.") + System.exit(1) + } else { + def yamlSlurper = new groovy.yaml.YamlSlurper() + repositoryPathsMapping = yamlSlurper.parse(repositoryPathsMappingFile) + } } // Read the Types from file -logger.logMessage("** Reading the Type Mapping definition.") +logger.logMessage("** Reading the Types Mapping definition.") if (props.APPLICATION_MEMBER_TYPE_MAPPING) { - def typesFile = new File(props.APPLICATION_MEMBER_TYPE_MAPPING) - if (!typesFile.exists()) { - logger.logMessage("*! [WARNING] File ${props.APPLICATION_MEMBER_TYPE_MAPPING} not found in the current working directory. All artifacts will use the 'UNKNOWN' type.") - } else { - types = loadMapFromFile(props.APPLICATION_MEMBER_TYPE_MAPPING) - } + def typesMappingFile = new File(props.APPLICATION_MEMBER_TYPE_MAPPING) + if (!typesMappingFile.exists()) { + logger.logMessage("*! [WARNING] File ${props.APPLICATION_MEMBER_TYPE_MAPPING} not found in the current working directory. All artifacts will use the 'UNKNOWN' type.") + } else { + def yamlSlurper = new groovy.yaml.YamlSlurper() + typesMapping = yamlSlurper.parse(typesMappingFile) + } } else { - logger.logMessage("*! [WARNING] No Types File provided. The 'UNKNOWN' type will be assigned by default to all artifacts.") + logger.logMessage("*! [WARNING] No Types Mapping file provided. The 'UNKNOWN' type will be assigned by default to all artifacts.") } logger.logMessage("** Loading the provided Applications Mapping files.") File applicationsMappingsDir = new File(props.DBB_MODELER_APPMAPPINGS_DIR) applicationsMappingsDir.eachFile(FILES) { applicationsMappingFile -> - logger.logMessage("*** Importing '${applicationsMappingFile.getName()}'") - def yamlSlurper = new groovy.yaml.YamlSlurper() - applicationsMapping = yamlSlurper.parse(applicationsMappingFile) - applicationsMapping.datasets.each() { dataset -> - ArrayList applicationsList = datasetsMap.get(dataset) - if (!applicationsList) { - applicationsList = new ArrayList() - datasetsMap.put(dataset, applicationsList) - } - applicationsMapping.applications.each() { application -> - applicationsList.add(application) - } - } + logger.logMessage("*** Importing '${applicationsMappingFile.getName()}'") + def yamlSlurper = new groovy.yaml.YamlSlurper() + applicationsMapping = yamlSlurper.parse(applicationsMappingFile) + applicationsMapping.datasets.each() { dataset -> + ArrayList applicationsList = datasetsMap.get(dataset) + if (!applicationsList) { + applicationsList = new ArrayList() + datasetsMap.put(dataset, applicationsList) + } + applicationsMapping.applications.each() { application -> + applicationsList.add(application) + } + } } logger.logMessage("** Iterating through the provided datasets and mapped applications.") datasetsMap.each() { dataset, applicationsList -> - String qdsn = constructPDSForZFileOperation(dataset) - if (ZFile.dsExists(qdsn)) { - def applications = applicationsList.collect { "'${it.application}'" } - logger.logMessage("**** Found '$dataset' referenced by applications ${applications.toString().replaceAll("\\[|\\]", "")}"); - try { - PdsDirectory directoryList = new PdsDirectory(qdsn) - Iterator directoryListIterator = directoryList.iterator(); - while (directoryListIterator.hasNext()) { - PdsDirectory.MemberInfo memberInfo = (PdsDirectory.MemberInfo) directoryListIterator.next(); - String member = (memberInfo.getName()); - addDatasetToMember(member, dataset) - def mappedApplication = findMappedApplicationFromMemberName(applicationsList, member) - logger.logMessage("***** '$dataset($member)' - Mapped Application: $mappedApplication"); - addDatasetMemberToApplication(mappedApplication, "$dataset($member)") - } - directoryList.close(); - } - catch (java.io.IOException exception) { - logger.logMessage("*! [ERROR] Problem when accessing the dataset '$qdsn'."); - } - } - else { - logger.logMessage("*! [ERROR] Dataset '$qdsn' does not exist."); - } + String qdsn = constructPDSForZFileOperation(dataset) + if (ZFile.dsExists(qdsn)) { + def applications = applicationsList.collect { "'${it.application}'" } + logger.logMessage("**** Found '$dataset' referenced by applications ${applications.toString().replaceAll("\\[|\\]", "")}"); + try { + PdsDirectory directoryList = new PdsDirectory(qdsn) + Iterator directoryListIterator = directoryList.iterator(); + while (directoryListIterator.hasNext()) { + PdsDirectory.MemberInfo memberInfo = (PdsDirectory.MemberInfo) directoryListIterator.next(); + String member = (memberInfo.getName()); + addDatasetToMember(member, dataset) + def mappedApplication = findMappedApplicationFromMemberName(applicationsList, member) + logger.logMessage("***** '$dataset($member)' - Mapped Application: $mappedApplication"); + addDatasetMemberToApplication(mappedApplication, "$dataset($member)") + } + directoryList.close(); + } + catch (java.io.IOException exception) { + logger.logMessage("*! [ERROR] Problem when accessing the dataset '$qdsn'."); + } + } + else { + logger.logMessage("*! [ERROR] Dataset '$qdsn' does not exist."); + } } DecimalFormat df = new DecimalFormat("###,###,###,###") logger.logMessage("** Generating Applications Configurations files.") applicationsToDatasetMembersMap.each() { application, members -> - logger.logMessage("** Generating Configuration files for application $application.") - generateApplicationFiles(application) - storageRequirements.put(application, calculateStorageSizeForMembers(members)) - - logger.logMessage("\tEstimated storage size of migrated members: ${df.format(storageRequirements.get(application))} bytes") + logger.logMessage("** Generating Configuration files for application $application.") + generateApplicationFiles(application) + storageRequirements.put(application, calculateStorageSizeForMembers(members)) + + logger.logMessage("\tEstimated storage size of migrated members: ${df.format(storageRequirements.get(application))} bytes") } def globalStorageRequirements = 0 storageRequirements.each() { application, storageRequirement -> - globalStorageRequirements = globalStorageRequirements + storageRequirement + globalStorageRequirements = globalStorageRequirements + storageRequirement } logger.logMessage("** Estimated storage size of all migrated members: ${df.format(globalStorageRequirements)} bytes") @@ -157,356 +159,361 @@ logger.close() /* parseArgs: parse arguments provided through CLI */ def parseArgs(String[] args) { - Properties configuration = new Properties() - String usage = 'extractApplications.groovy [options]' - String header = 'options:' - def cli = new CliBuilder(usage:usage,header:header) - cli.c(longOpt:'configFile', args:1, required:true, 'Path to the DBB Git Migration Modeler Configuration file (created by the Setup script)') - cli.l(longOpt:'logFile', args:1, required:false, 'Relative or absolute path to an output log file') - - def opts = cli.parse(args) - if (!args || !opts) { - cli.usage() - System.exit(1) - } - - if (opts.l) { - props.logFile = opts.l - logger.create(props.logFile) - } - - if (opts.c) { - props.configurationFilePath = opts.c - File configurationFile = new File(props.configurationFilePath) - if (configurationFile.exists()) { - configurationFile.withReader() { reader -> - configuration.load(reader) - } - } else { - logger.logMessage("*! [ERROR] The DBB Git Migration Modeler Configuration file '${opts.c}' does not exist. Exiting.") - System.exit(1) - } - } else { - logger.logMessage("*! [ERROR] The path to the DBB Git Migration Modeler Configuration file was not specified ('-c/--configFile' parameter). Exiting.") - System.exit(1) - } - - if (configuration.DBB_MODELER_APPCONFIG_DIR) { - File directory = new File(configuration.DBB_MODELER_APPCONFIG_DIR) - if (directory.exists()) { - props.DBB_MODELER_APPCONFIG_DIR = configuration.DBB_MODELER_APPCONFIG_DIR - } else { - logger.logMessage("*! [ERROR] The Configurations directory '${configuration.DBB_MODELER_APPCONFIG_DIR}' does not exist. Exiting.") - System.exit(1) - } - } else { - logger.logMessage("*! [ERROR] The Configurations directory must be specified in the DBB Git Migration Modeler Configuration file. Exiting.") - System.exit(1) - } - - if (configuration.DBB_MODELER_APPMAPPINGS_DIR) { - File directory = new File(configuration.DBB_MODELER_APPMAPPINGS_DIR) - if (directory.exists()) { - props.DBB_MODELER_APPMAPPINGS_DIR = configuration.DBB_MODELER_APPMAPPINGS_DIR - } else { - logger.logMessage("*! [ERROR] The Applications Mappings directory '${configuration.DBB_MODELER_APPMAPPINGS_DIR}' does not exist. Exiting.") - System.exit(1) - } - } else { - logger.logMessage("*! [ERROR] The Applications Mappings directory must be specified in the DBB Git Migration Modeler Configuration file. Exiting.") - System.exit(1) - } - - - if (configuration.DBB_MODELER_APPLICATION_DIR) { - File directory = new File(configuration.DBB_MODELER_APPLICATION_DIR) - if (directory.exists()) { - props.DBB_MODELER_APPLICATION_DIR = configuration.DBB_MODELER_APPLICATION_DIR - } else { - logger.logMessage("*! [ERROR] The Applications directory '${configuration.DBB_MODELER_APPLICATION_DIR}' does not exist. Exiting.") - System.exit(1) - } - } else { - logger.logMessage("*! [ERROR] The Applications directory must be specified in the DBB Git Migration Modeler Configuration file. Exiting.") - System.exit(1) - } - - if (configuration.REPOSITORY_PATH_MAPPING_FILE) { - File file = new File(configuration.REPOSITORY_PATH_MAPPING_FILE) - if (file.exists()) { - props.REPOSITORY_PATH_MAPPING_FILE = configuration.REPOSITORY_PATH_MAPPING_FILE - } else { - logger.logMessage("*! [ERROR] The Repository Paths Mapping file '${configuration.REPOSITORY_PATH_MAPPING_FILE}' does not exist. Exiting.") - System.exit(1) - } - } else { - logger.logMessage("*! [ERROR] The path to the Repository Paths Mapping file must be specified in the DBB Git Migration Modeler Configuration file. Exiting.") - System.exit(1) - } - - if (configuration.APPLICATION_MEMBER_TYPE_MAPPING) { - File file = new File(configuration.APPLICATION_MEMBER_TYPE_MAPPING) - if (file.exists()) { - props.APPLICATION_MEMBER_TYPE_MAPPING = configuration.APPLICATION_MEMBER_TYPE_MAPPING - } else { - logger.logMessage("*! [ERROR] The Types file '${configuration.APPLICATION_MEMBER_TYPE_MAPPING}' does not exist. Exiting.") - System.exit(1) - } - } - - if (configuration.SCAN_DATASET_MEMBERS) { - props.SCAN_DATASET_MEMBERS = configuration.SCAN_DATASET_MEMBERS - if (configuration.SCAN_DATASET_MEMBERS_ENCODING) { - props.SCAN_DATASET_MEMBERS_ENCODING = configuration.SCAN_DATASET_MEMBERS_ENCODING - } else { - props.SCAN_DATASET_MEMBERS_ENCODING = "IBM-1047" - } - } else { - props.SCAN_DATASET_MEMBERS = "false" - } - - logger.logMessage("** Script configuration:") - props.each() { k, v -> - logger.logMessage("\t$k -> $v") - } + Properties configuration = new Properties() + String usage = 'extractApplications.groovy [options]' + String header = 'options:' + def cli = new CliBuilder(usage:usage,header:header) + cli.c(longOpt:'configFile', args:1, required:true, 'Path to the DBB Git Migration Modeler Configuration file (created by the Setup script)') + cli.l(longOpt:'logFile', args:1, required:false, 'Relative or absolute path to an output log file') + + def opts = cli.parse(args) + if (!args || !opts) { + cli.usage() + System.exit(1) + } + + if (opts.l) { + props.logFile = opts.l + logger.create(props.logFile) + } + + if (opts.c) { + props.configurationFilePath = opts.c + File configurationFile = new File(props.configurationFilePath) + if (configurationFile.exists()) { + configurationFile.withReader() { reader -> + configuration.load(reader) + } + } else { + logger.logMessage("*! [ERROR] The DBB Git Migration Modeler Configuration file '${opts.c}' does not exist. Exiting.") + System.exit(1) + } + } else { + logger.logMessage("*! [ERROR] The path to the DBB Git Migration Modeler Configuration file was not specified ('-c/--configFile' parameter). Exiting.") + System.exit(1) + } + + if (configuration.DBB_MODELER_APPCONFIG_DIR) { + File directory = new File(configuration.DBB_MODELER_APPCONFIG_DIR) + if (directory.exists()) { + props.DBB_MODELER_APPCONFIG_DIR = configuration.DBB_MODELER_APPCONFIG_DIR + } else { + logger.logMessage("*! [ERROR] The Configurations directory '${configuration.DBB_MODELER_APPCONFIG_DIR}' does not exist. Exiting.") + System.exit(1) + } + } else { + logger.logMessage("*! [ERROR] The Configurations directory must be specified in the DBB Git Migration Modeler Configuration file. Exiting.") + System.exit(1) + } + + if (configuration.DBB_MODELER_APPMAPPINGS_DIR) { + File directory = new File(configuration.DBB_MODELER_APPMAPPINGS_DIR) + if (directory.exists()) { + props.DBB_MODELER_APPMAPPINGS_DIR = configuration.DBB_MODELER_APPMAPPINGS_DIR + } else { + logger.logMessage("*! [ERROR] The Applications Mappings directory '${configuration.DBB_MODELER_APPMAPPINGS_DIR}' does not exist. Exiting.") + System.exit(1) + } + } else { + logger.logMessage("*! [ERROR] The Applications Mappings directory must be specified in the DBB Git Migration Modeler Configuration file. Exiting.") + System.exit(1) + } + + + if (configuration.DBB_MODELER_APPLICATION_DIR) { + File directory = new File(configuration.DBB_MODELER_APPLICATION_DIR) + if (directory.exists()) { + props.DBB_MODELER_APPLICATION_DIR = configuration.DBB_MODELER_APPLICATION_DIR + } else { + logger.logMessage("*! [ERROR] The Applications directory '${configuration.DBB_MODELER_APPLICATION_DIR}' does not exist. Exiting.") + System.exit(1) + } + } else { + logger.logMessage("*! [ERROR] The Applications directory must be specified in the DBB Git Migration Modeler Configuration file. Exiting.") + System.exit(1) + } + + if (configuration.REPOSITORY_PATH_MAPPING_FILE) { + File file = new File(configuration.REPOSITORY_PATH_MAPPING_FILE) + if (file.exists()) { + props.REPOSITORY_PATH_MAPPING_FILE = configuration.REPOSITORY_PATH_MAPPING_FILE + } else { + logger.logMessage("*! [ERROR] The Repository Paths Mapping file '${configuration.REPOSITORY_PATH_MAPPING_FILE}' does not exist. Exiting.") + System.exit(1) + } + } else { + logger.logMessage("*! [ERROR] The path to the Repository Paths Mapping file must be specified in the DBB Git Migration Modeler Configuration file. Exiting.") + System.exit(1) + } + + if (configuration.APPLICATION_MEMBER_TYPE_MAPPING) { + File file = new File(configuration.APPLICATION_MEMBER_TYPE_MAPPING) + if (file.exists()) { + props.APPLICATION_MEMBER_TYPE_MAPPING = configuration.APPLICATION_MEMBER_TYPE_MAPPING + } else { + logger.logMessage("*! [ERROR] The Types file '${configuration.APPLICATION_MEMBER_TYPE_MAPPING}' does not exist. Exiting.") + System.exit(1) + } + } + + if (configuration.SCAN_DATASET_MEMBERS) { + props.SCAN_DATASET_MEMBERS = configuration.SCAN_DATASET_MEMBERS + if (configuration.SCAN_DATASET_MEMBERS_ENCODING) { + props.SCAN_DATASET_MEMBERS_ENCODING = configuration.SCAN_DATASET_MEMBERS_ENCODING + } else { + props.SCAN_DATASET_MEMBERS_ENCODING = "IBM-1047" + } + } else { + props.SCAN_DATASET_MEMBERS = "false" + } + + logger.logMessage("** Script configuration:") + props.each() { k, v -> + logger.logMessage("\t$k -> $v") + } } def constructPDSForZFileOperation(String PDS) { - return "//'${PDS}'" + return "//'${PDS}'" } def constructDatasetForZFileOperation(String PDS, String member) { - return "//'${PDS}($member)'" + return "//'${PDS}($member)'" } def isFilterOnMemberMatching(String memberName, String filter) { - StringBuilder expandedMemberNameStringBuilder = new StringBuilder(memberName); - while (expandedMemberNameStringBuilder.length() < 8) { - expandedMemberNameStringBuilder.append('.'); - } - String expandedMemberName = expandedMemberNameStringBuilder.toString(); - - StringBuilder expandedFilterStringBuilder = new StringBuilder(filter); - while (expandedFilterStringBuilder.length() < 8) { - expandedFilterStringBuilder.append('.'); - } - String expandedFilter = expandedFilterStringBuilder.toString(); - - StringBuilder result = new StringBuilder(); - int i = 0; - while (i < expandedMemberName.length() && i < 8) { - if (expandedFilter[i] != '.') { - result.append(expandedMemberName[i]) - } else { - result.append('.') - } - i++; - } - return result.toString().equalsIgnoreCase(expandedFilter); + StringBuilder expandedMemberNameStringBuilder = new StringBuilder(memberName); + while (expandedMemberNameStringBuilder.length() < 8) { + expandedMemberNameStringBuilder.append('.'); + } + String expandedMemberName = expandedMemberNameStringBuilder.toString(); + + StringBuilder expandedFilterStringBuilder = new StringBuilder(filter); + while (expandedFilterStringBuilder.length() < 8) { + expandedFilterStringBuilder.append('.'); + } + String expandedFilter = expandedFilterStringBuilder.toString(); + + StringBuilder result = new StringBuilder(); + int i = 0; + while (i < expandedMemberName.length() && i < 8) { + if (expandedFilter[i] != '.') { + result.append(expandedMemberName[i]) + } else { + result.append('.') + } + i++; + } + return result.toString().equalsIgnoreCase(expandedFilter); } +/* def isFilterOnMemberMatching(String memberName, String filter) { + println("INSIDE match - $memberName - $filter") + return memberName.matches(filter) +} */ + def generateApplicationFiles(String application) { - // If an existing DBB Migration Mapping file already exists in the CONFIG directory, - // we read it and store it into a HashMap where the key in the input dataset member - File mappingFile = new File(props.DBB_MODELER_APPCONFIG_DIR + '/' + application + ".mapping"); - HashMap mappings = new HashMap() - if (mappingFile.exists()) { - BufferedReader mappingReader = new BufferedReader(new FileReader(mappingFile)) - String line; - while((line = mappingReader.readLine()) != null) { - def lineSegments = line.split(' ') - mappings.put(lineSegments[0], lineSegments.tail().join(" ")) - } - mappingReader.close() - } - - // If an existing Application Descriptor file already exists in the CONFIG directory, - // we read it into an Application Descriptor object - File applicationDescriptorFile = new File(props.DBB_MODELER_APPCONFIG_DIR + '/' + application + ".yml") - def applicationDescriptor - if (applicationDescriptorFile.exists()) { - applicationDescriptor = applicationDescriptorUtils.readApplicationDescriptor(applicationDescriptorFile) - } else { - applicationDescriptor = applicationDescriptorUtils.createEmptyApplicationDescriptor() - } - - // Trying to find information about the application we are dealing with - foundApplication = findApplication(application) - if (foundApplication != null) { - applicationDescriptor.application = foundApplication.application - applicationDescriptor.description = foundApplication.description - applicationDescriptor.owner = foundApplication.owner - // Adding baseline to ApplicationDescriptor - applicationDescriptorUtils.addBaseline(applicationDescriptor, "main", "release", foundApplication.baseline) - applicationDescriptorUtils.addBaseline(applicationDescriptor, "release/${foundApplication.baseline}", "release", foundApplication.baseline) - } else { - applicationDescriptor.application = "UNASSIGNED" - applicationDescriptor.description = "Unassigned components" - applicationDescriptor.owner = "None" - applicationDescriptorUtils.addBaseline(applicationDescriptor, "main", "release", "rel-1.0.0") - applicationDescriptorUtils.addBaseline(applicationDescriptor, "release/rel-1.0.0", "release", "rel-1.0.0") - } - - // Main loop, iterating through the dataset members assigned to the current application - def datasetMembersCollection = applicationsToDatasetMembersMap.get(application) - // For each dataset member... - datasetMembersCollection.each () { datasetMember -> - // Get the dataset and the member separated - def (dataset, member) = getDatasetAndMember(datasetMember) - // Using the DBB Scanner if activated - String scannedLanguage, scannedFileType - if (props.SCAN_DATASET_MEMBERS && props.SCAN_DATASET_MEMBERS.toBoolean()) { - (scannedLanguage, scannedFileType) = scanDatasetMember(constructDatasetForZFileOperation(dataset, member)) - } - def lastQualifier = getLastQualifier(dataset) - def memberType = getType(member) - // Identifying the matching Repository Path - // based on 1) the scan result if enabled - // 2) the type if set - // 3) the last level qualifier of the containing dataset - def matchingRepositoryPath = repositoryPathsMapping.repositoryPaths.find {repositoryPath -> - (props.SCAN_DATASET_MEMBERS && props.SCAN_DATASET_MEMBERS.toBoolean() && repositoryPath.mvsMapping.scan ? repositoryPath.mvsMapping.scan.language.equals(scannedLanguage) && repositoryPath.mvsMapping.scan.fileType.equals(scannedFileType) : false) || - (repositoryPath.mvsMapping.types ? repositoryPath.mvsMapping.types.contains(memberType) : false) || - (repositoryPath.mvsMapping.datasetLastLevelQualifiers ? repositoryPath.mvsMapping.datasetLastLevelQualifiers.contains(lastQualifier) : false) - } - - def targetRepositoryPath - def pdsEncoding - def fileExtension - def artifactsType - def sourceGroup - def language - def languageProcessor - if (matchingRepositoryPath) { - // if Matching Repository Path found, we retrieve the information from the RepositoryPathsMapping file - if (matchingRepositoryPath.toLowerCase && matchingRepositoryPath.toLowerCase.toBoolean()) { - member = member.toLowerCase() - lastQualifier = lastQualifier.toLowerCase() - } - fileExtension = (matchingRepositoryPath.fileExtension) ? (matchingRepositoryPath.fileExtension) : lastQualifier - sourceGroup = (matchingRepositoryPath.sourceGroup) ? (matchingRepositoryPath.sourceGroup) : lastQualifier - language = (matchingRepositoryPath.language) ? (matchingRepositoryPath.language) : lastQualifier - languageProcessor = (matchingRepositoryPath.languageProcessor) ? (matchingRepositoryPath.languageProcessor) : lastQualifier + ".groovy" - targetRepositoryPath = (matchingRepositoryPath.repositoryPath) ? matchingRepositoryPath.repositoryPath.replaceAll('\\$application',application) : "$application/$lastQualifier" - pdsEncoding = (matchingRepositoryPath.encoding) ? (matchingRepositoryPath.encoding) : "IBM-1047" - artifactsType = (matchingRepositoryPath.artifactsType) ? (matchingRepositoryPath.artifactsType) : lastQualifier - - } else { - // if Matching Repository Path not found, we set default values based on the last qualifier of the dataset name - member = member.toLowerCase() - lastQualifier = lastQualifier.toLowerCase() - fileExtension = lastQualifier - sourceGroup = lastQualifier - language = lastQualifier - languageProcessor = lastQualifier + ".groovy" - targetRepositoryPath = "$application/$lastQualifier" - pdsEncoding = "IBM-1047" - artifactsType = lastQualifier - } - // Appending the dataset member to the Application Descriptor file - applicationDescriptorUtils.appendFileDefinition(applicationDescriptor, sourceGroup, language, languageProcessor, artifactsType, fileExtension, targetRepositoryPath, member, memberType, "undefined") - // Adding a line into the DBB Migration Mapping file - targetRepositoryPath = props.DBB_MODELER_APPLICATION_DIR + "/" + application + "/" + targetRepositoryPath - mappings.put(datasetMember, "$targetRepositoryPath/$member.$fileExtension pdsEncoding=$pdsEncoding") - } - - // Writing the DBB Migration Mapping file - try { - mappingFile.withWriter("IBM-1047") { writer -> - mappings.each() { datasetMember, target -> - writer.write("$datasetMember $target\n"); - } - } - Process process = "chtag -tc IBM-1047 ${mappingFile.getAbsolutePath()}".execute() - process.waitFor() - - } - catch (IOException e) { - e.printStackTrace(); - } - logger.logMessage("\tCreated DBB Migration Utility mapping file " + mappingFile.getAbsolutePath()); - // Writing the Application Descriptor file - applicationDescriptorUtils.writeApplicationDescriptor(applicationDescriptorFile, applicationDescriptor) - logger.logMessage("\tCreated Application Description file " + applicationDescriptorFile.getAbsolutePath()); + // If an existing DBB Migration Mapping file already exists in the CONFIG directory, + // we read it and store it into a HashMap where the key in the input dataset member + File mappingFile = new File(props.DBB_MODELER_APPCONFIG_DIR + '/' + application + ".mapping"); + HashMap mappings = new HashMap() + if (mappingFile.exists()) { + BufferedReader mappingReader = new BufferedReader(new FileReader(mappingFile)) + String line; + while((line = mappingReader.readLine()) != null) { + def lineSegments = line.split(' ') + mappings.put(lineSegments[0], lineSegments.tail().join(" ")) + } + mappingReader.close() + } + + // If an existing Application Descriptor file already exists in the CONFIG directory, + // we read it into an Application Descriptor object + File applicationDescriptorFile = new File(props.DBB_MODELER_APPCONFIG_DIR + '/' + application + ".yml") + def applicationDescriptor + if (applicationDescriptorFile.exists()) { + applicationDescriptor = applicationDescriptorUtils.readApplicationDescriptor(applicationDescriptorFile) + } else { + applicationDescriptor = applicationDescriptorUtils.createEmptyApplicationDescriptor() + } + + // Trying to find information about the application we are dealing with + foundApplication = findApplication(application) + if (foundApplication != null) { + applicationDescriptor.application = foundApplication.application + applicationDescriptor.description = foundApplication.description + applicationDescriptor.owner = foundApplication.owner + // Adding baseline to ApplicationDescriptor + applicationDescriptorUtils.addBaseline(applicationDescriptor, "main", "release", foundApplication.baseline) + applicationDescriptorUtils.addBaseline(applicationDescriptor, "release/${foundApplication.baseline}", "release", foundApplication.baseline) + } else { + applicationDescriptor.application = "UNASSIGNED" + applicationDescriptor.description = "Unassigned components" + applicationDescriptor.owner = "None" + applicationDescriptorUtils.addBaseline(applicationDescriptor, "main", "release", "rel-1.0.0") + applicationDescriptorUtils.addBaseline(applicationDescriptor, "release/rel-1.0.0", "release", "rel-1.0.0") + } + + // Main loop, iterating through the dataset members assigned to the current application + def datasetMembersCollection = applicationsToDatasetMembersMap.get(application) + // For each dataset member... + datasetMembersCollection.each () { datasetMember -> + // Get the dataset and the member separated + def (dataset, member) = getDatasetAndMember(datasetMember) + // Using the DBB Scanner if activated + String scannedLanguage, scannedFileType + if (props.SCAN_DATASET_MEMBERS && props.SCAN_DATASET_MEMBERS.toBoolean()) { + (scannedLanguage, scannedFileType) = scanDatasetMember(constructDatasetForZFileOperation(dataset, member)) + } + def lastQualifier = getLastQualifier(dataset) + def memberType = getTypes(member) + // Identifying the matching Repository Path + // based on 1) the scan result if enabled + // 2) the type if set + // 3) the last level qualifier of the containing dataset + def matchingRepositoryPath = repositoryPathsMapping.repositoryPaths.find {repositoryPath -> + (props.SCAN_DATASET_MEMBERS && props.SCAN_DATASET_MEMBERS.toBoolean() && repositoryPath.mvsMapping.scan ? repositoryPath.mvsMapping.scan.language.equals(scannedLanguage) && repositoryPath.mvsMapping.scan.fileType.equals(scannedFileType) : false) || + (repositoryPath.mvsMapping.types ? repositoryPath.mvsMapping.types.contains(memberType) : false) || + (repositoryPath.mvsMapping.datasetLastLevelQualifiers ? repositoryPath.mvsMapping.datasetLastLevelQualifiers.contains(lastQualifier) : false) + } + + def targetRepositoryPath + def pdsEncoding + def fileExtension + def artifactsType + def sourceGroup + def language + def languageProcessor + if (matchingRepositoryPath) { + // if Matching Repository Path found, we retrieve the information from the RepositoryPathsMapping file + if (matchingRepositoryPath.toLowerCase && matchingRepositoryPath.toLowerCase.toBoolean()) { + member = member.toLowerCase() + lastQualifier = lastQualifier.toLowerCase() + } + fileExtension = (matchingRepositoryPath.fileExtension) ? (matchingRepositoryPath.fileExtension) : lastQualifier + sourceGroup = (matchingRepositoryPath.sourceGroup) ? (matchingRepositoryPath.sourceGroup) : lastQualifier + language = (matchingRepositoryPath.language) ? (matchingRepositoryPath.language) : lastQualifier + languageProcessor = (matchingRepositoryPath.languageProcessor) ? (matchingRepositoryPath.languageProcessor) : lastQualifier + ".groovy" + targetRepositoryPath = (matchingRepositoryPath.repositoryPath) ? matchingRepositoryPath.repositoryPath.replaceAll('\\$application',application) : "$application/$lastQualifier" + pdsEncoding = (matchingRepositoryPath.encoding) ? (matchingRepositoryPath.encoding) : "IBM-1047" + artifactsType = (matchingRepositoryPath.artifactsType) ? (matchingRepositoryPath.artifactsType) : lastQualifier + + } else { + // if Matching Repository Path not found, we set default values based on the last qualifier of the dataset name + member = member.toLowerCase() + lastQualifier = lastQualifier.toLowerCase() + fileExtension = lastQualifier + sourceGroup = lastQualifier + language = lastQualifier + languageProcessor = lastQualifier + ".groovy" + targetRepositoryPath = "$application/$lastQualifier" + pdsEncoding = "IBM-1047" + artifactsType = lastQualifier + } + // Appending the dataset member to the Application Descriptor file + applicationDescriptorUtils.appendFileDefinition(applicationDescriptor, sourceGroup, language, languageProcessor, artifactsType, fileExtension, targetRepositoryPath, member, memberType, "undefined") + // Adding a line into the DBB Migration Mapping file + targetRepositoryPath = props.DBB_MODELER_APPLICATION_DIR + "/" + application + "/" + targetRepositoryPath + mappings.put(datasetMember, "$targetRepositoryPath/$member.$fileExtension pdsEncoding=$pdsEncoding") + } + + // Writing the DBB Migration Mapping file + try { + mappingFile.withWriter("IBM-1047") { writer -> + mappings.each() { datasetMember, target -> + writer.write("$datasetMember $target\n"); + } + } + Process process = "chtag -tc IBM-1047 ${mappingFile.getAbsolutePath()}".execute() + process.waitFor() + + } + catch (IOException e) { + e.printStackTrace(); + } + logger.logMessage("\tCreated DBB Migration Utility mapping file " + mappingFile.getAbsolutePath()); + // Writing the Application Descriptor file + applicationDescriptorUtils.writeApplicationDescriptor(applicationDescriptorFile, applicationDescriptor) + logger.logMessage("\tCreated Application Description file " + applicationDescriptorFile.getAbsolutePath()); } def addDatasetMemberToApplication(String application, String datasetMember) { - HashSet applicationHashSet = applicationsToDatasetMembersMap.get(application) - if (!application.equals("UNASSIGNED")) { - HashSet unassignedHashSet = applicationsToDatasetMembersMap.get("UNASSIGNED") - if (unassignedHashSet && unassignedHashSet.contains(datasetMember)) { - unassignedHashSet.remove(datasetMember) - } - if (!applicationHashSet) { - applicationHashSet = new HashSet(); - applicationsToDatasetMembersMap.put(application, applicationHashSet); - } - applicationHashSet.add(datasetMember); - } else { - HashSet foundDatasetMembers = new HashSet() - applicationsToDatasetMembersMap.forEach { searchedApplication, searchedApplicationHashSet -> - HashSet foundDatasetMembersInSearchedApplication = searchedApplicationHashSet.findAll { searchedDatasetMember -> - searchedDatasetMember.equals(datasetMember) && !searchedApplication.equals("UNASSIGNED") - } - foundDatasetMembers.addAll(foundDatasetMembersInSearchedApplication) - } - if (!foundDatasetMembers) { - if (!applicationHashSet) { - applicationHashSet = new HashSet(); - applicationsToDatasetMembersMap.put(application, applicationHashSet); - } - applicationHashSet.add(datasetMember); - } - } + HashSet applicationHashSet = applicationsToDatasetMembersMap.get(application) + if (!application.equals("UNASSIGNED")) { + HashSet unassignedHashSet = applicationsToDatasetMembersMap.get("UNASSIGNED") + if (unassignedHashSet && unassignedHashSet.contains(datasetMember)) { + unassignedHashSet.remove(datasetMember) + } + if (!applicationHashSet) { + applicationHashSet = new HashSet(); + applicationsToDatasetMembersMap.put(application, applicationHashSet); + } + applicationHashSet.add(datasetMember); + } else { + HashSet foundDatasetMembers = new HashSet() + applicationsToDatasetMembersMap.forEach { searchedApplication, searchedApplicationHashSet -> + HashSet foundDatasetMembersInSearchedApplication = searchedApplicationHashSet.findAll { searchedDatasetMember -> + searchedDatasetMember.equals(datasetMember) && !searchedApplication.equals("UNASSIGNED") + } + foundDatasetMembers.addAll(foundDatasetMembersInSearchedApplication) + } + if (!foundDatasetMembers) { + if (!applicationHashSet) { + applicationHashSet = new HashSet(); + applicationsToDatasetMembersMap.put(application, applicationHashSet); + } + applicationHashSet.add(datasetMember); + } + } } def findMappedApplicationFromMemberName(ArrayList applicationsList, String memberName) { - // Finding the owning application in the list of applications using the same dataset - def foundApplications = applicationsList.findAll { application -> - application.namingConventions.find { namingConvention -> - isFilterOnMemberMatching(memberName, namingConvention) - } - } - - if (foundApplications.size() == 1) { // one application claimed ownership - return foundApplications[0].application - } else if (foundApplications.size() > 1) { // multiple applications claimed ownership - logger.logMessage("*! [WARNING] Multiple applications claim ownership of member '$memberName':") - foundApplications.each { application -> - logger.logMessage("\t\tClaiming ownership: '${application.application}'") - } - logger.logMessage("*! [WARNING] The owner cannot be defined. Map '$memberName' to UNASSIGNED") - return "UNASSIGNED" - } else { // no match found - return "UNASSIGNED" - } + // Finding the owning application in the list of applications using the same dataset + def foundApplications = applicationsList.findAll { application -> + application.namingConventions.find { namingConvention -> + isFilterOnMemberMatching(memberName, namingConvention) + } + } + + if (foundApplications.size() == 1) { // one application claimed ownership + return foundApplications[0].application + } else if (foundApplications.size() > 1) { // multiple applications claimed ownership + logger.logMessage("*! [WARNING] Multiple applications claim ownership of member '$memberName':") + foundApplications.each { application -> + logger.logMessage("\t\tClaiming ownership: '${application.application}'") + } + logger.logMessage("*! [WARNING] The owner cannot be defined. Map '$memberName' to UNASSIGNED") + return "UNASSIGNED" + } else { // no match found + return "UNASSIGNED" + } } def findApplication(String applicationName) { - def applications = new ArrayList() - datasetsMap.each() { dataset, applicationsList -> - def foundApplications = applicationsList.findAll { - application -> application.application.equals(applicationName) - } - applications.addAll(foundApplications) - - } - applications.unique() - if (applications && applications.size() == 1) { - return applications[0] - } else { - return null - } + def applications = new ArrayList() + datasetsMap.each() { dataset, applicationsList -> + def foundApplications = applicationsList.findAll { + application -> application.application.equals(applicationName) + } + applications.addAll(foundApplications) + + } + applications.unique() + if (applications && applications.size() == 1) { + return applications[0] + } else { + return null + } } // Add dataset to the list of datasets where the member can be found def addDatasetToMember(String member, String dataset) { - ArrayList datasetsList = membersToDatasetsMap.get(member) - if (!datasetsList) { - datasetsList = new ArrayList() - membersToDatasetsMap.put(member, datasetsList) - } - datasetsList.add(dataset) + ArrayList datasetsList = membersToDatasetsMap.get(member) + if (!datasetsList) { + datasetsList = new ArrayList() + membersToDatasetsMap.put(member, datasetsList) + } + datasetsList.add(dataset) } /** @@ -515,95 +522,102 @@ def addDatasetToMember(String member, String dataset) { * For instance: BLD.LOAD(PGM1) --> [BLD.LOAD, PGM1] */ def getDatasetAndMember(String fullname) { - def ds,member; - def elements = fullname.split("[\\(\\)]"); - ds = elements[0]; - member = elements.size()>1? elements[1] : ""; - return [ds, member]; + def ds,member; + def elements = fullname.split("[\\(\\)]"); + ds = elements[0]; + member = elements.size()>1? elements[1] : ""; + return [ds, member]; } def getLastQualifier(String dataset) { - def qualifiers = dataset.split("\\."); - return qualifiers.last() + def qualifiers = dataset.split("\\."); + return qualifiers.last() } // Reads a HashMap from a file with comma separator (',') -def loadMapFromFile(String filePath) { - HashMap map = new HashMap<>(); - String line; - try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { - while ((line = reader.readLine()) != null) { - String[] keyValuePair = line.split(",", 2); - if (keyValuePair.length > 1) { - String key = keyValuePair[0].trim(); - String value = keyValuePair[1].trim().replaceAll(" ", ""); - map.put(key, value); - } - } - reader.close() - } catch (IOException e) { - e.printStackTrace(); - } - return map; -} - -def getType(String member) { - if (!types) { - return "UNKNOWN" - } else { - def type = types.get(member) - if (type) { - return type - } else { - return "UNKNOWN" - } - } +// def loadMapFromFile(String filePath) { +// HashMap map = new HashMap<>(); +// String line; +// try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { +// while ((line = reader.readLine()) != null) { +// String[] keyValuePair = line.split(",", 2); +// if (keyValuePair.length > 1) { +// String key = keyValuePair[0].trim(); +// String value = keyValuePair[1].trim().replaceAll(" ", ""); +// map.put(key, value); +// } +// } +// reader.close() +// } catch (IOException e) { +// e.printStackTrace(); +// } +// return map; +// } + +def getTypes(String member) { + if (!typesMapping) { + return "UNKNOWN" + } else { + ArrayList matchingTypes = new ArrayList() + typesMapping.types.each() { type -> + type.filePatterns.each() { filePattern -> + if (isFilterOnMemberMatching(member, filePattern)) { + matchingTypes.add(type.type) + } + } + } + if (matchingTypes) { + return matchingTypes.toString().replaceAll("[\\[\\] ]", "") + } else { + return "UNKNOWN" + } + } } def scanDatasetMember(String datasetMemberToScan) { - ZFile zFile = new ZFile(datasetMemberToScan, "r", ZFileConstants.FLAG_DISP_SHR) - InputStream zFileInputStream = zFile.getInputStream(); - // Init scanner - if (!scanner) scanner = initializeScanner() - // Scan file - Object scanMetadata = scanner.processSingleFile(zFileInputStream); - SingleFilesMetadata dmhfile = (SingleFilesMetadata) scanMetadata; - // Close file allocation - zFile.close() - - return [dmhfile.getLanguageCd(), dmhfile.getFileTypeCd()] + ZFile zFile = new ZFile(datasetMemberToScan, "r", ZFileConstants.FLAG_DISP_SHR) + InputStream zFileInputStream = zFile.getInputStream(); + // Init scanner + if (!scanner) scanner = initializeScanner() + // Scan file + Object scanMetadata = scanner.processSingleFile(zFileInputStream); + SingleFilesMetadata dmhfile = (SingleFilesMetadata) scanMetadata; + // Close file allocation + zFile.close() + + return [dmhfile.getLanguageCd(), dmhfile.getFileTypeCd()] } def initializeScanner() { - ScanProperties scanProperties = new ScanProperties(); - scanProperties.setCodePage(props.SCAN_DATASET_MEMBERS_ENCODING); - Dmh5210 dmh5210 = new Dmh5210(); - dmh5210.init(scanProperties); - return dmh5210; + ScanProperties scanProperties = new ScanProperties(); + scanProperties.setCodePage(props.SCAN_DATASET_MEMBERS_ENCODING); + Dmh5210 dmh5210 = new Dmh5210(); + dmh5210.init(scanProperties); + return dmh5210; } def calculateStorageSizeForMembers(HashSet datasetMembers) { - def storageSize = 0 - datasetMembers.forEach { datasetMember -> - storageSize = storageSize + estimateDatasetMemberSize(datasetMember) - } - return storageSize + def storageSize = 0 + datasetMembers.forEach { datasetMember -> + storageSize = storageSize + estimateDatasetMemberSize(datasetMember) + } + return storageSize } def estimateDatasetMemberSize(String datasetMember) { - ZFile file = new ZFile(constructPDSForZFileOperation(datasetMember), "r") - InputStreamReader streamReader = new InputStreamReader(file.getInputStream()) - long storageSize = 0 - long bytesSkipped = -1 - try { - while (bytesSkipped != 0) { - bytesSkipped = streamReader.skip(Long.MAX_VALUE) - storageSize = storageSize + bytesSkipped - } - file.close() - return storageSize - } catch (IOException exception) { - logger.logMessage("*! [WARNING] Unable to retrieve the estimated storage size for '$dataset($member)'") - return 0 - } + ZFile file = new ZFile(constructPDSForZFileOperation(datasetMember), "r") + InputStreamReader streamReader = new InputStreamReader(file.getInputStream()) + long storageSize = 0 + long bytesSkipped = -1 + try { + while (bytesSkipped != 0) { + bytesSkipped = streamReader.skip(Long.MAX_VALUE) + storageSize = storageSize + bytesSkipped + } + file.close() + return storageSize + } catch (IOException exception) { + logger.logMessage("*! [WARNING] Unable to retrieve the estimated storage size for '$dataset($member)'") + return 0 + } } \ No newline at end of file diff --git a/src/groovy/generateProperties.groovy b/src/groovy/generateZAppBuildProperties.groovy similarity index 99% rename from src/groovy/generateProperties.groovy rename to src/groovy/generateZAppBuildProperties.groovy index 3d7f335..d58ad98 100644 --- a/src/groovy/generateProperties.groovy +++ b/src/groovy/generateZAppBuildProperties.groovy @@ -30,11 +30,6 @@ import static java.nio.file.StandardCopyOption.* // Parse arguments from command-line parseArgs(args) -// Handle log file -if (props.logFile) { - logger.create(props.logFile) -} - def typesConfigurations // Build the Types Configuration object from Types Configurations file logger.logMessage("** Reading the Types Configurations definitions from '${props.TYPE_CONFIGURATIONS_FILE}'.") diff --git a/src/groovy/generateZBuilderProperties.groovy b/src/groovy/generateZBuilderProperties.groovy new file mode 100644 index 0000000..095c1f3 --- /dev/null +++ b/src/groovy/generateZBuilderProperties.groovy @@ -0,0 +1,373 @@ +/******************************************************************************** +* Licensed Materials - Property of IBM * +* (c) Copyright IBM Corporation 2018, 2024. All Rights Reserved. * +* * +* Note to U.S. Government Users Restricted Rights: * +* Use, duplication or disclosure restricted by GSA ADP Schedule * +* Contract with IBM Corp. * +********************************************************************************/ + +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import groovy.transform.* +import groovy.util.* +import groovy.time.* +import groovy.cli.commons.* +import groovy.yaml.YamlSlurper +import groovy.yaml.YamlBuilder +import java.util.Properties; +import java.nio.file.StandardCopyOption; +import java.nio.file.Files +import java.nio.file.Path + +@Field Properties props = new Properties() +@Field def applicationDescriptorUtils = loadScript(new File("utils/applicationDescriptorUtils.groovy")) +@Field def logger = loadScript(new File("utils/logger.groovy")) +@Field File applicationDescriptorFile +@Field def applicationDescriptor +@Field File applicationDBBAppYamlFile +@Field def applicationDBBAppYaml + +class ZBuilderApplication { + String name + ArrayList tasks +} + +class Task { + String task + ArrayList variables +} + +class Variable { + String name + String value + ArrayList forFiles +} + +class IncludeFile { + String file +} + +class DependencyPattern { + String languageExt + ArrayList dependencyPatterns +} + +/** + * Processing logic + */ + +// Parse arguments from command-line +parseArgs(args) + +def typesConfigurations +// Build the Types Configuration object from Types Configurations file +logger.logMessage("** Reading the Types Configurations definitions from '${props.TYPE_CONFIGURATIONS_FILE}'.") +def typesConfigurationsFile = new File(props.TYPE_CONFIGURATIONS_FILE) +if (!typesConfigurationsFile.exists()) { + logger.logMessage("!* [ERROR] the Types Configurations file '${props.TYPE_CONFIGURATIONS_FILE}' does not exist. Exiting.") + System.exit(1); +} else { + def yamlSlurper = new groovy.yaml.YamlSlurper() + typesConfigurations = yamlSlurper.parse(typesConfigurationsFile) +} + +// Parses the Application Descriptor File of the application, to retrieve the list of programs +applicationDescriptorFile = new File("${props.DBB_MODELER_APPLICATION_DIR}/${props.application}/applicationDescriptor.yml") +if (applicationDescriptorFile.exists()) { + applicationDescriptor = applicationDescriptorUtils.readApplicationDescriptor(applicationDescriptorFile) +} else { + logger.logMessage("!* [ERROR] The Application Descriptor file '${applicationDescriptorFile.getPath()}' does not exist. Exiting.") + System.exit(1) +} + +originalDBBAppYamlFile = new File("${props.DBB_MODELER_DEFAULT_GIT_CONFIG}/dbb-app.yaml") +if (originalDBBAppYamlFile.exists()) { + applicationDBBAppYamlFile = new File("${props.DBB_MODELER_APPLICATION_DIR}/${props.application}/dbb-app.yaml") + Files.copy(originalDBBAppYamlFile.toPath(), applicationDBBAppYamlFile.toPath(), StandardCopyOption.COPY_ATTRIBUTES, StandardCopyOption.REPLACE_EXISTING) + runShellCmd("chtag -t -c UTF-8 ${applicationDBBAppYamlFile.getAbsolutePath()}") + def yamlSlurper = new groovy.yaml.YamlSlurper() + applicationDBBAppYaml = yamlSlurper.parse(applicationDBBAppYamlFile) + applicationDBBAppYaml.application.name = props.application +} else { + logger.logMessage("!* [ERROR] The dbb-app.yaml sample file '${props.DBB_MODELER_DEFAULT_GIT_CONFIG}/dbb-app.yaml' does not exist. Exiting.") + System.exit(1) +} + +logger.logMessage("** Gathering the defined types for files.") + +// Internal map to collect all information : type <-> list of files mapped to that type +//HashMap> typesToFilesMap = new HashMap>() +// Internal map to collect all information : files <-> list of types for this file +HashMap> fileToTypesMap = new HashMap>() + +applicationDescriptor.sources.each { sourceGroup -> + repositoryPath = sourceGroup.repositoryPath + fileExtension = sourceGroup.fileExtension + sourceGroup.files.each { file -> + if (file.type) { + def types = file.type.split(",") + if (types != null && types.size() > 0 && !types[0].equals("UNKNOWN")) { + fileToTypesMap.put("${sourceGroup.repositoryPath}/${file.name}.${fileExtension}", types) + } + } + } +} + +if (fileToTypesMap && fileToTypesMap.size() > 0) { + logger.logMessage("** Generating zBuilder configuration files.") + + // Path to the zBuilder Configuration folder in the application's folder + def zBuilderConfigurationFolderPath = "${props.DBB_MODELER_APPLICATION_DIR}/${props.application}/zBuilder-config" + File zBuilderConfigurationFolder = new File(zBuilderConfigurationFolderPath) + if (!zBuilderConfigurationFolder.exists()) { + zBuilderConfigurationFolder.mkdirs() + } + + fileToTypesMap.each() { file, types -> + ZBuilderApplication zBuilderApplication = new ZBuilderApplication() + zBuilderApplication.name = props.application + zBuilderApplication.tasks = new ArrayList() + types.each() { type -> + typeConfiguration = typesConfigurations.typesConfigurations.find() { configuration -> + configuration.typeConfiguration.equals(type) + } + + if (typeConfiguration) { + typeConfiguration.tasks.each() { typeConfigurationTask -> + Task task = zBuilderApplication.tasks.find() { applicationTask -> + applicationTask.task.equals(typeConfigurationTask) + } + if (!task) { + task = new Task() + task.task = typeConfigurationTask + zBuilderApplication.tasks.add(task) + task.variables = new ArrayList() + } + + typeConfiguration.variables.each() { typeConfigurationVariable -> + Variable newVariable = new Variable() + newVariable.name = typeConfigurationVariable.name + newVariable.value = typeConfigurationVariable.value + newVariable.forFiles = new ArrayList() + newVariable.forFiles.add(file) + task.variables.add(newVariable) + } + } + // generating and writing the configuration file + File yamlFile = new File("${zBuilderConfigurationFolderPath}/${file}.yaml") + def yamlBuilder = new YamlBuilder() + yamlBuilder { + version "1.0.0" + application zBuilderApplication + } + File yamlFileParentFolder = yamlFile.getParentFile() + if (!yamlFileParentFolder.exists()) { + yamlFileParentFolder.mkdirs() + } + + yamlFile.withWriter("UTF-8") { writer -> + writer.write(yamlBuilder.toString()) + } + runShellCmd("chtag -t -c UTF-8 ${yamlFile.getAbsolutePath()}") + + if (!applicationDBBAppYaml.application.name) { + applicationDBBAppYaml.application.name = props.application + } + // Adding the file to the include files in the dbb-app.yaml file + if (!applicationDBBAppYaml.application.include) { + applicationDBBAppYaml.application.include = new ArrayList() + } + foundIncludeFiles = applicationDBBAppYaml.application.include.findAll() { foundIncludeFile -> + foundIncludeFile.file.equals("zBuilder-config/${file}.yaml" as String) + + } + if (!foundIncludeFiles) { + IncludeFile newIncludeFile = new IncludeFile() + newIncludeFile.file = "zBuilder-config/${file}.yaml" + applicationDBBAppYaml.application.include.add(newIncludeFile) + } + } else { + logger.logMessage("** [WARNING] No Type Configuration for type '${type}' found in '${props.TYPE_CONFIGURATIONS_FILE}'.") + } + } + } +} else { + logger.logMessage("** No Configuration type found for application '${props.application}'.") +} + +// generating dependencies path in configuration file + +def matchingSourcesGroup = applicationDescriptor.sources.findAll { source -> + source.artifactsType.equalsIgnoreCase("Program") +} +matchingSourcesGroup.each() { matchingMourceGroup -> + + def impactAnalysisTask = applicationDBBAppYaml.application.tasks.find() { task -> + task.task.equals("ImpactAnalysis") + } + if (impactAnalysisTask) { + def impactQueryPatterns = impactAnalysisTask.variables.find() { variable -> + variable.name.equals("impactQueryPatterns") + } + if (impactQueryPatterns) { + if (!impactQueryPatterns.value) { + impactQueryPatterns.value = new ArrayList() + } + if (matchingMourceGroup.name.equals("cobol")) { + DependencyPattern dependencyPattern = new DependencyPattern() + dependencyPattern.languageExt = matchingMourceGroup.fileExtension + dependencyPattern.dependencyPatterns = new ArrayList() + // BMS + def BMSGroup = applicationDescriptor.sources.find() { sourceGroup -> + sourceGroup.name.equals("bms") + } + if (BMSGroup) { + dependencyPattern.dependencyPatterns.add("\${APP_DIR_NAME}/${BMSGroup.repositoryPath}/*.${BMSGroup.fileExtension}") + } + // Cobol Copybook + def CopyGroup = applicationDescriptor.sources.find() { sourceGroup -> + sourceGroup.name.equals("copy") + } + if (CopyGroup) { + dependencyPattern.dependencyPatterns.add("\${APP_DIR_NAME}/${CopyGroup.repositoryPath}/*.${CopyGroup.fileExtension}") + } + // Cobol program + dependencyPattern.dependencyPatterns.add("\${APP_DIR_NAME}/${matchingMourceGroup.repositoryPath}/*.${matchingMourceGroup.fileExtension}") + impactQueryPatterns.value.add(dependencyPattern) + } + if (matchingMourceGroup.name.equals("link")) { + DependencyPattern dependencyPattern = new DependencyPattern() + dependencyPattern.languageExt = sourceGroup.fileExtension + dependencyPattern.dependencyPatterns = new ArrayList() + // Cobol program + def CobolGroup = applicationDescriptor.sources.find() { sourceGroup -> + sourceGroup.name.equals("cobol") + } + if (CobolGroup) { + dependencyPattern.dependencyPatterns.add("\${APP_DIR_NAME}/${CobolGroup.repositoryPath}/*.${CobolGroup.fileExtension}") + } + impactQueryPatterns.value.add(dependencyPattern) + } + } + } +} + + +// generating and writing the configuration file +def yamlBuilder = new YamlBuilder() + yamlBuilder { + version "1.0.0" + application applicationDBBAppYaml.application +} +applicationDBBAppYamlFile.withWriter("UTF-8") { writer -> + writer.write(yamlBuilder.toString()) +} + +// close logger file +logger.close() + + +/* ==== Utilities ==== */ + +/* parseArgs: parse arguments provided through CLI */ +def parseArgs(String[] args) { + Properties configuration = new Properties() + String usage = 'generateProperties.groovy [options]' + String header = 'options:' + def cli = new CliBuilder(usage:usage,header:header); + cli.a(longOpt:'application', args:1, required:true, 'Application name.') + cli.l(longOpt:'logFile', args:1, required:false, 'Relative or absolute path to an output log file') + cli.c(longOpt:'configFile', args:1, required:true, 'Path to the DBB Git Migration Modeler Configuration file (created by the Setup script)') + + def opts = cli.parse(args) + if (!args || !opts) { + cli.usage() + System.exit(1) + } + + if (opts.l) { + props.logFile = opts.l + logger.create(props.logFile) + } + + if (opts.a) { + props.application = opts.a + } else { + logger.logMessage("*! [ERROR] The Application name (option -a/--application) must be provided. Exiting.") + System.exit(1) + } + + if (opts.c) { + props.configurationFilePath = opts.c + File configurationFile = new File(props.configurationFilePath) + if (configurationFile.exists()) { + configurationFile.withReader() { reader -> + configuration.load(reader) + } + } else { + logger.logMessage("*! [ERROR] The DBB Git Migration Modeler Configuration file '${opts.c}' does not exist. Exiting.") + System.exit(1) + } + } else { + logger.logMessage("*! [ERROR] The path to the DBB Git Migration Modeler Configuration file was not specified ('-c/--configFile' parameter). Exiting.") + System.exit(1) + } + + if (configuration.DBB_MODELER_APPLICATION_DIR) { + File directory = new File(configuration.DBB_MODELER_APPLICATION_DIR) + if (directory.exists()) { + props.DBB_MODELER_APPLICATION_DIR = configuration.DBB_MODELER_APPLICATION_DIR + } else { + logger.logMessage("*! [ERROR] The Applications directory '${configuration.DBB_MODELER_APPLICATION_DIR}' does not exist. Exiting.") + System.exit(1) + } + } else { + logger.logMessage("*! [ERROR] The Applications directory must be specified in the DBB Git Migration Modeler Configuration file. Exiting.") + System.exit(1) + } + + if (configuration.TYPE_CONFIGURATIONS_FILE) { + File file = new File(configuration.TYPE_CONFIGURATIONS_FILE) + if (file.exists()) { + props.TYPE_CONFIGURATIONS_FILE = configuration.TYPE_CONFIGURATIONS_FILE + } else { + logger.logMessage("*! [ERROR] The Types Configurations file '${configuration.TYPE_CONFIGURATIONS_FILE}' does not exist. Exiting.") + System.exit(1) + } + } else { + logger.logMessage("*! [ERROR] The path to the Types Configurations file must be specified in the DBB Git Migration Modeler Configuration file. Exiting.") + System.exit(1) + } + + if (configuration.DBB_MODELER_DEFAULT_GIT_CONFIG) { + File file = new File(configuration.DBB_MODELER_DEFAULT_GIT_CONFIG) + if (file.exists()) { + props.DBB_MODELER_DEFAULT_GIT_CONFIG = configuration.DBB_MODELER_DEFAULT_GIT_CONFIG + } else { + logger.logMessage("*! [ERROR] The DBB Git Configuration samples folder '${configuration.DBB_MODELER_DEFAULT_GIT_CONFIG}' does not exist. Exiting.") + System.exit(1) + } + } else { + logger.logMessage("*! [ERROR] The path to the DBB Git Configuration samples folder must be specified in the DBB Git Migration Modeler Configuration file. Exiting.") + System.exit(1) + } + + logger.logMessage("** Script configuration:") + props.each() { k, v -> + logger.logMessage("\t$k -> $v") + } +} + +// Methods +def runShellCmd(String cmd){ + StringBuffer resp = new StringBuffer() + StringBuffer error = new StringBuffer() + + Process process = cmd.execute() + process.waitForProcessOutput(resp, error) + if (error) { + String warningMsg = "*! Failed to execute shell command $cmd" + println(warningMsg) + println(error) + } +} \ No newline at end of file diff --git a/src/groovy/utils/zappUtils.groovy b/src/groovy/utils/zappUtils.groovy index 34d60fc..37f5970 100644 --- a/src/groovy/utils/zappUtils.groovy +++ b/src/groovy/utils/zappUtils.groovy @@ -138,11 +138,11 @@ def writeZAppFile(File yamlFile) { } // write file - yamlFile.withWriter("IBM-1047") { writer -> + yamlFile.withWriter("UTF-8") { writer -> writer.write(yamlBuilder.toString()) } - Process process = "chtag -tc IBM-1047 ${yamlFile.getAbsolutePath()}".execute() + Process process = "chtag -tc UTF-8 ${yamlFile.getAbsolutePath()}".execute() process.waitFor() } diff --git a/src/scripts/Migration-Modeler-Start.sh b/src/scripts/Migration-Modeler-Start.sh index a580965..eaf75fc 100755 --- a/src/scripts/Migration-Modeler-Start.sh +++ b/src/scripts/Migration-Modeler-Start.sh @@ -133,7 +133,7 @@ if [ $rc -eq 0 ]; then echo echo "[PHASE] Extract applications from using Applications Mapping files located at '$DBB_MODELER_APPMAPPINGS_DIR'" if [[ $INTERACTIVE_RUN == "true" ]]; then - read -p "Do you want run the application extraction (Y/n): " variable + read -p "Do you want to run the application extraction (Y/n): " variable else variable="Y" fi @@ -187,7 +187,7 @@ if [ $rc -eq 0 ]; then echo echo "[PHASE] Generate build configuration" if [[ $INTERACTIVE_RUN == "true" ]]; then - read -p "Do you want to generate the dbb-zAppBuild configurations (Y/n): " variable + read -p "Do you want to generate the $BUILD_FRAMEWORK configurations (Y/n): " variable else variable="Y" fi diff --git a/src/scripts/utils/4-generateProperties.sh b/src/scripts/utils/4-generateProperties.sh index 32eacee..5a765af 100755 --- a/src/scripts/utils/4-generateProperties.sh +++ b/src/scripts/utils/4-generateProperties.sh @@ -65,7 +65,12 @@ if [ $rc -eq 0 ]; then echo "*******************************************************************" touch $DBB_MODELER_LOGS/4-$applicationDir-generateProperties.log chtag -tc IBM-1047 $DBB_MODELER_LOGS/4-$applicationDir-generateProperties.log - CMD="$DBB_HOME/bin/groovyz $DBB_MODELER_HOME/src/groovy/generateProperties.groovy \ + if [ "$BUILD_FRAMEWORK" = "zBuilder" ]; then + GENERATE_PROPERTIES_SCRIPT=$DBB_MODELER_HOME/src/groovy/generateZBuilderProperties.groovy + elif [ "$BUILD_FRAMEWORK" = "zAppBuild" ]; then + GENERATE_PROPERTIES_SCRIPT=$DBB_MODELER_HOME/src/groovy/generateZAppBuildProperties.groovy + fi + CMD="$DBB_HOME/bin/groovyz $GENERATE_PROPERTIES_SCRIPT \ --configFile $DBB_GIT_MIGRATION_MODELER_CONFIG_FILE \ --application $applicationDir \ --logFile $DBB_MODELER_LOGS/4-$applicationDir-generateProperties.log" diff --git a/src/scripts/utils/5-initApplicationRepositories.sh b/src/scripts/utils/5-initApplicationRepositories.sh index 49bd2e6..40fb0b8 100755 --- a/src/scripts/utils/5-initApplicationRepositories.sh +++ b/src/scripts/utils/5-initApplicationRepositories.sh @@ -237,36 +237,62 @@ if [ $rc -eq 0 ]; then # mkdir application log directory mkdir -p $DBB_MODELER_LOGS/$applicationDir + + if [ "$BUILD_FRAMEWORK" = "zBuilder" ]; then + if [ "$DBB_MODELER_METADATASTORE_TYPE" = "file" ]; then + unset METADATASTORE_OPTIONS + elif [ "$DBB_MODELER_METADATASTORE_TYPE" = "db2" ]; then + if [ -n "$DBB_MODELER_DB2_METADATASTORE_JDBC_PASSWORD" ]; then + declare METADATASTORE_OPTIONS="--dbid $DBB_MODELER_DB2_METADATASTORE_JDBC_ID --pw $DBB_MODELER_DB2_METADATASTORE_JDBC_PASSWORD" + elif [ -n "$DBB_MODELER_DB2_METADATASTORE_JDBC_PASSWORDFILE" ]; then + declare METADATASTORE_OPTIONS="--dbid $DBB_MODELER_DB2_METADATASTORE_JDBC_ID --dbpw $DBB_MODELER_DB2_METADATASTORE_JDBC_PASSWORDFILE" + fi + fi - if [ "$DBB_MODELER_METADATASTORE_TYPE" = "file" ]; then - declare METADATASTORE_OPTIONS="--propOverwrites createBuildOutputSubfolder=false,metadataStoreType=$DBB_MODELER_METADATASTORE_TYPE,metadataStoreFileLocation=$DBB_MODELER_FILE_METADATA_STORE_DIR" - elif [ "$DBB_MODELER_METADATASTORE_TYPE" = "db2" ]; then - if [ -n "$DBB_MODELER_DB2_METADATASTORE_JDBC_PASSWORD" ]; then - declare METADATASTORE_OPTIONS="--propOverwrites createBuildOutputSubfolder=false,metadataStoreType=$DBB_MODELER_METADATASTORE_TYPE,metadataStoreDb2ConnectionConf=$DBB_MODELER_DB2_METADATASTORE_CONFIG_FILE --id $DBB_MODELER_DB2_METADATASTORE_JDBC_ID --pw $DBB_MODELER_DB2_METADATASTORE_JDBC_PASSWORD" - elif [ -n "$DBB_MODELER_DB2_METADATASTORE_JDBC_PASSWORDFILE" ]; then - declare METADATASTORE_OPTIONS="--propOverwrites createBuildOutputSubfolder=false,metadataStoreType=$DBB_MODELER_METADATASTORE_TYPE,metadataStoreDb2ConnectionConf=$DBB_MODELER_DB2_METADATASTORE_CONFIG_FILE --id $DBB_MODELER_DB2_METADATASTORE_JDBC_ID --pwFile $DBB_MODELER_DB2_METADATASTORE_JDBC_PASSWORDFILE" + export DBB_BUILD=$DBB_ZBUILDER + cd $DBB_MODELER_APPLICATION_DIR/$applicationDir + CMD="$DBB_HOME/bin/dbb build full \ + --hlq $APPLICATION_ARTIFACTS_HLQ \ + --preview \ + ${METADATASTORE_OPTIONS}" + echo "** $CMD" >> $DBB_MODELER_LOGS/5-$applicationDir-initApplicationRepository.log + $CMD > $DBB_MODELER_LOGS/$applicationDir/build-preview-$applicationDir.log + rc=$? + if [ $rc -eq 0 ]; then + echo "** Preview Build of application '$applicationDir' completed successfully. rc="$rc + else + echo "*! [ERROR] Preview Build of application '$applicationDir' failed. rc="$rc + echo "** Build logs and reports available at '$DBB_MODELER_LOGS/$applicationDir'" + fi + elif [ "$BUILD_FRAMEWORK" = "zAppBuild" ]; then + if [ "$DBB_MODELER_METADATASTORE_TYPE" = "file" ]; then + declare METADATASTORE_OPTIONS="--propOverwrites createBuildOutputSubfolder=false,metadataStoreType=$DBB_MODELER_METADATASTORE_TYPE,metadataStoreFileLocation=$DBB_MODELER_FILE_METADATA_STORE_DIR" + elif [ "$DBB_MODELER_METADATASTORE_TYPE" = "db2" ]; then + if [ -n "$DBB_MODELER_DB2_METADATASTORE_JDBC_PASSWORD" ]; then + declare METADATASTORE_OPTIONS="--propOverwrites createBuildOutputSubfolder=false,metadataStoreType=$DBB_MODELER_METADATASTORE_TYPE,metadataStoreDb2ConnectionConf=$DBB_MODELER_DB2_METADATASTORE_CONFIG_FILE --id $DBB_MODELER_DB2_METADATASTORE_JDBC_ID --pw $DBB_MODELER_DB2_METADATASTORE_JDBC_PASSWORD" + elif [ -n "$DBB_MODELER_DB2_METADATASTORE_JDBC_PASSWORDFILE" ]; then + declare METADATASTORE_OPTIONS="--propOverwrites createBuildOutputSubfolder=false,metadataStoreType=$DBB_MODELER_METADATASTORE_TYPE,metadataStoreDb2ConnectionConf=$DBB_MODELER_DB2_METADATASTORE_CONFIG_FILE --id $DBB_MODELER_DB2_METADATASTORE_JDBC_ID --pwFile $DBB_MODELER_DB2_METADATASTORE_JDBC_PASSWORDFILE" + fi + fi + CMD="$DBB_HOME/bin/groovyz $DBB_ZAPPBUILD/build.groovy \ + --workspace $DBB_MODELER_APPLICATION_DIR/$applicationDir \ + --application $applicationDir \ + --outDir $DBB_MODELER_LOGS/$applicationDir \ + --fullBuild \ + --hlq $APPLICATION_ARTIFACTS_HLQ --preview \ + --logEncoding UTF-8 \ + --applicationCurrentBranch $APPLICATION_DEFAULT_BRANCH \ + ${METADATASTORE_OPTIONS} \ + --propFiles /var/dbb/dbb-zappbuild-config/build.properties,/var/dbb/dbb-zappbuild-config/datasets.properties" + echo "** $CMD" >> $DBB_MODELER_LOGS/5-$applicationDir-initApplicationRepository.log + $CMD > $DBB_MODELER_LOGS/$applicationDir/build-preview-$applicationDir.log + rc=$? + if [ $rc -eq 0 ]; then + echo "** Preview Build of application '$applicationDir' completed successfully. rc="$rc + else + echo "*! [ERROR] Preview Build of application '$applicationDir' failed. rc="$rc + echo "** Build logs and reports available at '$DBB_MODELER_LOGS/$applicationDir'" fi - fi - - - CMD="$DBB_HOME/bin/groovyz $DBB_ZAPPBUILD/build.groovy \ - --workspace $DBB_MODELER_APPLICATION_DIR/$applicationDir \ - --application $applicationDir \ - --outDir $DBB_MODELER_LOGS/$applicationDir \ - --fullBuild \ - --hlq $APPLICATION_ARTIFACTS_HLQ --preview \ - --logEncoding UTF-8 \ - --applicationCurrentBranch $APPLICATION_DEFAULT_BRANCH \ - ${METADATASTORE_OPTIONS} \ - --propFiles /var/dbb/dbb-zappbuild-config/build.properties,/var/dbb/dbb-zappbuild-config/datasets.properties" - echo "** $CMD" >> $DBB_MODELER_LOGS/5-$applicationDir-initApplicationRepository.log - $CMD > $DBB_MODELER_LOGS/$applicationDir/build-preview-$applicationDir.log - rc=$? - if [ $rc -eq 0 ]; then - echo "** Preview Build of application '$applicationDir' completed successfully. rc="$rc - else - echo "*! [ERROR] Preview Build of application '$applicationDir' failed. rc="$rc - echo "** Build logs and reports available at '$DBB_MODELER_LOGS/$applicationDir'" fi fi