Support for YAML-based Files-To-Types mapping

Setup.sh

@@ -101,7 +101,7 @@ DBB_MODELER_APPMAPPINGS_DIR="$DBB_MODELER_WORK/config/applications-mappings"
 # Reference to the repository paths mapping file
 REPOSITORY_PATH_MAPPING_FILE=$DBB_MODELER_WORK/config/repositoryPathsMapping.yaml
 # Reference to the type mapping file
-APPLICATION_MEMBER_TYPE_MAPPING=$DBB_MODELER_WORK/config/types/types.txt
+APPLICATION_TYPES_MAPPING=$DBB_MODELER_WORK/config/types/typesMapping.yaml
 # Reference to the type configuration file to generate build configuration
 TYPE_CONFIGURATIONS_FILE=$DBB_MODELER_WORK/config/types/typesConfigurations.yaml
 # Input files and configuration
@@ -151,7 +151,7 @@ PIPELINE_CI=
 
 # Arrays for configuration parameters, that will the Setup script will prompt the user for
 path_config_array=(DBB_MODELER_APPCONFIG_DIR DBB_MODELER_APPLICATION_DIR DBB_MODELER_LOGS DBB_MODELER_DEFAULT_APP_REPO_CONFIG)
-input_array=(DBB_MODELER_APPMAPPINGS_DIR REPOSITORY_PATH_MAPPING_FILE APPLICATION_MEMBER_TYPE_MAPPING TYPE_CONFIGURATIONS_FILE APPLICATION_ARTIFACTS_HLQ SCAN_CONTROL_TRANSFERS SCAN_DATASET_MEMBERS SCAN_DATASET_MEMBERS_ENCODING DBB_ZAPPBUILD DBB_COMMUNITY_REPO APPLICATION_DEFAULT_BRANCH MOVE_FILES_FLAG PUBLISH_ARTIFACTS INTERACTIVE_RUN)
+input_array=(DBB_MODELER_APPMAPPINGS_DIR REPOSITORY_PATH_MAPPING_FILE APPLICATION_TYPES_MAPPING TYPE_CONFIGURATIONS_FILE APPLICATION_ARTIFACTS_HLQ SCAN_CONTROL_TRANSFERS SCAN_DATASET_MEMBERS SCAN_DATASET_MEMBERS_ENCODING DBB_ZAPPBUILD DBB_COMMUNITY_REPO APPLICATION_DEFAULT_BRANCH MOVE_FILES_FLAG PUBLISH_ARTIFACTS INTERACTIVE_RUN)
 # Publishing options that are conditionally prompted, if PUBLISH_ARTIFACTS=true
 publishing_options=(ARTIFACT_REPOSITORY_SERVER_URL ARTIFACT_REPOSITORY_USER ARTIFACT_REPOSITORY_PASSWORD ARTIFACT_REPOSITORY_SUFFIX PIPELINE_USER PIPELINE_USER_GROUP)
 

samples/typesMapping.yaml

@@ -0,0 +1,24 @@
+## List of datasets members to which types are assigned
+datasetMembers:
+    - datasetMember: "DBEHM.MIG.COBOL(LGACDB01)"
+      ## For each dataset member, types is a list of types assigned
+      ## The types are then concatenated to form a Language configuration definition
+      types:
+          - "CBLCICSDB2"
+      ## Properties can also be specified to describe file-level overrides
+      ## These overrides are not yet used in the processing
+      properties:
+          - name: "COBPARM"
+            value: "AMODE(31)"
+    - datasetMember: "DBEHM.MIG.COBOL(LGACDB02)"
+      types:
+          - "CBLDB2"
+      properties:
+          - name: "COBPARM"
+            value: "AMODE(31)"
+    - datasetMember: "DBEHM.MIG.COBOL(LGACUS01)"
+      types:
+          - "CBLCICS"
+      properties:
+          - name: "COBPARM"
+            value: "AMODE(31)"

src/groovy/extractApplications.groovy

@@ -61,7 +61,7 @@ unassignedApplicationMappingConfiguration.application = "UNASSIGNED"
 @Field HashSet<String> filteredApplications = new HashSet<String>()
 
 // Types Configurations
-@Field HashMap<String, String> types
+@Field ArrayList typesMapping
 // script properties
 @Field Properties props = new Properties()
 @Field repositoryPathsMapping
@@ -100,13 +100,8 @@ if (props.REPOSITORY_PATH_MAPPING_FILE) {
 
 // Read the Types from file
 logger.logMessage("** Reading the Type Mapping definition.")
-if (props.APPLICATION_MEMBER_TYPE_MAPPING) {
-	def typesFile = new File(props.APPLICATION_MEMBER_TYPE_MAPPING)
-	if (!typesFile.exists()) {
-		logger.logMessage("*! [WARNING] File ${props.APPLICATION_MEMBER_TYPE_MAPPING} not found in the current working directory. All artifacts will use the 'UNKNOWN' type.")
-	} else {
-		types = fileUtils.loadTypes(props.APPLICATION_MEMBER_TYPE_MAPPING)
-	}
+if (props.APPLICATION_TYPES_MAPPING) {
+	typesMapping = loadTypesMapping(props.APPLICATION_TYPES_MAPPING)
 } else {
 	logger.logMessage("*! [WARNING] No Types File provided. The 'UNKNOWN' type will be assigned by default to all artifacts.")
 }
@@ -286,12 +281,12 @@ def parseArgs(String[] args) {
 		System.exit(1)
 	}
 
-	if (configuration.APPLICATION_MEMBER_TYPE_MAPPING) {
-		File file = new File(configuration.APPLICATION_MEMBER_TYPE_MAPPING)
+	if (configuration.APPLICATION_TYPES_MAPPING) {
+		File file = new File(configuration.APPLICATION_TYPES_MAPPING)
 		if (file.exists()) {
-			props.APPLICATION_MEMBER_TYPE_MAPPING = configuration.APPLICATION_MEMBER_TYPE_MAPPING
+			props.APPLICATION_TYPES_MAPPING = configuration.APPLICATION_TYPES_MAPPING
 		} else {
-			logger.logMessage("*! [ERROR] The Types file '${configuration.APPLICATION_MEMBER_TYPE_MAPPING}' does not exist. Exiting.")
+			logger.logMessage("*! [ERROR] The Types file '${configuration.APPLICATION_TYPES_MAPPING}' does not exist. Exiting.")
 			System.exit(1)
 		}
 	}
@@ -393,7 +388,7 @@ def generateApplicationFiles(ApplicationMappingConfiguration applicationConfigur
 			(scannedLanguage, scannedFileType) = scanDatasetMember(constructDatasetForZFileOperation(dataset, member))
 		}
 		def lastQualifier = getLastQualifier(dataset)
-		def memberType = fileUtils.getType(types, member.toUpperCase())
+		def memberType = getTypeForDatasetMember(typesMapping, datasetMember)
 		// Identifying the matching Repository Path
 		// based on 1) the scan result if enabled
 		// 2) the type if set
@@ -578,4 +573,31 @@ def estimateDatasetMemberSize(String datasetMember) {
 		logger.logMessage("*! [WARNING] Unable to retrieve the estimated storage size for '$dataset($member)'")
 		return 0
 	}
-}
+}
+
+// Reads a HashMap from the MEMBER_TYPE_MAPPING file with comma separator (',') and returns it
+def loadTypesMapping(String APPLICATION_TYPES_MAPPING) {
+    File typeMappingFile = new File(APPLICATION_TYPES_MAPPING)
+    if (!typeMappingFile.exists()) {
+        logger.logMessage("*! [WARNING] The Types Mapping file '$APPLICATION_TYPES_MAPPING' was not found.")
+        return null
+    } else {        
+        def yamlSlurper = new groovy.yaml.YamlSlurper()
+        return yamlSlurper.parse(typeMappingFile).datasetMembers
+    }
+}
+
+def getTypeForDatasetMember(ArrayList types, String datasetMember) {
+    if (!types) {
+        return "UNKNOWN"
+    } else {
+        def foundType = types.find { type ->
+          type.datasetMember.equalsIgnoreCase(datasetMember)
+        } 
+        if (foundType) {
+            return foundType.types.sort().join("-")
+        } else {
+            return "UNKNOWN"
+        }
+    }
+}

src/groovy/utils/fileUtils.groovy

@@ -14,43 +14,6 @@ import java.nio.file.*
 import groovy.cli.commons.*
 
 
-// Reads a HashMap from the MEMBER_TYPE_MAPPING file with comma separator (',') and returns it
-def loadTypes(String APPLICATION_MEMBER_TYPE_MAPPING) {
-	HashMap<String, String> types = new HashMap<>();
-	String line;
-	File applicationMemberTypeMappingFile = new File(APPLICATION_MEMBER_TYPE_MAPPING)
-	if (!applicationMemberTypeMappingFile.exists()) {
-		logger.logMessage("*! [WARNING] The Application Member Type Mapping file $APPLICATION_MEMBER_TYPE_MAPPING was not found. Exiting.")
-		System.exit(1)
-	} else {		
-		def yamlSlurper = new groovy.yaml.YamlSlurper()
-		applicationMemberTypeMappingFile.withReader("UTF-8") { reader ->
-			while ((line = reader.readLine()) != null) {
-				String[] keyValuePair = line.split(",", 2);
-				if (keyValuePair.length > 1) {
-					String key = keyValuePair[0].trim().toUpperCase();
-					String value = keyValuePair[1].trim().replaceAll(" ", "");
-					types.put(key, value);
-				}
-			}
-		}
-	}
-	return types
-}
-
-def getType(HashMap<String, String> types, String member) {
-	if (!types) {
-		return "UNKNOWN"
-	} else {
-		def type = types.get(member)
-		if (type) {
-			return type
-		} else {
-			return "UNKNOWN"
-		}
-	}
-}
-
 /*
  * relativizePath - converts an absolute path to a relative path from the workspace directory
  */

src/scripts/utils/0-validateConfiguration.sh

@@ -254,7 +254,7 @@ initializeWorkDirectory() {
 			fi	
 		fi
 		if [ $rc -eq 0 ]; then
-			echo "  [INFO] Copying sample Repository Paths mapping file to '$REPOSITORY_PATH_MAPPING_FILE'"
+			echo "  [INFO] Copying sample Repository Paths Mapping file to '$REPOSITORY_PATH_MAPPING_FILE'"
 			mkdir -p "$(dirname $REPOSITORY_PATH_MAPPING_FILE)"
 			cp $DBB_MODELER_HOME/samples/repositoryPathsMapping.yaml $REPOSITORY_PATH_MAPPING_FILE
 			command_rc=$?
@@ -265,13 +265,13 @@ initializeWorkDirectory() {
 			fi	
 		fi
 		if [ $rc -eq 0 ]; then
-			echo "  [INFO] Copying sample Types file to '$APPLICATION_MEMBER_TYPE_MAPPING'"
-			mkdir -p "$(dirname $APPLICATION_MEMBER_TYPE_MAPPING)"
-			cp $DBB_MODELER_HOME/samples/types.txt $APPLICATION_MEMBER_TYPE_MAPPING
+			echo "  [INFO] Copying sample Files to Types Mapping file to '$APPLICATION_TYPES_MAPPING'"
+			mkdir -p "$(dirname $APPLICATION_TYPES_MAPPING)"
+			cp $DBB_MODELER_HOME/samples/typesMapping.yaml $APPLICATION_TYPES_MAPPING
 			command_rc=$?
 			if [ $command_rc -ne 0 ]; then
 				rc=8
-				ERRMSG="[ERROR] Unable to copy sample Types file to '$APPLICATION_MEMBER_TYPE_MAPPING'."
+				ERRMSG="[ERROR] Unable to copy sample Types Mapping file to '$APPLICATION_TYPES_MAPPING'."
 				echo $ERRMSG
 			fi	
 		fi