20486: Implements shared deviations

howso.amlg

@@ -147,6 +147,7 @@
 	#!postProcessMap (null)
 	#!hasDependentFeatures (null)
 	#!dependentFeatureMap (null)
+	#!sharedDeviationsMap (null)
 	#!continuousToNominalDependenciesMap (null)
 	#!dependentsBoundaryMap (null)
 	#!dependentValuesCombinationsMap (null)
@@ -543,6 +544,10 @@
 			;assoc of feature -> { 'dependent_features' : [ list of dependent features ] }
 			!dependentFeatureMap (assoc)
 
+			;assoc of feature -> { 'shared_deviations_features' : primary feature key from group of shared deviations features }
+			;  This primary feature key is the first sorted feature in the group of features
+			!sharedDeviationsMap (assoc)
+
 			;assoc of continuous feature -> [ list of sorted nominal dependents ]
 			!continuousToNominalDependenciesMap (assoc)
 

howso/analysis.amlg

@@ -248,6 +248,7 @@
 										weight_feature weight_feature
 										use_case_weights use_case_weights
 										compute_null_uncertainties (false)
+										use_shared_deviations (true)
 									))
 							)
 

howso/analysis_weights.amlg

@@ -29,7 +29,7 @@
 		(while (< iteration num_iterations)
 			(assign (assoc
 				residuals_map
-					(call !ExpandResidualValuesToUncertainty  (assoc
+					(call !ExpandResidualValuesToUncertainty (assoc
 						feature_residuals_map
 							(call !CalculateFeatureResiduals (assoc
 								features features
@@ -51,6 +51,7 @@
 								compute_all_statistics use_deviations
 								;don't sparsify the confusion matrix so that SDM can be computed using full counts
 								confusion_matrix_min_count 0
+								use_shared_deviations (true)
 							))
 					))
 			))
@@ -208,7 +209,7 @@
 	)
 
 	;updates analyzed_hp_map
-	;run multiple iterations of grid search and resdiuals to either use with the inverse_residuals_as_weights flow
+	;run multiple iterations of grid search and residuals to either use with the inverse_residuals_as_weights flow
 	#!ConvergeIRW
 	(declare
 		(assoc
@@ -338,6 +339,7 @@
 									compute_all_statistics use_deviations
 									;don't sparsify the confusion matrix so that SDM can be computed using full counts
 									confusion_matrix_min_count 0
+									use_shared_deviations (true)
 								))
 						))
 					)

howso/attribute_maps.amlg

@@ -195,6 +195,194 @@
 		full_dependents_map
 	)
 
+	;Creates two assocs where
+	; shared_deviations_key_map <- mapping of shared deviations where the keys are each feature within a group of shared deviations, and the values are the feature names in
+	;   which their deviations are stored under. For each group of shared deviations, they are stored under the first sorted feature in their group.
+	;   e.g., a group of features with shared devations is [A, B, C], creates an mapping of A -> A, B -> A, C -> A
+	#!ProcessSharedDeviationsMap
+	(seq
+		(assign (assoc
+			shared_deviations_map
+				(append
+					(map
+						(lambda
+							(or
+								(get shared_deviations_map (current_index))
+								[(current_index  1)]
+							)
+						)
+						(zip (apply "append" (values shared_deviations_map)))
+					)
+					shared_deviations_map
+				)
+		))
+
+		;create lists of referenced features for each shared feature
+		(declare (assoc
+			shared_sets
+				(map
+					(lambda (let
+						(assoc
+							shared_features (current_value 1)
+							feature (current_index 1)
+						)
+						(append
+							feature
+							;keep lists of shared features that reference either this feature or any of its shared_features
+							(indices (filter
+								(lambda
+									(or
+										(contains_value shared_features (current_index))
+										(contains_value (current_value) feature)
+										(!=
+											(size shared_features)
+											(size (remove (zip shared_features) (current_value)) )
+										)
+									)
+								)
+								(remove shared_deviations_map feature)
+							))
+						)
+					))
+					shared_deviations_map
+				)
+		))
+
+
+		(declare (assoc
+			combined_shared_sets
+				(map
+					(lambda (let
+						(assoc
+							shared_set (zip (current_value 1))
+							num_features_in_set (size (current_value 1))
+							feature (current_index 1)
+						)
+
+						;append all the overlapping shared sets and make them unique by zipping up the appended features
+						(zip
+							(apply "append"
+								;if there's any overlap between shared_set and any other feature's shared_set, combine them
+								(values (filter
+									(lambda
+										(if (!= num_features_in_set (size (remove shared_set (current_value))))
+											(append shared_set (zip (current_value)))
+										)
+									)
+									(remove shared_sets feature)
+								))
+							)
+						)
+
+					))
+					shared_sets
+				)
+
+		))
+
+		(declare (assoc
+			unique_sets (values combined_shared_sets (true))
+		))
+
+		(declare (assoc
+			shared_deviations_key_map
+				(map
+					(lambda (let
+						(assoc feature (current_index 1))
+						;find the the shared set that this feature belongs to and output the first feature from that set
+						(first (indices
+							(first (filter
+								(lambda (contains_index (current_value) feature))
+								unique_sets
+							))
+						))
+					))
+					combined_shared_sets
+				)
+		))
+
+		(assoc
+			"shared_deviations_key_map" shared_deviations_key_map
+			"shared_deviations_groupings" unique_sets
+		)
+	)
+
+	;Validates the necessary parameters for shared deviations, making sure that no shared deviations features are unique nominals.
+	#!ValidateSharedDeviations
+	(let
+		(assoc error_messages (list))
+		;check to see if any features in a shared deviations group is a unique nominal
+		(if (size unique_nominals_set)
+			(declare (assoc
+				invalid_unique_shared_deviations_features
+					(filter
+						(lambda
+							(contains_value (indices unique_nominals_set) (current_index))
+						)
+						(get shared_deviations_map "shared_deviations_key_map")
+					)
+			))
+		)
+
+		(if (size invalid_unique_shared_deviations_features)
+			(accum (assoc
+				error_messages
+					(list (concat
+						"Features with shared deviations may not be unique nominals, as they have null deviations. "
+						"The following features are in a shared deviations group and is a unique nominal: "
+						(apply "concat" (trunc (weave (indices invalid_unique_shared_deviations_features) ", ")))
+						"."
+					))
+			))
+		)
+
+		;check to see if nominal features has a shared deviations with non-nominal features
+		(if (size nominals)
+			(declare (assoc
+				invalid_nominals_non_nominals_shared_deviations
+					(filter
+						(lambda
+							(let
+								(assoc
+									contains_nominals (false)
+									contains_non_nominals (false)
+								)
+								(map
+									(lambda
+										(if (contains_value nominals (current_value))
+											(assign (assoc contains_nominals (true)) )
+											(assign (assoc contains_non_nominals (true)) )
+										)
+									)
+									;indices of an assoc of shared deviations groups, i.e. {"a" (null) "b" (null)} -> ["a", "b"]
+									(indices (current_value))
+								)
+								(and
+									(= (true) contains_nominals)
+									(= (true) contains_non_nominals)
+								)
+							)
+						)
+						;list of assocs of shared deviations groups, i.e. [{"a" (null) "b" (null)} {"c" (null) "d" (null)}]
+						(get shared_deviations_map "shared_deviations_groupings")
+					)
+			))
+		)
+
+		(if (size invalid_nominals_non_nominals_shared_deviations)
+			(accum (assoc
+				error_messages
+					(list "Nominal features may not share deviations with non-nominal features.")
+			))
+		)
+
+		;warns if errors are present
+		(if (size error_messages)
+			(conclude (call !Return (assoc errors error_messages)))
+		)
+	)
+
+
 	#!ComposeContinuousToNominalDependenciesMap
 	(let
 		(assoc

howso/attributes.amlg

@@ -203,11 +203,17 @@
 			continuous_to_nominal_dependents_map (null)
 			nominal_to_nominal_dependents_map (null)
 			novel_substition_feature_set (null)
+			shared_deviations_map (null)
 		))
 
 		(call !UpdateAttributesForDateTimeDataTypes)
 
 		(assign (assoc
+			shared_deviations_map
+				(filter (map
+					(lambda (get (current_value) "shared_deviations") )
+					feature_attributes
+				))
 			ordinals_map
 				(filter
 					(lambda (= "ordinal" (get (current_value) "type")))
@@ -411,6 +417,13 @@
 			))
 		)
 
+		(if (size shared_deviations_map)
+			(seq
+				(assign (assoc shared_deviations_map (call !ProcessSharedDeviationsMap)))
+				(call !ValidateSharedDeviations)
+			)
+		)
+
 		(if (size post_process_map)
 			(assign (assoc has_post_processing (true)))
 		)
@@ -544,6 +557,7 @@
 
 			!queryDistanceTypeMap query_distance_type_map
 			!tsTimeFeature time_series_feature
+			!sharedDeviationsMap (get shared_deviations_map "shared_deviations_key_map")
 		))
 
 		(call !SetOrdinalFeatures (assoc ordinal_features ordinals))

howso/details_residuals.amlg

@@ -32,6 +32,7 @@
 			local_feature_metrics
 				(if (>= model_size 2)
 					(call !ExpandResidualValuesToUncertainty  (assoc
+						expand_shared_deviations (false)
 						feature_residuals_map
 							(call !CalculateFeatureResiduals (assoc
 								;only the features from the details