20486: Implements shared deviations

howso.amlg

@@ -149,6 +149,10 @@
 	#!postProcessMap (null)
 	#!hasDependentFeatures (null)
 	#!dependentFeatureMap (null)
+	#!sharedDeviationsMap (null)
+	#!sharedDeviationGroupByPrimaryMap (null)
+	#!sharedDeviationsNonPrimaryFeatures (null)
+	#!sharedDeviationsPrimaryFeatures (null)
 	#!continuousToNominalDependenciesMap (null)
 	#!dependentsBoundaryMap (null)
 	#!dependentValuesCombinationsMap (null)
@@ -229,6 +233,7 @@
 		"residuals"
 		"return_typing"
 		"series_store"
+		"shared_deviations"
 		"substitution"
 		"synthesis"
 		"synthesis_bounds"
@@ -556,6 +561,20 @@
 			;assoc of feature -> { 'dependent_features' : [ list of dependent features ] }
 			!dependentFeatureMap (assoc)
 
+			;assoc of feature ->  primary feature key from group of shared deviations features
+			;  This primary feature key is the first sorted feature in the group of features
+			!sharedDeviationsMap (assoc)
+
+			;list of all primary shared features
+			!sharedDeviationsPrimaryFeatures (list)
+
+			;list of all non-primary shared features in a group
+			!sharedDeviationsNonPrimaryFeatures (list)
+
+			;assoc of primary group feature -> list of shared features in its group
+			; e.g.,  { "a": ["a", "b"], "c": ["c", "d"] }
+			!sharedDeviationGroupByPrimaryMap (assoc)
+
 			;assoc of continuous feature -> [ list of sorted nominal dependents ]
 			!continuousToNominalDependenciesMap (assoc)
 

howso/analysis.amlg

@@ -293,6 +293,7 @@
 										weight_feature weight_feature
 										use_case_weights use_case_weights
 										compute_null_uncertainties (false)
+										use_shared_deviations (true)
 									))
 							)
 

howso/analysis_weights.amlg

@@ -29,7 +29,7 @@
 		(while (< iteration num_iterations)
 			(assign (assoc
 				residuals_map
-					(call !ExpandResidualValuesToUncertainty  (assoc
+					(call !ExpandResidualValuesToUncertainty (assoc
 						feature_residuals_map
 							(call !CalculateFeatureResiduals (assoc
 								features features
@@ -51,6 +51,9 @@
 								compute_all_statistics use_deviations
 								;don't sparsify the confusion matrix so that SDM can be computed using full counts
 								confusion_matrix_min_count 0
+								use_shared_deviations (true)
+								;don't create copies of confusion matrices for non-primary shared deviation features
+								expand_confusion_matrices (false)
 							))
 					))
 			))
@@ -215,7 +218,7 @@
 	)
 
 	;updates analyzed_hp_map
-	;run multiple iterations of grid search and resdiuals to either use with the inverse_residuals_as_weights flow
+	;run multiple iterations of grid search and residuals to either use with the inverse_residuals_as_weights flow
 	#!ConvergeIRW
 	(declare
 		(assoc
@@ -350,6 +353,9 @@
 									compute_all_statistics use_deviations
 									;don't sparsify the confusion matrix so that SDM can be computed using full counts
 									confusion_matrix_min_count 0
+									use_shared_deviations (true)
+									;don't create copies of confusion matrices for non-primary shared deviation features
+									expand_confusion_matrices (false)
 								))
 						))
 					)
@@ -436,7 +442,7 @@
 						(assoc feature (current_index 1))
 						;store class counts as weights (reciprocal of count) for all cases for each id feature
 						(map
-							(lambda (/ 1 (current_value)))
+							(lambda (/ 1 (current_value)) )
 
 							;grab the un-weighted count of each class
 							(compute_on_contained_entities (list

howso/attribute_maps.amlg

@@ -205,6 +205,207 @@
 		full_dependents_map
 	)
 
+	;Creates shared deviations model attributes:
+	; shared_deviations_key_map:  mapping of shared deviations where the keys are each feature within a group of shared deviations, and the values are their
+	; 	corresponding primary feature. For each group of shared deviations, they are stored under the first sorted feature in their group, a.k.a. the primary feature.
+	;   e.g., a group of features with shared devations is [A, B, C], creates an mapping of {A : A, B : A, C : A}
+	; shared_deviations_group_by_primary: mapping of primary key -> list of shared deviation features in the group, e.g. {A : [A, B, C]}
+	; shared_deviations_primary_features: list of all primary features, e.g., [A]
+	; shared_deviations_non_primary_features: list of all non-primary features  e.g., [B, C]
+	#!ProcessSharedDeviationsMap
+	(seq
+		(assign (assoc
+			shared_deviations_map
+				(append
+					(map
+						(lambda
+							(or
+								(get shared_deviations_map (current_index))
+								[(current_index  1)]
+							)
+						)
+						(zip (apply "append" (values shared_deviations_map)))
+					)
+					shared_deviations_map
+				)
+		))
+
+		;create lists of referenced features for each shared feature
+		(declare (assoc
+			shared_sets
+				(map
+					(lambda (let
+						(assoc
+							shared_features (current_value 1)
+							feature (current_index 1)
+						)
+						(append
+							feature
+							;keep lists of shared features that reference either this feature or any of its shared_features
+							(indices (filter
+								(lambda
+									(or
+										(contains_value shared_features (current_index))
+										(contains_value (current_value) feature)
+										(!=
+											(size shared_features)
+											(size (remove (zip shared_features) (current_value)) )
+										)
+									)
+								)
+								(remove shared_deviations_map feature)
+							))
+						)
+					))
+					shared_deviations_map
+				)
+		))
+
+
+		(declare (assoc
+			combined_shared_sets
+				(map
+					(lambda (let
+						(assoc
+							shared_set (zip (current_value 1))
+							num_features_in_set (size (current_value 1))
+							feature (current_index 1)
+						)
+
+						;append all the overlapping shared sets and make them unique by zipping up the appended features
+						(zip
+							(apply "append"
+								;if there's any overlap between shared_set and any other feature's shared_set, combine them
+								(values (filter
+									(lambda
+										(if (!= num_features_in_set (size (remove shared_set (current_value))))
+											(append shared_set (zip (current_value)))
+										)
+									)
+									(remove shared_sets feature)
+								))
+							)
+						)
+
+					))
+					shared_sets
+				)
+
+		))
+
+		(declare (assoc
+			unique_sets (values combined_shared_sets (true))
+		))
+
+		(declare (assoc
+			shared_deviations_key_map
+				(map
+					(lambda (let
+						(assoc feature (current_index 1))
+						;find the the shared set that this feature belongs to and output the first feature from that set
+						(first (indices
+							(first (filter
+								(lambda (contains_index (current_value) feature))
+								unique_sets
+							))
+						))
+					))
+					combined_shared_sets
+				)
+		))
+
+		(declare (assoc
+			primary_features (indices (zip (values shared_deviations_key_map)))
+		))
+
+		(assoc
+			"shared_deviations_key_map" shared_deviations_key_map
+			"shared_deviations_primary_features" primary_features
+			"shared_deviations_group_by_primary"
+				(map
+					(lambda (indices (get combined_shared_sets (current_index))) )
+					(zip primary_features)
+				)
+			"shared_deviations_non_primary_features" (indices (remove shared_deviations_key_map primary_features))
+		)
+	)
+
+	;Validates the necessary parameters for shared deviations, making sure that no shared deviations features are unique nominals.
+	#!ValidateSharedDeviations
+	(let
+		(assoc
+			error_messages (list)
+			invalid_unique_shared_deviations_features (null)
+			invalid_nominals_non_nominals_shared_deviations (null)
+		)
+		;check to see if any features in a shared deviations group is a unique nominal
+		(if (size unique_nominals_set)
+			(assign (assoc
+				invalid_unique_shared_deviations_features
+					(filter
+						(lambda (contains_index unique_nominals_set (current_value)))
+						(indices (get shared_deviations_map "shared_deviations_key_map"))
+					)
+			))
+		)
+
+		(if (size invalid_unique_shared_deviations_features)
+			(accum (assoc
+				error_messages
+					(list (concat
+						"Features with shared deviations may not be unique nominals, as they have null deviations. "
+						"The following features are in a shared deviations group and is a unique nominal: "
+						(apply "concat" (trunc (weave invalid_unique_shared_deviations_features ", ")))
+						"."
+					))
+			))
+		)
+
+		;check to see if nominal features has shared deviations with non-nominal features
+		(if (size nominals)
+			(assign (assoc
+				invalid_nominals_non_nominals_shared_deviations
+					;returns at least an empty assoc for each group of shared deviations, append only keeps the assocs with values
+					(apply "append"
+						(values (map
+							;grab the type of the primary feature, and then filter out all the ones that match. Should be left
+							; with a list of size 0 as all of the features should share the same type
+							(lambda
+								(let
+									(assoc
+										;the nominal type of the primary feature in the group of shared deviations
+										is_primary_nominal (contains_value nominals (current_index 1))
+									)
+									(filter
+										;if the current value's nominal type does not match the first value's return it
+										(lambda
+											(!= is_primary_nominal (contains_value nominals (current_value)) )
+										)
+										;list of shared deviations in the group ["a", "b"]
+										(current_value)
+									)
+								)
+							)
+							;assoc of shared deviations groups, i.e. {"a": ["a", "b"], "c": ["c", "d"] }
+							(get shared_deviations_map "shared_deviations_group_by_primary")
+						))
+					)
+			))
+		)
+
+		(if (size invalid_nominals_non_nominals_shared_deviations)
+			(accum (assoc
+				error_messages (list "Nominal features may not share deviations with non-nominal features.")
+			))
+		)
+
+		;warns if errors are present
+		(if (size error_messages)
+			(conclude (call !Return (assoc errors error_messages)))
+		)
+	)
+
+
 	#!ComposeContinuousToNominalDependenciesMap
 	(let
 		(assoc

howso/attributes.amlg

@@ -93,6 +93,10 @@
 			;					feature is non-sensitive, setting this parameter to true will bypass the 'subtype'
 			;					requirement. Only applicable to nominal features. Default is false
 			;
+			;	'shared_deviations': list or boolean, The shared deviations feature group. If a list, this feature will share deviations with
+			;					the features in this list. By default, derived lag values share deviations with the parent feature. If
+			;					false and a parent feature, derived lags of this feature will not share deviations.
+			;
 			;	'subtype': string, the type used in novel nominal substitution.
 			;
 			;	'original_type': string, original data type details. Used by clients to determine how to serialize and deserialize  feature data.
@@ -212,11 +216,17 @@
 			continuous_to_nominal_dependents_map (null)
 			nominal_to_nominal_dependents_map (null)
 			novel_substition_feature_set (null)
+			shared_deviations_map (null)
 		))
 
 		(call !UpdateAttributesForDateTimeDataTypes)
 
 		(assign (assoc
+			shared_deviations_map
+				(filter (map
+					(lambda (get (current_value) "shared_deviations") )
+					feature_attributes
+				))
 			ordinals_map
 				(filter
 					(lambda (= "ordinal" (get (current_value) "type")))
@@ -420,6 +430,13 @@
 			))
 		)
 
+		(if (size shared_deviations_map)
+			(seq
+				(assign (assoc shared_deviations_map (call !ProcessSharedDeviationsMap)))
+				(call !ValidateSharedDeviations)
+			)
+		)
+
 		(if (size post_process_map)
 			(assign (assoc has_post_processing (true)))
 		)
@@ -553,6 +570,11 @@
 
 			!queryDistanceTypeMap query_distance_type_map
 			!tsTimeFeature time_series_feature
+			!sharedDeviationsMap (get shared_deviations_map "shared_deviations_key_map")
+			!sharedDeviationGroupByPrimaryMap (get shared_deviations_map "shared_deviations_group_by_primary")
+			!sharedDeviationsPrimaryFeatures (get shared_deviations_map "shared_deviations_primary_features")
+			!sharedDeviationsNonPrimaryFeatures (get shared_deviations_map "shared_deviations_non_primary_features")
+
 		))
 
 		(call !SetOrdinalFeatures (assoc ordinal_features ordinals))