24048 deviation as ratio

module/analysis.amlg

@@ -721,7 +721,7 @@
 			features
 				;filter the context features to get just continuous
 				(filter
-					(lambda (= "continuous" (get !featureAttributes [(current_value 1) "type"])))
+					(lambda (contains_value [ "continuous_interval" "continuous_ratio"] (get !featureAttributes [(current_value 1) "type"])))
 					context_features
 				)
 			k_parameter .null

module/analysis_weights.amlg

@@ -422,6 +422,45 @@
 				initial_residual_values_map (call !ComputeInitialResiduals)
 			))
 		)
+		(declare (assoc
+			initial_ratios_map
+				(if !ratioMeasureMap
+					(let
+						(assoc
+							f_values_maps
+								(map
+									(lambda (retrieve_from_entity (current_value) !ratioMeasureMap) )
+									(call !AllCases)
+								)
+						)
+						;initial ratio computed as: harmonic mean of initial_deviation / case value for all cases
+						(map
+							(lambda (let
+								(assoc feature (current_index 1))
+								(generalized_mean
+									(map
+										(lambda
+											(/
+												;;;TODO: handle nulls and edit features
+												(get initial_residual_values_map feature)
+
+												;don't allow 0's by replacing them with half min gaps
+												(max
+													(abs (get (current_value) feature))
+													(get !cachedFeatureHalfMinGapMap feature)
+												)
+											)
+										)
+										f_values_maps
+									)
+									-1
+								)
+							))
+							!ratioMeasureMap
+						)
+					)
+				)
+		))
 
 		(declare (assoc
 			in_analyze .true
@@ -443,14 +482,15 @@
 				residuals_map
 					(if (= 0 iteration)
 						(assoc
-							"residual_map" (get_value initial_residual_values_map)
+							"residual_map" (modify initial_residual_values_map)
 							"ordinal_residual_map"
 								(if (size !ordinalFeaturesSet)
 									(filter
 										(lambda (contains_index !ordinalFeaturesRangesMap (current_index)))
-										(get_value initial_residual_values_map)
+										(modify initial_residual_values_map)
 									)
 								)
+							"ratios_map" initial_ratios_map
 						)
 
 						(call !ExpandResidualValuesToUncertainty  (assoc
@@ -494,6 +534,7 @@
 				))
 
 				(call !UpdateHyperparameters (assoc
+					ratios_map (get residuals_map "ratios_map")
 					feature_deviations (get residuals_map "residual_map")
 					confusion_matrix_map (get residuals_map ["prediction_stats" "confusion_matrix"])
 					ordinal_feature_deviations (get residuals_map "ordinal_residual_map")

module/attribute_maps.amlg

@@ -117,14 +117,14 @@
 				(lambda
 					(if
 						(and
-							(= "continuous" (get (current_value) "type"))
+							(contains_value ["continuous_interval" "continuous_ratio"] (get (current_value) "type"))
 							(= "formatted_date_time" (get (current_value) "data_type"))
 							(= .null (get (current_value) "date_time_format"))
 						)
 						(modify (current_value) "date_time_format" "%Y-%m-%dT%H:%M:%S")
 
 						(and
-							(= "continuous" (get (current_value) "type"))
+							(contains_value ["continuous_interval" "continuous_ratio"] (get (current_value) "type"))
 							(= .null (get (current_value) "date_time_format"))
 							(= "formatted_time" (get (current_value) "data_type"))
 						)
@@ -133,7 +133,7 @@
 						;force 'data_type' to 'formatted_date_time' for all features with a date_time_format
 						;that are not already set to 'formatted_time'
 						(and
-							(= "continuous" (get (current_value) "type"))
+							(contains_value ["continuous_interval" "continuous_ratio"] (get (current_value) "type"))
 							(!= .null (get (current_value) "date_time_format"))
 							(!= "formatted_time" (get (current_value) "data_type"))
 						)
@@ -611,8 +611,8 @@
 			(lambda
 				;only allowed data_type values are string, json and amalgam, all others are ignored
 				(if (and
-						(= "continuous" (get (last (current_value)) "type"))
-						(contains_value (list "string" "string_mixable" "json" "yaml" "amalgam") (get (last (current_value)) "data_type") )
+						(contains_value ["continuous_interval" "continuous_ratio"] (get (last (current_value)) "type"))
+						(contains_value ["string" "string_mixable" "json" "yaml" "amalgam"] (get (last (current_value)) "data_type") )
 					)
 					(list (modify (first (current_value 1))) (modify (get (last (current_value 1)) "data_type")))
 					;else don't output anything, map-filtering out any values that don't have datype defined
@@ -851,7 +851,7 @@
 
 					(contains_index cyclics_map (current_index))
 					{
-						"difference_type" "continuous"
+						"difference_type" "continuous_interval"
 						"data_type" "number"
 						"cycle_range" (modify (get cyclics_map (current_index 1)))
 					}
@@ -872,17 +872,18 @@
 
 									"number"
 								)
+							difference_type (get (current_value 1) "type")
 						)
 
 						(if (!= "code" data_type)
 							{
-								"difference_type" "continuous"
+								"difference_type" difference_type
 								"data_type" data_type
 							}
 
 							;continuous code specific datatypes are provided
 							{
-								"difference_type" "continuous"
+								"difference_type" difference_type
 								"data_type" data_type
 								"types_must_match"
 									;default to true

module/attributes.amlg

@@ -8,7 +8,8 @@
 		(assoc
 			;assoc of feature -> attributes, attributes defined below:
 			;
-			;	'type': string, one of 'continuous', 'ordinal' or 'nominal'. Default is 'continuous'.
+			;	'type': string, one of 'continuous_interval', 'continuous_ratio', 'ordinal' or 'nominal'. Default is 'continuous_interval'.
+			;		note: 'continuous' is supported but will be overwritten with 'continuous_interval'.
 			;
 			;	'bounds' : assoc of of bounds for feature value generation, keys of assoc defined below as:
 			;		'min': number or date string, the minimum value to be output
@@ -232,13 +233,28 @@
 				)
 		))
 
+		;backwards compatibility with type='continuous', overwrite with 'continuous_interval'
+		(assign (assoc
+			feature_attributes
+				(map
+					(lambda
+						(if (= "continuous" (get (current_value) "type"))
+							(modify (current_value) "type" "continuous_interval")
+							(current_value)
+						)
+					)
+					feature_attributes
+				)
+		))
+
 		(declare (assoc
 			ordinals (list)
 			nominals (list)
 
 			cyclics_map (assoc)
 			nominals_map (assoc)
 			ordinals_map (assoc)
+			continuous_map (assoc)
 			boundaries_map (assoc)
 
 			source_to_derived_feature_map (assoc)
@@ -294,6 +310,11 @@
 					(lambda (= "nominal" (get (current_value) "type")))
 					feature_attributes
 				)
+			continuous_map
+				(filter
+					(lambda (contains_value ["continuous_interval" "continuous_ratio"] (get (current_value) "type")))
+					feature_attributes
+				)
 			boundaries_map
 				(filter
 					(lambda (or (!= .null (get (current_value) "bounds")) (!= .null (get (current_value) "allow_null"))))
@@ -677,13 +698,27 @@
 			!sharedDeviationsPrimaryFeatures (get shared_deviations_map "shared_deviations_primary_features")
 			!sharedDeviationsNonPrimaryFeatures (get shared_deviations_map "shared_deviations_non_primary_features")
 
+			;;;TODO: may need to make a pass on unit tests to ensure they set feature attributes as continuous for all features as needed,
+			;and/or modify train and react_into_features so that anytime there are continuous features that have been added, they are also added to this assoc
+			!continuousMap (zip (indices continuous_map))
 		))
 		(call !UpdateDefinedFeatures)
 
 		(call !SetOrdinalFeatures (assoc ordinal_features ordinals))
 		(call !SetNominalFeatures (assoc nominal_features nominals))
 		(call !SetCyclicFeatures (assoc feature_attributes cyclics_map))
 
+		(declare (assoc
+			ratio_features_map
+				(filter
+					(lambda (= "continuous_ratio" (get (current_value) "type")))
+					continuous_map
+				)
+		))
+		(if (size ratio_features_map)
+			(assign_to_entities (assoc !ratioMeasureMap (zip (indices ratio_features_map)) ))
+		)
+
 		;overwrite computed deviations with user specified ones if user specified ones were larger
 		(if (> (size !userSpecifiedFeatureErrorsMap) 0)
 			(assign_to_entities (assoc
@@ -838,7 +873,7 @@
 										"ordinal"
 
 										;else
-										"continuous"
+										"continuous_interval"
 									)
 								decimal_places (get !featureRoundingMap (list (current_index 2) 1))
 								significant_digits (get !featureRoundingMap (list (current_index 2) 0))

module/conviction.amlg

@@ -264,10 +264,10 @@
 							(map
 								(lambda
 									(if (or (= "p_value_of_addition" (current_value)) (= "p_value_of_removal" (current_value)))
-										{"type" "continuous" "data_type" "number" "bounds" {"min" 0 "max" 1} }
+										{"type" "continuous_interval" "data_type" "number" "bounds" {"min" 0 "max" 1} }
 
 										;else convictions, entropy and distance contribution ranges are 0 to infinity
-										{"type" "continuous" "data_type" "number" "bounds" {"min" 0 "max" .infinity} }
+										{"type" "continuous_interval" "data_type" "number" "bounds" {"min" 0 "max" .infinity} }
 									)
 								)
 								(indices computed_map)
@@ -303,7 +303,7 @@
 							"weight_feature" (if use_case_weights weight_feature)
 						)
 					!autoAnalyzeEnabled .true
-					!queryFeatureAttributeTypeMap (append !queryFeatureAttributeTypeMap (zip (values computed_map) {"difference_type" "continuous" "data_type" "number"}) )
+					!queryFeatureAttributeTypeMap (append !queryFeatureAttributeTypeMap (zip (values computed_map) {"difference_type" "continuous_interval" "data_type" "number"}) )
 				))
 
 				;need to cache computed values to be able to approximate any of these listed computations during future train calls

module/derive_utilities.amlg

@@ -807,7 +807,7 @@
 							(lambda
 								(or
 									;categorical features are in the "delta_features" list...
-									(!= "continuous" (get !featureAttributes [(current_value 1) "type"]))
+									(contains_value ["ordinal" "nominal"] (get !featureAttributes [(current_value 1) "type"]))
 									(contains_value derived_action_features (get !derivedFeaturesMap (current_value)))
 								)
 							)

module/editing.amlg

@@ -466,7 +466,7 @@
 					(assign (assoc
 						feature_attributes
 							(assoc
-								"type" "continuous"
+								"type" "continuous_interval"
 								"bounds" (assoc "allow_null" .true)
 							)
 					))

module/feature_residuals.amlg

@@ -790,7 +790,79 @@
 			))
 		)
 
+		(declare (assoc
+			ratios_map
+				;only set/output ratios when computing deviations
+				(if (= "deviations" robust_residuals)
+					(let
+						(assoc
+							ratio_features (indices (keep residuals_map (indices !ratioMeasureMap)))
+						)
+						(if (size ratio_features)
+							(let
+								(assoc cont_feature_map (zip ratio_features) )
+
+								;list of maps for all the cases of feature -> value for all continuous features
+								(declare (assoc
+									f_values_maps
+										(map
+											(lambda (retrieve_from_entity (current_value) cont_feature_map) )
+											case_ids
+										)
+									feature_index_map (zip features (indices features))
+								))
+
+								(map
+									(lambda (let
+										(assoc
+											feature (current_index 1)
+											deviation_value (current_value 1)
+											;get the corresponding half of deviation value for each case
+											deviations
+												(map
+													(lambda (/ (first (current_value)) 2))
+													(get feature_residuals_lists (get feature_index_map (current_index 1)))
+												)
+										)
+
+										;does not affect nominal features
+										(if (contains_index !nominalsMap feature)
+											(current_value)
+
+											(generalized_mean
+												(map
+													(lambda
+														;divide deviation by feature value
+														(/
+															;;;TODO: handle null tuples and edit features
+															(or (get deviations (current_index)) deviation_value)
+
+															;don't allow 0s by replacing them with half min gaps
+															(max
+																(abs (get (current_value) feature))
+																(get !cachedFeatureHalfMinGapMap feature)
+															)
+														)
+
+													)
+													f_values_maps
+												)
+												;type of mean 1=arithmetic, 0=geometric, -1=harmonic
+												-1
+											)
+
+										)
+									))
+									(keep residuals_map ratio_features)
+								)
+							)
+						)
+					)
+				)
+		))
+
 		(assoc
+			"ratios_map" ratios_map
 			"residual_map" residuals_map
 			"ordinal_residual_map" ordinal_residuals_map
 			"hyperparam_map" hyperparam_map
@@ -1913,7 +1985,7 @@
 						(lambda
 							(or
 								;categorical features are in the "delta_features" list...
-								(!= "continuous" (get !featureAttributes [(current_value 1) "type"]))
+								(contains_value ["ordinal" "nominal"] (get !featureAttributes [(current_value 1) "type"]))
 								(contains_value derived_action_features (get !derivedFeaturesMap (current_value)))
 							)
 						)
@@ -2046,7 +2118,7 @@
 											feat_index (get feature_index_map (current_value 1))
 										)
 
-										(if (= "continuous" (get !featureAttributes [feat_to_diff "type"]))
+										(if (contains_index !continuousMap feat_to_diff)
 											(call !ComputeDiffTuple (assoc
 												feature feat_to_diff
 												feature_has_continuous_nulls (contains_index continuous_with_nulls_map feat_to_diff)